作者 yangfu

高级查询优化

... ... @@ -11,4 +11,5 @@ require (
github.com/go-redis/redis v6.14.2+incompatible
github.com/google/uuid v1.1.1
github.com/linmadan/egglib-go v0.0.0-20210827085852-177fa745932d
github.com/stretchr/testify v1.7.0
)
... ...
... ... @@ -5,26 +5,26 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/allied-creation-gateway/pkg/util/advance"
)
type Model struct {
type (
Model struct {
Columns []advance.Column `json:"columns"`
MapColumn advance.MapColumn `json:"-"`
Name string `json:"name"`
}
}
ModelInterface interface {
ModelName() string
Columns() []advance.Column
}
)
var registerModels = make(map[string]Model)
func RegisModel(m Model) {
if _, ok := registerModels[m.Name]; ok {
panic("register modes exists:" + m.Name)
}
registerModels[m.Name] = m
}
// AdvancedQuerySql 高级查询语句生成
func AdvancedQuerySql(model string, quires advance.AdvancedQueries) string {
if len(quires) == 0 {
return ""
}
fixQueries := fixAdvanceQueries(model, quires)
fixQueries := mergeDuplicateQueries(model, quires)
sql, err := advance.AdvancedQuerySql(fixQueries)
if err != nil {
log.Logger.Error(err.Error())
... ... @@ -32,7 +32,7 @@ func AdvancedQuerySql(model string, quires advance.AdvancedQueries) string {
return sql
}
func fixAdvanceQueries(model string, quires advance.AdvancedQueries) []advance.AdvancedQuery {
func mergeDuplicateQueries(model string, quires advance.AdvancedQueries) []advance.AdvancedQuery {
m, ok := GetModel(model)
response := make([]advance.AdvancedQuery, 0)
mapResponse := make(map[string]advance.AdvancedQuery)
... ... @@ -57,11 +57,6 @@ func fixAdvanceQueries(model string, quires advance.AdvancedQueries) []advance.A
return response
}
func GetModel(name string) (Model, bool) {
m, ok := registerModels[name]
return m, ok
}
func NewModel(m ModelInterface) Model {
return Model{
Name: m.ModelName(),
... ... @@ -70,14 +65,22 @@ func NewModel(m ModelInterface) Model {
}
}
type ModelInterface interface {
ModelName() string
Columns() []advance.Column
func RegisModel(m Model) {
if _, ok := registerModels[m.Name]; ok {
panic("register modes exists:" + m.Name)
}
registerModels[m.Name] = m
}
func GetModel(name string) (Model, bool) {
m, ok := registerModels[name]
return m, ok
}
/*User*/
type UserModel struct{}
// 实现接口 ModelInterface
func (u UserModel) ModelName() string { return "user" }
func (u UserModel) Columns() []advance.Column {
return []advance.Column{
... ...
... ... @@ -160,7 +160,7 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
exceptSql string
ok bool
}{
// in
// in (equal)
{
col: columnChar,
name: "in zero item",
... ... @@ -212,6 +212,19 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
ok: true,
},
// not in (not equal)
{
col: columnNumber,
name: "not equal many item (number)",
ins: []Expr{
{OpChar: NotEqual, Value: []interface{}{1, 2, 3, 4}},
{OpChar: NotEqual, Value: []interface{}{5, 6}},
},
except: nil,
exceptSql: "(age not in (1,2,3,4,5,6))",
ok: true,
},
// range
{
col: columnChar,
... ... @@ -241,7 +254,7 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
{OpChar: LessThanEqual, Value: []interface{}{Infinity, 80}},
},
except: nil,
exceptSql: "(( age <= 80 ))",
exceptSql: "(( age <= 60 ))",
ok: true,
},
{
... ... @@ -254,7 +267,19 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
{OpChar: LessThanEqual, Value: []interface{}{Infinity, 70}},
},
except: nil,
exceptSql: "(( age <= 70 ) or ( age >= 80 ))",
exceptSql: "", // 或集 (( age <= 70 ) or ( age >= 80 )) 并集 ""
ok: true,
},
{
col: columnNumber,
name: "range many item (Between $<=n<=$)",
ins: []Expr{
{OpChar: GreaterThanEqual, Value: []interface{}{80, Infinity}},
{OpChar: LessThanEqual, Value: []interface{}{Infinity, 200}},
{OpChar: LessThan, Value: []interface{}{Infinity, 150}},
},
except: nil,
exceptSql: "(( age >= 80 and age < 150 ))",
ok: true,
},
{
... ... @@ -275,9 +300,10 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
{OpChar: Range, Value: []interface{}{120, 220}, LeftOp: GreaterThanEqual, RightOp: LessThan},
{OpChar: Range, Value: []interface{}{100, 200}, LeftOp: GreaterThanEqual, RightOp: LessThanEqual},
{OpChar: Range, Value: []interface{}{80, 100}, LeftOp: GreaterThan, RightOp: LessThanEqual},
{OpChar: Range, Value: []interface{}{300, 500}, LeftOp: GreaterThan, RightOp: LessThanEqual},
},
except: nil,
exceptSql: "(( age > 80 and age < 220 ))",
exceptSql: "(( age > 80 and age < 220 ) or ( age > 300 and age <= 500 ))",
ok: true,
},
... ... @@ -322,6 +348,18 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
exceptSql: "((age::text like '%10%' or age::text like '%20%'))",
ok: true,
},
// error input
{
col: columnChar,
name: "invalid input",
ins: []Expr{
{OpChar: Like, Value: []interface{}{"dsger?*"}},
},
except: nil,
exceptSql: "",
ok: false,
},
}
for i := range tables {
... ... @@ -333,7 +371,6 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
}
t.Run(tables[i].name, func(t *testing.T) {
sql, err := AdvancedQuerySql(q)
assert.Equal(t, tables[i].except, err)
assert.Equal(t, tables[i].exceptSql, sql)
if tables[i].ok {
assert.Nil(t, err)
... ...
... ... @@ -3,6 +3,8 @@ package advance
import (
"bytes"
"fmt"
"regexp"
"strconv"
"strings"
)
... ... @@ -65,7 +67,7 @@ func ComputeColumnExpr(queries []AdvancedQuery) []ColumnExprResult {
}
func JoinColumnExprNumber(expr []Expr) []ExprResult {
var ec = []exprCompute{NewInExprCompute(expr, ValueNumber), NewRangeExprCompute(expr, ValueNumber), NewNotEqualExprCompute(expr, ValueNumber), NewLikeExprCompute(expr, ValueNumber)}
var ec = []exprCompute{NewInExprCompute(expr, ValueNumber), NewRangeExprCompute(expr, ValueNumber), NewLessGreaterExprCompute(expr, ValueNumber), NewNotEqualExprCompute(expr, ValueNumber), NewLikeExprCompute(expr, ValueNumber)}
return joinExprResult(ec)
}
... ... @@ -98,12 +100,15 @@ func (m MapColumn) FindItem(col string) Column {
type PgSqlGenerator struct{}
func (p PgSqlGenerator) SqlGen(q ColumnExprResult) string {
func (p PgSqlGenerator) SqlGen(q ColumnExprResult) (string, error) {
sql := bytes.NewBuffer(nil)
sql.WriteString("(")
var wheres []string
for i := range q.Result {
item := q.Result[i]
if err := p.PreCheckAndFormat(&item); err != nil {
return "", err
}
var where string
switch item.OpChar {
case In:
... ... @@ -124,9 +129,34 @@ func (p PgSqlGenerator) SqlGen(q ColumnExprResult) string {
}
sql.WriteString(strings.Join(wheres, sepOr))
sql.WriteString(")")
return sql.String()
return sql.String(), nil
}
func (p PgSqlGenerator) PreCheckAndFormat(rs *ExprResult) error {
if rs.ValueType == ValueNumber || rs.ValueType == ValueDate {
for i := range rs.Value {
v := rs.Value[i]
if isInfinity(v) {
continue
}
if _, err := strconv.ParseFloat(fmt.Sprintf("%v", v), 64); err != nil {
return fmt.Errorf("不是有效的数值类型:%v", v)
}
}
}
if rs.ValueType == ValueChars {
for i := range rs.Value {
v, ok := rs.Value[i].(string)
err := fmt.Errorf("不是有效的字符串类型:%v", v)
if !ok {
return err
}
if ok, e := regexp.MatchString("[!%&()*+,-/=?^`'{|}~]", v); ok || e != nil {
return fmt.Errorf("非法字符:%v", v)
}
}
}
return nil
}
func (p PgSqlGenerator) In(c Column, values []interface{}) string {
if len(values) < 1 {
return ""
... ... @@ -207,11 +237,18 @@ func AdvancedQuerySql(queries []AdvancedQuery) (string, error) {
sql := bytes.NewBuffer(nil)
var wheres []string
for i := range results {
wheres = append(wheres, gen.SqlGen(results[i]))
condition, err := gen.SqlGen(results[i])
if err != nil {
return "", err
}
wheres = append(wheres, condition)
}
sql.WriteString(strings.Join(wheres, sepAnd))
// 空条件 ()
if len(sql.String()) == 2 {
if sql.String() == "()" {
return "", nil
}
if sql.String() == "(())" {
return "", nil
}
return sql.String(), nil
... ...
... ... @@ -53,10 +53,10 @@ type (
}
Expr struct {
// 操作符
OpChar OpType `json:"oc"`
OpChar OpType `json:"op"`
// 如果 OpChar=range ,LeftOp、RightOp需要有值
LeftOp OpType `json:"loc"` // "<= <"
RightOp OpType `json:"roc"` // ">= >"
LeftOp OpType `json:"lop"` // "<= <"
RightOp OpType `json:"rop"` // ">= >"
// 值
Value []interface{} `json:"values"`
}
... ... @@ -98,6 +98,10 @@ type (
valueType ValueType
OpType OpType
}
LessGreaterExprCompute struct {
expr []Expr
valueType ValueType
}
)
// in合并
... ... @@ -151,7 +155,7 @@ func combine(arr []interface{}) []interface{} {
func NewRangeExprCompute(expr []Expr, valueType ValueType) exprCompute {
rec := RangeNumberExprCompute{valueType: valueType}
for i := range expr {
if expr[i].OpChar == Range || expr[i].OpChar == LessThanEqual || expr[i].OpChar == GreaterThanEqual || expr[i].OpChar == LessThan || expr[i].OpChar == GreaterThan {
if expr[i].OpChar == Range { // || expr[i].OpChar == LessThanEqual || expr[i].OpChar == GreaterThanEqual || expr[i].OpChar == LessThan || expr[i].OpChar == GreaterThan
rec.expr = append(rec.expr, expr[i])
}
}
... ... @@ -178,14 +182,14 @@ func (ex RangeNumberExprCompute) Append(result []ExprResult) []ExprResult {
arr.RightOp = ex.expr[0].RightOp
}
for i := 1; i < len(ex.expr); i++ {
if !arr.NumberCompare(ex.expr[i]) {
if !arr.Compare(ex.expr[i]) {
result = append(result, *arr)
arr.Value = ex.expr[i].Value
arr.LeftOp = ex.expr[i].OpChar
arr.RightOp = ex.expr[i].OpChar
//if len(ex.expr[0].LeftOp) != 0 {
// arr.LeftOp = ex.expr[0].LeftOp
//}
if len(ex.expr[0].LeftOp) != 0 {
arr.LeftOp = ex.expr[0].LeftOp
}
if len(ex.expr[0].RightOp) != 0 {
arr.RightOp = ex.expr[0].RightOp
}
... ... @@ -196,6 +200,60 @@ func (ex RangeNumberExprCompute) Append(result []ExprResult) []ExprResult {
return result
}
//范围合并
func NewLessGreaterExprCompute(expr []Expr, valueType ValueType) exprCompute {
rec := LessGreaterExprCompute{valueType: valueType}
for i := range expr {
if expr[i].OpChar == LessThanEqual || expr[i].OpChar == LessThan || expr[i].OpChar == GreaterThanEqual || expr[i].OpChar == GreaterThan {
rec.expr = append(rec.expr, expr[i])
}
}
if len(rec.expr) == 0 {
return nil
}
var exprSort = exprSortable(rec.expr)
sort.Sort(exprSort)
rec.expr = exprSort
return rec
}
func (ex LessGreaterExprCompute) Append(result []ExprResult) []ExprResult {
arr := &ExprResult{
OpChar: Range,
ValueType: ex.valueType,
Value: ex.expr[0].Value,
LeftOp: ex.expr[0].OpChar,
RightOp: ex.expr[0].OpChar,
}
x0, x1 := toFloat64(arr.Value[0], arr.Value[1])
for i := 1; i < len(ex.expr); i++ {
compare := ex.expr[i]
y0, y1 := toFloat64(compare.Value[0], compare.Value[1])
if compare.OpChar == LessThanEqual || compare.OpChar == LessThan {
if isInfinity(x1) {
arr.Value[1] = y0
arr.RightOp = compare.OpChar
} else if x1 > y1 {
arr.Value[0] = y0
arr.LeftOp = compare.OpChar
}
}
if compare.OpChar == GreaterThan || compare.OpChar == GreaterThanEqual {
if isInfinity(x0) {
arr.Value[0] = y0
arr.LeftOp = compare.OpChar
} else if y0 > x0 {
arr.Value[0] = y0
arr.LeftOp = compare.OpChar
}
}
}
if _, ok := less(arr.Value[0], arr.Value[1]); !ok {
arr.Value[0], arr.Value[1] = Infinity, Infinity
}
result = append(result, *arr)
return result
}
// recent范围
func NewRecentDateExprCompute(expr []Expr, valueType ValueType) exprCompute {
inExpr := RecentDateExprCompute{valueType: valueType}
... ... @@ -268,18 +326,24 @@ func (ex NotEqualExprCompute) Append(result []ExprResult) []ExprResult {
return result
}
func (er *ExprResult) NumberCompare(expr Expr) bool {
if len(expr.Value) != 2 {
// Compare 比较两个范围的值
// eg: [80,&] [&,200]
// 排序 [&,200]
// [80,&]
// false:开启新的范围
// true:继续这个范围
func (er *ExprResult) Compare(compare Expr) bool {
if len(compare.Value) != 2 {
return false
}
_, x2 := toFloat64(er.Value[0], er.Value[1])
y1, _ := toFloat64(expr.Value[0], expr.Value[1])
if y1 <= x2 {
er.Value[1] = max(er.Value[1], expr.Value[1])
if isEqual(er.Value[1], expr.Value[1]) {
er.RightOp = expr.OpChar
if len(expr.RightOp) != 0 {
er.RightOp = expr.RightOp
_, x1 := toFloat64(er.Value[0], er.Value[1])
y0, _ := toFloat64(compare.Value[0], compare.Value[1])
if y0 <= x1 {
er.Value[1] = max(er.Value[1], compare.Value[1])
if isEqual(er.Value[1], compare.Value[1]) {
er.RightOp = compare.OpChar
if len(compare.RightOp) != 0 {
er.RightOp = compare.RightOp
}
}
return true
... ... @@ -349,6 +413,20 @@ func max(x, y interface{}) interface{} {
return math.Max(fx, fy)
}
func less(x, y interface{}) (interface{}, bool) {
if isInfinity(x) {
return x, true
}
if isInfinity(y) {
return y, true
}
fx, fy := toFloat64(x, y)
if fx < fy {
return x, true
}
return fy, false
}
func isInfinity(val interface{}) bool {
v := fmt.Sprintf("%v", val)
inf := fmt.Sprintf("%v", Infinity)
... ...