高级查询优化

@@ -11,4 +11,5 @@ require (
 	github.com/go-redis/redis v6.14.2+incompatible
 	github.com/google/uuid v1.1.1
 	github.com/linmadan/egglib-go v0.0.0-20210827085852-177fa745932d
+	github.com/stretchr/testify v1.7.0
 )

@@ -5,26 +5,26 @@ import (
 	"gitlab.fjmaimaimai.com/allied-creation/allied-creation-gateway/pkg/util/advance"
 )
 
-type Model struct {
+type (
+	Model struct {
 		Columns   []advance.Column  `json:"columns"`
 		MapColumn advance.MapColumn `json:"-"`
 		Name      string            `json:"name"`
-}
+	}
+	ModelInterface interface {
+		ModelName() string
+		Columns() []advance.Column
+	}
+)
 
 var registerModels = make(map[string]Model)
 
-func RegisModel(m Model) {
-	if _, ok := registerModels[m.Name]; ok {
-		panic("register modes exists:" + m.Name)
-	}
-	registerModels[m.Name] = m
-}
-
+// AdvancedQuerySql 高级查询语句生成
 func AdvancedQuerySql(model string, quires advance.AdvancedQueries) string {
 	if len(quires) == 0 {
 		return ""
 	}
-	fixQueries := fixAdvanceQueries(model, quires)
+	fixQueries := mergeDuplicateQueries(model, quires)
 	sql, err := advance.AdvancedQuerySql(fixQueries)
 	if err != nil {
 		log.Logger.Error(err.Error())
@@ -32,7 +32,7 @@ func AdvancedQuerySql(model string, quires advance.AdvancedQueries) string {
 	return sql
 }
 
-func fixAdvanceQueries(model string, quires advance.AdvancedQueries) []advance.AdvancedQuery {
+func mergeDuplicateQueries(model string, quires advance.AdvancedQueries) []advance.AdvancedQuery {
 	m, ok := GetModel(model)
 	response := make([]advance.AdvancedQuery, 0)
 	mapResponse := make(map[string]advance.AdvancedQuery)
@@ -57,11 +57,6 @@ func fixAdvanceQueries(model string, quires advance.AdvancedQueries) []advance.A
 	return response
 }
 
-func GetModel(name string) (Model, bool) {
-	m, ok := registerModels[name]
-	return m, ok
-}
-
 func NewModel(m ModelInterface) Model {
 	return Model{
 		Name:      m.ModelName(),
@@ -70,14 +65,22 @@ func NewModel(m ModelInterface) Model {
 	}
 }
 
-type ModelInterface interface {
-	ModelName() string
-	Columns() []advance.Column
+func RegisModel(m Model) {
+	if _, ok := registerModels[m.Name]; ok {
+		panic("register modes exists:" + m.Name)
+	}
+	registerModels[m.Name] = m
+}
+
+func GetModel(name string) (Model, bool) {
+	m, ok := registerModels[name]
+	return m, ok
 }
 
 /*User*/
 type UserModel struct{}
 
+// 实现接口 ModelInterface
 func (u UserModel) ModelName() string { return "user" }
 func (u UserModel) Columns() []advance.Column {
 	return []advance.Column{

@@ -160,7 +160,7 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
 		exceptSql string
 		ok        bool
 	}{
-		// in
+		// in (equal)
 		{
 			col:  columnChar,
 			name: "in zero item",
@@ -212,6 +212,19 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
 			ok:        true,
 		},
 
+		// not in (not equal)
+		{
+			col:  columnNumber,
+			name: "not equal many item (number)",
+			ins: []Expr{
+				{OpChar: NotEqual, Value: []interface{}{1, 2, 3, 4}},
+				{OpChar: NotEqual, Value: []interface{}{5, 6}},
+			},
+			except:    nil,
+			exceptSql: "(age not in (1,2,3,4,5,6))",
+			ok:        true,
+		},
+
 		// range
 		{
 			col:  columnChar,
@@ -241,7 +254,7 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
 				{OpChar: LessThanEqual, Value: []interface{}{Infinity, 80}},
 			},
 			except:    nil,
-			exceptSql: "(( age <= 80 ))",
+			exceptSql: "(( age <= 60 ))",
 			ok:        true,
 		},
 		{
@@ -254,7 +267,19 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
 				{OpChar: LessThanEqual, Value: []interface{}{Infinity, 70}},
 			},
 			except:    nil,
-			exceptSql: "(( age <= 70 ) or ( age >= 80 ))",
+			exceptSql: "", // 或集  (( age <= 70 ) or ( age >= 80 ))  并集 ""
+			ok:        true,
+		},
+		{
+			col:  columnNumber,
+			name: "range many item (Between $<=n<=$)",
+			ins: []Expr{
+				{OpChar: GreaterThanEqual, Value: []interface{}{80, Infinity}},
+				{OpChar: LessThanEqual, Value: []interface{}{Infinity, 200}},
+				{OpChar: LessThan, Value: []interface{}{Infinity, 150}},
+			},
+			except:    nil,
+			exceptSql: "(( age >= 80  and  age < 150 ))",
 			ok:        true,
 		},
 		{
@@ -275,9 +300,10 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
 				{OpChar: Range, Value: []interface{}{120, 220}, LeftOp: GreaterThanEqual, RightOp: LessThan},
 				{OpChar: Range, Value: []interface{}{100, 200}, LeftOp: GreaterThanEqual, RightOp: LessThanEqual},
 				{OpChar: Range, Value: []interface{}{80, 100}, LeftOp: GreaterThan, RightOp: LessThanEqual},
+				{OpChar: Range, Value: []interface{}{300, 500}, LeftOp: GreaterThan, RightOp: LessThanEqual},
 			},
 			except:    nil,
-			exceptSql: "(( age > 80  and  age < 220 ))",
+			exceptSql: "(( age > 80  and  age < 220 ) or ( age > 300  and  age <= 500 ))",
 			ok:        true,
 		},
 
@@ -322,6 +348,18 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
 			exceptSql: "((age::text like '%10%' or age::text like '%20%'))",
 			ok:        true,
 		},
+
+		// error input
+		{
+			col:  columnChar,
+			name: "invalid input",
+			ins: []Expr{
+				{OpChar: Like, Value: []interface{}{"dsger?*"}},
+			},
+			except:    nil,
+			exceptSql: "",
+			ok:        false,
+		},
 	}
 
 	for i := range tables {
@@ -333,7 +371,6 @@ func TestAdvancedQuerySql_PG(t *testing.T) {
 		}
 		t.Run(tables[i].name, func(t *testing.T) {
 			sql, err := AdvancedQuerySql(q)
-			assert.Equal(t, tables[i].except, err)
 			assert.Equal(t, tables[i].exceptSql, sql)
 			if tables[i].ok {
 				assert.Nil(t, err)

@@ -3,6 +3,8 @@ package advance
 import (
 	"bytes"
 	"fmt"
+	"regexp"
+	"strconv"
 	"strings"
 )
 
@@ -65,7 +67,7 @@ func ComputeColumnExpr(queries []AdvancedQuery) []ColumnExprResult {
 }
 
 func JoinColumnExprNumber(expr []Expr) []ExprResult {
-	var ec = []exprCompute{NewInExprCompute(expr, ValueNumber), NewRangeExprCompute(expr, ValueNumber), NewNotEqualExprCompute(expr, ValueNumber), NewLikeExprCompute(expr, ValueNumber)}
+	var ec = []exprCompute{NewInExprCompute(expr, ValueNumber), NewRangeExprCompute(expr, ValueNumber), NewLessGreaterExprCompute(expr, ValueNumber), NewNotEqualExprCompute(expr, ValueNumber), NewLikeExprCompute(expr, ValueNumber)}
 	return joinExprResult(ec)
 }
 
@@ -98,12 +100,15 @@ func (m MapColumn) FindItem(col string) Column {
 
 type PgSqlGenerator struct{}
 
-func (p PgSqlGenerator) SqlGen(q ColumnExprResult) string {
+func (p PgSqlGenerator) SqlGen(q ColumnExprResult) (string, error) {
 	sql := bytes.NewBuffer(nil)
 	sql.WriteString("(")
 	var wheres []string
 	for i := range q.Result {
 		item := q.Result[i]
+		if err := p.PreCheckAndFormat(&item); err != nil {
+			return "", err
+		}
 		var where string
 		switch item.OpChar {
 		case In:
@@ -124,9 +129,34 @@ func (p PgSqlGenerator) SqlGen(q ColumnExprResult) string {
 	}
 	sql.WriteString(strings.Join(wheres, sepOr))
 	sql.WriteString(")")
-	return sql.String()
+	return sql.String(), nil
+}
+func (p PgSqlGenerator) PreCheckAndFormat(rs *ExprResult) error {
+	if rs.ValueType == ValueNumber || rs.ValueType == ValueDate {
+		for i := range rs.Value {
+			v := rs.Value[i]
+			if isInfinity(v) {
+				continue
+			}
+			if _, err := strconv.ParseFloat(fmt.Sprintf("%v", v), 64); err != nil {
+				return fmt.Errorf("不是有效的数值类型:%v", v)
+			}
+		}
+	}
+	if rs.ValueType == ValueChars {
+		for i := range rs.Value {
+			v, ok := rs.Value[i].(string)
+			err := fmt.Errorf("不是有效的字符串类型:%v", v)
+			if !ok {
+				return err
+			}
+			if ok, e := regexp.MatchString("[!%&()*+,-/=?^`'{|}~]", v); ok || e != nil {
+				return fmt.Errorf("非法字符:%v", v)
+			}
+		}
+	}
+	return nil
 }
-
 func (p PgSqlGenerator) In(c Column, values []interface{}) string {
 	if len(values) < 1 {
 		return ""
@@ -207,11 +237,18 @@ func AdvancedQuerySql(queries []AdvancedQuery) (string, error) {
 	sql := bytes.NewBuffer(nil)
 	var wheres []string
 	for i := range results {
-		wheres = append(wheres, gen.SqlGen(results[i]))
+		condition, err := gen.SqlGen(results[i])
+		if err != nil {
+			return "", err
+		}
+		wheres = append(wheres, condition)
 	}
 	sql.WriteString(strings.Join(wheres, sepAnd))
 	// 空条件 ()
-	if len(sql.String()) == 2 {
+	if sql.String() == "()" {
+		return "", nil
+	}
+	if sql.String() == "(())" {
 		return "", nil
 	}
 	return sql.String(), nil

@@ -53,10 +53,10 @@ type (
 	}
 	Expr struct {
 		// 操作符
-		OpChar OpType `json:"oc"`
+		OpChar OpType `json:"op"`
 		// 如果 OpChar=range ,LeftOp、RightOp需要有值
-		LeftOp  OpType `json:"loc"` // "<= <"
-		RightOp OpType `json:"roc"` // ">= >"
+		LeftOp  OpType `json:"lop"` // "<= <"
+		RightOp OpType `json:"rop"` // ">= >"
 		// 值
 		Value []interface{} `json:"values"`
 	}
@@ -98,6 +98,10 @@ type (
 		valueType ValueType
 		OpType    OpType
 	}
+	LessGreaterExprCompute struct {
+		expr      []Expr
+		valueType ValueType
+	}
 )
 
 // in合并
@@ -151,7 +155,7 @@ func combine(arr []interface{}) []interface{} {
 func NewRangeExprCompute(expr []Expr, valueType ValueType) exprCompute {
 	rec := RangeNumberExprCompute{valueType: valueType}
 	for i := range expr {
-		if expr[i].OpChar == Range || expr[i].OpChar == LessThanEqual || expr[i].OpChar == GreaterThanEqual || expr[i].OpChar == LessThan || expr[i].OpChar == GreaterThan {
+		if expr[i].OpChar == Range { // || expr[i].OpChar == LessThanEqual || expr[i].OpChar == GreaterThanEqual || expr[i].OpChar == LessThan || expr[i].OpChar == GreaterThan
 			rec.expr = append(rec.expr, expr[i])
 		}
 	}
@@ -178,14 +182,14 @@ func (ex RangeNumberExprCompute) Append(result []ExprResult) []ExprResult {
 		arr.RightOp = ex.expr[0].RightOp
 	}
 	for i := 1; i < len(ex.expr); i++ {
-		if !arr.NumberCompare(ex.expr[i]) {
+		if !arr.Compare(ex.expr[i]) {
 			result = append(result, *arr)
 			arr.Value = ex.expr[i].Value
 			arr.LeftOp = ex.expr[i].OpChar
 			arr.RightOp = ex.expr[i].OpChar
-			//if len(ex.expr[0].LeftOp) != 0 {
-			//	arr.LeftOp = ex.expr[0].LeftOp
-			//}
+			if len(ex.expr[0].LeftOp) != 0 {
+				arr.LeftOp = ex.expr[0].LeftOp
+			}
 			if len(ex.expr[0].RightOp) != 0 {
 				arr.RightOp = ex.expr[0].RightOp
 			}
@@ -196,6 +200,60 @@ func (ex RangeNumberExprCompute) Append(result []ExprResult) []ExprResult {
 	return result
 }
 
+//范围合并
+func NewLessGreaterExprCompute(expr []Expr, valueType ValueType) exprCompute {
+	rec := LessGreaterExprCompute{valueType: valueType}
+	for i := range expr {
+		if expr[i].OpChar == LessThanEqual || expr[i].OpChar == LessThan || expr[i].OpChar == GreaterThanEqual || expr[i].OpChar == GreaterThan {
+			rec.expr = append(rec.expr, expr[i])
+		}
+	}
+	if len(rec.expr) == 0 {
+		return nil
+	}
+	var exprSort = exprSortable(rec.expr)
+	sort.Sort(exprSort)
+	rec.expr = exprSort
+	return rec
+}
+func (ex LessGreaterExprCompute) Append(result []ExprResult) []ExprResult {
+	arr := &ExprResult{
+		OpChar:    Range,
+		ValueType: ex.valueType,
+		Value:     ex.expr[0].Value,
+		LeftOp:    ex.expr[0].OpChar,
+		RightOp:   ex.expr[0].OpChar,
+	}
+	x0, x1 := toFloat64(arr.Value[0], arr.Value[1])
+	for i := 1; i < len(ex.expr); i++ {
+		compare := ex.expr[i]
+		y0, y1 := toFloat64(compare.Value[0], compare.Value[1])
+		if compare.OpChar == LessThanEqual || compare.OpChar == LessThan {
+			if isInfinity(x1) {
+				arr.Value[1] = y0
+				arr.RightOp = compare.OpChar
+			} else if x1 > y1 {
+				arr.Value[0] = y0
+				arr.LeftOp = compare.OpChar
+			}
+		}
+		if compare.OpChar == GreaterThan || compare.OpChar == GreaterThanEqual {
+			if isInfinity(x0) {
+				arr.Value[0] = y0
+				arr.LeftOp = compare.OpChar
+			} else if y0 > x0 {
+				arr.Value[0] = y0
+				arr.LeftOp = compare.OpChar
+			}
+		}
+	}
+	if _, ok := less(arr.Value[0], arr.Value[1]); !ok {
+		arr.Value[0], arr.Value[1] = Infinity, Infinity
+	}
+	result = append(result, *arr)
+	return result
+}
+
 // recent范围
 func NewRecentDateExprCompute(expr []Expr, valueType ValueType) exprCompute {
 	inExpr := RecentDateExprCompute{valueType: valueType}
@@ -268,18 +326,24 @@ func (ex NotEqualExprCompute) Append(result []ExprResult) []ExprResult {
 	return result
 }
 
-func (er *ExprResult) NumberCompare(expr Expr) bool {
-	if len(expr.Value) != 2 {
+// Compare 比较两个范围的值
+// eg: [80,&] [&,200]
+// 排序 [&,200]
+//     [80,&]
+// false:开启新的范围
+// true:继续这个范围
+func (er *ExprResult) Compare(compare Expr) bool {
+	if len(compare.Value) != 2 {
 		return false
 	}
-	_, x2 := toFloat64(er.Value[0], er.Value[1])
-	y1, _ := toFloat64(expr.Value[0], expr.Value[1])
-	if y1 <= x2 {
-		er.Value[1] = max(er.Value[1], expr.Value[1])
-		if isEqual(er.Value[1], expr.Value[1]) {
-			er.RightOp = expr.OpChar
-			if len(expr.RightOp) != 0 {
-				er.RightOp = expr.RightOp
+	_, x1 := toFloat64(er.Value[0], er.Value[1])
+	y0, _ := toFloat64(compare.Value[0], compare.Value[1])
+	if y0 <= x1 {
+		er.Value[1] = max(er.Value[1], compare.Value[1])
+		if isEqual(er.Value[1], compare.Value[1]) {
+			er.RightOp = compare.OpChar
+			if len(compare.RightOp) != 0 {
+				er.RightOp = compare.RightOp
 			}
 		}
 		return true
@@ -349,6 +413,20 @@ func max(x, y interface{}) interface{} {
 	return math.Max(fx, fy)
 }
 
+func less(x, y interface{}) (interface{}, bool) {
+	if isInfinity(x) {
+		return x, true
+	}
+	if isInfinity(y) {
+		return y, true
+	}
+	fx, fy := toFloat64(x, y)
+	if fx < fy {
+		return x, true
+	}
+	return fy, false
+}
+
 func isInfinity(val interface{}) bool {
 	v := fmt.Sprintf("%v", val)
 	inf := fmt.Sprintf("%v", Infinity)