Merge branch 'feat_calc_excel_expr' into test

@@ -16,7 +16,7 @@ import (
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/event"
 )
 
-const Version = "v1.2.0"
+const Version = "v1.3.0"
 
 func main() {
 	defer func() {

@@ -13,6 +13,7 @@ import (
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
 	"strings"
 	"time"
 )
@@ -441,7 +442,7 @@ func (querySetService *QuerySetService) CalculateItemPreview(ctx *domain.Context
 	if q.Formula.MixTableModel() {
 		q.Formula.ExprSql = q.Formula.Complete()
 	}
-	_, result := GetItemValues(transactionContext, q)
+	_, result := GetItemValues(ctx, transactionContext, q)
 	if err := transactionContext.CommitTransaction(); err != nil {
 		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
 	}
@@ -464,7 +465,7 @@ func (querySetService *QuerySetService) CalculateItemExport(ctx *domain.Context,
 	defer func() {
 		transactionContext.RollbackTransaction()
 	}()
-	querySet, result := GetItemValues(transactionContext, q)
+	querySet, result := GetItemValues(ctx, transactionContext, q)
 	if querySet == nil {
 		return nil, err
 	}
@@ -490,7 +491,7 @@ func (querySetService *QuerySetService) CalculateItemExport(ctx *domain.Context,
 	}, err
 }
 
-func GetItemValues(transactionContext application.TransactionContext, q *query.CalculateItemPreviewQuery) (*domain.QuerySet, []itemValue) {
+func GetItemValues(ctx *domain.Context, transactionContext application.TransactionContext, q *query.CalculateItemPreviewQuery) (*domain.QuerySet, []itemValue) {
 	_, querySet, err := factory.FastPgQuerySet(transactionContext, q.QuerySetId)
 	if err != nil {
 		return nil, nil
@@ -499,8 +500,24 @@ func GetItemValues(transactionContext application.TransactionContext, q *query.C
 	if q.Formula.MixTableModel() {
 		q.Formula.ExprSql = q.Formula.Complete()
 	}
-	value := starrocks.CalculateItemValue(starrocks.DB, q.Formula)
-
+	var value string
+	if q.Formula.ExprMode == domain.ExprModeExcelFunction {
+		svr, _ := factory.FastQuerySetServices(transactionContext)
+		dataTable, err := svr.LoadCalculateItemData(ctx, nil, &domain.FieldFormulaExpr{
+			FieldExpr: *q.Formula,
+			ExprMode:  q.Formula.ExprMode,
+		})
+		if err != nil {
+			log.Logger.Error(err.Error())
+		}
+		if dataTable != nil && len(dataTable.Data) > 0 {
+			if len(dataTable.Data[0]) > 0 {
+				value = dataTable.Data[0][0]
+			}
+		}
+	} else {
+		value = starrocks.CalculateItemValue(starrocks.DB, q.Formula)
+	}
 	var result = make([]itemValue, 0)
 	result = append(result, itemValue{
 		Name:  querySet.Name,

@@ -10,6 +10,7 @@ import (
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/dto"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/query"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain/astexpr"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
@@ -427,7 +428,8 @@ func (tableService *TableService) ValidExprSql(ctx *domain.Context, cmd *command
 	if err := cmd.ValidateCommand(); err != nil {
 		return nil, application.ThrowError(application.ARG_ERROR, err.Error())
 	}
-
+	switch cmd.FieldExpr.ExprMode {
+	case domain.ExprModeSql:
 		set := collection.NewSet()
 		for _, f := range cmd.TableFields {
 			set.AddStr(f.TableSqlName)
@@ -436,9 +438,6 @@ func (tableService *TableService) ValidExprSql(ctx *domain.Context, cmd *command
 			cmd.ExprSql = cmd.Complete()
 		}
 		selectValue := cmd.ExprSql
-	//if _, parseErr := strconv.ParseFloat(cmd.ExprSql, 64); parseErr != nil {
-	//	selectValue = "'" + selectValue + "'"
-	//}
 		if len(cmd.ExprSql) == 0 {
 			selectValue = "''"
 		}
@@ -453,6 +452,16 @@ func (tableService *TableService) ValidExprSql(ctx *domain.Context, cmd *command
 				"result": tx.Error.Error(),
 			}, nil
 		}
+	case domain.ExprModeExcelFunction:
+		_, err := astexpr.NewExprAST(cmd.ExprSql)
+		if err != nil {
+			return map[string]string{
+				"result": err.Error(),
+			}, nil
+		}
+	default:
+	}
+
 	return struct{}{}, nil
 }
 

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/go-gota/gota/dataframe"
 	"github.com/go-gota/gota/series"
+	"github.com/shopspring/decimal"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
 	"strings"
@@ -16,6 +17,17 @@ type Calculator struct {
 }
 
 func NewCalculator(expr string) (*Calculator, error) {
+	ar, err := NewExprAST(expr)
+	if err != nil {
+		return nil, err
+	}
+	cal := &Calculator{
+		ExprAST: ar,
+	}
+	return cal, nil
+}
+
+func NewExprAST(expr string) (ExprAST, error) {
 	toks, err := ParseToken(expr)
 	if err != nil {
 		return nil, err
@@ -28,11 +40,7 @@ func NewCalculator(expr string) (*Calculator, error) {
 	if ast.Err != nil {
 		return nil, ast.Err
 	}
-
-	cal := &Calculator{
-		ExprAST: ar,
-	}
-	return cal, nil
+	return ar, nil
 }
 
 func (cal *Calculator) SetDataTable(t *domain.DataTable) *Calculator {
@@ -106,19 +114,21 @@ func (cal *Calculator) callDef(name string, args []*param) *param {
 
 func (cal *Calculator) sum(params ...*param) *param {
 	var res = make([]string, 0)
-	var total float64
+	var total = decimal.NewFromFloat(0)
 	for _, p := range params {
 		for _, v := range p.data {
-			total += utils.NewNumberString(v).MustFloat64()
+			dv, _ := decimal.NewFromString(v)
+			total = total.Add(dv)
 		}
 	}
-	res = append(res, utils.AssertString(total))
+	res = append(res, total.String())
 	return NewResult(res)
 }
 
 func (cal *Calculator) sumifs(params ...*param) *param {
 	var list = make([]series.Series, 0)
 	var filters = make([]dataframe.F, 0)
+	var groupBy = make([]string, 0)
 	for i := 0; i < len(params)-1; i++ {
 		col := colName(i)
 		if i == 0 {
@@ -126,15 +136,27 @@ func (cal *Calculator) sumifs(params ...*param) *param {
 			continue
 		}
 		if i%2 == 1 {
-			list = append(list, series.New(params[i+1].Data(), series.String, col))
-			if f, ok := cal.resolverFilter(col, params[i]); ok {
+			list = append(list, series.New(params[i].Data(), series.String, col))
+			// TODO 类型是行字段判断为按行分组
+			if params[i+1].Len() > 1 {
+				groupBy = append(groupBy, col)
+			} else {
+				if f, ok := cal.resolverFilter(col, params[i+1]); ok {
 					filters = append(filters, f)
 				}
+			}
 			i++
 		}
 	}
 	df := dataframe.New(list...)
 	df = df.FilterAggregation(dataframe.And, filters...)
+	if len(groupBy) > 0 {
+		groups := df.GroupBy(groupBy...)
+		df = groups.Aggregation([]dataframe.AggregationType{dataframe.Aggregation_SUM}, []string{"A0"})
+		s := df.Col("A0_SUM")
+		return NewResult(toArrayFloat(s.Records())) //4000.00 需要格式化掉后缀 .00
+	}
+
 	s := df.Col("A0")
 	return NewResult(s.Records())
 }
@@ -220,17 +242,21 @@ func (cal *Calculator) OpCalc(op string, lp *param, rp *param) *param {
 }
 
 func opCalc(op, v1, v2 string) string {
-	fv1 := utils.NumberString(v1).MustFloat64()
-	fv2 := utils.NumberString(v2).MustFloat64()
+	//fv1 := utils.NumberString(v1).MustFloat64()
+	//fv2 := utils.NumberString(v2).MustFloat64()
+
+	fv1, _ := decimal.NewFromString(v1)
+	fv2, _ := decimal.NewFromString(v2)
 	switch op {
 	case "+":
-		return utils.AssertString(fv1 + fv2)
+
+		return utils.AssertString(fv1.Add(fv2).String())
 	case "-":
-		return utils.AssertString(fv1 - fv2)
+		return utils.AssertString(fv1.Sub(fv2).String()) // utils.Round(fv1-fv2, 15)
 	case "*":
-		return utils.AssertString(fv1 * fv2)
+		return utils.AssertString(fv1.Mul(fv2).String())
 	case "/":
-		return utils.AssertString(fv1 / fv2)
+		return utils.AssertString(fv1.Div(fv2).String())
 	}
 	return ""
 }
@@ -252,3 +278,10 @@ func NewResult(data []string) *param {
 		data: data,
 	}
 }
+
+func toArrayFloat(list []string) []string {
+	for i := range list {
+		list[i] = utils.AssertString(utils.NewNumberString(list[i]).MustFloat64())
+	}
+	return list
+}

@@ -125,19 +125,27 @@ func TestSumIfCalculator(t *testing.T) {
 		want []string
 	}{
 		{
-			expr: `sum(sumifs(销售明细.业绩,"3月",销售明细.月份))`,
+			expr: `sum(sumifs(销售明细.业绩,销售明细.月份,"3月"))`,
 			want: []string{"40000"},
 		},
 		{
-			expr: `sum(sumifs(销售明细.业绩,"3月",销售明细.月份,"<25000",销售明细.业绩))`,
+			expr: `sum(sumifs(销售明细.业绩,销售明细.月份,"3月",销售明细.业绩,"<25000"))`,
 			want: []string{"40000"},
 		},
 		{
-			expr: `sum(sumifs(销售明细.业绩,"3*",销售明细.月份))`,
+			expr: `sum(sumifs(销售明细.业绩,销售明细.月份,"3*"))`,
 			want: []string{"40000"},
 		},
 		{
-			expr: `sum(sumifs(销售明细.业绩,"*月",销售明细.月份))`,
+			expr: `sum(sumifs(销售明细.业绩,销售明细.月份,"*月"))`,
+			want: []string{"50000"},
+		},
+		{
+			expr: `sumifs(销售明细.业绩,销售明细.月份,销售明细.月份)`,
+			want: []string{"10000", "40000"},
+		},
+		{
+			expr: `sum(sumifs(销售明细.业绩,销售明细.月份,销售明细.月份))`,
 			want: []string{"50000"},
 		},
 	}

@@ -202,7 +202,7 @@ func (p *Parser) isDigitNum(c rune) bool {
 }
 
 func (p *Parser) isChar(c rune) bool {
-	return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '.' || c == '"' || isChineseCharacter(c)
+	return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '.' || c == '"' || isChineseCharacter(c) || c == '_' //|| p.isDigitNum(c)
 	//判断是汉字
 }
 

@@ -89,6 +89,10 @@ func (t *DataTable) Values(f *Field) []string {
 			index = i
 			break
 		}
+		if t.Fields[i].Name == f.SQLName {
+			index = i
+			break
+		}
 	}
 	if index < 0 {
 		return res

@@ -5,6 +5,11 @@ import (
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
 )
 
+const (
+	ExprModeSql = iota
+	ExprModeExcelFunction
+)
+
 var (
 	ErrorNotFound = fmt.Errorf("没有此资源")
 )

@@ -38,6 +38,16 @@ func (f *Field) SqlTypeEqual(compare *Field) bool {
 	return false
 }
 
+func (f *Field) SqlTypeToDB() string {
+	if f.SQLType == Float.ToString() {
+		return DECIMALV2.ToString()
+	}
+	if f.SQLType == Date.ToString() || f.SQLType == Datetime.ToString() {
+		return "timestamp"
+	}
+	return f.SQLType
+}
+
 func (f *Field) Valid() error {
 	if _, ok := SQLTypeMap[strings.ToUpper(f.SQLType)]; !ok {
 		return fmt.Errorf("unknown sql type:%v", f.SQLType)

@@ -68,6 +68,7 @@ func (c ConditionExpr) ExprHuman() string {
 }
 
 type FieldFormulaExpr struct {
+	ExprMode int `json:"exprMode"` // 表达式模式 0:sql 1:excel function
 	FieldExpr
 }
 
@@ -93,6 +94,7 @@ type SelectExprGroup struct { // 查询表达式
 }
 
 type FieldExpr struct {
+	ExprMode    int          `json:"exprMode,omitempty"`
 	TableFields []TableField `json:"tableFields"`
 	ExprHuman   string       `json:"exprHuman"`
 	ExprSql     string       `json:"exprSql"`

@@ -2,12 +2,12 @@ package domainService
 
 import (
 	"bytes"
-
 	"github.com/beego/beego/v2/client/httplib"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api/bytelib"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
 )
 
 type ByteCoreService struct {
@@ -147,6 +147,12 @@ func (ptr *ByteCoreService) FormulasGenerate(param domain.ReqFormulasGenerate) (
 	//	}
 	//	return &domain.DataFormulasGenerate{}, nil
 	//}
+	if param.QuerySet.Type == domain.CalculateItem.ToString() {
+		if param.QuerySet.QueryComponents[0].Formula.ExprMode != 0 {
+			err := ptr.ExcelExprCalcPersistence(param.QuerySet.QueryComponents[0].Formula, param, true)
+			return &domain.DataFormulasGenerate{}, err
+		}
+	}
 	if param.QuerySet.Type == domain.CalculateSet.ToString() {
 		_, err := param.QuerySetService.(*QuerySetService).LoadCalculateSetData(param.Context.(*domain.Context), param.QuerySet, param.QueryComponents)
 		if err != nil {
@@ -162,6 +168,24 @@ func (ptr *ByteCoreService) FormulasClear(param domain.ReqFormulasClear) (*domai
 	return apiByteLib.FormulasClear(param)
 }
 
+func (ptr *ByteCoreService) ExcelExprCalcPersistence(expr *domain.FieldFormulaExpr, param domain.ReqFormulasGenerate, persistence bool) error {
+	if len(param.QueryComponents) == 0 {
+		return nil
+	}
+	// 加载Tables数据
+	q := param.QueryComponents[0]
+	result, err := param.QuerySetService.(*QuerySetService).LoadCalculateItemData(param.Context.(*domain.Context), param.Table, q.Formula)
+	if err != nil {
+		return err
+	}
+	if persistence {
+		starrocks.DropView(starrocks.DB, param.Table.SQLName)
+		starrocks.DropTable(starrocks.DB, param.Table.SQLName)
+		return starrocks.Exec(starrocks.DB, starrocks.CreateTableSql(param.Table.SQLName, param.Table.Fields(false), result.Data[0]))
+	}
+	return nil
+}
+
 //////////////
 // 字库核心
 //////////////

@@ -176,6 +176,8 @@ func (ptr *QuerySetService) UpdateCalculateItem(ctx *domain.Context, qs *domain.
 		QuerySet:        qs,
 		Table:           table,
 		QueryComponents: queryComponents,
+		QuerySetService: ptr,
+		Context:         ctx,
 	})
 	if err != nil {
 		return err
@@ -751,18 +753,12 @@ func (ptr *QuerySetService) CreateOrUpdateCalculateItemTable(ctx *domain.Context
 	dependencyTables := querySet.GetDependencyTables(queryComponents)
 	tableRepository, _ := repository.NewTableRepository(ptr.transactionContext)
 	queryComponent := queryComponents[0]
-	//！！！warning：每个字段默认需要带函数，没有就补全max()
-	//if queryComponent.Formula.MixTableModel() {
-	//	queryComponent.Formula.Complete()
-	//}
 	builder := NewDataFieldsBuilder()
-	//field := DataField(querySet.Name, domain.String.ToString(), domain.MainTableField, 1)
 	field := builder.NewDataField(querySet.Name, domain.String.ToString(), domain.MainTableField)
 	if len(queryComponent.Formula.TableFields) > 0 {
 		field.SQLType = queryComponent.Formula.TableFields[0].FieldSQLType
 	}
 	var table *domain.Table = NewCopyTable(domain.TableType(querySet.Type), querySet.Name, []*domain.Field{field}, 1).WithContext(ctx).WithPrefix(strings.ToLower(querySet.Type))
-	//table.PK = nil
 	if querySet.QuerySetInfo.BindTableId > 0 {
 		table, err = tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableId": querySet.QuerySetInfo.BindTableId})
 		if err != nil {

+package domainService
+
+import (
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain/astexpr"
+)
+
+func (ptr *QuerySetService) LoadCalculateItemData(ctx *domain.Context, t *domain.Table, formula *domain.FieldFormulaExpr) (*domain.DataTable, error) {
+	var (
+		res = &domain.DataTable{}
+		err error
+	)
+
+	calc, err := astexpr.NewCalculator(formula.ExprSql)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(formula.TableFields) > 0 {
+		var tableId = formula.TableFields[0].TableId
+		mapTable, _ := ptr.loadDataTables(ctx, []int{tableId})
+		calc.SetDataTable(mapTable[tableId])
+	}
+	result, err := calc.ExprASTResult(calc.ExprAST)
+	if err != nil {
+		return nil, err
+	}
+
+	res.Data = [][]string{
+		result.Data(),
+	}
+	if t != nil {
+		res.Fields = t.Fields(false)
+	}
+	res.Total = int64(len(res.Data))
+	// 数据持久化
+	return res, nil
+}

@@ -225,10 +225,10 @@ func FastDataTable(options starrocks.QueryOptions) (*domain.DataTable, error) {
 }
 
 func (ptr *QuerySetService) LoadDataTables(ctx *domain.Context, cells []*domain.LayoutCell) (map[int]*domain.DataTable, error) {
-	var (
-		dataTables         = make(map[int]*domain.DataTable)
-		tableRepository, _ = repository.NewTableRepository(ptr.transactionContext)
-	)
+	//var (
+	//	dataTables         = make(map[int]*domain.DataTable)
+	//	tableRepository, _ = repository.NewTableRepository(ptr.transactionContext)
+	//)
 	tableIds := collection.NewSet()
 	for _, cell := range cells {
 		if cell.Data == nil || cell.Data.TableField == nil || cell.Data.TableField.TableId == 0 {
@@ -236,8 +236,34 @@ func (ptr *QuerySetService) LoadDataTables(ctx *domain.Context, cells []*domain.
 		}
 		tableIds.AddInt(cell.Data.TableField.TableId)
 	}
-	if len(tableIds.KeysInt()) > 0 {
-		_, tables, err := tableRepository.Find(map[string]interface{}{"context": ctx, "tableIds": tableIds.KeysInt()})
+	//if len(tableIds.KeysInt()) > 0 {
+	//	_, tables, err := tableRepository.Find(map[string]interface{}{"context": ctx, "tableIds": tableIds.KeysInt()})
+	//	if err != nil {
+	//		return nil, err
+	//	}
+	//	for _, t := range tables {
+	//		if _, ok := dataTables[t.TableId]; ok {
+	//			continue
+	//		}
+	//		dataTable, err := FastTable(t)
+	//		if err != nil {
+	//			log.Logger.Error(err.Error())
+	//			return nil, fmt.Errorf("获取【%s】出现异常:%s", t.Name, err.Error())
+	//		}
+	//		dataTable.Fields = t.DataFields
+	//		dataTables[t.TableId] = dataTable
+	//	}
+	//}
+	return ptr.loadDataTables(ctx, tableIds.KeysInt())
+}
+
+func (ptr *QuerySetService) loadDataTables(ctx *domain.Context, tableIds []int) (map[int]*domain.DataTable, error) {
+	var (
+		dataTables         = make(map[int]*domain.DataTable)
+		tableRepository, _ = repository.NewTableRepository(ptr.transactionContext)
+	)
+	if len(tableIds) > 0 {
+		_, tables, err := tableRepository.Find(map[string]interface{}{"context": ctx, "tableIds": tableIds})
 		if err != nil {
 			return nil, err
 		}

@@ -112,3 +112,50 @@ create view {{.ViewName}}(
 	})
 	return html.UnescapeString(buf.String())
 }
+
+func DropTable(db *gorm.DB, tableName string) error {
+	tx := db.Exec("DROP TABLE IF EXISTS " + tableName)
+	return tx.Error
+}
+
+func CreateTableSql(viewName string, fields []*domain.Field, data []string) string {
+	sql := `
+## DROP VIEW IF EXISTS {{.ViewName}};
+## DROP TABLE IF EXISTS {{.ViewName}};
+Create TABLE {{.ViewName}} (
+id BIGINT,
+{{.Field}} 
+) 
+UNIQUE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 3
+PROPERTIES("replication_num"="1");
+
+TRUNCATE TABLE {{.ViewName}};
+
+insert into {{.ViewName}} Values {{.Values}}
+`
+	tmp := template.New("ViewCreator")
+	tmp.Parse(sql)
+
+	buf := bytes.NewBuffer(nil)
+	bufField := bytes.NewBuffer(nil)
+	for i, f := range fields {
+		bufField.WriteString(fmt.Sprintf(`%s VARCHAR(65533)`, f.SQLName)) //, f.SqlTypeToDB()
+		if i != len(fields)-1 {
+			bufField.WriteString(",\n")
+		}
+	}
+
+	bufInsert := bytes.NewBuffer(nil)
+	for _, d := range data {
+		id, _ := utils.NewSnowflakeId()
+		bufInsert.WriteString(fmt.Sprintf("(%d,'%s')", id, d))
+	}
+
+	tmp.Execute(buf, map[string]interface{}{
+		"ViewName": viewName,
+		"Field":    bufField.String(),
+		"Values":   bufInsert.String(),
+	})
+	return html.UnescapeString(buf.String())
+}