作者 yangfu

Merge branch 'feat_calc_excel_expr' into test

... ... @@ -16,7 +16,7 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/event"
)
const Version = "v1.2.0"
const Version = "v1.3.0"
func main() {
defer func() {
... ...
... ... @@ -13,6 +13,7 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
"strings"
"time"
)
... ... @@ -441,7 +442,7 @@ func (querySetService *QuerySetService) CalculateItemPreview(ctx *domain.Context
if q.Formula.MixTableModel() {
q.Formula.ExprSql = q.Formula.Complete()
}
_, result := GetItemValues(transactionContext, q)
_, result := GetItemValues(ctx, transactionContext, q)
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
... ... @@ -464,7 +465,7 @@ func (querySetService *QuerySetService) CalculateItemExport(ctx *domain.Context,
defer func() {
transactionContext.RollbackTransaction()
}()
querySet, result := GetItemValues(transactionContext, q)
querySet, result := GetItemValues(ctx, transactionContext, q)
if querySet == nil {
return nil, err
}
... ... @@ -490,7 +491,7 @@ func (querySetService *QuerySetService) CalculateItemExport(ctx *domain.Context,
}, err
}
func GetItemValues(transactionContext application.TransactionContext, q *query.CalculateItemPreviewQuery) (*domain.QuerySet, []itemValue) {
func GetItemValues(ctx *domain.Context, transactionContext application.TransactionContext, q *query.CalculateItemPreviewQuery) (*domain.QuerySet, []itemValue) {
_, querySet, err := factory.FastPgQuerySet(transactionContext, q.QuerySetId)
if err != nil {
return nil, nil
... ... @@ -499,8 +500,24 @@ func GetItemValues(transactionContext application.TransactionContext, q *query.C
if q.Formula.MixTableModel() {
q.Formula.ExprSql = q.Formula.Complete()
}
value := starrocks.CalculateItemValue(starrocks.DB, q.Formula)
var value string
if q.Formula.ExprMode == domain.ExprModeExcelFunction {
svr, _ := factory.FastQuerySetServices(transactionContext)
dataTable, err := svr.LoadCalculateItemData(ctx, nil, &domain.FieldFormulaExpr{
FieldExpr: *q.Formula,
ExprMode: q.Formula.ExprMode,
})
if err != nil {
log.Logger.Error(err.Error())
}
if dataTable != nil && len(dataTable.Data) > 0 {
if len(dataTable.Data[0]) > 0 {
value = dataTable.Data[0][0]
}
}
} else {
value = starrocks.CalculateItemValue(starrocks.DB, q.Formula)
}
var result = make([]itemValue, 0)
result = append(result, itemValue{
Name: querySet.Name,
... ...
... ... @@ -10,6 +10,7 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/dto"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/query"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain/astexpr"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
... ... @@ -427,32 +428,40 @@ func (tableService *TableService) ValidExprSql(ctx *domain.Context, cmd *command
if err := cmd.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
set := collection.NewSet()
for _, f := range cmd.TableFields {
set.AddStr(f.TableSqlName)
}
if cmd.MixTableModel() {
cmd.ExprSql = cmd.Complete()
}
selectValue := cmd.ExprSql
//if _, parseErr := strconv.ParseFloat(cmd.ExprSql, 64); parseErr != nil {
// selectValue = "'" + selectValue + "'"
//}
if len(cmd.ExprSql) == 0 {
selectValue = "''"
}
sql := "select " + selectValue + " as expr"
if len(set.KeysStr()) > 0 {
sql += " from " + strings.Join(set.KeysStr(), ",")
sql += " limit 1"
}
tx := starrocks.DB.Exec(sql)
if tx.Error != nil {
return map[string]string{
"result": tx.Error.Error(),
}, nil
switch cmd.FieldExpr.ExprMode {
case domain.ExprModeSql:
set := collection.NewSet()
for _, f := range cmd.TableFields {
set.AddStr(f.TableSqlName)
}
if cmd.MixTableModel() {
cmd.ExprSql = cmd.Complete()
}
selectValue := cmd.ExprSql
if len(cmd.ExprSql) == 0 {
selectValue = "''"
}
sql := "select " + selectValue + " as expr"
if len(set.KeysStr()) > 0 {
sql += " from " + strings.Join(set.KeysStr(), ",")
sql += " limit 1"
}
tx := starrocks.DB.Exec(sql)
if tx.Error != nil {
return map[string]string{
"result": tx.Error.Error(),
}, nil
}
case domain.ExprModeExcelFunction:
_, err := astexpr.NewExprAST(cmd.ExprSql)
if err != nil {
return map[string]string{
"result": err.Error(),
}, nil
}
default:
}
return struct{}{}, nil
}
... ...
... ... @@ -4,6 +4,7 @@ import (
"fmt"
"github.com/go-gota/gota/dataframe"
"github.com/go-gota/gota/series"
"github.com/shopspring/decimal"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
"strings"
... ... @@ -16,6 +17,17 @@ type Calculator struct {
}
func NewCalculator(expr string) (*Calculator, error) {
ar, err := NewExprAST(expr)
if err != nil {
return nil, err
}
cal := &Calculator{
ExprAST: ar,
}
return cal, nil
}
func NewExprAST(expr string) (ExprAST, error) {
toks, err := ParseToken(expr)
if err != nil {
return nil, err
... ... @@ -28,11 +40,7 @@ func NewCalculator(expr string) (*Calculator, error) {
if ast.Err != nil {
return nil, ast.Err
}
cal := &Calculator{
ExprAST: ar,
}
return cal, nil
return ar, nil
}
func (cal *Calculator) SetDataTable(t *domain.DataTable) *Calculator {
... ... @@ -106,19 +114,21 @@ func (cal *Calculator) callDef(name string, args []*param) *param {
func (cal *Calculator) sum(params ...*param) *param {
var res = make([]string, 0)
var total float64
var total = decimal.NewFromFloat(0)
for _, p := range params {
for _, v := range p.data {
total += utils.NewNumberString(v).MustFloat64()
dv, _ := decimal.NewFromString(v)
total = total.Add(dv)
}
}
res = append(res, utils.AssertString(total))
res = append(res, total.String())
return NewResult(res)
}
func (cal *Calculator) sumifs(params ...*param) *param {
var list = make([]series.Series, 0)
var filters = make([]dataframe.F, 0)
var groupBy = make([]string, 0)
for i := 0; i < len(params)-1; i++ {
col := colName(i)
if i == 0 {
... ... @@ -126,15 +136,27 @@ func (cal *Calculator) sumifs(params ...*param) *param {
continue
}
if i%2 == 1 {
list = append(list, series.New(params[i+1].Data(), series.String, col))
if f, ok := cal.resolverFilter(col, params[i]); ok {
filters = append(filters, f)
list = append(list, series.New(params[i].Data(), series.String, col))
// TODO 类型是行字段判断为按行分组
if params[i+1].Len() > 1 {
groupBy = append(groupBy, col)
} else {
if f, ok := cal.resolverFilter(col, params[i+1]); ok {
filters = append(filters, f)
}
}
i++
}
}
df := dataframe.New(list...)
df = df.FilterAggregation(dataframe.And, filters...)
if len(groupBy) > 0 {
groups := df.GroupBy(groupBy...)
df = groups.Aggregation([]dataframe.AggregationType{dataframe.Aggregation_SUM}, []string{"A0"})
s := df.Col("A0_SUM")
return NewResult(toArrayFloat(s.Records())) //4000.00 需要格式化掉后缀 .00
}
s := df.Col("A0")
return NewResult(s.Records())
}
... ... @@ -220,17 +242,21 @@ func (cal *Calculator) OpCalc(op string, lp *param, rp *param) *param {
}
func opCalc(op, v1, v2 string) string {
fv1 := utils.NumberString(v1).MustFloat64()
fv2 := utils.NumberString(v2).MustFloat64()
//fv1 := utils.NumberString(v1).MustFloat64()
//fv2 := utils.NumberString(v2).MustFloat64()
fv1, _ := decimal.NewFromString(v1)
fv2, _ := decimal.NewFromString(v2)
switch op {
case "+":
return utils.AssertString(fv1 + fv2)
return utils.AssertString(fv1.Add(fv2).String())
case "-":
return utils.AssertString(fv1 - fv2)
return utils.AssertString(fv1.Sub(fv2).String()) // utils.Round(fv1-fv2, 15)
case "*":
return utils.AssertString(fv1 * fv2)
return utils.AssertString(fv1.Mul(fv2).String())
case "/":
return utils.AssertString(fv1 / fv2)
return utils.AssertString(fv1.Div(fv2).String())
}
return ""
}
... ... @@ -252,3 +278,10 @@ func NewResult(data []string) *param {
data: data,
}
}
func toArrayFloat(list []string) []string {
for i := range list {
list[i] = utils.AssertString(utils.NewNumberString(list[i]).MustFloat64())
}
return list
}
... ...
... ... @@ -125,19 +125,27 @@ func TestSumIfCalculator(t *testing.T) {
want []string
}{
{
expr: `sum(sumifs(销售明细.业绩,"3月",销售明细.月份))`,
expr: `sum(sumifs(销售明细.业绩,销售明细.月份,"3月"))`,
want: []string{"40000"},
},
{
expr: `sum(sumifs(销售明细.业绩,"3月",销售明细.月份,"<25000",销售明细.业绩))`,
expr: `sum(sumifs(销售明细.业绩,销售明细.月份,"3月",销售明细.业绩,"<25000"))`,
want: []string{"40000"},
},
{
expr: `sum(sumifs(销售明细.业绩,"3*",销售明细.月份))`,
expr: `sum(sumifs(销售明细.业绩,销售明细.月份,"3*"))`,
want: []string{"40000"},
},
{
expr: `sum(sumifs(销售明细.业绩,"*月",销售明细.月份))`,
expr: `sum(sumifs(销售明细.业绩,销售明细.月份,"*月"))`,
want: []string{"50000"},
},
{
expr: `sumifs(销售明细.业绩,销售明细.月份,销售明细.月份)`,
want: []string{"10000", "40000"},
},
{
expr: `sum(sumifs(销售明细.业绩,销售明细.月份,销售明细.月份))`,
want: []string{"50000"},
},
}
... ...
... ... @@ -202,7 +202,7 @@ func (p *Parser) isDigitNum(c rune) bool {
}
func (p *Parser) isChar(c rune) bool {
return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '.' || c == '"' || isChineseCharacter(c)
return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '.' || c == '"' || isChineseCharacter(c) || c == '_' //|| p.isDigitNum(c)
//判断是汉字
}
... ...
... ... @@ -89,6 +89,10 @@ func (t *DataTable) Values(f *Field) []string {
index = i
break
}
if t.Fields[i].Name == f.SQLName {
index = i
break
}
}
if index < 0 {
return res
... ...
... ... @@ -5,6 +5,11 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
)
const (
ExprModeSql = iota
ExprModeExcelFunction
)
var (
ErrorNotFound = fmt.Errorf("没有此资源")
)
... ...
... ... @@ -38,6 +38,16 @@ func (f *Field) SqlTypeEqual(compare *Field) bool {
return false
}
func (f *Field) SqlTypeToDB() string {
if f.SQLType == Float.ToString() {
return DECIMALV2.ToString()
}
if f.SQLType == Date.ToString() || f.SQLType == Datetime.ToString() {
return "timestamp"
}
return f.SQLType
}
func (f *Field) Valid() error {
if _, ok := SQLTypeMap[strings.ToUpper(f.SQLType)]; !ok {
return fmt.Errorf("unknown sql type:%v", f.SQLType)
... ...
... ... @@ -68,6 +68,7 @@ func (c ConditionExpr) ExprHuman() string {
}
type FieldFormulaExpr struct {
ExprMode int `json:"exprMode"` // 表达式模式 0:sql 1:excel function
FieldExpr
}
... ... @@ -93,6 +94,7 @@ type SelectExprGroup struct { // 查询表达式
}
type FieldExpr struct {
ExprMode int `json:"exprMode,omitempty"`
TableFields []TableField `json:"tableFields"`
ExprHuman string `json:"exprHuman"`
ExprSql string `json:"exprSql"`
... ...
... ... @@ -2,12 +2,12 @@ package domainService
import (
"bytes"
"github.com/beego/beego/v2/client/httplib"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api/bytelib"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
)
type ByteCoreService struct {
... ... @@ -147,6 +147,12 @@ func (ptr *ByteCoreService) FormulasGenerate(param domain.ReqFormulasGenerate) (
// }
// return &domain.DataFormulasGenerate{}, nil
//}
if param.QuerySet.Type == domain.CalculateItem.ToString() {
if param.QuerySet.QueryComponents[0].Formula.ExprMode != 0 {
err := ptr.ExcelExprCalcPersistence(param.QuerySet.QueryComponents[0].Formula, param, true)
return &domain.DataFormulasGenerate{}, err
}
}
if param.QuerySet.Type == domain.CalculateSet.ToString() {
_, err := param.QuerySetService.(*QuerySetService).LoadCalculateSetData(param.Context.(*domain.Context), param.QuerySet, param.QueryComponents)
if err != nil {
... ... @@ -162,6 +168,24 @@ func (ptr *ByteCoreService) FormulasClear(param domain.ReqFormulasClear) (*domai
return apiByteLib.FormulasClear(param)
}
func (ptr *ByteCoreService) ExcelExprCalcPersistence(expr *domain.FieldFormulaExpr, param domain.ReqFormulasGenerate, persistence bool) error {
if len(param.QueryComponents) == 0 {
return nil
}
// 加载Tables数据
q := param.QueryComponents[0]
result, err := param.QuerySetService.(*QuerySetService).LoadCalculateItemData(param.Context.(*domain.Context), param.Table, q.Formula)
if err != nil {
return err
}
if persistence {
starrocks.DropView(starrocks.DB, param.Table.SQLName)
starrocks.DropTable(starrocks.DB, param.Table.SQLName)
return starrocks.Exec(starrocks.DB, starrocks.CreateTableSql(param.Table.SQLName, param.Table.Fields(false), result.Data[0]))
}
return nil
}
//////////////
// 字库核心
//////////////
... ...
... ... @@ -176,6 +176,8 @@ func (ptr *QuerySetService) UpdateCalculateItem(ctx *domain.Context, qs *domain.
QuerySet: qs,
Table: table,
QueryComponents: queryComponents,
QuerySetService: ptr,
Context: ctx,
})
if err != nil {
return err
... ... @@ -751,18 +753,12 @@ func (ptr *QuerySetService) CreateOrUpdateCalculateItemTable(ctx *domain.Context
dependencyTables := querySet.GetDependencyTables(queryComponents)
tableRepository, _ := repository.NewTableRepository(ptr.transactionContext)
queryComponent := queryComponents[0]
//!!!warning:每个字段默认需要带函数,没有就补全max()
//if queryComponent.Formula.MixTableModel() {
// queryComponent.Formula.Complete()
//}
builder := NewDataFieldsBuilder()
//field := DataField(querySet.Name, domain.String.ToString(), domain.MainTableField, 1)
field := builder.NewDataField(querySet.Name, domain.String.ToString(), domain.MainTableField)
if len(queryComponent.Formula.TableFields) > 0 {
field.SQLType = queryComponent.Formula.TableFields[0].FieldSQLType
}
var table *domain.Table = NewCopyTable(domain.TableType(querySet.Type), querySet.Name, []*domain.Field{field}, 1).WithContext(ctx).WithPrefix(strings.ToLower(querySet.Type))
//table.PK = nil
if querySet.QuerySetInfo.BindTableId > 0 {
table, err = tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableId": querySet.QuerySetInfo.BindTableId})
if err != nil {
... ...
package domainService
import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain/astexpr"
)
func (ptr *QuerySetService) LoadCalculateItemData(ctx *domain.Context, t *domain.Table, formula *domain.FieldFormulaExpr) (*domain.DataTable, error) {
var (
res = &domain.DataTable{}
err error
)
calc, err := astexpr.NewCalculator(formula.ExprSql)
if err != nil {
return nil, err
}
if len(formula.TableFields) > 0 {
var tableId = formula.TableFields[0].TableId
mapTable, _ := ptr.loadDataTables(ctx, []int{tableId})
calc.SetDataTable(mapTable[tableId])
}
result, err := calc.ExprASTResult(calc.ExprAST)
if err != nil {
return nil, err
}
res.Data = [][]string{
result.Data(),
}
if t != nil {
res.Fields = t.Fields(false)
}
res.Total = int64(len(res.Data))
// 数据持久化
return res, nil
}
... ...
... ... @@ -225,10 +225,10 @@ func FastDataTable(options starrocks.QueryOptions) (*domain.DataTable, error) {
}
func (ptr *QuerySetService) LoadDataTables(ctx *domain.Context, cells []*domain.LayoutCell) (map[int]*domain.DataTable, error) {
var (
dataTables = make(map[int]*domain.DataTable)
tableRepository, _ = repository.NewTableRepository(ptr.transactionContext)
)
//var (
// dataTables = make(map[int]*domain.DataTable)
// tableRepository, _ = repository.NewTableRepository(ptr.transactionContext)
//)
tableIds := collection.NewSet()
for _, cell := range cells {
if cell.Data == nil || cell.Data.TableField == nil || cell.Data.TableField.TableId == 0 {
... ... @@ -236,8 +236,34 @@ func (ptr *QuerySetService) LoadDataTables(ctx *domain.Context, cells []*domain.
}
tableIds.AddInt(cell.Data.TableField.TableId)
}
if len(tableIds.KeysInt()) > 0 {
_, tables, err := tableRepository.Find(map[string]interface{}{"context": ctx, "tableIds": tableIds.KeysInt()})
//if len(tableIds.KeysInt()) > 0 {
// _, tables, err := tableRepository.Find(map[string]interface{}{"context": ctx, "tableIds": tableIds.KeysInt()})
// if err != nil {
// return nil, err
// }
// for _, t := range tables {
// if _, ok := dataTables[t.TableId]; ok {
// continue
// }
// dataTable, err := FastTable(t)
// if err != nil {
// log.Logger.Error(err.Error())
// return nil, fmt.Errorf("获取【%s】出现异常:%s", t.Name, err.Error())
// }
// dataTable.Fields = t.DataFields
// dataTables[t.TableId] = dataTable
// }
//}
return ptr.loadDataTables(ctx, tableIds.KeysInt())
}
func (ptr *QuerySetService) loadDataTables(ctx *domain.Context, tableIds []int) (map[int]*domain.DataTable, error) {
var (
dataTables = make(map[int]*domain.DataTable)
tableRepository, _ = repository.NewTableRepository(ptr.transactionContext)
)
if len(tableIds) > 0 {
_, tables, err := tableRepository.Find(map[string]interface{}{"context": ctx, "tableIds": tableIds})
if err != nil {
return nil, err
}
... ...
... ... @@ -112,3 +112,50 @@ create view {{.ViewName}}(
})
return html.UnescapeString(buf.String())
}
func DropTable(db *gorm.DB, tableName string) error {
tx := db.Exec("DROP TABLE IF EXISTS " + tableName)
return tx.Error
}
func CreateTableSql(viewName string, fields []*domain.Field, data []string) string {
sql := `
## DROP VIEW IF EXISTS {{.ViewName}};
## DROP TABLE IF EXISTS {{.ViewName}};
Create TABLE {{.ViewName}} (
id BIGINT,
{{.Field}}
)
UNIQUE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 3
PROPERTIES("replication_num"="1");
TRUNCATE TABLE {{.ViewName}};
insert into {{.ViewName}} Values {{.Values}}
`
tmp := template.New("ViewCreator")
tmp.Parse(sql)
buf := bytes.NewBuffer(nil)
bufField := bytes.NewBuffer(nil)
for i, f := range fields {
bufField.WriteString(fmt.Sprintf(`%s VARCHAR(65533)`, f.SQLName)) //, f.SqlTypeToDB()
if i != len(fields)-1 {
bufField.WriteString(",\n")
}
}
bufInsert := bytes.NewBuffer(nil)
for _, d := range data {
id, _ := utils.NewSnowflakeId()
bufInsert.WriteString(fmt.Sprintf("(%d,'%s')", id, d))
}
tmp.Execute(buf, map[string]interface{}{
"ViewName": viewName,
"Field": bufField.String(),
"Values": bufInsert.String(),
})
return html.UnescapeString(buf.String())
}
... ...