作者 yangfu

feat: ast expr version 2,byte to rune

... ... @@ -11,6 +11,7 @@ require (
github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/gavv/httpexpect v2.0.0+incompatible
github.com/go-gota/gota v0.12.0
github.com/go-pg/pg/v10 v10.10.6
github.com/go-redis/redis v6.15.9+incompatible
github.com/google/go-querystring v1.1.0 // indirect
... ... @@ -29,6 +30,7 @@ require (
github.com/stretchr/testify v1.7.1
github.com/valyala/fasthttp v1.38.0 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect
github.com/xuri/excelize/v2 v2.6.0
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect
github.com/yudai/gojsondiff v1.0.0 // indirect
... ...
... ... @@ -57,9 +57,10 @@ func init() {
"rounddown": {-1, defNone},
"roundup": {-1, defNone},
"count": {-1, defNone},
"countif": {-1, defNone},
"countifs": {-1, defNone},
//"&": {-1, defNone},
"concat": {-1, defNone},
"sumifs": {-1, defNone},
}
}
... ...
... ... @@ -34,8 +34,9 @@ func NewFieldExprAST(val string) FieldExprAST {
ExprType: TypeFieldExprAST,
Str: val,
Field: &domain.TableField{
FieldName: filed,
TableName: table,
FieldName: filed,
FieldSqlName: filed,
TableName: table,
},
}
}
... ...
package astexpr
import (
"fmt"
"github.com/go-gota/gota/dataframe"
"github.com/go-gota/gota/series"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
"strings"
)
type Calculator struct {
ExprAST ExprAST
DataTable *domain.DataTable
Result []string
}
func NewCalculator(expr string) (*Calculator, error) {
toks, err := ParseToken(expr)
if err != nil {
return nil, err
}
ast := NewAST(toks, expr)
if ast.Err != nil {
return nil, ast.Err
}
ar := ast.ParseExpression()
if ast.Err != nil {
return nil, ast.Err
}
cal := &Calculator{
ExprAST: ar,
}
return cal, nil
}
func (cal *Calculator) SetDataTable(t *domain.DataTable) *Calculator {
cal.DataTable = t
return cal
}
func (cal *Calculator) Exec() error {
return nil
}
func (cal *Calculator) ExprASTResult(ast ExprAST) (*param, error) {
switch ast.(type) {
case BinaryExprAST:
var l, r *param
var err error
ast := ast.(BinaryExprAST)
l, err = cal.ExprASTResult(ast.Lhs)
if err != nil {
return nil, err
}
r, err = cal.ExprASTResult(ast.Rhs)
if err != nil {
return nil, err
}
switch ast.Op {
case "+", "-", "*", "/", "%":
return cal.OpCalc(ast.Op, l, r), nil
default:
}
case NumberExprAST:
f := ast.(NumberExprAST)
return NewResult([]string{f.Str}), nil
case ValueExprAST:
f := ast.(ValueExprAST)
return NewResult([]string{f.Val}), nil
case FieldExprAST:
f := ast.(FieldExprAST)
values := cal.DataTable.Values(&domain.Field{SQLName: f.Field.FieldSqlName})
return NewResult(values), nil
case FunCallerExprAST:
f := ast.(FunCallerExprAST)
//def := defFunc[f.Name]
//def.fun(f.Args...)
args := make([]*param, 0)
for i := range f.Args {
argValue, err := cal.ExprASTResult(f.Args[i])
if err != nil {
return nil, err
}
args = append(args, argValue)
}
return cal.callDef(f.Name, args), nil
}
return nil, nil
}
func (cal *Calculator) callDef(name string, args []*param) *param {
switch strings.ToLower(name) {
case "sum":
return cal.sum(args...)
case "sumifs":
return cal.sumifs(args...)
case "countifs":
return cal.countifs(args...)
}
return cal.sum(args...)
}
func (cal *Calculator) sum(params ...*param) *param {
var res = make([]string, 0)
var total float64
for _, p := range params {
for _, v := range p.data {
total += utils.NewNumberString(v).MustFloat64()
}
}
res = append(res, utils.AssertString(total))
return NewResult(res)
}
func (cal *Calculator) sumifs(params ...*param) *param {
var list = make([]series.Series, 0)
var filters = make([]dataframe.F, 0)
for i := 0; i < len(params)-1; i++ {
col := colName(i)
if i == 0 {
list = append(list, series.New(params[i].Data(), series.Float, col))
continue
}
if i%2 == 1 {
list = append(list, series.New(params[i+1].Data(), series.String, col))
if f, ok := cal.resolverFilter(col, params[i]); ok {
filters = append(filters, f)
}
i++
}
}
df := dataframe.New(list...)
df = df.FilterAggregation(dataframe.And, filters...)
s := df.Col("A0")
return NewResult(s.Records())
}
func (cal *Calculator) countifs(params ...*param) *param {
var list = make([]series.Series, 0)
var filters = make([]dataframe.F, 0)
for i := 0; i < len(params)-1; i++ {
col := colName(i)
if i%2 == 0 {
list = append(list, series.New(params[i].Data(), series.String, col))
if f, ok := cal.resolverFilter(col, params[i+1]); ok {
filters = append(filters, f)
}
i++
}
}
df := dataframe.New(list...)
df = df.FilterAggregation(dataframe.And, filters...)
count := df.Col("A0").Len()
return NewResult([]string{fmt.Sprintf("%d", count)})
}
func (cal *Calculator) resolverFilter(key string, param *param) (dataframe.F, bool) {
if len(param.Data()) == 1 {
condition := param.Data()[0]
tokens, _ := ParseToken(formatTok(condition))
switch tokens[0].Type {
case Operator, CompareOperator:
if tokens[0].Tok == "*" {
return dataframe.F{Colname: key, Comparator: series.CompFunc, Comparando: func(el series.Element) bool {
return strings.Contains(el.String(), strings.Trim(formatTok(condition), "*"))
}}, true
}
return dataframe.F{Colname: key, Comparator: series.Comparator(tokens[0].Tok), Comparando: formatTok(tokens[1].Tok)}, true
case Identifier, Literal, StringArgs:
if tokens[len(tokens)-1].Tok == "*" || tokens[0].Tok == "*" {
return dataframe.F{Colname: key, Comparator: series.CompFunc, Comparando: func(el series.Element) bool {
return strings.Contains(el.String(), strings.Trim(formatTok(condition), "*"))
}}, true
}
return dataframe.F{Colname: key, Comparator: series.Eq, Comparando: formatTok(condition)}, true
}
}
return dataframe.F{}, false
}
func colName(i int) string {
return fmt.Sprintf("A%v", i)
}
func formatTok(tok string) string {
return strings.Trim(tok, `"`)
}
func (cal *Calculator) OpCalc(op string, lp *param, rp *param) *param {
var res = make([]string, 0)
temp := make([]string, 0)
temp = lp.Data()
l := lp.Data()
r := rp.Data()
if lp.Len() < rp.Len() {
l = r
r = temp
}
rIsSingleValue := len(r) == 1
var rValue string
if rIsSingleValue {
rValue = r[0]
}
for i, lValue := range l {
if rIsSingleValue {
res = append(res, opCalc(op, lValue, rValue))
continue
}
if i >= len(r) {
break
}
res = append(res, opCalc(op, lValue, r[i]))
}
return NewResult(res)
}
func opCalc(op, v1, v2 string) string {
fv1 := utils.NumberString(v1).MustFloat64()
fv2 := utils.NumberString(v2).MustFloat64()
switch op {
case "+":
return utils.AssertString(fv1 + fv2)
case "-":
return utils.AssertString(fv1 - fv2)
case "*":
return utils.AssertString(fv1 * fv2)
case "/":
return utils.AssertString(fv1 / fv2)
}
return ""
}
type param struct {
data []string
}
func (p *param) Len() int {
return len(p.data)
}
func (p *param) Data() []string {
return p.data
}
func NewResult(data []string) *param {
return &param{
data: data,
}
}
... ...
package astexpr
import (
"github.com/stretchr/testify/assert"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"testing"
)
var table = &domain.DataTable{
Fields: []*domain.Field{
{
Name: "业绩",
SQLName: "业绩",
},
{
Name: "绩效",
SQLName: "绩效",
},
{
Name: "月份",
SQLName: "月份",
},
},
Data: [][]string{
{
"10000", "0.90", "2月",
},
{
"20000", "0.95", "3月",
},
{
"20000", "0.95", "3月",
},
},
}
var table1 = &domain.DataTable{
Fields: []*domain.Field{
{
Name: "产品",
SQLName: "产品",
},
{
Name: "季度",
SQLName: "季度",
},
{
Name: "数量",
SQLName: "数量",
},
},
Data: [][]string{
{
"苹果", "第一季度", "800",
},
{
"香蕉", "第一季度", "906",
},
{
"西瓜", "第一季度", "968",
},
{
"菠萝", "第一季度", "227",
},
{
"芒果", "第一季度", "612",
},
{
"苹果", "第二季度", "530",
},
{
"香蕉", "第二季度", "950",
},
{
"西瓜", "第二季度", "533",
},
{
"菠萝", "第二季度", "642",
},
{
"芒果", "第二季度", "489",
},
},
}
func TestSumCalculator(t *testing.T) {
inputs := []struct {
expr string
want []string
}{
{
expr: `sum(1000+销售明细.业绩*销售明细.绩效)`,
want: []string{"50000"},
},
{
expr: `sum(销售明细.业绩)`,
want: []string{"50000"},
},
{
expr: `sum(销售明细.业绩/10)`,
want: []string{"5000"},
},
{
expr: `sum(10000,20000,20000)`,
want: []string{"50000"},
},
}
for _, expr := range inputs {
calc, err := NewCalculator(expr.expr)
if err != nil {
t.Fatal(err)
}
calc.SetDataTable(table)
got, err := calc.ExprASTResult(calc.ExprAST)
if err != nil {
t.Fatal(err)
}
assert.Equal(t, expr.want, got.data)
}
}
func TestSumIfCalculator(t *testing.T) {
inputs := []struct {
expr string
want []string
}{
{
expr: `sum(sumifs(销售明细.业绩,"3月",销售明细.月份))`,
want: []string{"40000"},
},
{
expr: `sum(sumifs(销售明细.业绩,"3月",销售明细.月份,"<25000",销售明细.业绩))`,
want: []string{"40000"},
},
{
expr: `sum(sumifs(销售明细.业绩,"3*",销售明细.月份))`,
want: []string{"40000"},
},
{
expr: `sum(sumifs(销售明细.业绩,"*月",销售明细.月份))`,
want: []string{"50000"},
},
}
for _, expr := range inputs {
calc, err := NewCalculator(expr.expr)
if err != nil {
t.Fatal(err)
}
calc.SetDataTable(table)
got, err := calc.ExprASTResult(calc.ExprAST)
if err != nil {
t.Fatal(err)
}
assert.Equal(t, expr.want, got.data)
}
}
func TestCountIfCalculator(t *testing.T) {
inputs := []struct {
expr string
want []string
}{
{
expr: `countifs(水果季度.产品,"苹果")`,
want: []string{"2"},
},
{
expr: `countifs(水果季度.产品,"*果")`,
want: []string{"4"},
},
{
expr: `countifs(水果季度.季度,"第一季度",水果季度.数量,">600")`,
want: []string{"4"},
},
{
expr: `countifs(水果季度.产品,"*果",水果季度.数量,">600")`,
want: []string{"2"},
},
}
for _, expr := range inputs {
calc, err := NewCalculator(expr.expr)
if err != nil {
t.Fatal(err)
}
calc.SetDataTable(table1)
got, err := calc.ExprASTResult(calc.ExprAST)
if err != nil {
t.Fatal(err)
}
assert.Equal(t, expr.want, got.data)
}
}
... ...
package astexpr
import (
"fmt"
"testing"
"github.com/linmadan/egglib-go/utils/json"
"github.com/stretchr/testify/assert"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"testing"
)
func TestAstExprUnmarshalJSON(t *testing.T) {
... ... @@ -97,16 +99,35 @@ func TestAstExprUnmarshalJSON(t *testing.T) {
func TestAstExprParse(t *testing.T) {
funs := []struct {
Name string
Exp []string
Name string
Exp []string
Debug bool
}{
{
"多级嵌套",
[]string{`COUNTIF(销售明细.业绩,"<=1000")-COUNTIF(销售明细.业绩,"<=100")`},
"COUNTIF 多级嵌套",
[]string{
`COUNTIF(销售明细.业绩,"<=1000")-COUNTIF(销售明细.业绩,"<=100")`,
`SUM(1/(COUNTIF(销售明细.业绩,"<=1000")-COUNTIF(销售明细.业绩,"<=100")))`,
},
true,
},
{
"COUNTIF 多级嵌套",
[]string{
`COUNTIF(销售明细.业绩,"<=1000")-COUNTIF(销售明细.业绩,"<=100")`,
`SUM(COUNTIF(销售明细.业绩,"<=1000")-COUNTIF(销售明细.业绩,"<=100"))`,
},
false,
},
}
for _, f := range funs {
if !f.Debug {
continue
}
fmt.Println("测试项目", f.Name)
fmt.Println()
for _, exp := range f.Exp {
fmt.Println("表达式:", exp)
r, err := Parse(exp)
if err != nil {
t.Error(err)
... ...
... ... @@ -37,20 +37,22 @@ type Token struct {
}
type Parser struct {
Source string
ch byte
offset int
Source string
SourceRunes []rune
ch rune
offset int
err error
}
func ParseToken(s string) ([]*Token, error) {
p := &Parser{
Source: s,
err: nil,
ch: s[0],
Source: s,
SourceRunes: []rune(s),
err: nil,
//ch: s[0],
}
p.ch = p.SourceRunes[0]
toks := p.parse()
if p.err != nil {
return nil, p.err
... ... @@ -71,7 +73,7 @@ func (p *Parser) parse() []*Token {
}
func (p *Parser) nextTok() *Token {
if p.offset >= len(p.Source) || p.err != nil {
if p.offset >= len(p.SourceRunes) || p.err != nil {
return nil
}
var err error
... ... @@ -105,7 +107,7 @@ func (p *Parser) nextTok() *Token {
for p.isCompareWordChar(p.ch) && p.nextCh() == nil {
}
tok = &Token{
Tok: p.Source[start:p.offset],
Tok: string(p.SourceRunes[start:p.offset]),
Type: CompareOperator,
}
tok.Offset = start
... ... @@ -128,24 +130,24 @@ func (p *Parser) nextTok() *Token {
'8',
'9':
for p.isDigitNum(p.ch) && p.nextCh() == nil {
if (p.ch == '-' || p.ch == '+') && p.Source[p.offset-1] != 'e' {
if (p.ch == '-' || p.ch == '+') && p.SourceRunes[p.offset-1] != 'e' {
break
}
}
tok = &Token{
Tok: strings.ReplaceAll(p.Source[start:p.offset], "_", ""),
Tok: strings.ReplaceAll(string(p.SourceRunes[start:p.offset]), "_", ""),
Type: Literal,
}
tok.Offset = start
case '"':
for (p.isDigitNum(p.ch) || p.isChar(p.ch) || p.isCompareWordChar(p.ch)) && p.nextCh() == nil {
for (p.isDigitNum(p.ch) || p.isChar(p.ch) || p.isCompareWordChar(p.ch) || p.ch == '*') && p.nextCh() == nil {
if p.ch == '"' {
break
}
}
err = p.nextCh()
tok = &Token{
Tok: p.Source[start:p.offset],
Tok: string(p.SourceRunes[start:p.offset]),
Type: StringArgs,
}
tok.Offset = start
... ... @@ -162,7 +164,7 @@ func (p *Parser) nextTok() *Token {
for p.isWordChar(p.ch) && p.nextCh() == nil {
}
tok = &Token{
Tok: p.Source[start:p.offset],
Tok: string(p.SourceRunes[start:p.offset]),
Type: Identifier,
}
tok.Offset = start
... ... @@ -179,14 +181,14 @@ func (p *Parser) nextTok() *Token {
func (p *Parser) nextCh() error {
p.offset++
if p.offset < len(p.Source) {
p.ch = p.Source[p.offset]
if p.offset < len(p.SourceRunes) {
p.ch = p.SourceRunes[p.offset]
return nil
}
return errors.New("EOF")
}
func (p *Parser) isWhitespace(c byte) bool {
func (p *Parser) isWhitespace(c rune) bool {
return c == ' ' ||
c == '\t' ||
c == '\n' ||
... ... @@ -195,18 +197,23 @@ func (p *Parser) isWhitespace(c byte) bool {
c == '\r'
}
func (p *Parser) isDigitNum(c byte) bool {
func (p *Parser) isDigitNum(c rune) bool {
return '0' <= c && c <= '9' || c == '.' || c == '_' || c == 'e' || c == '-' || c == '+'
}
func (p *Parser) isChar(c byte) bool {
return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '.' || c == '"'
func (p *Parser) isChar(c rune) bool {
return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '.' || c == '"' || isChineseCharacter(c)
//判断是汉字
}
func (p *Parser) isWordChar(c byte) bool {
func (p *Parser) isWordChar(c rune) bool {
return p.isChar(c) || '0' <= c && c <= '9'
}
func (p *Parser) isCompareWordChar(c byte) bool {
func (p *Parser) isCompareWordChar(c rune) bool {
return c == '=' || c == '<' || c == '>'
}
func isChineseCharacter(c rune) bool {
return len([]byte(string(c))) > 2
}
... ...
package astexpr
import (
"bytes"
"encoding/json"
"fmt"
"testing"
... ... @@ -267,12 +268,14 @@ func Parse(s string) (r float64, err error) {
err = e.(error)
}
}()
if ar != nil {
fmt.Printf("ExprAST: %+v\n", ar)
}
arData, _ := json.Marshal(ar)
fmt.Printf("%s\n", string(arData))
buf := bytes.NewBuffer(nil)
enc := json.NewEncoder(buf)
enc.SetEscapeHTML(false)
enc.Encode(ar)
// if ar != nil {
// fmt.Printf("ExprAST: %+v\n", ar)
// }
fmt.Println(buf.String())
return 0, err
}
... ...
... ... @@ -160,7 +160,7 @@ func (t TableType) ToString() string {
}
func (t TableType) TableStatusEditable() bool {
return t == SchemaTable || t == CalculateItem || t == CalculateTable || t == CalculateSet
return t == SchemaTable || t == CalculateItem || t == CalculateSet
}
func (t TableType) TableHasGroup() bool {
... ...
... ... @@ -133,6 +133,25 @@ func (querySet *QuerySet) GetDependencyTables(queryComponents []*QueryComponent)
return res
}
func (querySet *QuerySet) Valid(queryComponents []*QueryComponent) error {
switch querySet.Type {
case CalculateTable.ToString():
if len(queryComponents) == 0 {
return fmt.Errorf("行、值不能同时为空")
}
qc := queryComponents[0]
set := collection.NewSet()
for _, f := range qc.Aggregation.AggregationFields() {
if !set.Contains(f.DisplayName) {
set.AddStr(f.DisplayName)
} else {
return fmt.Errorf("字段'%s'存在重名,请进行重命名", f.DisplayName)
}
}
}
return nil
}
type QuerySets []*QuerySet
func (querySets QuerySets) ToMap() map[int]*QuerySet {
... ...
... ... @@ -188,7 +188,7 @@ func NewTableAppendRequest(param domain.ReqAppendData) TableAppendRequest {
OriginalTableId: intToString(param.FileId),
CheckoutTableFileUrl: param.FileUrl,
DatabaseTableName: param.Table.SQLName,
ColumnSchemas: DomainFieldsToColumnSchemas(param.ExcelTable.DataFields),
ColumnSchemas: DomainFieldsToColumnSchemas(param.From), //param.ExcelTable.DataFields
FieldSchemas: ToFieldSchemas(param.Table.DataFields),
SchemaMap: make(map[string]domain.ColumnSchema),
}
... ...
... ... @@ -8,6 +8,7 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"reflect"
"strings"
"time"
)
... ... @@ -273,6 +274,9 @@ func (ptr *QuerySetService) PreviewPrepare(ctx *domain.Context, querySetId int,
if err != nil {
return nil, err
}
if err = querySet.Valid(queryComponents); err != nil {
return nil, err
}
if !queryComponentsHasEdit(ctx, querySet, queryComponents) && querySet.QuerySetInfo.BindTableId > 0 {
if t, _ := tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableId": querySet.QuerySetInfo.BindTableId}); t != nil {
return t, nil
... ... @@ -655,6 +659,29 @@ func aggregationEditLog(ctx *domain.Context, querySet *domain.QuerySet, queryCom
return res
}
func aggregationHasEdit(ctx *domain.Context, querySet *domain.QuerySet, queryComponents []*domain.QueryComponent) bool {
if len(queryComponents) > 0 && len(querySet.QueryComponents) > 0 {
oldQC := querySet.QueryComponents[0]
newQC := queryComponents[0]
if oldQC.Aggregation == nil || newQC.Aggregation == nil {
return false
}
c1 := make([]string, 0)
for _, f := range oldQC.Aggregation.AggregationFields() {
c1 = append(c1, f.Expr.ExprSql)
}
c2 := make([]string, 0)
for _, f := range newQC.Aggregation.AggregationFields() {
c2 = append(c2, f.Expr.ExprSql)
}
if !reflect.DeepEqual(c1, c2) {
return true
}
}
return false
}
func queryComponentsHasEdit(ctx *domain.Context, querySet *domain.QuerySet, queryComponents []*domain.QueryComponent) bool {
logs := selectsEditLog(ctx, querySet, queryComponents)
if len(logs) > 0 {
... ... @@ -668,6 +695,9 @@ func queryComponentsHasEdit(ctx *domain.Context, querySet *domain.QuerySet, quer
if len(logs) > 0 {
return true
}
if aggregationHasEdit(ctx, querySet, queryComponents) {
return true
}
for _, item := range queryComponents {
if len(item.Id) == 0 {
return true
... ... @@ -757,6 +787,9 @@ func (ptr *QuerySetService) CreateOrUpdateCalculateTable(ctx *domain.Context, qu
err error
foundMasterTable *domain.Table
)
if err = querySet.Valid(queryComponents); err != nil {
return nil, err
}
dependencyTables := querySet.GetDependencyTables(queryComponents)
tableRepository, _ := repository.NewTableRepository(ptr.transactionContext)
foundMasterTable, err = tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableId": masterTable.TableId})
... ...
... ... @@ -85,6 +85,7 @@ func (ptr *AppendDataToTableService) AppendData(ctx *domain.Context, fileId int,
"result": fmt.Sprintf("字段【%s】的类型与导入数据表的类型不匹配", toField.Name),
}, nil
}
fromField.SQLType = toField.SQLType // 兼容 INT BIGINT
requestData.To = append(requestData.To, toField)
requestData.From = append(requestData.From, fromField)
}
... ...