allied-creation / character-library-metadata-bastion · 提交

转到一个项目

作者 yangfu
提交 24725118c1785938bcab76de36da1d59393cc975 1 个父辈 86c9e321

fix:check table row duplicate

内嵌 并排对比
正在显示 5 个修改的文件 包含 227 行增加 和 7 行删除
@@ -10,9 +10,9 @@ import ( @@ -10,9 +10,9 @@ import (
10 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/dto" 10 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/dto"
11 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/query" 11 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/query"
12 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain" 12 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
  13 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
13 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks" 14 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
14 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils" 15 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
15 - "strconv"  
16 "strings" 16 "strings"
17 ) 17 )
18 18
@@ -433,9 +433,9 @@ func (tableService *TableService) ValidExprSql(ctx *domain.Context, cmd *command @@ -433,9 +433,9 @@ func (tableService *TableService) ValidExprSql(ctx *domain.Context, cmd *command
433 set.AddStr(f.TableSqlName) 433 set.AddStr(f.TableSqlName)
434 } 434 }
435 selectValue := cmd.ExprSql 435 selectValue := cmd.ExprSql
436 - if _, parseErr := strconv.ParseFloat(cmd.ExprSql, 64); parseErr != nil {  
437 - selectValue = "'" + selectValue + "'"  
438 - } 436 + //if _, parseErr := strconv.ParseFloat(cmd.ExprSql, 64); parseErr != nil {
  437 + // selectValue = "'" + selectValue + "'"
  438 + //}
439 sql := "select " + selectValue + " as expr" 439 sql := "select " + selectValue + " as expr"
440 if len(set.KeysStr()) > 0 { 440 if len(set.KeysStr()) > 0 {
441 sql += " from " + strings.Join(set.KeysStr(), ",") 441 sql += " from " + strings.Join(set.KeysStr(), ",")
@@ -551,6 +551,127 @@ func (tableService *TableService) CheckRowDuplicateV2(ctx *domain.Context, cmd * @@ -551,6 +551,127 @@ func (tableService *TableService) CheckRowDuplicateV2(ctx *domain.Context, cmd *
551 }, nil 551 }, nil
552 } 552 }
553 553
  554 +func (tableService *TableService) CheckRowDuplicateV3(ctx *domain.Context, cmd *command.CheckRowDuplicateCommand) (interface{}, error) {
  555 + var defaultResponse = map[string]interface{}{
  556 + "rowDuplicateFlag": false,
  557 + }
  558 + if cmd.TableId == 0 {
  559 + return defaultResponse, nil
  560 + }
  561 + if err := cmd.ValidateCommand(); err != nil {
  562 + return nil, application.ThrowError(application.ARG_ERROR, err.Error())
  563 + }
  564 + transactionContext, err := factory.CreateTransactionContext(nil)
  565 + if err != nil {
  566 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  567 + }
  568 + if err := transactionContext.StartTransaction(); err != nil {
  569 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  570 + }
  571 + defer func() {
  572 + transactionContext.RollbackTransaction()
  573 + }()
  574 +
  575 + var table *domain.Table
  576 + _, table, err = factory.FastPgTable(transactionContext, cmd.TableId)
  577 + if err != nil {
  578 + return nil, factory.FastError(err)
  579 + }
  580 +
  581 + querySetRepository, _, _ := factory.FastPgQuerySet(transactionContext, 0)
  582 + querySet, _ := querySetRepository.FindOne(map[string]interface{}{"BindTableId": table.TableId, "context": ctx})
  583 +
  584 + if querySet == nil {
  585 + cache := redis.NewTableQuerySetCacheService()
  586 + if item, cacheItemErr := cache.GetTableQuerySetCache(redis.KeyTableQuerySet(cmd.TableId)); cacheItemErr == nil {
  587 + querySet = &domain.QuerySet{QuerySetId: item.QuerySetId, QueryComponents: item.QueryComponents}
  588 + }
  589 + }
  590 +
  591 + if querySet == nil {
  592 + return defaultResponse, nil
  593 + }
  594 +
  595 + if len(querySet.QueryComponents) == 1 {
  596 + return defaultResponse, nil
  597 + }
  598 + var mapQueryComponents = make(map[int][]*domain.QueryComponent)
  599 + for _, queryComponent := range querySet.QueryComponents {
  600 + if _, ok := mapQueryComponents[queryComponent.MasterTable.TableId]; !ok {
  601 + mapQueryComponents[queryComponent.MasterTable.TableId] = make([]*domain.QueryComponent, 0)
  602 + }
  603 + mapQueryComponents[queryComponent.MasterTable.TableId] = append(mapQueryComponents[queryComponent.MasterTable.TableId], queryComponent)
  604 + }
  605 + for _, v := range mapQueryComponents {
  606 + if len(v) <= 1 {
  607 + continue
  608 + }
  609 + total, duplicateTotal, err := starrocks.WrapQueryHasDuplicateRowBySql(CountQueryComponents(v), starrocks.DB)()
  610 + if err != nil {
  611 + return nil, factory.FastError(err)
  612 + }
  613 + if total != duplicateTotal {
  614 + return map[string]interface{}{
  615 + "tableName": table.Name,
  616 + "rowDuplicateFlag": total != duplicateTotal,
  617 + "rowTotal": total,
  618 + "rowDuplicateTotal": total - duplicateTotal,
  619 + }, nil
  620 + }
  621 + }
  622 + if err := transactionContext.CommitTransaction(); err != nil {
  623 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  624 + }
  625 + return defaultResponse, nil
  626 +}
  627 +
  628 +func CountQueryComponents(queryComponents []*domain.QueryComponent) string {
  629 + var sql = "select count(id) c1,count(DISTINCT(id)) c2 from ("
  630 + for _, q := range queryComponents {
  631 + sql += "\n"
  632 + sql += ViewQueryComponent(q)
  633 + sql += "\n"
  634 + sql += "union all"
  635 + }
  636 + sql = strings.TrimSuffix(sql, "union all")
  637 + sql += ") a"
  638 + return sql
  639 +}
  640 +
  641 +func ViewQueryComponent(queryComponent *domain.QueryComponent) string {
  642 + sql := "select * from " + queryComponent.MasterTable.SQLName
  643 + for i, c := range queryComponent.Conditions {
  644 + if i == 0 {
  645 + sql += " where "
  646 + }
  647 + if len(c.FieldRight.TableFields) > 0 && queryComponent.MasterTable.TableId != c.FieldRight.TableFields[0].TableId {
  648 + continue
  649 + }
  650 + sql += FormatCondition(c)
  651 + sql += " and"
  652 + }
  653 + sql = strings.TrimSuffix(sql, "and")
  654 + return sql
  655 +}
  656 +
  657 +func FormatCondition(c domain.ConditionExpr) string {
  658 + //if len(c.FieldRight.TableFields) > 0 {
  659 + // return fmt.Sprintf("%s %s %s", c.FieldRight.ExprSql, FormatOp(c.OperatorSymbol), FormatByOp(c.OperatorSymbol, c.FieldRight.ExprSql))
  660 + //}
  661 + //if domain.SQLType(c.FieldLeft.TableFields[0].FieldSQLType).IsString() {
  662 + // return fmt.Sprintf("%s %s '%s'", c.FieldRight.ExprSql, FormatOp(c.OperatorSymbol), FormatByOp(c.OperatorSymbol, c.FieldRight.ExprSql))
  663 + //}
  664 + return fmt.Sprintf("%s %s %s", c.FieldLeft.ExprSql, FormatOp(c.OperatorSymbol), FormatByOp(c.OperatorSymbol, c.FieldRight.ExprSql))
  665 +}
  666 +
  667 +func FormatOp(op string) string {
  668 + return op
  669 +}
  670 +
  671 +func FormatByOp(op string, exprSql string) string {
  672 + return exprSql
  673 +}
  674 +
554 func NewTableService(options map[string]interface{}) *TableService { 675 func NewTableService(options map[string]interface{}) *TableService {
555 newTableService := &TableService{} 676 newTableService := &TableService{}
556 return newTableService 677 return newTableService
@@ -5,6 +5,7 @@ import ( @@ -5,6 +5,7 @@ import (
5 pgTransaction "github.com/linmadan/egglib-go/transaction/pg" 5 pgTransaction "github.com/linmadan/egglib-go/transaction/pg"
6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain" 6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/dao" 7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/dao"
  8 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
8 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository" 9 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository"
9 "strings" 10 "strings"
10 "time" 11 "time"
@@ -148,6 +149,7 @@ func (ptr *QuerySetService) PreviewPrepare(ctx *domain.Context, querySetId int, @@ -148,6 +149,7 @@ func (ptr *QuerySetService) PreviewPrepare(ctx *domain.Context, querySetId int,
148 return t, nil 149 return t, nil
149 } 150 }
150 } 151 }
  152 +
151 // 验证 153 // 验证
152 if err = ptr.validQueryComponents(queryComponents); err != nil { 154 if err = ptr.validQueryComponents(queryComponents); err != nil {
153 return nil, err 155 return nil, err
@@ -178,7 +180,11 @@ func (ptr *QuerySetService) PreviewPrepare(ctx *domain.Context, querySetId int, @@ -178,7 +180,11 @@ func (ptr *QuerySetService) PreviewPrepare(ctx *domain.Context, querySetId int,
178 if err != nil { 180 if err != nil {
179 return nil, err 181 return nil, err
180 } 182 }
181 - 183 + // set cache
  184 + cache := redis.NewTableQuerySetCacheService()
  185 + if err = cache.SetTableQuerySetCache(redis.KeyTableQuerySet(table.TableId), redis.NewTableQuerySetCache(querySet.QuerySetId, table.TableId, queryComponents)); err != nil {
  186 + return nil, err
  187 + }
182 // 调用底层的组装sql 188 // 调用底层的组装sql
183 if _, err = ByteCore.FormulasGenerate(domain.ReqFormulasGenerate{ 189 if _, err = ByteCore.FormulasGenerate(domain.ReqFormulasGenerate{
184 QuerySet: querySet, 190 QuerySet: querySet,
@@ -346,7 +352,19 @@ func conditionsEditLog(ctx *domain.Context, querySet *domain.QuerySet, queryComp @@ -346,7 +352,19 @@ func conditionsEditLog(ctx *domain.Context, querySet *domain.QuerySet, queryComp
346 352
347 func queryComponentsHasEdit(ctx *domain.Context, querySet *domain.QuerySet, queryComponents []*domain.QueryComponent) bool { 353 func queryComponentsHasEdit(ctx *domain.Context, querySet *domain.QuerySet, queryComponents []*domain.QueryComponent) bool {
348 logs := selectsEditLog(ctx, querySet, queryComponents) 354 logs := selectsEditLog(ctx, querySet, queryComponents)
349 - return len(logs) != 0 355 + if len(logs) > 0 {
  356 + return true
  357 + }
  358 + logs = conditionsEditLog(ctx, querySet, queryComponents)
  359 + if len(logs) > 0 {
  360 + return true
  361 + }
  362 + for _, item := range queryComponents {
  363 + if len(item.Id) == 0 {
  364 + return true
  365 + }
  366 + }
  367 + return false
350 } 368 }
351 369
352 func selectsEditLog(ctx *domain.Context, querySet *domain.QuerySet, queryComponents []*domain.QueryComponent) []FastSourceLog { 370 func selectsEditLog(ctx *domain.Context, querySet *domain.QuerySet, queryComponents []*domain.QueryComponent) []FastSourceLog {
  1 +package redis
  2 +
  3 +import (
  4 + "fmt"
  5 + "github.com/linmadan/egglib-go/utils/json"
  6 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
  7 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
  8 +)
  9 +
  10 +const (
  11 + TableQuerySetExpire = 3600 * 24
  12 +)
  13 +
  14 +func KeyTableQuerySet(tableId int) string {
  15 + return fmt.Sprintf("%v:queryset:temporary:%v", constant.CACHE_PREFIX, tableId)
  16 +}
  17 +
  18 +type TableQuerySetCache struct {
  19 + QuerySetId int `json:"querySetId"`
  20 + TableId int `json:"tableId"`
  21 + QueryComponents []*domain.QueryComponent `json:"queryComponents"`
  22 +}
  23 +
  24 +func NewTableQuerySetCache(querySetId int, tableId int, queryComponents []*domain.QueryComponent) TableQuerySetCache {
  25 + return TableQuerySetCache{
  26 + QuerySetId: querySetId,
  27 + TableId: tableId,
  28 + QueryComponents: queryComponents,
  29 + }
  30 +}
  31 +
  32 +type TableQuerySetCacheService struct {
  33 +}
  34 +
  35 +func (s *TableQuerySetCacheService) SetTableQuerySetCache(key string, cache TableQuerySetCache) error {
  36 + err := ZeroCoreRedis.Setex(key, json.MarshalToString(cache), TemporaryFileExpire)
  37 + return err
  38 +}
  39 +
  40 +func (s *TableQuerySetCacheService) GetTableQuerySetCache(key string) (*TableQuerySetCache, error) {
  41 + var res = &TableQuerySetCache{}
  42 + ok, err := ZeroCoreRedis.Exists(key)
  43 + if !ok {
  44 + return nil, domain.ErrorNotFound
  45 + }
  46 + if err != nil {
  47 + return nil, err
  48 + }
  49 + data, err := ZeroCoreRedis.Get(key)
  50 + if err != nil {
  51 + return nil, err
  52 + }
  53 + err = json.UnmarshalFromString(data, res)
  54 + if err != nil {
  55 + return nil, err
  56 + }
  57 + return res, nil
  58 +}
  59 +
  60 +func NewTableQuerySetCacheService() *TableQuerySetCacheService {
  61 + return &TableQuerySetCacheService{}
  62 +}
@@ -367,3 +367,22 @@ func WrapQueryHasDuplicateRowByIDWithDB(params QueryOptions, db *gorm.DB) func() @@ -367,3 +367,22 @@ func WrapQueryHasDuplicateRowByIDWithDB(params QueryOptions, db *gorm.DB) func()
367 return total, duplicateTotal, nil 367 return total, duplicateTotal, nil
368 } 368 }
369 } 369 }
  370 +
  371 +func WrapQueryHasDuplicateRowBySql(sql string, db *gorm.DB) func() (int64, int64, error) {
  372 + return func() (int64, int64, error) {
  373 + var total int64
  374 + var duplicateTotal int64
  375 + query := db.Raw(sql)
  376 + row := query.Row()
  377 + if row.Err() != nil {
  378 + return total, duplicateTotal, row.Err()
  379 + }
  380 + if err := row.Scan(&total, &duplicateTotal); err != nil {
  381 + return total, duplicateTotal, err
  382 + }
  383 + if total == duplicateTotal {
  384 + return total, duplicateTotal, nil
  385 + }
  386 + return total, duplicateTotal, nil
  387 + }
  388 +}
@@ -187,7 +187,7 @@ func (controller *TableController) CheckRowDuplicate() { @@ -187,7 +187,7 @@ func (controller *TableController) CheckRowDuplicate() {
187 tableService := service.NewTableService(nil) 187 tableService := service.NewTableService(nil)
188 cmd := &command.CheckRowDuplicateCommand{} 188 cmd := &command.CheckRowDuplicateCommand{}
189 Must(controller.Unmarshal(cmd)) 189 Must(controller.Unmarshal(cmd))
190 - data, err := tableService.CheckRowDuplicateV2(ParseContext(controller.BaseController), cmd) 190 + data, err := tableService.CheckRowDuplicateV3(ParseContext(controller.BaseController), cmd)
191 controller.Response(data, err) 191 controller.Response(data, err)
192 } 192 }
193 193