Merge remote-tracking branch 'origin/test'

@@ -541,3 +541,18 @@ extractMethod: 提取方法(1.by-date:按日期 2.by-number:按数值)
 ```
 convertType:转换类型 STRING 数值: INT 小数: FLOAT 日期: DATE 时间: DATETIME
 ```
+
+## 优化点
+- [] 0.测试服务、数据库的上限（QPS，TPS）查询瓶颈的接口
+
+- [] 1.模型详情缓存（tables、query_set）
+
+- [] 2.列表搜索缓存 （tables、query_set）,减轻数据库压力
+
+```
+更新、删除、重命名
+get list:queryset:cxxx:*
+删除所有匹配的缓存
+
+消息队列更新事件、确保缓存一定移除成功、缓存时间控制
+```

@@ -21,3 +21,4 @@ STARROCKS_PORT = 9030
 
 BLACK_LIST_USER = 0
 BLACK_LIST_COMPANY = 1612991734952759296
+WHITE_LIST_USERS = 22,23
\ No newline at end of file

@@ -18,4 +18,7 @@ CREATE INDEX IF NOT EXISTS idx_logs_company_id_operator_name ON metadata.logs US
 CREATE INDEX IF NOT EXISTS idx_logs_company_id_created_at ON metadata.logs USING btree((context->>'companyId'),created_at);
 
 /*mapping_rules*/
-CREATE INDEX IF NOT EXISTS idx_mapping_rules_company_id_table_id_file_id ON metadata.mapping_rules USING btree((context->>'companyId'),table_id,file_id);
\ No newline at end of file
+CREATE INDEX IF NOT EXISTS idx_mapping_rules_company_id_table_id_file_id ON metadata.mapping_rules USING btree((context->>'companyId'),table_id,file_id);
+
+/*query_sets*/
+CREATE INDEX IF NOT EXISTS idx_query_sets_company_id_type_deleted_at ON metadata.query_sets USING btree((context->>'companyId'),type,deleted_at);
\ No newline at end of file

@@ -117,6 +117,8 @@ spec:
           value: "1"
         - name: BLACK_LIST_COMPANY
           value: "1646025721363042304"
+        - name: WHITE_LIST_USERS
+          value: "0"
       volumes:
       - name: accesslogs
         emptyDir: {}
\ No newline at end of file

@@ -45,5 +45,7 @@ require (
 
 replace (
 	github.com/extrame/xls v0.0.1 => github.com/tiptok/xls v1.0.1
-	github.com/linmadan/egglib-go v0.0.0-20210313060205-8b5e456b11f7 => github.com/tiptok/egglib-go v0.0.0-20220421085958-9682d0ac42c1
+	//github.com/linmadan/egglib-go v0.0.0-20210313060205-8b5e456b11f7 => github.com/tiptok/egglib-go v0.0.0-20220421085958-9682d0ac42c1
+	github.com/linmadan/egglib-go v0.0.0-20210313060205-8b5e456b11f7 => github.com/tiptok/egglib-go v1.0.0
+
 )

@@ -311,12 +311,12 @@ func (querySetService *QuerySetService) SearchQuerySet(ctx *domain.Context, sear
 	if err != nil {
 		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
 	}
-	if err := transactionContext.StartTransaction(); err != nil {
-		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	}
-	defer func() {
-		transactionContext.RollbackTransaction()
-	}()
+	//if err := transactionContext.StartTransaction(); err != nil {
+	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
+	//}
+	//defer func() {
+	//	transactionContext.RollbackTransaction()
+	//}()
 
 	QuerySetRepository, _, _ := factory.FastPgQuerySet(transactionContext, 0)
 	options := utils.ObjectToMap(searchQuerySetQuery)
@@ -361,9 +361,9 @@ func (querySetService *QuerySetService) SearchQuerySet(ctx *domain.Context, sear
 
 	var result = dto.NewQuerySetDtoList(querySets)
 
-	if err := transactionContext.CommitTransaction(); err != nil {
-		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	}
+	//if err := transactionContext.CommitTransaction(); err != nil {
+	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
+	//}
 	return map[string]interface{}{
 		"count":     count,
 		"querySets": result,

@@ -426,6 +426,15 @@ func (tableService *TableService) ApplyOn(ctx *domain.Context, cmd *command.Appl
 	} else {
 		table.TableInfo.SetApplyOn(cmd.Module)
 	}
+	if !ctx.Access() {
+		for i := range cmd.Modules {
+			// 字库应用于数控目前限制在3万条记录以内，在字库的应用于数控按钮上增加判断，如果超过3万条，
+			// 错误提示：普通用户数据限制在3万条记录，如需使用更多数据，请联系管理员，需针对公司配置应用记录上限数值，未配置默认3万条
+			if cmd.Modules[i] == domain.ModuleDigitalCenter && table.RowCount > 30000 {
+				return nil, factory.FastError(fmt.Errorf("普通用户数据限制在3万条记录，如需使用更多数据，请联系管理员，需针对公司配置应用记录上限数值，未配置默认3万条"))
+			}
+		}
+	}
 
 	table, err = tableRepository.Save(table)
 	if err != nil {

@@ -19,12 +19,12 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
 	if err != nil {
 		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
 	}
-	if err := transactionContext.StartTransaction(); err != nil {
-		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	}
-	defer func() {
-		transactionContext.RollbackTransaction()
-	}()
+	//if err := transactionContext.StartTransaction(); err != nil {
+	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
+	//}
+	//defer func() {
+	//	transactionContext.RollbackTransaction()
+	//}()
 
 	tableRepository, _, _ := factory.FastPgTable(transactionContext, 0)
 	_, tables, err := tableRepository.Find(utils.ObjectToMap(searchQuery))
@@ -112,9 +112,9 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
 		groupItem.LoadGroup(querySetGroup)
 		response = append(response, groupItem)
 	}
-	if err := transactionContext.CommitTransaction(); err != nil {
-		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	}
+	//if err := transactionContext.CommitTransaction(); err != nil {
+	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
+	//}
 
 	sort.Slice(response, func(i, j int) bool {
 		item1 := response[i]

@@ -21,12 +21,12 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
 	if err != nil {
 		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
 	}
-	if err := transactionContext.StartTransaction(); err != nil {
-		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	}
-	defer func() {
-		transactionContext.RollbackTransaction()
-	}()
+	//if err := transactionContext.StartTransaction(); err != nil {
+	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
+	//}
+	//defer func() {
+	//	transactionContext.RollbackTransaction()
+	//}()
 	var dataTable *domain.DataTable
 	var table *domain.Table
 	var cacheMiss bool
@@ -91,8 +91,8 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
 		// 存储缓存
 		cache.SetDataTable(table.TableId, dataTable)
 	}
-	if err := transactionContext.CommitTransaction(); err != nil {
-		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	}
+	//if err := transactionContext.CommitTransaction(); err != nil {
+	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
+	//}
 	return response, nil
 }

@@ -2,6 +2,8 @@ package constant
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
 )
 
 var SERVICE_NAME = "character-library-metadata-bastion"
@@ -26,8 +28,9 @@ var BYTE_CORE_HOST = "http://192.168.100.34:8303"
 
 var AUTH_SERVER_HOST = "http://digital-platform-dev.fjmaimaimai.com"
 
-var BlacklistUser int64
-var BlackListCompany int64
+var BLACK_LIST_USER int64
+var BLACK_LIST_COMPANY int64
+var WHITE_LIST_USERS []int
 
 //var CUSTOMER_ACCOUNT = []int64{3129687560814592, 3129687690100739, 3492238958608384}
 
@@ -53,6 +56,12 @@ func init() {
 	PPROF_ON = Configurator.DefaultBool("PPROF_ON", PPROF_ON)
 	CACHE_PREFIX = SERVICE_NAME + ":" + SERVICE_ENV
 
-	BlacklistUser = Configurator.DefaultInt64("BLACK_LIST_USER", BlacklistUser)
-	BlackListCompany = Configurator.DefaultInt64("BLACK_LIST_COMPANY", BlackListCompany)
+	BLACK_LIST_USER = Configurator.DefaultInt64("BLACK_LIST_USER", BLACK_LIST_USER)
+	BLACK_LIST_COMPANY = Configurator.DefaultInt64("BLACK_LIST_COMPANY", BLACK_LIST_COMPANY)
+
+	whiteListUsers := strings.Split(Configurator.DefaultString("WHITE_LIST_USERS", ""), ",")
+	for _, userId := range whiteListUsers {
+		v, _ := strconv.Atoi(userId)
+		WHITE_LIST_USERS = append(WHITE_LIST_USERS, v)
+	}
 }

 package domain
 
+import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
+
 type Context struct {
 	// 公司
 	CompanyId int `json:"companyId"`
@@ -38,3 +40,12 @@ const (
 	ContextWithLogLevel = "WithLogLevel"
 	ContextWithLogMsg   = "WithLogMsg"
 )
+
+func (c *Context) Access() bool {
+	for _, userId := range constant.WHITE_LIST_USERS {
+		if userId == c.OperatorId {
+			return true
+		}
+	}
+	return false
+}

@@ -71,7 +71,7 @@ func (t *DataTable) OptionalValue(args ...string) []string {
 			if len(match) > 0 && !strings.Contains(t.Data[i][0], match) {
 				continue
 			}
-			if filedType == Float.ToString() {
+			if filedType == Float.ToString() || filedType == Date.ToString() {
 				values = append(values, RoundFieldValue(&Field{SQLType: filedType}, t.Data[i][0]))
 				continue
 			}

@@ -272,6 +272,10 @@ func RoundFieldValue(f *Field, v string) string {
 		}
 		return utils.AssertString(fv) //fmt.Sprintf("%v", fv)
 	}
+	// TODO:计算表 Count(日期)类型是日期类型,0 的话下面日期解析成当前时间，计算表类型需要修改
+	if fv, err := strconv.ParseFloat(v, 64); err == nil {
+		return utils.AssertString(fv)
+	}
 	if f.SQLType == Datetime.ToString() {
 		fv, err := xtime.Parse(v)
 		if err != nil {
@@ -336,6 +340,11 @@ func MakeToInterfaces(fields []*Field) func([]string) []interface{} {
 				if fields[i].SQLType == Float.ToString() {
 					v = RoundFieldValue(fields[i], v)
 				}
+				if fields[i].SQLType == Date.ToString() {
+					v = RoundFieldValue(fields[i], v)
+					output[i] = v
+					continue
+				}
 				convValue, err := ValueToType(v, fields[i].SQLType)
 				if err == nil {
 					output[i] = convValue

@@ -140,7 +140,12 @@ func NewFormulaCalculate(table *domain.Table, queryComponent *domain.QueryCompon
 	}
 	if queryComponent.Formula != nil {
 		formula := queryComponent.Formula
-		res.DatabaseTableName = formula.TableFields[0].TableSqlName
+		// res.DatabaseTableName = formula.TableFields[0].TableSqlName
+		// TODO:支持常量
+		res.DatabaseTableName = ""
+		if len(formula.TableFields) > 0 {
+			res.DatabaseTableName = formula.TableFields[0].TableSqlName
+		}
 		exprSql := formula.ExprSql
 		if queryComponent.Formula.MixTableModel() {
 			exprSql = queryComponent.Formula.Complete()
@@ -153,6 +158,14 @@ func NewFormulaCalculate(table *domain.Table, queryComponent *domain.QueryCompon
 				CalculateFieldName:  table.DataFields[0].SQLName,
 			})
 		}
+		if len(queryComponent.Formula.TableFields) == 0 && len(queryComponent.Formula.ExprSql) > 0 {
+			res.FormulaCalculateFields = append(res.FormulaCalculateFields, &FormulaCalculateField{
+				DatabaseTableName:   "",
+				FieldSchema:         NewFieldSchema(domain.TableField{}),
+				CalculateExpression: exprSql,
+				CalculateFieldName:  table.DataFields[0].SQLName,
+			})
+		}
 		//res.FormulaCalculateFields = append(res.FormulaCalculateFields, &FormulaCalculateField{
 		//	DatabaseTableName:   res.DatabaseTableName,
 		//	FieldSchema:         NewFieldSchemaFromField(table.DataFields[0]),

@@ -46,6 +46,17 @@ func (f *TemporaryFileInfo) SetFile(file *domain.File) *TemporaryFileInfo {
 }
 
 func (f *TemporaryFileInfo) SetFields(fields []*domain.Field) *TemporaryFileInfo {
+	// 保留原有字段的类型（底层拆分的时候类型会变掉，无法处理，此处做特殊处理）
+	for i := range fields {
+		for j := range f.Fields {
+			if f.Fields[j].Name == fields[i].Name {
+				if f.Fields[j].SQLType != fields[i].SQLType {
+					fields[i].SQLType = f.Fields[j].SQLType
+					break
+				}
+			}
+		}
+	}
 	f.Fields = fields
 	return f
 }

@@ -128,7 +128,7 @@ func (repository *QuerySetRepository) Remove(querySet *domain.QuerySet) (*domain
 	return querySet, nil
 }
 func (repository *QuerySetRepository) FindOne(queryOptions map[string]interface{}) (*domain.QuerySet, error) {
-	tx := repository.transactionContext.PgTx
+	tx := repository.transactionContext.DB()
 	querySetModel := new(models.QuerySet)
 	query := sqlbuilder.BuildQuery(tx.Model(querySetModel), queryOptions)
 	WhereContext(query, queryOptions)
@@ -152,7 +152,7 @@ func (repository *QuerySetRepository) FindOne(queryOptions map[string]interface{
 	}
 }
 func (repository *QuerySetRepository) Find(queryOptions map[string]interface{}) (int64, []*domain.QuerySet, error) {
-	tx := repository.transactionContext.PgTx
+	tx := repository.transactionContext.DB()
 	var querySetModels []*models.QuerySet
 	querySets := make([]*domain.QuerySet, 0)
 	query := sqlbuilder.BuildQuery(tx.Model(&querySetModels), queryOptions)

@@ -133,7 +133,7 @@ func (repository *TableRepository) Remove(table *domain.Table) (*domain.Table, e
 	return table, nil
 }
 func (repository *TableRepository) FindOne(queryOptions map[string]interface{}) (*domain.Table, error) {
-	tx := repository.transactionContext.PgTx
+	tx := repository.transactionContext.DB()
 	tableModel := new(models.Table)
 	query := sqlbuilder.BuildQuery(tx.Model(tableModel), queryOptions)
 	WhereContext(query, queryOptions)
@@ -161,7 +161,7 @@ func (repository *TableRepository) FindOne(queryOptions map[string]interface{})
 	}
 }
 func (repository *TableRepository) Find(queryOptions map[string]interface{}) (int64, []*domain.Table, error) {
-	tx := repository.transactionContext.PgTx
+	tx := repository.transactionContext.DB()
 	var tableModels []*models.Table
 	tables := make([]*domain.Table, 0)
 	query := sqlbuilder.BuildQuery(tx.Model(&tableModels), queryOptions)

@@ -6,12 +6,14 @@ import (
 	"github.com/beego/beego/v2/server/web"
 	"github.com/beego/beego/v2/server/web/context"
 	"github.com/linmadan/egglib-go/web/beego/filters"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/controllers"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	. "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
 	_ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/routers"
@@ -24,6 +26,7 @@ func init() {
 	web.BConfig.Listen.HTTPPort = 8080
 	web.BConfig.Listen.EnableAdmin = false
 	web.BConfig.WebConfig.CommentRouterPath = "/pkg/port/beego/routers"
+	web.BConfig.Log.AccessLogs = true
 	if os.Getenv("RUN_MODE") != "" {
 		web.BConfig.RunMode = os.Getenv("RUN_MODE")
 	}
@@ -53,14 +56,18 @@ func init() {
 
 	web.InsertFilter("/*", web.BeforeRouter, filters.AllowCors())
 	web.InsertFilter("/*", web.BeforeRouter, JwtFilter())
+	web.InsertFilter("/*", web.BeforeRouter, RequestCostBefore())
 	web.InsertFilter("/*", web.BeforeExec, controllers.BlacklistFilter(controllers.BlacklistRouters))
 	web.InsertFilter("/*", web.BeforeExec, CreateRequestLogFilter(true)) //  filters.CreateRequstLogFilter(Logger)
-	web.InsertFilter("/*", web.AfterExec, filters.CreateResponseLogFilter(Logger), web.WithReturnOnOutput(false))
+	if constant.SERVICE_ENV == "dev" {                                   //|| web.BConfig.RunMode =="test"
+		web.InsertFilter("/*", web.AfterExec, filters.CreateResponseLogFilter(Logger), web.WithReturnOnOutput(false))
+	}
+	web.InsertFilter("/*", web.AfterExec, RequestCostAfter(150), web.WithReturnOnOutput(false))
 }
 
 func CreateRequestLogFilter(console bool) func(ctx *context.Context) {
 	return func(ctx *context.Context) {
-		msg := fmt.Sprintf("beego | %v | %v \n %v", ctx.Input.Method(), ctx.Input.URL(), string(ctx.Input.RequestBody))
+		msg := fmt.Sprintf("beego | %v | %v | %v \n %v", ctx.Input.Method(), strings.Split(ctx.Request.RemoteAddr, ":")[0], ctx.Input.URL(), string(ctx.Input.RequestBody))
 		logs.Debug(msg)
 		if console {
 			fmt.Println(msg)
@@ -89,3 +96,23 @@ func JwtFilter() func(ctx *context.Context) {
 		}
 	}
 }
+func RequestCostBefore() func(ctx *context.Context) {
+	return func(ctx *context.Context) {
+		ctx.Input.SetData("cost-begin", time.Now().UnixMilli())
+	}
+}
+
+func RequestCostAfter(maxCost int64) func(ctx *context.Context) {
+	return func(ctx *context.Context) {
+		t := ctx.Input.GetData("cost-begin")
+		if t != nil {
+			costBegin := t.(int64)
+			costEnd := time.Now().UnixMilli()
+			cost := costEnd - costBegin
+			if cost > 0 && maxCost > 0 && cost > maxCost {
+				msg := fmt.Sprintf("beego | %v | %v | %v | 耗时:%v \n %v", ctx.Input.Method(), strings.Split(ctx.Request.RemoteAddr, ":")[0], ctx.Input.URL(), time.Duration(cost)*time.Millisecond, string(ctx.Input.RequestBody))
+				logs.Warn(msg)
+			}
+		}
+	}
+}

@@ -129,9 +129,9 @@ func BlacklistFilter(black map[string]bool) func(ctx *context.Context) {
 			if !ok {
 				return
 			}
-			if userToken.UserId > 0 && userToken.UserId == constant.BlacklistUser {
+			if userToken.UserId > 0 && userToken.UserId == constant.BLACK_LIST_USER {
 				goto CheckBlackList
-			} else if userToken.CompanyId > 0 && userToken.UserId == 0 && userToken.CompanyId == constant.BlackListCompany {
+			} else if userToken.CompanyId > 0 && userToken.UserId == 0 && userToken.CompanyId == constant.BLACK_LIST_COMPANY {
 				goto CheckBlackList
 			} else {
 				return