作者 yangfu

chore: some optimize

# fix #3549 #3592
# add whitelist auth
... ... @@ -21,3 +21,4 @@ STARROCKS_PORT = 9030
BLACK_LIST_USER = 0
BLACK_LIST_COMPANY = 1612991734952759296
WHITE_LIST_USERS = 22,23
\ No newline at end of file
... ...
... ... @@ -19,3 +19,6 @@ CREATE INDEX IF NOT EXISTS idx_logs_company_id_created_at ON metadata.logs USING
/*mapping_rules*/
CREATE INDEX IF NOT EXISTS idx_mapping_rules_company_id_table_id_file_id ON metadata.mapping_rules USING btree((context->>'companyId'),table_id,file_id);
/*query_sets*/
CREATE INDEX IF NOT EXISTS idx_query_sets_company_id_type_deleted_at ON metadata.query_sets USING btree((context->>'companyId'),type,deleted_at);
\ No newline at end of file
... ...
... ... @@ -117,6 +117,8 @@ spec:
value: "1"
- name: BLACK_LIST_COMPANY
value: "1646025721363042304"
- name: WHITE_LIST_USERS
value: "0"
volumes:
- name: accesslogs
emptyDir: {}
\ No newline at end of file
... ...
... ... @@ -426,6 +426,15 @@ func (tableService *TableService) ApplyOn(ctx *domain.Context, cmd *command.Appl
} else {
table.TableInfo.SetApplyOn(cmd.Module)
}
if !ctx.Access() {
for i := range cmd.Modules {
// 字库应用于数控目前限制在3万条记录以内,在字库的应用于数控按钮上增加判断,如果超过3万条,
// 错误提示:普通用户数据限制在3万条记录,如需使用更多数据,请联系管理员,需针对公司配置应用记录上限数值,未配置默认3万条
if cmd.Modules[i] == domain.ModuleDigitalCenter && table.RowCount > 30000 {
return nil, factory.FastError(fmt.Errorf("普通用户数据限制在3万条记录,如需使用更多数据,请联系管理员,需针对公司配置应用记录上限数值,未配置默认3万条"))
}
}
}
table, err = tableRepository.Save(table)
if err != nil {
... ...
... ... @@ -2,6 +2,8 @@ package constant
import (
"fmt"
"strconv"
"strings"
)
var SERVICE_NAME = "character-library-metadata-bastion"
... ... @@ -26,8 +28,9 @@ var BYTE_CORE_HOST = "http://192.168.100.34:8303"
var AUTH_SERVER_HOST = "http://digital-platform-dev.fjmaimaimai.com"
var BlacklistUser int64
var BlackListCompany int64
var BLACK_LIST_USER int64
var BLACK_LIST_COMPANY int64
var WHITE_LIST_USERS []int
//var CUSTOMER_ACCOUNT = []int64{3129687560814592, 3129687690100739, 3492238958608384}
... ... @@ -53,6 +56,12 @@ func init() {
PPROF_ON = Configurator.DefaultBool("PPROF_ON", PPROF_ON)
CACHE_PREFIX = SERVICE_NAME + ":" + SERVICE_ENV
BlacklistUser = Configurator.DefaultInt64("BLACK_LIST_USER", BlacklistUser)
BlackListCompany = Configurator.DefaultInt64("BLACK_LIST_COMPANY", BlackListCompany)
BLACK_LIST_USER = Configurator.DefaultInt64("BLACK_LIST_USER", BLACK_LIST_USER)
BLACK_LIST_COMPANY = Configurator.DefaultInt64("BLACK_LIST_COMPANY", BLACK_LIST_COMPANY)
whiteListUsers := strings.Split(Configurator.DefaultString("WHITE_LIST_USERS", ""), ",")
for _, userId := range whiteListUsers {
v, _ := strconv.Atoi(userId)
WHITE_LIST_USERS = append(WHITE_LIST_USERS, v)
}
}
... ...
package domain
import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
type Context struct {
// 公司
CompanyId int `json:"companyId"`
... ... @@ -38,3 +40,12 @@ const (
ContextWithLogLevel = "WithLogLevel"
ContextWithLogMsg = "WithLogMsg"
)
func (c *Context) Access() bool {
for _, userId := range constant.WHITE_LIST_USERS {
if userId == c.OperatorId {
return true
}
}
return false
}
... ...
... ... @@ -141,6 +141,11 @@ func NewFormulaCalculate(table *domain.Table, queryComponent *domain.QueryCompon
if queryComponent.Formula != nil {
formula := queryComponent.Formula
res.DatabaseTableName = formula.TableFields[0].TableSqlName
// TODO:支持常量
//res.DatabaseTableName = ""
//if len(formula.TableFields) > 0 {
// res.DatabaseTableName = formula.TableFields[0].TableSqlName
//}
exprSql := formula.ExprSql
if queryComponent.Formula.MixTableModel() {
exprSql = queryComponent.Formula.Complete()
... ...
... ... @@ -46,6 +46,17 @@ func (f *TemporaryFileInfo) SetFile(file *domain.File) *TemporaryFileInfo {
}
func (f *TemporaryFileInfo) SetFields(fields []*domain.Field) *TemporaryFileInfo {
// 保留原有字段的类型(底层拆分的时候类型会变掉,无法处理,此处做特殊处理)
for i := range fields {
for j := range f.Fields {
if f.Fields[j].Name == fields[i].Name {
if f.Fields[j].SQLType != fields[i].SQLType {
fields[i].SQLType = f.Fields[j].SQLType
break
}
}
}
}
f.Fields = fields
return f
}
... ...
... ... @@ -6,12 +6,14 @@ import (
"github.com/beego/beego/v2/server/web"
"github.com/beego/beego/v2/server/web/context"
"github.com/linmadan/egglib-go/web/beego/filters"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/controllers"
"net/http"
"os"
"strconv"
"strings"
"time"
. "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
_ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/routers"
... ... @@ -53,9 +55,13 @@ func init() {
web.InsertFilter("/*", web.BeforeRouter, filters.AllowCors())
web.InsertFilter("/*", web.BeforeRouter, JwtFilter())
web.InsertFilter("/*", web.BeforeRouter, RequestCostBefore())
web.InsertFilter("/*", web.BeforeExec, controllers.BlacklistFilter(controllers.BlacklistRouters))
web.InsertFilter("/*", web.BeforeExec, CreateRequestLogFilter(true)) // filters.CreateRequstLogFilter(Logger)
if constant.SERVICE_ENV == "dev" { //|| web.BConfig.RunMode =="test"
web.InsertFilter("/*", web.AfterExec, filters.CreateResponseLogFilter(Logger), web.WithReturnOnOutput(false))
}
web.InsertFilter("/*", web.AfterExec, RequestCostAfter(150), web.WithReturnOnOutput(false))
}
func CreateRequestLogFilter(console bool) func(ctx *context.Context) {
... ... @@ -89,3 +95,23 @@ func JwtFilter() func(ctx *context.Context) {
}
}
}
func RequestCostBefore() func(ctx *context.Context) {
return func(ctx *context.Context) {
ctx.Input.SetData("cost-begin", time.Now().UnixMilli())
}
}
func RequestCostAfter(maxCost int64) func(ctx *context.Context) {
return func(ctx *context.Context) {
t := ctx.Input.GetData("cost-begin")
if t != nil {
costBegin := t.(int64)
costEnd := time.Now().UnixMilli()
cost := costEnd - costBegin
if cost > 0 && maxCost > 0 && cost > maxCost {
msg := fmt.Sprintf("beego | %v | %v | 耗时:%v \n %v", ctx.Input.Method(), ctx.Input.URL(), time.Duration(cost)*time.Millisecond, string(ctx.Input.RequestBody))
logs.Warn(msg)
}
}
}
}
... ...
... ... @@ -129,9 +129,9 @@ func BlacklistFilter(black map[string]bool) func(ctx *context.Context) {
if !ok {
return
}
if userToken.UserId > 0 && userToken.UserId == constant.BlacklistUser {
if userToken.UserId > 0 && userToken.UserId == constant.BLACK_LIST_USER {
goto CheckBlackList
} else if userToken.CompanyId > 0 && userToken.UserId == 0 && userToken.CompanyId == constant.BlackListCompany {
} else if userToken.CompanyId > 0 && userToken.UserId == 0 && userToken.CompanyId == constant.BLACK_LIST_COMPANY {
goto CheckBlackList
} else {
return
... ...