作者 Administrator

合并分支 'test' 到 'master'

Test



查看合并请求 !2
正在显示 43 个修改的文件 包含 795 行增加294 行删除
... ... @@ -39,9 +39,7 @@ spec:
operator: In
values:
- cn-hangzhou.i-bp1djh1xn7taumbue1ze
- cn-hangzhou.i-bp1djh1xn7taumbue1zd
- cn-hangzhou.i-bp1euf5u1ph9kbhtndhb
- cn-hangzhou.i-bp1hyp5oips9cdwxxgxy
containers:
- name: character-library-metadata-bastion
image: 192.168.0.243:5000/mmm/character-library-metadata-bastion:dev
... ...
... ... @@ -42,7 +42,7 @@ func (querySetService *QuerySetService) CalculateSetPreview(ctx *domain.Context,
response.ObjectType = querySet.Type
response.Name = querySet.Name
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false, true), int64(len(dataTable.Data)))
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
... ...
... ... @@ -494,7 +494,7 @@ func GetItemValues(transactionContext application.TransactionContext, q *query.C
var result = make([]itemValue, 0)
result = append(result, itemValue{
Name: querySet.Name,
Value: value,
Value: domain.RoundValue(value),
})
for _, f := range q.Formula.TableFields {
... ... @@ -511,7 +511,7 @@ func GetItemValues(transactionContext application.TransactionContext, q *query.C
})
result = append(result, itemValue{
Name: f.FieldName,
Value: value,
Value: domain.RoundValue(value),
})
}
return querySet, result
... ...
... ... @@ -20,7 +20,10 @@ type TablePreviewCommand struct {
}
func (cmd *TablePreviewCommand) Valid(validation *validation.Validation) {
if cmd.UseCache && cmd.PageSize==0{
cmd.PageNumber = 1
cmd.PageSize = 10000 //默认缓存前10000条
}
if cmd.PageSize > 0 {
cmd.Where.PageNumber = cmd.PageNumber
cmd.Where.PageSize = cmd.PageSize
... ...
... ... @@ -71,7 +71,10 @@ func (tableService *TableService) FieldOptionalValues(ctx *domain.Context, cmd *
if table.TableType == domain.SubTable.ToString() && field.Flag == domain.ManualField {
return empty, nil
}
match := cmd.Match
if !domain.SQLType(field.SQLType).IsString() {
match = ""
}
options := &starrocks.QueryOptions{
Table: table,
TableName: table.SQLName,
... ... @@ -80,7 +83,7 @@ func (tableService *TableService) FieldOptionalValues(ctx *domain.Context, cmd *
{
Condition: domain.Condition{
Field: field,
Like: cmd.Match,
Like: match,
Order: "ASC",
},
Distinct: true,
... ... @@ -107,8 +110,9 @@ func (tableService *TableService) FieldOptionalValues(ctx *domain.Context, cmd *
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
values := dataTable.OptionalValue(cmd.Match, field.SQLType)
return map[string]interface{}{
"values": dataTable.OptionalValue(),
"total": dataTable.Total,
"values": values,
"total": len(values),
}, nil
}
... ...
... ... @@ -11,6 +11,9 @@ import (
)
func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command.TablePreviewCommand) (interface{}, error) {
if cmd.TableId == 0 {
return &dto.TablePreviewDto{}, nil
}
if err := cmd.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
... ... @@ -68,6 +71,10 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
if err != nil {
return nil, factory.FastError(err)
}
// 计算项只返回第一个值
// if table.TableType==domain.CalculateItem.ToString() && dataTable!=nil && len(dataTable.Data)>0{
// dataTable.Data = dataTable.Data[:1]
// }
response.Load(table, dataTable, domain.ObjectMetaTable)
}
}
... ...
... ... @@ -28,9 +28,6 @@ type (
IsFromOriginalTable bool `json:"isFromOriginalTable"`
TableFileUrl string `json:"tableFileUrl"`
ColumnSchemas []ColumnSchema `json:"columnSchemas"`
//PageNumber int `json:"pageNumber"`
//PageSize int `json:"pageSize"`
//QueryParameters []QueryParameter `json:"queryParameters"`
SortParameters map[string]interface{} `json:"sortParameters"`
}
... ... @@ -50,14 +47,6 @@ type (
ColumnContents []string `json:"columnContents"`
IsContainContent bool `json:"isContainContent"`
}
//Field struct {
// // 索引序号
// Index int `json:"index"`
// // 名称
// Name string `json:"name"`
// // 对应数据库类型
// Type string `json:"sqlType"`
//}
InValidCell struct {
Col string `json:"col"`
... ... @@ -157,11 +146,6 @@ type (
MainTable *Table
Table *Table
CopyToTable *Table
// 数据列
//DataFields []*Field `json:"dataFields"`
// 手动添加的列
//ManualFields []*Field `json:"manualFields"`
}
DataCopyTable struct {
... ... @@ -249,32 +233,13 @@ func ToFields(fields []*Field) []*Field {
return result
}
//func ToFieldData(fields []*Field, data [][]string, byteName bool) []map[string]string {
// var result = make([]map[string]string, 0)
// var key string
// for _, d := range data {
// var item = make(map[string]string)
// for j, f := range fields {
// key = f.Name
// if byteName {
// key = f.Name
// }
// if len(d) >= j {
// item[key] = d[j]
// } else {
// item[key] = ""
// }
// }
// result = append(result, item)
// }
// return result
//}
type (
ReqFormulasGenerate struct {
QuerySet *QuerySet
Table *Table
QueryComponents []*QueryComponent
QuerySetService interface{}
Context interface{}
}
DataFormulasGenerate struct {
... ...
... ... @@ -9,7 +9,7 @@ type Context struct {
OperatorName string `json:"operatorName"`
// 租户 (个人、企业)
TenantId int `json:"tenantId"`
// 附加数据
data map[string]interface{}
}
... ...
package domain
import "strings"
type DataTable struct {
Fields []*Field `json:"fields"`
Data [][]string `json:"data"`
... ... @@ -51,19 +53,28 @@ type Condition struct {
Order string `json:"order"`
}
func (t *DataTable) OptionalValue() []string {
//set := make(map[string]string)
func (t *DataTable) OptionalValue(args ...string) []string {
var values = make([]string, 0)
match := ""
filedType := ""
if len(args) > 0 {
match = args[0]
}
if len(args) > 1 {
filedType = args[1]
}
if len(t.Data) > 0 && len(t.Data[0]) == 1 {
for i := range t.Data {
if len(t.Data[i]) == 0 {
continue
}
//if _, ok := set[t.Data[i][0]]; ok {
// continue
//} else {
// set[t.Data[i][0]] = ""
//}
if len(match) > 0 && !strings.Contains(t.Data[i][0], match) {
continue
}
if filedType == Float.ToString() {
values = append(values, RoundFieldValue(&Field{SQLType: filedType}, t.Data[i][0]))
continue
}
values = append(values, t.Data[i][0])
}
}
... ... @@ -85,7 +96,7 @@ func (t *DataTable) Values(f *Field) []string {
for i := range t.Data {
for j := range t.Data[i] {
if j == index {
res = append(res, t.Data[i][j])
res = append(res, RoundFieldValue(f, t.Data[i][j]))
break
}
}
... ...
... ... @@ -125,6 +125,8 @@ var (
Int SQLType = "INT"
BigInt SQLType = "BIGINT"
Float SQLType = "FLOAT"
DECIMALV2 SQLType = "DECIMAL(20,15)" //"DECIMALV2"
DECIMAL279 SQLType = "DECIMAL(20,15)"
Date SQLType = "DATE"
Datetime SQLType = "DATETIME"
)
... ... @@ -134,6 +136,7 @@ var SQLTypeMap = map[string]string{
Int.ToString(): "整数",
BigInt.ToString(): "整数",
Float.ToString(): "小数",
DECIMALV2.ToString(): "小数",
Date.ToString(): "日期",
Datetime.ToString(): "日期时间",
}
... ...
... ... @@ -4,6 +4,7 @@ import (
"fmt"
"github.com/linmadan/egglib-go/utils/xtime"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
"strconv"
"strings"
)
... ... @@ -185,6 +186,16 @@ func ValueToType(value string, sqlType string) (interface{}, error) {
if err != nil {
err = fmt.Errorf("[%v]不是有效的浮点数类型", value)
}
case DECIMALV2.ToString():
toTypeVal, err = numberString.Float64()
if err != nil {
err = fmt.Errorf("[%v]不是有效的浮点数类型", value)
}
case DECIMAL279.ToString():
toTypeVal, err = numberString.Float64()
if err != nil {
err = fmt.Errorf("[%v]不是有效的浮点数类型", value)
}
case Date.ToString():
toTypeVal, err = xtime.Parse(value)
if err != nil {
... ... @@ -203,9 +214,13 @@ func ValueToType(value string, sqlType string) (interface{}, error) {
return toTypeVal, err
}
func ToFieldData(fields []*Field, data [][]string, byName bool) []map[string]string {
func ToFieldData(fields []*Field, data [][]string, byName bool, configs ...bool) []map[string]string {
var result = make([]map[string]string, 0)
var key string
var ignoreEmptyString = false
if len(configs) > 0 {
ignoreEmptyString = configs[0]
}
for _, d := range data {
var item = make(map[string]string)
for j, f := range fields {
... ... @@ -213,8 +228,12 @@ func ToFieldData(fields []*Field, data [][]string, byName bool) []map[string]str
if byName {
key = f.Name
}
if ignoreEmptyString && len(d[j]) == 0 {
continue
}
if len(d) >= j {
item[key] = d[j]
// 处理精度问题
item[key] = RoundFieldValue(f, d[j]) //d[j]
} else {
item[key] = ""
}
... ... @@ -224,6 +243,35 @@ func ToFieldData(fields []*Field, data [][]string, byName bool) []map[string]str
return result
}
// RoundFieldValue 字段值精度处理
func RoundFieldValue(f *Field, v string) string {
if f.SQLType == Float.ToString() {
fv, err := strconv.ParseFloat(v, 64)
if err != nil {
return v
}
return fmt.Sprintf("%v", fv)
}
return v
//if f.SQLType != DECIMALV2.ToString() {
// return v
//}
//fv, err := strconv.ParseFloat(v, 64)
//if err != nil {
// return v
//}
//fv = utils.Round(fv, 6)
//return fmt.Sprintf("%v", fv)
}
func RoundValue(v string) string {
fv, err := strconv.ParseFloat(v, 64)
if err != nil {
return v
}
return fmt.Sprintf("%v", fv)
}
func GripData(data []map[string]string, total int64) map[string]interface{} {
if len(data) == 0 {
data = make([]map[string]string, 0)
... ... @@ -250,6 +298,10 @@ func MakeToInterfaces(fields []*Field) func([]string) []interface{} {
output := make([]interface{}, len(input))
for i, v := range input {
if i < len(fields) {
// 处理精度问题
if fields[i].SQLType == Float.ToString() {
v = RoundFieldValue(fields[i], v)
}
convValue, err := ValueToType(v, fields[i].SQLType)
if err == nil {
output[i] = convValue
... ...
package domain
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestRoundFieldValue(t *testing.T) {
floatFiled := &Field{
SQLType: Float.ToString(),
}
inputs := []struct {
v string
f *Field
want string
}{
{
v: "0.12359999999999999",
f: floatFiled,
want: "0.1236",
},
{
v: "0.12360000000000001",
f: floatFiled,
want: "0.1236",
},
{
v: "0.12359999999999995",
f: floatFiled,
want: "0.1236",
},
}
for _, input := range inputs {
got := RoundFieldValue(input.f, input.v)
assert.Equal(t, input.want, got)
}
}
... ...
package domain
import (
"path"
"path/filepath"
"strings"
"time"
)
... ... @@ -16,8 +16,6 @@ type File struct {
FileInfo *FileInfo `json:"fileInfo"`
// 源文件Id(FileType为TemporaryFile或VerifiedFile时有值)
SourceFileId int `json:"sourceFileId"`
// 操作人
// Operator string `json:"operator"`
// 创建时间
CreatedAt time.Time `json:"createdAt"`
// 更新时间
... ... @@ -60,8 +58,13 @@ func (file *File) Update(data map[string]interface{}) error {
}
func FileName(fileName string) string {
if len(fileName) == 0 {
return ""
}
base := filepath.Base(fileName)
return strings.Split(base, ".")[0]
suffix := path.Ext(fileName)
prefix := base[0 : len(base)-len(suffix)]
return prefix
}
func (file *File) CopyTo(fileType FileType, ctx *Context) *File {
... ... @@ -78,7 +81,6 @@ func (file *File) CopyTo(fileType FileType, ctx *Context) *File {
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
SourceFileId: file.FileId,
//Operator: file.Operator,
Context: ctx,
}
return t
... ...
... ... @@ -8,8 +8,6 @@ type FileInfo struct {
Url string `json:"url"`
// 文件大小
FileSize int `json:"fileSize"`
// 文件编号,以固定字符“F”+4位年+2位月+2位日+三位流水,每天编号从001开始
// FileCode string `json:"fileCode"`
// 后缀扩展
Ext string `json:"ext"`
// 记录数
... ...
package domain
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestFileName(t *testing.T) {
inputs := []struct {
input string
want string
}{
{
input: "device/sdk/CMakeLists.txt",
want: "CMakeLists",
},
{
input: "",
want: "",
},
{
input: "CMakeLists.txt",
want: "CMakeLists",
},
{
input: "d:\\CMakeLists.txt",
want: "CMakeLists",
},
}
for _, input := range inputs {
got := FileName(input.input)
assert.Equal(t, input.want, got)
}
}
... ...
... ... @@ -45,6 +45,5 @@ func (log *Log) Identify() interface{} {
}
func (log *Log) Update(data map[string]interface{}) error {
return nil
}
... ...
... ... @@ -2,7 +2,7 @@ package domain
import "time"
// 匹配规则配置
// MappingRule 匹配规则配置
type MappingRule struct {
// 匹配规则ID
MappingRuleId int `json:"mappingRuleId"`
... ... @@ -53,7 +53,6 @@ type MappingField struct {
func NewMappingFields(all []*Field, some []*Field) []*MappingField {
var result []*MappingField
//allMap :=(Fields)(all).ToMap()
someMap := (Fields)(some).ToMap()
for _, f := range all {
item := &MappingField{
... ...
... ... @@ -8,7 +8,7 @@ import (
"time"
)
// 查询集合
// QuerySet 查询集合
type QuerySet struct {
// 查询集合ID
QuerySetId int `json:"querySetId"`
... ...
... ... @@ -76,7 +76,6 @@ type SelectExpr struct { // 查询表达式
FieldLeft FieldExpr `json:"fieldLeft"`
FieldRight FieldExpr `json:"fieldRight"` // has value when type is equal to 1
Type string `json:"type"` // 1.拆分 2.拆方赋值 3.正常赋值
//SubGroup []SelectExpr `json:"subGroup,omitempty"`
}
func (s SelectExpr) Equal(compare SelectExpr) bool {
... ... @@ -94,7 +93,6 @@ type SelectExprGroup struct { // 查询表达式
}
type FieldExpr struct {
//LabelColumns []LabelColumn `json:"labelColumns"`
TableFields []TableField `json:"tableFields"`
ExprHuman string `json:"exprHuman"`
ExprSql string `json:"exprSql"`
... ... @@ -104,15 +102,11 @@ func (expr *FieldExpr) Complete() string {
exprSql := expr.ExprSql
for _, f := range expr.TableFields {
sql := fmt.Sprintf("%s.%s", f.TableSqlName, f.FieldSqlName)
//zhSql := fmt.Sprintf("%s.%s", f.TableName, f.FieldSqlName)
sub := fmt.Sprintf("max(%s.%s)", f.TableSqlName, f.FieldSqlName)
//zhSub := fmt.Sprintf("max(%s.%s)", f.TableName, f.FieldSqlName)
if strings.Contains(expr.ExprSql, fmt.Sprintf("(%s)", sql)) {
continue
}
exprSql = strings.ReplaceAll(exprSql, sql, sub)
//expr.ExprSql = strings.ReplaceAll(expr.ExprSql, sql, sub)
//expr.ExprHuman = strings.ReplaceAll(expr.ExprHuman, zhSql, zhSub)
}
return exprSql
}
... ... @@ -227,3 +221,20 @@ func SelectsExprToMapById(items []SelectExpr) map[string]SelectExpr {
}
return res
}
type QueryComponents []*QueryComponent
func (list QueryComponents) IsEmpty(t TableType) bool {
if len(list) == 0 {
return true
}
for _, q := range list {
switch t {
case CalculateTable:
if q.Aggregation != nil && len(q.Aggregation.RowFields) == 0 && len(q.Aggregation.ValueFields) == 0 {
return true
}
}
}
return false
}
... ...
package domain
import (
"github.com/zeromicro/go-zero/core/collection"
"sort"
"strings"
)
const (
CellTypeTable = "Table"
CellTypeTableField = "TableField"
... ... @@ -38,24 +44,12 @@ type Location struct {
}
type LayoutCellData struct {
//Table *Table `json:"table,omitempty"`
//Field *Field `json:"field,omitempty"`
TableField *TableField `json:"tableField"`
Text string `json:"text,omitempty"`
}
func (l *LayoutRuleItem) LayoutCells() []*LayoutCell {
var cells = make([]*LayoutCell, 0)
//for i, rows := range l.Cells {
// for j, item := range rows {
// if item.Type == CellTypeNull || item.Type == "" {
// continue
// }
// item.X = i
// item.Y = j
// cells = append(cells, item)
// }
//}
for _, item := range l.Cells {
item.X = item.Position.X
item.Y = item.Position.Y
... ... @@ -63,3 +57,19 @@ func (l *LayoutRuleItem) LayoutCells() []*LayoutCell {
}
return cells
}
type LayoutCells []*LayoutCell
func (cells LayoutCells) CellsRange(direction string) []int {
list := collection.NewSet()
for i := range cells {
if strings.ToLower(direction) == "x" {
list.Add(cells[i].X)
} else {
list.Add(cells[i].Y)
}
}
sortList := list.KeysInt()
sort.Ints(sortList)
return sortList
}
... ...
... ... @@ -2,5 +2,4 @@ package domain
type QuerySetInfo struct {
BindTableId int // 查询集绑定的表
//DependentTables []int // 依赖的表
}
... ...
... ... @@ -67,16 +67,39 @@ func (table *Table) TableIdString() string {
func (table *Table) WithContext(ctx *Context) *Table {
rand.Seed(time.Now().Unix())
table.SQLName = fmt.Sprintf("%v_t%v_c%v", table.SQLName, rand.Intn(1000000), ctx.CompanyId)
table.SQLName = fmt.Sprintf("%v_t%v_c%v", limitStringLen(table.SQLName, 40), rand.Intn(1000000), limitStringLen(fmt.Sprintf("%d", ctx.CompanyId), 8))
table.Context = ctx
return table
}
func limitStringLen(s string, l int) string {
result := s
subLength := l / 2
if len(result) > l {
result = result[:subLength] + result[len(result)-subLength:]
}
return result
}
func (table *Table) WithPrefix(prefix string) *Table {
if strings.HasPrefix(table.SQLName, "_") {
table.SQLName = fmt.Sprintf("%v%v", strings.ToLower(prefix), table.SQLName)
return table
}
table.SQLName = fmt.Sprintf("%v_%v", strings.ToLower(prefix), table.SQLName)
return table
}
func (table *Table) ApplyDefaultModule() *Table {
if table.TableType == CalculateTable.ToString() || table.TableType == SubProcessTable.ToString() {
table.TableInfo.SetApplyOn(ModuleQuerySetCenter)
}
if table.TableType == SchemaTable.ToString() {
table.TableInfo.SetApplyOn(ModuleQuerySetCenter | ModuleCalculateCenter)
}
return table
}
func (table *Table) WithParentId(parentId int) *Table {
table.ParentId = parentId
return table
... ... @@ -91,44 +114,52 @@ func (table *Table) Update(data map[string]interface{}) error {
return nil
}
func (t *Table) Fields(includePK bool) []*Field {
func (table *Table) Fields(includePK bool) []*Field {
var fields []*Field
if includePK && t.PK != nil {
fields = append(fields, t.PK)
if includePK && table.PK != nil {
fields = append(fields, table.PK)
}
fields = append(fields, t.DataFields...)
if t.TableType == SubTable.ToString() {
fields = append(fields, t.ManualFields...)
fields = append(fields, table.DataFields...)
if table.TableType == SubTable.ToString() {
fields = append(fields, table.ManualFields...)
}
t.fields = fields
table.fields = fields
return t.fields
return table.fields
}
func (t *Table) MatchField(field *Field) (*Field, bool) {
if len(t.fields) == 0 {
t.fields = t.Fields(true)
func (table *Table) MatchField(field *Field) (*Field, bool) {
if len(table.fields) == 0 {
table.fields = table.Fields(true)
}
mField := (Fields)(t.fields).ToMap()
mField := (Fields)(table.fields).ToMap()
if v, ok := mField[field.Name]; ok {
return v, true
}
return nil, false
}
func (t *Table) DependencyTables() []int {
if t.TableInfo == nil {
func (table *Table) DependencyTables() []int {
if table.TableInfo == nil {
return []int{}
}
return t.TableInfo.DependencyTables
return table.TableInfo.DependencyTables
}
func (t *Table) AssertTableType(types ...TableType) bool {
func (table *Table) AssertTableType(types ...TableType) bool {
for _, item := range types {
if t.TableType == item.ToString() {
if table.TableType == item.ToString() {
return true
}
}
return false
}
func TableTypesToStringList(list ...TableType) []string {
var result = make([]string, 0)
for _, item := range list {
result = append(result, item.ToString())
}
return result
}
... ...
package domain
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestLimitStringLen(t *testing.T){
inputs:=[]struct{
input string
length int
want string
}{
{
input: "123456789",
length: 6,
want: "123789",
},
{
input: "123456789",
length: 7,
want: "123789",
},
{
input: "123456789",
length: 10,
want: "123456789",
},
{
input: "pai_xu_ce_shi_zhu_biao_fu_zhi",
length: 20,
want: "pai_xu_ce_iao_fu_zhi",
},
}
for _,input:=range inputs{
got:=limitStringLen(input.input,input.length)
assert.Equal(t,input.want,got)
}
}
\ No newline at end of file
... ...
package domain
// 用户对象
// User 用户对象
type User struct {
// 用户Id 用户唯一标识
Id int `json:"id"`
... ...
package bytelib
import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
func DomainFieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema {
result := make([]domain.ColumnSchema, 0)
for _, f := range fields {
res := domain.ColumnSchema{
ColumnName: f.Name,
ColumnType: f.SQLType,
}
if convertFiledSQLType(f.SQLType) {
res.ColumnType = domain.DECIMAL279.ToString()
}
result = append(result, res)
}
return result
}
func ToFieldSchemas(fields []*domain.Field) []FieldSchema {
result := make([]FieldSchema, 0)
for _, f := range fields {
res := FieldSchema{
FieldZhName: f.Name,
FieldEnName: f.SQLName,
FieldType: f.SQLType,
FieldDescription: f.Description,
IsAllowNull: true,
}
if convertFiledSQLType(f.SQLType) {
res.FieldType = domain.DECIMAL279.ToString()
}
result = append(result, res)
}
return result
}
func NewFieldSchema(f domain.TableField) FieldSchema {
var res = FieldSchema{
FieldZhName: f.FieldName,
FieldEnName: f.FieldSqlName,
FieldType: f.FieldSQLType,
FieldDescription: "",
IsAllowNull: true,
}
if convertFiledSQLType(f.FieldSQLType) {
res.FieldType = domain.DECIMAL279.ToString()
}
return res
}
func NewFieldSchemaFromField(f *domain.Field) FieldSchema {
var res = FieldSchema{
FieldZhName: f.Name,
FieldEnName: f.SQLName,
FieldType: f.SQLType,
FieldDescription: "",
IsAllowNull: true,
}
if convertFiledSQLType(f.SQLType) {
res.FieldType = domain.DECIMAL279.ToString()
}
return res
}
func convertFiledSQLType(sqlType string) bool {
if sqlType == domain.Float.ToString() || sqlType == domain.DECIMAL279.ToString() {
return true
}
return false
}
func FieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema {
result := make([]domain.ColumnSchema, 0)
for _, f := range fields {
result = append(result, domain.ColumnSchema{
ColumnName: f.Name,
ColumnType: f.SQLType,
})
}
return result
}
... ...
... ... @@ -96,30 +96,6 @@ type ResponseCheckoutTablesSave struct {
CheckoutTableUrl string `json:"checkoutTableUrl"`
}
func FieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema {
result := make([]domain.ColumnSchema, 0)
for _, f := range fields {
result = append(result, domain.ColumnSchema{
ColumnName: f.Name,
ColumnType: f.SQLType,
})
}
return result
}
func DomainFieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema {
result := make([]domain.ColumnSchema, 0)
for _, f := range fields {
result = append(result, domain.ColumnSchema{
ColumnName: f.Name,
ColumnType: f.SQLType, // domain.String.ToString(),
})
}
return result
}
func ToDataLoadDataTable(data DataCheckoutTables) *domain.DataLoadDataTable {
response := &domain.DataLoadDataTable{
PageNumber: data.PageNumber,
... ... @@ -181,28 +157,6 @@ func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable)
return request
}
func ToFieldSchemas(fields []*domain.Field) []FieldSchema {
result := make([]FieldSchema, 0)
for _, f := range fields {
result = append(result, FieldSchema{
FieldZhName: f.Name,
FieldEnName: f.SQLName,
FieldType: f.SQLType,
FieldDescription: f.Description,
IsAllowNull: true,
})
}
return result
}
func ToFieldSchemaEnNames(fields []*domain.Field) []string {
result := make([]string, 0)
for _, f := range fields {
result = append(result, f.SQLName)
}
return result
}
type (
TableAppendRequest struct {
//MasterTableId string `json:"masterTableId"`
... ... @@ -243,10 +197,6 @@ func NewTableAppendRequest(param domain.ReqAppendData) TableAppendRequest {
req.SchemaMap[param.To[i].SQLName] = columnSchemas[i]
}
}
//if len(param.From) > 0 {
// req.ColumnSchemas = DomainFieldsToColumnSchemas(param.From)
// req.FieldSchemas = ToFieldSchemas(param.To)
//}
return req
}
... ...
... ... @@ -162,12 +162,7 @@ func NewFormulaCalculate(table *domain.Table, queryComponent *domain.QueryCompon
}
if queryComponent.Aggregation != nil {
res.DatabaseTableName = queryComponent.MasterTable.SQLName
for i, f := range queryComponent.Aggregation.ValueFields {
tableField, ok := table.MatchField(&domain.Field{Name: f.DisplayName})
if !ok {
continue
}
if i == 0 {
if len(queryComponent.Aggregation.ValueFields) > 0 || len(queryComponent.Aggregation.RowFields) > 0 {
res.FormulaCalculateFields = append(res.FormulaCalculateFields, &FormulaCalculateField{
DatabaseTableName: queryComponent.MasterTable.SQLName,
FieldSchema: NewFieldSchemaFromField(&domain.Field{
... ... @@ -179,6 +174,11 @@ func NewFormulaCalculate(table *domain.Table, queryComponent *domain.QueryCompon
CalculateFieldName: "id",
})
}
for _, f := range queryComponent.Aggregation.ValueFields {
tableField, ok := table.MatchField(&domain.Field{Name: f.DisplayName})
if !ok {
continue
}
res.FormulaCalculateFields = append(res.FormulaCalculateFields, &FormulaCalculateField{
DatabaseTableName: queryComponent.MasterTable.SQLName,
FieldSchema: NewFieldSchemaFromField(f.Field),
... ... @@ -252,28 +252,6 @@ func NewFormulaField(f domain.FieldExpr, args ...interface{}) FormulaField {
return res
}
func NewFieldSchema(f domain.TableField) FieldSchema {
var res = FieldSchema{
FieldZhName: f.FieldName,
FieldEnName: f.FieldSqlName,
FieldType: f.FieldSQLType,
FieldDescription: "",
IsAllowNull: true,
}
return res
}
func NewFieldSchemaFromField(f *domain.Field) FieldSchema {
var res = FieldSchema{
FieldZhName: f.Name,
FieldEnName: f.SQLName,
FieldType: f.SQLType,
FieldDescription: "",
IsAllowNull: true,
}
return res
}
func NewFormulaDataHandleRule(s domain.SelectExprGroup) FormulaDataHandleRule {
var res = FormulaDataHandleRule{
RuleType: 1,
... ...
... ... @@ -9,6 +9,7 @@ import (
)
var DefaultCache = cache.New(12*time.Hour, 1*time.Hour)
var TableCacheExpire = 24 * 60 * 60 * 2 // 两天过期
func KeyCompanyUser(companyId int, userId int) string {
return fmt.Sprintf("local:cache:user:%d:%d", companyId, userId)
... ... @@ -26,15 +27,19 @@ func SetDataTable(tableId int, table *domain.DataTable) {
DefaultDataTableCacheService.SetDataTable(tableId, table)
}
var DefaultDataTableCacheService = &DataTableCacheService{}
// 1:优化点 redis 一万条数据占用内存 10M 无法大量使用,需要内存+本地文件形式
//var DefaultDataTableCacheService DataTableCacheService= &DataTableRedisCacheService{}
type DataTableCacheService struct {
var DefaultDataTableCacheService DataTableCacheService = &DataTableLocalCacheService{}
type DataTableLocalCacheService struct {
}
func (svr *DataTableCacheService) KeyDataTable(tableId int) string {
func (svr *DataTableLocalCacheService) KeyDataTable(tableId int) string {
return fmt.Sprintf("local:cache:table:%d", tableId)
}
func (svr *DataTableCacheService) GetDataTable(tableId int) (*domain.DataTable, bool) {
func (svr *DataTableLocalCacheService) GetDataTable(tableId int) (*domain.DataTable, bool) {
v, ok := DefaultCache.Get(KeyDataTable(tableId))
if !ok {
log.Logger.Info(fmt.Sprintf("【缓存】 miss cache %v", KeyDataTable(tableId)))
... ... @@ -43,11 +48,20 @@ func (svr *DataTableCacheService) GetDataTable(tableId int) (*domain.DataTable,
log.Logger.Info(fmt.Sprintf("【缓存】 hit cache %v", KeyDataTable(tableId)))
return v.(*domain.DataTable), true
}
func (svr *DataTableCacheService) SetDataTable(tableId int, table *domain.DataTable) {
func (svr *DataTableLocalCacheService) SetDataTable(tableId int, table *domain.DataTable) {
log.Logger.Info(fmt.Sprintf("【缓存】 set cache %v", KeyDataTable(tableId)))
DefaultCache.Set(KeyDataTable(tableId), table, 24*time.Hour*30)
DefaultCache.Set(KeyDataTable(tableId), table, time.Duration(TableCacheExpire)*time.Second)
}
func (svr *DataTableCacheService) DeleteDataTable(tableId int) {
func (svr *DataTableLocalCacheService) DeleteDataTable(tableId int) {
log.Logger.Info(fmt.Sprintf("【缓存】 delete cache %v", KeyDataTable(tableId)))
DefaultCache.Delete(KeyDataTable(tableId))
}
type DataTableCacheService interface {
KeyDataTable(tableId int) string
GetDataTable(tableId int) (*domain.DataTable, bool)
SetDataTable(tableId int, table *domain.DataTable)
DeleteDataTable(tableId int)
}
... ...
package cache
import (
"fmt"
"github.com/linmadan/egglib-go/utils/json"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
)
type DataTableRedisCacheService struct {
}
func (svr *DataTableRedisCacheService) KeyDataTable(tableId int) string {
return fmt.Sprintf("%s:cache:table:%d", constant.CACHE_PREFIX, tableId)
}
func (svr *DataTableRedisCacheService) GetDataTable(tableId int) (*domain.DataTable, bool) {
key := svr.KeyDataTable(tableId)
data, err := redis.ZeroCoreRedis.Get(key)
if err == redis.NotFound || len(data) == 0 {
log.Logger.Info(fmt.Sprintf("【缓存】 miss cache %v", key))
return nil, false
}
var table = &domain.DataTable{}
err = json.UnmarshalFromString(data, table)
if err != nil {
log.Logger.Error(fmt.Sprintf("【缓存】 miss cache %v,err %v", key, err.Error()))
return nil, false
}
log.Logger.Info(fmt.Sprintf("【缓存】 hit cache %v", key))
return table, true
}
func (svr *DataTableRedisCacheService) SetDataTable(tableId int, table *domain.DataTable) {
key := svr.KeyDataTable(tableId)
log.Logger.Info(fmt.Sprintf("【缓存】 set cache %v", key))
redis.ZeroCoreRedis.Setex(key, json.MarshalToString(table), TableCacheExpire)
}
func (svr *DataTableRedisCacheService) DeleteDataTable(tableId int) {
key := svr.KeyDataTable(tableId)
log.Logger.Info(fmt.Sprintf("【缓存】 delete cache %v", key))
redis.ZeroCoreRedis.Del(key)
}
... ...
... ... @@ -2,6 +2,7 @@ package domainService
import (
"bytes"
"github.com/beego/beego/v2/client/httplib"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
... ... @@ -146,6 +147,13 @@ func (ptr *ByteCoreService) FormulasGenerate(param domain.ReqFormulasGenerate) (
// }
// return &domain.DataFormulasGenerate{}, nil
//}
if param.QuerySet.Type == domain.CalculateSet.ToString() {
_, err := param.QuerySetService.(*QuerySetService).LoadCalculateSetData(param.Context.(*domain.Context), param.QuerySet, param.QueryComponents)
if err != nil {
return nil, err
}
return &domain.DataFormulasGenerate{}, nil
}
return apiByteLib.FormulasGenerate(param)
}
... ...
... ... @@ -33,8 +33,7 @@ func (ptr *FlushDataTableService) Flush(ctx *domain.Context, fileId int, table *
table = NewTable(domain.ExcelTable, file.FileInfo.Name, table.DataFields, table.RowCount).WithContext(ctx)
// 通知底层保存、进行回调
var response *domain.DataSaveTable
response, err = ByteCore.SaveTable(domain.ReqSaveTable{FileId: fileId, Table: table})
if err != nil {
if response, err = ByteCore.SaveTable(domain.ReqSaveTable{FileId: fileId, Table: table}); err != nil {
return nil, err
}
// 来自源文件的
... ... @@ -154,12 +153,11 @@ func NewTable(tableType domain.TableType, fileName string, dataFields []*domain.
func NewCopyTable(tableType domain.TableType, fileName string, dataFields []*domain.Field, rowCount int) *domain.Table {
var table = &domain.Table{}
// New Table
table.TableType = tableType.ToString()
table.Name = fileName
table.SQLName = pin(fileName) //SQLTableName()
table.SQLName = pin(fileName)
table.PK = PK()
if table.TableType == domain.CalculateTable.ToString() || table.TableType == domain.CalculateItem.ToString() {
if table.TableType == domain.CalculateItem.ToString() {
table.PK = nil
}
table.DataFieldIndex = len(dataFields)
... ... @@ -192,7 +190,7 @@ func DataField(name string, sqlType string, flag int, index int) *domain.Field {
return &domain.Field{
Index: index,
Name: name,
SQLName: fmt.Sprintf("%v_c%d", pin(name), index), //fieldName(index),
SQLName: fmt.Sprintf("%v_c%d", pin(name), index),
SQLType: sqlType,
Description: "",
Flag: flag,
... ...
... ... @@ -25,17 +25,17 @@ func (ptr *GenerateMainTableService) GenerateTable(ctx *domain.Context, fileId i
return nil, fmt.Errorf("文件未校验")
}
duplicateTable, err := tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableName": tableName,
"tableTypes":[]string{string(domain.MainTable),string(domain.SubTable),string(domain.SideTable)}})
"tableTypes": []string{string(domain.MainTable), string(domain.SubTable), string(domain.SideTable)}})
if err == nil && duplicateTable != nil {
return nil, fmt.Errorf("表名称重复")
}
mainTable := NewTable(domain.MainTable, tableName, table.DataFields, table.RowCount).WithContext(ctx)
_, err = tableRepository.Save(mainTable)
if err != nil {
mainTable := NewTable(domain.MainTable, tableName, table.DataFields, table.RowCount).
WithContext(ctx).
WithPrefix(domain.MainTable.ToString())
if _, err = tableRepository.Save(mainTable); err != nil {
return nil, err
}
// 日志
if err = FastLog(ptr.transactionContext, domain.CommonLog, mainTable.TableId, &GenerateMainTableLog{
LogEntry: domain.NewLogEntry(tableName, domain.MainTable.ToString(), domain.GenerateMainTable, ctx),
... ...
... ... @@ -91,7 +91,7 @@ type Log interface {
var _ Log = (*FileUploadSuccessLog)(nil)
// 1.1文件上传成功
// FileUploadSuccessLog 1.1文件上传成功
type FileUploadSuccessLog struct {
domain.LogEntry
}
... ... @@ -100,7 +100,7 @@ func (l *FileUploadSuccessLog) Content() string {
return fmt.Sprintf("上传成功")
}
// 1.2文件上传失败
// FileUploadFailLog 1.2文件上传失败
type FileUploadFailLog struct {
domain.LogEntry
Reason string
... ... @@ -110,7 +110,7 @@ func (l *FileUploadFailLog) Content() string {
return fmt.Sprintf("上传失败,失败原因:%s", l.Reason)
}
// 2.文件校验
// FileVerifyLog 2.文件校验
type FileVerifyLog struct {
domain.LogEntry
// 错误信息
... ... @@ -127,7 +127,7 @@ func (l *FileVerifyLog) Content() string {
return msg
}
// 3.主表生成日志
// GenerateMainTableLog 3.主表生成日志
type GenerateMainTableLog struct {
domain.LogEntry
// 表名
... ... @@ -141,7 +141,7 @@ func (l *GenerateMainTableLog) Content() string {
return msg
}
// 4.主表拆分
// SpiltMainTableLog 4.主表拆分
type SpiltMainTableLog struct {
domain.LogEntry
Reserve []*domain.Field
... ... @@ -175,7 +175,7 @@ func (l *SpiltMainTableLog) fieldNames(fields []*domain.Field) []string {
return names
}
// 5.分表编辑
// SubTableEditLog 5.分表编辑
type SubTableEditLog struct {
domain.LogEntry
... ... @@ -208,7 +208,7 @@ func (l *SubTableEditLog) fieldNames(fields []*domain.Field) []string {
return names
}
// 6.表复制日志
// CopyTableLog 6.表复制日志
type CopyTableLog struct {
domain.LogEntry
// 表名
... ... @@ -220,7 +220,7 @@ func (l *CopyTableLog) Content() string {
return msg
}
// 7.编辑记录
// RowAddLog 7.编辑记录
type RowAddLog struct {
domain.LogEntry
}
... ... @@ -239,10 +239,8 @@ type RowUpdateLog struct {
func (l *RowUpdateLog) Content() string {
change := ""
//index := l.Number + l.Where.Offset()
for _, f := range l.FieldValue {
if f.OldValue != f.Value {
//change += fmt.Sprintf("%v字段%v行的值从%v更改为%v;", f.Field.Name, index, f.OldValue, f.Value)
change += fmt.Sprintf("【%v】字段的值从“%v”更改为“%v”;", f.Field.Name, f.OldValue, f.Value)
}
}
... ... @@ -261,7 +259,6 @@ type RowRemoveLog struct {
func (l *RowRemoveLog) Content() string {
index := l.DeleteRowCount
//msg := fmt.Sprintf("删除%v行数据;筛选件:%v",index,"")
msg := fmt.Sprintf("删除%v行数据;", index)
filters := make([]string, 0)
inArgs := func(args []string) string {
... ... @@ -281,7 +278,7 @@ func (l *RowRemoveLog) Content() string {
return msg
}
// 8.表删除日志
// DeleteTableLog 8.表删除日志
type DeleteTableLog struct {
domain.LogEntry
// 表名
... ... @@ -302,7 +299,7 @@ func (l *DeleteTableLog) Content() string {
return msg
}
// 9.数据追加日志
// AppendDataToTableLog 9.数据追加日志
type AppendDataToTableLog struct {
domain.LogEntry
Table *domain.Table
... ... @@ -323,7 +320,8 @@ func (l *AppendDataToTableLog) Content() string {
return msg
}
/*步骤日志*/
/* *********************************************步骤日志************************************************** */
type ExcelTableEditLog struct {
domain.LogEntry
// 操作名称
... ...
... ... @@ -278,6 +278,9 @@ func (ptr *QuerySetService) PreviewPrepare(ctx *domain.Context, querySetId int,
return t, nil
}
}
if domain.QueryComponents(queryComponents).IsEmpty(domain.TableType(querySet.Type)) {
return &domain.Table{}, nil
}
// 验证
if err = ptr.validQueryComponents(queryComponents); err != nil {
... ... @@ -308,9 +311,6 @@ func (ptr *QuerySetService) PreviewPrepare(ctx *domain.Context, querySetId int,
var table *domain.Table = NewCopyTable(domain.TableType(domain.TemporaryTable), querySet.Name, domain.RangeFields(fields, domain.ChangeFieldFlag), 0).
WithContext(ctx).
WithPrefix(strings.ToLower(string(domain.TemporaryTable)))
if querySet.Type == domain.CalculateTable.ToString() {
table.PK = nil
}
// 循环依赖判断
if err = ptr.validDependentCircle(ctx, querySet, queryComponents); err != nil {
return nil, err
... ... @@ -937,29 +937,26 @@ func (ptr *QuerySetService) Copy(ctx *domain.Context, querySetId int, t string,
if err != nil {
return nil, err
}
copyTable := NewCopyTable(domain.TableType(t), name, table.Fields(false), 0).WithContext(ctx).WithPrefix(qs.Type)
copyTable := NewCopyTable(domain.TableType(t), name, table.Fields(false), 0).
WithContext(ctx).
WithPrefix(qs.Type).
ApplyDefaultModule()
copyTable, err = tableRepository.Save(copyTable)
if err != nil {
return nil, err
}
// 调用底层的组装sql
formulasGenerateResponse, err := ByteCore.FormulasGenerate(domain.ReqFormulasGenerate{
_, err = ByteCore.FormulasGenerate(domain.ReqFormulasGenerate{
QuerySet: qs,
Table: copyTable,
QueryComponents: qs.QueryComponents,
QuerySetService: ptr,
Context: ctx,
})
if err != nil {
return nil, err
}
if len(formulasGenerateResponse.FormulaName) > 0 && formulasGenerateResponse.FormulaName != table.SQLName {
//copyTable.SQLName = formulasGenerateResponse.FormulaName
//tableRepository, _ := repository.NewTableRepository(ptr.transactionContext)
//copyTable, err = tableRepository.Save(copyTable)
//if err != nil {
// return nil, err
//}
}
copy.QuerySetInfo.BindTableId = copyTable.TableId
}
copy, err = querySetRepository.Save(copy)
... ...
... ... @@ -2,16 +2,18 @@ package domainService
import (
"fmt"
"sort"
"strings"
"github.com/zeromicro/go-zero/core/collection"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
"strings"
)
const DefaultExpandNum = 10000
const MaxExpandNum = 50000
const DefaultExpandNum = 1000
const MaxExpandNum = 5000
func (ptr *QuerySetService) LoadCalculateSetData(ctx *domain.Context, qs *domain.QuerySet, queryComponents []*domain.QueryComponent) (*domain.DataTable, error) {
var (
... ... @@ -26,7 +28,13 @@ func (ptr *QuerySetService) LoadCalculateSetData(ctx *domain.Context, qs *domain
// 加载Tables数据
q := queryComponents[0]
cells := q.Layout.LayoutCells()
dataTables = ptr.LoadDataTables(ctx, cells)
if len(cells) == 0 {
return res, nil
}
dataTables, err = ptr.LoadDataTables(ctx, cells)
if err != nil {
return nil, err
}
// 设置数据
dt := &DataLayoutDataTable{
DataTable: res,
... ... @@ -38,10 +46,10 @@ func (ptr *QuerySetService) LoadCalculateSetData(ctx *domain.Context, qs *domain
cells[i].Length = length
cells[i].BlockData = blockData
}
// 根据数据修改便宜
// 根据数据修改偏移
CellsLocationAdjust(cells)
// 数据布局
res, err = DataLayout(res, dataTables, cells)
res, err = DataLayout(res, cells)
if err != nil {
return nil, err
}
... ... @@ -60,18 +68,6 @@ func CellsLocationAdjust(cells []*domain.LayoutCell) {
yMin = cells[0].Y
yMax = cells[0].Y
}
min := func(a, b int) int {
if a > b {
return b
}
return a
}
max := func(a, b int) int {
if a < b {
return b
}
return a
}
for i := 1; i <= len(cells)-1; i++ {
cell := cells[i]
xMin = min(xMin, cell.X)
... ... @@ -111,7 +107,7 @@ func CellsLocationAdjust(cells []*domain.LayoutCell) {
}
//move = cell.Length
//c = cell
if max(move, cell.Length) != move {
if max(move, cell.Length) != move && c == nil {
c = cell
}
move = max(move, cell.Length)
... ... @@ -121,6 +117,61 @@ func CellsLocationAdjust(cells []*domain.LayoutCell) {
}
}
func CellsLocationAdjustV1(cells []*domain.LayoutCell) {
yList := cellsRange(cells, "y")
xList := cellsRange(cells, "x")
for i := 0; i < len(yList); i++ {
j := yList[i]
move := 0
var c *domain.LayoutCell
for _, cell := range cells {
if cell.Y != j {
continue
}
if cell.Direction != domain.DirectionRight {
continue
}
if max(move, cell.Length) != move {
c = cell
}
move = max(move, cell.Length)
}
ChangeLocation(cells, domain.DirectionRight, j, move, c)
}
for j := 0; j < len(xList); j++ {
i := xList[j]
move := 0
var c *domain.LayoutCell
for _, cell := range cells {
if cell.X != i {
continue
}
if cell.Direction != domain.DirectionDown {
continue
}
if max(move, cell.Length) != move && c == nil {
c = cell
}
move = max(move, cell.Length)
}
ChangeLocation(cells, domain.DirectionDown, i, move, c)
}
}
func cellsRange(cells []*domain.LayoutCell, direction string) []int {
list := collection.NewSet()
for i := range cells {
if strings.ToLower(direction) == "x" {
list.Add(cells[i].X)
} else {
list.Add(cells[i].Y)
}
}
sortList := list.KeysInt()
sort.Ints(sortList)
return sortList
}
func ChangeLocation(cells []*domain.LayoutCell, direction string, position, move int, c *domain.LayoutCell) {
// log.Logger.Info("修改定位点")
if move == 0 {
... ... @@ -148,6 +199,8 @@ func FastTable(table *domain.Table) (*domain.DataTable, error) {
TableName: table.SQLName,
Select: table.Fields(false),
}
options.SetDefaultOrder()
options.SetOffsetLimit(1, 10000)
var dataTable *domain.DataTable
dataTable, err = FastDataTable(options)
if err != nil {
... ... @@ -164,7 +217,6 @@ func FastDataTable(options starrocks.QueryOptions) (*domain.DataTable, error) {
if err != nil {
return nil, err
}
dataTable.Total, err = starrocks.WrapQueryCountWithDB(options, starrocks.DB)()
if err != nil {
return nil, err
... ... @@ -172,7 +224,7 @@ func FastDataTable(options starrocks.QueryOptions) (*domain.DataTable, error) {
return dataTable, nil
}
func (ptr *QuerySetService) LoadDataTables(ctx *domain.Context, cells []*domain.LayoutCell) map[int]*domain.DataTable {
func (ptr *QuerySetService) LoadDataTables(ctx *domain.Context, cells []*domain.LayoutCell) (map[int]*domain.DataTable, error) {
var (
dataTables = make(map[int]*domain.DataTable)
tableRepository, _ = repository.NewTableRepository(ptr.transactionContext)
... ... @@ -187,28 +239,27 @@ func (ptr *QuerySetService) LoadDataTables(ctx *domain.Context, cells []*domain.
if len(tableIds.KeysInt()) > 0 {
_, tables, err := tableRepository.Find(map[string]interface{}{"context": ctx, "tableIds": tableIds.KeysInt()})
if err != nil {
return nil
return nil, err
}
for _, t := range tables {
if _, ok := dataTables[t.TableId]; ok {
continue
}
dataTable, e := FastTable(t)
if e != nil {
log.Logger.Error(e.Error())
continue
dataTable, err := FastTable(t)
if err != nil {
log.Logger.Error(err.Error())
return nil, fmt.Errorf("获取【%s】出现异常:%s", t.Name, err.Error())
}
dataTable.Fields = t.DataFields
dataTables[t.TableId] = dataTable
}
}
return dataTables
return dataTables, nil
}
func DataLayout(res *domain.DataTable, dataTables map[int]*domain.DataTable, cells []*domain.LayoutCell) (*domain.DataTable, error) {
func DataLayout(res *domain.DataTable, cells []*domain.LayoutCell) (*domain.DataTable, error) {
dt := &DataLayoutDataTable{
DataTable: res,
//MapDataTables: dataTables,
unprocessed: cells,
}
dt.Init(DefaultExpandNum)
... ... @@ -218,14 +269,6 @@ func DataLayout(res *domain.DataTable, dataTables map[int]*domain.DataTable, cel
}
cell := dt.unprocessed[0]
dt.unprocessed = dt.unprocessed[1:]
//blockData, length := dt.BlockData(cell)
//if err := dt.Expand(cell, length); err != nil {
// return nil, err
//}
//dt.addByLocation(cell, blockData)
//blockData, length := dt.BlockData(cell)
// 当前单元格子 影响其他格子坐标
//dt.changeUnProcessedLocation(cell, cell.Length)
if err := dt.Expand(cell, cell.Length); err != nil {
return nil, err
}
... ... @@ -273,8 +316,13 @@ func (l *Location) UpdateY(y int) {
}
}
func (d *DataLayoutDataTable) StartCell() {
func (l *Location) Update(x int, y int, compare func(int, int) int) {
if x != 0 {
l.X = compare(l.X, x)
}
if y != 0 {
l.Y = compare(l.Y, y)
}
}
func (d *DataLayoutDataTable) addByLocation(cell *domain.LayoutCell, blockData []string) {
... ... @@ -287,23 +335,19 @@ func (d *DataLayoutDataTable) addByLocation(cell *domain.LayoutCell, blockData [
for i := range blockData {
d.DataTable.Data[cell.X][cell.Y+i] = blockData[i]
}
d.PointEnd.UpdateX(cell.X)
d.PointEnd.UpdateY(cell.Y + len(blockData) - 1)
d.PointEnd.Update(cell.X, cell.Y+len(blockData)-1, max)
case domain.DirectionDown:
for i := range blockData {
d.DataTable.Data[cell.X+i][cell.Y] = blockData[i]
}
d.PointEnd.UpdateX(cell.X + len(blockData) - 1)
d.PointEnd.UpdateY(cell.Y)
d.PointEnd.Update(cell.X+len(blockData)-1, cell.Y, max)
case domain.DirectionNone:
d.DataTable.Data[cell.X][cell.Y] = blockData[0]
d.PointEnd.UpdateX(cell.X)
d.PointEnd.UpdateY(cell.Y)
d.PointEnd.Update(cell.X, cell.Y, max)
}
}
func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutCell, length int) {
// log.Logger.Info("修改定位点")
for _, cell := range d.unprocessed {
switch lastCell.Direction {
case domain.DirectionRight:
... ... @@ -315,7 +359,6 @@ func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutC
cell.X += length - 1
}
}
// log.Logger.Info(fmt.Sprintf("%s %s X:%d Y:%d", cell.Data.Field.SQLName, cell.Direction, cell.X, cell.Y))
}
}
... ... @@ -329,7 +372,7 @@ func (d *DataLayoutDataTable) BlockData(cells *domain.LayoutCell) ([]string, int
if !ok {
return block, 0
}
values := table.Values(&domain.Field{SQLName: cells.Data.TableField.FieldSqlName})
values := table.Values(&domain.Field{SQLName: cells.Data.TableField.FieldSqlName, SQLType: cells.Data.TableField.FieldSQLType})
if len(values) == 0 {
return block, 0
}
... ... @@ -411,3 +454,16 @@ func (d *DataLayoutDataTable) CellOutRange(cell *domain.LayoutCell, length int)
}
return false
}
func min(a, b int) int {
if a > b {
return b
}
return a
}
func max(a, b int) int {
if a < b {
return b
}
return a
}
... ...
... ... @@ -13,6 +13,7 @@ func TestDataLayout(t *testing.T) {
cells []*domain.LayoutCell
flag Location
title string
debug bool
}{
{
title: "配置组多组混合",
... ... @@ -410,6 +411,93 @@ func TestDataLayout(t *testing.T) {
flag: Location{X: 2, Y: 1},
},
{
title: "测试用例4",
cells: []*domain.LayoutCell{
// 分组一
{
X: 0,
Y: 0,
Length: 5,
ImageData: "a",
Direction: domain.DirectionDown,
},
{
X: 0,
Y: 1,
Length: 5,
ImageData: "b",
Direction: domain.DirectionDown,
},
// ,
// {
// X: 1,
// Y: 1,
// Length: 5,
// ImageData: "c",
// Direction: domain.DirectionRight,
// },
// {
// X: 2,
// Y: 1,
// Length: 5,
// ImageData: "d",
// Direction: domain.DirectionRight,
// },
},
flag: Location{X: 0, Y: 0},
},
{
title: "测试用例5",
cells: []*domain.LayoutCell{
// 分组一
{
X: 0,
Y: 0,
Length: 5,
ImageData: "a",
Direction: domain.DirectionDown,
},
{
X: 0,
Y: 1,
Length: 6,
ImageData: "b",
Direction: domain.DirectionDown,
},
{
X: 1,
Y: 0,
Length: 1,
ImageData: "t",
Direction: domain.DirectionNone,
},
{
X: 1,
Y: 1,
Length: 1,
ImageData: "x",
Direction: domain.DirectionNone,
},
// ,
// {
// X: 1,
// Y: 1,
// Length: 5,
// ImageData: "c",
// Direction: domain.DirectionRight,
// },
// {
// X: 2,
// Y: 1,
// Length: 5,
// ImageData: "d",
// Direction: domain.DirectionRight,
// },
},
flag: Location{X: 0, Y: 0},
debug:true,
},
{
title: "测试用例6",
cells: []*domain.LayoutCell{
// 分组一
... ... @@ -445,13 +533,17 @@ func TestDataLayout(t *testing.T) {
}
}
}
debugItem:= false
for _, input := range inputs {
if debugItem && !input.debug{
continue
}
padding(input.cells)
// 根据数据修改位移
CellsLocationAdjust(input.cells)
// 数据布局
res := &domain.DataTable{}
res, err := DataLayout(res, nil, input.cells)
res, err := DataLayout(res, input.cells)
if err != nil {
assert.NoError(t, err)
}
... ... @@ -500,6 +592,20 @@ b | | | | | | | | |
b | | | | | | | | |
| | | | | d | d | d | d | d
a | b
a | b
a | b
a | b
a | b
a | b
a | b
a | b
a | b
a | b
| b
t | x
a | | | | |
| c | c | c | c | c
| d | d | d | d | d
... ...
... ... @@ -40,7 +40,7 @@ func (ptr *AddTableStructService) AddTableStruct(ctx *domain.Context, parentTabl
fields = MappingFields(mainTable, fields)
dataFields := (domain.Fields)(fields).Select(map[string]interface{}{"flag": domain.MainTableField})
manualFields := (domain.Fields)(fields).Select(map[string]interface{}{"flag": domain.ManualField})
table := NewTable(domain.SubTable, name, fields, mainTable.RowCount).WithContext(ctx)
table := NewTable(domain.SubTable, name, fields, mainTable.RowCount).WithContext(ctx).WithPrefix(string(domain.SubTable))
table.DataFieldIndex = mainTable.DataFieldIndex
table.DataFields = dataFields
table.ManualFields = manualFields
... ...
... ... @@ -2,6 +2,7 @@ package domainService
import (
"fmt"
pgTransaction "github.com/linmadan/egglib-go/transaction/pg"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository"
... ... @@ -28,9 +29,6 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab
if err != nil {
return nil, err
}
if !(table.TableType == domain.MainTable.ToString() || table.TableType == domain.SubTable.ToString()) {
return nil, fmt.Errorf("主表、分表才允许复制")
}
var mainTable *domain.Table
dataFields := table.DataFields
if table.TableType == domain.SubTable.ToString() {
... ... @@ -45,7 +43,10 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab
}
}
// 验证表名是否重复
duplicateTable, err := tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableName": tableName})
duplicateTable, err := tableRepository.FindOne(map[string]interface{}{
"context": ctx,
"tableName": tableName,
"tableTypes": domain.TableTypesToStringList(domain.MainTable, domain.SubTable, domain.SideTable)})
if err == nil && duplicateTable != nil {
return nil, fmt.Errorf("表名称重复")
}
... ... @@ -53,7 +54,8 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab
sideTable := NewCopyTable(domain.SideTable, tableName, dataFields, table.RowCount).
WithContext(ctx).
WithParentId(table.TableId).
WithDataFieldIndex(table.DataFieldIndex)
WithDataFieldIndex(table.DataFieldIndex).
WithPrefix(domain.SideTable.ToString())
if sideTable, err = tableRepository.Save(sideTable); err != nil {
return nil, err
}
... ...
... ... @@ -62,8 +62,9 @@ func (ptr *UpdateTableStructService) UpdateTableStruct(ctx *domain.Context, tabl
return nil, err
}
}
// Log
defer func() {
AsyncEvent(domain.NewEventTable(ctx, domain.TableDataEditEvent).WithTable(table))
}()
// 日志
if err = FastLog(ptr.transactionContext, domain.CommonLog, table.TableId, &SubTableEditLog{
LogEntry: domain.NewLogEntry(table.Name, table.TableType, domain.EditSubTable, ctx),
... ...
... ... @@ -13,6 +13,8 @@ func KeyExportTable(ctx *domain.Context, tableId int) string {
var ZeroCoreRedis *redis.Redis
var NotFound = redis.ErrEmptyKey
func InitZeroCoreRedis() {
ZeroCoreRedis = redis.New(constant.REDIS_HOST+":"+constant.REDIS_PORT, redis.WithPass(constant.REDIS_AUTH))
}
... ...
... ... @@ -3,8 +3,8 @@ package starrocks
import (
"bytes"
"fmt"
"github.com/google/uuid"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
"gorm.io/gorm"
"html"
"html/template"
... ... @@ -15,8 +15,12 @@ func Insert(db *gorm.DB, tableName string, fields []*domain.FieldValue) error {
for _, f := range fields {
if f.Field.Flag == domain.PKField && f.Value == "" {
//continue
id, _ := uuid.NewUUID()
f.Value = id.String()
//id, _ := uuid.NewUUID()
// idString := id.String()
// 需要调用分布式id生成
id, _ := utils.NewSnowflakeId()
idString := fmt.Sprintf("%d", id)
f.Value = idString
}
value[f.Field.SQLName] = f.TypeValue()
}
... ...
... ... @@ -128,17 +128,23 @@ func (o *QueryOptions) AdditionOptionsByTable(table *domain.Table) *QueryOptions
}
type Condition struct {
params QueryOptions
domain.Condition
Distinct bool
DisableFormat bool
}
func (c Condition) SetWhere(params QueryOptions, q *gorm.DB) {
c.SetParams(params)
if len(c.Like) > 0 {
q.Where(fmt.Sprintf("%v like '%%%v%%'", c.FormatIfNull(params, c.Field), c.Like))
}
if len(c.In) > 0 {
q.Where(fmt.Sprintf("%v in %v", c.FormatIfNull(params, c.Field), c.InArgs(c.In)))
if c.Field.SQLType == domain.Float.ToString() {
q.Where(fmt.Sprintf("%v in %v", c.CastType(c.Field.SQLName, domain.DECIMALV2.ToString()), c.InArgs(c.In)))
} else {
q.Where(fmt.Sprintf("%v in %v", c.CastType(c.FormatIfNull(params, c.Field), "string"), c.InArgs(c.In)))
}
}
if len(c.Ex) > 0 {
in := c.InArgs(c.Ex)
... ... @@ -166,13 +172,18 @@ func (c Condition) SetWhere(params QueryOptions, q *gorm.DB) {
}
}
if c.Distinct {
q.Distinct(c.Field.SQLName)
// 需要优化
q.Distinct(c.FormatFiled(c.Field))
}
if len(c.Order) > 0 {
q.Order(fmt.Sprintf("%v %v", c.Field.SQLName, c.Order))
}
}
func (c *Condition) SetParams(params QueryOptions) {
c.params = params
}
func (c Condition) FormatIfNull(params QueryOptions, f *domain.Field) string {
if params.Table != nil && params.Table.TableType == domain.ObjectDBTable {
return f.SQLName
... ... @@ -183,6 +194,42 @@ func (c Condition) FormatIfNull(params QueryOptions, f *domain.Field) string {
return f.SQLName
}
func (c Condition) FormatFiled(f *domain.Field) string {
return formatFiled(c.Field)
}
func (c Condition) CastType(sql, t string) string {
if c.params.Table != nil && c.params.Table.TableType == domain.ObjectDBTable {
return sql
}
return castType(sql, t)
}
func (c Condition) CastTypeByField(f *domain.Field, t string) string {
if c.params.Table != nil && c.params.Table.TableType == domain.ObjectDBTable {
return f.SQLName
}
if f.SQLType == domain.Float.ToString() || f.SQLType == domain.DECIMAL279.ToString() {
return castTypeAlias(f.SQLName, domain.DECIMALV2.ToString())
}
return castType(f.SQLName, t)
}
func formatFiled(f *domain.Field) string {
//if f.SQLType == domain.Float.ToString() || f.SQLType == domain.DECIMAL279.ToString() {
// return castTypeAlias(f.SQLName, domain.DECIMALV2.ToString())
//}
return f.SQLName
}
func castType(sql, t string) string {
return fmt.Sprintf("cast(%v as %v)", sql, t)
}
func castTypeAlias(sql, t string) string {
return fmt.Sprintf("cast(%v as %v) %v", sql, t, sql)
}
var opMap = map[string]string{
"=": "=",
">": ">",
... ... @@ -292,7 +339,7 @@ func queryWithoutLimitOffset(query *gorm.DB, params QueryOptions) {
fields = append(fields, "'' "+f.SQLName)
continue
}
fields = append(fields, f.SQLName)
fields = append(fields, formatFiled(f))
}
query.Select(strings.Join(fields, ","))
}
... ...
... ... @@ -117,3 +117,10 @@ func TestRound(t *testing.T) {
t.Logf("%.1f", Truncate(99.99, 1))
t.Logf("%v", Truncate(99, 0))
}
func TestNewSnowflakeId(t *testing.T) {
id, _ := NewSnowflakeId()
for i := 0; i < 100; i++ {
t.Log(id / 2)
}
}
... ...