作者 yangfu

fix: some optimize

... ... @@ -20,7 +20,7 @@ type CreateFileCommand struct {
// 文件来源
FileFrom string `json:"-"`
// AppKey
AppKey string `json:"-" valid:"Required"`
AppKey string `json:"-"`
}
var MaxFileSize = 50 * 1024 * 1024
... ...
... ... @@ -5,10 +5,12 @@ import (
"fmt"
"github.com/beego/beego/v2/client/httplib"
"github.com/linmadan/egglib-go/core/application"
"github.com/linmadan/egglib-go/transaction/pg"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/dto"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/domainService"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
)
... ... @@ -63,6 +65,22 @@ func (fileService *FileService) ResetHeaderRow(ctx *domain.Context, loadDataTabl
}
loadDataTableService, _ := factory.CreateLoadDataTableService(transactionContext)
data, err := loadDataTableService.RePreview(ctx, loadDataTableCommand.FileId, temporaryFile.Fields, loadDataTableCommand.Where)
// 处理错误
level := domain.LevelInfo
errMsg := ""
if err != nil {
level = domain.LevelError
errMsg = err.Error()
}
if logErr := domainService.FastLog(transactionContext.(*pg.TransactionContext),
domain.VerifiedStepLog, temporaryFile.FileId, &domainService.ExcelTableResetHeaderLog{
LogEntry: domain.NewLogEntry(temporaryFile.FileName, domain.VerifiedFile.ToString(), domain.FileVerify,
ctx.WithValue(domain.ContextWithLogLevel, level).
WithValue(domain.ContextWithLogMsg, errMsg)),
HeaderRow: domain.GetHeaderRow(loadDataTableCommand.HeaderRow),
}); logErr != nil {
return nil, logErr
}
if err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
... ... @@ -134,6 +152,13 @@ func (fileService *FileService) EditDataTable(ctx *domain.Context, editDataTable
if editDataTableCommand.Action == "remove-column" && len(temporaryFile.Fields) == len(editDataTableCommand.ProcessFields) {
return nil, factory.FastError(fmt.Errorf("请至少保留一个数据列"))
}
if editDataTableCommand.Action == "rename-column" {
targetColumn := editDataTableCommand.ProcessFieldNames[0]
newColumnName := editDataTableCommand.Params["newColumnName"].(string)
if len(temporaryFile.MatchFields([]string{newColumnName})) > 0 && newColumnName != targetColumn {
return nil, factory.FastError(fmt.Errorf("已存在相同名称,修改无效"))
}
}
// allowAction := func(fields []*domain.Field, action string) error {
// for _, f := range fields {
// if f.SQLType != string(domain.String) &&
... ... @@ -178,6 +203,9 @@ func (fileService *FileService) FlushDataTable(ctx *domain.Context, flushDataTab
if err != nil {
return nil, factory.FastError(err)
}
if err = temporaryFile.Valid(); err != nil {
return nil, factory.FastError(err)
}
if _, err := flushDataTableService.Flush(ctx, flushDataTableCommand.ObjectId, &domain.Table{
DataFields: temporaryFile.Fields,
... ...
... ... @@ -41,6 +41,12 @@ func (searchQuery *SearchTableQuery) Valid(validation *validation.Validation) {
if searchQuery.ParentTableId > 0 && searchQuery.ParentId == 0 {
searchQuery.ParentId = searchQuery.ParentTableId
}
if searchQuery.Module == 4 {
searchQuery.FilterRules = append(searchQuery.FilterRules, &FilterRule{
TableType: domain.SchemaTable.ToString(),
Status: domain.StatusOn,
})
}
}
func (searchQuery *SearchTableQuery) ValidateQuery() error {
... ...
... ... @@ -19,12 +19,6 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
//if err := transactionContext.StartTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
//defer func() {
// transactionContext.RollbackTransaction()
//}()
tableRepository, _, _ := factory.FastPgTable(transactionContext, 0)
_, tables, err := tableRepository.Find(utils.ObjectToMap(searchQuery))
... ... @@ -40,7 +34,9 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
item.SetDetailStructInfo(table)
}
item.Flag = domain.FlagSet
if item.TableType == domain.MainTable.ToString() || item.TableType == domain.SubTable.ToString() || item.TableType == domain.SideTable.ToString() {
if item.TableType == domain.MainTable.ToString() ||
item.TableType == domain.SubTable.ToString() ||
item.TableType == domain.SideTable.ToString() {
item.ParentId = 0
item.Status = domain.StatusOn
}
... ... @@ -55,23 +51,33 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
querySetRepository, _, _ := factory.FastPgQuerySet(transactionContext, 0)
_, querySets, _ := querySetRepository.Find(map[string]interface{}{"context": searchQuery.Context})
if !searchQuery.ReturnGroupItem {
querySets = make([]*domain.QuerySet, 0)
var (
querySetMapById = make(map[int]*domain.QuerySet)
// BindTableId , parentId
querySetMapByTableId = make(map[int]*domain.QuerySet)
)
if searchQuery.ReturnGroupItem {
for _, qs := range querySets {
querySetMapById[qs.QuerySetId] = qs
}
}
// BindTableId , parentId
querySetMapByTableId := make(map[int]*domain.QuerySet)
for _, qs := range querySets {
if qs.QuerySetInfo.BindTableId == 0 {
continue
}
querySetMapByTableId[qs.QuerySetInfo.BindTableId] = qs
}
querySetMapById := make(map[int]*domain.QuerySet)
for _, qs := range querySets {
querySetMapById[qs.QuerySetId] = qs
}
var response = make([]*dto.TableObjectDto, 0)
for index, t := range result {
v, ok := querySetMapByTableId[t.TableId]
if !ok {
continue
}
result[index].Update(v)
}
// 分组
querySetMapGroup := make(map[int]bool)
querySetGroups := make([]*domain.QuerySet, 0)
... ... @@ -79,7 +85,7 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
if filterTableByFilterRule(t, searchQuery) {
continue
}
if !domain.TableType(t.TableType).TableHasGroup() {
if !domain.TableType(t.TableType).TableIsSplitByGroup() {
response = append(response, t)
continue
}
... ... @@ -112,9 +118,6 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
groupItem.LoadGroup(querySetGroup)
response = append(response, groupItem)
}
//if err := transactionContext.CommitTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
sort.Slice(response, func(i, j int) bool {
item1 := response[i]
... ...
package service
import (
"fmt"
"github.com/linmadan/egglib-go/core/application"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command"
... ... @@ -8,6 +9,7 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/cache"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
)
func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command.TablePreviewCommand) (interface{}, error) {
... ... @@ -21,12 +23,6 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
//if err := transactionContext.StartTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
//defer func() {
// transactionContext.RollbackTransaction()
//}()
var dataTable *domain.DataTable
var table *domain.Table
var cacheMiss bool
... ... @@ -57,7 +53,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
return nil, factory.FastError(err)
}
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, pageData(cmd.PageNumber, cmd.PageSize, dataTable.Data), false), int64(len(dataTable.Data)))
default:
var options = starrocks.QueryOptions{
Table: table,
... ... @@ -77,22 +73,34 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
// }
response.Load(table, dataTable, domain.ObjectMetaTable)
}
}
switch table.TableType {
case domain.CalculateSet.ToString():
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
default:
response.Load(table, dataTable, domain.ObjectMetaTable)
} else {
switch table.TableType {
case domain.CalculateSet.ToString():
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, pageData(cmd.PageNumber, cmd.PageSize, dataTable.Data), false), int64(len(dataTable.Data)))
default:
response.Load(table, dataTable, domain.ObjectMetaTable)
}
}
if cacheMiss && dataTable != nil {
// 存储缓存
cache.SetDataTable(table.TableId, dataTable)
}
//if err := transactionContext.CommitTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
return response, nil
}
func pageData(pageNumber, pageSize int, data [][]string) [][]string {
if pageNumber == 0 || pageSize == 0 {
return data
}
offset := (pageNumber - 1) * pageSize
if len(data) < offset {
return [][]string{}
}
if len(data) < offset+pageSize {
pageSize = len(data) - offset
}
log.Logger.Info(fmt.Sprintf("%v %v %v", len(data), offset, pageSize))
return data[offset : offset+pageSize]
}
... ...
... ... @@ -10,6 +10,10 @@ const (
ExprModeExcelFunction
)
const (
MaxQueryRow = 1000
)
var (
ErrorNotFound = fmt.Errorf("没有此资源")
)
... ... @@ -170,7 +174,7 @@ func (t TableType) TableStatusEditable() bool {
return t == SchemaTable || t == CalculateItem || t == CalculateSet
}
func (t TableType) TableHasGroup() bool {
func (t TableType) TableIsSplitByGroup() bool {
return t == SchemaTable || t == SubProcessTable || t == CalculateItem || t == CalculateTable || t == CalculateSet
}
... ...
... ... @@ -348,6 +348,19 @@ func (l *ExcelTableEditLog) Content() string {
return msg
}
type ExcelTableResetHeaderLog struct {
domain.LogEntry
// 操作名称
// OperateName string
// 操作列
HeaderRow int
}
func (l *ExcelTableResetHeaderLog) Content() string {
msg := fmt.Sprintf("修改标题行位置:%v行", l.HeaderRow)
return msg
}
/* *********************************************拆解模块************************************************** */
type CreateQuerySetLog struct {
... ...
... ... @@ -368,11 +368,18 @@ func (d *DataLayoutDataTable) addByLocation(cell *domain.LayoutCell, blockData [
}
d.PointEnd.Update(cell.X+len(blockData)-1, cell.Y, max)
case domain.DirectionNone:
d.DataTable.Data[cell.X][cell.Y] = blockData[0]
d.DataTable.Data[cell.X][cell.Y] = safeBlockData(blockData, 0)
d.PointEnd.Update(cell.X, cell.Y, max)
}
}
func safeBlockData(data []string, index int) string {
if len(data) < index+1 {
return ""
}
return data[index]
}
func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutCell, length int) {
for _, cell := range d.unprocessed {
switch lastCell.Direction {
... ... @@ -389,7 +396,7 @@ func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutC
}
func (d *DataLayoutDataTable) BlockData(cells *domain.LayoutCell) ([]string, int) {
var block []string
var block = make([]string, 0)
if cells.Type == domain.CellTypeText {
data := []string{cells.Data.Text}
return data, 1
... ...
... ... @@ -5,6 +5,7 @@ import (
"github.com/linmadan/egglib-go/utils/json"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"strings"
)
const (
... ... @@ -19,6 +20,7 @@ type TemporaryFileInfo struct {
OriginalFileId int `json:"originalFileId"`
FileId int `json:"fileId"`
FileType string `json:"fileType"`
FileName string `json:"fileName"`
Total int `json:"total"`
HeaderRow int `json:"headerRow"`
Fields []*domain.Field `json:"fields"`
... ... @@ -29,6 +31,15 @@ type TemporaryFileInfo struct {
ConvertTypeErrors []ConvertTypeError `json:"convertTypeErrors"`
}
func (f *TemporaryFileInfo) Valid() error {
for _, f := range f.Fields {
if strings.HasPrefix(f.Name, "Unnamed:") {
return fmt.Errorf("存在空字段 `%s` 请重命名", f.Name)
}
}
return nil
}
func (f *TemporaryFileInfo) MatchFields(columns []string) []*domain.Field {
mapFields := (domain.Fields)(f.Fields).ToMap()
var result = make([]*domain.Field, 0)
... ... @@ -43,6 +54,7 @@ func (f *TemporaryFileInfo) MatchFields(columns []string) []*domain.Field {
func (f *TemporaryFileInfo) SetFile(file *domain.File) *TemporaryFileInfo {
f.FileId = file.FileId
f.FileType = file.FileType
f.FileName = file.FileInfo.Name
return f
}
... ...
... ... @@ -120,6 +120,9 @@ func (controller *FileController) SearchSourceFile() {
fileService := service.NewFileService(nil)
cmd := &query.SearchFileQuery{}
Must(controller.Unmarshal(cmd))
if cmd.PageSize == 0 {
cmd.PageSize = domain.MaxQueryRow
}
cmd.FileType = domain.SourceFile
cmd.Context = ParseContext(controller.BaseController)
data, err := fileService.SearchFile(cmd)
... ... @@ -130,6 +133,9 @@ func (controller *FileController) SearchAppSourceFile() {
fileService := service.NewFileService(nil)
cmd := &query.SearchFileQuery{}
Must(controller.Unmarshal(cmd))
if cmd.PageSize == 0 {
cmd.PageSize = domain.MaxQueryRow
}
cmd.FileType = domain.SourceFile
data, err := fileService.SearchAppFile(ParseContext(controller.BaseController), cmd)
controller.Response(data, err)
... ... @@ -146,6 +152,9 @@ func (controller *FileController) SearchVerifiedFile() {
fileService := service.NewFileService(nil)
cmd := &query.SearchFileQuery{}
Must(controller.Unmarshal(cmd))
if cmd.PageSize == 0 {
cmd.PageSize = domain.MaxQueryRow
}
cmd.FileType = domain.VerifiedFile
cmd.Context = ParseContext(controller.BaseController)
data, err := fileService.SearchFile(cmd)
... ...