fix: some optimize

@@ -20,7 +20,7 @@ type CreateFileCommand struct {
 	// 文件来源
 	FileFrom string `json:"-"`
 	// AppKey
-	AppKey string `json:"-" valid:"Required"`
+	AppKey string `json:"-"`
 }
 
 var MaxFileSize = 50 * 1024 * 1024

@@ -5,10 +5,12 @@ import (
 	"fmt"
 	"github.com/beego/beego/v2/client/httplib"
 	"github.com/linmadan/egglib-go/core/application"
+	"github.com/linmadan/egglib-go/transaction/pg"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/command"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/dto"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/domainService"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
 )
@@ -63,6 +65,22 @@ func (fileService *FileService) ResetHeaderRow(ctx *domain.Context, loadDataTabl
 	}
 	loadDataTableService, _ := factory.CreateLoadDataTableService(transactionContext)
 	data, err := loadDataTableService.RePreview(ctx, loadDataTableCommand.FileId, temporaryFile.Fields, loadDataTableCommand.Where)
+	// 处理错误
+	level := domain.LevelInfo
+	errMsg := ""
+	if err != nil {
+		level = domain.LevelError
+		errMsg = err.Error()
+	}
+	if logErr := domainService.FastLog(transactionContext.(*pg.TransactionContext),
+		domain.VerifiedStepLog, temporaryFile.FileId, &domainService.ExcelTableResetHeaderLog{
+			LogEntry: domain.NewLogEntry(temporaryFile.FileName, domain.VerifiedFile.ToString(), domain.FileVerify,
+				ctx.WithValue(domain.ContextWithLogLevel, level).
+					WithValue(domain.ContextWithLogMsg, errMsg)),
+			HeaderRow: domain.GetHeaderRow(loadDataTableCommand.HeaderRow),
+		}); logErr != nil {
+		return nil, logErr
+	}
 	if err != nil {
 		return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
 	}
@@ -134,6 +152,13 @@ func (fileService *FileService) EditDataTable(ctx *domain.Context, editDataTable
 	if editDataTableCommand.Action == "remove-column" && len(temporaryFile.Fields) == len(editDataTableCommand.ProcessFields) {
 		return nil, factory.FastError(fmt.Errorf("请至少保留一个数据列"))
 	}
+	if editDataTableCommand.Action == "rename-column" {
+		targetColumn := editDataTableCommand.ProcessFieldNames[0]
+		newColumnName := editDataTableCommand.Params["newColumnName"].(string)
+		if len(temporaryFile.MatchFields([]string{newColumnName})) > 0 && newColumnName != targetColumn {
+			return nil, factory.FastError(fmt.Errorf("已存在相同名称，修改无效"))
+		}
+	}
 	// allowAction := func(fields []*domain.Field, action string) error {
 	// 	for _, f := range fields {
 	// 		if f.SQLType != string(domain.String) &&
@@ -178,6 +203,9 @@ func (fileService *FileService) FlushDataTable(ctx *domain.Context, flushDataTab
 	if err != nil {
 		return nil, factory.FastError(err)
 	}
+	if err = temporaryFile.Valid(); err != nil {
+		return nil, factory.FastError(err)
+	}
 
 	if _, err := flushDataTableService.Flush(ctx, flushDataTableCommand.ObjectId, &domain.Table{
 		DataFields: temporaryFile.Fields,

@@ -41,6 +41,12 @@ func (searchQuery *SearchTableQuery) Valid(validation *validation.Validation) {
 	if searchQuery.ParentTableId > 0 && searchQuery.ParentId == 0 {
 		searchQuery.ParentId = searchQuery.ParentTableId
 	}
+	if searchQuery.Module == 4 {
+		searchQuery.FilterRules = append(searchQuery.FilterRules, &FilterRule{
+			TableType: domain.SchemaTable.ToString(),
+			Status:    domain.StatusOn,
+		})
+	}
 }
 
 func (searchQuery *SearchTableQuery) ValidateQuery() error {

@@ -19,12 +19,6 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
 	if err != nil {
 		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
 	}
-	//if err := transactionContext.StartTransaction(); err != nil {
-	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	//}
-	//defer func() {
-	//	transactionContext.RollbackTransaction()
-	//}()
 
 	tableRepository, _, _ := factory.FastPgTable(transactionContext, 0)
 	_, tables, err := tableRepository.Find(utils.ObjectToMap(searchQuery))
@@ -40,7 +34,9 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
 			item.SetDetailStructInfo(table)
 		}
 		item.Flag = domain.FlagSet
-		if item.TableType == domain.MainTable.ToString() || item.TableType == domain.SubTable.ToString() || item.TableType == domain.SideTable.ToString() {
+		if item.TableType == domain.MainTable.ToString() ||
+			item.TableType == domain.SubTable.ToString() ||
+			item.TableType == domain.SideTable.ToString() {
 			item.ParentId = 0
 			item.Status = domain.StatusOn
 		}
@@ -55,23 +51,33 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
 
 	querySetRepository, _, _ := factory.FastPgQuerySet(transactionContext, 0)
 	_, querySets, _ := querySetRepository.Find(map[string]interface{}{"context": searchQuery.Context})
-	if !searchQuery.ReturnGroupItem {
-		querySets = make([]*domain.QuerySet, 0)
+
+	var (
+		querySetMapById = make(map[int]*domain.QuerySet)
+		// BindTableId , parentId
+		querySetMapByTableId = make(map[int]*domain.QuerySet)
+	)
+	if searchQuery.ReturnGroupItem {
+		for _, qs := range querySets {
+			querySetMapById[qs.QuerySetId] = qs
+		}
 	}
-	// BindTableId , parentId
-	querySetMapByTableId := make(map[int]*domain.QuerySet)
+
 	for _, qs := range querySets {
 		if qs.QuerySetInfo.BindTableId == 0 {
 			continue
 		}
 		querySetMapByTableId[qs.QuerySetInfo.BindTableId] = qs
 	}
-	querySetMapById := make(map[int]*domain.QuerySet)
-	for _, qs := range querySets {
-		querySetMapById[qs.QuerySetId] = qs
-	}
 
 	var response = make([]*dto.TableObjectDto, 0)
+	for index, t := range result {
+		v, ok := querySetMapByTableId[t.TableId]
+		if !ok {
+			continue
+		}
+		result[index].Update(v)
+	}
 	// 分组
 	querySetMapGroup := make(map[int]bool)
 	querySetGroups := make([]*domain.QuerySet, 0)
@@ -79,7 +85,7 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
 		if filterTableByFilterRule(t, searchQuery) {
 			continue
 		}
-		if !domain.TableType(t.TableType).TableHasGroup() {
+		if !domain.TableType(t.TableType).TableIsSplitByGroup() {
 			response = append(response, t)
 			continue
 		}
@@ -112,9 +118,6 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
 		groupItem.LoadGroup(querySetGroup)
 		response = append(response, groupItem)
 	}
-	//if err := transactionContext.CommitTransaction(); err != nil {
-	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	//}
 
 	sort.Slice(response, func(i, j int) bool {
 		item1 := response[i]

 package service
 
 import (
+	"fmt"
 	"github.com/linmadan/egglib-go/core/application"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command"
@@ -8,6 +9,7 @@ import (
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/cache"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
 )
 
 func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command.TablePreviewCommand) (interface{}, error) {
@@ -21,12 +23,6 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
 	if err != nil {
 		return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
 	}
-	//if err := transactionContext.StartTransaction(); err != nil {
-	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	//}
-	//defer func() {
-	//	transactionContext.RollbackTransaction()
-	//}()
 	var dataTable *domain.DataTable
 	var table *domain.Table
 	var cacheMiss bool
@@ -57,7 +53,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
 				return nil, factory.FastError(err)
 			}
 			response.Fields = dataTable.Fields
-			response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
+			response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, pageData(cmd.PageNumber, cmd.PageSize, dataTable.Data), false), int64(len(dataTable.Data)))
 		default:
 			var options = starrocks.QueryOptions{
 				Table:     table,
@@ -77,22 +73,34 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
 			// }
 			response.Load(table, dataTable, domain.ObjectMetaTable)
 		}
-	}
-
-	switch table.TableType {
-	case domain.CalculateSet.ToString():
-		response.Fields = dataTable.Fields
-		response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
-	default:
-		response.Load(table, dataTable, domain.ObjectMetaTable)
+	} else {
+		switch table.TableType {
+		case domain.CalculateSet.ToString():
+			response.Fields = dataTable.Fields
+			response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, pageData(cmd.PageNumber, cmd.PageSize, dataTable.Data), false), int64(len(dataTable.Data)))
+		default:
+			response.Load(table, dataTable, domain.ObjectMetaTable)
+		}
 	}
 
 	if cacheMiss && dataTable != nil {
 		// 存储缓存
 		cache.SetDataTable(table.TableId, dataTable)
 	}
-	//if err := transactionContext.CommitTransaction(); err != nil {
-	//	return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
-	//}
 	return response, nil
 }
+
+func pageData(pageNumber, pageSize int, data [][]string) [][]string {
+	if pageNumber == 0 || pageSize == 0 {
+		return data
+	}
+	offset := (pageNumber - 1) * pageSize
+	if len(data) < offset {
+		return [][]string{}
+	}
+	if len(data) < offset+pageSize {
+		pageSize = len(data) - offset
+	}
+	log.Logger.Info(fmt.Sprintf("%v %v %v", len(data), offset, pageSize))
+	return data[offset : offset+pageSize]
+}

@@ -10,6 +10,10 @@ const (
 	ExprModeExcelFunction
 )
 
+const (
+	MaxQueryRow = 1000
+)
+
 var (
 	ErrorNotFound = fmt.Errorf("没有此资源")
 )
@@ -170,7 +174,7 @@ func (t TableType) TableStatusEditable() bool {
 	return t == SchemaTable || t == CalculateItem || t == CalculateSet
 }
 
-func (t TableType) TableHasGroup() bool {
+func (t TableType) TableIsSplitByGroup() bool {
 	return t == SchemaTable || t == SubProcessTable || t == CalculateItem || t == CalculateTable || t == CalculateSet
 }
 

@@ -348,6 +348,19 @@ func (l *ExcelTableEditLog) Content() string {
 	return msg
 }
 
+type ExcelTableResetHeaderLog struct {
+	domain.LogEntry
+	// 操作名称
+	// OperateName string
+	// 操作列
+	HeaderRow int
+}
+
+func (l *ExcelTableResetHeaderLog) Content() string {
+	msg := fmt.Sprintf("修改标题行位置：%v行", l.HeaderRow)
+	return msg
+}
+
 /* *********************************************拆解模块************************************************** */
 
 type CreateQuerySetLog struct {

@@ -368,11 +368,18 @@ func (d *DataLayoutDataTable) addByLocation(cell *domain.LayoutCell, blockData [
 		}
 		d.PointEnd.Update(cell.X+len(blockData)-1, cell.Y, max)
 	case domain.DirectionNone:
-		d.DataTable.Data[cell.X][cell.Y] = blockData[0]
+		d.DataTable.Data[cell.X][cell.Y] = safeBlockData(blockData, 0)
 		d.PointEnd.Update(cell.X, cell.Y, max)
 	}
 }
 
+func safeBlockData(data []string, index int) string {
+	if len(data) < index+1 {
+		return ""
+	}
+	return data[index]
+}
+
 func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutCell, length int) {
 	for _, cell := range d.unprocessed {
 		switch lastCell.Direction {
@@ -389,7 +396,7 @@ func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutC
 }
 
 func (d *DataLayoutDataTable) BlockData(cells *domain.LayoutCell) ([]string, int) {
-	var block []string
+	var block = make([]string, 0)
 	if cells.Type == domain.CellTypeText {
 		data := []string{cells.Data.Text}
 		return data, 1

@@ -5,6 +5,7 @@ import (
 	"github.com/linmadan/egglib-go/utils/json"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
+	"strings"
 )
 
 const (
@@ -19,6 +20,7 @@ type TemporaryFileInfo struct {
 	OriginalFileId int             `json:"originalFileId"`
 	FileId         int             `json:"fileId"`
 	FileType       string          `json:"fileType"`
+	FileName       string          `json:"fileName"`
 	Total          int             `json:"total"`
 	HeaderRow      int             `json:"headerRow"`
 	Fields         []*domain.Field `json:"fields"`
@@ -29,6 +31,15 @@ type TemporaryFileInfo struct {
 	ConvertTypeErrors []ConvertTypeError `json:"convertTypeErrors"`
 }
 
+func (f *TemporaryFileInfo) Valid() error {
+	for _, f := range f.Fields {
+		if strings.HasPrefix(f.Name, "Unnamed:") {
+			return fmt.Errorf("存在空字段 `%s` 请重命名", f.Name)
+		}
+	}
+	return nil
+}
+
 func (f *TemporaryFileInfo) MatchFields(columns []string) []*domain.Field {
 	mapFields := (domain.Fields)(f.Fields).ToMap()
 	var result = make([]*domain.Field, 0)
@@ -43,6 +54,7 @@ func (f *TemporaryFileInfo) MatchFields(columns []string) []*domain.Field {
 func (f *TemporaryFileInfo) SetFile(file *domain.File) *TemporaryFileInfo {
 	f.FileId = file.FileId
 	f.FileType = file.FileType
+	f.FileName = file.FileInfo.Name
 	return f
 }
 

@@ -120,6 +120,9 @@ func (controller *FileController) SearchSourceFile() {
 	fileService := service.NewFileService(nil)
 	cmd := &query.SearchFileQuery{}
 	Must(controller.Unmarshal(cmd))
+	if cmd.PageSize == 0 {
+		cmd.PageSize = domain.MaxQueryRow
+	}
 	cmd.FileType = domain.SourceFile
 	cmd.Context = ParseContext(controller.BaseController)
 	data, err := fileService.SearchFile(cmd)
@@ -130,6 +133,9 @@ func (controller *FileController) SearchAppSourceFile() {
 	fileService := service.NewFileService(nil)
 	cmd := &query.SearchFileQuery{}
 	Must(controller.Unmarshal(cmd))
+	if cmd.PageSize == 0 {
+		cmd.PageSize = domain.MaxQueryRow
+	}
 	cmd.FileType = domain.SourceFile
 	data, err := fileService.SearchAppFile(ParseContext(controller.BaseController), cmd)
 	controller.Response(data, err)
@@ -146,6 +152,9 @@ func (controller *FileController) SearchVerifiedFile() {
 	fileService := service.NewFileService(nil)
 	cmd := &query.SearchFileQuery{}
 	Must(controller.Unmarshal(cmd))
+	if cmd.PageSize == 0 {
+		cmd.PageSize = domain.MaxQueryRow
+	}
 	cmd.FileType = domain.VerifiedFile
 	cmd.Context = ParseContext(controller.BaseController)
 	data, err := fileService.SearchFile(cmd)