作者 yangfu

feat: row data edit

... ... @@ -180,4 +180,10 @@
- [ ] 50W 120s 读取数据库:30s 保存文件:10s 下载:30M/500K=60S;RAR压缩 24M/500k=50S
- [ ] 20W ..
- [ ] 10W ..
- [ ] 保存单个文件、压缩 | 保存多个文件、压缩
\ No newline at end of file
- [ ] 保存单个文件、压缩 | 保存多个文件、压缩
## 讨论事项
- [ ] 校验动作,参数模型讨论
- [ ] 校验日志错误(标红)
- [ ] 校验完毕应答实体,类型修改即使错误,也要返回修改完毕的表
\ No newline at end of file
... ...
... ... @@ -2,7 +2,6 @@ package command
import (
"fmt"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"reflect"
"strings"
... ... @@ -13,9 +12,9 @@ type FlushDataTableCommand struct {
// 文件ID
FileId int `cname:"文件ID" json:"objectId" valid:"Required"`
// 记录数
RowCount int `cname:"记录数" json:"rowCount" valid:"Required"`
//RowCount int `cname:"记录数" json:"rowCount" valid:"Required"`
// 数据列
DataFields []*domain.Field `cname:"数据列" json:"fields" valid:"Required"`
//DataFields []*domain.Field `cname:"数据列" json:"fields" valid:"Required"`
}
func (flushDataTableCommand *FlushDataTableCommand) Valid(validation *validation.Validation) {
... ...
... ... @@ -6,6 +6,7 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/dto"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
)
// FilePreview 加载表格数据
... ... @@ -85,6 +86,12 @@ func (fileService *FileService) EditDataTable(ctx *domain.Context, editDataTable
transactionContext.RollbackTransaction()
}()
cache := redis.NewFileCacheService()
temporaryFile, err := cache.Get(redis.KeyTemporaryFileInfo(editDataTableCommand.FileId))
if err != nil {
return nil, factory.FastError(err)
}
editDataTableCommand.Fields = temporaryFile.Fields
editDataTableService, _ := factory.CreateEditDataTableService(transactionContext)
_, err = editDataTableService.Edit(ctx, editDataTableCommand.EditTableRequest)
if err != nil {
... ... @@ -112,16 +119,23 @@ func (fileService *FileService) FlushDataTable(ctx *domain.Context, flushDataTab
transactionContext.RollbackTransaction()
}()
flushDataTableService, _ := factory.CreateFlushDataTableService(transactionContext)
fields := make([]*domain.Field, 0)
for _, f := range flushDataTableCommand.DataFields {
fields = append(fields, &domain.Field{
Name: f.Name,
SQLType: f.SQLType,
})
//fields := make([]*domain.Field, 0)
//for _, f := range flushDataTableCommand.DataFields {
// fields = append(fields, &domain.Field{
// Name: f.Name,
// SQLType: f.SQLType,
// })
//}
cache := redis.NewFileCacheService()
temporaryFile, err := cache.Get(redis.KeyTemporaryFileInfo(flushDataTableCommand.FileId))
if err != nil {
return nil, factory.FastError(err)
}
if _, err := flushDataTableService.Flush(ctx, flushDataTableCommand.FileId, &domain.Table{
DataFields: fields,
RowCount: flushDataTableCommand.RowCount,
DataFields: temporaryFile.Fields,
RowCount: temporaryFile.Total,
}); err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
... ...
... ... @@ -21,6 +21,10 @@ type VerifiedStepLogDto struct {
//OperatorName string `json:"operatorName"`
// 创建时间
CreatedAt string `json:"createdAt"`
// 错误级别
Level string `json:"level"`
// 错误信息
Error string `json:"error"`
}
func (d *VerifiedStepLogDto) Load(m *domain.Log) {
... ... @@ -32,4 +36,9 @@ func (d *VerifiedStepLogDto) Load(m *domain.Log) {
d.Content = m.Content
//d.OperatorName = m.OperatorName
d.CreatedAt = m.CreatedAt.Local().Format("2006-01-02 15:04:05")
d.Level = m.Entry.Level
d.Error = m.Entry.Error
if len(d.Level) == 0 {
d.Level = domain.LevelInfo.ToString()
}
}
... ...
... ... @@ -12,6 +12,7 @@ import (
type TablePreviewCommand struct {
// 表Id
TableId int `cname:"表Id" json:"objectId" valid:"Required"`
ObjectType string `json:"objectType"`
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
Where domain.Where `json:"where"`
... ...
... ... @@ -17,6 +17,7 @@ type RowEditCommand struct {
RemoveList []*domain.FieldValues `json:"removeList"`
AddList []*domain.FieldValues `json:"addList"`
Where domain.Where `json:"where"`
DataList []map[string]string `json:"dataList"`
}
func (cmd *RowEditCommand) Valid(validation *validation.Validation) {
... ...
package command
import (
"fmt"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"reflect"
"strings"
"github.com/beego/beego/v2/core/validation"
)
type RowEditCommandV2 struct {
// 表Id
TableId int `cname:"表Id" json:"tableId" valid:"Required"`
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
// 数据列
UpdateList []map[string]string `json:"updateList"`
RemoveList []map[string]string `json:"removeList"`
AddList []map[string]string `json:"addList"`
Where domain.Where `json:"where"`
}
func (cmd *RowEditCommandV2) Valid(validation *validation.Validation) {
if cmd.PageSize > 0 {
cmd.Where.PageSize = cmd.PageSize
cmd.Where.PageNumber = cmd.PageNumber
}
}
func (cmd *RowEditCommandV2) ValidateCommand() error {
valid := validation.Validation{}
b, err := valid.Valid(cmd)
if err != nil {
return err
}
if !b {
elem := reflect.TypeOf(cmd).Elem()
for _, validErr := range valid.Errors {
field, isExist := elem.FieldByName(validErr.Field)
if isExist {
return fmt.Errorf(strings.Replace(validErr.Message, validErr.Field, field.Tag.Get("cname"), -1))
} else {
return fmt.Errorf(validErr.Message)
}
}
}
return nil
}
... ...
... ... @@ -28,3 +28,13 @@ func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, obj
//d.Total = dataTable.Total
return d
}
func ToFieldDataByPK(m *domain.Table, dataTable *domain.DataTable) map[string]map[string]string {
var dataListByPk = make(map[string]map[string]string)
for _, item := range domain.ToFieldData(m.Fields(true), dataTable.Data, false) {
if v, ok := item[domain.DefaultPkField]; ok {
dataListByPk[v] = item
}
}
return dataListByPk
}
... ...
package service
import (
"github.com/linmadan/egglib-go/core/application"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
)
func (tableService *TableService) RowEdit(ctx *domain.Context, cmd *command.RowEditCommand) (interface{}, error) {
if err := cmd.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
editDataService, _ := factory.CreateTableEditDataService(transactionContext)
_, err = editDataService.RowEdit(ctx, domain.EditDataRequest{
TableId: cmd.TableId,
Where: cmd.Where,
UpdateList: cmd.UpdateList,
AddList: cmd.AddList,
RemoveList: cmd.RemoveList,
})
if err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return struct{}{}, nil
}
... ... @@ -8,6 +8,7 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/pg"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
... ... @@ -103,9 +104,16 @@ func (tableService *TableService) ExportDataTableV2(ctx *domain.Context, cmd *co
defer locker.Release()
var table *domain.Table
//var mainTable *domain.Table
_, table, err = factory.FastPgTable(transactionContext, cmd.TableId)
if err != nil {
return nil, factory.FastError(err)
if cmd.ObjectType == domain.ObjectDBTable {
table = domain.DBTables[cmd.TableId]
if table == nil {
return nil, factory.FastError(fmt.Errorf("表不存在"))
}
} else {
_, table, err = factory.FastPgTable(transactionContext, cmd.TableId)
if err != nil {
return nil, factory.FastError(err)
}
}
data, err := exportTableTo(ctx, cmd, table, 10000)
... ... @@ -123,8 +131,14 @@ func exportTableTo(ctx *domain.Context, cmd *command.TablePreviewCommand, table
var options = starrocks.QueryOptions{
TableName: table.SQLName,
Select: table.Fields(false),
//Table: table,
}
count, err := starrocks.QueryCount(options)
db := starrocks.DB
if table.TableType == domain.ObjectDBTable {
db = pg.GormDB
}
count, err := starrocks.WrapQueryCountWithDB(options, db)()
if err != nil {
return nil, factory.FastError(err)
}
... ... @@ -138,6 +152,7 @@ func exportTableTo(ctx *domain.Context, cmd *command.TablePreviewCommand, table
//Where: cmd.Where.Conditions,
Offset: i * blockSize,
Limit: blockSize,
Table: table,
}
options.SetCondition(cmd.Where.Conditions).SetDefaultOrder()
source <- Query{
... ... @@ -150,7 +165,7 @@ func exportTableTo(ctx *domain.Context, cmd *command.TablePreviewCommand, table
// mapper
query := item.(Query)
var dataTable *domain.DataTable
dataTable, err = starrocks.Query(query.Options, starrocks.WrapQueryFuncWithDB(starrocks.DB))
dataTable, err = starrocks.Query(query.Options, starrocks.WrapQueryFuncWithDB(db))
if err != nil {
log.Logger.Error(err.Error(), map[string]interface{}{"mapper": query})
return
... ...
... ... @@ -31,10 +31,23 @@ func (tableService *TableService) FieldOptionalValues(ctx *domain.Context, cmd *
var db *gorm.DB
switch cmd.ObjectType {
case domain.ObjectFile:
table, err = tableRepository.FindOne(map[string]interface{}{"ctx": ctx, "tableId": cmd.ObjectId})
byteCore, _ := factory.CreateByteCoreService(transactionContext)
response, err := byteCore.FieldOptionalValues(domain.ReqFieldOptionalValues{
FileId: cmd.ObjectId,
Field: &cmd.Field,
Match: cmd.Match,
Where: domain.Where{
PageNumber: cmd.PageNumber,
PageSize: cmd.PageSize,
},
})
if err != nil {
return nil, factory.FastError(err)
}
return map[string]interface{}{
"values": response.Values,
"total": response.Total,
}, nil
case domain.ObjectMetaTable:
table, err = tableRepository.FindOne(map[string]interface{}{"ctx": ctx, "tableId": cmd.ObjectId})
if err != nil {
... ... @@ -50,6 +63,7 @@ func (tableService *TableService) FieldOptionalValues(ctx *domain.Context, cmd *
if !ok {
return nil, factory.FastError(fmt.Errorf("列:%v 不存在", cmd.Field.Name))
}
options := &starrocks.QueryOptions{
TableName: table.SQLName,
Select: []*domain.Field{field},
... ...
package service
import (
"github.com/linmadan/egglib-go/core/application"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/dto"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
)
func (tableService *TableService) RowEdit(ctx *domain.Context, cmd *command.RowEditCommand) (interface{}, error) {
if err := cmd.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
//editDataService, _ := factory.CreateTableEditDataService(transactionContext)
//_, err = editDataService.RowEdit(ctx, domain.EditDataRequest{
// TableId: cmd.TableId,
// Where: cmd.Where,
// UpdateList: cmd.UpdateList,
// AddList: cmd.AddList,
// RemoveList: cmd.RemoveList,
//})
if err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return struct{}{}, nil
}
func (tableService *TableService) RowEditV2(ctx *domain.Context, cmd *command.RowEditCommandV2) (interface{}, error) {
if err := cmd.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
var table *domain.Table
_, table, err = factory.FastPgTable(transactionContext, cmd.TableId)
if err != nil {
return nil, factory.FastError(err)
}
var options = starrocks.QueryOptions{
TableName: table.SQLName,
Select: table.Fields(true),
}
options.SetCondition(cmd.Where.Conditions).SetDefaultOrder()
options.SetOffsetLimit(cmd.Where.PageNumber, cmd.Where.PageSize)
var dataTable *domain.DataTable
dataTable, err = factory.FastDataTable(options)
if err != nil {
return nil, factory.FastError(err)
}
editDataService, _ := factory.CreateTableEditDataService(transactionContext)
_, err = editDataService.RowEdit(ctx, domain.EditDataRequest{
TableId: cmd.TableId,
Table: table,
Where: cmd.Where,
UpdateList: MapArrayToFieldValues(cmd.UpdateList, table, dataTable, true),
AddList: MapArrayToFieldValues(cmd.AddList, table, dataTable, false),
RemoveList: MapArrayToFieldValues(cmd.RemoveList, table, dataTable, true),
})
if err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return struct{}{}, nil
}
func MapArrayToFieldValues(list []map[string]string, table *domain.Table, dataTable *domain.DataTable, mustMatch bool) []*domain.FieldValues {
var result = make([]*domain.FieldValues, 0)
history := dto.ToFieldDataByPK(table, dataTable)
mapField := domain.Fields(table.Fields(true)).ToMapBySqlName()
for _, m := range list {
var fieldValues = &domain.FieldValues{
FieldValues: make([]*domain.FieldValue, 0),
}
matchItem, ok := history[m[domain.DefaultPkField]]
if mustMatch {
if !ok {
continue
}
}
if _, ok := m[domain.DefaultPkField]; !ok {
m[domain.DefaultPkField] = ""
}
for key, value := range m {
field, ok := mapField[key]
if !ok || field.Flag == domain.ManualField {
continue
}
fieldValue := &domain.FieldValue{
Field: field,
Value: value,
}
if mustMatch {
if oldValue, ok := matchItem[key]; ok {
fieldValue.OldValue = oldValue
}
}
fieldValues.FieldValues = append(fieldValues.FieldValues, fieldValue)
}
result = append(result, fieldValues)
}
return result
}
... ...
... ... @@ -45,5 +45,5 @@ func init() {
HTTP_PORT = Configurator.DefaultInt("HTTP_PORT", HTTP_PORT)
SERVICE_NAME = fmt.Sprintf("%v-%v", SERVICE_NAME, SERVICE_ENV)
PPROF_ON = Configurator.DefaultBool("PPROF_ON", PPROF_ON)
CACHE_PREFIX = SERVICE_NAME
CACHE_PREFIX = SERVICE_NAME + ":" + SERVICE_ENV
}
... ...
... ... @@ -11,6 +11,7 @@ type ByteLibService interface {
DeleteTable(param ReqDeleteTable) (*DataDeleteTable, error)
CancelFile(param ReqCancelFile) (*DataCancelFile, error)
EditTableData(param ReqEditTableData) (*DataEditTableData, error)
FieldOptionalValues(param ReqFieldOptionalValues) (*DataFieldOptionalValues, error)
}
type (
... ... @@ -26,8 +27,8 @@ type (
ColumnSchemas []ColumnSchema `json:"columnSchemas"`
//PageNumber int `json:"pageNumber"`
//PageSize int `json:"pageSize"`
QueryParameters map[string]interface{} `json:"queryParameters"`
SortParameters map[string]interface{} `json:"sortParameters"`
//QueryParameters []QueryParameter `json:"queryParameters"`
SortParameters map[string]interface{} `json:"sortParameters"`
}
DataLoadDataTable struct {
... ... @@ -41,6 +42,11 @@ type (
InValidCells []InValidCell `json:"inValidCells"`
}
QueryParameter struct {
ColumnName string `json:"columnName"`
ColumnContents []string `json:"columnContents"`
IsContainContent bool `json:"isContainContent"`
}
//Field struct {
// // 索引序号
// Index int `json:"index"`
... ... @@ -183,6 +189,21 @@ type (
)
type (
ReqFieldOptionalValues struct {
FileId int
Field *Field
Where Where
Match string
}
DataFieldOptionalValues struct {
Field *Field
Values []string
Total int
}
)
type (
ReqCancelFile struct {
}
... ...
... ... @@ -9,4 +9,26 @@ type Context struct {
OperatorName string `json:"operatorName"`
// 租户 (个人、企业)
TenantId int `json:"tenantId"`
data map[string]interface{}
}
func (c *Context) WithValue(key string, value interface{}) *Context {
if _, ok := c.data[key]; ok {
return c
}
c.data[key] = value
return c
}
func (c *Context) GetValue(key string) (interface{}, bool) {
if v, ok := c.data[key]; ok {
return v, true
}
return nil, false
}
const (
ContextWithLogLevel = "WithLogLevel"
ContextWithLogMsg = "WithLogMsg"
)
... ...
... ... @@ -75,7 +75,8 @@ type TableEditDataService interface {
RowEdit(ctx *Context, request EditDataRequest) (interface{}, error)
}
type EditDataRequest struct {
TableId int `json:"tableId"`
TableId int `json:"tableId"`
Table *Table
UpdateList []*FieldValues `json:"updateList"`
RemoveList []*FieldValues `json:"removeList"`
AddList []*FieldValues `json:"addList"`
... ...
... ... @@ -210,3 +210,19 @@ const (
ObjectMetaTable = "MetaTable"
ObjectDBTable = "DBTable"
)
type LogLevel string
const (
LevelInfo LogLevel = "info"
LevelError LogLevel = "error"
LevelWarn LogLevel = "warn"
)
func (t LogLevel) ToString() string {
return string(t)
}
const (
DefaultPkField = "id"
)
... ...
... ... @@ -45,6 +45,14 @@ func (fields Fields) ToMap() map[string]*Field {
return m
}
func (fields Fields) ToMapBySqlName() map[string]*Field {
m := make(map[string]*Field)
for i := range fields {
m[fields[i].SQLName] = fields[i]
}
return m
}
func (fields Fields) NameArrayString() []string {
m := make([]string, 0)
for i := range fields {
... ... @@ -104,7 +112,7 @@ func FieldsChange(oldFields []*Field, newFields []*Field) (reserve []*Field, del
}
type FieldValues struct {
Number int `json:"number"`
Number int `json:"rowIndex"`
FieldValues []*FieldValue `json:"fieldValues"`
}
... ...
... ... @@ -12,8 +12,11 @@ type LogEntry struct {
Content string `json:"content"`
// 操作人名称
OperatorName string `json:"operatorName"`
ctx *Context `json:"-"`
// 错误级别
Level string `json:"level"`
// 错误信息
Error string `json:"error"`
ctx *Context `json:"-"`
}
func (l LogEntry) Entry() LogEntry {
... ...
... ... @@ -226,3 +226,22 @@ func (gateway ApiByteLib) EditTableData(param domain.ReqEditTableData) (*domain.
}
return &data, nil
}
// FieldOptionalValues 字段可选值
func (gateway ApiByteLib) FieldOptionalValues(param domain.ReqFieldOptionalValues) (*domain.DataFieldOptionalValues, error) {
url := gateway.Host() + "/checkout-tables/filter-column-data"
method := "post"
var data CheckoutTablesFilterColumnDataResponse
request := NewCheckoutTablesFilterColumnDataRequest(param)
err := gateway.FastDoRequest(url, method, request, &data)
if err != nil {
return nil, err
}
var response = domain.DataFieldOptionalValues{
//TableName: data.MasterTableName,
Field: param.Field,
Values: data.ShowData,
Total: data.DataCount,
}
return &response, nil
}
... ...
... ... @@ -7,14 +7,15 @@ import (
)
type RequestCheckoutTablesQuery struct {
OriginalTableId string `json:"originalTableId"`
IsFromOriginalTable bool `json:"isFromOriginalTable"`
TableFileUrl string `json:"tableFileUrl"`
ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
QueryParameters map[string]interface{} `json:"queryParameters"`
SortParameters map[string]interface{} `json:"sortParameters"`
OriginalTableId string `json:"originalTableId"`
IsFromOriginalTable bool `json:"isFromOriginalTable"`
TableFileUrl string `json:"tableFileUrl"`
ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
QueryParameters []domain.QueryParameter `json:"queryParameters"`
//QueryParameters map[string]interface{} `json:"queryParameters"`
SortParameters map[string]interface{} `json:"sortParameters"`
}
type DataCheckoutTables struct {
... ... @@ -35,8 +36,10 @@ func NewRequestCheckoutTablesQuery(param domain.ReqLoadDataTable) RequestCheckou
ColumnSchemas: param.ColumnSchemas,
PageNumber: param.PageNumber,
PageSize: param.PageSize,
QueryParameters: param.QueryParameters,
SortParameters: param.SortParameters,
//QueryParameters: param.QueryParameters,
QueryParameters: make([]domain.QueryParameter, 0),
//QueryParameters: make(map[string]interface{}),
SortParameters: param.SortParameters,
}
}
... ... @@ -287,3 +290,30 @@ func NewSplitTableRequest(param domain.ReqSplitTable) SplitTableRequest {
SplitTableFieldSchemas: ToFieldSchemas(param.ToSubTable.DataFields),
}
}
type (
CheckoutTablesFilterColumnDataRequest struct {
OriginalTableId string `json:"originalTableId"`
domain.ColumnSchema
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
FilterParameters map[string]string `json:"filterParameters"`
}
CheckoutTablesFilterColumnDataResponse struct {
DataCount int `json:"dataCount"`
ShowData []string `json:"showData"`
}
)
func NewCheckoutTablesFilterColumnDataRequest(param domain.ReqFieldOptionalValues) CheckoutTablesFilterColumnDataRequest {
return CheckoutTablesFilterColumnDataRequest{
OriginalTableId: intToString(param.FileId),
ColumnSchema: domain.ColumnSchema{
ColumnName: param.Field.Name,
ColumnType: param.Field.SQLType,
},
PageNumber: param.Where.PageNumber,
PageSize: param.Where.PageSize,
FilterParameters: map[string]string{"keyWord": param.Match},
}
}
... ...
... ... @@ -121,6 +121,11 @@ func (ptr *ByteCoreService) SplitTable(param domain.ReqSplitTable) (*domain.Data
return apiByteLib.SplitTable(param)
}
func (ptr *ByteCoreService) FieldOptionalValues(param domain.ReqFieldOptionalValues) (*domain.DataFieldOptionalValues, error) {
apiByteLib := bytelib.NewApiByteLib(constant.BYTE_CORE_HOST)
return apiByteLib.FieldOptionalValues(param)
}
//////////////
// 字库核心
//////////////
... ...
... ... @@ -25,7 +25,9 @@ func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableR
var operateName string = req.Action
// 日志
if err = FastLog(ptr.transactionContext, domain.VerifiedStepLog, file.FileId, &ExcelTableEditLog{
LogEntry: domain.NewLogEntry(file.FileInfo.Name, domain.VerifiedFile.ToString(), domain.FileVerify, ctx),
LogEntry: domain.NewLogEntry(file.FileInfo.Name, domain.VerifiedFile.ToString(), domain.FileVerify,
ctx.WithValue(domain.ContextWithLogLevel, domain.LevelInfo).
WithValue(domain.ContextWithLogMsg, "")),
ProcessFields: req.ProcessFields,
OperateName: operateName,
}); err != nil {
... ... @@ -46,6 +48,8 @@ func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableR
return nil, err
}
// 1.有修改表类型的,更新缓存数据列类型
return response, nil
}
... ...
... ... @@ -5,6 +5,7 @@ import (
pgTransaction "github.com/linmadan/egglib-go/transaction/pg"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api/bytelib"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository"
)
... ... @@ -50,12 +51,17 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie
IsFromOriginalTable: isSourceFile,
TableFileUrl: fileUrl,
ColumnSchemas: bytelib.DomainFieldsToColumnSchemas(fields),
QueryParameters: make(map[string]interface{}),
SortParameters: make(map[string]interface{}),
//QueryParameters: make([]domain.QueryParameter, 0),
SortParameters: make(map[string]interface{}),
})
if err != nil {
return nil, err
}
cache := redis.NewFileCacheService()
_, err = cache.Update(redis.KeyTemporaryFileInfo(file.FileId), file, response.Fields, response.Total)
if err != nil {
return nil, err
}
var responseDto = &FilePreviewDto{}
responseDto.Load(file.FileId, response)
return responseDto, nil
... ...
... ... @@ -5,6 +5,7 @@ import (
pgTransaction "github.com/linmadan/egglib-go/transaction/pg"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"strings"
"time"
)
... ... @@ -43,6 +44,13 @@ func (ptr *PGLogService) Log(logType domain.LogType, sourceId int, logEntry Log)
CreatedAt: time.Now(),
Context: logEntry.Context(),
}
if v, ok := logEntry.Context().GetValue(domain.ContextWithLogLevel); ok {
log.Entry.Level = string(v.(domain.LogLevel))
}
if v, ok := logEntry.Context().GetValue(domain.ContextWithLogLevel); ok {
log.Entry.Error = v.(string)
}
_, err := logRepository.Save(log)
return err
}
... ... @@ -207,10 +215,11 @@ type RowUpdateLog struct {
func (l *RowUpdateLog) Content() string {
change := ""
index := l.Number + l.Where.Offset()
//index := l.Number + l.Where.Offset()
for _, f := range l.FieldValue {
if f.OldValue != f.Value {
change += fmt.Sprintf("%v字段%v行的值从%v更改为%v;", f.Field.Name, index, f.OldValue, f.Value)
//change += fmt.Sprintf("%v字段%v行的值从%v更改为%v;", f.Field.Name, index, f.OldValue, f.Value)
change += fmt.Sprintf("【%v】字段的值从“%v”更改为“%v”;", f.Field.Name, f.OldValue, f.Value)
}
}
if len(change) == 0 {
... ... @@ -222,14 +231,29 @@ func (l *RowUpdateLog) Content() string {
type RowRemoveLog struct {
domain.LogEntry
FieldValue *domain.FieldValues
Where domain.Where
DeleteRowCount int
Where domain.Where
}
func (l *RowRemoveLog) Content() string {
index := l.FieldValue.Number + l.Where.Offset()
index := l.DeleteRowCount
//msg := fmt.Sprintf("删除%v行数据;筛选件:%v",index,"")
msg := fmt.Sprintf("删除%v行数据;", index)
filters := make([]string, 0)
inArgs := func(args []string) string {
return strings.Join(args, "、")
}
for _, c := range l.Where.Conditions {
if len(c.In) > 0 {
filters = append(filters, fmt.Sprintf("【%v】 包含 %v", c.Field.Name, inArgs(starrocks.ArrayInterfaceToString(c.In))))
}
if len(c.Ex) > 0 {
filters = append(filters, fmt.Sprintf("【%v】 不包含 %v", c.Field.Name, inArgs(starrocks.ArrayInterfaceToString(c.Ex))))
}
}
if len(filters) > 0 {
msg += "筛选件:" + strings.Join(filters, "|")
}
return msg
}
... ...
... ... @@ -24,26 +24,33 @@ func NewTableEditDataService(transactionContext *pgTransaction.TransactionContex
func (ptr *TableEditDataService) RowEdit(ctx *domain.Context, request domain.EditDataRequest) (interface{}, error) {
tableRepository, _ := repository.NewTableRepository(ptr.transactionContext)
table, err := tableRepository.FindOne(map[string]interface{}{"tableId": request.TableId})
if err != nil {
return nil, err
var table *domain.Table = request.Table
var err error
if table == nil {
table, err = tableRepository.FindOne(map[string]interface{}{"tableId": request.TableId})
if err != nil {
return nil, err
}
}
if table.TableType != domain.SideTable.ToString() {
return nil, fmt.Errorf("副表才允许编辑数据")
}
//if table.TableType == domain.SubTable.ToString(){
// mainTable, err := tableRepository.FindOne(map[string]interface{}{"tableId": request.TableId})
// if err != nil {
// return nil, err
// }
// table.SQLName = mainTable.SQLName
//}
for _, l := range request.AddList {
ptr.add(ctx, table, l, request.Where)
}
for _, l := range request.RemoveList {
ptr.remove(ctx, table, l, request.Where)
if len(request.RemoveList) > 0 {
// 日志
if err = FastLog(ptr.transactionContext, domain.CommonLog, table.TableId, &RowRemoveLog{
LogEntry: domain.NewLogEntry(table.Name, table.TableType, domain.RowEdit, ctx),
DeleteRowCount: len(request.RemoveList),
}); err != nil {
return nil, err
}
for _, l := range request.RemoveList {
ptr.remove(ctx, table, l, request.Where)
}
}
for _, l := range request.UpdateList {
ptr.update(ctx, table, l, request.Where)
... ... @@ -72,15 +79,6 @@ func (ptr *TableEditDataService) add(ctx *domain.Context, table *domain.Table, l
func (ptr *TableEditDataService) remove(ctx *domain.Context, table *domain.Table, list *domain.FieldValues, where domain.Where) error {
var err error
// 日志
if err = FastLog(ptr.transactionContext, domain.CommonLog, table.TableId, &RowRemoveLog{
LogEntry: domain.NewLogEntry(table.Name, table.TableType, domain.RowEdit, ctx),
FieldValue: list,
Where: where,
}); err != nil {
return err
}
// 删除记录
if err = starrocks.Delete(starrocks.DB, table.SQLName, list.FieldValues); err != nil {
return err
... ... @@ -93,7 +91,9 @@ func (ptr *TableEditDataService) update(ctx *domain.Context, table *domain.Table
// 日志
if err = FastLog(ptr.transactionContext, domain.CommonLog, table.TableId, &RowUpdateLog{
LogEntry: domain.NewLogEntry(table.Name, table.TableType, domain.RowEdit, ctx),
LogEntry: domain.NewLogEntry(table.Name, table.TableType, domain.RowEdit, ctx),
FieldValue: list.FieldValues,
Number: list.Number,
}); err != nil {
return err
}
... ...
package redis
import (
"fmt"
"github.com/linmadan/egglib-go/utils/json"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
)
const (
TemporaryFileExpire = 3600 * 10
)
func KeyTemporaryFileInfo(fileId int) string {
return fmt.Sprintf("%v:file:temporary:%v", constant.CACHE_PREFIX, fileId)
}
type TemporaryFileInfo struct {
FileId int `json:"fileId"`
FileType string `json:"fileType"`
Total int `json:"total"`
Fields []*domain.Field `json:"fields"`
// 编辑表错误,有错误不允许保存成校验文件
// 行记录错误
// 列类型有错误
// 表整体错误
}
func (f *TemporaryFileInfo) SetFile(file *domain.File) *TemporaryFileInfo {
f.FileId = file.FileId
f.FileType = file.FileType
return f
}
func (f *TemporaryFileInfo) SetFields(fields []*domain.Field) *TemporaryFileInfo {
f.Fields = fields
return f
}
func (f *TemporaryFileInfo) SetTotal(total int) *TemporaryFileInfo {
f.Total = total
return f
}
type FileCacheService struct {
}
func (s *FileCacheService) Update(key string, file *domain.File, fields []*domain.Field, total int) (*TemporaryFileInfo, error) {
ok, err := ZeroCoreRedis.Exists(key)
var response = &TemporaryFileInfo{}
if err != nil {
return response, err
}
if !ok {
response.SetFile(file).SetFields(fields).SetTotal(total)
return response, ZeroCoreRedis.Setex(key, json.MarshalToString(response), TemporaryFileExpire)
}
data, err := ZeroCoreRedis.Get(key)
if err != nil {
return nil, err
}
err = json.UnmarshalFromString(data, response)
if err != nil {
return nil, err
}
response.SetFields(fields)
err = ZeroCoreRedis.Setex(key, json.MarshalToString(response), TemporaryFileExpire)
if err != nil {
return nil, err
}
return response, err
}
func (s *FileCacheService) Get(key string) (*TemporaryFileInfo, error) {
var response = &TemporaryFileInfo{}
ok, err := ZeroCoreRedis.Exists(key)
if err != nil {
return nil, err
}
if !ok {
return nil, fmt.Errorf("临时文件信息缓存不存在")
}
data, err := ZeroCoreRedis.Get(key)
if err != nil {
return nil, err
}
err = json.UnmarshalFromString(data, response)
if err != nil {
return nil, err
}
return response, nil
}
func NewFileCacheService() *FileCacheService {
return &FileCacheService{}
}
... ...
... ... @@ -8,7 +8,7 @@ import (
)
func KeyExportTable(ctx *domain.Context, tableId int) string {
return fmt.Sprintf("%v.lock-table.%v,%v", constant.SERVICE_ENV, tableId, ctx.OperatorId)
return fmt.Sprintf("%v:lock-table:%v:%v", constant.CACHE_PREFIX, tableId, ctx.OperatorId)
}
var ZeroCoreRedis *redis.Redis
... ...
... ... @@ -2,6 +2,7 @@ package starrocks
import (
"fmt"
"github.com/google/uuid"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gorm.io/gorm"
)
... ... @@ -10,7 +11,9 @@ func Insert(db *gorm.DB, tableName string, fields []*domain.FieldValue) error {
value := map[string]interface{}{}
for _, f := range fields {
if f.Field.Flag == domain.PKField && f.Value == "" {
continue
//continue
id, _ := uuid.NewUUID()
f.Value = id.String()
}
value[f.Field.SQLName] = f.TypeValue()
}
... ...
... ... @@ -4,6 +4,7 @@ import (
"database/sql"
"fmt"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
"gorm.io/gorm"
"reflect"
"strings"
... ... @@ -23,6 +24,7 @@ func Query(params QueryOptions, queryFunc func(params QueryOptions) (*sql.Rows,
}
type QueryOptions struct {
Table *domain.Table
TableName string
Select []*domain.Field
Where []Condition
... ... @@ -60,6 +62,15 @@ func (o *QueryOptions) SetDefaultOrder() *QueryOptions {
}
// 没有排序的加一个排序,才能分页
if !hasOrder {
if o.Table != nil {
o.Where = append(o.Where, Condition{
Condition: domain.Condition{
Field: o.Table.PK,
Order: "ASC",
},
})
return o
}
o.Where = append(o.Where, Condition{
Condition: domain.Condition{
Field: domain.PK(),
... ... @@ -207,3 +218,11 @@ func WrapQueryCountWithDB(params QueryOptions, db *gorm.DB) func() (int64, error
return total, query.Error
}
}
func ArrayInterfaceToString(args []interface{}) []string {
result := make([]string, 0)
for _, arg := range args {
result = append(result, utils.AssertString(arg))
}
return result
}
... ...
... ... @@ -156,7 +156,7 @@ func (controller *TableController) AddTableStruct() {
func (controller *TableController) ExportDataTable() {
tableService := service.NewTableService(nil)
cmd := &command.TablePreviewCommand{}
controller.Unmarshal(cmd)
Must(controller.Unmarshal(cmd))
data, err := tableService.ExportDataTableV2(ParseContext(controller.BaseController), cmd)
controller.Response(data, err)
}
... ... @@ -164,7 +164,7 @@ func (controller *TableController) ExportDataTable() {
func (controller *TableController) TablePreview() {
tableService := service.NewTableService(nil)
cmd := &command.TablePreviewCommand{}
controller.Unmarshal(cmd)
Must(controller.Unmarshal(cmd))
data, err := tableService.TablePreview(ParseContext(controller.BaseController), cmd)
controller.Response(data, err)
}
... ... @@ -172,7 +172,7 @@ func (controller *TableController) TablePreview() {
func (controller *TableController) DBTablePreview() {
tableService := service.NewTableService(nil)
cmd := &command.DBTablePreviewCommand{}
controller.Unmarshal(cmd)
Must(controller.Unmarshal(cmd))
data, err := tableService.DBTablePreview(ParseContext(controller.BaseController), cmd)
controller.Response(data, err)
}
... ... @@ -180,7 +180,7 @@ func (controller *TableController) DBTablePreview() {
func (controller *TableController) FieldOptionalValues() {
tableService := service.NewTableService(nil)
cmd := &command.FieldOptionalValuesCommand{}
controller.Unmarshal(cmd)
Must(controller.Unmarshal(cmd))
data, err := tableService.FieldOptionalValues(ParseContext(controller.BaseController), cmd)
controller.Response(data, err)
}
... ... @@ -191,21 +191,21 @@ func (controller *TableController) Preview() {
cmd := &struct {
ObjectType string `json:"objectType"`
}{}
controller.Unmarshal(cmd)
Must(controller.Unmarshal(cmd))
var data interface{}
var err error
switch cmd.ObjectType {
case domain.ObjectFile:
cmd := &filecommand.LoadDataTableCommand{}
controller.Unmarshal(cmd)
Must(controller.Unmarshal(cmd))
data, err = fileService.FilePreview(ParseContext(controller.BaseController), cmd)
case domain.ObjectMetaTable:
cmd := &command.TablePreviewCommand{}
controller.Unmarshal(cmd)
Must(controller.Unmarshal(cmd))
data, err = tableService.TablePreview(ParseContext(controller.BaseController), cmd)
case domain.ObjectDBTable:
cmd := &command.DBTablePreviewCommand{}
controller.Unmarshal(cmd)
Must(controller.Unmarshal(cmd))
data, err = tableService.DBTablePreview(ParseContext(controller.BaseController), cmd)
default:
... ... @@ -215,8 +215,8 @@ func (controller *TableController) Preview() {
func (controller *TableController) RowEdit() {
tableService := service.NewTableService(nil)
cmd := &command.RowEditCommand{}
cmd := &command.RowEditCommandV2{}
Must(controller.Unmarshal(cmd))
data, err := tableService.RowEdit(ParseContext(controller.BaseController), cmd)
data, err := tableService.RowEditV2(ParseContext(controller.BaseController), cmd)
controller.Response(data, err)
}
... ...