allied-creation / character-library-metadata-bastion · 提交

转到一个项目

作者 yangfu
提交 5cafe51ae4682b980527f787dcb2a0616657c691 2 个父辈 84fb62d6 c0263eef

Merge branch 'test'

内嵌 并排对比
正在显示 61 个修改的文件 包含 1443 行增加 和 174 行删除
ALTER TABLE files ADD file_from TEXT;
ALTER TABLE files ADD app_key TEXT;
Update files set file_from = 'ByteBankWebClient';
CREATE INDEX IF NOT EXISTS idx_files_app_key ON metadata.files USING btree(app_key);
\ No newline at end of file
... ...
... ... @@ -4,6 +4,7 @@ go 1.16
require (
github.com/ajg/form v1.5.1 // indirect
github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible // indirect
github.com/beego/beego/v2 v2.0.1
github.com/bwmarrin/snowflake v0.3.0
github.com/dgrijalva/jwt-go v3.2.0+incompatible
... ...
package service
import (
"github.com/linmadan/egglib-go/core/application"
pgTransaction "github.com/linmadan/egglib-go/transaction/pg"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/event/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/cache"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/domainService"
)
func (tableEventService *TableEventService) DigitalPlatformEventSubscribe(ctx *domain.Context, cmd *command.TableEventCommand) (interface{}, error) {
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
//if err := transactionContext.StartTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
//defer func() {
// transactionContext.RollbackTransaction()
//}()
var (
dataChanged = true
structChanged = true
)
data := cmd.EventTable
tableId := 0
switch data.Type {
case domain.TableDataImportEvent, domain.TableDataEditEvent, domain.TableDeleteEvent:
// dataChanged = true
tableId = data.Table.TableId
case domain.QuerySetUpdateEvent:
tableId = data.QuerySet.QuerySetInfo.BindTableId
// structChanged = true
}
if tableId == 0 {
return nil, nil
}
var notifyData = struct {
DataChanged bool `json:"dataChanged"`
StructChanged bool `json:"structChanged"`
TableId int `json:"tableId"`
Event string `json:"event"`
TableAffectedList []int `json:"tableAffectedList"`
}{
DataChanged: dataChanged,
StructChanged: structChanged,
TableId: tableId,
Event: data.Type.ToString(),
}
// tableId 相关联的
tableRepository, _, _ := factory.FastPgTable(transactionContext, 0)
_, tables, err := tableRepository.Find(map[string]interface{}{"context": data.Context, "tableTypesNotIn": []string{domain.TemporaryTable.ToString(), domain.ExcelTable.ToString()}})
if err != nil {
return nil, err
}
tableDependencyService, _ := domainService.NewTableDependencyService(transactionContext.(*pgTransaction.TransactionContext))
tableDependTree := tableDependencyService.TableDependTree(tables, tableId)
tree := tableDependTree.Tree
//tableService := tableservice.NewTableService(nil)
for i := range tree {
cache.DefaultDataTableCacheService.DeleteDataTable(tree[i])
// fresh cache
//tableService.TablePreview(data.Context, &tablecommand.TablePreviewCommand{
// TableId: tree[i],
// ObjectType: domain.ObjectMetaTable,
// PageSize: 10000,
// PageNumber: 0,
// UseCache: true,
//})
notifyData.TableAffectedList = append(notifyData.TableAffectedList, tree[i])
}
//if err := transactionContext.CommitTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
return nil, nil
}
... ...
package command
import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
type AppTableFileAppendDataCommand struct {
Name string `json:"name"`
// name 字段中文名
Fields []*domain.Field `json:"fields"`
// 数据列表 key:name(字段中文名) value:值（字符串类型）
Data []map[string]string `json:"data"`
AppKey string `json:"appKey"`
}
... ...
package command
import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
type CreateAppTableFileCommand struct {
Name string `json:"name"`
// name 字段中文名
Fields []*domain.Field `json:"fields"`
// 数据列表 key:name(字段中文名) value:值（字符串类型）
Data []map[string]string `json:"data"`
}
... ...
package command
type DeleteAppTableFileCommand struct {
Name string `json:"name"`
AppKey string `json:"appKey"`
}
... ...
... ... @@ -17,6 +17,10 @@ type CreateFileCommand struct {
Url string `cname:"文件地址" json:"url" valid:"Required"`
// 文件大小 单位KB
FileSize int `cname:"文件大小" json:"fileSize" valid:"Required"`
// 文件来源
FileFrom string `json:"-"`
// AppKey
AppKey string `json:"-"`
}
var MaxFileSize = 50 * 1024 * 1024
... ...
... ... @@ -15,6 +15,7 @@ type EditDataTableCommand struct {
//
//Fields []*domain.Field
domain.EditTableRequest
HeaderRow int `json:"headerRow"` // 行号 默认:0
}
func (editDataTableCommand *EditDataTableCommand) Valid(validation *validation.Validation) {
... ... @@ -30,6 +31,7 @@ func (editDataTableCommand *EditDataTableCommand) Valid(validation *validation.V
validation.Error("文件ID不能为空")
return
}
editDataTableCommand.Where.HeaderRow = domain.SetHeaderRow(editDataTableCommand.HeaderRow)
}
func (editDataTableCommand *EditDataTableCommand) ValidateCommand() error {
... ...
... ... @@ -23,6 +23,7 @@ func (loadDataTableCommand *LoadDataTableCommand) Valid(validation *validation.V
if loadDataTableCommand.PageSize == 0 {
loadDataTableCommand.PageSize = 20
}
loadDataTableCommand.HeaderRow = domain.SetHeaderRow(loadDataTableCommand.HeaderRow)
}
func (loadDataTableCommand *LoadDataTableCommand) ValidateCommand() error {
... ...
package command
import (
"fmt"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"reflect"
"strings"
"github.com/beego/beego/v2/core/validation"
)
type ResetTableHeaderCommand struct {
// 文件ID
FileId int `cname:"文件ID" json:"objectId" valid:"Required"`
domain.Where
}
func (loadDataTableCommand *ResetTableHeaderCommand) Valid(validation *validation.Validation) {
loadDataTableCommand.HeaderRow = domain.SetHeaderRow(loadDataTableCommand.HeaderRow)
}
func (loadDataTableCommand *ResetTableHeaderCommand) ValidateCommand() error {
valid := validation.Validation{}
b, err := valid.Valid(loadDataTableCommand)
if err != nil {
return err
}
if !b {
elem := reflect.TypeOf(loadDataTableCommand).Elem()
for _, validErr := range valid.Errors {
field, isExist := elem.FieldByName(validErr.Field)
if isExist {
return fmt.Errorf(strings.Replace(validErr.Message, validErr.Field, field.Tag.Get("cname"), -1))
} else {
return fmt.Errorf(validErr.Message)
}
}
}
return nil
}
... ...
... ... @@ -18,6 +18,10 @@ type FileDto struct {
Ext string `json:"ext"`
// 创建时间
Time string `json:"time"`
// 行号
HeaderRow int `json:"headerRow"`
// 所属应用
AppKey string `json:"appKey"`
}
func (d *FileDto) Load(f *domain.File) *FileDto {
... ... @@ -30,5 +34,14 @@ func (d *FileDto) Load(f *domain.File) *FileDto {
d.FileType = f.FileType
d.Ext = f.FileInfo.Ext
d.Time = xtime.New(f.UpdatedAt).Local().Format("2006-01-02 15:04:05")
d.HeaderRow = domain.GetHeaderRow(f.FileInfo.HeaderRow)
d.AppKey = f.AppKey
return d
}
type AppDto struct {
AppId int64 `json:"appId"`
AppKey string `json:"appKey"`
AppName string `json:"appName"`
Files []*FileDto `json:"files"`
}
... ...
package query
type ListAppTableFileCommand struct {
Name string `json:"name"`
AppKey string `json:"appKey"`
}
... ...
... ... @@ -17,11 +17,12 @@ type SearchFileQuery struct {
// 页码
// PageNumber int `cname:"页码" json:"pageNumber,omitempty"`
// 页数
FileName string `cname:"文件名称" json:"fileName,omitempty"`
PageSize int `cname:"页数" json:"pageSize,omitempty"`
LastId int `cname:"最后一条记录ID" json:"lastId"`
FileType domain.FileType `cname:"文件类型" json:"fileType" valid:"Required"`
Context *domain.Context
FileName string `cname:"文件名称" json:"fileName,omitempty"`
PageSize int `cname:"页数" json:"pageSize,omitempty"`
LastId int `cname:"最后一条记录ID" json:"lastId"`
FileType domain.FileType `cname:"文件类型" json:"fileType" valid:"Required"`
InAppKeys []string `json:"inAppKeys"`
Context *domain.Context
}
func (cmd *SearchFileQuery) Valid(validation *validation.Validation) {
... ...
package service
import (
"bytes"
"errors"
"fmt"
"github.com/beego/beego/v2/client/httplib"
"github.com/linmadan/egglib-go/core/application"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/query"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api/apilib"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
"os"
"strings"
"time"
)
func (fileService *FileService) CreateAppTableFile(ctx *domain.Context, cmd *command.CreateAppTableFileCommand) (*command.CreateFileCommand, error) {
response := &command.CreateFileCommand{}
var (
titles = make([]string, 0)
dataList = make([][]string, 0)
)
for _, filed := range cmd.Fields {
titles = append(titles, filed.Name)
}
for i := range cmd.Data {
row := make([]string, 0)
for _, filed := range titles {
if v, ok := cmd.Data[i][filed]; ok {
row = append(row, v)
} else {
row = append(row, "")
}
}
dataList = append(dataList, row)
}
fileUpload, err := saveFile(cmd.Name, titles, dataList, nil)
if err != nil {
return nil, factory.FastError(err)
}
response.Name = cmd.Name
if !strings.HasSuffix(response.Name, domain.XLSX) {
response.Name = response.Name + domain.XLSX
}
response.Url = fileUpload.Url
response.FileSize = int(fileUpload.FileSize)
response.FileFrom = domain.FileFromDigitalAppClient
return response, nil
}
func saveFile(name string, title []string, dataList [][]string, toInterfaces func([]string) []interface{}) (FileUpload, error) {
var (
response = FileUpload{}
err error
)
var writerTo = excel.NewXLXSWriterTo(title, dataList)
if toInterfaces != nil {
writerTo.ToInterfaces = toInterfaces
}
filename := fmt.Sprintf("%v_%v.xlsx", name, time.Now().Format("060102150405"))
path := fmt.Sprintf("public/%v", filename)
if err = writerTo.Save(path); err != nil {
return response, factory.FastError(err)
}
api := apilib.NewApiAuthLib(constant.OPEN_API_HOST)
uploadResponse, err := api.Upload(apilib.RequestUpload{
UploadFileMap: map[string]string{"file": path},
})
if err != nil {
return response, err
}
if stat, err := os.Stat(path); err == nil {
response.FileSize = stat.Size()
}
response.Url = uploadResponse.Path
response.FileName = name
response.Ext = domain.XLSX
return response, nil
}
type FileUpload struct {
Url string `json:"url"`
Ext string `json:"ext"`
FileName string `json:"fileName"`
FileSize int64 `json:"fileSize"`
}
func (fileService *FileService) DeleteAppTableFile(ctx *domain.Context, cmd *command.DeleteAppTableFileCommand) (interface{}, error) {
//if err := cmd.ValidateCommand(); err != nil {
// return nil, application.ThrowError(application.ARG_ERROR, err.Error())
//}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
fileRepository, file, _ := factory.FastPgFile(transactionContext, 0)
file, err = fileRepository.FindOne(map[string]interface{}{"appKey": cmd.AppKey, "fileName": cmd.Name, "fileType": domain.SourceFile})
if err == domain.ErrorNotFound {
return nil, factory.FastError(errors.New("文件不存在"))
}
if err != nil {
return nil, factory.FastError(err)
}
if _, err := fileRepository.Remove(file); err != nil {
return nil, factory.FastError(err)
}
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return struct{}{}, nil
}
func (fileService *FileService) AppTableFileAppendData(ctx *domain.Context, cmd *command.AppTableFileAppendDataCommand) (interface{}, error) {
//if err := cmd.ValidateCommand(); err != nil {
// return nil, application.ThrowError(application.ARG_ERROR, err.Error())
//}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
fileRepository, file, _ := factory.FastPgFile(transactionContext, 0)
file, err = fileRepository.FindOne(map[string]interface{}{"appKey": cmd.AppKey, "fileName": cmd.Name, "fileType": domain.SourceFile})
if err == domain.ErrorNotFound {
return nil, factory.FastError(errors.New("文件不存在"))
}
if err != nil {
return nil, factory.FastError(err)
}
// 下载文件
f, err := httplib.Get(domain.ConvertFileUrlToInternal(file.FileInfo.Url)).Bytes()
if err != nil {
return nil, factory.FastError(err)
}
reader := bytes.NewReader(f)
var importer *excel.Importer = excel.NewExcelImportByFile(file.FileInfo.Ext)
data, err := importer.OpenExcelFromIoReader(reader)
if err != nil {
return nil, factory.FastError(err)
}
titles := importer.Reader().Header().Columns
for _, f := range cmd.Fields {
found := false
for _, column := range titles {
if column == f.Name {
found = true
break
}
}
if !found {
titles = append(titles, f.Name)
}
}
// 填充旧数据
// 追加文件
for i := range data {
if len(data[i]) < len(titles) {
for j := 0; j < (len(titles) - len(data[i])); j++ {
data[i] = append(data[i], "")
}
}
}
for i := range cmd.Data {
row := make([]string, 0)
for _, filed := range titles {
if v, ok := cmd.Data[i][filed]; ok {
row = append(row, v)
} else {
row = append(row, "")
}
}
data = append(data, row)
}
// 上传文件
fileUpload, err := saveFile(cmd.Name, titles, data, nil)
if err != nil {
return nil, factory.FastError(err)
}
// 更新文件
file.FileInfo.Url = fileUpload.Url
file.FileInfo.FileSize = int(fileUpload.FileSize)
file.FileInfo.RowCount = len(data)
_, err = fileRepository.Save(file)
if err != nil {
return nil, factory.FastError(err)
}
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return struct{}{}, nil
}
func (fileService *FileService) AppTableFileList(ctx *domain.Context, cmd *query.ListAppTableFileCommand) (interface{}, error) {
return fileService.GetAppFile(ctx, cmd.AppKey, cmd.Name)
}
... ...
... ... @@ -4,14 +4,14 @@ import (
"bytes"
"fmt"
"github.com/beego/beego/v2/client/httplib"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
"time"
"github.com/linmadan/egglib-go/core/application"
"github.com/linmadan/egglib-go/transaction/pg"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/dto"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/domainService"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
)
... ... @@ -44,6 +44,54 @@ func (fileService *FileService) FilePreview(ctx *domain.Context, loadDataTableCo
return data, nil
}
func (fileService *FileService) ResetHeaderRow(ctx *domain.Context, loadDataTableCommand *command.ResetTableHeaderCommand) (interface{}, error) {
if err := loadDataTableCommand.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
cache := redis.NewFileCacheService()
temporaryFile, err := cache.Get(redis.KeyTemporaryFileInfo(loadDataTableCommand.FileId))
if err != nil {
return nil, factory.FastError(err)
}
loadDataTableService, _ := factory.CreateLoadDataTableService(transactionContext)
data, err := loadDataTableService.RePreview(ctx, loadDataTableCommand.FileId, temporaryFile.Fields, loadDataTableCommand.Where)
// 处理错误
level := domain.LevelInfo
errMsg := ""
if err != nil {
level = domain.LevelError
errMsg = err.Error()
}
if logErr := domainService.FastLog(transactionContext.(*pg.TransactionContext),
domain.VerifiedStepLog, temporaryFile.FileId, &domainService.ExcelTableResetHeaderLog{
LogEntry: domain.NewLogEntry(temporaryFile.FileName, domain.VerifiedFile.ToString(), domain.FileVerify,
ctx.WithValue(domain.ContextWithLogLevel, level).
WithValue(domain.ContextWithLogMsg, errMsg)),
HeaderRow: domain.GetHeaderRow(loadDataTableCommand.HeaderRow),
}); logErr != nil {
return nil, logErr
}
if err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return data, nil
}
// PrepareTemporaryFile 准备临时文件
func (fileService *FileService) PrepareTemporaryFile(ctx *domain.Context, cmd *command.PrepareTemporaryFileCommand) (interface{}, error) {
if err := cmd.ValidateCommand(); err != nil {
... ... @@ -104,6 +152,13 @@ func (fileService *FileService) EditDataTable(ctx *domain.Context, editDataTable
if editDataTableCommand.Action == "remove-column" && len(temporaryFile.Fields) == len(editDataTableCommand.ProcessFields) {
return nil, factory.FastError(fmt.Errorf("请至少保留一个数据列"))
}
if editDataTableCommand.Action == "rename-column" {
targetColumn := editDataTableCommand.ProcessFieldNames[0]
newColumnName := editDataTableCommand.Params["newColumnName"].(string)
if len(temporaryFile.MatchFields([]string{newColumnName})) > 0 && newColumnName != targetColumn {
return nil, factory.FastError(fmt.Errorf("已存在相同名称，修改无效"))
}
}
// allowAction := func(fields []*domain.Field, action string) error {
// for _, f := range fields {
// if f.SQLType != string(domain.String) &&
... ... @@ -148,10 +203,14 @@ func (fileService *FileService) FlushDataTable(ctx *domain.Context, flushDataTab
if err != nil {
return nil, factory.FastError(err)
}
if err = temporaryFile.Valid(); err != nil {
return nil, factory.FastError(err)
}
if _, err := flushDataTableService.Flush(ctx, flushDataTableCommand.ObjectId, &domain.Table{
DataFields: temporaryFile.Fields,
RowCount: temporaryFile.Total,
HeaderRow: temporaryFile.HeaderRow,
}); err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
... ... @@ -266,11 +325,7 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
return nil, factory.FastError(err)
}
var response = struct {
Url string `json:"url"`
Ext string `json:"ext"`
FileName string `json:"fileName"`
}{}
var response = FileUpload{}
if file.FileType == domain.SourceFile.ToString() {
response.Url = file.FileInfo.Url
response.Ext = domain.XLSX
... ... @@ -282,7 +337,7 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
return nil, factory.FastError(err)
}
f, err := httplib.Get(file.FileInfo.Url).Bytes()
f, err := httplib.Get(domain.ConvertFileUrlToInternal(file.FileInfo.Url)).Bytes()
if err != nil {
return nil, factory.FastError(err)
}
... ... @@ -292,17 +347,46 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
if err != nil {
return nil, factory.FastError(err)
}
filename := fmt.Sprintf("%v_%v.xlsx", file.FileInfo.Name, time.Now().Format("060102150405"))
path := fmt.Sprintf("public/%v", filename)
writerTo := excel.NewXLXSWriterTo(importer.Reader().Header().Columns, data)
writerTo.ToInterfaces = domain.MakeToInterfaces(table.DataFields)
if err := writerTo.Save(path); err != nil {
response, err = saveFile(file.FileInfo.Name, importer.Reader().Header().Columns, data, domain.MakeToInterfaces(table.DataFields))
if err != nil {
return nil, factory.FastError(err)
}
response.Url = domain.DownloadUrl(filename)
response.FileName = file.FileInfo.Name
response.Ext = domain.XLSX
//filename := fmt.Sprintf("%v_%v.xlsx", file.FileInfo.Name, time.Now().Format("060102150405"))
//path := fmt.Sprintf("public/%v", filename)
//writerTo := excel.NewXLXSWriterTo(importer.Reader().Header().Columns, data)
//writerTo.ToInterfaces = domain.MakeToInterfaces(table.DataFields)
//if err := writerTo.Save(path); err != nil {
// return nil, factory.FastError(err)
//}
//
//var (
// config = utils.RouterConfig{
// OssEndPoint: "oss-cn-hangzhou.aliyuncs-internal.com",
// AccessKeyID: "LTAI4Fz1LUBW2fXp6QWaJHRS",
// AccessKeySecret: "aLZXwK8pgrs10Ws03qcN7NsrSXFVsg",
// BuckName: "byte-bank",
// }
// key = fmt.Sprintf("byte-bank/%v/%v", time.Now().Format("2006-01-02"), filename)
//)
//bucket, bucketErr := utils.NewBucket(config)
//if bucketErr == nil && bucket != nil {
// log.Logger.Info(fmt.Sprintf("end-point:%v key:%v", config.OssEndPoint, key))
// f, _ := os.Open(path)
// if err = utils.CreateObjects(bucket, utils.Object{
// Key: key,
// Value: f,
// }); err != nil {
// log.Logger.Error(err.Error())
// } else {
// response.Url = domain.ConvertInternalFileUrlToPublic(fmt.Sprintf("https://%v.%v/%v", config.BuckName, config.OssEndPoint, key))
// }
//}
//if len(response.Url) == 0 {
// response.Url = domain.DownloadUrl(filename)
//}
//response.FileName = file.FileInfo.Name
//response.Ext = domain.XLSX
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
... ...
... ... @@ -2,7 +2,10 @@ package service
import (
"fmt"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api/authlib"
"path/filepath"
"strings"
"time"
"github.com/linmadan/egglib-go/core/application"
... ... @@ -49,6 +52,8 @@ func (fileService *FileService) CreateFile(ctx *domain.Context, createFileComman
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
Context: ctx,
FileFrom: createFileCommand.FileFrom,
AppKey: createFileCommand.AppKey,
}
fileRepository, _, _ := factory.FastPgFile(transactionContext, 0)
... ... @@ -213,6 +218,126 @@ func (fileService *FileService) SearchFile(listFileQuery *query.SearchFileQuery)
}, nil
}
// 返回文件服务列表
func (fileService *FileService) SearchAppFile(ctx *domain.Context, listFileQuery *query.SearchFileQuery) (interface{}, error) {
if err := listFileQuery.ValidateQuery(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
var fileRepository, _, _ = factory.FastPgFile(transactionContext, 0)
apiAuthLib := authlib.NewApiAuthLib(constant.AUTH_SERVER_HOST).WithToken(ctx.AccessToken)
response, err := apiAuthLib.MeAppInfo(authlib.RequestUserMeQuery{UserId: ctx.TenantId})
if err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
inAppKeys := make([]string, 0)
for _, app := range response.Apps {
inAppKeys = append(inAppKeys, app.AppKey)
}
var (
fileDtos = make([]*dto.FileDto, 0)
total int64
)
if len(inAppKeys) > 0 {
queryOptions := utils.ObjectToMap(listFileQuery)
queryOptions["inAppKeys"] = inAppKeys
queryOptions["limit"] = 1000
queryOptions["fileName"] = listFileQuery.FileName
count, files, err := fileRepository.Find(queryOptions)
if err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
for _, file := range files {
var item = &dto.FileDto{}
item.Load(file)
fileDtos = append(fileDtos, item)
}
total = count
}
var apps = make([]*dto.AppDto, 0)
for _, app := range response.Apps {
if len(listFileQuery.FileName) > 0 && !strings.Contains(app.AppName, listFileQuery.FileName) {
continue
}
files := make([]*dto.FileDto, 0)
for _, file := range fileDtos {
if file.AppKey == app.AppKey {
files = append(files, file)
}
}
apps = append(apps, &dto.AppDto{
AppId: app.AppId,
AppKey: app.AppKey,
AppName: app.AppName,
Files: files,
})
}
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return map[string]interface{}{
"apps": apps,
"count": total,
}, nil
}
// GetAppFile 返回应用对应的文件服务列表
func (fileService *FileService) GetAppFile(ctx *domain.Context, appKey string, fileName string) (interface{}, error) {
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
var fileRepository, _, _ = factory.FastPgFile(transactionContext, 0)
var (
fileDtos = make([]*dto.FileDto, 0)
total int64
)
queryOptions := make(map[string]interface{})
queryOptions["fileType"] = domain.SourceFile
queryOptions["inAppKeys"] = []string{appKey}
queryOptions["limit"] = 100
count, files, err := fileRepository.Find(queryOptions)
if err != nil {
return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
}
for _, file := range files {
var item = &dto.FileDto{}
if fileName != "" && file.FileInfo.Name != fileName {
continue
}
item.Load(file)
fileDtos = append(fileDtos, item)
}
total = count
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return map[string]interface{}{
"count": total,
"files": fileDtos,
}, nil
}
// 移除文件服务
func (fileService *FileService) RemoveFile(ctx *domain.Context, removeFileCommand *command.RemoveFileCommand) (interface{}, error) {
if err := removeFileCommand.ValidateCommand(); err != nil {
... ...
... ... @@ -17,10 +17,11 @@ type TablePreviewCommand struct {
PageSize int `json:"pageSize"`
Where domain.Where `json:"where"`
UseCache bool `json:"useCache"`
HiddenData bool `json:"hiddenData"` // 隐藏数据，只返回结构
}
func (cmd *TablePreviewCommand) Valid(validation *validation.Validation) {
if cmd.UseCache && cmd.PageSize == 0 {
if cmd.UseCache && cmd.PageSize == 0 && cmd.Where.PageSize == 0 {
cmd.PageNumber = 1
cmd.PageSize = 30000 //默认缓存前30000条
}
... ...
... ... @@ -16,6 +16,7 @@ type TablePreviewDto struct {
Fields []*domain.Field `json:"fields"`
Data interface{} `json:"grid"`
//Total int64 `json:"total"`
HiddenData bool `json:"-"`
}
func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, objectType string) *TablePreviewDto {
... ... @@ -26,6 +27,12 @@ func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, obj
d.Fields = dataTable.MatchFields(m.Fields(true))
d.Data = domain.GripData(domain.ToFieldData(m.Fields(true), dataTable.Data, false), dataTable.Total)
//d.Total = dataTable.Total
if d.HiddenData {
d.Data = map[string]interface{}{
"list": make([]map[string]string, 0),
"total": 0,
}
}
return d
}
... ...
... ... @@ -41,6 +41,12 @@ func (searchQuery *SearchTableQuery) Valid(validation *validation.Validation) {
if searchQuery.ParentTableId > 0 && searchQuery.ParentId == 0 {
searchQuery.ParentId = searchQuery.ParentTableId
}
if searchQuery.Module == 4 {
searchQuery.FilterRules = append(searchQuery.FilterRules, &FilterRule{
TableType: domain.SchemaTable.ToString(),
Status: domain.StatusOn,
})
}
}
func (searchQuery *SearchTableQuery) ValidateQuery() error {
... ...
... ... @@ -117,7 +117,9 @@ func (tableService *TableService) RowsDelete(ctx *domain.Context, cmd *command.R
if err != nil {
return nil, factory.FastError(err)
}
defer func() {
domainService.AsyncEvent(domain.NewEventTable(ctx, domain.TableDataEditEvent).WithTable(table))
}()
var options = starrocks.QueryOptions{
TableName: table.SQLName,
Select: []*domain.Field{domain.PK()}, //table.Fields(true),
... ...
... ... @@ -19,12 +19,6 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
//if err := transactionContext.StartTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
//defer func() {
// transactionContext.RollbackTransaction()
//}()
tableRepository, _, _ := factory.FastPgTable(transactionContext, 0)
_, tables, err := tableRepository.Find(utils.ObjectToMap(searchQuery))
... ... @@ -40,7 +34,9 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
item.SetDetailStructInfo(table)
}
item.Flag = domain.FlagSet
if item.TableType == domain.MainTable.ToString() || item.TableType == domain.SubTable.ToString() || item.TableType == domain.SideTable.ToString() {
if item.TableType == domain.MainTable.ToString() ||
item.TableType == domain.SubTable.ToString() ||
item.TableType == domain.SideTable.ToString() {
item.ParentId = 0
item.Status = domain.StatusOn
}
... ... @@ -55,23 +51,33 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
querySetRepository, _, _ := factory.FastPgQuerySet(transactionContext, 0)
_, querySets, _ := querySetRepository.Find(map[string]interface{}{"context": searchQuery.Context})
if !searchQuery.ReturnGroupItem {
querySets = make([]*domain.QuerySet, 0)
var (
querySetMapById = make(map[int]*domain.QuerySet)
// BindTableId , parentId
querySetMapByTableId = make(map[int]*domain.QuerySet)
)
if searchQuery.ReturnGroupItem {
for _, qs := range querySets {
querySetMapById[qs.QuerySetId] = qs
}
}
// BindTableId , parentId
querySetMapByTableId := make(map[int]*domain.QuerySet)
for _, qs := range querySets {
if qs.QuerySetInfo.BindTableId == 0 {
continue
}
querySetMapByTableId[qs.QuerySetInfo.BindTableId] = qs
}
querySetMapById := make(map[int]*domain.QuerySet)
for _, qs := range querySets {
querySetMapById[qs.QuerySetId] = qs
}
var response = make([]*dto.TableObjectDto, 0)
for index, t := range result {
v, ok := querySetMapByTableId[t.TableId]
if !ok {
continue
}
result[index].Update(v)
}
// 分组
querySetMapGroup := make(map[int]bool)
querySetGroups := make([]*domain.QuerySet, 0)
... ... @@ -79,7 +85,7 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
if filterTableByFilterRule(t, searchQuery) {
continue
}
if !domain.TableType(t.TableType).TableHasGroup() {
if !domain.TableType(t.TableType).TableIsSplitByGroup() {
response = append(response, t)
continue
}
... ... @@ -112,9 +118,6 @@ func (tableService *TableService) TableObjectSearch(searchQuery *query.SearchTab
groupItem.LoadGroup(querySetGroup)
response = append(response, groupItem)
}
//if err := transactionContext.CommitTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
sort.Slice(response, func(i, j int) bool {
item1 := response[i]
... ...
package service
import (
"fmt"
"github.com/linmadan/egglib-go/core/application"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command"
... ... @@ -8,6 +9,7 @@ import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/cache"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
)
func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command.TablePreviewCommand) (interface{}, error) {
... ... @@ -21,12 +23,6 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
//if err := transactionContext.StartTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
//defer func() {
// transactionContext.RollbackTransaction()
//}()
var dataTable *domain.DataTable
var table *domain.Table
var cacheMiss bool
... ... @@ -42,7 +38,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
cacheMiss = true
}
}
response := &dto.TablePreviewDto{}
response := &dto.TablePreviewDto{HiddenData: cmd.HiddenData}
if dataTable == nil {
switch table.TableType {
case domain.CalculateSet.ToString():
... ... @@ -57,7 +53,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
return nil, factory.FastError(err)
}
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, pageData(cmd.Where.PageNumber, cmd.Where.PageSize, dataTable.Data), false), int64(len(dataTable.Data)))
default:
var options = starrocks.QueryOptions{
Table: table,
... ... @@ -77,22 +73,34 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
// }
response.Load(table, dataTable, domain.ObjectMetaTable)
}
}
switch table.TableType {
case domain.CalculateSet.ToString():
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
default:
response.Load(table, dataTable, domain.ObjectMetaTable)
} else {
switch table.TableType {
case domain.CalculateSet.ToString():
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, pageData(cmd.Where.PageNumber, cmd.Where.PageSize, dataTable.Data), false), int64(len(dataTable.Data)))
default:
response.Load(table, dataTable, domain.ObjectMetaTable)
}
}
if cacheMiss && dataTable != nil {
// 存储缓存
cache.SetDataTable(table.TableId, dataTable)
}
//if err := transactionContext.CommitTransaction(); err != nil {
// return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
//}
return response, nil
}
func pageData(pageNumber, pageSize int, data [][]string) [][]string {
if pageNumber == 0 || pageSize == 0 {
return data
}
offset := (pageNumber - 1) * pageSize
if len(data) < offset {
return [][]string{}
}
if len(data) < offset+pageSize {
pageSize = len(data) - offset
}
log.Logger.Info(fmt.Sprintf("%v %v %v", len(data), offset, pageSize))
return data[offset : offset+pageSize]
}
... ...
... ... @@ -28,6 +28,8 @@ var BYTE_CORE_HOST = "http://192.168.100.34:8303"
var AUTH_SERVER_HOST = "http://digital-platform-dev.fjmaimaimai.com"
var OPEN_API_HOST = "http://mmm-open-api-test.fjmaimaimai.com"
var BLACK_LIST_USER int64
var BLACK_LIST_COMPANY int64
var WHITE_LIST_USERS []int
... ... @@ -52,6 +54,7 @@ func init() {
SERVICE_ENV = Configurator.DefaultString("SERVICE_ENV", SERVICE_ENV)
HTTP_PORT = Configurator.DefaultInt("HTTP_PORT", HTTP_PORT)
AUTH_SERVER_HOST = Configurator.DefaultString("AUTH_SERVER_HOST", AUTH_SERVER_HOST)
OPEN_API_HOST = Configurator.DefaultString("OPEN_API_HOST", OPEN_API_HOST)
SERVICE_NAME = fmt.Sprintf("%v-%v", SERVICE_NAME, SERVICE_ENV)
PPROF_ON = Configurator.DefaultBool("PPROF_ON", PPROF_ON)
CACHE_PREFIX = SERVICE_NAME + ":" + SERVICE_ENV
... ...
... ... @@ -111,6 +111,7 @@ type (
ProcessFields []*Field `json:"processFields"`
Action string `json:"action"`
Params map[string]interface{} `json:"params"`
HeaderRow int `json:"headerRow"`
}
DataEditDataTable struct {
... ...
... ... @@ -11,6 +11,10 @@ type Context struct {
OperatorName string `json:"operatorName"`
// 租户 （个人、企业）
TenantId int `json:"tenantId"`
// 应用键值
AppKey string `json:"appKey"`
// Token
AccessToken string `json:"-"`
// 附加数据
data map[string]interface{}
}
... ...
... ... @@ -11,6 +11,7 @@ type DataTable struct {
type Where struct {
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
HeaderRow int `json:"headerRow"` // 行号 默认:0
Conditions []Condition `json:"conditions"`
}
... ...
... ... @@ -19,6 +19,7 @@ type TableService interface {
type PreviewDataTableService interface {
Preview(ctx *Context, fileId int, fields []*Field, where Where) (interface{}, error)
RePreview(ctx *Context, fileId int, fields []*Field, where Where) (interface{}, error)
CreateTemporaryFile(ctx *Context, fileId int) (*File, error)
GetFileId() int
}
... ...
... ... @@ -10,6 +10,10 @@ const (
ExprModeExcelFunction
)
const (
MaxQueryRow = 1000
)
var (
ErrorNotFound = fmt.Errorf("没有此资源")
)
... ... @@ -170,7 +174,7 @@ func (t TableType) TableStatusEditable() bool {
return t == SchemaTable || t == CalculateItem || t == CalculateSet
}
func (t TableType) TableHasGroup() bool {
func (t TableType) TableIsSplitByGroup() bool {
return t == SchemaTable || t == SubProcessTable || t == CalculateItem || t == CalculateTable || t == CalculateSet
}
... ... @@ -419,6 +423,17 @@ func (t LogLevel) ToString() string {
return string(t)
}
type FileFromType string
const (
FileFromByteBankWebClient = "ByteBankWebClient"
FileFromDigitalAppClient = "DigitalAppClient"
)
func (t FileFromType) ToString() string {
return string(t)
}
const (
DefaultPkField = "id"
)
... ...
... ... @@ -6,6 +6,7 @@ const (
InvalidSign = 903
InvalidClientId = 904
InvalidUUid = 905
InvalidApp = 906
)
var CodeMsg = map[int]string{
... ... @@ -14,4 +15,5 @@ var CodeMsg = map[int]string{
InvalidSign: "sign 签名无效，需重新登录手机 APP",
InvalidClientId: "client id 或 client secret 无效，需强制更新手机 APP",
InvalidUUid: "uuid 无效",
InvalidApp: "AppKey或者Token无效",
}
... ...
... ... @@ -26,6 +26,10 @@ type File struct {
Version int `json:"version"`
// 扩展
Context *Context `json:"context"`
// 文件来源
FileFrom string `json:"fileFrom"`
// 来源是 DigitalAppClient 时有值
AppKey string `json:"appKey"`
}
type FileRepository interface {
... ... @@ -42,15 +46,16 @@ func (file *File) Identify() interface{} {
return file.FileId
}
func (file *File) UpdateFileUrl(url string) {
func (file *File) UpdateFileUrl(url string) *File {
if len(url) == 0 {
return
return file
}
if url == file.FileInfo.Url {
return
return file
}
file.FileInfo.Ext = filepath.Ext(url)
file.FileInfo.Url = url
return file
}
func (file *File) Update(data map[string]interface{}) error {
... ... @@ -81,7 +86,37 @@ func (file *File) CopyTo(fileType FileType, ctx *Context) *File {
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
SourceFileId: file.FileId,
FileFrom: file.FileFrom,
Context: ctx,
}
return t
}
func (file *File) SetHeaderRow(headerRow int) *File {
//file.FileInfo.HeaderRow = headerRow
return file
}
func (file *File) SetContext(context *Context) *File {
//file.FileInfo.HeaderRow = headerRow
file.Context = context
return file
}
func (file *File) GetHeaderRow() int {
return file.FileInfo.HeaderRow
}
func SetHeaderRow(headerRow int) int {
if headerRow-1 < 0 {
return 0
}
return headerRow - 1
}
func GetHeaderRow(headerRow int) int {
if headerRow == 0 {
return 1
}
return headerRow + 1
}
... ...
package domain
import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"strings"
)
// FileInfo 文件信息
type FileInfo struct {
// 名称
... ... @@ -14,4 +19,34 @@ type FileInfo struct {
RowCount int `json:"rowCount"`
// 表结构ID
TableId int `json:"tableId"`
// 行号
HeaderRow int `json:"headerRow"`
}
// ConvertFileUrlToInternal 传递内网地址
// byte-bank.oss-cn-hangzhou 当oss与字库同属于这个节点下面，使用内网进行传输
func ConvertFileUrlToInternal(fileUrl string) string {
if constant.SERVICE_ENV != "prod" {
return fileUrl
}
var bucketRegion = "https://byte-bank.oss-cn-hangzhou"
var bucketInternalRegion = "https://byte-bank.oss-cn-hangzhou-internal"
if strings.HasPrefix(fileUrl, bucketRegion) {
return strings.Replace(fileUrl, bucketRegion, bucketInternalRegion, 1)
}
return fileUrl
}
// ConvertInternalFileUrlToPublic 转内网地址为外网地址
func ConvertInternalFileUrlToPublic(fileUrl string) string {
//var bucketRegion = "https://byte-bank.oss-cn-hangzhou"
//var bucketInternalRegion = "https://byte-bank.oss-cn-hangzhou-internal"
//if strings.HasPrefix(fileUrl, bucketInternalRegion) {
// return strings.Replace(fileUrl, bucketInternalRegion, bucketRegion, 1)
//}
var internal = "-internal"
if strings.Contains(fileUrl, internal) {
return strings.Replace(fileUrl, internal, "", 1)
}
return fileUrl
}
... ...
... ... @@ -45,6 +45,9 @@ type Table struct {
Context *Context `json:"context"`
// 表信息
TableInfo *TableInfo `json:"tableInfo"`
// 表头行号 从0开始
HeaderRow int `json:"-"`
}
type TableRepository interface {
... ...
package apilib
import (
"fmt"
"github.com/beego/beego/v2/core/logs"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api"
"net/http"
"time"
)
type OpenApiLib struct {
Token string
api.BaseServiceGateway
}
func (gateway *OpenApiLib) WithToken(token string) *OpenApiLib {
gateway.Token = token
return gateway
}
func (gateway *OpenApiLib) DefaultHeader() http.Header {
var header = make(map[string][]string)
header["x-mmm-accesstoken"] = []string{gateway.Token}
header["x-mmm-appproject"] = []string{"byte-bank"}
return header
}
func NewApiAuthLib(host string) *OpenApiLib {
gt := api.NewBaseServiceGateway(host)
gt.ConnectTimeout = 360 * time.Second
gt.ReadWriteTimeout = 360 * time.Second
gt.Interceptor = func(msg string) {
logs.Debug(msg)
}
gt.ServiceName = "【开发接口】"
return &OpenApiLib{
BaseServiceGateway: gt,
}
}
func (gateway *OpenApiLib) Upload(param RequestUpload) (*DataUploadItem, error) {
url := gateway.Host() + "/v1/vod/putObject"
method := "post"
var data DataUpload
err := gateway.FastDoRequest(url, method, struct{}{}, &data, api.WithHeader(gateway.DefaultHeader()), api.WithFileMap(param.UploadFileMap))
if err != nil {
return nil, err
}
if len(data) > 0 {
return data[0], nil
}
return nil, fmt.Errorf("上传失败")
}
... ...
package apilib
type DataUpload []*DataUploadItem
type DataUploadItem struct {
Host string `json:"host"`
Key string `json:"key"`
Path string `json:"path"`
FileName string `json:"fileName"`
}
type RequestUpload struct {
UploadFileMap map[string]string
}
... ...
... ... @@ -48,6 +48,17 @@ func (gateway *ApiAuthLib) MeInfo(param RequestUserMeQuery) (*DataUserMe, error)
return &data, nil
}
func (gateway *ApiAuthLib) MeAppInfo(param RequestUserMeQuery) (*DataUserAppInfo, error) {
url := gateway.Host() + "/v1/user/me-app-info"
method := "get"
var data DataUserAppInfo
err := gateway.FastDoRequest(url, method, param, &data, api.WithHeader(gateway.DefaultHeader()))
if err != nil {
return nil, err
}
return &data, nil
}
func (gateway *ApiAuthLib) LoginCheck(param RequestLoginCheck) (*DataLoginCheck, error) {
url := gateway.Host() + "/v1/login/check?token=" + param.Token
method := "get"
... ... @@ -64,3 +75,14 @@ func (gateway *ApiAuthLib) LoginCheck(param RequestLoginCheck) (*DataLoginCheck,
}
return &data, nil
}
func (gateway *ApiAuthLib) AppLogin(param RequestAppLogin) (*DataAppLogin, error) {
url := gateway.Host() + "/v1/login/app-login"
method := "post"
var data DataAppLogin
err := gateway.FastDoRequest(url, method, param, &data, api.WithHeader(gateway.DefaultHeader()))
if err != nil {
return nil, err
}
return &data, nil
}
... ...
... ... @@ -50,3 +50,24 @@ type DataLoginCheck struct {
Code int `json:"code"`
Msg string `json:"msg"`
}
type (
RequestAppLogin struct {
AppKey string `json:"appKey" valid:"Required"` // 应用键值
Token string `json:"token" valid:"Required"` // 凭证
}
DataAppLogin struct {
AppEnabled bool `json:"appEnabled"`
}
)
type (
DataUserAppInfo struct {
Apps []AppItem `json:"apps"`
}
AppItem struct {
AppId int64
AppKey string
AppName string
}
)
... ...
... ... @@ -58,6 +58,9 @@ func (gateway BaseServiceGateway) CreateRequest(url string, method string, optio
request.Header(k, strings.Join(v, ";"))
}
}
for k, v := range options.FileMap {
request.PostFile(k, v)
}
return request.SetTimeout(gateway.ConnectTimeout, gateway.ReadWriteTimeout)
}
... ... @@ -104,11 +107,14 @@ func (gateway BaseServiceGateway) FastDoRequest(url, method string, param interf
func (gateway BaseServiceGateway) DoRequest(requestParam Request, val interface{}, options *RequestOptions) error {
r := gateway.CreateRequest(requestParam.Url, requestParam.Method, options)
req, err := r.JSONBody(requestParam.Param)
if err != nil {
return err
var err error
if len(options.FileMap) == 0 {
r, err = r.JSONBody(requestParam.Param)
if err != nil {
return err
}
}
byteResult, err := req.Bytes()
byteResult, err := r.Bytes()
if err != nil {
gateway.InterceptSimple(requestParam.Url, string(byteResult), err)
return err
... ... @@ -149,6 +155,8 @@ func NewBaseServiceGateway(host string) BaseServiceGateway {
type RequestOptions struct {
Header http.Header
// key:form key value:path
FileMap map[string]string
}
type Option func(o *RequestOptions)
... ... @@ -158,3 +166,9 @@ func WithHeader(header http.Header) Option {
o.Header = header
}
}
func WithFileMap(v map[string]string) Option {
return func(o *RequestOptions) {
o.FileMap = v
}
}
... ...
package bytelib
import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
import (
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
)
func DomainFieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema {
result := make([]domain.ColumnSchema, 0)
... ...
... ... @@ -7,15 +7,17 @@ import (
)
type RequestCheckoutTablesQuery struct {
OriginalTableId string `json:"originalTableId"`
IsFromOriginalTable bool `json:"isFromOriginalTable"`
TableFileUrl string `json:"tableFileUrl"`
ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
QueryParameters []domain.QueryParameter `json:"queryParameters"`
OriginalTableId string `json:"originalTableId"`
IsFromOriginalTable bool `json:"isFromOriginalTable"`
TableFileUrl string `json:"tableFileUrl"`
TableFileUrlInternal string `json:"tableFileUrlInternal"`
ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
QueryParameters []domain.QueryParameter `json:"queryParameters"`
//QueryParameters map[string]interface{} `json:"queryParameters"`
SortParameters map[string]interface{} `json:"sortParameters"`
HeaderRow int `json:"headerRow"`
}
type DataCheckoutTables struct {
... ... @@ -37,17 +39,22 @@ func NewRequestCheckoutTablesQuery(param domain.ReqLoadDataTable) RequestCheckou
if param.IsFromOriginalTable {
tableFileUrl = param.TableFileUrl
}
if param.HeaderRow > 0 {
isSourceFile = true
}
return RequestCheckoutTablesQuery{
OriginalTableId: param.OriginalTableId,
IsFromOriginalTable: isSourceFile,
TableFileUrl: tableFileUrl,
ColumnSchemas: param.ColumnSchemas,
PageNumber: param.PageNumber,
PageSize: param.PageSize,
OriginalTableId: param.OriginalTableId,
IsFromOriginalTable: isSourceFile,
TableFileUrl: domain.ConvertFileUrlToInternal(tableFileUrl),
TableFileUrlInternal: tableFileUrl,
ColumnSchemas: param.ColumnSchemas,
PageNumber: param.PageNumber,
PageSize: param.PageSize,
//QueryParameters: param.QueryParameters,
QueryParameters: make([]domain.QueryParameter, 0),
//QueryParameters: make(map[string]interface{}),
SortParameters: param.SortParameters,
HeaderRow: param.HeaderRow,
}
}
... ... @@ -61,6 +68,7 @@ type RequestCheckoutTablesPreProccess struct {
PageSize int `json:"pageSize"`
QueryParameters interface{} `json:"queryParameters"`
SortParameters map[string]interface{} `json:"sortParameters"`
HeaderRow int `json:"headerRow"`
}
func NewRequestCheckoutTablesPreProccess(param domain.ReqEditDataTable) RequestCheckoutTablesPreProccess {
... ... @@ -73,6 +81,7 @@ func NewRequestCheckoutTablesPreProccess(param domain.ReqEditDataTable) RequestC
PageSize: 20, //param.PageSize,
QueryParameters: []interface{}{},
SortParameters: make(map[string]interface{}),
HeaderRow: param.HeaderRow,
}
if len(param.Params) == 0 {
... ... @@ -126,12 +135,13 @@ func ToDataLoadDataTable(data DataCheckoutTables) *domain.DataLoadDataTable {
type (
RequestCheckoutTablesGenerateMasterTable struct {
OriginalTableId string `json:"originalTableId"`
CheckoutTableFileUrl string `json:"checkoutTableFileUrl"`
ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
MasterTableName string `json:"masterTableName"`
FieldSchemas []FieldSchema `json:"fieldSchemas"`
KeyFieldEnNames []string `json:"keyFieldEnNames"`
OriginalTableId string `json:"originalTableId"`
CheckoutTableFileUrl string `json:"checkoutTableFileUrl"`
CheckoutTableFileUrlInternal string `json:"checkoutTableFileUrlInternal"`
ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
MasterTableName string `json:"masterTableName"`
FieldSchemas []FieldSchema `json:"fieldSchemas"`
KeyFieldEnNames []string `json:"keyFieldEnNames"`
}
DataCheckoutTablesGenerateMasterTable struct {
MasterTableName string `json:"masterTableName"`
... ... @@ -147,12 +157,13 @@ type (
func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable) RequestCheckoutTablesGenerateMasterTable {
request := RequestCheckoutTablesGenerateMasterTable{
OriginalTableId: fmt.Sprintf("%v", param.FileId),
CheckoutTableFileUrl: param.FileUrl,
ColumnSchemas: DomainFieldsToColumnSchemas(param.Table.DataFields),
MasterTableName: param.Table.SQLName,
FieldSchemas: ToFieldSchemas(param.Table.DataFields),
KeyFieldEnNames: []string{param.Table.PK.SQLName},
OriginalTableId: fmt.Sprintf("%v", param.FileId),
CheckoutTableFileUrl: domain.ConvertFileUrlToInternal(param.FileUrl),
CheckoutTableFileUrlInternal: param.FileUrl,
ColumnSchemas: DomainFieldsToColumnSchemas(param.Table.DataFields),
MasterTableName: param.Table.SQLName,
FieldSchemas: ToFieldSchemas(param.Table.DataFields),
KeyFieldEnNames: []string{param.Table.PK.SQLName},
}
return request
}
... ... @@ -160,12 +171,13 @@ func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable)
type (
TableAppendRequest struct {
//MasterTableId string `json:"masterTableId"`
OriginalTableId string `json:"originalTableId"`
CheckoutTableFileUrl string `json:"checkoutTableFileUrl"`
DatabaseTableName string `json:"databaseTableName"`
ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
FieldSchemas []FieldSchema `json:"fieldSchemas"`
SchemaMap map[string]domain.ColumnSchema `json:"schemaMap"`
OriginalTableId string `json:"originalTableId"`
CheckoutTableFileUrl string `json:"checkoutTableFileUrl"`
CheckoutTableFileUrlInternal string `json:"checkoutTableFileUrlInternal"`
DatabaseTableName string `json:"databaseTableName"`
ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
FieldSchemas []FieldSchema `json:"fieldSchemas"`
SchemaMap map[string]domain.ColumnSchema `json:"schemaMap"`
}
MasterTablesAppendRequest struct {
... ... @@ -185,12 +197,13 @@ type (
func NewTableAppendRequest(param domain.ReqAppendData) TableAppendRequest {
columnSchemas := DomainFieldsToColumnSchemas(param.From)
req := TableAppendRequest{
OriginalTableId: intToString(param.FileId),
CheckoutTableFileUrl: param.FileUrl,
DatabaseTableName: param.Table.SQLName,
ColumnSchemas: DomainFieldsToColumnSchemas(param.ExcelTable.DataFields), //这里主要需要传递原文件所有字段 param.From
FieldSchemas: ToFieldSchemas(param.Table.DataFields),
SchemaMap: make(map[string]domain.ColumnSchema),
OriginalTableId: intToString(param.FileId),
CheckoutTableFileUrl: domain.ConvertFileUrlToInternal(param.FileUrl),
CheckoutTableFileUrlInternal: param.FileUrl,
DatabaseTableName: param.Table.SQLName,
ColumnSchemas: DomainFieldsToColumnSchemas(param.ExcelTable.DataFields), //这里主要需要传递原文件所有字段 param.From
FieldSchemas: ToFieldSchemas(param.Table.DataFields),
SchemaMap: make(map[string]domain.ColumnSchema),
}
for i := 0; i < len(param.To); i++ {
if len(columnSchemas) > i {
... ...
... ... @@ -31,6 +31,7 @@ func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableR
ProcessFields: domain.ToFields(req.ProcessFields),
Action: req.Action,
Params: req.Params,
HeaderRow: req.Where.HeaderRow,
})
if err != nil {
return nil, err
... ...
... ... @@ -29,6 +29,7 @@ func (ptr *FlushDataTableService) Flush(ctx *domain.Context, fileId int, table *
if err != nil {
return nil, fmt.Errorf("源文件不存在")
}
sourceFile.SetHeaderRow(table.HeaderRow)
// New Table
table = NewTable(domain.ExcelTable, file.FileInfo.Name, table.DataFields, table.RowCount).WithContext(ctx)
// 通知底层保存、进行回调
... ... @@ -40,7 +41,7 @@ func (ptr *FlushDataTableService) Flush(ctx *domain.Context, fileId int, table *
// 临时文件 -》校验文件
var newUrl string
if response != nil {
newUrl = response.Url
newUrl = domain.ConvertInternalFileUrlToPublic(response.Url)
}
log.Logger.Info("更新文件地址", map[string]interface{}{"from_url": file.FileInfo.Url, "to_url": newUrl, "sourceFileId": file.SourceFileId})
switch sourceFile.FileType {
... ... @@ -72,9 +73,15 @@ func (ptr *FlushDataTableService) flushSourceFile(ctx *domain.Context, table *do
if err != nil {
return err
}
if _, err = fileRepository.Save(sourceFile); err != nil {
return err
}
file.FileInfo.TableId = table.TableId
file.FileType = domain.VerifiedFile.ToString()
file.UpdateFileUrl(url)
file.UpdateFileUrl(url).SetHeaderRow(sourceFile.FileInfo.HeaderRow)
if file.FileFrom == domain.FileFromDigitalAppClient {
file.SetContext(ctx)
}
if file, err = fileRepository.Save(file); err != nil {
return err
}
... ...
... ... @@ -14,6 +14,50 @@ type PreviewDataTableService struct {
transactionContext *pgTransaction.TransactionContext
}
// RePreview 重新预览
func (ptr *PreviewDataTableService) RePreview(ctx *domain.Context, fileId int, fields []*domain.Field, where domain.Where) (interface{}, error) {
fileRepository, _ := repository.NewFileRepository(ptr.transactionContext)
file, err := fileRepository.FindOne(map[string]interface{}{"fileId": fileId})
if err != nil {
return nil, fmt.Errorf("校验文件不存在")
}
isSourceFile := true
fileUrl := file.FileInfo.Url
fileCache := redis.NewFileCacheService()
tempFile, _ := fileCache.Get(redis.KeyTemporaryFileInfo(fileId))
if tempFile == nil {
return nil, fmt.Errorf("临时文件不存在")
}
// Load Data From Excel(python api)
byteCore, _ := CreateByteCoreService()
response, err := byteCore.LoadDataTable(domain.ReqLoadDataTable{
FileId: file.FileId,
FileName: file.FileInfo.Name,
Url: file.FileInfo.Url,
Ext: file.FileInfo.Ext,
Where: where,
OriginalTableId: fmt.Sprintf("%v", file.FileId),
IsFromOriginalTable: isSourceFile,
TableFileUrl: fileUrl,
ColumnSchemas: bytelib.DomainFieldsToColumnSchemas(fields),
SortParameters: make(map[string]interface{}),
})
if err != nil {
return nil, err
}
// 强制刷新列
fields = response.Fields
cache := redis.NewFileCacheService()
tempFile, err = cache.Update(redis.KeyTemporaryFileInfo(file.FileId), file, fields, response.Total, redis.WithHeaderRow(where.HeaderRow))
if err != nil {
return nil, err
}
var responseDto = &FilePreviewDto{}
//responseDto.Load(file.FileId, response, tempFile)
return responseDto, nil
}
// Preview 预览 【data-table】
func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fields []*domain.Field, where domain.Where) (interface{}, error) {
fileRepository, _ := repository.NewFileRepository(ptr.transactionContext)
... ... @@ -98,6 +142,7 @@ type FilePreviewDto struct {
Data interface{} `json:"grid"`
PageNumber int `json:"pageNumber"`
InValidCells []domain.InValidCell `json:"inValidCells"`
HeaderRow int `json:"headerRow"`
}
func (d *FilePreviewDto) Load(fileId int, m *domain.DataLoadDataTable, file *redis.TemporaryFileInfo) {
... ... @@ -137,6 +182,7 @@ func (d *FilePreviewDto) Load(fileId int, m *domain.DataLoadDataTable, file *red
}
d.InValidCells = domain.NewInValidCells(fields, mapData)
d.HeaderRow = domain.GetHeaderRow(file.HeaderRow)
}
func (ptr *PreviewDataTableService) GetFileId() int {
... ...
... ... @@ -348,6 +348,19 @@ func (l *ExcelTableEditLog) Content() string {
return msg
}
type ExcelTableResetHeaderLog struct {
domain.LogEntry
// 操作名称
// OperateName string
// 操作列
HeaderRow int
}
func (l *ExcelTableResetHeaderLog) Content() string {
msg := fmt.Sprintf("修改标题行位置：%v行", l.HeaderRow)
return msg
}
/* *********************************************拆解模块************************************************** */
type CreateQuerySetLog struct {
... ...
... ... @@ -368,11 +368,18 @@ func (d *DataLayoutDataTable) addByLocation(cell *domain.LayoutCell, blockData [
}
d.PointEnd.Update(cell.X+len(blockData)-1, cell.Y, max)
case domain.DirectionNone:
d.DataTable.Data[cell.X][cell.Y] = blockData[0]
d.DataTable.Data[cell.X][cell.Y] = safeBlockData(blockData, 0)
d.PointEnd.Update(cell.X, cell.Y, max)
}
}
func safeBlockData(data []string, index int) string {
if len(data) < index+1 {
return ""
}
return data[index]
}
func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutCell, length int) {
for _, cell := range d.unprocessed {
switch lastCell.Direction {
... ... @@ -389,7 +396,7 @@ func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutC
}
func (d *DataLayoutDataTable) BlockData(cells *domain.LayoutCell) ([]string, int) {
var block []string
var block = make([]string, 0)
if cells.Type == domain.CellTypeText {
data := []string{cells.Data.Text}
return data, 1
... ...
... ... @@ -27,4 +27,8 @@ type File struct {
Version int `comment:"版本"`
// 扩展
Context *domain.Context `json:"context"`
// 文件来源
FileFrom string
// 来源是 DigitalAppClient 时有值
AppKey string
}
... ...
... ... @@ -17,5 +17,7 @@ func TransformToFileDomainModelFromPgModels(fileModel *models.File) (*domain.Fil
DeletedAt: fileModel.DeletedAt,
Version: fileModel.Version,
Context: fileModel.Context,
AppKey: fileModel.AppKey,
FileFrom: fileModel.FileFrom,
}, nil
}
... ...
... ... @@ -5,6 +5,7 @@ import (
"github.com/linmadan/egglib-go/utils/json"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"strings"
)
const (
... ... @@ -19,7 +20,9 @@ type TemporaryFileInfo struct {
OriginalFileId int `json:"originalFileId"`
FileId int `json:"fileId"`
FileType string `json:"fileType"`
FileName string `json:"fileName"`
Total int `json:"total"`
HeaderRow int `json:"headerRow"`
Fields []*domain.Field `json:"fields"`
// 编辑表错误，有错误不允许保存成校验文件
// 行记录错误
... ... @@ -28,6 +31,15 @@ type TemporaryFileInfo struct {
ConvertTypeErrors []ConvertTypeError `json:"convertTypeErrors"`
}
func (f *TemporaryFileInfo) Valid() error {
for _, f := range f.Fields {
if strings.HasPrefix(f.Name, "Unnamed:") {
return fmt.Errorf("存在空字段 `%s` 请重命名", f.Name)
}
}
return nil
}
func (f *TemporaryFileInfo) MatchFields(columns []string) []*domain.Field {
mapFields := (domain.Fields)(f.Fields).ToMap()
var result = make([]*domain.Field, 0)
... ... @@ -42,6 +54,7 @@ func (f *TemporaryFileInfo) MatchFields(columns []string) []*domain.Field {
func (f *TemporaryFileInfo) SetFile(file *domain.File) *TemporaryFileInfo {
f.FileId = file.FileId
f.FileType = file.FileType
f.FileName = file.FileInfo.Name
return f
}
... ... @@ -66,6 +79,11 @@ func (f *TemporaryFileInfo) SetTotal(total int) *TemporaryFileInfo {
return f
}
func (f *TemporaryFileInfo) SetHeaderRow(headerRow int) *TemporaryFileInfo {
f.HeaderRow = headerRow
return f
}
func (f *TemporaryFileInfo) AddConvertTypeError(e ConvertTypeError) *TemporaryFileInfo {
f.RemoveConvertTypeError(e)
f.addConvertTypeError(e)
... ... @@ -99,46 +117,52 @@ type ConvertTypeError struct {
type FileCacheService struct {
}
func (s *FileCacheService) Update(key string, file *domain.File, fields []*domain.Field, total int, errors ...FileCacheOptionsFunc) (*TemporaryFileInfo, error) {
func (s *FileCacheService) Update(key string, file *domain.File, fields []*domain.Field, total int, option ...FileCacheOptionsFunc) (*TemporaryFileInfo, error) {
options := NewFileCacheOptions(option...)
ok, err := ZeroCoreRedis.Exists(key)
var response = &TemporaryFileInfo{}
var tmpFile = &TemporaryFileInfo{}
if err != nil {
return response, err
return tmpFile, err
}
if !ok {
response.SetFile(file).SetFields(fields).SetTotal(total)
return response, ZeroCoreRedis.Setex(key, json.MarshalToString(response), TemporaryFileExpire)
tmpFile.SetFile(file).SetFields(fields).SetTotal(total)
if options.HasSetHeaderRow {
tmpFile.SetHeaderRow(options.HeaderRow)
}
return tmpFile, ZeroCoreRedis.Setex(key, json.MarshalToString(tmpFile), TemporaryFileExpire)
}
data, err := ZeroCoreRedis.Get(key)
if err != nil {
return nil, err
}
err = json.UnmarshalFromString(data, response)
err = json.UnmarshalFromString(data, tmpFile)
if err != nil {
return nil, err
}
response.SetFields(fields)
tmpFile.SetFields(fields)
if options.HasSetHeaderRow {
tmpFile.SetHeaderRow(options.HeaderRow)
}
options := NewFileCacheOptions(errors...)
for i := range options.AddConvertTypeErrors {
response.AddConvertTypeError(options.AddConvertTypeErrors[i])
tmpFile.AddConvertTypeError(options.AddConvertTypeErrors[i])
}
for i := range options.RemoveConvertTypeErrors {
convertType := options.RemoveConvertTypeErrors[i]
response.RemoveConvertTypeError(options.RemoveConvertTypeErrors[i])
for j := range response.Fields {
if response.Fields[j].Name == convertType.FieldName {
response.Fields[j].SQLType = convertType.ToType
tmpFile.RemoveConvertTypeError(options.RemoveConvertTypeErrors[i])
for j := range tmpFile.Fields {
if tmpFile.Fields[j].Name == convertType.FieldName {
tmpFile.Fields[j].SQLType = convertType.ToType
break
}
}
}
err = ZeroCoreRedis.Setex(key, json.MarshalToString(response), TemporaryFileExpire)
err = ZeroCoreRedis.Setex(key, json.MarshalToString(tmpFile), TemporaryFileExpire)
if err != nil {
return nil, err
}
return response, err
return tmpFile, err
}
func (s *FileCacheService) UpdateField(key string, file *domain.File, errors ...FileCacheOptionsFunc) (*TemporaryFileInfo, error) {
... ... @@ -209,6 +233,8 @@ type FileCacheOptions struct {
//OriginalFileId int
RemoveConvertTypeErrors []ConvertTypeError
AddConvertTypeErrors []ConvertTypeError
HeaderRow int
HasSetHeaderRow bool
}
type FileCacheOptionsFunc func(o *FileCacheOptions)
... ... @@ -225,6 +251,13 @@ func WithAddConvertTypeErrors(errors []ConvertTypeError) FileCacheOptionsFunc {
}
}
func WithHeaderRow(headerRow int) FileCacheOptionsFunc {
return func(o *FileCacheOptions) {
o.HeaderRow = headerRow
o.HasSetHeaderRow = true
}
}
//func WithOriginalFileId(originalFileId int) FileCacheOptionsFunc {
// return func(o *FileCacheOptions) {
// o.OriginalFileId = originalFileId
... ...
... ... @@ -28,6 +28,8 @@ func (repository *FileRepository) Save(file *domain.File) (*domain.File, error)
"deleted_at",
"version",
"context",
"file_from",
"app_key",
}
insertFieldsSnippet := sqlbuilder.SqlFieldsSnippet(sqlbuilder.RemoveSqlFields(sqlBuildFields, "file_id", "deleted_at"))
insertPlaceHoldersSnippet := sqlbuilder.SqlPlaceHoldersSnippet(sqlbuilder.RemoveSqlFields(sqlBuildFields, "file_id", "deleted_at"))
... ... @@ -48,6 +50,8 @@ func (repository *FileRepository) Save(file *domain.File) (*domain.File, error)
&file.DeletedAt,
&file.Version,
&file.Context,
&file.FileFrom,
&file.AppKey,
),
fmt.Sprintf("INSERT INTO metadata.files (%s) VALUES (%s) RETURNING %s", insertFieldsSnippet, insertPlaceHoldersSnippet, returningFieldsSnippet),
file.FileType,
... ... @@ -58,6 +62,8 @@ func (repository *FileRepository) Save(file *domain.File) (*domain.File, error)
file.UpdatedAt,
file.Version,
file.Context,
file.FileFrom,
file.AppKey,
); err != nil {
return file, err
}
... ... @@ -76,6 +82,8 @@ func (repository *FileRepository) Save(file *domain.File) (*domain.File, error)
&file.DeletedAt,
&file.Version,
&file.Context,
&file.FileFrom,
&file.AppKey,
),
fmt.Sprintf("UPDATE metadata.files SET %s WHERE file_id=? and version=? RETURNING %s", updateFieldsSnippet, returningFieldsSnippet),
file.FileType,
... ... @@ -86,6 +94,8 @@ func (repository *FileRepository) Save(file *domain.File) (*domain.File, error)
file.UpdatedAt,
file.Version,
file.Context,
file.FileFrom,
file.AppKey,
file.Identify(),
oldVersion,
); err != nil {
... ... @@ -111,6 +121,7 @@ func (repository *FileRepository) FindOne(queryOptions map[string]interface{}) (
WhereContext(query, queryOptions)
query.SetWhereByQueryOption("file_info->>'name' = ?", "fileName")
query.SetWhereByQueryOption("file_type = ?", "fileType")
query.SetWhereByQueryOption("app_key = ?", "appKey")
if err := query.First(); err != nil {
if err.Error() == "pg: no rows in result set" {
return nil, domain.ErrorNotFound
... ... @@ -138,6 +149,9 @@ func (repository *FileRepository) Find(queryOptions map[string]interface{}) (int
if v, ok := queryOptions["notInFileIds"]; ok && len(v.([]int)) > 0 {
query.Where(`file_id not in (?)`, pg.In(v.([]int)))
}
if v, ok := queryOptions["inAppKeys"]; ok && len(v.([]string)) > 0 {
query.Where(`app_key in (?)`, pg.In(v.([]string)))
}
if v, ok := queryOptions["updatedAtBegin"]; ok && !v.(time.Time).IsZero() {
query.Where(`updated_at>?`, v.(time.Time))
}
... ...
... ... @@ -368,6 +368,12 @@ func WrapDeleteFuncWithDB(db *gorm.DB) func(QueryOptions) (int64, error) {
continue
}
idList = append(idList, row[0])
if len(idList) > 5000 {
c := Condition{}
sql := fmt.Sprintf("delete from %v where id in %v", params.TableName, c.InArgs(idList))
query = db.Exec(sql)
idList = make([]string, 0)
}
}
if len(idList) == 0 {
return 0, nil
... ...
package utils
import (
"fmt"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/google/uuid"
"io"
"path"
"time"
)
type RouterConfig struct {
OssEndPoint string
AccessKeyID string
AccessKeySecret string
BuckName string
AppProject string
RegionID string
RoleArn string
}
type (
CreateStsAuthRequest struct {
Files []string `json:"files"`
}
CreateStsAuthResponse struct {
Certificate interface{} `json:"certificate"`
Files []BuckObject `json:"files"`
}
Object struct {
Key string
Value io.Reader
}
BuckObject struct {
DefaultHost string `json:"host"`
Key string `json:"key"`
Path string `json:"path"`
FileName string `json:"fileName"`
}
)
func NewBucket(config RouterConfig) (*oss.Bucket, error) {
client, err := oss.New(config.OssEndPoint, config.AccessKeyID, config.AccessKeySecret)
if err != nil {
return nil, err
}
bucket, err := client.Bucket(config.BuckName)
if err != nil {
return nil, err
}
return bucket, nil
}
func CreateObjects(bucket *oss.Bucket, objects ...Object) error {
for _, object := range objects {
err := bucket.PutObject(object.Key, object.Value)
if err != nil {
return err
}
}
return nil
}
func DeleteObjects(bucket *oss.Bucket, objects ...string) error {
for _, object := range objects {
err := bucket.DeleteObject(object)
if err != nil {
return err
}
}
return nil
}
func GetFileName(projectName, filename string) string {
date := time.Now().Format("20060102")
ext := path.Ext(filename)
if len(projectName) == 0 {
projectName = "default"
}
uid, _ := uuid.NewUUID()
filename = fmt.Sprintf("%v%v", uid.String(), ext)
sourcePath := fmt.Sprintf("%v/%v/%v", projectName, date, filename)
return sourcePath
}
... ...
... ... @@ -7,10 +7,8 @@ import (
"github.com/beego/beego/v2/server/web/context"
"github.com/linmadan/egglib-go/web/beego/filters"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api/authlib"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/controllers"
"net/http"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/middleware"
"os"
"strconv"
"strings"
... ... @@ -56,7 +54,7 @@ func init() {
}
web.InsertFilter("/*", web.BeforeRouter, filters.AllowCors())
web.InsertFilter("/*", web.BeforeRouter, JwtFilter())
web.InsertFilter("/data/*", web.BeforeRouter, middleware.JwtFilter())
web.InsertFilter("/*", web.BeforeRouter, RequestCostBefore())
web.InsertFilter("/*", web.BeforeExec, controllers.BlacklistFilter(controllers.BlacklistRouters))
web.InsertFilter("/*", web.BeforeExec, CreateRequestLogFilter(true)) // filters.CreateRequstLogFilter(Logger)
... ... @@ -76,48 +74,6 @@ func CreateRequestLogFilter(console bool) func(ctx *context.Context) {
}
}
func JwtFilter() func(ctx *context.Context) {
authLib := authlib.NewApiAuthLib(constant.AUTH_SERVER_HOST)
authLib.BaseServiceGateway.ConnectTimeout = 200 * time.Millisecond
authLib.BaseServiceGateway.ReadWriteTimeout = 200 * time.Millisecond
return func(ctx *context.Context) {
//token := ctx.Request.Header.Get("Authorization")
token := ctx.Request.Header.Get("x-mmm-accesstoken")
if len(token) > 0 {
token = strings.TrimPrefix(token, "Bearer ")
userToken := &domain.UserToken{}
err := userToken.ParseToken(token)
if err != nil {
ctx.Output.SetStatus(http.StatusOK)
ctx.Output.JSON(WithCodeMsgResponse(domain.InvalidRefreshToken), false, false)
return
}
if userToken.UserId > 0 && userToken.CompanyId > 0 {
loginCheckResponse, _ := authLib.LoginCheck(authlib.RequestLoginCheck{Token: token})
if loginCheckResponse != nil && loginCheckResponse.Code == 901 {
ctx.Output.SetStatus(http.StatusOK)
ctx.Output.JSON(WithCodeMsgResponse(domain.InvalidRefreshToken), false, false)
return
}
}
ctx.Input.SetData("UserToken", userToken)
ctx.Input.SetData("Accesstoken", token)
}
}
}
func WithCodeMsgResponse(code int) map[string]interface{} {
msg := "token 过期或无效，需刷新令牌"
if codeMsg, ok := domain.CodeMsg[code]; ok {
msg = codeMsg
}
return map[string]interface{}{
"msg": msg,
"code": code,
"data": struct{}{},
}
}
func RequestCostBefore() func(ctx *context.Context) {
return func(ctx *context.Context) {
ctx.Input.SetData("cost-begin", time.Now().UnixMilli())
... ...
... ... @@ -43,6 +43,22 @@ func Must(err error) {
}
}
func ParseAppKey(c beego.BaseController) string {
appKey := c.Ctx.Input.GetData("AppKey")
if appKey == nil {
return ""
}
return appKey.(string)
}
func ParseAccessToken(c beego.BaseController) string {
token := c.Ctx.Input.GetData("Accesstoken")
if token == nil {
return ""
}
return token.(string)
}
func ParseContext(c beego.BaseController) *domain.Context {
var companyId int = 1598224576532189184
var userId int = 1
... ... @@ -84,6 +100,7 @@ END:
CompanyId: companyId,
OperatorId: userId,
OperatorName: userName,
AccessToken: ParseAccessToken(c),
TenantId: 1,
}
return ctx
... ...
... ... @@ -16,11 +16,55 @@ func (controller *FileController) CreateFile() {
fileService := service.NewFileService(nil)
createFileCommand := &command.CreateFileCommand{}
controller.Unmarshal(createFileCommand)
createFileCommand.FileFrom = domain.FileFromByteBankWebClient
ctx := ParseContext(controller.BaseController)
data, err := fileService.CreateFile(ctx, createFileCommand)
controller.Response(data, err)
}
func (controller *FileController) CreateAppTableFile() {
fileService := service.NewFileService(nil)
createDigitalAppFileCommand := &command.CreateAppTableFileCommand{}
controller.Unmarshal(createDigitalAppFileCommand)
ctx := ParseContext(controller.BaseController)
createFileCommand, err := fileService.CreateAppTableFile(ctx, createDigitalAppFileCommand)
if err != nil {
controller.Response(nil, err)
return
}
createFileCommand.AppKey = ParseAppKey(controller.BaseController)
data, err := fileService.CreateFile(&domain.Context{}, createFileCommand)
controller.Response(data, err)
}
func (controller *FileController) DeleteAppTableFile() {
fileService := service.NewFileService(nil)
cmd := &command.DeleteAppTableFileCommand{}
controller.Unmarshal(cmd)
cmd.AppKey = ParseAppKey(controller.BaseController)
data, err := fileService.DeleteAppTableFile(&domain.Context{}, cmd)
controller.Response(data, err)
}
func (controller *FileController) AppendDataAppTableFile() {
fileService := service.NewFileService(nil)
cmd := &command.AppTableFileAppendDataCommand{}
controller.Unmarshal(cmd)
cmd.AppKey = ParseAppKey(controller.BaseController)
data, err := fileService.AppTableFileAppendData(&domain.Context{}, cmd)
controller.Response(data, err)
}
func (controller *FileController) ListAppTableFile() {
fileService := service.NewFileService(nil)
cmd := &query.ListAppTableFileCommand{}
controller.Unmarshal(cmd)
cmd.AppKey = ParseAppKey(controller.BaseController)
data, err := fileService.AppTableFileList(&domain.Context{}, cmd)
controller.Response(data, err)
}
func (controller *FileController) UpdateFile() {
fileService := service.NewFileService(nil)
updateFileCommand := &command.UpdateFileCommand{}
... ... @@ -76,16 +120,41 @@ func (controller *FileController) SearchSourceFile() {
fileService := service.NewFileService(nil)
cmd := &query.SearchFileQuery{}
Must(controller.Unmarshal(cmd))
if cmd.PageSize == 0 {
cmd.PageSize = domain.MaxQueryRow
}
cmd.FileType = domain.SourceFile
cmd.Context = ParseContext(controller.BaseController)
data, err := fileService.SearchFile(cmd)
controller.Response(data, err)
}
func (controller *FileController) SearchAppSourceFile() {
fileService := service.NewFileService(nil)
cmd := &query.SearchFileQuery{}
Must(controller.Unmarshal(cmd))
if cmd.PageSize == 0 {
cmd.PageSize = domain.MaxQueryRow
}
cmd.FileType = domain.SourceFile
data, err := fileService.SearchAppFile(ParseContext(controller.BaseController), cmd)
controller.Response(data, err)
}
func (controller *FileController) GetAppFile() {
fileService := service.NewFileService(nil)
appKey := controller.GetString("app_key", "")
data, err := fileService.GetAppFile(ParseContext(controller.BaseController), appKey, "")
controller.Response(data, err)
}
func (controller *FileController) SearchVerifiedFile() {
fileService := service.NewFileService(nil)
cmd := &query.SearchFileQuery{}
Must(controller.Unmarshal(cmd))
if cmd.PageSize == 0 {
cmd.PageSize = domain.MaxQueryRow
}
cmd.FileType = domain.VerifiedFile
cmd.Context = ParseContext(controller.BaseController)
data, err := fileService.SearchFile(cmd)
... ...
... ... @@ -248,6 +248,7 @@ func (controller *TableController) Preview() {
fileService := fileservice.NewFileService(nil)
cmd := &struct {
ObjectType string `json:"objectType"`
Action string `json:"action"`
}{}
Must(controller.Unmarshal(cmd))
var data interface{}
... ... @@ -271,6 +272,14 @@ func (controller *TableController) Preview() {
controller.Response(data, err)
}
func (controller *TableController) TableResetHeaderRow() {
fileService := fileservice.NewFileService(nil)
cmd := &filecommand.ResetTableHeaderCommand{}
Must(controller.Unmarshal(cmd))
data, err := fileService.ResetHeaderRow(ParseContext(controller.BaseController), cmd)
controller.Response(data, err)
}
func (controller *TableController) RowEdit() {
tableService := service.NewTableService(nil)
cmd := &command.RowEditCommandV2{}
... ...
package middleware
import (
"github.com/beego/beego/v2/server/web/context"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api/authlib"
"net/http"
"time"
)
func AppAccessFilter() func(ctx *context.Context) {
authLib := authlib.NewApiAuthLib(constant.AUTH_SERVER_HOST)
authLib.BaseServiceGateway.ConnectTimeout = 200 * time.Millisecond
authLib.BaseServiceGateway.ReadWriteTimeout = 200 * time.Millisecond
return func(ctx *context.Context) {
token := ctx.Request.Header.Get("x-mmm-accesstoken")
appKey := ctx.Request.Header.Get("x-mmm-appkey")
if len(appKey) == 0 || len(token) == 0 {
ctx.Output.SetStatus(http.StatusOK)
ctx.Output.JSON(WithCodeMsgResponse(domain.InvalidApp), false, false)
return
}
response, err := authLib.AppLogin(authlib.RequestAppLogin{
AppKey: appKey,
Token: token,
})
if err != nil {
ctx.Output.SetStatus(http.StatusOK)
ctx.Output.JSON(WithCodeMsgResponse(domain.InvalidApp), false, false)
return
}
if !response.AppEnabled {
ctx.Output.SetStatus(http.StatusOK)
ctx.Output.JSON(WithCodeMsgResponse(domain.InvalidApp), false, false)
return
}
ctx.Input.SetData("AppKey", appKey)
}
}
... ...
package middleware
import (
"github.com/beego/beego/v2/server/web/context"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/api/authlib"
"net/http"
"strings"
"time"
)
func JwtFilter() func(ctx *context.Context) {
authLib := authlib.NewApiAuthLib(constant.AUTH_SERVER_HOST)
authLib.BaseServiceGateway.ConnectTimeout = 200 * time.Millisecond
authLib.BaseServiceGateway.ReadWriteTimeout = 200 * time.Millisecond
return func(ctx *context.Context) {
//token := ctx.Request.Header.Get("Authorization")
token := ctx.Request.Header.Get("x-mmm-accesstoken")
if len(token) > 0 {
token = strings.TrimPrefix(token, "Bearer ")
userToken := &domain.UserToken{}
err := userToken.ParseToken(token)
if err != nil {
ctx.Output.SetStatus(http.StatusOK)
ctx.Output.JSON(WithCodeMsgResponse(domain.InvalidRefreshToken), false, false)
return
}
if userToken.UserId > 0 && userToken.CompanyId > 0 {
loginCheckResponse, _ := authLib.LoginCheck(authlib.RequestLoginCheck{Token: token})
if loginCheckResponse != nil && loginCheckResponse.Code == 901 {
ctx.Output.SetStatus(http.StatusOK)
ctx.Output.JSON(WithCodeMsgResponse(domain.InvalidRefreshToken), false, false)
return
}
}
ctx.Input.SetData("UserToken", userToken)
ctx.Input.SetData("Accesstoken", token)
}
}
}
func WithCodeMsgResponse(code int) map[string]interface{} {
msg := "token 过期或无效，需刷新令牌"
if codeMsg, ok := domain.CodeMsg[code]; ok {
msg = codeMsg
}
return map[string]interface{}{
"msg": msg,
"code": code,
"data": struct{}{},
}
}
... ...
package routers
import (
"github.com/beego/beego/v2/server/web"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/controllers"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/middleware"
)
func init() {
web.InsertFilter("/api/app-table-file/*", web.BeforeRouter, middleware.AppAccessFilter())
web.Router("/api/app-table-file/create", &controllers.FileController{}, "Post:CreateAppTableFile")
web.Router("/api/app-table-file/delete", &controllers.FileController{}, "Post:DeleteAppTableFile")
web.Router("/api/app-table-file/append-data", &controllers.FileController{}, "Post:AppendDataAppTableFile")
web.Router("/api/app-table-file/list", &controllers.FileController{}, "Post:ListAppTableFile")
}
... ...
... ... @@ -3,8 +3,10 @@ package routers
import (
"github.com/beego/beego/v2/server/web"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/controllers"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego/middleware"
)
func init() {
web.InsertFilter("/static/*", web.BeforeRouter, middleware.JwtFilter())
web.Router("/static/:filename", &controllers.DownloadFileController{}, "*:DownloadHandle")
}
... ...
... ... @@ -15,11 +15,14 @@ func init() {
web.Router("/data/files/check-status", &controllers.FileController{}, "Post:CheckFileVerifyStatus")
web.Router("/data/files/search", &controllers.FileController{}, "Post:SearchFile")
web.Router("/data/files/search-source-file", &controllers.FileController{}, "Post:SearchSourceFile")
web.Router("/data/files/search-app-source-file", &controllers.FileController{}, "Post:SearchAppSourceFile")
web.Router("/data/files/search-verified-file", &controllers.FileController{}, "Post:SearchVerifiedFile")
web.Router("/data/files/cancel-verifying-file", &controllers.FileController{}, "Post:CancelVerifyingFile")
web.Router("/data/files/prepare-temporary-file", &controllers.FileController{}, "Post:PrepareTemporaryFile")
web.Router("/data/files/export-file", &controllers.FileController{}, "Post:ExportFile")
web.Router("/data/files/app-file", &controllers.FileController{}, "Get:GetAppFile")
web.Router("/data/file-preview", &controllers.FileController{}, "Post:FilePreview")
web.Router("/data/edit-data-table", &controllers.FileController{}, "Post:EditDataTable")
web.Router("/data/flush-data-table", &controllers.FileController{}, "Post:FlushDataTable")
... ...
... ... @@ -42,5 +42,7 @@ func init() {
web.Router("/business/db-table-preview", tableController, "Post:DBTablePreview")
web.Router("/data/table-preview", tableController, "Post:Preview")
web.Router("/data/reset-header-row", tableController, "Post:TableResetHeaderRow")
web.Router("/data/tables/exec/:name", tableController, "Get:ExecScript")
}
... ...
... ... @@ -19,5 +19,9 @@ func tableDataChangeHandler(e event.Event) error {
_, err := svr.Handler(nil, &command.TableEventCommand{
EventTable: et,
})
_, err = svr.DigitalPlatformEventSubscribe(nil, &command.TableEventCommand{
EventTable: et,
})
return err
}
... ...