allied-creation / character-library-metadata-bastion · 提交

转到一个项目

作者 yangfu
提交 f2a9846fa9818d42895c87bf10f1ab8e96976386 1 个父辈 400be627

refactor: optimize some core operate

内嵌 并排对比
正在显示 18 个修改的文件 包含 352 行增加 和 42 行删除
@@ -155,14 +155,14 @@ @@ -155,14 +155,14 @@
155 "params": ["产品名2"] 155 "params": ["产品名2"]
156 } 156 }
157 ``` 157 ```
158 -- [ ] 数据预览 1 158 +- [x] 数据预览 1
159 - [ ] 表格编辑 1 159 - [ ] 表格编辑 1
160 - [ ] 保存校验文件 (文件地址) 1 160 - [ ] 保存校验文件 (文件地址) 1
161 -- [ ] 生成主表 1  
162 -- [ ] 表复制 （副表）1  
163 -- [ ] 追加数据 （主表、副表） 161 +- [x] 生成主表 1
  162 +- [x] 表复制 （副表）1
  163 +- [x] 追加数据 （主表、副表）
164 - [ ] 表删除 （主表、副表）~~、分表~~ 164 - [ ] 表删除 （主表、副表）~~、分表~~
165 -- [ ] 表拆分 1 165 +- [x] 表拆分 1
166 - [ ] 更新表结构（分表）1 166 - [ ] 更新表结构（分表）1
167 - [ ] 编辑、添加、删除表数据（副表） 1 167 - [ ] 编辑、添加、删除表数据（副表） 1
168 - [ ] 取消校验 168 - [ ] 取消校验
@@ -171,3 +171,13 @@ @@ -171,3 +171,13 @@
171 171
172 - [x] 隔天清理校验中的文件 172 - [x] 隔天清理校验中的文件
173 - [x] 隔天清理public临时文件 173 - [x] 隔天清理public临时文件
  174 +
  175 +## 表数据导出
  176 +
  177 +- [ ] 加锁,只允许当前用户同时只能发起一次导出命令 ,3min过期
  178 +- [ ] 单次拉取数量 MR
  179 + - [ ] 100W ..
  180 + - [ ] 50W 120s 读取数据库:30s 保存文件:10s 下载:30M/500K=60S;RAR压缩 24M/500k=50S
  181 + - [ ] 20W ..
  182 + - [ ] 10W ..
  183 +- [ ] 保存单个文件、压缩 | 保存多个文件、压缩
@@ -3,7 +3,6 @@ module gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion @@ -3,7 +3,6 @@ module gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion
3 go 1.16 3 go 1.16
4 4
5 require ( 5 require (
6 - github.com/Shopify/sarama v1.30.0 // indirect  
7 github.com/ajg/form v1.5.1 // indirect 6 github.com/ajg/form v1.5.1 // indirect
8 github.com/beego/beego/v2 v2.0.1 7 github.com/beego/beego/v2 v2.0.1
9 github.com/bwmarrin/snowflake v0.3.0 8 github.com/bwmarrin/snowflake v0.3.0
@@ -13,18 +12,13 @@ require ( @@ -13,18 +12,13 @@ require (
13 github.com/gavv/httpexpect v2.0.0+incompatible 12 github.com/gavv/httpexpect v2.0.0+incompatible
14 github.com/go-pg/pg/v10 v10.10.6 13 github.com/go-pg/pg/v10 v10.10.6
15 github.com/go-redis/redis v6.15.9+incompatible 14 github.com/go-redis/redis v6.15.9+incompatible
16 - github.com/google/go-cmp v0.5.7 // indirect  
17 github.com/google/go-querystring v1.1.0 // indirect 15 github.com/google/go-querystring v1.1.0 // indirect
18 github.com/google/uuid v1.3.0 16 github.com/google/uuid v1.3.0
19 github.com/imkira/go-interpol v1.1.0 // indirect 17 github.com/imkira/go-interpol v1.1.0 // indirect
20 github.com/linmadan/egglib-go v0.0.0-20210313060205-8b5e456b11f7 18 github.com/linmadan/egglib-go v0.0.0-20210313060205-8b5e456b11f7
21 - github.com/mattn/go-colorable v0.1.9 // indirect  
22 - github.com/mattn/go-isatty v0.0.14 // indirect  
23 - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect  
24 github.com/moul/http2curl v1.0.0 // indirect 19 github.com/moul/http2curl v1.0.0 // indirect
25 github.com/onsi/ginkgo v1.16.5 20 github.com/onsi/ginkgo v1.16.5
26 github.com/onsi/gomega v1.18.1 21 github.com/onsi/gomega v1.18.1
27 - github.com/prometheus/client_golang v1.12.2 // indirect  
28 github.com/sergi/go-diff v1.2.0 // indirect 22 github.com/sergi/go-diff v1.2.0 // indirect
29 github.com/shopspring/decimal v1.3.1 23 github.com/shopspring/decimal v1.3.1
30 github.com/smartystreets/goconvey v1.7.2 // indirect 24 github.com/smartystreets/goconvey v1.7.2 // indirect
@@ -36,12 +30,8 @@ require ( @@ -36,12 +30,8 @@ require (
36 github.com/yudai/gojsondiff v1.0.0 // indirect 30 github.com/yudai/gojsondiff v1.0.0 // indirect
37 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect 31 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
38 github.com/yudai/pp v2.0.1+incompatible // indirect 32 github.com/yudai/pp v2.0.1+incompatible // indirect
39 - go.uber.org/automaxprocs v1.5.1 // indirect  
40 - golang.org/x/net v0.0.0-20220421235706-1d1ef9303861 // indirect  
41 - golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 // indirect 33 + github.com/zeromicro/go-zero v1.3.4
42 golang.org/x/text v0.3.7 34 golang.org/x/text v0.3.7
43 - golang.org/x/tools v0.1.5 // indirect  
44 - google.golang.org/protobuf v1.28.0 // indirect  
45 gorm.io/driver/mysql v1.3.6 35 gorm.io/driver/mysql v1.3.6
46 gorm.io/driver/postgres v1.3.9 36 gorm.io/driver/postgres v1.3.9
47 gorm.io/gorm v1.23.8 37 gorm.io/gorm v1.23.8
@@ -28,6 +28,7 @@ func main() { @@ -28,6 +28,7 @@ func main() {
28 28
29 log.InitLogHook(constant.ENABLE_KAFKA_LOG, true) 29 log.InitLogHook(constant.ENABLE_KAFKA_LOG, true)
30 redis.InitRedis() 30 redis.InitRedis()
  31 + redis.InitZeroCoreRedis()
31 pg.Init() 32 pg.Init()
32 if err := starrocks.Init(); err != nil { 33 if err := starrocks.Init(); err != nil {
33 log.Logger.Error(err.Error()) 34 log.Logger.Error(err.Error())
  1 +package command
  2 +
  3 +import (
  4 + "fmt"
  5 + "reflect"
  6 + "strings"
  7 +
  8 + "github.com/beego/beego/v2/core/validation"
  9 +)
  10 +
  11 +type PrepareTemporaryFileCommand struct {
  12 + // 文件ID
  13 + FileId int `cname:"文件ID" json:"fileId" valid:"Required"`
  14 +}
  15 +
  16 +func (cmd *PrepareTemporaryFileCommand) Valid(validation *validation.Validation) {
  17 +
  18 +}
  19 +
  20 +func (cmd *PrepareTemporaryFileCommand) ValidateCommand() error {
  21 + valid := validation.Validation{}
  22 + b, err := valid.Valid(cmd)
  23 + if err != nil {
  24 + return err
  25 + }
  26 + if !b {
  27 + elem := reflect.TypeOf(cmd).Elem()
  28 + for _, validErr := range valid.Errors {
  29 + field, isExist := elem.FieldByName(validErr.Field)
  30 + if isExist {
  31 + return fmt.Errorf(strings.Replace(validErr.Message, validErr.Field, field.Tag.Get("cname"), -1))
  32 + } else {
  33 + return fmt.Errorf(validErr.Message)
  34 + }
  35 + }
  36 + }
  37 + return nil
  38 +}
@@ -4,6 +4,7 @@ import ( @@ -4,6 +4,7 @@ import (
4 "github.com/linmadan/egglib-go/core/application" 4 "github.com/linmadan/egglib-go/core/application"
5 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory" 5 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/command" 6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/command"
  7 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/file/dto"
7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain" 8 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
8 ) 9 )
9 10
@@ -37,6 +38,37 @@ func (fileService *FileService) FilePreview(ctx *domain.Context, loadDataTableCo @@ -37,6 +38,37 @@ func (fileService *FileService) FilePreview(ctx *domain.Context, loadDataTableCo
37 return data, nil 38 return data, nil
38 } 39 }
39 40
  41 +// PrepareTemporaryFile 准备临时文件
  42 +func (fileService *FileService) PrepareTemporaryFile(ctx *domain.Context, cmd *command.PrepareTemporaryFileCommand) (interface{}, error) {
  43 + if err := cmd.ValidateCommand(); err != nil {
  44 + return nil, application.ThrowError(application.ARG_ERROR, err.Error())
  45 + }
  46 + transactionContext, err := factory.CreateTransactionContext(nil)
  47 + if err != nil {
  48 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  49 + }
  50 + if err := transactionContext.StartTransaction(); err != nil {
  51 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  52 + }
  53 + defer func() {
  54 + transactionContext.RollbackTransaction()
  55 + }()
  56 +
  57 + loadDataTableService, _ := factory.CreateLoadDataTableService(transactionContext)
  58 + data, err := loadDataTableService.CreateTemporaryFile(ctx, cmd.FileId)
  59 + if err != nil {
  60 + return nil, application.ThrowError(application.INTERNAL_SERVER_ERROR, err.Error())
  61 + }
  62 +
  63 + if err := transactionContext.CommitTransaction(); err != nil {
  64 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  65 + }
  66 +
  67 + fileDto := &dto.FileDto{}
  68 + fileDto.Load(data)
  69 + return fileDto, nil
  70 +}
  71 +
40 // EditDataTable 编辑表格数据 72 // EditDataTable 编辑表格数据
41 func (fileService *FileService) EditDataTable(ctx *domain.Context, editDataTableCommand *command.EditDataTableCommand) (interface{}, error) { 73 func (fileService *FileService) EditDataTable(ctx *domain.Context, editDataTableCommand *command.EditDataTableCommand) (interface{}, error) {
42 if err := editDataTableCommand.ValidateCommand(); err != nil { 74 if err := editDataTableCommand.ValidateCommand(); err != nil {
@@ -12,12 +12,16 @@ import ( @@ -12,12 +12,16 @@ import (
12 type TablePreviewCommand struct { 12 type TablePreviewCommand struct {
13 // 表Id 13 // 表Id
14 TableId int `cname:"表Id" json:"objectId" valid:"Required"` 14 TableId int `cname:"表Id" json:"objectId" valid:"Required"`
15 - 15 + PageNumber int `json:"pageNumber"`
  16 + PageSize int `json:"pageSize"`
16 Where domain.Where `json:"where"` 17 Where domain.Where `json:"where"`
17 } 18 }
18 19
19 func (cmd *TablePreviewCommand) Valid(validation *validation.Validation) { 20 func (cmd *TablePreviewCommand) Valid(validation *validation.Validation) {
20 - 21 + if cmd.PageSize > 0 {
  22 + cmd.Where.PageNumber = cmd.PageNumber
  23 + cmd.Where.PageSize = cmd.PageSize
  24 + }
21 } 25 }
22 26
23 func (cmd *TablePreviewCommand) ValidateCommand() error { 27 func (cmd *TablePreviewCommand) ValidateCommand() error {
@@ -13,11 +13,17 @@ type DBTablePreviewCommand struct { @@ -13,11 +13,17 @@ type DBTablePreviewCommand struct {
13 // 表Id 13 // 表Id
14 ObjectId int `cname:"表Id" json:"objectId" valid:"Required"` 14 ObjectId int `cname:"表Id" json:"objectId" valid:"Required"`
15 //ObjectType string `json:"objectType"` 15 //ObjectType string `json:"objectType"`
  16 + // 适配外层
  17 + PageNumber int `json:"pageNumber"`
  18 + PageSize int `json:"pageSize"`
16 Where domain.Where `json:"where"` 19 Where domain.Where `json:"where"`
17 } 20 }
18 21
19 func (cmd *DBTablePreviewCommand) Valid(validation *validation.Validation) { 22 func (cmd *DBTablePreviewCommand) Valid(validation *validation.Validation) {
20 - 23 + if cmd.PageSize > 0 {
  24 + cmd.Where.PageNumber = cmd.PageNumber
  25 + cmd.Where.PageSize = cmd.PageSize
  26 + }
21 } 27 }
22 28
23 func (cmd *DBTablePreviewCommand) ValidateCommand() error { 29 func (cmd *DBTablePreviewCommand) ValidateCommand() error {
@@ -3,11 +3,14 @@ package service @@ -3,11 +3,14 @@ package service
3 import ( 3 import (
4 "fmt" 4 "fmt"
5 "github.com/linmadan/egglib-go/core/application" 5 "github.com/linmadan/egglib-go/core/application"
  6 + "github.com/zeromicro/go-zero/core/mr"
6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory" 7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command" 8 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command"
8 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain" 9 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
9 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel" 10 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
  11 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
10 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks" 12 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
  13 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
11 "time" 14 "time"
12 ) 15 )
13 16
@@ -29,21 +32,21 @@ func (tableService *TableService) ExportDataTable(ctx *domain.Context, cmd *comm @@ -29,21 +32,21 @@ func (tableService *TableService) ExportDataTable(ctx *domain.Context, cmd *comm
29 // TODO：加锁 同一个用户同一个时间点只允许一次下载 32 // TODO：加锁 同一个用户同一个时间点只允许一次下载
30 33
31 var table *domain.Table 34 var table *domain.Table
32 - var mainTable *domain.Table 35 + //var mainTable *domain.Table
33 _, table, err = factory.FastPgTable(transactionContext, cmd.TableId) 36 _, table, err = factory.FastPgTable(transactionContext, cmd.TableId)
34 if err != nil { 37 if err != nil {
35 return nil, factory.FastError(err) 38 return nil, factory.FastError(err)
36 } 39 }
37 - if table.TableType == domain.SubTable.ToString() {  
38 - _, mainTable, err = factory.FastPgTable(transactionContext, cmd.TableId)  
39 - if err != nil {  
40 - return nil, factory.FastError(err)  
41 - }  
42 - } else {  
43 - mainTable = table  
44 - } 40 + //if table.TableType == domain.SubTable.ToString() {
  41 + // _, mainTable, err = factory.FastPgTable(transactionContext, cmd.TableId)
  42 + // if err != nil {
  43 + // return nil, factory.FastError(err)
  44 + // }
  45 + //} else {
  46 + // mainTable = table
  47 + //}
45 var options = starrocks.QueryOptions{ 48 var options = starrocks.QueryOptions{
46 - TableName: mainTable.SQLName, 49 + TableName: table.SQLName,
47 Select: table.Fields(true), 50 Select: table.Fields(true),
48 } 51 }
49 // 待优化分批下载，压缩 52 // 待优化分批下载，压缩
@@ -71,3 +74,143 @@ func (tableService *TableService) ExportDataTable(ctx *domain.Context, cmd *comm @@ -71,3 +74,143 @@ func (tableService *TableService) ExportDataTable(ctx *domain.Context, cmd *comm
71 "count": count, 74 "count": count,
72 }, nil 75 }, nil
73 } 76 }
  77 +
  78 +func (tableService *TableService) ExportDataTableV2(ctx *domain.Context, cmd *command.TablePreviewCommand) (interface{}, error) {
  79 + if err := cmd.ValidateCommand(); err != nil {
  80 + return nil, application.ThrowError(application.ARG_ERROR, err.Error())
  81 + }
  82 + transactionContext, err := factory.CreateTransactionContext(nil)
  83 + if err != nil {
  84 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  85 + }
  86 + if err := transactionContext.StartTransaction(); err != nil {
  87 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  88 + }
  89 + defer func() {
  90 + transactionContext.RollbackTransaction()
  91 + }()
  92 +
  93 + // TODO：加锁 同一个用户同一个时间点只允许一次下载
  94 + locker := redis.NewLock(redis.KeyExportTable(ctx, cmd.TableId))
  95 + locker.SetExpire(60 * 2)
  96 + ok, err := locker.Acquire()
  97 + if err != nil {
  98 + return nil, factory.FastError(err)
  99 + }
  100 + if !ok {
  101 + return nil, factory.FastError(fmt.Errorf("点击过于频繁,请稍后再试"))
  102 + }
  103 + defer locker.Release()
  104 + var table *domain.Table
  105 + //var mainTable *domain.Table
  106 + _, table, err = factory.FastPgTable(transactionContext, cmd.TableId)
  107 + if err != nil {
  108 + return nil, factory.FastError(err)
  109 + }
  110 +
  111 + data, err := exportTableTo(ctx, cmd, table, 10000)
  112 + if err != nil {
  113 + return nil, factory.FastError(err)
  114 + }
  115 +
  116 + if err := transactionContext.CommitTransaction(); err != nil {
  117 + return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
  118 + }
  119 + return data, nil
  120 +}
  121 +
  122 +func exportTableTo(ctx *domain.Context, cmd *command.TablePreviewCommand, table *domain.Table, blockSize int) (interface{}, error) {
  123 + var options = starrocks.QueryOptions{
  124 + TableName: table.SQLName,
  125 + Select: table.Fields(false),
  126 + }
  127 + count, err := starrocks.QueryCount(options)
  128 + if err != nil {
  129 + return nil, factory.FastError(err)
  130 + }
  131 + generate := func(source chan<- interface{}) {
  132 + // generator
  133 + t := int(count)/blockSize + 1
  134 + for i := 0; i < t; i++ {
  135 + options := starrocks.QueryOptions{
  136 + TableName: table.SQLName,
  137 + Select: table.Fields(false),
  138 + //Where: cmd.Where.Conditions,
  139 + Offset: i * blockSize,
  140 + Limit: blockSize,
  141 + }
  142 + options.SetCondition(cmd.Where.Conditions).SetDefaultOrder()
  143 + source <- Query{
  144 + Index: i,
  145 + Options: options,
  146 + }
  147 + }
  148 + }
  149 + mapper := func(item interface{}, writer mr.Writer, cancel func(error)) {
  150 + // mapper
  151 + query := item.(Query)
  152 + var dataTable *domain.DataTable
  153 + dataTable, err = starrocks.Query(query.Options, starrocks.WrapQueryFuncWithDB(starrocks.DB))
  154 + if err != nil {
  155 + log.Logger.Error(err.Error(), map[string]interface{}{"mapper": query})
  156 + return
  157 + }
  158 + writer.Write(QueryResult{
  159 + Index: query.Index,
  160 + DataTable: dataTable,
  161 + })
  162 + }
  163 + reducer := func(pipe <-chan interface{}, writer mr.Writer, cancel func(error)) {
  164 + // reducer
  165 + var data = make(map[int]QueryResult)
  166 + for i := range pipe {
  167 + item := i.(QueryResult)
  168 + data[item.Index] = item
  169 + }
  170 + writer.Write(data)
  171 + }
  172 + result, err := mr.MapReduce(generate, mapper, reducer, mr.WithWorkers(5))
  173 + if err != nil {
  174 + return nil, err
  175 + }
  176 + dataTable := resultToDataTable(result)
  177 +
  178 + filename := fmt.Sprintf("%v_%v.xlsx", table.Name, time.Now().Format("060102150405"))
  179 + path := fmt.Sprintf("public/%v", filename)
  180 + excelWriter := excel.NewXLXSWriterTo(domain.Fields(table.Fields(false)).NameArrayString(), dataTable.Data) //
  181 + if err = excelWriter.Save(path); err != nil {
  182 + return nil, factory.FastError(err)
  183 + }
  184 +
  185 + return map[string]interface{}{
  186 + "url": domain.DownloadUrl(filename),
  187 + "count": count,
  188 + }, err
  189 +}
  190 +
  191 +func resultToDataTable(result interface{}) *domain.DataTable {
  192 + var dataTable *domain.DataTable = &domain.DataTable{}
  193 + queryResult := result.(map[int]QueryResult)
  194 + for i := 0; i < len(queryResult); i++ {
  195 + if v, ok := queryResult[i]; ok {
  196 + if dataTable == nil {
  197 + dataTable = v.DataTable
  198 + continue
  199 + }
  200 + dataTable.Total += v.DataTable.Total
  201 + dataTable.Data = append(dataTable.Data, v.DataTable.Data...)
  202 + }
  203 + }
  204 + return dataTable
  205 +}
  206 +
  207 +type Query struct {
  208 + Index int
  209 + Data struct{}
  210 + Options starrocks.QueryOptions
  211 +}
  212 +
  213 +type QueryResult struct {
  214 + Index int
  215 + DataTable *domain.DataTable
  216 +}
@@ -34,7 +34,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command @@ -34,7 +34,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
34 TableName: table.SQLName, 34 TableName: table.SQLName,
35 Select: table.Fields(true), 35 Select: table.Fields(true),
36 } 36 }
37 - options.SetCondition(cmd.Where.Conditions) 37 + options.SetCondition(cmd.Where.Conditions).SetDefaultOrder()
38 options.SetOffsetLimit(cmd.Where.PageNumber, cmd.Where.PageSize) 38 options.SetOffsetLimit(cmd.Where.PageNumber, cmd.Where.PageSize)
39 var dataTable *domain.DataTable 39 var dataTable *domain.DataTable
40 dataTable, err = factory.FastDataTable(options) 40 dataTable, err = factory.FastDataTable(options)
@@ -19,6 +19,7 @@ type TableService interface { @@ -19,6 +19,7 @@ type TableService interface {
19 19
20 type PreviewDataTableService interface { 20 type PreviewDataTableService interface {
21 Preview(ctx *Context, fileId int, fields []*Field, where Where) (interface{}, error) 21 Preview(ctx *Context, fileId int, fields []*Field, where Where) (interface{}, error)
  22 + CreateTemporaryFile(ctx *Context, fileId int) (*File, error)
22 GetFileId() int 23 GetFileId() int
23 } 24 }
24 25
@@ -202,3 +202,14 @@ func GripData(data []map[string]string, total int64) map[string]interface{} { @@ -202,3 +202,14 @@ func GripData(data []map[string]string, total int64) map[string]interface{} {
202 "total": total, 202 "total": total,
203 } 203 }
204 } 204 }
  205 +
  206 +func PK() *Field {
  207 + return &Field{
  208 + Index: 0,
  209 + Name: "序号",
  210 + SQLName: "id",
  211 + SQLType: String.ToString(),
  212 + Description: "主键",
  213 + Flag: PKField,
  214 + }
  215 +}
@@ -18,21 +18,24 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie @@ -18,21 +18,24 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie
18 fileRepository, _ := repository.NewFileRepository(ptr.transactionContext) 18 fileRepository, _ := repository.NewFileRepository(ptr.transactionContext)
19 file, err := fileRepository.FindOne(map[string]interface{}{"fileId": fileId}) 19 file, err := fileRepository.FindOne(map[string]interface{}{"fileId": fileId})
20 if err != nil { 20 if err != nil {
21 - return nil, fmt.Errorf("文件不存在") 21 + return nil, fmt.Errorf("校验文件不存在")
22 } 22 }
23 isSourceFile := false 23 isSourceFile := false
24 - fileUrl := "" 24 + fileUrl := file.FileInfo.Url
25 // Copy to TemporaryFile 25 // Copy to TemporaryFile
26 if file.FileType != domain.TemporaryFile.ToString() { 26 if file.FileType != domain.TemporaryFile.ToString() {
27 - file = file.CopyTo(domain.TemporaryFile, ctx)  
28 - if file, err = fileRepository.Save(file); err != nil {  
29 - return nil, err 27 + //file = file.CopyTo(domain.TemporaryFile, ctx)
  28 + //if file, err = fileRepository.Save(file); err != nil {
  29 + // return nil, err
  30 + //}
  31 + //isSourceFile = true
  32 + //fileUrl = file.FileInfo.Url
  33 + return nil, fmt.Errorf("校验文件不存在")
30 } 34 }
  35 + ptr.FileId = file.FileId
  36 + if len(fields) == 0 {
31 isSourceFile = true 37 isSourceFile = true
32 - fileUrl = file.FileInfo.Url  
33 } 38 }
34 - //TEST  
35 - ptr.FileId = file.FileId  
36 39
37 // Load Data From Excel(python api) 40 // Load Data From Excel(python api)
38 byteCore, _ := CreateByteCoreService() 41 byteCore, _ := CreateByteCoreService()
@@ -58,6 +61,22 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie @@ -58,6 +61,22 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie
58 return responseDto, nil 61 return responseDto, nil
59 } 62 }
60 63
  64 +func (ptr *PreviewDataTableService) CreateTemporaryFile(ctx *domain.Context, fileId int) (*domain.File, error) {
  65 + fileRepository, _ := repository.NewFileRepository(ptr.transactionContext)
  66 + file, err := fileRepository.FindOne(map[string]interface{}{"fileId": fileId})
  67 + if err != nil {
  68 + return nil, fmt.Errorf("文件不存在")
  69 + }
  70 + if !(file.FileType == domain.SourceFile.ToString() || file.FileType == domain.VerifiedFile.ToString()) {
  71 + return nil, fmt.Errorf("源文件/校验文件才可以创建临时文件")
  72 + }
  73 + file = file.CopyTo(domain.TemporaryFile, ctx)
  74 + if file, err = fileRepository.Save(file); err != nil {
  75 + return nil, err
  76 + }
  77 + return file, nil
  78 +}
  79 +
61 type FilePreviewDto struct { 80 type FilePreviewDto struct {
62 ObjectId int `json:"objectId"` 81 ObjectId int `json:"objectId"`
63 ObjectType string `json:"objectType"` 82 ObjectType string `json:"objectType"`
@@ -8,11 +8,11 @@ import ( @@ -8,11 +8,11 @@ import (
8 "github.com/go-pg/pg/v10/orm" 8 "github.com/go-pg/pg/v10/orm"
9 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant" 9 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
10 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/pg/models" 10 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/pg/models"
  11 + liblog "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
11 "gorm.io/driver/postgres" 12 "gorm.io/driver/postgres"
12 "gorm.io/gorm" 13 "gorm.io/gorm"
13 "gorm.io/gorm/logger" 14 "gorm.io/gorm/logger"
14 "log" 15 "log"
15 - "os"  
16 "time" 16 "time"
17 ) 17 )
18 18
@@ -50,7 +50,7 @@ func Init() { @@ -50,7 +50,7 @@ func Init() {
50 50
51 var err error 51 var err error
52 newLogger := logger.New( 52 newLogger := logger.New(
53 - log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer 53 + liblog.GormLogWriter{Module: "【StarRocks】"}, // io writer
54 logger.Config{ 54 logger.Config{
55 SlowThreshold: time.Second, // Slow SQL threshold 55 SlowThreshold: time.Second, // Slow SQL threshold
56 LogLevel: logger.Info, // Log level 56 LogLevel: logger.Info, // Log level
  1 +package redis
  2 +
  3 +import (
  4 + "fmt"
  5 + "github.com/zeromicro/go-zero/core/stores/redis"
  6 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
  7 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
  8 +)
  9 +
  10 +func KeyExportTable(ctx *domain.Context, tableId int) string {
  11 + return fmt.Sprintf("%v.lock-table.%v,%v", constant.SERVICE_ENV, tableId, ctx.OperatorId)
  12 +}
  13 +
  14 +var ZeroCoreRedis *redis.Redis
  15 +
  16 +func InitZeroCoreRedis() {
  17 + ZeroCoreRedis = redis.New(constant.REDIS_HOST+":"+constant.REDIS_PORT, redis.WithPass(constant.REDIS_AUTH))
  18 +}
  19 +
  20 +func NewLock(key string) *redis.RedisLock {
  21 + return redis.NewRedisLock(ZeroCoreRedis, key)
  22 +}
@@ -42,12 +42,32 @@ func (o *QueryOptions) SetOffsetLimit(pageNumber, pageSize int) { @@ -42,12 +42,32 @@ func (o *QueryOptions) SetOffsetLimit(pageNumber, pageSize int) {
42 o.Limit = pageSize 42 o.Limit = pageSize
43 } 43 }
44 44
45 -func (o *QueryOptions) SetCondition(conditions []domain.Condition) { 45 +func (o *QueryOptions) SetCondition(conditions []domain.Condition) *QueryOptions {
46 for _, c := range conditions { 46 for _, c := range conditions {
47 o.Where = append(o.Where, Condition{ 47 o.Where = append(o.Where, Condition{
48 Condition: c, 48 Condition: c,
49 }) 49 })
50 } 50 }
  51 + return o
  52 +}
  53 +
  54 +func (o *QueryOptions) SetDefaultOrder() *QueryOptions {
  55 + hasOrder := false
  56 + for _, c := range o.Where {
  57 + if len(c.Order) > 0 {
  58 + hasOrder = true
  59 + }
  60 + }
  61 + // 没有排序的加一个排序，才能分页
  62 + if !hasOrder {
  63 + o.Where = append(o.Where, Condition{
  64 + Condition: domain.Condition{
  65 + Field: domain.PK(),
  66 + Order: "ASC",
  67 + },
  68 + })
  69 + }
  70 + return o
51 } 71 }
52 72
53 type Condition struct { 73 type Condition struct {
@@ -179,6 +199,10 @@ func WrapQueryCountWithDB(params QueryOptions, db *gorm.DB) func() (int64, error @@ -179,6 +199,10 @@ func WrapQueryCountWithDB(params QueryOptions, db *gorm.DB) func() (int64, error
179 var total int64 199 var total int64
180 query := db.Table(params.TableName) 200 query := db.Table(params.TableName)
181 queryWithoutLimitOffset(query, params) 201 queryWithoutLimitOffset(query, params)
  202 + if params.Context != nil {
  203 + query.Where(fmt.Sprintf("context->>'companyId'='%v'", params.Context.CompanyId))
  204 + //query.Where("context->>'companyId'='?'", params.Context.CompanyId)
  205 + }
182 query.Count(&total) 206 query.Count(&total)
183 return total, query.Error 207 return total, query.Error
184 } 208 }
@@ -98,6 +98,14 @@ func (controller *FileController) FilePreview() { @@ -98,6 +98,14 @@ func (controller *FileController) FilePreview() {
98 controller.Response(data, err) 98 controller.Response(data, err)
99 } 99 }
100 100
  101 +func (controller *FileController) PrepareTemporaryFile() {
  102 + fileService := service.NewFileService(nil)
  103 + loadDataTableCommand := &command.PrepareTemporaryFileCommand{}
  104 + controller.Unmarshal(loadDataTableCommand)
  105 + data, err := fileService.PrepareTemporaryFile(ParseContext(controller.BaseController), loadDataTableCommand)
  106 + controller.Response(data, err)
  107 +}
  108 +
101 func (controller *FileController) EditDataTable() { 109 func (controller *FileController) EditDataTable() {
102 fileService := service.NewFileService(nil) 110 fileService := service.NewFileService(nil)
103 editDataTableCommand := &command.EditDataTableCommand{} 111 editDataTableCommand := &command.EditDataTableCommand{}
@@ -157,7 +157,7 @@ func (controller *TableController) ExportDataTable() { @@ -157,7 +157,7 @@ func (controller *TableController) ExportDataTable() {
157 tableService := service.NewTableService(nil) 157 tableService := service.NewTableService(nil)
158 cmd := &command.TablePreviewCommand{} 158 cmd := &command.TablePreviewCommand{}
159 controller.Unmarshal(cmd) 159 controller.Unmarshal(cmd)
160 - data, err := tableService.ExportDataTable(ParseContext(controller.BaseController), cmd) 160 + data, err := tableService.ExportDataTableV2(ParseContext(controller.BaseController), cmd)
161 controller.Response(data, err) 161 controller.Response(data, err)
162 } 162 }
163 163
@@ -15,6 +15,7 @@ func init() { @@ -15,6 +15,7 @@ func init() {
15 web.Router("/data/files/search-source-file", &controllers.FileController{}, "Post:SearchSourceFile") 15 web.Router("/data/files/search-source-file", &controllers.FileController{}, "Post:SearchSourceFile")
16 web.Router("/data/files/search-verified-file", &controllers.FileController{}, "Post:SearchVerifiedFile") 16 web.Router("/data/files/search-verified-file", &controllers.FileController{}, "Post:SearchVerifiedFile")
17 web.Router("/data/files/cancel-verifying-file", &controllers.FileController{}, "Post:CancelVerifyingFile") 17 web.Router("/data/files/cancel-verifying-file", &controllers.FileController{}, "Post:CancelVerifyingFile")
  18 + web.Router("/data/files/prepare-temporary-file", &controllers.FileController{}, "Post:PrepareTemporaryFile")
18 19
19 web.Router("/data/file-preview", &controllers.FileController{}, "Post:FilePreview") 20 web.Router("/data/file-preview", &controllers.FileController{}, "Post:FilePreview")
20 web.Router("/data/edit-data-table", &controllers.FileController{}, "Post:EditDataTable") 21 web.Router("/data/edit-data-table", &controllers.FileController{}, "Post:EditDataTable")