allied-creation / character-library-metadata-bastion · 提交

转到一个项目

作者 yangfu
提交 1086755cd3502a34d50254b6003c94e3df7f6ba1 1 个父辈 67873ec7

feat:big_excel_support

内嵌 并排对比
正在显示 10 个修改的文件 包含 206 行增加 和 42 行删除
@@ -4,6 +4,7 @@ go 1.16 @@ -4,6 +4,7 @@ go 1.16
4 4
5 require ( 5 require (
6 github.com/ajg/form v1.5.1 // indirect 6 github.com/ajg/form v1.5.1 // indirect
  7 + github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible // indirect
7 github.com/beego/beego/v2 v2.0.1 8 github.com/beego/beego/v2 v2.0.1
8 github.com/bwmarrin/snowflake v0.3.0 9 github.com/bwmarrin/snowflake v0.3.0
9 github.com/dgrijalva/jwt-go v3.2.0+incompatible 10 github.com/dgrijalva/jwt-go v3.2.0+incompatible
@@ -5,6 +5,9 @@ import ( @@ -5,6 +5,9 @@ import (
5 "fmt" 5 "fmt"
6 "github.com/beego/beego/v2/client/httplib" 6 "github.com/beego/beego/v2/client/httplib"
7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel" 7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
  8 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
  9 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
  10 + "os"
8 "time" 11 "time"
9 12
10 "github.com/linmadan/egglib-go/core/application" 13 "github.com/linmadan/egglib-go/core/application"
@@ -315,7 +318,7 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp @@ -315,7 +318,7 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
315 return nil, factory.FastError(err) 318 return nil, factory.FastError(err)
316 } 319 }
317 320
318 - f, err := httplib.Get(file.FileInfo.Url).Bytes() 321 + f, err := httplib.Get(domain.ConvertFileUrlToInternal(file.FileInfo.Url)).Bytes()
319 if err != nil { 322 if err != nil {
320 return nil, factory.FastError(err) 323 return nil, factory.FastError(err)
321 } 324 }
@@ -333,7 +336,31 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp @@ -333,7 +336,31 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
333 return nil, factory.FastError(err) 336 return nil, factory.FastError(err)
334 } 337 }
335 338
336 - response.Url = domain.DownloadUrl(filename) 339 + var (
  340 + config = utils.RouterConfig{
  341 + OssEndPoint: "oss-cn-hangzhou.aliyuncs-internal.com",
  342 + AccessKeyID: "LTAI4Fz1LUBW2fXp6QWaJHRS",
  343 + AccessKeySecret: "aLZXwK8pgrs10Ws03qcN7NsrSXFVsg",
  344 + BuckName: "byte-bank",
  345 + }
  346 + key = fmt.Sprintf("byte-bank/%v/%v", time.Now().Format("2006-01-02"), filename)
  347 + )
  348 + bucket, bucketErr := utils.NewBucket(config)
  349 + if bucketErr == nil && bucket != nil {
  350 + log.Logger.Info(fmt.Sprintf("end-point:%v key:%v", config.OssEndPoint, key))
  351 + f, _ := os.Open(path)
  352 + if err = utils.CreateObjects(bucket, utils.Object{
  353 + Key: key,
  354 + Value: f,
  355 + }); err != nil {
  356 + log.Logger.Error(err.Error())
  357 + } else {
  358 + response.Url = domain.ConvertInternalFileUrlToPublic(fmt.Sprintf("https://%v.%v/%v", config.BuckName, config.OssEndPoint, key))
  359 + }
  360 + }
  361 + if len(response.Url) == 0 {
  362 + response.Url = domain.DownloadUrl(filename)
  363 + }
337 response.FileName = file.FileInfo.Name 364 response.FileName = file.FileInfo.Name
338 response.Ext = domain.XLSX 365 response.Ext = domain.XLSX
339 366
@@ -17,6 +17,7 @@ type TablePreviewCommand struct { @@ -17,6 +17,7 @@ type TablePreviewCommand struct {
17 PageSize int `json:"pageSize"` 17 PageSize int `json:"pageSize"`
18 Where domain.Where `json:"where"` 18 Where domain.Where `json:"where"`
19 UseCache bool `json:"useCache"` 19 UseCache bool `json:"useCache"`
  20 + HiddenData bool `json:"hiddenData"` // 隐藏数据，只返回结构
20 } 21 }
21 22
22 func (cmd *TablePreviewCommand) Valid(validation *validation.Validation) { 23 func (cmd *TablePreviewCommand) Valid(validation *validation.Validation) {
@@ -16,6 +16,7 @@ type TablePreviewDto struct { @@ -16,6 +16,7 @@ type TablePreviewDto struct {
16 Fields []*domain.Field `json:"fields"` 16 Fields []*domain.Field `json:"fields"`
17 Data interface{} `json:"grid"` 17 Data interface{} `json:"grid"`
18 //Total int64 `json:"total"` 18 //Total int64 `json:"total"`
  19 + HiddenData bool `json:"-"`
19 } 20 }
20 21
21 func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, objectType string) *TablePreviewDto { 22 func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, objectType string) *TablePreviewDto {
@@ -26,6 +27,12 @@ func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, obj @@ -26,6 +27,12 @@ func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, obj
26 d.Fields = dataTable.MatchFields(m.Fields(true)) 27 d.Fields = dataTable.MatchFields(m.Fields(true))
27 d.Data = domain.GripData(domain.ToFieldData(m.Fields(true), dataTable.Data, false), dataTable.Total) 28 d.Data = domain.GripData(domain.ToFieldData(m.Fields(true), dataTable.Data, false), dataTable.Total)
28 //d.Total = dataTable.Total 29 //d.Total = dataTable.Total
  30 + if d.HiddenData {
  31 + d.Data = map[string]interface{}{
  32 + "list": make([]map[string]string, 0),
  33 + "total": 0,
  34 + }
  35 + }
29 return d 36 return d
30 } 37 }
31 38
@@ -42,7 +42,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command @@ -42,7 +42,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
42 cacheMiss = true 42 cacheMiss = true
43 } 43 }
44 } 44 }
45 - response := &dto.TablePreviewDto{} 45 + response := &dto.TablePreviewDto{HiddenData: cmd.HiddenData}
46 if dataTable == nil { 46 if dataTable == nil {
47 switch table.TableType { 47 switch table.TableType {
48 case domain.CalculateSet.ToString(): 48 case domain.CalculateSet.ToString():
1 package domain 1 package domain
2 2
  3 +import (
  4 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
  5 + "strings"
  6 +)
  7 +
3 // FileInfo 文件信息 8 // FileInfo 文件信息
4 type FileInfo struct { 9 type FileInfo struct {
5 // 名称 10 // 名称
@@ -17,3 +22,31 @@ type FileInfo struct { @@ -17,3 +22,31 @@ type FileInfo struct {
17 // 行号 22 // 行号
18 HeaderRow int `json:"headerRow"` 23 HeaderRow int `json:"headerRow"`
19 } 24 }
  25 +
  26 +// ConvertFileUrlToInternal 传递内网地址
  27 +// byte-bank.oss-cn-hangzhou 当oss与字库同属于这个节点下面，使用内网进行传输
  28 +func ConvertFileUrlToInternal(fileUrl string) string {
  29 + if constant.SERVICE_ENV != "prod" {
  30 + return fileUrl
  31 + }
  32 + var bucketRegion = "https://byte-bank.oss-cn-hangzhou"
  33 + var bucketInternalRegion = "https://byte-bank.oss-cn-hangzhou-internal"
  34 + if strings.HasPrefix(fileUrl, bucketRegion) {
  35 + return strings.Replace(fileUrl, bucketRegion, bucketInternalRegion, 1)
  36 + }
  37 + return fileUrl
  38 +}
  39 +
  40 +// ConvertInternalFileUrlToPublic 转内网地址为外网地址
  41 +func ConvertInternalFileUrlToPublic(fileUrl string) string {
  42 + //var bucketRegion = "https://byte-bank.oss-cn-hangzhou"
  43 + //var bucketInternalRegion = "https://byte-bank.oss-cn-hangzhou-internal"
  44 + //if strings.HasPrefix(fileUrl, bucketInternalRegion) {
  45 + // return strings.Replace(fileUrl, bucketInternalRegion, bucketRegion, 1)
  46 + //}
  47 + var internal = "-internal"
  48 + if strings.Contains(fileUrl, internal) {
  49 + return strings.Replace(fileUrl, internal, "", 1)
  50 + }
  51 + return fileUrl
  52 +}
1 package bytelib 1 package bytelib
2 2
3 -import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain" 3 +import (
  4 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
  5 +)
4 6
5 func DomainFieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema { 7 func DomainFieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema {
6 result := make([]domain.ColumnSchema, 0) 8 result := make([]domain.ColumnSchema, 0)
@@ -7,13 +7,14 @@ import ( @@ -7,13 +7,14 @@ import (
7 ) 7 )
8 8
9 type RequestCheckoutTablesQuery struct { 9 type RequestCheckoutTablesQuery struct {
10 - OriginalTableId string `json:"originalTableId"`  
11 - IsFromOriginalTable bool `json:"isFromOriginalTable"`  
12 - TableFileUrl string `json:"tableFileUrl"`  
13 - ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`  
14 - PageNumber int `json:"pageNumber"`  
15 - PageSize int `json:"pageSize"`  
16 - QueryParameters []domain.QueryParameter `json:"queryParameters"` 10 + OriginalTableId string `json:"originalTableId"`
  11 + IsFromOriginalTable bool `json:"isFromOriginalTable"`
  12 + TableFileUrl string `json:"tableFileUrl"`
  13 + TableFileUrlInternal string `json:"tableFileUrlInternal"`
  14 + ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
  15 + PageNumber int `json:"pageNumber"`
  16 + PageSize int `json:"pageSize"`
  17 + QueryParameters []domain.QueryParameter `json:"queryParameters"`
17 //QueryParameters map[string]interface{} `json:"queryParameters"` 18 //QueryParameters map[string]interface{} `json:"queryParameters"`
18 SortParameters map[string]interface{} `json:"sortParameters"` 19 SortParameters map[string]interface{} `json:"sortParameters"`
19 HeaderRow int `json:"headerRow"` 20 HeaderRow int `json:"headerRow"`
@@ -42,12 +43,13 @@ func NewRequestCheckoutTablesQuery(param domain.ReqLoadDataTable) RequestCheckou @@ -42,12 +43,13 @@ func NewRequestCheckoutTablesQuery(param domain.ReqLoadDataTable) RequestCheckou
42 isSourceFile = true 43 isSourceFile = true
43 } 44 }
44 return RequestCheckoutTablesQuery{ 45 return RequestCheckoutTablesQuery{
45 - OriginalTableId: param.OriginalTableId,  
46 - IsFromOriginalTable: isSourceFile,  
47 - TableFileUrl: tableFileUrl,  
48 - ColumnSchemas: param.ColumnSchemas,  
49 - PageNumber: param.PageNumber,  
50 - PageSize: param.PageSize, 46 + OriginalTableId: param.OriginalTableId,
  47 + IsFromOriginalTable: isSourceFile,
  48 + TableFileUrl: domain.ConvertFileUrlToInternal(tableFileUrl),
  49 + TableFileUrlInternal: tableFileUrl,
  50 + ColumnSchemas: param.ColumnSchemas,
  51 + PageNumber: param.PageNumber,
  52 + PageSize: param.PageSize,
51 //QueryParameters: param.QueryParameters, 53 //QueryParameters: param.QueryParameters,
52 QueryParameters: make([]domain.QueryParameter, 0), 54 QueryParameters: make([]domain.QueryParameter, 0),
53 //QueryParameters: make(map[string]interface{}), 55 //QueryParameters: make(map[string]interface{}),
@@ -133,12 +135,13 @@ func ToDataLoadDataTable(data DataCheckoutTables) *domain.DataLoadDataTable { @@ -133,12 +135,13 @@ func ToDataLoadDataTable(data DataCheckoutTables) *domain.DataLoadDataTable {
133 135
134 type ( 136 type (
135 RequestCheckoutTablesGenerateMasterTable struct { 137 RequestCheckoutTablesGenerateMasterTable struct {
136 - OriginalTableId string `json:"originalTableId"`  
137 - CheckoutTableFileUrl string `json:"checkoutTableFileUrl"`  
138 - ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`  
139 - MasterTableName string `json:"masterTableName"`  
140 - FieldSchemas []FieldSchema `json:"fieldSchemas"`  
141 - KeyFieldEnNames []string `json:"keyFieldEnNames"` 138 + OriginalTableId string `json:"originalTableId"`
  139 + CheckoutTableFileUrl string `json:"checkoutTableFileUrl"`
  140 + CheckoutTableFileUrlInternal string `json:"checkoutTableFileUrlInternal"`
  141 + ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
  142 + MasterTableName string `json:"masterTableName"`
  143 + FieldSchemas []FieldSchema `json:"fieldSchemas"`
  144 + KeyFieldEnNames []string `json:"keyFieldEnNames"`
142 } 145 }
143 DataCheckoutTablesGenerateMasterTable struct { 146 DataCheckoutTablesGenerateMasterTable struct {
144 MasterTableName string `json:"masterTableName"` 147 MasterTableName string `json:"masterTableName"`
@@ -154,12 +157,13 @@ type ( @@ -154,12 +157,13 @@ type (
154 157
155 func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable) RequestCheckoutTablesGenerateMasterTable { 158 func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable) RequestCheckoutTablesGenerateMasterTable {
156 request := RequestCheckoutTablesGenerateMasterTable{ 159 request := RequestCheckoutTablesGenerateMasterTable{
157 - OriginalTableId: fmt.Sprintf("%v", param.FileId),  
158 - CheckoutTableFileUrl: param.FileUrl,  
159 - ColumnSchemas: DomainFieldsToColumnSchemas(param.Table.DataFields),  
160 - MasterTableName: param.Table.SQLName,  
161 - FieldSchemas: ToFieldSchemas(param.Table.DataFields),  
162 - KeyFieldEnNames: []string{param.Table.PK.SQLName}, 160 + OriginalTableId: fmt.Sprintf("%v", param.FileId),
  161 + CheckoutTableFileUrl: domain.ConvertFileUrlToInternal(param.FileUrl),
  162 + CheckoutTableFileUrlInternal: param.FileUrl,
  163 + ColumnSchemas: DomainFieldsToColumnSchemas(param.Table.DataFields),
  164 + MasterTableName: param.Table.SQLName,
  165 + FieldSchemas: ToFieldSchemas(param.Table.DataFields),
  166 + KeyFieldEnNames: []string{param.Table.PK.SQLName},
163 } 167 }
164 return request 168 return request
165 } 169 }
@@ -167,12 +171,13 @@ func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable) @@ -167,12 +171,13 @@ func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable)
167 type ( 171 type (
168 TableAppendRequest struct { 172 TableAppendRequest struct {
169 //MasterTableId string `json:"masterTableId"` 173 //MasterTableId string `json:"masterTableId"`
170 - OriginalTableId string `json:"originalTableId"`  
171 - CheckoutTableFileUrl string `json:"checkoutTableFileUrl"`  
172 - DatabaseTableName string `json:"databaseTableName"`  
173 - ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`  
174 - FieldSchemas []FieldSchema `json:"fieldSchemas"`  
175 - SchemaMap map[string]domain.ColumnSchema `json:"schemaMap"` 174 + OriginalTableId string `json:"originalTableId"`
  175 + CheckoutTableFileUrl string `json:"checkoutTableFileUrl"`
  176 + CheckoutTableFileUrlInternal string `json:"checkoutTableFileUrlInternal"`
  177 + DatabaseTableName string `json:"databaseTableName"`
  178 + ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
  179 + FieldSchemas []FieldSchema `json:"fieldSchemas"`
  180 + SchemaMap map[string]domain.ColumnSchema `json:"schemaMap"`
176 } 181 }
177 182
178 MasterTablesAppendRequest struct { 183 MasterTablesAppendRequest struct {
@@ -192,12 +197,13 @@ type ( @@ -192,12 +197,13 @@ type (
192 func NewTableAppendRequest(param domain.ReqAppendData) TableAppendRequest { 197 func NewTableAppendRequest(param domain.ReqAppendData) TableAppendRequest {
193 columnSchemas := DomainFieldsToColumnSchemas(param.From) 198 columnSchemas := DomainFieldsToColumnSchemas(param.From)
194 req := TableAppendRequest{ 199 req := TableAppendRequest{
195 - OriginalTableId: intToString(param.FileId),  
196 - CheckoutTableFileUrl: param.FileUrl,  
197 - DatabaseTableName: param.Table.SQLName,  
198 - ColumnSchemas: DomainFieldsToColumnSchemas(param.ExcelTable.DataFields), //这里主要需要传递原文件所有字段 param.From  
199 - FieldSchemas: ToFieldSchemas(param.Table.DataFields),  
200 - SchemaMap: make(map[string]domain.ColumnSchema), 200 + OriginalTableId: intToString(param.FileId),
  201 + CheckoutTableFileUrl: domain.ConvertFileUrlToInternal(param.FileUrl),
  202 + CheckoutTableFileUrlInternal: param.FileUrl,
  203 + DatabaseTableName: param.Table.SQLName,
  204 + ColumnSchemas: DomainFieldsToColumnSchemas(param.ExcelTable.DataFields), //这里主要需要传递原文件所有字段 param.From
  205 + FieldSchemas: ToFieldSchemas(param.Table.DataFields),
  206 + SchemaMap: make(map[string]domain.ColumnSchema),
201 } 207 }
202 for i := 0; i < len(param.To); i++ { 208 for i := 0; i < len(param.To); i++ {
203 if len(columnSchemas) > i { 209 if len(columnSchemas) > i {
@@ -41,7 +41,7 @@ func (ptr *FlushDataTableService) Flush(ctx *domain.Context, fileId int, table * @@ -41,7 +41,7 @@ func (ptr *FlushDataTableService) Flush(ctx *domain.Context, fileId int, table *
41 // 临时文件 -》校验文件 41 // 临时文件 -》校验文件
42 var newUrl string 42 var newUrl string
43 if response != nil { 43 if response != nil {
44 - newUrl = response.Url 44 + newUrl = domain.ConvertInternalFileUrlToPublic(response.Url)
45 } 45 }
46 log.Logger.Info("更新文件地址", map[string]interface{}{"from_url": file.FileInfo.Url, "to_url": newUrl, "sourceFileId": file.SourceFileId}) 46 log.Logger.Info("更新文件地址", map[string]interface{}{"from_url": file.FileInfo.Url, "to_url": newUrl, "sourceFileId": file.SourceFileId})
47 switch sourceFile.FileType { 47 switch sourceFile.FileType {
  1 +package utils
  2 +
  3 +import (
  4 + "fmt"
  5 + "github.com/aliyun/aliyun-oss-go-sdk/oss"
  6 + "github.com/google/uuid"
  7 + "io"
  8 + "path"
  9 + "time"
  10 +)
  11 +
  12 +type RouterConfig struct {
  13 + OssEndPoint string
  14 + AccessKeyID string
  15 + AccessKeySecret string
  16 + BuckName string
  17 + AppProject string
  18 +
  19 + RegionID string
  20 + RoleArn string
  21 +}
  22 +
  23 +type (
  24 + CreateStsAuthRequest struct {
  25 + Files []string `json:"files"`
  26 + }
  27 + CreateStsAuthResponse struct {
  28 + Certificate interface{} `json:"certificate"`
  29 + Files []BuckObject `json:"files"`
  30 + }
  31 +
  32 + Object struct {
  33 + Key string
  34 + Value io.Reader
  35 + }
  36 + BuckObject struct {
  37 + DefaultHost string `json:"host"`
  38 + Key string `json:"key"`
  39 + Path string `json:"path"`
  40 + FileName string `json:"fileName"`
  41 + }
  42 +)
  43 +
  44 +func NewBucket(config RouterConfig) (*oss.Bucket, error) {
  45 + client, err := oss.New(config.OssEndPoint, config.AccessKeyID, config.AccessKeySecret)
  46 + if err != nil {
  47 + return nil, err
  48 + }
  49 + bucket, err := client.Bucket(config.BuckName)
  50 + if err != nil {
  51 + return nil, err
  52 + }
  53 +
  54 + return bucket, nil
  55 +}
  56 +
  57 +func CreateObjects(bucket *oss.Bucket, objects ...Object) error {
  58 + for _, object := range objects {
  59 + err := bucket.PutObject(object.Key, object.Value)
  60 + if err != nil {
  61 + return err
  62 + }
  63 + }
  64 + return nil
  65 +}
  66 +
  67 +func DeleteObjects(bucket *oss.Bucket, objects ...string) error {
  68 + for _, object := range objects {
  69 + err := bucket.DeleteObject(object)
  70 + if err != nil {
  71 + return err
  72 + }
  73 + }
  74 + return nil
  75 +}
  76 +
  77 +func GetFileName(projectName, filename string) string {
  78 + date := time.Now().Format("20060102")
  79 + ext := path.Ext(filename)
  80 + if len(projectName) == 0 {
  81 + projectName = "default"
  82 + }
  83 + uid, _ := uuid.NewUUID()
  84 + filename = fmt.Sprintf("%v%v", uid.String(), ext)
  85 + sourcePath := fmt.Sprintf("%v/%v/%v", projectName, date, filename)
  86 + return sourcePath
  87 +}