feat:big_excel_support

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/ajg/form v1.5.1 // indirect
+	github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible // indirect
 	github.com/beego/beego/v2 v2.0.1
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible

@@ -5,6 +5,9 @@ import (
 	"fmt"
 	"github.com/beego/beego/v2/client/httplib"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
+	"os"
 	"time"
 
 	"github.com/linmadan/egglib-go/core/application"
@@ -315,7 +318,7 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
 		return nil, factory.FastError(err)
 	}
 
-	f, err := httplib.Get(file.FileInfo.Url).Bytes()
+	f, err := httplib.Get(domain.ConvertFileUrlToInternal(file.FileInfo.Url)).Bytes()
 	if err != nil {
 		return nil, factory.FastError(err)
 	}
@@ -333,7 +336,31 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
 		return nil, factory.FastError(err)
 	}
 
-	response.Url = domain.DownloadUrl(filename)
+	var (
+		config = utils.RouterConfig{
+			OssEndPoint:     "oss-cn-hangzhou.aliyuncs-internal.com",
+			AccessKeyID:     "LTAI4Fz1LUBW2fXp6QWaJHRS",
+			AccessKeySecret: "aLZXwK8pgrs10Ws03qcN7NsrSXFVsg",
+			BuckName:        "byte-bank",
+		}
+		key = fmt.Sprintf("byte-bank/%v/%v", time.Now().Format("2006-01-02"), filename)
+	)
+	bucket, bucketErr := utils.NewBucket(config)
+	if bucketErr == nil && bucket != nil {
+		log.Logger.Info(fmt.Sprintf("end-point:%v key:%v", config.OssEndPoint, key))
+		f, _ := os.Open(path)
+		if err = utils.CreateObjects(bucket, utils.Object{
+			Key:   key,
+			Value: f,
+		}); err != nil {
+			log.Logger.Error(err.Error())
+		} else {
+			response.Url = domain.ConvertInternalFileUrlToPublic(fmt.Sprintf("https://%v.%v/%v", config.BuckName, config.OssEndPoint, key))
+		}
+	}
+	if len(response.Url) == 0 {
+		response.Url = domain.DownloadUrl(filename)
+	}
 	response.FileName = file.FileInfo.Name
 	response.Ext = domain.XLSX
 

@@ -17,6 +17,7 @@ type TablePreviewCommand struct {
 	PageSize   int          `json:"pageSize"`
 	Where      domain.Where `json:"where"`
 	UseCache   bool         `json:"useCache"`
+	HiddenData bool         `json:"hiddenData"` // 隐藏数据，只返回结构
 }
 
 func (cmd *TablePreviewCommand) Valid(validation *validation.Validation) {

@@ -16,6 +16,7 @@ type TablePreviewDto struct {
 	Fields []*domain.Field `json:"fields"`
 	Data   interface{}     `json:"grid"`
 	//Total  int64               `json:"total"`
+	HiddenData bool `json:"-"`
 }
 
 func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, objectType string) *TablePreviewDto {
@@ -26,6 +27,12 @@ func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, obj
 	d.Fields = dataTable.MatchFields(m.Fields(true))
 	d.Data = domain.GripData(domain.ToFieldData(m.Fields(true), dataTable.Data, false), dataTable.Total)
 	//d.Total = dataTable.Total
+	if d.HiddenData {
+		d.Data = map[string]interface{}{
+			"list":  make([]map[string]string, 0),
+			"total": 0,
+		}
+	}
 	return d
 }
 

@@ -42,7 +42,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
 			cacheMiss = true
 		}
 	}
-	response := &dto.TablePreviewDto{}
+	response := &dto.TablePreviewDto{HiddenData: cmd.HiddenData}
 	if dataTable == nil {
 		switch table.TableType {
 		case domain.CalculateSet.ToString():

 package domain
 
+import (
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
+	"strings"
+)
+
 // FileInfo 文件信息
 type FileInfo struct {
 	// 名称
@@ -17,3 +22,31 @@ type FileInfo struct {
 	// 行号
 	HeaderRow int `json:"headerRow"`
 }
+
+// ConvertFileUrlToInternal 传递内网地址
+// byte-bank.oss-cn-hangzhou 当oss与字库同属于这个节点下面，使用内网进行传输
+func ConvertFileUrlToInternal(fileUrl string) string {
+	if constant.SERVICE_ENV != "prod" {
+		return fileUrl
+	}
+	var bucketRegion = "https://byte-bank.oss-cn-hangzhou"
+	var bucketInternalRegion = "https://byte-bank.oss-cn-hangzhou-internal"
+	if strings.HasPrefix(fileUrl, bucketRegion) {
+		return strings.Replace(fileUrl, bucketRegion, bucketInternalRegion, 1)
+	}
+	return fileUrl
+}
+
+// ConvertInternalFileUrlToPublic 转内网地址为外网地址
+func ConvertInternalFileUrlToPublic(fileUrl string) string {
+	//var bucketRegion = "https://byte-bank.oss-cn-hangzhou"
+	//var bucketInternalRegion = "https://byte-bank.oss-cn-hangzhou-internal"
+	//if strings.HasPrefix(fileUrl, bucketInternalRegion) {
+	//	return strings.Replace(fileUrl, bucketInternalRegion, bucketRegion, 1)
+	//}
+	var internal = "-internal"
+	if strings.Contains(fileUrl, internal) {
+		return strings.Replace(fileUrl, internal, "", 1)
+	}
+	return fileUrl
+}

 package bytelib
 
-import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
+import (
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
+)
 
 func DomainFieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema {
 	result := make([]domain.ColumnSchema, 0)

@@ -7,13 +7,14 @@ import (
 )
 
 type RequestCheckoutTablesQuery struct {
-	OriginalTableId     string                  `json:"originalTableId"`
-	IsFromOriginalTable bool                    `json:"isFromOriginalTable"`
-	TableFileUrl        string                  `json:"tableFileUrl"`
-	ColumnSchemas       []domain.ColumnSchema   `json:"columnSchemas"`
-	PageNumber          int                     `json:"pageNumber"`
-	PageSize            int                     `json:"pageSize"`
-	QueryParameters     []domain.QueryParameter `json:"queryParameters"`
+	OriginalTableId      string                  `json:"originalTableId"`
+	IsFromOriginalTable  bool                    `json:"isFromOriginalTable"`
+	TableFileUrl         string                  `json:"tableFileUrl"`
+	TableFileUrlInternal string                  `json:"tableFileUrlInternal"`
+	ColumnSchemas        []domain.ColumnSchema   `json:"columnSchemas"`
+	PageNumber           int                     `json:"pageNumber"`
+	PageSize             int                     `json:"pageSize"`
+	QueryParameters      []domain.QueryParameter `json:"queryParameters"`
 	//QueryParameters map[string]interface{} `json:"queryParameters"`
 	SortParameters map[string]interface{} `json:"sortParameters"`
 	HeaderRow      int                    `json:"headerRow"`
@@ -42,12 +43,13 @@ func NewRequestCheckoutTablesQuery(param domain.ReqLoadDataTable) RequestCheckou
 		isSourceFile = true
 	}
 	return RequestCheckoutTablesQuery{
-		OriginalTableId:     param.OriginalTableId,
-		IsFromOriginalTable: isSourceFile,
-		TableFileUrl:        tableFileUrl,
-		ColumnSchemas:       param.ColumnSchemas,
-		PageNumber:          param.PageNumber,
-		PageSize:            param.PageSize,
+		OriginalTableId:      param.OriginalTableId,
+		IsFromOriginalTable:  isSourceFile,
+		TableFileUrl:         domain.ConvertFileUrlToInternal(tableFileUrl),
+		TableFileUrlInternal: tableFileUrl,
+		ColumnSchemas:        param.ColumnSchemas,
+		PageNumber:           param.PageNumber,
+		PageSize:             param.PageSize,
 		//QueryParameters:     param.QueryParameters,
 		QueryParameters: make([]domain.QueryParameter, 0),
 		//QueryParameters: make(map[string]interface{}),
@@ -133,12 +135,13 @@ func ToDataLoadDataTable(data DataCheckoutTables) *domain.DataLoadDataTable {
 
 type (
 	RequestCheckoutTablesGenerateMasterTable struct {
-		OriginalTableId      string                `json:"originalTableId"`
-		CheckoutTableFileUrl string                `json:"checkoutTableFileUrl"`
-		ColumnSchemas        []domain.ColumnSchema `json:"columnSchemas"`
-		MasterTableName      string                `json:"masterTableName"`
-		FieldSchemas         []FieldSchema         `json:"fieldSchemas"`
-		KeyFieldEnNames      []string              `json:"keyFieldEnNames"`
+		OriginalTableId              string                `json:"originalTableId"`
+		CheckoutTableFileUrl         string                `json:"checkoutTableFileUrl"`
+		CheckoutTableFileUrlInternal string                `json:"checkoutTableFileUrlInternal"`
+		ColumnSchemas                []domain.ColumnSchema `json:"columnSchemas"`
+		MasterTableName              string                `json:"masterTableName"`
+		FieldSchemas                 []FieldSchema         `json:"fieldSchemas"`
+		KeyFieldEnNames              []string              `json:"keyFieldEnNames"`
 	}
 	DataCheckoutTablesGenerateMasterTable struct {
 		MasterTableName string `json:"masterTableName"`
@@ -154,12 +157,13 @@ type (
 
 func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable) RequestCheckoutTablesGenerateMasterTable {
 	request := RequestCheckoutTablesGenerateMasterTable{
-		OriginalTableId:      fmt.Sprintf("%v", param.FileId),
-		CheckoutTableFileUrl: param.FileUrl,
-		ColumnSchemas:        DomainFieldsToColumnSchemas(param.Table.DataFields),
-		MasterTableName:      param.Table.SQLName,
-		FieldSchemas:         ToFieldSchemas(param.Table.DataFields),
-		KeyFieldEnNames:      []string{param.Table.PK.SQLName},
+		OriginalTableId:              fmt.Sprintf("%v", param.FileId),
+		CheckoutTableFileUrl:         domain.ConvertFileUrlToInternal(param.FileUrl),
+		CheckoutTableFileUrlInternal: param.FileUrl,
+		ColumnSchemas:                DomainFieldsToColumnSchemas(param.Table.DataFields),
+		MasterTableName:              param.Table.SQLName,
+		FieldSchemas:                 ToFieldSchemas(param.Table.DataFields),
+		KeyFieldEnNames:              []string{param.Table.PK.SQLName},
 	}
 	return request
 }
@@ -167,12 +171,13 @@ func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable)
 type (
 	TableAppendRequest struct {
 		//MasterTableId        string                `json:"masterTableId"`
-		OriginalTableId      string                         `json:"originalTableId"`
-		CheckoutTableFileUrl string                         `json:"checkoutTableFileUrl"`
-		DatabaseTableName    string                         `json:"databaseTableName"`
-		ColumnSchemas        []domain.ColumnSchema          `json:"columnSchemas"`
-		FieldSchemas         []FieldSchema                  `json:"fieldSchemas"`
-		SchemaMap            map[string]domain.ColumnSchema `json:"schemaMap"`
+		OriginalTableId              string                         `json:"originalTableId"`
+		CheckoutTableFileUrl         string                         `json:"checkoutTableFileUrl"`
+		CheckoutTableFileUrlInternal string                         `json:"checkoutTableFileUrlInternal"`
+		DatabaseTableName            string                         `json:"databaseTableName"`
+		ColumnSchemas                []domain.ColumnSchema          `json:"columnSchemas"`
+		FieldSchemas                 []FieldSchema                  `json:"fieldSchemas"`
+		SchemaMap                    map[string]domain.ColumnSchema `json:"schemaMap"`
 	}
 
 	MasterTablesAppendRequest struct {
@@ -192,12 +197,13 @@ type (
 func NewTableAppendRequest(param domain.ReqAppendData) TableAppendRequest {
 	columnSchemas := DomainFieldsToColumnSchemas(param.From)
 	req := TableAppendRequest{
-		OriginalTableId:      intToString(param.FileId),
-		CheckoutTableFileUrl: param.FileUrl,
-		DatabaseTableName:    param.Table.SQLName,
-		ColumnSchemas:        DomainFieldsToColumnSchemas(param.ExcelTable.DataFields), //这里主要需要传递原文件所有字段 param.From
-		FieldSchemas:         ToFieldSchemas(param.Table.DataFields),
-		SchemaMap:            make(map[string]domain.ColumnSchema),
+		OriginalTableId:              intToString(param.FileId),
+		CheckoutTableFileUrl:         domain.ConvertFileUrlToInternal(param.FileUrl),
+		CheckoutTableFileUrlInternal: param.FileUrl,
+		DatabaseTableName:            param.Table.SQLName,
+		ColumnSchemas:                DomainFieldsToColumnSchemas(param.ExcelTable.DataFields), //这里主要需要传递原文件所有字段 param.From
+		FieldSchemas:                 ToFieldSchemas(param.Table.DataFields),
+		SchemaMap:                    make(map[string]domain.ColumnSchema),
 	}
 	for i := 0; i < len(param.To); i++ {
 		if len(columnSchemas) > i {

@@ -41,7 +41,7 @@ func (ptr *FlushDataTableService) Flush(ctx *domain.Context, fileId int, table *
 	// 临时文件 -》校验文件
 	var newUrl string
 	if response != nil {
-		newUrl = response.Url
+		newUrl = domain.ConvertInternalFileUrlToPublic(response.Url)
 	}
 	log.Logger.Info("更新文件地址", map[string]interface{}{"from_url": file.FileInfo.Url, "to_url": newUrl, "sourceFileId": file.SourceFileId})
 	switch sourceFile.FileType {

+package utils
+
+import (
+	"fmt"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/google/uuid"
+	"io"
+	"path"
+	"time"
+)
+
+type RouterConfig struct {
+	OssEndPoint     string
+	AccessKeyID     string
+	AccessKeySecret string
+	BuckName        string
+	AppProject      string
+
+	RegionID string
+	RoleArn  string
+}
+
+type (
+	CreateStsAuthRequest struct {
+		Files []string `json:"files"`
+	}
+	CreateStsAuthResponse struct {
+		Certificate interface{}  `json:"certificate"`
+		Files       []BuckObject `json:"files"`
+	}
+
+	Object struct {
+		Key   string
+		Value io.Reader
+	}
+	BuckObject struct {
+		DefaultHost string `json:"host"`
+		Key         string `json:"key"`
+		Path        string `json:"path"`
+		FileName    string `json:"fileName"`
+	}
+)
+
+func NewBucket(config RouterConfig) (*oss.Bucket, error) {
+	client, err := oss.New(config.OssEndPoint, config.AccessKeyID, config.AccessKeySecret)
+	if err != nil {
+		return nil, err
+	}
+	bucket, err := client.Bucket(config.BuckName)
+	if err != nil {
+		return nil, err
+	}
+
+	return bucket, nil
+}
+
+func CreateObjects(bucket *oss.Bucket, objects ...Object) error {
+	for _, object := range objects {
+		err := bucket.PutObject(object.Key, object.Value)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func DeleteObjects(bucket *oss.Bucket, objects ...string) error {
+	for _, object := range objects {
+		err := bucket.DeleteObject(object)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func GetFileName(projectName, filename string) string {
+	date := time.Now().Format("20060102")
+	ext := path.Ext(filename)
+	if len(projectName) == 0 {
+		projectName = "default"
+	}
+	uid, _ := uuid.NewUUID()
+	filename = fmt.Sprintf("%v%v", uid.String(), ext)
+	sourcePath := fmt.Sprintf("%v/%v/%v", projectName, date, filename)
+	return sourcePath
+}