feat:big_excel_support

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/ajg/form v1.5.1 // indirect
+	github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible // indirect
 	github.com/beego/beego/v2 v2.0.1
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible

@@ -5,6 +5,9 @@ import (
 	"fmt"
 	"github.com/beego/beego/v2/client/httplib"
 	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/utils"
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
+	"os"
 	"time"
 
 	"github.com/linmadan/egglib-go/core/application"
@@ -315,7 +318,7 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
 		return nil, factory.FastError(err)
 	}
 
-	f, err := httplib.Get(file.FileInfo.Url).Bytes()
+	f, err := httplib.Get(domain.ConvertFileUrlToInternal(file.FileInfo.Url)).Bytes()
 	if err != nil {
 		return nil, factory.FastError(err)
 	}
@@ -333,7 +336,31 @@ func (fileService *FileService) ExportFile(ctx *domain.Context, cmd *command.Exp
 		return nil, factory.FastError(err)
 	}
 
+	var (
+		config = utils.RouterConfig{
+			OssEndPoint:     "oss-cn-hangzhou.aliyuncs-internal.com",
+			AccessKeyID:     "LTAI4Fz1LUBW2fXp6QWaJHRS",
+			AccessKeySecret: "aLZXwK8pgrs10Ws03qcN7NsrSXFVsg",
+			BuckName:        "byte-bank",
+		}
+		key = fmt.Sprintf("byte-bank/%v/%v", time.Now().Format("2006-01-02"), filename)
+	)
+	bucket, bucketErr := utils.NewBucket(config)
+	if bucketErr == nil && bucket != nil {
+		log.Logger.Info(fmt.Sprintf("end-point:%v key:%v", config.OssEndPoint, key))
+		f, _ := os.Open(path)
+		if err = utils.CreateObjects(bucket, utils.Object{
+			Key:   key,
+			Value: f,
+		}); err != nil {
+			log.Logger.Error(err.Error())
+		} else {
+			response.Url = domain.ConvertInternalFileUrlToPublic(fmt.Sprintf("https://%v.%v/%v", config.BuckName, config.OssEndPoint, key))
+		}
+	}
+	if len(response.Url) == 0 {
 		response.Url = domain.DownloadUrl(filename)
+	}
 	response.FileName = file.FileInfo.Name
 	response.Ext = domain.XLSX
 

@@ -17,6 +17,7 @@ type TablePreviewCommand struct {
 	PageSize   int          `json:"pageSize"`
 	Where      domain.Where `json:"where"`
 	UseCache   bool         `json:"useCache"`
+	HiddenData bool         `json:"hiddenData"` // 隐藏数据，只返回结构
 }
 
 func (cmd *TablePreviewCommand) Valid(validation *validation.Validation) {

@@ -16,6 +16,7 @@ type TablePreviewDto struct {
 	Fields []*domain.Field `json:"fields"`
 	Data   interface{}     `json:"grid"`
 	//Total  int64               `json:"total"`
+	HiddenData bool `json:"-"`
 }
 
 func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, objectType string) *TablePreviewDto {
@@ -26,6 +27,12 @@ func (d *TablePreviewDto) Load(m *domain.Table, dataTable *domain.DataTable, obj
 	d.Fields = dataTable.MatchFields(m.Fields(true))
 	d.Data = domain.GripData(domain.ToFieldData(m.Fields(true), dataTable.Data, false), dataTable.Total)
 	//d.Total = dataTable.Total
+	if d.HiddenData {
+		d.Data = map[string]interface{}{
+			"list":  make([]map[string]string, 0),
+			"total": 0,
+		}
+	}
 	return d
 }
 

@@ -42,7 +42,7 @@ func (tableService *TableService) TablePreview(ctx *domain.Context, cmd *command
 			cacheMiss = true
 		}
 	}
-	response := &dto.TablePreviewDto{}
+	response := &dto.TablePreviewDto{HiddenData: cmd.HiddenData}
 	if dataTable == nil {
 		switch table.TableType {
 		case domain.CalculateSet.ToString():

 package domain
 
+import (
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
+	"strings"
+)
+
 // FileInfo 文件信息
 type FileInfo struct {
 	// 名称
@@ -17,3 +22,31 @@ type FileInfo struct {
 	// 行号
 	HeaderRow int `json:"headerRow"`
 }
+
+// ConvertFileUrlToInternal 传递内网地址
+// byte-bank.oss-cn-hangzhou 当oss与字库同属于这个节点下面，使用内网进行传输
+func ConvertFileUrlToInternal(fileUrl string) string {
+	if constant.SERVICE_ENV != "prod" {
+		return fileUrl
+	}
+	var bucketRegion = "https://byte-bank.oss-cn-hangzhou"
+	var bucketInternalRegion = "https://byte-bank.oss-cn-hangzhou-internal"
+	if strings.HasPrefix(fileUrl, bucketRegion) {
+		return strings.Replace(fileUrl, bucketRegion, bucketInternalRegion, 1)
+	}
+	return fileUrl
+}
+
+// ConvertInternalFileUrlToPublic 转内网地址为外网地址
+func ConvertInternalFileUrlToPublic(fileUrl string) string {
+	//var bucketRegion = "https://byte-bank.oss-cn-hangzhou"
+	//var bucketInternalRegion = "https://byte-bank.oss-cn-hangzhou-internal"
+	//if strings.HasPrefix(fileUrl, bucketInternalRegion) {
+	//	return strings.Replace(fileUrl, bucketInternalRegion, bucketRegion, 1)
+	//}
+	var internal = "-internal"
+	if strings.Contains(fileUrl, internal) {
+		return strings.Replace(fileUrl, internal, "", 1)
+	}
+	return fileUrl
+}

 package bytelib
 
-import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
+import (
+	"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
+)
 
 func DomainFieldsToColumnSchemas(fields []*domain.Field) []domain.ColumnSchema {
 	result := make([]domain.ColumnSchema, 0)

@@ -10,6 +10,7 @@ type RequestCheckoutTablesQuery struct {
 	OriginalTableId      string                  `json:"originalTableId"`
 	IsFromOriginalTable  bool                    `json:"isFromOriginalTable"`
 	TableFileUrl         string                  `json:"tableFileUrl"`
+	TableFileUrlInternal string                  `json:"tableFileUrlInternal"`
 	ColumnSchemas        []domain.ColumnSchema   `json:"columnSchemas"`
 	PageNumber           int                     `json:"pageNumber"`
 	PageSize             int                     `json:"pageSize"`
@@ -44,7 +45,8 @@ func NewRequestCheckoutTablesQuery(param domain.ReqLoadDataTable) RequestCheckou
 	return RequestCheckoutTablesQuery{
 		OriginalTableId:      param.OriginalTableId,
 		IsFromOriginalTable:  isSourceFile,
-		TableFileUrl:        tableFileUrl,
+		TableFileUrl:         domain.ConvertFileUrlToInternal(tableFileUrl),
+		TableFileUrlInternal: tableFileUrl,
 		ColumnSchemas:        param.ColumnSchemas,
 		PageNumber:           param.PageNumber,
 		PageSize:             param.PageSize,
@@ -135,6 +137,7 @@ type (
 	RequestCheckoutTablesGenerateMasterTable struct {
 		OriginalTableId              string                `json:"originalTableId"`
 		CheckoutTableFileUrl         string                `json:"checkoutTableFileUrl"`
+		CheckoutTableFileUrlInternal string                `json:"checkoutTableFileUrlInternal"`
 		ColumnSchemas                []domain.ColumnSchema `json:"columnSchemas"`
 		MasterTableName              string                `json:"masterTableName"`
 		FieldSchemas                 []FieldSchema         `json:"fieldSchemas"`
@@ -155,7 +158,8 @@ type (
 func NewRequestCheckoutTablesGenerateMasterTable(param domain.ReqGenerateTable) RequestCheckoutTablesGenerateMasterTable {
 	request := RequestCheckoutTablesGenerateMasterTable{
 		OriginalTableId:              fmt.Sprintf("%v", param.FileId),
-		CheckoutTableFileUrl: param.FileUrl,
+		CheckoutTableFileUrl:         domain.ConvertFileUrlToInternal(param.FileUrl),
+		CheckoutTableFileUrlInternal: param.FileUrl,
 		ColumnSchemas:                DomainFieldsToColumnSchemas(param.Table.DataFields),
 		MasterTableName:              param.Table.SQLName,
 		FieldSchemas:                 ToFieldSchemas(param.Table.DataFields),
@@ -169,6 +173,7 @@ type (
 		//MasterTableId        string                `json:"masterTableId"`
 		OriginalTableId              string                         `json:"originalTableId"`
 		CheckoutTableFileUrl         string                         `json:"checkoutTableFileUrl"`
+		CheckoutTableFileUrlInternal string                         `json:"checkoutTableFileUrlInternal"`
 		DatabaseTableName            string                         `json:"databaseTableName"`
 		ColumnSchemas                []domain.ColumnSchema          `json:"columnSchemas"`
 		FieldSchemas                 []FieldSchema                  `json:"fieldSchemas"`
@@ -193,7 +198,8 @@ func NewTableAppendRequest(param domain.ReqAppendData) TableAppendRequest {
 	columnSchemas := DomainFieldsToColumnSchemas(param.From)
 	req := TableAppendRequest{
 		OriginalTableId:              intToString(param.FileId),
-		CheckoutTableFileUrl: param.FileUrl,
+		CheckoutTableFileUrl:         domain.ConvertFileUrlToInternal(param.FileUrl),
+		CheckoutTableFileUrlInternal: param.FileUrl,
 		DatabaseTableName:            param.Table.SQLName,
 		ColumnSchemas:                DomainFieldsToColumnSchemas(param.ExcelTable.DataFields), //这里主要需要传递原文件所有字段 param.From
 		FieldSchemas:                 ToFieldSchemas(param.Table.DataFields),

@@ -41,7 +41,7 @@ func (ptr *FlushDataTableService) Flush(ctx *domain.Context, fileId int, table *
 	// 临时文件 -》校验文件
 	var newUrl string
 	if response != nil {
-		newUrl = response.Url
+		newUrl = domain.ConvertInternalFileUrlToPublic(response.Url)
 	}
 	log.Logger.Info("更新文件地址", map[string]interface{}{"from_url": file.FileInfo.Url, "to_url": newUrl, "sourceFileId": file.SourceFileId})
 	switch sourceFile.FileType {

+package utils
+
+import (
+	"fmt"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/google/uuid"
+	"io"
+	"path"
+	"time"
+)
+
+type RouterConfig struct {
+	OssEndPoint     string
+	AccessKeyID     string
+	AccessKeySecret string
+	BuckName        string
+	AppProject      string
+
+	RegionID string
+	RoleArn  string
+}
+
+type (
+	CreateStsAuthRequest struct {
+		Files []string `json:"files"`
+	}
+	CreateStsAuthResponse struct {
+		Certificate interface{}  `json:"certificate"`
+		Files       []BuckObject `json:"files"`
+	}
+
+	Object struct {
+		Key   string
+		Value io.Reader
+	}
+	BuckObject struct {
+		DefaultHost string `json:"host"`
+		Key         string `json:"key"`
+		Path        string `json:"path"`
+		FileName    string `json:"fileName"`
+	}
+)
+
+func NewBucket(config RouterConfig) (*oss.Bucket, error) {
+	client, err := oss.New(config.OssEndPoint, config.AccessKeyID, config.AccessKeySecret)
+	if err != nil {
+		return nil, err
+	}
+	bucket, err := client.Bucket(config.BuckName)
+	if err != nil {
+		return nil, err
+	}
+
+	return bucket, nil
+}
+
+func CreateObjects(bucket *oss.Bucket, objects ...Object) error {
+	for _, object := range objects {
+		err := bucket.PutObject(object.Key, object.Value)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func DeleteObjects(bucket *oss.Bucket, objects ...string) error {
+	for _, object := range objects {
+		err := bucket.DeleteObject(object)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func GetFileName(projectName, filename string) string {
+	date := time.Now().Format("20060102")
+	ext := path.Ext(filename)
+	if len(projectName) == 0 {
+		projectName = "default"
+	}
+	uid, _ := uuid.NewUUID()
+	filename = fmt.Sprintf("%v%v", uid.String(), ext)
+	sourcePath := fmt.Sprintf("%v/%v/%v", projectName, date, filename)
+	return sourcePath
+}