作者 yangfu

feat: calculate set

package service
import (
"fmt"
"github.com/linmadan/egglib-go/core/application"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/querySet/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/dto"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/excel"
"time"
)
func (querySetService *QuerySetService) CalculateSetPreview(ctx *domain.Context, updateQuerySetCommand *command.UpdateQuerySetCommand) (interface{}, error) {
if err := updateQuerySetCommand.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
var querySet *domain.QuerySet
_, querySet, err = factory.FastPgQuerySet(transactionContext, updateQuerySetCommand.QuerySetId)
if err != nil {
return nil, factory.FastError(err)
}
svr, _ := factory.FastQuerySetServices(transactionContext)
var dataTable *domain.DataTable
dataTable, err = svr.LoadCalculateSetData(ctx, querySet, updateQuerySetCommand.QueryComponents)
if err != nil {
return nil, factory.FastError(err)
}
response := (&dto.TablePreviewDto{})
response.ObjectType = querySet.Type
response.Name = querySet.Name
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return response, nil
}
func (querySetService *QuerySetService) CalculateSetExport(ctx *domain.Context, updateQuerySetCommand *command.UpdateQuerySetCommand) (interface{}, error) {
if err := updateQuerySetCommand.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
var querySet *domain.QuerySet
_, querySet, err = factory.FastPgQuerySet(transactionContext, updateQuerySetCommand.QuerySetId)
if err != nil {
return nil, factory.FastError(err)
}
svr, _ := factory.FastQuerySetServices(transactionContext)
var dataTable *domain.DataTable
dataTable, err = svr.LoadCalculateSetData(ctx, querySet, updateQuerySetCommand.QueryComponents)
if err != nil {
return nil, factory.FastError(err)
}
var fields []string
//for i := range dataTable.Fields {
// fields = append(fields, dataTable.Fields[i].Name)
//}
filename := fmt.Sprintf("%v_%v.xlsx", querySet.Name, time.Now().Format("060102150405"))
path := fmt.Sprintf("public/%v", filename)
excelWriter := excel.NewXLXSWriterTo(fields, dataTable.Data)
excelWriter.IgnoreTitle = true
if err = excelWriter.Save(path); err != nil {
return nil, factory.FastError(err)
}
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return map[string]interface{}{
"url": domain.DownloadUrl(filename),
}, err
}
... ...
package service
import (
"github.com/linmadan/egglib-go/core/application"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/factory"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/command"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/table/dto"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
)
func (tableService *TableService) CalculateTablePreview(ctx *domain.Context, cmd *command.TablePreviewCommand) (interface{}, error) {
if err := cmd.ValidateCommand(); err != nil {
return nil, application.ThrowError(application.ARG_ERROR, err.Error())
}
transactionContext, err := factory.CreateTransactionContext(nil)
if err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
if err := transactionContext.StartTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
defer func() {
transactionContext.RollbackTransaction()
}()
var table *domain.QuerySet
_, table, err = factory.FastPgQuerySet(transactionContext, cmd.TableId)
if err != nil {
return nil, factory.FastError(err)
}
svr, _ := factory.FastQuerySetServices(transactionContext)
var dataTable *domain.DataTable
dataTable, err = svr.LoadCalculateSetData(ctx, table, table.QueryComponents)
if err != nil {
return nil, factory.FastError(err)
}
response := (&dto.TablePreviewDto{})
response.Fields = dataTable.Fields
response.Data = domain.GripData(domain.ToFieldData(dataTable.Fields, dataTable.Data, false), int64(len(dataTable.Data)))
if err := transactionContext.CommitTransaction(); err != nil {
return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
}
return response, nil
}
... ...
... ... @@ -70,6 +70,29 @@ func (t *DataTable) OptionalValue() []string {
return values
}
func (t *DataTable) Values(f *Field) []string {
var res = make([]string, 0)
index := -1
for i := range t.Fields {
if t.Fields[i].SQLName == f.SQLName {
index = i
break
}
}
if index < 0 {
return res
}
for i := range t.Data {
for j := range t.Data[i] {
if j == index {
res = append(res, t.Data[i][j])
break
}
}
}
return res
}
func (t *DataTable) MatchFields(from []*Field) []*Field {
return from
}
... ...
... ... @@ -112,6 +112,7 @@ var ObjectTypeMap = map[string]string{
SubProcessTable.ToString(): "子过程",
CalculateItem.ToString(): "计算项",
CalculateTable.ToString(): "计算表",
CalculateSet.ToString(): "计算集",
}
var (
... ...
... ... @@ -35,6 +35,7 @@ type QueryComponent struct {
Description string `json:"description"`
Formula *FieldFormulaExpr `json:"formula"`
Aggregation *AggregationRule `json:"aggregation"`
Layout *LayoutRule `json:"layout"`
}
func (qc QueryComponent) AllSelectExpr() []SelectExpr {
... ...
package domain
const (
CellTypeTable = "Table"
CellTypeTableField = "TableField"
CellTypeText = "Text"
CellTypeNull = "Null"
)
const (
DirectionNone = "Null"
DirectionRight = "Right"
DirectionDown = "Down"
)
type LayoutRule struct {
Layout
}
type Layout struct {
Cells [][]*LayoutCell `json:"cells"`
}
type LayoutCell struct {
Type string `json:"type,omitempty"` // Table TableField Text Null
Data *LayoutCellData `json:"data,omitempty"`
Direction string `json:"direction,omitempty"` // 向右:Right 向下:Down
X int `json:"-,omitempty"`
Y int `json:"-,omitempty"`
Length int `json:"-"`
}
type LayoutCellData struct {
//Table *Table `json:"table,omitempty"`
//Field *Field `json:"field,omitempty"`
TableField *TableField `json:"tableField"`
Text string `json:"text,omitempty"`
}
func (l *Layout) LayoutCells() []*LayoutCell {
var cells = make([]*LayoutCell, 0)
for i, rows := range l.Cells {
for j, item := range rows {
if item.Type == CellTypeNull || item.Type == "" {
continue
}
item.X = i
item.Y = j
cells = append(cells, item)
}
}
return cells
}
... ...
... ... @@ -104,6 +104,9 @@ func (ptr *QuerySetService) Update(ctx *domain.Context, querySetId int, queryCom
if qs.Type == domain.CalculateTable.ToString() {
return ptr.UpdateCalculateTable(ctx, qs, queryComponents)
}
if qs.Type == domain.CalculateSet.ToString() {
return ptr.UpdateCalculateSet(ctx, qs, queryComponents)
}
return nil
}
... ... @@ -235,6 +238,31 @@ func (ptr *QuerySetService) UpdateCalculateTable(ctx *domain.Context, qs *domain
return nil
}
func (ptr *QuerySetService) UpdateCalculateSet(ctx *domain.Context, qs *domain.QuerySet, queryComponents []*domain.QueryComponent) error {
//var err error
res, err := ptr.LoadCalculateSetData(ctx, qs, queryComponents)
if err != nil {
return err
}
querySetRepository, _ := repository.NewQuerySetRepository(ptr.transactionContext)
table, err := ptr.CreateOrUpdateCalculateSet(ctx, qs, res, queryComponents)
if err != nil {
return err
}
// 生成日志
//if err = ptr.UpdateQuerySetLog(ctx, qs, queryComponents); err != nil {
// return err
//}
//保存
qs.Update(queryComponents, table.TableId)
_, err = querySetRepository.Save(qs)
if err != nil {
return err
}
return nil
}
func (ptr *QuerySetService) PreviewPrepare(ctx *domain.Context, querySetId int, queryComponents []*domain.QueryComponent) (*domain.Table, error) {
querySetRepository, _ := repository.NewQuerySetRepository(ptr.transactionContext)
querySet, err := querySetRepository.FindOne(map[string]interface{}{"querySetId": querySetId})
... ... @@ -771,6 +799,32 @@ func (ptr *QuerySetService) CreateOrUpdateCalculateTable(ctx *domain.Context, qu
return table, nil
}
// CreateOrUpdateCalculateSet 计算集
func (ptr *QuerySetService) CreateOrUpdateCalculateSet(ctx *domain.Context, querySet *domain.QuerySet, dataTable *domain.DataTable, queryComponents []*domain.QueryComponent) (*domain.Table, error) {
var (
err error
)
dependencyTables := querySet.GetDependencyTables(queryComponents)
tableRepository, _ := repository.NewTableRepository(ptr.transactionContext)
var table *domain.Table = NewCopyTable(domain.TableType(querySet.Type), querySet.Name, dataTable.Fields, 0).WithContext(ctx).WithPrefix(strings.ToLower(querySet.Type))
if querySet.QuerySetInfo.BindTableId > 0 {
table, err = tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableId": querySet.QuerySetInfo.BindTableId})
if err != nil {
return nil, err
}
table.DataFields = dataTable.Fields
table.UpdatedAt = time.Now()
}
table.TableInfo.ApplyOnModule = domain.ModuleAll
table.TableInfo.DependencyTables = dependencyTables
table, err = tableRepository.Save(table)
if err != nil {
return nil, err
}
return table, nil
}
func (ptr *QuerySetService) Rename(ctx *domain.Context, querySetId int, name string) error {
querySetRepository, _ := repository.NewQuerySetRepository(ptr.transactionContext)
qs, err := querySetRepository.FindOne(map[string]interface{}{"querySetId": querySetId})
... ...
package domainService
import (
"fmt"
"github.com/zeromicro/go-zero/core/collection"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
"gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
"strings"
)
const DefaultExpandNum = 10000
const MaxExpandNum = 50000
func (ptr *QuerySetService) LoadCalculateSetData(ctx *domain.Context, qs *domain.QuerySet, queryComponents []*domain.QueryComponent) (*domain.DataTable, error) {
var (
res = &domain.DataTable{}
dataTables = make(map[int]*domain.DataTable)
err error
)
if len(queryComponents) == 0 {
return res, nil
}
// 加载Tables数据
q := queryComponents[0]
cells := q.Layout.LayoutCells()
dataTables = ptr.LoadDataTables(ctx, cells)
// 数据布局
res, err = DataLayout(res, dataTables, cells)
if err != nil {
return nil, err
}
// 数据持久化
return res, nil
}
func FastTable(table *domain.Table) (*domain.DataTable, error) {
var err error
var options = starrocks.QueryOptions{
Table: table,
TableName: table.SQLName,
Select: table.Fields(false),
}
var dataTable *domain.DataTable
dataTable, err = FastDataTable(options)
if err != nil {
return nil, err
}
return dataTable, nil
}
func FastDataTable(options starrocks.QueryOptions) (*domain.DataTable, error) {
var err error
// 待优化分批下载,压缩
var dataTable *domain.DataTable
dataTable, err = starrocks.Query(options, starrocks.WrapQueryFuncWithDB(starrocks.DB))
if err != nil {
return nil, err
}
dataTable.Total, err = starrocks.WrapQueryCountWithDB(options, starrocks.DB)()
if err != nil {
return nil, err
}
return dataTable, nil
}
func (ptr *QuerySetService) LoadDataTables(ctx *domain.Context, cells []*domain.LayoutCell) map[int]*domain.DataTable {
var (
dataTables = make(map[int]*domain.DataTable)
tableRepository, _ = repository.NewTableRepository(ptr.transactionContext)
)
tableIds := collection.NewSet()
for _, cell := range cells {
if cell.Data == nil || cell.Data.TableField == nil || cell.Data.TableField.TableId == 0 {
continue
}
tableIds.AddInt(cell.Data.TableField.TableId)
}
if len(tableIds.KeysInt()) > 0 {
_, tables, err := tableRepository.Find(map[string]interface{}{"context": ctx, "tableIds": tableIds.KeysInt()})
if err != nil {
return nil
}
for _, t := range tables {
if _, ok := dataTables[t.TableId]; ok {
continue
}
dataTable, e := FastTable(t)
if e != nil {
log.Logger.Error(e.Error())
continue
}
dataTable.Fields = t.DataFields
dataTables[t.TableId] = dataTable
}
}
return dataTables
}
func DataLayout(res *domain.DataTable, dataTables map[int]*domain.DataTable, cells []*domain.LayoutCell) (*domain.DataTable, error) {
dt := &DataLayoutDataTable{
DataTable: res,
MapDataTables: dataTables,
unprocessed: cells,
}
dt.Init(DefaultExpandNum)
for {
if len(dt.unprocessed) == 0 {
break
}
cell := dt.unprocessed[0]
dt.unprocessed = dt.unprocessed[1:]
blockData, length := dt.BlockData(cell)
if err := dt.Expand(cell, length); err != nil {
return nil, err
}
dt.addByLocation(cell, blockData)
// 当前单元格子 影响其他格子坐标
dt.changeUnProcessedLocation(cell, length)
dt.processed = append(dt.processed, cell)
dt.LastCell = cell
}
dt.Shrink()
return dt.DataTable, nil
}
type DataLayoutDataTable struct {
PointBegin *Location
PointEnd *Location
MaxX int
MaxY int
DataTable *domain.DataTable
MapDataTables map[int]*domain.DataTable
processed []*domain.LayoutCell
unprocessed []*domain.LayoutCell
LastCell *domain.LayoutCell
}
type Location struct {
X int
Y int
}
func NewLocation(x, y int) *Location {
return &Location{
X: x,
Y: y,
}
}
func (l *Location) UpdateX(x int) {
if l.X <= x {
l.X = x
}
}
func (l *Location) UpdateY(y int) {
if l.Y <= y {
l.Y = y
}
}
func (d *DataLayoutDataTable) StartCell() {
}
func (d *DataLayoutDataTable) addByLocation(cell *domain.LayoutCell, blockData []string) {
if d.PointBegin == nil {
d.PointBegin = NewLocation(cell.X, cell.Y)
d.PointEnd = NewLocation(cell.X, cell.Y)
}
switch cell.Direction {
case domain.DirectionRight:
for i := range blockData {
d.DataTable.Data[cell.X][cell.Y+i] = blockData[i]
}
d.PointEnd.UpdateX(cell.X)
d.PointEnd.UpdateY(cell.Y + len(blockData) - 1)
case domain.DirectionDown:
for i := range blockData {
d.DataTable.Data[cell.X+i][cell.Y] = blockData[i]
}
d.PointEnd.UpdateX(cell.X + len(blockData) - 1)
d.PointEnd.UpdateY(cell.Y)
case domain.DirectionNone:
d.DataTable.Data[cell.X][cell.Y] = blockData[0]
d.PointEnd.UpdateX(cell.X)
d.PointEnd.UpdateY(cell.Y)
}
}
func (d *DataLayoutDataTable) changeUnProcessedLocation(lastCell *domain.LayoutCell, length int) {
// log.Logger.Info("修改定位点")
for _, cell := range d.unprocessed {
switch lastCell.Direction {
case domain.DirectionRight:
if cell.X >= lastCell.X && cell.Y > lastCell.Y {
cell.Y += length - 1
}
case domain.DirectionDown:
if cell.X > lastCell.X && cell.Y >= lastCell.Y {
cell.X += length - 1
}
}
// log.Logger.Info(fmt.Sprintf("%s %s X:%d Y:%d", cell.Data.Field.SQLName, cell.Direction, cell.X, cell.Y))
}
}
func (d *DataLayoutDataTable) BlockData(cells *domain.LayoutCell) ([]string, int) {
var block []string
if cells.Type == domain.CellTypeText {
data := []string{cells.Data.Text}
return data, 1
}
table, ok := d.MapDataTables[cells.Data.TableField.TableId]
if !ok {
return block, 0
}
values := table.Values(&domain.Field{SQLName: cells.Data.TableField.FieldSqlName})
if len(values) == 0 {
return block, 0
}
if cells.Data.TableField != nil && strings.HasPrefix(cells.Data.TableField.TableSqlName, strings.ToLower(domain.CalculateItem.ToString())) {
return values[:1], 1
}
if cells.Direction == domain.DirectionNone {
return values[:1], 1
}
return values, len(values)
}
func (d *DataLayoutDataTable) Shrink() {
x := d.PointEnd.X - d.PointBegin.X
y := d.PointEnd.Y - d.PointBegin.Y
data := make([][]string, x+1)
for i := range data {
data[i] = make([]string, y+1)
}
iData := 0
for i := d.PointBegin.X; i <= d.PointEnd.X; i++ {
jData := 0
for j := d.PointBegin.Y; j <= d.PointEnd.Y; j++ {
data[iData][jData] = d.DataTable.Data[i][j]
jData++
}
iData++
}
d.DataTable.Data = data
for i := 0; i <= y; i++ {
d.DataTable.Fields = append(d.DataTable.Fields, &domain.Field{
Name: fmt.Sprintf("列%d", i),
SQLName: fmt.Sprintf("col%d", i),
SQLType: domain.String.ToString(),
})
}
}
func (d *DataLayoutDataTable) Expand(cell *domain.LayoutCell, length int) error {
if !d.CellOutRange(cell, length) {
return nil
}
expandLength := DefaultExpandNum
if expandLength < length {
expandLength = length
}
if d.MaxX+expandLength > MaxExpandNum {
return fmt.Errorf("布局元素超过限制 %d", MaxExpandNum)
}
copyData := make([][]string, d.MaxX+length)
for i := range copyData {
copyData[i] = make([]string, d.MaxY+length)
}
copy(copyData, d.DataTable.Data)
d.DataTable.Data = copyData
return nil
}
func (d *DataLayoutDataTable) Init(length int) {
initData := make([][]string, length)
for i := range initData {
initData[i] = make([]string, length)
}
d.DataTable.Data = initData
d.AddRange(length)
}
func (d *DataLayoutDataTable) AddRange(length int) {
d.MaxY += length
d.MaxX += length
}
func (d *DataLayoutDataTable) CellOutRange(cell *domain.LayoutCell, length int) bool {
if cell.X+length > d.MaxX {
return true
}
if cell.Y+length > d.MaxY {
return true
}
return false
}
... ...
... ... @@ -14,6 +14,7 @@ type XLXSWriterTo struct {
data [][]string
title []string
ToInterfaces func([]string) []interface{}
IgnoreTitle bool
}
func (wt *XLXSWriterTo) WriteTo(w io.Writer) (n int64, err error) {
... ... @@ -33,7 +34,7 @@ func (wt *XLXSWriterTo) Save(fileName string) error {
}
file, err = wt.newFile()
if err != nil {
return nil
return err
}
return file.SaveAs(fileName)
}
... ... @@ -68,12 +69,18 @@ func (wt *XLXSWriterTo) newFile() (*excelize.File, error) {
}
if len(wt.title) == 0 {
return nil, fmt.Errorf("未设置数据表头")
}
if err := streamWriter.SetRow("A1", stringsToInterfaces(wt.title)); err != nil {
return nil, err
if !wt.IgnoreTitle {
return nil, fmt.Errorf("未设置数据表头")
}
} else {
if err := streamWriter.SetRow("A1", stringsToInterfaces(wt.title)); err != nil {
return nil, err
}
}
var rowID = 2
if wt.IgnoreTitle {
rowID = 1
}
for i := 0; i < len(wt.data); i++ {
row := stringsToInterfaces(wt.data[i])
if wt.ToInterfaces != nil {
... ...
... ... @@ -42,7 +42,7 @@ func Must(err error) {
}
func ParseContext(c beego.BaseController) *domain.Context {
var companyId int = 1
var companyId int = 1598224576532189184
var userId int = 1
var userName string = "管理员"
if token := c.Ctx.Input.GetData("UserToken"); token != nil {
... ...
... ... @@ -130,3 +130,19 @@ func (controller *QuerySetController) SearchQuerySet() {
data, err := querySetService.SearchQuerySet(ParseContext(controller.BaseController), searchQuerySetQuery)
controller.Response(data, err)
}
func (controller *QuerySetController) CalculateSetPreview() {
querySetService := service.NewQuerySetService(nil)
updateQuerySetCommand := &command.UpdateQuerySetCommand{}
Must(controller.Unmarshal(updateQuerySetCommand))
data, err := querySetService.CalculateSetPreview(ParseContext(controller.BaseController), updateQuerySetCommand)
controller.Response(data, err)
}
func (controller *QuerySetController) CalculateSetExport() {
querySetService := service.NewQuerySetService(nil)
updateQuerySetCommand := &command.UpdateQuerySetCommand{}
Must(controller.Unmarshal(updateQuerySetCommand))
data, err := querySetService.CalculateSetExport(ParseContext(controller.BaseController), updateQuerySetCommand)
controller.Response(data, err)
}
... ...
... ... @@ -31,6 +31,8 @@ func init() {
web.Router("/data/query-sets/formula/rename", &controllers.QuerySetController{}, "Post:Rename")
web.Router("/data/query-sets/formula/search", &controllers.QuerySetController{}, "Post:SearchQuerySet")
web.Router("/data/query-sets/formula/calculate-table-preview-prepare", &controllers.QuerySetController{}, "Post:PreviewPrepare")
web.Router("/data/query-sets/formula/calculate-set-preview", &controllers.QuerySetController{}, "Post:CalculateSetPreview")
web.Router("/data/query-sets/formula/calculate-set-export", &controllers.QuerySetController{}, "Post:CalculateSetExport")
web.Router("/data/query-sets/formula/calculate-item-preview", &controllers.QuerySetController{}, "Post:CalculateItemPreview")
web.Router("/data/query-sets/formula/calculate-item-export", &controllers.QuerySetController{}, "Post:CalculateItemExport")
... ...