allied-creation / character-library-metadata-bastion · 提交

转到一个项目

作者 yangfu
提交 5f8d5462651f24770c801531276c8af59e59c8b9 1 个父辈 1e58d10c

chore: refactor

内嵌 并排对比
正在显示 26 个修改的文件 包含 688 行增加 和 102 行删除
@@ -3,7 +3,7 @@ @@ -3,7 +3,7 @@
3 ## 源文件管理 3 ## 源文件管理
4 4
5 - 文件列表 - list 5 - 文件列表 - list
6 -- 上传 - 上传oss / 创建file 6 +- 上传 - 上传 oss / 创建 file
7 - 加载 - loadDataTable 7 - 加载 - loadDataTable
8 - 编辑 - editDataTable 8 - 编辑 - editDataTable
9 - 持久化 - flushDataTable 9 - 持久化 - flushDataTable
@@ -11,7 +11,6 @@ @@ -11,7 +11,6 @@
11 - 删除 - delete 11 - 删除 - delete
12 - 操作日志 - log 12 - 操作日志 - log
13 13
14 -  
15 - editDataTable params 列表 14 - editDataTable params 列表
16 15
17 ### 加载表格数据 loadDataTable - 查询 16 ### 加载表格数据 loadDataTable - 查询
@@ -25,9 +24,9 @@ @@ -25,9 +24,9 @@
25 "index": 1, 24 "index": 1,
26 "name": "产品名称" 25 "name": "产品名称"
27 }, 26 },
28 - "in": ["a","b"],  
29 - "ex": ["c","d"],  
30 - "sort": ["a","asc"] 27 + "in": ["a", "b"],
  28 + "ex": ["c", "d"],
  29 + "sort": ["a", "asc"]
31 } 30 }
32 ] 31 ]
33 } 32 }
@@ -42,17 +41,19 @@ @@ -42,17 +41,19 @@
42 "name": "产品名称" 41 "name": "产品名称"
43 }, 42 },
44 "operation": { 43 "operation": {
45 - "desc": ["拆分","按字符数"], 44 + "desc": ["拆分", "按字符数"],
46 "code": "split_by_char_number" 45 "code": "split_by_char_number"
47 }, 46 },
48 "params": [] 47 "params": []
49 } 48 }
50 ``` 49 ```
  50 +
51 精简 51 精简
  52 +
52 ```json 53 ```json
53 { 54 {
54 "field": "产品名称", 55 "field": "产品名称",
55 - "desc": ["拆分","按字符数"], 56 + "desc": ["拆分", "按字符数"],
56 "operationCode": "split_by_char_number", 57 "operationCode": "split_by_char_number",
57 "params": [] 58 "params": []
58 } 59 }
@@ -60,7 +61,6 @@ @@ -60,7 +61,6 @@
60 61
61 `params 列表` 62 `params 列表`
62 63
63 -  
64 ### 数据展示 64 ### 数据展示
65 65
66 ```json 66 ```json
@@ -80,22 +80,10 @@ @@ -80,22 +80,10 @@
80 } 80 }
81 ], 81 ],
82 "dataRows": [ 82 "dataRows": [
83 - [  
84 - "素面",  
85 - 200  
86 - ],  
87 - [  
88 - "冻豆腐",  
89 - 400  
90 - ],  
91 - [  
92 - "冻豆腐1",  
93 - 300  
94 - ],  
95 - [  
96 - "冻豆2",  
97 - "A"  
98 - ] 83 + ["素面", 200],
  84 + ["冻豆腐", 400],
  85 + ["冻豆腐1", 300],
  86 + ["冻豆2", "A"]
99 ], 87 ],
100 "total": 100, 88 "total": 100,
101 "pageNumber": 1, 89 "pageNumber": 1,
@@ -111,7 +99,6 @@ @@ -111,7 +99,6 @@
111 } 99 }
112 ``` 100 ```
113 101
114 -  
115 ## 表关联关系 102 ## 表关联关系
116 103
117 - [x] 可追加数据的表列表 /tables/search-appended-list 104 - [x] 可追加数据的表列表 /tables/search-appended-list
@@ -135,6 +122,7 @@ @@ -135,6 +122,7 @@
135 - [x] 校验步骤日志 /log/verified-step-Log 122 - [x] 校验步骤日志 /log/verified-step-Log
136 123
137 ## 数据预览 124 ## 数据预览
  125 +
138 - [x] 表数据预览（格式） /table/preview 126 - [x] 表数据预览（格式） /table/preview
139 - [x] 表数据自定义查询 /table/preview where conditions 升序、降序 包含、不包含 127 - [x] 表数据自定义查询 /table/preview where conditions 升序、降序 包含、不包含
140 - [x] 表数据字段可选值搜索 /table/field-optional 文本匹配 128 - [x] 表数据字段可选值搜索 /table/field-optional 文本匹配
@@ -147,37 +135,29 @@ @@ -147,37 +135,29 @@
147 135
148 ## 底层字库接口 136 ## 底层字库接口
149 137
150 -```json  
151 -{  
152 - "file": {},  
153 - "fields": [],  
154 - "action":"filed_rename",  
155 - "params": ["产品名2"]  
156 -}  
157 -```  
158 - [x] 数据预览 1 138 - [x] 数据预览 1
159 -- [ ] 表格编辑 1  
160 -- [ ] 保存校验文件 (文件地址) 1 139 +- [x] 表格编辑 1
  140 +- [x] 保存校验文件 (文件地址) 1
161 - [x] 生成主表 1 141 - [x] 生成主表 1
162 - [x] 表复制 （副表）1 142 - [x] 表复制 （副表）1
163 - [x] 追加数据 （主表、副表） 143 - [x] 追加数据 （主表、副表）
164 - [ ] 表删除 （主表、副表）~~、分表~~ 144 - [ ] 表删除 （主表、副表）~~、分表~~
165 - [x] 表拆分 1 145 - [x] 表拆分 1
166 -- [ ] 更新表结构（分表）1  
167 -- [ ] 编辑、添加、删除表数据（副表） 1 146 +- [x] 更新表结构（分表）1
  147 +- [x] 编辑、添加、删除表数据（副表） 1
168 - [ ] 取消校验 148 - [ ] 取消校验
169 149
170 ## 定时作业 150 ## 定时作业
171 151
172 - [x] 隔天清理校验中的文件 152 - [x] 隔天清理校验中的文件
173 -- [x] 隔天清理public临时文件 153 +- [x] 隔天清理 public 临时文件
174 154
175 ## 表数据导出 155 ## 表数据导出
176 156
177 -- [ ] 加锁,只允许当前用户同时只能发起一次导出命令 ,3min过期 157 +- [ ] 加锁,只允许当前用户同时只能发起一次导出命令 ,3min 过期
178 - [ ] 单次拉取数量 MR 158 - [ ] 单次拉取数量 MR
179 - [ ] 100W .. 159 - [ ] 100W ..
180 - - [ ] 50W 120s 读取数据库:30s 保存文件:10s 下载:30M/500K=60S;RAR压缩 24M/500k=50S 160 + - [ ] 50W 120s 读取数据库:30s 保存文件:10s 下载:30M/500K=60S;RAR 压缩 24M/500k=50S
181 - [ ] 20W .. 161 - [ ] 20W ..
182 - [ ] 10W .. 162 - [ ] 10W ..
183 - [ ] 保存单个文件、压缩 | 保存多个文件、压缩 163 - [ ] 保存单个文件、压缩 | 保存多个文件、压缩
@@ -187,3 +167,282 @@ @@ -187,3 +167,282 @@
187 - [ ] 校验动作，参数模型讨论 167 - [ ] 校验动作，参数模型讨论
188 - [ ] 校验日志错误（标红） 168 - [ ] 校验日志错误（标红）
189 - [ ] 校验完毕应答实体，类型修改即使错误，也要返回修改完毕的表 169 - [ ] 校验完毕应答实体，类型修改即使错误，也要返回修改完毕的表
  170 +
  171 +## 参数说明
  172 +
  173 +### 通用格式
  174 +
  175 +```json
  176 +{
  177 + "objectId": 1,
  178 + "processFields": [],
  179 + "action": "xx",
  180 + "params": {}
  181 +}
  182 +```
  183 +
  184 +processFields：操作字段
  185 +
  186 +### 常规
  187 +
  188 +1. 删除列
  189 +
  190 +```json
  191 +{
  192 + "action": "remove-column",
  193 + "params": {}
  194 +}
  195 +```
  196 +
  197 +2. 复制列
  198 +
  199 +```json
  200 +{
  201 + "action": "copy-column",
  202 + "params": {}
  203 +}
  204 +```
  205 +
  206 +3. 重命名
  207 +
  208 +```json
  209 +{
  210 + "action": "rename-column",
  211 + "params": {
  212 + "newColumnName": "新的列名称"
  213 + }
  214 +}
  215 +```
  216 +
  217 +4. 替换值
  218 +
  219 +```json
  220 +{
  221 + "action": "replace-column",
  222 + "params": {
  223 + "replaceMethod": "replace",
  224 + "searchValue": "搜索值",
  225 + "replaceValue": "替换值"
  226 + }
  227 +}
  228 +```
  229 +
  230 +参数说明
  231 +
  232 +```
  233 +replaceMethod: 替换方法(1.replace:替换值 2.add-prefix:添加前缀 3.add-postfix:添加后缀 4.remove-prefix:去除前缀 5.remove-postfix:去除后缀 6.remove-chars:去除固定字符 7.clean:清除)
  234 +searchValue: 搜索值-replace,remove-prefix,remove-postfix,remove-chars参数
  235 +replaceValue: 替换值-replace,add-prefix,add-postfix参数
  236 +```
  237 +
  238 +### 格式 formatMethod
  239 +
  240 +1. 大写
  241 +
  242 +```json
  243 +{
  244 + "action": "format-column",
  245 + "params": {
  246 + "formatMethod": "upper"
  247 + }
  248 +}
  249 +```
  250 +
  251 +参数说明
  252 +
  253 +```
  254 +formatMethod: 格式化方法(1.upper:大写2.lower:小写3.capitalize:首字母大写4.strip:修整)
  255 +```
  256 +
  257 +2. 小写
  258 +
  259 +```json
  260 +{
  261 + "action": "format-column",
  262 + "params": {
  263 + "formatMethod": "lower"
  264 + }
  265 +}
  266 +```
  267 +
  268 +3. 首字母大写
  269 +
  270 +```json
  271 +{
  272 + "action": "format-column",
  273 + "params": {
  274 + "formatMethod": "capitalize"
  275 + }
  276 +}
  277 +```
  278 +
  279 +4. 清除
  280 +
  281 +```json
  282 +{
  283 + "action": "replace-column",
  284 + "params": {
  285 + "replaceMethod": "clean"
  286 + }
  287 +}
  288 +```
  289 +
  290 +5. 修整
  291 +
  292 +```json
  293 +{
  294 + "action": "format-column",
  295 + "params": {
  296 + "formatMethod": "strip"
  297 + }
  298 +}
  299 +```
  300 +
  301 +6. 添加前缀
  302 +
  303 +```json
  304 +{
  305 + "action": "replace-column",
  306 + "params": {
  307 + "replaceMethod": "add-prefix",
  308 + "replaceValue": "前缀值"
  309 + }
  310 +}
  311 +```
  312 +
  313 +7. 添加后缀
  314 +
  315 +```json
  316 +{
  317 + "action": "replace-column",
  318 + "params": {
  319 + "replaceMethod": "add-postfix",
  320 + "replaceValue": "后缀值"
  321 + }
  322 +}
  323 +```
  324 +
  325 +8. 去除前缀
  326 +
  327 +```json
  328 +{
  329 + "action": "replace-column",
  330 + "params": {
  331 + "replaceMethod": "remove-prefix",
  332 + "searchValue": "前缀值"
  333 + }
  334 +}
  335 +```
  336 +
  337 +9. 去除后最
  338 +
  339 +```json
  340 +{
  341 + "action": "replace-column",
  342 + "params": {
  343 + "replaceMethod": "remove-postfix",
  344 + "searchValue": "后缀值"
  345 + }
  346 +}
  347 +```
  348 +
  349 +10. 去除固定字符
  350 +
  351 +```json
  352 +{
  353 + "action": "replace-column",
  354 + "params": {
  355 + "replaceMethod": "remove-chars",
  356 + "searchValue": "字符"
  357 + }
  358 +}
  359 +```
  360 +
  361 +### 拆分 split-column
  362 +
  363 +1. 按分隔符
  364 +
  365 +```json
  366 +{
  367 + "action": "split-column",
  368 + "params": {
  369 + "splitMethod": "separator",
  370 + "separator": "|",
  371 + "splitDirection": "left",
  372 + "splitCount": "1"
  373 + }
  374 +}
  375 +```
  376 +
  377 +参数说明
  378 +
  379 +```
  380 +separator: 分割符号 ‘|’
  381 +splitDirection: 拆分方向(1.left:从左边 2.right:从右边)-separator专属参数
  382 +splitCount: 拆分次数-separator专属参数
  383 +```
  384 +
  385 +2. 按字符数
  386 +
  387 +```json
  388 +{
  389 + "action": "split-column",
  390 + "params": {
  391 + "splitMethod": "char-length",
  392 + "charLength": 10
  393 + }
  394 +}
  395 +```
  396 +
  397 +参数说明
  398 +
  399 +```
  400 +charLength: 字符长度-char-length专属参数
  401 +```
  402 +
  403 +### 提取 extract-column
  404 +
  405 +1. 按日期
  406 +
  407 +```json
  408 +{
  409 + "action": "extract-column",
  410 + "params": {
  411 + "extractMethod": "by-date"
  412 + }
  413 +}
  414 +```
  415 +
  416 +参数说明
  417 +
  418 +```
  419 +extractMethod: 提取方法(1.by-date:按日期 2.by-number:按数值)
  420 +```
  421 +
  422 +2. 按数值 action
  423 +
  424 +```json
  425 +{
  426 + "action": "extract-column",
  427 + "params": {
  428 + "extractMethod": "by-number"
  429 + }
  430 +}
  431 +```
  432 +
  433 +### 修改字段类型
  434 +
  435 +```json
  436 +{
  437 + "action": "convert-column-type",
  438 + "params": {
  439 + "convertType": "STRING"
  440 + }
  441 +}
  442 +```
  443 +
  444 +参数说明
  445 +
  446 +```
  447 +convertType:转换类型 STRING 数值: INT 小数: FLOAT 日期: DATE 时间: DATETIME
  448 +```
@@ -96,7 +96,7 @@ spec: @@ -96,7 +96,7 @@ spec:
96 - name: ERROR_BASE_CODE_MULTIPLE 96 - name: ERROR_BASE_CODE_MULTIPLE
97 value: "2000" 97 value: "2000"
98 - name: ENABLE_KAFKA_LOG 98 - name: ENABLE_KAFKA_LOG
99 - value: "true" 99 + value: "false"
100 - name: HTTP_PORT 100 - name: HTTP_PORT
101 value: "8082" 101 value: "8082"
102 - name: SERVICE_ENV 102 - name: SERVICE_ENV
@@ -5,14 +5,12 @@ import ( @@ -5,14 +5,12 @@ import (
5 "github.com/beego/beego/v2/server/web" 5 "github.com/beego/beego/v2/server/web"
6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/crontab" 6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/application/crontab"
7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant" 7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
  8 + _ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"
8 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/pg" 9 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/pg"
9 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis" 10 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
  11 + _ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
10 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks" 12 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/starrocks"
11 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log" 13 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
12 - "time"  
13 -  
14 - _ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/constant"  
15 - _ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"  
16 _ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log" 14 _ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/log"
17 _ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego" 15 _ "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/port/beego"
18 ) 16 )
@@ -37,7 +35,6 @@ func main() { @@ -37,7 +35,6 @@ func main() {
37 cron.StartCrontabTask() 35 cron.StartCrontabTask()
38 defer cron.StopCrontabTask() 36 defer cron.StopCrontabTask()
39 37
40 - time.Sleep(time.Second)  
41 log.Logger.Info("Service:" + constant.SERVICE_NAME) 38 log.Logger.Info("Service:" + constant.SERVICE_NAME)
42 log.Logger.Info("Version:" + Version) 39 log.Logger.Info("Version:" + Version)
43 log.Logger.Info("server start!") 40 log.Logger.Info("server start!")
@@ -18,7 +18,7 @@ type EditDataTableCommand struct { @@ -18,7 +18,7 @@ type EditDataTableCommand struct {
18 } 18 }
19 19
20 func (editDataTableCommand *EditDataTableCommand) Valid(validation *validation.Validation) { 20 func (editDataTableCommand *EditDataTableCommand) Valid(validation *validation.Validation) {
21 - if len(editDataTableCommand.ProcessFields) == 0 { 21 + if len(editDataTableCommand.ProcessFieldNames) == 0 {
22 validation.Error("未选择操作列") 22 validation.Error("未选择操作列")
23 return 23 return
24 } 24 }
@@ -10,7 +10,7 @@ import ( @@ -10,7 +10,7 @@ import (
10 10
11 type FlushDataTableCommand struct { 11 type FlushDataTableCommand struct {
12 // 文件ID 12 // 文件ID
13 - FileId int `cname:"文件ID" json:"objectId" valid:"Required"` 13 + ObjectId int `cname:"文件ID" json:"objectId" valid:"Required"`
14 // 记录数 14 // 记录数
15 //RowCount int `cname:"记录数" json:"rowCount" valid:"Required"` 15 //RowCount int `cname:"记录数" json:"rowCount" valid:"Required"`
16 // 数据列 16 // 数据列
  1 +package dto
  2 +
  3 +import "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
  4 +
  5 +type EditDataTableDto struct {
  6 + Code int `json:"errNo,omitempty"`
  7 + Error string `json:"errMsg,omitempty"`
  8 + InValidCells []domain.InValidCell `json:"inValidCells,omitempty"`
  9 +}
  10 +
  11 +func (d *EditDataTableDto) Load(m *domain.DataEditDataTable) *EditDataTableDto {
  12 + d.Error = ""
  13 + if len(m.InValidCells) > 0 {
  14 + d.Error = "类型转换错误"
  15 + d.Code = 1001 // 1001:类型转换错误
  16 + }
  17 + //d.InValidCells = m.InValidCells
  18 + return d
  19 +}
@@ -92,15 +92,16 @@ func (fileService *FileService) EditDataTable(ctx *domain.Context, editDataTable @@ -92,15 +92,16 @@ func (fileService *FileService) EditDataTable(ctx *domain.Context, editDataTable
92 return nil, factory.FastError(err) 92 return nil, factory.FastError(err)
93 } 93 }
94 editDataTableCommand.Fields = temporaryFile.Fields 94 editDataTableCommand.Fields = temporaryFile.Fields
  95 + editDataTableCommand.ProcessFields = temporaryFile.MatchFields(editDataTableCommand.ProcessFieldNames)
95 editDataTableService, _ := factory.CreateEditDataTableService(transactionContext) 96 editDataTableService, _ := factory.CreateEditDataTableService(transactionContext)
96 - _, err = editDataTableService.Edit(ctx, editDataTableCommand.EditTableRequest) 97 + response, err := editDataTableService.Edit(ctx, editDataTableCommand.EditTableRequest)
97 if err != nil { 98 if err != nil {
98 return nil, factory.FastError(err) 99 return nil, factory.FastError(err)
99 } 100 }
100 if err := transactionContext.CommitTransaction(); err != nil { 101 if err := transactionContext.CommitTransaction(); err != nil {
101 return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error()) 102 return nil, application.ThrowError(application.TRANSACTION_ERROR, err.Error())
102 } 103 }
103 - return struct{}{}, nil 104 + return (&dto.EditDataTableDto{}).Load(response), nil
104 } 105 }
105 106
106 // FlushDataTable 持久化表格数据 107 // FlushDataTable 持久化表格数据
@@ -128,12 +129,12 @@ func (fileService *FileService) FlushDataTable(ctx *domain.Context, flushDataTab @@ -128,12 +129,12 @@ func (fileService *FileService) FlushDataTable(ctx *domain.Context, flushDataTab
128 //} 129 //}
129 130
130 cache := redis.NewFileCacheService() 131 cache := redis.NewFileCacheService()
131 - temporaryFile, err := cache.Get(redis.KeyTemporaryFileInfo(flushDataTableCommand.FileId)) 132 + temporaryFile, err := cache.Get(redis.KeyTemporaryFileInfo(flushDataTableCommand.ObjectId))
132 if err != nil { 133 if err != nil {
133 return nil, factory.FastError(err) 134 return nil, factory.FastError(err)
134 } 135 }
135 136
136 - if _, err := flushDataTableService.Flush(ctx, flushDataTableCommand.FileId, &domain.Table{ 137 + if _, err := flushDataTableService.Flush(ctx, flushDataTableCommand.ObjectId, &domain.Table{
137 DataFields: temporaryFile.Fields, 138 DataFields: temporaryFile.Fields,
138 RowCount: temporaryFile.Total, 139 RowCount: temporaryFile.Total,
139 }); err != nil { 140 }); err != nil {
@@ -47,7 +47,7 @@ func (cmd *SearchLogCommand) Valid(validation *validation.Validation) { @@ -47,7 +47,7 @@ func (cmd *SearchLogCommand) Valid(validation *validation.Validation) {
47 } 47 }
48 if cmd.Year > 0 && cmd.Month > 0 && cmd.Day > 0 { 48 if cmd.Year > 0 && cmd.Month > 0 && cmd.Day > 0 {
49 cmd.BeginTime = time.Date(cmd.Year, time.Month(cmd.Month), cmd.Day, 0, 0, 0, 0, time.Local) 49 cmd.BeginTime = time.Date(cmd.Year, time.Month(cmd.Month), cmd.Day, 0, 0, 0, 0, time.Local)
50 - cmd.EndTime = cmd.BeginTime.AddDate(0, 0, cmd.Day) 50 + cmd.EndTime = cmd.BeginTime.AddDate(0, 0, 1)
51 } 51 }
52 } 52 }
53 53
@@ -150,6 +150,7 @@ type ( @@ -150,6 +150,7 @@ type (
150 150
151 type ( 151 type (
152 ReqCopyTable struct { 152 ReqCopyTable struct {
  153 + MainTable *Table
153 Table *Table 154 Table *Table
154 CopyToTable *Table 155 CopyToTable *Table
155 } 156 }
@@ -14,6 +14,9 @@ type Context struct { @@ -14,6 +14,9 @@ type Context struct {
14 } 14 }
15 15
16 func (c *Context) WithValue(key string, value interface{}) *Context { 16 func (c *Context) WithValue(key string, value interface{}) *Context {
  17 + if c.data == nil {
  18 + c.data = make(map[string]interface{})
  19 + }
17 if _, ok := c.data[key]; ok { 20 if _, ok := c.data[key]; ok {
18 return c 21 return c
19 } 22 }
@@ -22,6 +25,9 @@ func (c *Context) WithValue(key string, value interface{}) *Context { @@ -22,6 +25,9 @@ func (c *Context) WithValue(key string, value interface{}) *Context {
22 } 25 }
23 26
24 func (c *Context) GetValue(key string) (interface{}, bool) { 27 func (c *Context) GetValue(key string) (interface{}, bool) {
  28 + if c.data == nil {
  29 + return nil, false
  30 + }
25 if v, ok := c.data[key]; ok { 31 if v, ok := c.data[key]; ok {
26 return v, true 32 return v, true
27 } 33 }
@@ -24,7 +24,7 @@ type PreviewDataTableService interface { @@ -24,7 +24,7 @@ type PreviewDataTableService interface {
24 } 24 }
25 25
26 type EditDataTableService interface { 26 type EditDataTableService interface {
27 - Edit(ctx *Context, param EditTableRequest) (interface{}, error) 27 + Edit(ctx *Context, param EditTableRequest) (*DataEditDataTable, error)
28 } 28 }
29 29
30 type FlushDataTableService interface { 30 type FlushDataTableService interface {
@@ -65,7 +65,8 @@ type AddTableStructService interface { @@ -65,7 +65,8 @@ type AddTableStructService interface {
65 type EditTableRequest struct { 65 type EditTableRequest struct {
66 FileId int `json:"objectId"` 66 FileId int `json:"objectId"`
67 Fields []*Field `json:"fields"` 67 Fields []*Field `json:"fields"`
68 - ProcessFields []*Field `json:"processFields"` 68 + ProcessFieldNames []string `json:"processFields"`
  69 + ProcessFields []*Field `json:"-" `
69 Where Where `json:"where"` 70 Where Where `json:"where"`
70 Action string `json:"action"` 71 Action string `json:"action"`
71 Params map[string]interface{} `json:"params"` 72 Params map[string]interface{} `json:"params"`
@@ -77,6 +77,7 @@ var ( @@ -77,6 +77,7 @@ var (
77 var ( 77 var (
78 String SQLType = "STRING" 78 String SQLType = "STRING"
79 Int SQLType = "INT" 79 Int SQLType = "INT"
  80 + BigInt SQLType = "BIGINT"
80 Float SQLType = "FLOAT" 81 Float SQLType = "FLOAT"
81 Date SQLType = "DATE" 82 Date SQLType = "DATE"
82 Datetime SQLType = "DATETIME" 83 Datetime SQLType = "DATETIME"
@@ -226,3 +227,98 @@ func (t LogLevel) ToString() string { @@ -226,3 +227,98 @@ func (t LogLevel) ToString() string {
226 const ( 227 const (
227 DefaultPkField = "id" 228 DefaultPkField = "id"
228 ) 229 )
  230 +
  231 +type Action struct {
  232 + // 操作名
  233 + Desc string `json:"desc"`
  234 + Name string `json:"name"`
  235 +}
  236 +
  237 +const (
  238 + RemoveColumn = "remove-column"
  239 + CopyColumn = "copy-column"
  240 + RenameColumn = "rename-column"
  241 + ReplaceColumn = "replace-column"
  242 + ConvertColumnType = "convert-column-type"
  243 +
  244 + FormatColumn = "format-column"
  245 + SplitColumn = "split-column"
  246 + ExtractColumn = "extract-column"
  247 +)
  248 +
  249 +var MapActionCommon = map[string]Action{
  250 + // 常规
  251 + RemoveColumn: {
  252 + Desc: "删除列", Name: RemoveColumn,
  253 + },
  254 + CopyColumn: {
  255 + Desc: "复制列", Name: CopyColumn,
  256 + },
  257 + RenameColumn: {
  258 + Desc: "重命名", Name: RenameColumn,
  259 + },
  260 + ConvertColumnType: {
  261 + Desc: "数据类型更改", Name: ConvertColumnType,
  262 + },
  263 +}
  264 +
  265 +var MapActionFormat = map[string]Action{
  266 + // 格式
  267 + "upper": {
  268 + Desc: "大写", Name: "upper",
  269 + },
  270 + "lower": {
  271 + Desc: "小写", Name: "lower",
  272 + },
  273 + "capitalize": {
  274 + Desc: "首字母大写", Name: "capitalize",
  275 + },
  276 + "f4": {
  277 + Desc: "删除列", Name: "cc",
  278 + },
  279 + "strip": {
  280 + Desc: "休整", Name: "strip",
  281 + },
  282 +}
  283 +
  284 +var MapActionReplaceColumn = map[string]Action{
  285 + "add-prefix": {
  286 + Desc: "添加前缀", Name: "add-prefix",
  287 + },
  288 + "add-postfix": {
  289 + Desc: "添加后缀", Name: "add-postfix",
  290 + },
  291 + "remove-prefix": {
  292 + Desc: "去除前缀", Name: "remove-prefix",
  293 + },
  294 + "remove-postfix": {
  295 + Desc: "去除后缀", Name: "remove-postfix",
  296 + },
  297 + "remove-chars": {
  298 + Desc: "去除固定字符", Name: "remove-chars",
  299 + },
  300 + "replace": {
  301 + Desc: "替换值", Name: "replace",
  302 + },
  303 + "clean": {
  304 + Desc: "清除", Name: "clean",
  305 + },
  306 +}
  307 +
  308 +var MapActionSplitColumn = map[string]Action{
  309 + "separator": {
  310 + Desc: "按按分隔符拆分", Name: "separator",
  311 + },
  312 + "char-length": {
  313 + Desc: "按字符数拆分", Name: "char-length",
  314 + },
  315 +}
  316 +
  317 +var MapActionExtractColumn = map[string]Action{
  318 + "by-date": {
  319 + Desc: "按日期提取", Name: "by-date",
  320 + },
  321 + "by-number": {
  322 + Desc: "按数值提取", Name: "by-number",
  323 + },
  324 +}
@@ -35,6 +35,16 @@ func (f *Field) Valid() error { @@ -35,6 +35,16 @@ func (f *Field) Valid() error {
35 return nil 35 return nil
36 } 36 }
37 37
  38 +func (f *Field) Copy() *Field {
  39 + return &Field{
  40 + Index: f.Index,
  41 + Name: f.Name,
  42 + SQLName: f.SQLName,
  43 + SQLType: f.SQLType,
  44 + Flag: f.Flag,
  45 + }
  46 +}
  47 +
38 type Fields []*Field 48 type Fields []*Field
39 49
40 func (fields Fields) ToMap() map[string]*Field { 50 func (fields Fields) ToMap() map[string]*Field {
@@ -163,17 +173,26 @@ func ValueToType(value string, sqlType string) (interface{}, error) { @@ -163,17 +173,26 @@ func ValueToType(value string, sqlType string) (interface{}, error) {
163 if err != nil { 173 if err != nil {
164 err = fmt.Errorf("[%v]不是有效的数值类型", value) 174 err = fmt.Errorf("[%v]不是有效的数值类型", value)
165 } 175 }
  176 + case BigInt.ToString():
  177 + toTypeVal, err = numberString.Int()
  178 + if err != nil {
  179 + err = fmt.Errorf("[%v]不是有效的数值类型", value)
  180 + }
166 case Float.ToString(): 181 case Float.ToString():
167 toTypeVal, err = numberString.Float64() 182 toTypeVal, err = numberString.Float64()
  183 + if err != nil {
168 err = fmt.Errorf("[%v]不是有效的浮点数类型", value) 184 err = fmt.Errorf("[%v]不是有效的浮点数类型", value)
  185 + }
169 case Date.ToString(): 186 case Date.ToString():
170 toTypeVal, err = xtime.Parse(value) 187 toTypeVal, err = xtime.Parse(value)
  188 + if err != nil {
171 err = fmt.Errorf("[%v]不是有效的日期类型", value) 189 err = fmt.Errorf("[%v]不是有效的日期类型", value)
172 - return nil, nil 190 + }
173 case Datetime.ToString(): 191 case Datetime.ToString():
174 toTypeVal, err = xtime.Parse(value) 192 toTypeVal, err = xtime.Parse(value)
  193 + if err != nil {
175 err = fmt.Errorf("[%v]不是有效的时间类型", value) 194 err = fmt.Errorf("[%v]不是有效的时间类型", value)
176 - return nil, nil 195 + }
177 default: 196 default:
178 return nil, fmt.Errorf("unknow sql type :%v", sqlType) 197 return nil, fmt.Errorf("unknow sql type :%v", sqlType)
179 } 198 }
@@ -67,6 +67,16 @@ func (table *Table) WithContext(ctx *Context) *Table { @@ -67,6 +67,16 @@ func (table *Table) WithContext(ctx *Context) *Table {
67 return table 67 return table
68 } 68 }
69 69
  70 +func (table *Table) WithParentId(parentId int) *Table {
  71 + table.ParentId = parentId
  72 + return table
  73 +}
  74 +
  75 +func (table *Table) WithDataFieldIndex(dataFieldIndex int) *Table {
  76 + table.DataFieldIndex = dataFieldIndex
  77 + return table
  78 +}
  79 +
70 func (table *Table) Update(data map[string]interface{}) error { 80 func (table *Table) Update(data map[string]interface{}) error {
71 return nil 81 return nil
72 } 82 }
@@ -109,6 +109,7 @@ func (gateway ApiByteLib) CopyTable(param domain.ReqCopyTable) (*domain.DataCopy @@ -109,6 +109,7 @@ func (gateway ApiByteLib) CopyTable(param domain.ReqCopyTable) (*domain.DataCopy
109 } 109 }
110 url = gateway.Host() + "/master-tables/copy" 110 url = gateway.Host() + "/master-tables/copy"
111 case domain.SubTable.ToString(): 111 case domain.SubTable.ToString():
  112 + copyRequest.DatabaseTableName = param.MainTable.SQLName // 分表复制时 传入主表的表名
112 request = CopyMasterTableRequest{ 113 request = CopyMasterTableRequest{
113 MasterTableId: intToString(param.Table.TableId), 114 MasterTableId: intToString(param.Table.TableId),
114 CopyTableRequest: copyRequest, 115 CopyTableRequest: copyRequest,
@@ -25,7 +25,7 @@ type DataCheckoutTables struct { @@ -25,7 +25,7 @@ type DataCheckoutTables struct {
25 PageSize int `json:"pageSize"` 25 PageSize int `json:"pageSize"`
26 DataCount int `json:"dataCount"` 26 DataCount int `json:"dataCount"`
27 ShowData [][]string `json:"showData"` 27 ShowData [][]string `json:"showData"`
28 - AbnormalProccessIndexes [][]interface{} `json:"abnormalProccessIndexes"` 28 + AbnormalProccessIndexes [][]string `json:"abnormalProccessIndexes"`
29 } 29 }
30 30
31 func NewRequestCheckoutTablesQuery(param domain.ReqLoadDataTable) RequestCheckoutTablesQuery { 31 func NewRequestCheckoutTablesQuery(param domain.ReqLoadDataTable) RequestCheckoutTablesQuery {
@@ -48,10 +48,10 @@ type RequestCheckoutTablesPreProccess struct { @@ -48,10 +48,10 @@ type RequestCheckoutTablesPreProccess struct {
48 ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"` 48 ColumnSchemas []domain.ColumnSchema `json:"columnSchemas"`
49 PreProccessColumnNames []string `json:"preProccessColumnNames"` 49 PreProccessColumnNames []string `json:"preProccessColumnNames"`
50 PreProccessActionName string `json:"preProccessActionName"` 50 PreProccessActionName string `json:"preProccessActionName"`
51 - PreProccessActionParameterValues map[string]string `json:"preProccessActionParameterValues"` 51 + PreProccessActionParameterValues map[string]interface{} `json:"preProccessActionParameterValues"`
52 PageNumber int `json:"pageNumber"` 52 PageNumber int `json:"pageNumber"`
53 PageSize int `json:"pageSize"` 53 PageSize int `json:"pageSize"`
54 - QueryParameters map[string]interface{} `json:"queryParameters"` 54 + QueryParameters interface{} `json:"queryParameters"`
55 SortParameters map[string]interface{} `json:"sortParameters"` 55 SortParameters map[string]interface{} `json:"sortParameters"`
56 } 56 }
57 57
@@ -60,13 +60,17 @@ func NewRequestCheckoutTablesPreProccess(param domain.ReqEditDataTable) RequestC @@ -60,13 +60,17 @@ func NewRequestCheckoutTablesPreProccess(param domain.ReqEditDataTable) RequestC
60 OriginalTableId: fmt.Sprintf("%v", param.FileId), 60 OriginalTableId: fmt.Sprintf("%v", param.FileId),
61 ColumnSchemas: FieldsToColumnSchemas(param.Fields), 61 ColumnSchemas: FieldsToColumnSchemas(param.Fields),
62 PreProccessActionName: param.Action, 62 PreProccessActionName: param.Action,
63 - PreProccessActionParameterValues: make(map[string]string), 63 + PreProccessActionParameterValues: param.Params,
64 PageNumber: param.PageNumber, 64 PageNumber: param.PageNumber,
65 PageSize: param.PageSize, 65 PageSize: param.PageSize,
66 - QueryParameters: make(map[string]interface{}), 66 + QueryParameters: []interface{}{},
67 SortParameters: make(map[string]interface{}), 67 SortParameters: make(map[string]interface{}),
68 } 68 }
69 69
  70 + if len(param.Params) == 0 {
  71 + req.PreProccessActionParameterValues = make(map[string]interface{})
  72 + }
  73 +
70 for _, v := range param.ProcessFields { 74 for _, v := range param.ProcessFields {
71 req.PreProccessColumnNames = append(req.PreProccessColumnNames, v.Name) 75 req.PreProccessColumnNames = append(req.PreProccessColumnNames, v.Name)
72 } 76 }
@@ -120,6 +124,17 @@ func ToDataLoadDataTable(data DataCheckoutTables) *domain.DataLoadDataTable { @@ -120,6 +124,17 @@ func ToDataLoadDataTable(data DataCheckoutTables) *domain.DataLoadDataTable {
120 SQLType: f.ColumnType, 124 SQLType: f.ColumnType,
121 }) 125 })
122 } 126 }
  127 +
  128 + for _, invalid := range data.AbnormalProccessIndexes {
  129 + if len(invalid) == 3 {
  130 + index, _ := strconv.Atoi(invalid[0])
  131 + response.InValidCells = append(response.InValidCells, domain.InValidCell{
  132 + Index: index,
  133 + Col: invalid[1],
  134 + Err: invalid[2],
  135 + })
  136 + }
  137 + }
123 return response 138 return response
124 } 139 }
125 140
@@ -4,6 +4,7 @@ import ( @@ -4,6 +4,7 @@ import (
4 "fmt" 4 "fmt"
5 pgTransaction "github.com/linmadan/egglib-go/transaction/pg" 5 pgTransaction "github.com/linmadan/egglib-go/transaction/pg"
6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain" 6 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/domain"
  7 + "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/redis"
7 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository" 8 "gitlab.fjmaimaimai.com/allied-creation/character-library-metadata-bastion/pkg/infrastructure/repository"
8 ) 9 )
9 10
@@ -12,7 +13,7 @@ type EditDataTableService struct { @@ -12,7 +13,7 @@ type EditDataTableService struct {
12 } 13 }
13 14
14 // Edit 表结构编辑 【data-table】 15 // Edit 表结构编辑 【data-table】
15 -func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableRequest) (interface{}, error) { 16 +func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableRequest) (*domain.DataEditDataTable, error) {
16 fileRepository, _ := repository.NewFileRepository(ptr.transactionContext) 17 fileRepository, _ := repository.NewFileRepository(ptr.transactionContext)
17 file, err := fileRepository.FindOne(map[string]interface{}{"fileId": req.FileId}) 18 file, err := fileRepository.FindOne(map[string]interface{}{"fileId": req.FileId})
18 if err != nil { 19 if err != nil {
@@ -21,20 +22,7 @@ func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableR @@ -21,20 +22,7 @@ func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableR
21 if file.FileType != domain.TemporaryFile.ToString() { 22 if file.FileType != domain.TemporaryFile.ToString() {
22 return nil, fmt.Errorf("文件未校验") 23 return nil, fmt.Errorf("文件未校验")
23 } 24 }
24 - // TODO：操作名映射  
25 - var operateName string = req.Action  
26 - // 日志  
27 - if err = FastLog(ptr.transactionContext, domain.VerifiedStepLog, file.FileId, &ExcelTableEditLog{  
28 - LogEntry: domain.NewLogEntry(file.FileInfo.Name, domain.VerifiedFile.ToString(), domain.FileVerify,  
29 - ctx.WithValue(domain.ContextWithLogLevel, domain.LevelInfo).  
30 - WithValue(domain.ContextWithLogMsg, "")),  
31 - ProcessFields: req.ProcessFields,  
32 - OperateName: operateName,  
33 - }); err != nil {  
34 - return nil, err  
35 - }  
36 // 通知底层进行文件表编辑 25 // 通知底层进行文件表编辑
37 -  
38 response, err := ByteCore.EditTable(domain.ReqEditDataTable{ 26 response, err := ByteCore.EditTable(domain.ReqEditDataTable{
39 FileId: file.FileId, 27 FileId: file.FileId,
40 PageNumber: req.Where.PageNumber, 28 PageNumber: req.Where.PageNumber,
@@ -46,13 +34,77 @@ func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableR @@ -46,13 +34,77 @@ func (ptr *EditDataTableService) Edit(ctx *domain.Context, req domain.EditTableR
46 }) 34 })
47 if err != nil { 35 if err != nil {
48 return nil, err 36 return nil, err
  37 + //log.Logger.Error(err.Error())
  38 + }
  39 + // 处理错误
  40 + level := domain.LevelInfo
  41 + errMsg := ""
  42 + if len(response.InValidCells) > 0 {
  43 + level = domain.LevelError
  44 + errMsg = response.InValidCells[0].Err
  45 + }
  46 +
  47 + // 日志
  48 + var operateName string = actionName(req.Action, req.Params)
  49 + if err = FastLog(ptr.transactionContext, domain.VerifiedStepLog, file.FileId, &ExcelTableEditLog{
  50 + LogEntry: domain.NewLogEntry(file.FileInfo.Name, domain.VerifiedFile.ToString(), domain.FileVerify,
  51 + ctx.WithValue(domain.ContextWithLogLevel, level).
  52 + WithValue(domain.ContextWithLogMsg, errMsg)),
  53 + ProcessFields: req.ProcessFields,
  54 + OperateName: operateName,
  55 + }); err != nil {
  56 + return nil, err
49 } 57 }
50 58
51 // 1.有修改表类型的，更新缓存数据列类型 59 // 1.有修改表类型的，更新缓存数据列类型
  60 + if response != nil && len(response.Fields) > 0 {
  61 + // 特殊处理修改类型错误
  62 + options := make([]redis.FileCacheOptionsFunc, 0)
  63 + if req.Action == "convert-column-type" {
  64 + var toType = req.Params["convertType"].(string)
  65 + var fieldName = req.ProcessFieldNames[0]
  66 + if level == domain.LevelError {
  67 + options = append(options, redis.WithAddConvertTypeErrors([]redis.ConvertTypeError{{FieldName: fieldName, ErrMsg: errMsg, ToType: toType}}))
  68 + } else {
  69 + options = append(options, redis.WithRemoveConvertTypeErrors([]redis.ConvertTypeError{{FieldName: fieldName, ErrMsg: errMsg, ToType: toType}}))
  70 + }
  71 + }
52 72
  73 + cacheService := redis.NewFileCacheService()
  74 + if _, err := cacheService.Update(redis.KeyTemporaryFileInfo(file.FileId), file, response.Fields, response.Total, options...); err != nil {
  75 + return nil, err
  76 + }
  77 + }
53 return response, nil 78 return response, nil
54 } 79 }
55 80
  81 +func actionName(action string, params map[string]interface{}) string {
  82 + if v, ok := domain.MapActionCommon[action]; ok {
  83 + return v.Desc
  84 + }
  85 + if params != nil {
  86 + switch action {
  87 + case domain.FormatColumn:
  88 + if v, ok := domain.MapActionFormat[params["formatMethod"].(string)]; ok {
  89 + return v.Desc
  90 + }
  91 + case domain.SplitColumn:
  92 + if v, ok := domain.MapActionSplitColumn[params["splitMethod"].(string)]; ok {
  93 + return v.Desc
  94 + }
  95 + case domain.ExtractColumn:
  96 + if v, ok := domain.MapActionExtractColumn[params["extractMethod"].(string)]; ok {
  97 + return v.Desc
  98 + }
  99 + case domain.ReplaceColumn:
  100 + if v, ok := domain.MapActionReplaceColumn[params["replaceMethod"].(string)]; ok {
  101 + return v.Desc
  102 + }
  103 + }
  104 + }
  105 + return action
  106 +}
  107 +
56 func NewEditDataTableService(transactionContext *pgTransaction.TransactionContext) (*EditDataTableService, error) { 108 func NewEditDataTableService(transactionContext *pgTransaction.TransactionContext) (*EditDataTableService, error) {
57 if transactionContext == nil { 109 if transactionContext == nil {
58 return nil, fmt.Errorf("transactionContext参数不能为nil") 110 return nil, fmt.Errorf("transactionContext参数不能为nil")
@@ -149,6 +149,22 @@ func NewTable(tableType domain.TableType, fileName string, dataFields []*domain. @@ -149,6 +149,22 @@ func NewTable(tableType domain.TableType, fileName string, dataFields []*domain.
149 return table 149 return table
150 } 150 }
151 151
  152 +func NewCopyTable(tableType domain.TableType, fileName string, dataFields []*domain.Field, rowCount int) *domain.Table {
  153 + var table = &domain.Table{}
  154 + // New Table
  155 + table.TableType = tableType.ToString()
  156 + table.Name = fileName
  157 + table.SQLName = pin(fileName) //SQLTableName()
  158 + table.PK = PK()
  159 + table.DataFieldIndex = len(dataFields)
  160 + table.DataFields = dataFields
  161 + table.ManualFields = make([]*domain.Field, 0)
  162 + table.CreatedAt = time.Now()
  163 + table.UpdatedAt = time.Now()
  164 + table.RowCount = rowCount
  165 + return table
  166 +}
  167 +
152 func SQLTableName() string { 168 func SQLTableName() string {
153 id, _ := uuid.NewUUID() 169 id, _ := uuid.NewUUID()
154 return id.String() 170 return id.String()
@@ -21,21 +21,16 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie @@ -21,21 +21,16 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie
21 if err != nil { 21 if err != nil {
22 return nil, fmt.Errorf("校验文件不存在") 22 return nil, fmt.Errorf("校验文件不存在")
23 } 23 }
24 - isSourceFile := false 24 + isSourceFile := true
25 fileUrl := file.FileInfo.Url 25 fileUrl := file.FileInfo.Url
26 - // Copy to TemporaryFile  
27 if file.FileType != domain.TemporaryFile.ToString() { 26 if file.FileType != domain.TemporaryFile.ToString() {
28 - //file = file.CopyTo(domain.TemporaryFile, ctx)  
29 - //if file, err = fileRepository.Save(file); err != nil {  
30 - // return nil, err  
31 - //}  
32 - //isSourceFile = true  
33 - //fileUrl = file.FileInfo.Url  
34 return nil, fmt.Errorf("校验文件不存在") 27 return nil, fmt.Errorf("校验文件不存在")
35 } 28 }
36 ptr.FileId = file.FileId 29 ptr.FileId = file.FileId
37 - if len(fields) == 0 {  
38 - isSourceFile = true 30 + fileCache := redis.NewFileCacheService()
  31 + if tempFile, _ := fileCache.Get(redis.KeyTemporaryFileInfo(fileId)); tempFile != nil {
  32 + isSourceFile = false
  33 + fields = tempFile.Fields
39 } 34 }
40 35
41 // Load Data From Excel(python api) 36 // Load Data From Excel(python api)
@@ -58,12 +53,12 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie @@ -58,12 +53,12 @@ func (ptr *PreviewDataTableService) Preview(ctx *domain.Context, fileId int, fie
58 return nil, err 53 return nil, err
59 } 54 }
60 cache := redis.NewFileCacheService() 55 cache := redis.NewFileCacheService()
61 - _, err = cache.Update(redis.KeyTemporaryFileInfo(file.FileId), file, response.Fields, response.Total) 56 + tempFile, err := cache.Update(redis.KeyTemporaryFileInfo(file.FileId), file, response.Fields, response.Total)
62 if err != nil { 57 if err != nil {
63 return nil, err 58 return nil, err
64 } 59 }
65 var responseDto = &FilePreviewDto{} 60 var responseDto = &FilePreviewDto{}
66 - responseDto.Load(file.FileId, response) 61 + responseDto.Load(file.FileId, response, tempFile)
67 return responseDto, nil 62 return responseDto, nil
68 } 63 }
69 64
@@ -93,16 +88,27 @@ type FilePreviewDto struct { @@ -93,16 +88,27 @@ type FilePreviewDto struct {
93 InValidCells []domain.InValidCell `json:"inValidCells"` 88 InValidCells []domain.InValidCell `json:"inValidCells"`
94 } 89 }
95 90
96 -func (d *FilePreviewDto) Load(fileId int, m *domain.DataLoadDataTable) { 91 +func (d *FilePreviewDto) Load(fileId int, m *domain.DataLoadDataTable, file *redis.TemporaryFileInfo) {
97 d.ObjectId = fileId 92 d.ObjectId = fileId
98 d.ObjectType = domain.ObjectFile 93 d.ObjectType = domain.ObjectFile
99 d.TableType = domain.ExcelTable.ToString() 94 d.TableType = domain.ExcelTable.ToString()
100 d.Fields = m.Fields 95 d.Fields = m.Fields
101 - 96 + var fields []*domain.Field
102 mapData := domain.ToFieldData(m.Fields, m.Data, true) 97 mapData := domain.ToFieldData(m.Fields, m.Data, true)
103 d.Data = domain.GripData(mapData, int64(m.Total)) 98 d.Data = domain.GripData(mapData, int64(m.Total))
104 d.PageNumber = m.PageNumber 99 d.PageNumber = m.PageNumber
105 - d.InValidCells = domain.NewInValidCells(m.Fields, mapData) 100 +
  101 + for _, f := range m.Fields {
  102 + copyField := f.Copy()
  103 + for _, e := range file.ConvertTypeErrors {
  104 + if e.FieldName == copyField.Name {
  105 + copyField.SQLType = e.ToType
  106 + }
  107 + }
  108 + fields = append(fields, copyField)
  109 + }
  110 +
  111 + d.InValidCells = domain.NewInValidCells(fields, mapData)
106 } 112 }
107 113
108 func (ptr *PreviewDataTableService) GetFileId() int { 114 func (ptr *PreviewDataTableService) GetFileId() int {
@@ -48,7 +48,7 @@ func (ptr *PGLogService) Log(logType domain.LogType, sourceId int, logEntry Log) @@ -48,7 +48,7 @@ func (ptr *PGLogService) Log(logType domain.LogType, sourceId int, logEntry Log)
48 if v, ok := logEntry.Context().GetValue(domain.ContextWithLogLevel); ok { 48 if v, ok := logEntry.Context().GetValue(domain.ContextWithLogLevel); ok {
49 log.Entry.Level = string(v.(domain.LogLevel)) 49 log.Entry.Level = string(v.(domain.LogLevel))
50 } 50 }
51 - if v, ok := logEntry.Context().GetValue(domain.ContextWithLogLevel); ok { 51 + if v, ok := logEntry.Context().GetValue(domain.ContextWithLogMsg); ok {
52 log.Entry.Error = v.(string) 52 log.Entry.Error = v.(string)
53 } 53 }
54 _, err := logRepository.Save(log) 54 _, err := logRepository.Save(log)
@@ -28,8 +28,15 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab @@ -28,8 +28,15 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab
28 if err != nil { 28 if err != nil {
29 return nil, err 29 return nil, err
30 } 30 }
31 - if table.TableType != domain.MainTable.ToString() {  
32 - return nil, fmt.Errorf("主表才允许复制") 31 + if !(table.TableType == domain.MainTable.ToString() || table.TableType == domain.SubTable.ToString()) {
  32 + return nil, fmt.Errorf("主表、分表才允许复制")
  33 + }
  34 + var mainTable *domain.Table
  35 + if table.TableType == domain.SubTable.ToString() {
  36 + mainTable, err = tableRepository.FindOne(map[string]interface{}{"tableId": table.ParentId})
  37 + if err != nil {
  38 + return nil, err
  39 + }
33 } 40 }
34 // 验证表名是否重复 41 // 验证表名是否重复
35 duplicateTable, err := tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableName": tableName}) 42 duplicateTable, err := tableRepository.FindOne(map[string]interface{}{"context": ctx, "tableName": tableName})
@@ -37,8 +44,10 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab @@ -37,8 +44,10 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab
37 return nil, fmt.Errorf("表名称重复") 44 return nil, fmt.Errorf("表名称重复")
38 } 45 }
39 46
40 - sideTable := NewTable(domain.SideTable, tableName, table.DataFields, table.RowCount).WithContext(ctx)  
41 - sideTable.ParentId = table.TableId 47 + sideTable := NewCopyTable(domain.SideTable, tableName, table.DataFields, table.RowCount).
  48 + WithContext(ctx).
  49 + WithParentId(table.TableId).
  50 + WithDataFieldIndex(table.DataFieldIndex)
42 if sideTable, err = tableRepository.Save(sideTable); err != nil { 51 if sideTable, err = tableRepository.Save(sideTable); err != nil {
43 return nil, err 52 return nil, err
44 } 53 }
@@ -51,7 +60,7 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab @@ -51,7 +60,7 @@ func (ptr *CopyDataTableService) CopyTable(ctx *domain.Context, tableId int, tab
51 } 60 }
52 61
53 // 通知底层 62 // 通知底层
54 - if _, err = ByteCore.CopyTable(domain.ReqCopyTable{Table: table, CopyToTable: sideTable}); err != nil { 63 + if _, err = ByteCore.CopyTable(domain.ReqCopyTable{Table: table, MainTable: mainTable, CopyToTable: sideTable}); err != nil {
55 return nil, err 64 return nil, err
56 } 65 }
57 return nil, nil 66 return nil, nil
@@ -24,6 +24,18 @@ type TemporaryFileInfo struct { @@ -24,6 +24,18 @@ type TemporaryFileInfo struct {
24 // 行记录错误 24 // 行记录错误
25 // 列类型有错误 25 // 列类型有错误
26 // 表整体错误 26 // 表整体错误
  27 + ConvertTypeErrors []ConvertTypeError `json:"convertTypeErrors"`
  28 +}
  29 +
  30 +func (f *TemporaryFileInfo) MatchFields(columns []string) []*domain.Field {
  31 + mapFields := (domain.Fields)(f.Fields).ToMap()
  32 + var result = make([]*domain.Field, 0)
  33 + for _, c := range columns {
  34 + if v, ok := mapFields[c]; ok {
  35 + result = append(result, v)
  36 + }
  37 + }
  38 + return result
27 } 39 }
28 40
29 func (f *TemporaryFileInfo) SetFile(file *domain.File) *TemporaryFileInfo { 41 func (f *TemporaryFileInfo) SetFile(file *domain.File) *TemporaryFileInfo {
@@ -42,10 +54,40 @@ func (f *TemporaryFileInfo) SetTotal(total int) *TemporaryFileInfo { @@ -42,10 +54,40 @@ func (f *TemporaryFileInfo) SetTotal(total int) *TemporaryFileInfo {
42 return f 54 return f
43 } 55 }
44 56
  57 +func (f *TemporaryFileInfo) AddConvertTypeError(e ConvertTypeError) *TemporaryFileInfo {
  58 + f.RemoveConvertTypeError(e)
  59 + f.addConvertTypeError(e)
  60 + return f
  61 +}
  62 +
  63 +func (f *TemporaryFileInfo) addConvertTypeError(e ConvertTypeError) *TemporaryFileInfo {
  64 + f.RemoveConvertTypeError(e)
  65 + f.ConvertTypeErrors = append(f.ConvertTypeErrors, e)
  66 + return f
  67 +}
  68 +
  69 +func (f *TemporaryFileInfo) RemoveConvertTypeError(e ConvertTypeError) *TemporaryFileInfo {
  70 + var newErrors = make([]ConvertTypeError, 0)
  71 + for _, item := range f.ConvertTypeErrors {
  72 + if item.FieldName == e.FieldName {
  73 + continue
  74 + }
  75 + newErrors = append(newErrors, item)
  76 + }
  77 + f.ConvertTypeErrors = newErrors
  78 + return f
  79 +}
  80 +
  81 +type ConvertTypeError struct {
  82 + FieldName string `json:"fieldName"`
  83 + ErrMsg string `json:"errMsg"`
  84 + ToType string `json:"toType"`
  85 +}
  86 +
45 type FileCacheService struct { 87 type FileCacheService struct {
46 } 88 }
47 89
48 -func (s *FileCacheService) Update(key string, file *domain.File, fields []*domain.Field, total int) (*TemporaryFileInfo, error) { 90 +func (s *FileCacheService) Update(key string, file *domain.File, fields []*domain.Field, total int, errors ...FileCacheOptionsFunc) (*TemporaryFileInfo, error) {
49 ok, err := ZeroCoreRedis.Exists(key) 91 ok, err := ZeroCoreRedis.Exists(key)
50 var response = &TemporaryFileInfo{} 92 var response = &TemporaryFileInfo{}
51 if err != nil { 93 if err != nil {
@@ -64,6 +106,15 @@ func (s *FileCacheService) Update(key string, file *domain.File, fields []*domai @@ -64,6 +106,15 @@ func (s *FileCacheService) Update(key string, file *domain.File, fields []*domai
64 return nil, err 106 return nil, err
65 } 107 }
66 response.SetFields(fields) 108 response.SetFields(fields)
  109 +
  110 + options := NewFileCacheOptions(errors...)
  111 + for i := range options.AddConvertTypeErrors {
  112 + response.AddConvertTypeError(options.AddConvertTypeErrors[i])
  113 + }
  114 + for i := range options.RemoveConvertTypeErrors {
  115 + response.RemoveConvertTypeError(options.RemoveConvertTypeErrors[i])
  116 + }
  117 +
67 err = ZeroCoreRedis.Setex(key, json.MarshalToString(response), TemporaryFileExpire) 118 err = ZeroCoreRedis.Setex(key, json.MarshalToString(response), TemporaryFileExpire)
68 if err != nil { 119 if err != nil {
69 return nil, err 120 return nil, err
@@ -94,3 +145,30 @@ func (s *FileCacheService) Get(key string) (*TemporaryFileInfo, error) { @@ -94,3 +145,30 @@ func (s *FileCacheService) Get(key string) (*TemporaryFileInfo, error) {
94 func NewFileCacheService() *FileCacheService { 145 func NewFileCacheService() *FileCacheService {
95 return &FileCacheService{} 146 return &FileCacheService{}
96 } 147 }
  148 +
  149 +type FileCacheOptions struct {
  150 + RemoveConvertTypeErrors []ConvertTypeError
  151 + AddConvertTypeErrors []ConvertTypeError
  152 +}
  153 +
  154 +type FileCacheOptionsFunc func(o *FileCacheOptions)
  155 +
  156 +func WithRemoveConvertTypeErrors(errors []ConvertTypeError) FileCacheOptionsFunc {
  157 + return func(o *FileCacheOptions) {
  158 + o.RemoveConvertTypeErrors = errors
  159 + }
  160 +}
  161 +
  162 +func WithAddConvertTypeErrors(errors []ConvertTypeError) FileCacheOptionsFunc {
  163 + return func(o *FileCacheOptions) {
  164 + o.AddConvertTypeErrors = errors
  165 + }
  166 +}
  167 +
  168 +func NewFileCacheOptions(options ...FileCacheOptionsFunc) *FileCacheOptions {
  169 + option := &FileCacheOptions{}
  170 + for i := range options {
  171 + options[i](option)
  172 + }
  173 + return option
  174 +}
@@ -118,7 +118,7 @@ func (repository *FileRepository) FindOne(queryOptions map[string]interface{}) ( @@ -118,7 +118,7 @@ func (repository *FileRepository) FindOne(queryOptions map[string]interface{}) (
118 query.SetWhereByQueryOption("file.file_id = ?", "fileId") 118 query.SetWhereByQueryOption("file.file_id = ?", "fileId")
119 if err := query.First(); err != nil { 119 if err := query.First(); err != nil {
120 if err.Error() == "pg: no rows in result set" { 120 if err.Error() == "pg: no rows in result set" {
121 - return nil, fmt.Errorf("没有此资源") 121 + return nil, domain.ErrorNotFound
122 } else { 122 } else {
123 return nil, err 123 return nil, err
124 } 124 }
@@ -125,7 +125,7 @@ func (repository *LogRepository) FindOne(queryOptions map[string]interface{}) (* @@ -125,7 +125,7 @@ func (repository *LogRepository) FindOne(queryOptions map[string]interface{}) (*
125 query.SetWhereByQueryOption("log.log_id = ?", "logId") 125 query.SetWhereByQueryOption("log.log_id = ?", "logId")
126 if err := query.First(); err != nil { 126 if err := query.First(); err != nil {
127 if err.Error() == "pg: no rows in result set" { 127 if err.Error() == "pg: no rows in result set" {
128 - return nil, fmt.Errorf("没有此资源") 128 + return nil, domain.ErrorNotFound
129 } else { 129 } else {
130 return nil, err 130 return nil, err
131 } 131 }
@@ -126,7 +126,7 @@ func (repository *MappingRuleRepository) FindOne(queryOptions map[string]interfa @@ -126,7 +126,7 @@ func (repository *MappingRuleRepository) FindOne(queryOptions map[string]interfa
126 query.SetWhereByQueryOption("mapping_rule.mapping_rule_id = ?", "mappingRuleId") 126 query.SetWhereByQueryOption("mapping_rule.mapping_rule_id = ?", "mappingRuleId")
127 if err := query.First(); err != nil { 127 if err := query.First(); err != nil {
128 if err.Error() == "pg: no rows in result set" { 128 if err.Error() == "pg: no rows in result set" {
129 - return nil, fmt.Errorf("没有此资源") 129 + return nil, domain.ErrorNotFound
130 } else { 130 } else {
131 return nil, err 131 return nil, err
132 } 132 }
@@ -147,7 +147,7 @@ func (repository *TableRepository) FindOne(queryOptions map[string]interface{}) @@ -147,7 +147,7 @@ func (repository *TableRepository) FindOne(queryOptions map[string]interface{})
147 query.SetWhereByQueryOption("parent_id = ?", "parentId") 147 query.SetWhereByQueryOption("parent_id = ?", "parentId")
148 if err := query.First(); err != nil { 148 if err := query.First(); err != nil {
149 if err.Error() == "pg: no rows in result set" { 149 if err.Error() == "pg: no rows in result set" {
150 - return nil, fmt.Errorf("没有此资源") 150 + return nil, domain.ErrorNotFound
151 } else { 151 } else {
152 return nil, err 152 return nil, err
153 } 153 }