mmm-go / go-stash · 提交

转到一个项目

作者 kevin
提交 b14d515d26300f9a6badb8d464fd024e602df201 1 个父辈 6a6052f6

rename to match producer/consumer in kafka

内嵌 并排对比
正在显示 7 个修改的文件 包含 22 行增加 和 16 行删除
@@ -3,13 +3,14 @@ module github.com/tal-tech/go-stash @@ -3,13 +3,14 @@ module github.com/tal-tech/go-stash
3 go 1.13 3 go 1.13
4 4
5 require ( 5 require (
  6 + github.com/dsymonds/gotoc v0.0.0-20160928043926-5aebcfc91819 // indirect
6 github.com/fortytw2/leaktest v1.3.0 // indirect 7 github.com/fortytw2/leaktest v1.3.0 // indirect
7 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 8 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
8 github.com/json-iterator/go v1.1.10 9 github.com/json-iterator/go v1.1.10
9 github.com/mailru/easyjson v0.7.3 // indirect 10 github.com/mailru/easyjson v0.7.3 // indirect
10 github.com/olivere/elastic v6.2.34+incompatible 11 github.com/olivere/elastic v6.2.34+incompatible
11 github.com/stretchr/testify v1.5.1 12 github.com/stretchr/testify v1.5.1
12 - github.com/tal-tech/go-queue v1.0.1  
13 - github.com/tal-tech/go-zero v1.0.20 13 + github.com/tal-tech/go-queue v1.0.2
  14 + github.com/tal-tech/go-zero v1.0.21
14 github.com/vjeantet/jodaTime v1.0.0 15 github.com/vjeantet/jodaTime v1.0.0
15 ) 16 )
@@ -40,6 +40,7 @@ github.com/dsymonds/gotoc v0.0.0-20160928043926-5aebcfc91819/go.mod h1:MvzMVHq8B @@ -40,6 +40,7 @@ github.com/dsymonds/gotoc v0.0.0-20160928043926-5aebcfc91819/go.mod h1:MvzMVHq8B
40 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 40 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
41 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= 41 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
42 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 42 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
  43 +github.com/emicklei/proto v1.9.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A=
43 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 44 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
44 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 45 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
45 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 46 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -229,6 +230,8 @@ github.com/tal-tech/go-queue v1.0.0 h1:FRRHuWbSUwyfXpQgK4rNa+9iJT0jN6+WBj5OjZWst @@ -229,6 +230,8 @@ github.com/tal-tech/go-queue v1.0.0 h1:FRRHuWbSUwyfXpQgK4rNa+9iJT0jN6+WBj5OjZWst
229 github.com/tal-tech/go-queue v1.0.0/go.mod h1:vffND44z/ta5szc5rD0cEAK7qphrgT2rrTxghQ1JWDE= 230 github.com/tal-tech/go-queue v1.0.0/go.mod h1:vffND44z/ta5szc5rD0cEAK7qphrgT2rrTxghQ1JWDE=
230 github.com/tal-tech/go-queue v1.0.1 h1:TSWauhHGxOX1lAQ5p2xQn86eI1GtamiP0kAIQAiS8DY= 231 github.com/tal-tech/go-queue v1.0.1 h1:TSWauhHGxOX1lAQ5p2xQn86eI1GtamiP0kAIQAiS8DY=
231 github.com/tal-tech/go-queue v1.0.1/go.mod h1:V3cpvoQDKfVpPDvCiPDAZTdR/6RYuFhNGhJeXl4ak9w= 232 github.com/tal-tech/go-queue v1.0.1/go.mod h1:V3cpvoQDKfVpPDvCiPDAZTdR/6RYuFhNGhJeXl4ak9w=
  233 +github.com/tal-tech/go-queue v1.0.2 h1:A9V321RMHNITfRvmIvKrBAK3BfoykDNooAttk7pUg1o=
  234 +github.com/tal-tech/go-queue v1.0.2/go.mod h1:gQK4Eg8pqel8Z9r1hjlSXbJFavLeJQVyTSwBKeAnpm8=
232 github.com/tal-tech/go-zero v1.0.8 h1:Wca6UVi5+Pr1GOFCpvcB178EzlecLlKEgGb3v8Ln1dQ= 235 github.com/tal-tech/go-zero v1.0.8 h1:Wca6UVi5+Pr1GOFCpvcB178EzlecLlKEgGb3v8Ln1dQ=
233 github.com/tal-tech/go-zero v1.0.8/go.mod h1:/e0i8rMFzFO6Lha+UG9/nkzLSvv5dyYCCN+TFP1JcB0= 236 github.com/tal-tech/go-zero v1.0.8/go.mod h1:/e0i8rMFzFO6Lha+UG9/nkzLSvv5dyYCCN+TFP1JcB0=
234 github.com/tal-tech/go-zero v1.0.12 h1:ymdKjhNOBaO5Ddqa/XVo4qRwMNsB3LOKWj6UdFJN5rc= 237 github.com/tal-tech/go-zero v1.0.12 h1:ymdKjhNOBaO5Ddqa/XVo4qRwMNsB3LOKWj6UdFJN5rc=
@@ -239,6 +242,8 @@ github.com/tal-tech/go-zero v1.0.14 h1:a8vH2pMCEt+sackLDZEIYL/u7DhRY7SBbN04ATgzU @@ -239,6 +242,8 @@ github.com/tal-tech/go-zero v1.0.14 h1:a8vH2pMCEt+sackLDZEIYL/u7DhRY7SBbN04ATgzU
239 github.com/tal-tech/go-zero v1.0.14/go.mod h1:y2wBHTkxNJw79K9/wCSeDKzv2pCT6x45oOmXEsJdQK8= 242 github.com/tal-tech/go-zero v1.0.14/go.mod h1:y2wBHTkxNJw79K9/wCSeDKzv2pCT6x45oOmXEsJdQK8=
240 github.com/tal-tech/go-zero v1.0.20 h1:BluZPak0minwoNqhSvtlBuwjDK3Q7Vgke8YSQJS2S+w= 243 github.com/tal-tech/go-zero v1.0.20 h1:BluZPak0minwoNqhSvtlBuwjDK3Q7Vgke8YSQJS2S+w=
241 github.com/tal-tech/go-zero v1.0.20/go.mod h1:NgINotJQNboHp9OrqhNzgmk8WbklEkx13tvT4MSPVe4= 244 github.com/tal-tech/go-zero v1.0.20/go.mod h1:NgINotJQNboHp9OrqhNzgmk8WbklEkx13tvT4MSPVe4=
  245 +github.com/tal-tech/go-zero v1.0.21 h1:IB0c6zmkhUERcagSLPrlVT4oA74yE+l2w2K1ujW0sY8=
  246 +github.com/tal-tech/go-zero v1.0.21/go.mod h1:llP5PQjnATfnzZo/lo5unjR41njzoL3lkGO/KXbnisw=
242 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 247 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
243 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 248 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
244 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 249 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -17,7 +17,7 @@ gostash -f etc/config.yaml @@ -17,7 +17,7 @@ gostash -f etc/config.yaml
17 config.yaml example as below: 17 config.yaml example as below:
18 18
19 ```yaml 19 ```yaml
20 -Processors: 20 +Clusters:
21 - Input: 21 - Input:
22 Kafka: 22 Kafka:
23 Name: gostash 23 Name: gostash
@@ -27,7 +27,7 @@ Processors: @@ -27,7 +27,7 @@ Processors:
27 Topics: 27 Topics:
28 - k8slog 28 - k8slog
29 Group: pro 29 Group: pro
30 - NumProducers: 16 30 + Consumers: 16
31 Filters: 31 Filters:
32 - Action: drop 32 - Action: drop
33 Conditions: 33 Conditions:
@@ -37,14 +37,14 @@ type ( @@ -37,14 +37,14 @@ type (
37 Group string 37 Group string
38 Topics []string 38 Topics []string
39 Offset string `json:",options=first|last,default=last"` 39 Offset string `json:",options=first|last,default=last"`
40 - NumConns int `json:",default=1"`  
41 - NumProducers int `json:",default=8"`  
42 - NumConsumers int `json:",default=8"` 40 + Conns int `json:",default=1"`
  41 + Consumers int `json:",default=8"`
  42 + Processors int `json:",default=8"`
43 MinBytes int `json:",default=10240"` // 10K 43 MinBytes int `json:",default=10240"` // 10K
44 MaxBytes int `json:",default=10485760"` // 10M 44 MaxBytes int `json:",default=10485760"` // 10M
45 } 45 }
46 46
47 - Processor struct { 47 + Cluster struct {
48 Input struct { 48 Input struct {
49 Kafka KafkaConf 49 Kafka KafkaConf
50 } 50 }
@@ -55,7 +55,7 @@ type ( @@ -55,7 +55,7 @@ type (
55 } 55 }
56 56
57 Config struct { 57 Config struct {
58 - Processors []Processor 58 + Clusters []Cluster
59 GracePeriod time.Duration `json:",default=10s"` 59 GracePeriod time.Duration `json:",default=10s"`
60 } 60 }
61 ) 61 )
1 -Processors: 1 +Clusters:
2 - Input: 2 - Input:
3 Kafka: 3 Kafka:
4 Name: gostash 4 Name: gostash
@@ -8,7 +8,7 @@ Processors: @@ -8,7 +8,7 @@ Processors:
8 Topics: 8 Topics:
9 - k8slog 9 - k8slog
10 Group: pro 10 Group: pro
11 - NumProducers: 16 11 + Consumers: 16
12 Filters: 12 Filters:
13 - Action: drop 13 - Action: drop
14 Conditions: 14 Conditions:
@@ -14,7 +14,7 @@ const ( @@ -14,7 +14,7 @@ const (
14 14
15 type FilterFunc func(map[string]interface{}) map[string]interface{} 15 type FilterFunc func(map[string]interface{}) map[string]interface{}
16 16
17 -func CreateFilters(p config.Processor) []FilterFunc { 17 +func CreateFilters(p config.Cluster) []FilterFunc {
18 var filters []FilterFunc 18 var filters []FilterFunc
19 19
20 for _, f := range p.Filters { 20 for _, f := range p.Filters {
@@ -28,9 +28,9 @@ func toKqConf(c config.KafkaConf) []kq.KqConf { @@ -28,9 +28,9 @@ func toKqConf(c config.KafkaConf) []kq.KqConf {
28 Group: c.Group, 28 Group: c.Group,
29 Topic: topic, 29 Topic: topic,
30 Offset: c.Offset, 30 Offset: c.Offset,
31 - NumConns: c.NumConns,  
32 - NumProducers: c.NumProducers,  
33 - NumConsumers: c.NumConsumers, 31 + Conns: c.Conns,
  32 + Consumers: c.Consumers,
  33 + Processors: c.Processors,
34 MinBytes: c.MinBytes, 34 MinBytes: c.MinBytes,
35 MaxBytes: c.MaxBytes, 35 MaxBytes: c.MaxBytes,
36 }) 36 })
@@ -49,7 +49,7 @@ func main() { @@ -49,7 +49,7 @@ func main() {
49 group := service.NewServiceGroup() 49 group := service.NewServiceGroup()
50 defer group.Stop() 50 defer group.Stop()
51 51
52 - for _, processor := range c.Processors { 52 + for _, processor := range c.Clusters {
53 client, err := elastic.NewClient( 53 client, err := elastic.NewClient(
54 elastic.SetSniff(false), 54 elastic.SetSniff(false),
55 elastic.SetURL(processor.Output.ElasticSearch.Hosts...), 55 elastic.SetURL(processor.Output.ElasticSearch.Hosts...),