mmm-go / go-stash · 提交

转到一个项目

作者 kevin
提交 9bb8dde7dd7b3b49106b372fef8d1045cf91cb69 1 个父辈 8dbc1a56

support multiple topics

内嵌 并排对比
正在显示 6 个修改的文件 包含 40 行增加 和 22 行删除
1 module github.com/tal-tech/go-stash 1 module github.com/tal-tech/go-stash
2 2
3 -go 1.14 3 +go 1.13
4 4
5 require ( 5 require (
6 github.com/fortytw2/leaktest v1.3.0 // indirect 6 github.com/fortytw2/leaktest v1.3.0 // indirect
@@ -9,5 +9,5 @@ require ( @@ -9,5 +9,5 @@ require (
9 github.com/mailru/easyjson v0.7.3 // indirect 9 github.com/mailru/easyjson v0.7.3 // indirect
10 github.com/olivere/elastic v6.2.34+incompatible 10 github.com/olivere/elastic v6.2.34+incompatible
11 github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328 11 github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328
12 - github.com/tal-tech/go-zero v1.0.8 12 + github.com/tal-tech/go-zero v1.0.12
13 ) 13 )
1 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 1 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
2 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
  3 +github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
3 github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= 4 github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
4 github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= 5 github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
5 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 6 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -59,6 +60,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 @@ -59,6 +60,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
59 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 60 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
60 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= 61 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
61 github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 62 github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
  63 +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
62 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 64 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
63 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 65 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
64 github.com/go-xorm/builder v0.3.4/go.mod h1:KxkQkNN1DpPKTedxXyTQcmH+rXfvk4LZ9SOOBoZBAxw= 66 github.com/go-xorm/builder v0.3.4/go.mod h1:KxkQkNN1DpPKTedxXyTQcmH+rXfvk4LZ9SOOBoZBAxw=
@@ -109,6 +111,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6 @@ -109,6 +111,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6
109 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 111 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
110 github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= 112 github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
111 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 113 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
  114 +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
112 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 115 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
113 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 116 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
114 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 117 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -137,6 +140,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -137,6 +140,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
137 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 140 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
138 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 141 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
139 github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc= 142 github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
  143 +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
140 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 144 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
141 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= 145 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
142 github.com/mailru/easyjson v0.7.3 h1:M6wcO9gFHCIPynXGu4iA+NMs//FCgFUWR2jxqV3/+Xk= 146 github.com/mailru/easyjson v0.7.3 h1:M6wcO9gFHCIPynXGu4iA+NMs//FCgFUWR2jxqV3/+Xk=
@@ -151,6 +155,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky @@ -151,6 +155,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
151 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 155 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
152 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 156 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
153 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 157 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
  158 +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
154 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 159 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
155 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 160 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
156 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 161 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -219,6 +224,8 @@ github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328 h1:rWeU2ZWx/Jwmd @@ -219,6 +224,8 @@ github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328 h1:rWeU2ZWx/Jwmd
219 github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328/go.mod h1:vffND44z/ta5szc5rD0cEAK7qphrgT2rrTxghQ1JWDE= 224 github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328/go.mod h1:vffND44z/ta5szc5rD0cEAK7qphrgT2rrTxghQ1JWDE=
220 github.com/tal-tech/go-zero v1.0.8 h1:Wca6UVi5+Pr1GOFCpvcB178EzlecLlKEgGb3v8Ln1dQ= 225 github.com/tal-tech/go-zero v1.0.8 h1:Wca6UVi5+Pr1GOFCpvcB178EzlecLlKEgGb3v8Ln1dQ=
221 github.com/tal-tech/go-zero v1.0.8/go.mod h1:/e0i8rMFzFO6Lha+UG9/nkzLSvv5dyYCCN+TFP1JcB0= 226 github.com/tal-tech/go-zero v1.0.8/go.mod h1:/e0i8rMFzFO6Lha+UG9/nkzLSvv5dyYCCN+TFP1JcB0=
  227 +github.com/tal-tech/go-zero v1.0.12 h1:ymdKjhNOBaO5Ddqa/XVo4qRwMNsB3LOKWj6UdFJN5rc=
  228 +github.com/tal-tech/go-zero v1.0.12/go.mod h1:y2wBHTkxNJw79K9/wCSeDKzv2pCT6x45oOmXEsJdQK8=
222 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 229 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
223 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 230 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
224 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 231 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -28,7 +28,7 @@ type ( @@ -28,7 +28,7 @@ type (
28 Fields []string `json:",optional"` 28 Fields []string `json:",optional"`
29 } 29 }
30 30
31 - Config struct { 31 + Processor struct {
32 Input struct { 32 Input struct {
33 Kafka kq.KqConf 33 Kafka kq.KqConf
34 } 34 }
@@ -36,6 +36,10 @@ type ( @@ -36,6 +36,10 @@ type (
36 Output struct { 36 Output struct {
37 ElasticSearch ElasticSearchConf 37 ElasticSearch ElasticSearchConf
38 } 38 }
  39 + }
  40 +
  41 + Config struct {
  42 + Processors []Processor
39 GracePeriod time.Duration `json:",default=10s"` 43 GracePeriod time.Duration `json:",default=10s"`
40 } 44 }
41 ) 45 )
1 ----  
2 -Input: 1 +Processors:
  2 +- Input:
3 Kafka: 3 Kafka:
4 Name: gostash 4 Name: gostash
5 Brokers: 5 Brokers:
6 - - 172.16.186.16:19092  
7 - - 172.16.186.17:19092 6 + - ":172.16.186.16:19092"
  7 + - "172.16.186.17:19092"
8 Topic: k8slog 8 Topic: k8slog
9 Group: pro 9 Group: pro
10 NumProducers: 16 10 NumProducers: 16
11 -Filters: 11 + Filters:
12 - Action: drop 12 - Action: drop
13 Conditions: 13 Conditions:
14 - Key: k8s_container_name 14 - Key: k8s_container_name
@@ -34,9 +34,9 @@ Filters: @@ -34,9 +34,9 @@ Filters:
34 - prospector 34 - prospector
35 - source 35 - source
36 - stream 36 - stream
37 -Output: 37 + Output:
38 ElasticSearch: 38 ElasticSearch:
39 Hosts: 39 Hosts:
40 - - 172.16.141.4:9200  
41 - - 172.16.141.5:9200  
42 - DailyIndexPrefix: k8s_pro- 40 + - "172.16.141.4:9200"
  41 + - "172.16.141.5:9200"
  42 + DailyIndexPrefix: "k8s_pro-"
@@ -13,10 +13,10 @@ const ( @@ -13,10 +13,10 @@ const (
13 13
14 type FilterFunc func(map[string]interface{}) map[string]interface{} 14 type FilterFunc func(map[string]interface{}) map[string]interface{}
15 15
16 -func CreateFilters(c config.Config) []FilterFunc { 16 +func CreateFilters(p config.Processor) []FilterFunc {
17 var filters []FilterFunc 17 var filters []FilterFunc
18 18
19 - for _, f := range c.Filters { 19 + for _, f := range p.Filters {
20 switch f.Action { 20 switch f.Action {
21 case filterDrop: 21 case filterDrop:
22 filters = append(filters, DropFilter(f.Conditions)) 22 filters = append(filters, DropFilter(f.Conditions))
@@ -13,6 +13,7 @@ import ( @@ -13,6 +13,7 @@ import (
13 "github.com/tal-tech/go-zero/core/conf" 13 "github.com/tal-tech/go-zero/core/conf"
14 "github.com/tal-tech/go-zero/core/logx" 14 "github.com/tal-tech/go-zero/core/logx"
15 "github.com/tal-tech/go-zero/core/proc" 15 "github.com/tal-tech/go-zero/core/proc"
  16 + "github.com/tal-tech/go-zero/core/service"
16 ) 17 )
17 18
18 const dateFormat = "2006.01.02" 19 const dateFormat = "2006.01.02"
@@ -26,16 +27,20 @@ func main() { @@ -26,16 +27,20 @@ func main() {
26 conf.MustLoad(*configFile, &c) 27 conf.MustLoad(*configFile, &c)
27 proc.SetTimeoutToForceQuit(c.GracePeriod) 28 proc.SetTimeoutToForceQuit(c.GracePeriod)
28 29
  30 + group := service.NewServiceGroup()
  31 + defer group.Stop()
  32 +
  33 + for _, processor := range c.Processors {
29 client, err := elastic.NewClient( 34 client, err := elastic.NewClient(
30 elastic.SetSniff(false), 35 elastic.SetSniff(false),
31 - elastic.SetURL(c.Output.ElasticSearch.Hosts...), 36 + elastic.SetURL(processor.Output.ElasticSearch.Hosts...),
32 ) 37 )
33 logx.Must(err) 38 logx.Must(err)
34 39
35 - indexFormat := c.Output.ElasticSearch.DailyIndexPrefix + dateFormat 40 + indexFormat := processor.Output.ElasticSearch.DailyIndexPrefix + dateFormat
36 var loc *time.Location 41 var loc *time.Location
37 - if len(c.Output.ElasticSearch.TimeZone) > 0 {  
38 - loc, err = time.LoadLocation(c.Output.ElasticSearch.TimeZone) 42 + if len(processor.Output.ElasticSearch.TimeZone) > 0 {
  43 + loc, err = time.LoadLocation(processor.Output.ElasticSearch.TimeZone)
39 logx.Must(err) 44 logx.Must(err)
40 } else { 45 } else {
41 loc = time.Local 46 loc = time.Local
@@ -44,13 +49,15 @@ func main() { @@ -44,13 +49,15 @@ func main() {
44 return t.In(loc).Format(indexFormat) 49 return t.In(loc).Format(indexFormat)
45 }) 50 })
46 51
47 - filters := filter.CreateFilters(c)  
48 - writer, err := es.NewWriter(c.Output.ElasticSearch, indexer) 52 + filters := filter.CreateFilters(processor)
  53 + writer, err := es.NewWriter(processor.Output.ElasticSearch, indexer)
49 logx.Must(err) 54 logx.Must(err)
50 55
51 handle := handler.NewHandler(writer) 56 handle := handler.NewHandler(writer)
52 handle.AddFilters(filters...) 57 handle.AddFilters(filters...)
53 handle.AddFilters(filter.AddUriFieldFilter("url", "uri")) 58 handle.AddFilters(filter.AddUriFieldFilter("url", "uri"))
54 - q := kq.MustNewQueue(c.Input.Kafka, handle)  
55 - q.Start() 59 + group.Add(kq.MustNewQueue(processor.Input.Kafka, handle))
  60 + }
  61 +
  62 + group.Start()
56 } 63 }