mmm-go / go-stash · 提交

转到一个项目

作者 kevin
提交 9bb8dde7dd7b3b49106b372fef8d1045cf91cb69 1 个父辈 8dbc1a56

support multiple topics

内嵌 并排对比
正在显示 6 个修改的文件 包含 89 行增加 和 71 行删除
1 module github.com/tal-tech/go-stash 1 module github.com/tal-tech/go-stash
2 2
3 -go 1.14 3 +go 1.13
4 4
5 require ( 5 require (
6 github.com/fortytw2/leaktest v1.3.0 // indirect 6 github.com/fortytw2/leaktest v1.3.0 // indirect
@@ -9,5 +9,5 @@ require ( @@ -9,5 +9,5 @@ require (
9 github.com/mailru/easyjson v0.7.3 // indirect 9 github.com/mailru/easyjson v0.7.3 // indirect
10 github.com/olivere/elastic v6.2.34+incompatible 10 github.com/olivere/elastic v6.2.34+incompatible
11 github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328 11 github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328
12 - github.com/tal-tech/go-zero v1.0.8 12 + github.com/tal-tech/go-zero v1.0.12
13 ) 13 )
1 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 1 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
2 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
  3 +github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
3 github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= 4 github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
4 github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= 5 github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
5 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 6 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -59,6 +60,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 @@ -59,6 +60,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
59 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 60 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
60 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= 61 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
61 github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 62 github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
  63 +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
62 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 64 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
63 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 65 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
64 github.com/go-xorm/builder v0.3.4/go.mod h1:KxkQkNN1DpPKTedxXyTQcmH+rXfvk4LZ9SOOBoZBAxw= 66 github.com/go-xorm/builder v0.3.4/go.mod h1:KxkQkNN1DpPKTedxXyTQcmH+rXfvk4LZ9SOOBoZBAxw=
@@ -109,6 +111,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6 @@ -109,6 +111,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6
109 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 111 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
110 github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= 112 github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
111 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 113 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
  114 +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
112 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 115 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
113 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 116 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
114 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 117 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -137,6 +140,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -137,6 +140,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
137 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 140 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
138 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 141 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
139 github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc= 142 github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
  143 +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
140 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 144 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
141 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= 145 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
142 github.com/mailru/easyjson v0.7.3 h1:M6wcO9gFHCIPynXGu4iA+NMs//FCgFUWR2jxqV3/+Xk= 146 github.com/mailru/easyjson v0.7.3 h1:M6wcO9gFHCIPynXGu4iA+NMs//FCgFUWR2jxqV3/+Xk=
@@ -151,6 +155,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky @@ -151,6 +155,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
151 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 155 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
152 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 156 github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
153 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 157 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
  158 +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
154 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 159 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
155 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 160 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
156 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 161 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -219,6 +224,8 @@ github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328 h1:rWeU2ZWx/Jwmd @@ -219,6 +224,8 @@ github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328 h1:rWeU2ZWx/Jwmd
219 github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328/go.mod h1:vffND44z/ta5szc5rD0cEAK7qphrgT2rrTxghQ1JWDE= 224 github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328/go.mod h1:vffND44z/ta5szc5rD0cEAK7qphrgT2rrTxghQ1JWDE=
220 github.com/tal-tech/go-zero v1.0.8 h1:Wca6UVi5+Pr1GOFCpvcB178EzlecLlKEgGb3v8Ln1dQ= 225 github.com/tal-tech/go-zero v1.0.8 h1:Wca6UVi5+Pr1GOFCpvcB178EzlecLlKEgGb3v8Ln1dQ=
221 github.com/tal-tech/go-zero v1.0.8/go.mod h1:/e0i8rMFzFO6Lha+UG9/nkzLSvv5dyYCCN+TFP1JcB0= 226 github.com/tal-tech/go-zero v1.0.8/go.mod h1:/e0i8rMFzFO6Lha+UG9/nkzLSvv5dyYCCN+TFP1JcB0=
  227 +github.com/tal-tech/go-zero v1.0.12 h1:ymdKjhNOBaO5Ddqa/XVo4qRwMNsB3LOKWj6UdFJN5rc=
  228 +github.com/tal-tech/go-zero v1.0.12/go.mod h1:y2wBHTkxNJw79K9/wCSeDKzv2pCT6x45oOmXEsJdQK8=
222 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 229 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
223 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 230 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
224 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 231 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -28,7 +28,7 @@ type ( @@ -28,7 +28,7 @@ type (
28 Fields []string `json:",optional"` 28 Fields []string `json:",optional"`
29 } 29 }
30 30
31 - Config struct { 31 + Processor struct {
32 Input struct { 32 Input struct {
33 Kafka kq.KqConf 33 Kafka kq.KqConf
34 } 34 }
@@ -36,6 +36,10 @@ type ( @@ -36,6 +36,10 @@ type (
36 Output struct { 36 Output struct {
37 ElasticSearch ElasticSearchConf 37 ElasticSearch ElasticSearchConf
38 } 38 }
  39 + }
  40 +
  41 + Config struct {
  42 + Processors []Processor
39 GracePeriod time.Duration `json:",default=10s"` 43 GracePeriod time.Duration `json:",default=10s"`
40 } 44 }
41 ) 45 )
1 ----  
2 -Input:  
3 - Kafka:  
4 - Name: gostash  
5 - Brokers:  
6 - - 172.16.186.16:19092  
7 - - 172.16.186.17:19092  
8 - Topic: k8slog  
9 - Group: pro  
10 - NumProducers: 16  
11 -Filters:  
12 - - Action: drop  
13 - Conditions:  
14 - - Key: k8s_container_name  
15 - Value: "-rpc"  
16 - Type: contains  
17 - - Key: level  
18 - Value: info  
19 - Type: match  
20 - Op: and  
21 - - Action: remove_field  
22 - Fields:  
23 - - message  
24 - - _source  
25 - - _type  
26 - - _score  
27 - - _id  
28 - - "@version"  
29 - - topic  
30 - - index  
31 - - beat  
32 - - docker_container  
33 - - offset  
34 - - prospector  
35 - - source  
36 - - stream  
37 -Output:  
38 - ElasticSearch:  
39 - Hosts:  
40 - - 172.16.141.4:9200  
41 - - 172.16.141.5:9200  
42 - DailyIndexPrefix: k8s_pro- 1 +Processors:
  2 +- Input:
  3 + Kafka:
  4 + Name: gostash
  5 + Brokers:
  6 + - ":172.16.186.16:19092"
  7 + - "172.16.186.17:19092"
  8 + Topic: k8slog
  9 + Group: pro
  10 + NumProducers: 16
  11 + Filters:
  12 + - Action: drop
  13 + Conditions:
  14 + - Key: k8s_container_name
  15 + Value: "-rpc"
  16 + Type: contains
  17 + - Key: level
  18 + Value: info
  19 + Type: match
  20 + Op: and
  21 + - Action: remove_field
  22 + Fields:
  23 + - message
  24 + - _source
  25 + - _type
  26 + - _score
  27 + - _id
  28 + - "@version"
  29 + - topic
  30 + - index
  31 + - beat
  32 + - docker_container
  33 + - offset
  34 + - prospector
  35 + - source
  36 + - stream
  37 + Output:
  38 + ElasticSearch:
  39 + Hosts:
  40 + - "172.16.141.4:9200"
  41 + - "172.16.141.5:9200"
  42 + DailyIndexPrefix: "k8s_pro-"
@@ -13,10 +13,10 @@ const ( @@ -13,10 +13,10 @@ const (
13 13
14 type FilterFunc func(map[string]interface{}) map[string]interface{} 14 type FilterFunc func(map[string]interface{}) map[string]interface{}
15 15
16 -func CreateFilters(c config.Config) []FilterFunc { 16 +func CreateFilters(p config.Processor) []FilterFunc {
17 var filters []FilterFunc 17 var filters []FilterFunc
18 18
19 - for _, f := range c.Filters { 19 + for _, f := range p.Filters {
20 switch f.Action { 20 switch f.Action {
21 case filterDrop: 21 case filterDrop:
22 filters = append(filters, DropFilter(f.Conditions)) 22 filters = append(filters, DropFilter(f.Conditions))
@@ -13,6 +13,7 @@ import ( @@ -13,6 +13,7 @@ import (
13 "github.com/tal-tech/go-zero/core/conf" 13 "github.com/tal-tech/go-zero/core/conf"
14 "github.com/tal-tech/go-zero/core/logx" 14 "github.com/tal-tech/go-zero/core/logx"
15 "github.com/tal-tech/go-zero/core/proc" 15 "github.com/tal-tech/go-zero/core/proc"
  16 + "github.com/tal-tech/go-zero/core/service"
16 ) 17 )
17 18
18 const dateFormat = "2006.01.02" 19 const dateFormat = "2006.01.02"
@@ -26,31 +27,37 @@ func main() { @@ -26,31 +27,37 @@ func main() {
26 conf.MustLoad(*configFile, &c) 27 conf.MustLoad(*configFile, &c)
27 proc.SetTimeoutToForceQuit(c.GracePeriod) 28 proc.SetTimeoutToForceQuit(c.GracePeriod)
28 29
29 - client, err := elastic.NewClient(  
30 - elastic.SetSniff(false),  
31 - elastic.SetURL(c.Output.ElasticSearch.Hosts...),  
32 - )  
33 - logx.Must(err) 30 + group := service.NewServiceGroup()
  31 + defer group.Stop()
34 32
35 - indexFormat := c.Output.ElasticSearch.DailyIndexPrefix + dateFormat  
36 - var loc *time.Location  
37 - if len(c.Output.ElasticSearch.TimeZone) > 0 {  
38 - loc, err = time.LoadLocation(c.Output.ElasticSearch.TimeZone) 33 + for _, processor := range c.Processors {
  34 + client, err := elastic.NewClient(
  35 + elastic.SetSniff(false),
  36 + elastic.SetURL(processor.Output.ElasticSearch.Hosts...),
  37 + )
39 logx.Must(err) 38 logx.Must(err)
40 - } else {  
41 - loc = time.Local 39 +
  40 + indexFormat := processor.Output.ElasticSearch.DailyIndexPrefix + dateFormat
  41 + var loc *time.Location
  42 + if len(processor.Output.ElasticSearch.TimeZone) > 0 {
  43 + loc, err = time.LoadLocation(processor.Output.ElasticSearch.TimeZone)
  44 + logx.Must(err)
  45 + } else {
  46 + loc = time.Local
  47 + }
  48 + indexer := es.NewIndex(client, func(t time.Time) string {
  49 + return t.In(loc).Format(indexFormat)
  50 + })
  51 +
  52 + filters := filter.CreateFilters(processor)
  53 + writer, err := es.NewWriter(processor.Output.ElasticSearch, indexer)
  54 + logx.Must(err)
  55 +
  56 + handle := handler.NewHandler(writer)
  57 + handle.AddFilters(filters...)
  58 + handle.AddFilters(filter.AddUriFieldFilter("url", "uri"))
  59 + group.Add(kq.MustNewQueue(processor.Input.Kafka, handle))
42 } 60 }
43 - indexer := es.NewIndex(client, func(t time.Time) string {  
44 - return t.In(loc).Format(indexFormat)  
45 - })  
46 -  
47 - filters := filter.CreateFilters(c)  
48 - writer, err := es.NewWriter(c.Output.ElasticSearch, indexer)  
49 - logx.Must(err)  
50 -  
51 - handle := handler.NewHandler(writer)  
52 - handle.AddFilters(filters...)  
53 - handle.AddFilters(filter.AddUriFieldFilter("url", "uri"))  
54 - q := kq.MustNewQueue(c.Input.Kafka, handle)  
55 - q.Start() 61 +
  62 + group.Start()
56 } 63 }