mmm-go / go-stash · 提交

转到一个项目

作者 kevin
提交 66b19124ed625c8b6560fad363d0729cc9b2a527 1 个父辈 bf301e65

use kq from go-queue

内嵌 并排对比
正在显示 8 个修改的文件 包含 20 行增加 和 413 行删除
@@ -3,11 +3,13 @@ module github.com/tal-tech/go-stash @@ -3,11 +3,13 @@ module github.com/tal-tech/go-stash
3 go 1.14 3 go 1.14
4 4
5 require ( 5 require (
  6 + github.com/DataDog/zstd v1.4.0 // indirect
6 github.com/fortytw2/leaktest v1.3.0 // indirect 7 github.com/fortytw2/leaktest v1.3.0 // indirect
7 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 8 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
8 github.com/json-iterator/go v1.1.10 9 github.com/json-iterator/go v1.1.10
9 github.com/mailru/easyjson v0.7.3 // indirect 10 github.com/mailru/easyjson v0.7.3 // indirect
10 github.com/olivere/elastic v6.2.34+incompatible 11 github.com/olivere/elastic v6.2.34+incompatible
11 - github.com/segmentio/kafka-go v0.3.5  
12 - github.com/tal-tech/go-zero v1.0.3 12 + github.com/segmentio/kafka-go v0.4.2
  13 + github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328
  14 + github.com/tal-tech/go-zero v1.0.8
13 ) 15 )
@@ -11,6 +11,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF @@ -11,6 +11,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
11 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= 11 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
12 github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= 12 github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
13 github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= 13 github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
  14 +github.com/beanstalkd/go-beanstalk v0.1.0/go.mod h1:/G8YTyChOtpOArwLTQPY1CHB+i212+av35bkPXXj56Y=
14 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 15 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
15 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 16 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
16 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 17 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -36,6 +37,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c @@ -36,6 +37,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
36 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 37 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
37 github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= 38 github.com/dchest/siphash v1.2.1/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
38 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 39 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
  40 +github.com/dsymonds/gotoc v0.0.0-20160928043926-5aebcfc91819/go.mod h1:MvzMVHq8BH2Ji/o8TGDocVA70byvLrAgFTxkEnmjO4Y=
39 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 41 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
40 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= 42 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
41 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 43 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
@@ -61,6 +63,8 @@ github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dT @@ -61,6 +63,8 @@ github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dT
61 github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 63 github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
62 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 64 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
63 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 65 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
  66 +github.com/go-xorm/builder v0.3.4/go.mod h1:KxkQkNN1DpPKTedxXyTQcmH+rXfvk4LZ9SOOBoZBAxw=
  67 +github.com/go-xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:56xuuqnHyryaerycW3BfssRdxQstACi0Epw/yC5E2xM=
64 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 68 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
65 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 69 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
66 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 70 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -105,6 +109,7 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf @@ -105,6 +109,7 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
105 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 109 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
106 github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= 110 github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0=
107 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 111 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
  112 +github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
108 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 113 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
109 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 114 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
110 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 115 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -122,6 +127,8 @@ github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2F @@ -122,6 +127,8 @@ github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2F
122 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 127 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
123 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 128 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
124 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 129 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
  130 +github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=
  131 +github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
125 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 132 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
126 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 133 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
127 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 134 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -193,6 +200,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR @@ -193,6 +200,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
193 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 200 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
194 github.com/segmentio/kafka-go v0.3.5 h1:2JVT1inno7LxEASWj+HflHh5sWGfM0gkRiLAxkXhGG4= 201 github.com/segmentio/kafka-go v0.3.5 h1:2JVT1inno7LxEASWj+HflHh5sWGfM0gkRiLAxkXhGG4=
195 github.com/segmentio/kafka-go v0.3.5/go.mod h1:OT5KXBPbaJJTcvokhWR2KFmm0niEx3mnccTwjmLvSi4= 202 github.com/segmentio/kafka-go v0.3.5/go.mod h1:OT5KXBPbaJJTcvokhWR2KFmm0niEx3mnccTwjmLvSi4=
  203 +github.com/segmentio/kafka-go v0.4.2 h1:QXZ6q9Bu1JkAJQ/CQBb2Av8pFRG8LQ0kWCrLXgQyL8c=
  204 +github.com/segmentio/kafka-go v0.4.2/go.mod h1:Inh7PqOsxmfgasV8InZYKVXWsdjcCq2d9tFV75GLbuM=
196 github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= 205 github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
197 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= 206 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
198 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 207 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -210,8 +219,12 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV @@ -210,8 +219,12 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
210 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 219 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
211 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 220 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
212 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 221 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
  222 +github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328 h1:rWeU2ZWx/Jwmd77IQtd7k3RMZIh8e7i4yzztrt4kPLM=
  223 +github.com/tal-tech/go-queue v0.0.0-20200901073541-0da84ebed328/go.mod h1:vffND44z/ta5szc5rD0cEAK7qphrgT2rrTxghQ1JWDE=
213 github.com/tal-tech/go-zero v1.0.3 h1:B7yo8yMh27ZfEpjM7s6tPKGaLtSBufIPxTPMwjsiDxI= 224 github.com/tal-tech/go-zero v1.0.3 h1:B7yo8yMh27ZfEpjM7s6tPKGaLtSBufIPxTPMwjsiDxI=
214 github.com/tal-tech/go-zero v1.0.3/go.mod h1:MZHttkq0yf8kUd3TGaGvwUpiRIw9eD2iNFFfkx4YgG4= 225 github.com/tal-tech/go-zero v1.0.3/go.mod h1:MZHttkq0yf8kUd3TGaGvwUpiRIw9eD2iNFFfkx4YgG4=
  226 +github.com/tal-tech/go-zero v1.0.8 h1:Wca6UVi5+Pr1GOFCpvcB178EzlecLlKEgGb3v8Ln1dQ=
  227 +github.com/tal-tech/go-zero v1.0.8/go.mod h1:/e0i8rMFzFO6Lha+UG9/nkzLSvv5dyYCCN+TFP1JcB0=
215 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 228 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
216 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 229 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
217 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 230 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -222,6 +235,7 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= @@ -222,6 +235,7 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
222 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 235 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
223 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 236 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
224 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= 237 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
  238 +github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY=
225 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 239 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
226 github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= 240 github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
227 go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= 241 go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
1 -package kq  
2 -  
3 -import "github.com/tal-tech/go-zero/core/service"  
4 -  
5 -const (  
6 - firstOffset = "first"  
7 - lastOffset = "last"  
8 -)  
9 -  
10 -type KqConf struct {  
11 - service.ServiceConf  
12 - Brokers []string  
13 - Group string  
14 - Topic string  
15 - Offset string `json:",options=first|last,default=last"`  
16 - NumConns int `json:",default=1"`  
17 - NumProducers int `json:",default=8"`  
18 - NumConsumers int `json:",default=8"`  
19 - MinBytes int `json:",default=10240"` // 10K  
20 - MaxBytes int `json:",default=10485760"` // 10M  
21 -}  
1 -package kq  
2 -  
3 -import (  
4 - "context"  
5 - "strconv"  
6 - "time"  
7 -  
8 - "github.com/segmentio/kafka-go"  
9 - "github.com/segmentio/kafka-go/snappy"  
10 - "github.com/tal-tech/go-zero/core/executors"  
11 - "github.com/tal-tech/go-zero/core/logx"  
12 -)  
13 -  
14 -type (  
15 - PushOption func(options *chunkOptions)  
16 -  
17 - Pusher struct {  
18 - produer *kafka.Writer  
19 - topic string  
20 - executor *executors.ChunkExecutor  
21 - }  
22 -  
23 - chunkOptions struct {  
24 - chunkSize int  
25 - flushInterval time.Duration  
26 - }  
27 -)  
28 -  
29 -func NewPusher(addrs []string, topic string, opts ...PushOption) *Pusher {  
30 - producer := kafka.NewWriter(kafka.WriterConfig{  
31 - Brokers: addrs,  
32 - Topic: topic,  
33 - Balancer: &kafka.LeastBytes{},  
34 - CompressionCodec: snappy.NewCompressionCodec(),  
35 - })  
36 -  
37 - pusher := &Pusher{  
38 - produer: producer,  
39 - topic: topic,  
40 - }  
41 - pusher.executor = executors.NewChunkExecutor(func(tasks []interface{}) {  
42 - chunk := make([]kafka.Message, len(tasks))  
43 - for i := range tasks {  
44 - chunk[i] = tasks[i].(kafka.Message)  
45 - }  
46 - if err := pusher.produer.WriteMessages(context.Background(), chunk...); err != nil {  
47 - logx.Error(err)  
48 - }  
49 - }, newOptions(opts)...)  
50 -  
51 - return pusher  
52 -}  
53 -  
54 -func (p *Pusher) Close() error {  
55 - return p.produer.Close()  
56 -}  
57 -  
58 -func (p *Pusher) Name() string {  
59 - return p.topic  
60 -}  
61 -  
62 -func (p *Pusher) Push(v string) error {  
63 - msg := kafka.Message{  
64 - Key: []byte(strconv.FormatInt(time.Now().UnixNano(), 10)),  
65 - Value: []byte(v),  
66 - }  
67 - if p.executor != nil {  
68 - return p.executor.Add(msg, len(v))  
69 - } else {  
70 - return p.produer.WriteMessages(context.Background(), msg)  
71 - }  
72 -}  
73 -  
74 -func WithChunkSize(chunkSize int) PushOption {  
75 - return func(options *chunkOptions) {  
76 - options.chunkSize = chunkSize  
77 - }  
78 -}  
79 -  
80 -func WithFlushInterval(interval time.Duration) PushOption {  
81 - return func(options *chunkOptions) {  
82 - options.flushInterval = interval  
83 - }  
84 -}  
85 -  
86 -func newOptions(opts []PushOption) []executors.ChunkOption {  
87 - var options chunkOptions  
88 - for _, opt := range opts {  
89 - opt(&options)  
90 - }  
91 -  
92 - var chunkOpts []executors.ChunkOption  
93 - if options.chunkSize > 0 {  
94 - chunkOpts = append(chunkOpts, executors.WithChunkBytes(options.chunkSize))  
95 - }  
96 - if options.flushInterval > 0 {  
97 - chunkOpts = append(chunkOpts, executors.WithFlushInterval(options.flushInterval))  
98 - }  
99 - return chunkOpts  
100 -}  
1 -package kq  
2 -  
3 -import (  
4 - "context"  
5 - "io"  
6 - "log"  
7 - "time"  
8 -  
9 - "github.com/segmentio/kafka-go"  
10 - _ "github.com/segmentio/kafka-go/gzip"  
11 - _ "github.com/segmentio/kafka-go/lz4"  
12 - _ "github.com/segmentio/kafka-go/snappy"  
13 - "github.com/tal-tech/go-zero/core/logx"  
14 - "github.com/tal-tech/go-zero/core/queue"  
15 - "github.com/tal-tech/go-zero/core/service"  
16 - "github.com/tal-tech/go-zero/core/stat"  
17 - "github.com/tal-tech/go-zero/core/threading"  
18 - "github.com/tal-tech/go-zero/core/timex"  
19 -)  
20 -  
21 -const (  
22 - defaultCommitInterval = time.Second  
23 - defaultMaxWait = time.Second  
24 -)  
25 -  
26 -type (  
27 - ConsumeHandle func(key, value string) error  
28 -  
29 - ConsumeHandler interface {  
30 - Consume(key, value string) error  
31 - }  
32 -  
33 - queueOptions struct {  
34 - commitInterval time.Duration  
35 - maxWait time.Duration  
36 - metrics *stat.Metrics  
37 - }  
38 -  
39 - QueueOption func(*queueOptions)  
40 -  
41 - kafkaQueue struct {  
42 - c KqConf  
43 - consumer *kafka.Reader  
44 - handler ConsumeHandler  
45 - channel chan kafka.Message  
46 - producerRoutines *threading.RoutineGroup  
47 - consumerRoutines *threading.RoutineGroup  
48 - metrics *stat.Metrics  
49 - }  
50 -  
51 - kafkaQueues struct {  
52 - queues []queue.MessageQueue  
53 - group *service.ServiceGroup  
54 - }  
55 -)  
56 -  
57 -func MustNewQueue(c KqConf, handler ConsumeHandler, opts ...QueueOption) queue.MessageQueue {  
58 - q, err := NewQueue(c, handler, opts...)  
59 - if err != nil {  
60 - log.Fatal(err)  
61 - }  
62 -  
63 - return q  
64 -}  
65 -  
66 -func NewQueue(c KqConf, handler ConsumeHandler, opts ...QueueOption) (queue.MessageQueue, error) {  
67 - if err := c.SetUp(); err != nil {  
68 - return nil, err  
69 - }  
70 -  
71 - var options queueOptions  
72 - for _, opt := range opts {  
73 - opt(&options)  
74 - }  
75 - ensureQueueOptions(c, &options)  
76 -  
77 - if c.NumConns < 1 {  
78 - c.NumConns = 1  
79 - }  
80 - q := kafkaQueues{  
81 - group: service.NewServiceGroup(),  
82 - }  
83 - for i := 0; i < c.NumConns; i++ {  
84 - q.queues = append(q.queues, newKafkaQueue(c, handler, options))  
85 - }  
86 -  
87 - return q, nil  
88 -}  
89 -  
90 -func newKafkaQueue(c KqConf, handler ConsumeHandler, options queueOptions) queue.MessageQueue {  
91 - var offset int64  
92 - if c.Offset == firstOffset {  
93 - offset = kafka.FirstOffset  
94 - } else {  
95 - offset = kafka.LastOffset  
96 - }  
97 - consumer := kafka.NewReader(kafka.ReaderConfig{  
98 - Brokers: c.Brokers,  
99 - GroupID: c.Group,  
100 - Topic: c.Topic,  
101 - StartOffset: offset,  
102 - MinBytes: c.MinBytes, // 10KB  
103 - MaxBytes: c.MaxBytes, // 10MB  
104 - MaxWait: options.maxWait,  
105 - CommitInterval: options.commitInterval,  
106 - })  
107 -  
108 - return &kafkaQueue{  
109 - c: c,  
110 - consumer: consumer,  
111 - handler: handler,  
112 - channel: make(chan kafka.Message),  
113 - producerRoutines: threading.NewRoutineGroup(),  
114 - consumerRoutines: threading.NewRoutineGroup(),  
115 - metrics: options.metrics,  
116 - }  
117 -}  
118 -  
119 -func (q *kafkaQueue) Start() {  
120 - q.startConsumers()  
121 - q.startProducers()  
122 -  
123 - q.producerRoutines.Wait()  
124 - close(q.channel)  
125 - q.consumerRoutines.Wait()  
126 -}  
127 -  
128 -func (q *kafkaQueue) Stop() {  
129 - q.consumer.Close()  
130 - logx.Close()  
131 -}  
132 -  
133 -func (q *kafkaQueue) consumeOne(key, val string) error {  
134 - startTime := timex.Now()  
135 - err := q.handler.Consume(key, val)  
136 - q.metrics.Add(stat.Task{  
137 - Duration: timex.Since(startTime),  
138 - })  
139 - return err  
140 -}  
141 -  
142 -func (q *kafkaQueue) startConsumers() {  
143 - for i := 0; i < q.c.NumConsumers; i++ {  
144 - q.consumerRoutines.Run(func() {  
145 - for msg := range q.channel {  
146 - if err := q.consumeOne(string(msg.Key), string(msg.Value)); err != nil {  
147 - logx.Errorf("Error on consuming: %s, error: %v", string(msg.Value), err)  
148 - }  
149 - }  
150 - })  
151 - }  
152 -}  
153 -  
154 -func (q *kafkaQueue) startProducers() {  
155 - for i := 0; i < q.c.NumProducers; i++ {  
156 - q.producerRoutines.Run(func() {  
157 - for {  
158 - msg, err := q.consumer.ReadMessage(context.Background())  
159 - // io.EOF means consumer closed  
160 - // io.ErrClosedPipe means committing messages on the consumer,  
161 - // kafka will refire the messages on uncommitted messages, ignore  
162 - if err == io.EOF || err == io.ErrClosedPipe {  
163 - return  
164 - }  
165 - if err != nil {  
166 - logx.Errorf("Error on reading mesage, %q", err.Error())  
167 - continue  
168 - }  
169 - q.channel <- msg  
170 - }  
171 - })  
172 - }  
173 -}  
174 -  
175 -func (q kafkaQueues) Start() {  
176 - for _, each := range q.queues {  
177 - q.group.Add(each)  
178 - }  
179 - q.group.Start()  
180 -}  
181 -  
182 -func (q kafkaQueues) Stop() {  
183 - q.group.Stop()  
184 -}  
185 -  
186 -func WithCommitInterval(interval time.Duration) QueueOption {  
187 - return func(options *queueOptions) {  
188 - options.commitInterval = interval  
189 - }  
190 -}  
191 -  
192 -func WithHandle(handle ConsumeHandle) ConsumeHandler {  
193 - return innerConsumeHandler{  
194 - handle: handle,  
195 - }  
196 -}  
197 -  
198 -func WithMaxWait(wait time.Duration) QueueOption {  
199 - return func(options *queueOptions) {  
200 - options.maxWait = wait  
201 - }  
202 -}  
203 -  
204 -func WithMetrics(metrics *stat.Metrics) QueueOption {  
205 - return func(options *queueOptions) {  
206 - options.metrics = metrics  
207 - }  
208 -}  
209 -  
210 -type innerConsumeHandler struct {  
211 - handle ConsumeHandle  
212 -}  
213 -  
214 -func (ch innerConsumeHandler) Consume(k, v string) error {  
215 - return ch.handle(k, v)  
216 -}  
217 -  
218 -func ensureQueueOptions(c KqConf, options *queueOptions) {  
219 - if options.commitInterval == 0 {  
220 - options.commitInterval = defaultCommitInterval  
221 - }  
222 - if options.maxWait == 0 {  
223 - options.maxWait = defaultMaxWait  
224 - }  
225 - if options.metrics == nil {  
226 - options.metrics = stat.NewMetrics(c.Name)  
227 - }  
228 -}  
@@ -3,7 +3,7 @@ package config @@ -3,7 +3,7 @@ package config
3 import ( 3 import (
4 "time" 4 "time"
5 5
6 - "github.com/tal-tech/go-stash/kq" 6 + "github.com/tal-tech/go-queue/kq"
7 ) 7 )
8 8
9 type ( 9 type (
1 -{  
2 - "Input": {  
3 - "Kafka": {  
4 - "Name": "gostash",  
5 - "Brokers": [  
6 - "172.16.186.16:19092",  
7 - "172.16.186.17:19092"  
8 - ],  
9 - "Topic": "k8slog",  
10 - "Group": "pro",  
11 - "NumProducers": 16  
12 - }  
13 - },  
14 - "Filters": [  
15 - {  
16 - "Action": "drop",  
17 - "Conditions": [  
18 - {  
19 - "Key": "k8s_container_name",  
20 - "Value": "-rpc",  
21 - "Type": "contains"  
22 - },  
23 - {  
24 - "Key": "level",  
25 - "Value": "info",  
26 - "Type": "match",  
27 - "Op": "and"  
28 - }  
29 - ]  
30 - },  
31 - {  
32 - "Action": "remove_field",  
33 - "Fields": [  
34 - "message",  
35 - "_source",  
36 - "_type",  
37 - "_score",  
38 - "_id",  
39 - "@version",  
40 - "topic",  
41 - "index",  
42 - "beat",  
43 - "docker_container",  
44 - "offset",  
45 - "prospector",  
46 - "source",  
47 - "stream"  
48 - ]  
49 - }  
50 - ],  
51 - "Output": {  
52 - "ElasticSearch": {  
53 - "Hosts": [  
54 - "172.16.141.4:9200",  
55 - "172.16.141.5:9200"  
56 - ],  
57 - "DailyIndexPrefix": "k8s_pro-"  
58 - }  
59 - }  
60 -}  
@@ -5,7 +5,7 @@ import ( @@ -5,7 +5,7 @@ import (
5 "time" 5 "time"
6 6
7 "github.com/olivere/elastic" 7 "github.com/olivere/elastic"
8 - "github.com/tal-tech/go-stash/kq" 8 + "github.com/tal-tech/go-queue/kq"
9 "github.com/tal-tech/go-stash/stash/config" 9 "github.com/tal-tech/go-stash/stash/config"
10 "github.com/tal-tech/go-stash/stash/es" 10 "github.com/tal-tech/go-stash/stash/es"
11 "github.com/tal-tech/go-stash/stash/filter" 11 "github.com/tal-tech/go-stash/stash/filter"