正在显示
3 个修改的文件
包含
18 行增加
和
20 行删除
@@ -3,7 +3,6 @@ package es | @@ -3,7 +3,6 @@ package es | ||
3 | import ( | 3 | import ( |
4 | "context" | 4 | "context" |
5 | 5 | ||
6 | - jsoniter "github.com/json-iterator/go" | ||
7 | "github.com/olivere/elastic" | 6 | "github.com/olivere/elastic" |
8 | "github.com/tal-tech/go-stash/stash/config" | 7 | "github.com/tal-tech/go-stash/stash/config" |
9 | "github.com/tal-tech/go-zero/core/executors" | 8 | "github.com/tal-tech/go-zero/core/executors" |
@@ -14,7 +13,6 @@ type ( | @@ -14,7 +13,6 @@ type ( | ||
14 | Writer struct { | 13 | Writer struct { |
15 | docType string | 14 | docType string |
16 | client *elastic.Client | 15 | client *elastic.Client |
17 | - indexer *Index | ||
18 | inserter *executors.ChunkExecutor | 16 | inserter *executors.ChunkExecutor |
19 | } | 17 | } |
20 | 18 | ||
@@ -24,7 +22,7 @@ type ( | @@ -24,7 +22,7 @@ type ( | ||
24 | } | 22 | } |
25 | ) | 23 | ) |
26 | 24 | ||
27 | -func NewWriter(c config.ElasticSearchConf, indexer *Index) (*Writer, error) { | 25 | +func NewWriter(c config.ElasticSearchConf) (*Writer, error) { |
28 | client, err := elastic.NewClient( | 26 | client, err := elastic.NewClient( |
29 | elastic.SetSniff(false), | 27 | elastic.SetSniff(false), |
30 | elastic.SetURL(c.Hosts...), | 28 | elastic.SetURL(c.Hosts...), |
@@ -37,20 +35,12 @@ func NewWriter(c config.ElasticSearchConf, indexer *Index) (*Writer, error) { | @@ -37,20 +35,12 @@ func NewWriter(c config.ElasticSearchConf, indexer *Index) (*Writer, error) { | ||
37 | writer := Writer{ | 35 | writer := Writer{ |
38 | docType: c.DocType, | 36 | docType: c.DocType, |
39 | client: client, | 37 | client: client, |
40 | - indexer: indexer, | ||
41 | } | 38 | } |
42 | writer.inserter = executors.NewChunkExecutor(writer.execute, executors.WithChunkBytes(c.MaxChunkBytes)) | 39 | writer.inserter = executors.NewChunkExecutor(writer.execute, executors.WithChunkBytes(c.MaxChunkBytes)) |
43 | return &writer, nil | 40 | return &writer, nil |
44 | } | 41 | } |
45 | 42 | ||
46 | -func (w *Writer) Write(m map[string]interface{}) error { | ||
47 | - bs, err := jsoniter.Marshal(m) | ||
48 | - if err != nil { | ||
49 | - return err | ||
50 | - } | ||
51 | - | ||
52 | - index := w.indexer.GetIndex(m) | ||
53 | - val := string(bs) | 43 | +func (w *Writer) Write(index, val string) error { |
54 | return w.inserter.Add(valueWithIndex{ | 44 | return w.inserter.Add(valueWithIndex{ |
55 | index: index, | 45 | index: index, |
56 | val: val, | 46 | val: val, |
@@ -8,12 +8,14 @@ import ( | @@ -8,12 +8,14 @@ import ( | ||
8 | 8 | ||
9 | type MessageHandler struct { | 9 | type MessageHandler struct { |
10 | writer *es.Writer | 10 | writer *es.Writer |
11 | + indexer *es.Index | ||
11 | filters []filter.FilterFunc | 12 | filters []filter.FilterFunc |
12 | } | 13 | } |
13 | 14 | ||
14 | -func NewHandler(writer *es.Writer) *MessageHandler { | 15 | +func NewHandler(writer *es.Writer, indexer *es.Index) *MessageHandler { |
15 | return &MessageHandler{ | 16 | return &MessageHandler{ |
16 | - writer: writer, | 17 | + writer: writer, |
18 | + indexer: indexer, | ||
17 | } | 19 | } |
18 | } | 20 | } |
19 | 21 | ||
@@ -29,11 +31,17 @@ func (mh *MessageHandler) Consume(_, val string) error { | @@ -29,11 +31,17 @@ func (mh *MessageHandler) Consume(_, val string) error { | ||
29 | return err | 31 | return err |
30 | } | 32 | } |
31 | 33 | ||
34 | + index := mh.indexer.GetIndex(m) | ||
32 | for _, proc := range mh.filters { | 35 | for _, proc := range mh.filters { |
33 | if m = proc(m); m == nil { | 36 | if m = proc(m); m == nil { |
34 | return nil | 37 | return nil |
35 | } | 38 | } |
36 | } | 39 | } |
37 | 40 | ||
38 | - return mh.writer.Write(m) | 41 | + bs, err := jsoniter.Marshal(m) |
42 | + if err != nil { | ||
43 | + return err | ||
44 | + } | ||
45 | + | ||
46 | + return mh.writer.Write(index, string(bs)) | ||
39 | } | 47 | } |
@@ -35,6 +35,10 @@ func main() { | @@ -35,6 +35,10 @@ func main() { | ||
35 | ) | 35 | ) |
36 | logx.Must(err) | 36 | logx.Must(err) |
37 | 37 | ||
38 | + filters := filter.CreateFilters(processor) | ||
39 | + writer, err := es.NewWriter(processor.Output.ElasticSearch) | ||
40 | + logx.Must(err) | ||
41 | + | ||
38 | var loc *time.Location | 42 | var loc *time.Location |
39 | if len(processor.Output.ElasticSearch.TimeZone) > 0 { | 43 | if len(processor.Output.ElasticSearch.TimeZone) > 0 { |
40 | loc, err = time.LoadLocation(processor.Output.ElasticSearch.TimeZone) | 44 | loc, err = time.LoadLocation(processor.Output.ElasticSearch.TimeZone) |
@@ -43,11 +47,7 @@ func main() { | @@ -43,11 +47,7 @@ func main() { | ||
43 | loc = time.Local | 47 | loc = time.Local |
44 | } | 48 | } |
45 | indexer := es.NewIndex(client, processor.Output.ElasticSearch.Index, loc) | 49 | indexer := es.NewIndex(client, processor.Output.ElasticSearch.Index, loc) |
46 | - filters := filter.CreateFilters(processor) | ||
47 | - writer, err := es.NewWriter(processor.Output.ElasticSearch, indexer) | ||
48 | - logx.Must(err) | ||
49 | - | ||
50 | - handle := handler.NewHandler(writer) | 50 | + handle := handler.NewHandler(writer, indexer) |
51 | handle.AddFilters(filters...) | 51 | handle.AddFilters(filters...) |
52 | handle.AddFilters(filter.AddUriFieldFilter("url", "uri")) | 52 | handle.AddFilters(filter.AddUriFieldFilter("url", "uri")) |
53 | group.Add(kq.MustNewQueue(processor.Input.Kafka, handle)) | 53 | group.Add(kq.MustNewQueue(processor.Input.Kafka, handle)) |
-
请 注册 或 登录 后发表评论