正在显示
3 个修改的文件
包含
18 行增加
和
20 行删除
| @@ -3,7 +3,6 @@ package es | @@ -3,7 +3,6 @@ package es | ||
| 3 | import ( | 3 | import ( |
| 4 | "context" | 4 | "context" |
| 5 | 5 | ||
| 6 | - jsoniter "github.com/json-iterator/go" | ||
| 7 | "github.com/olivere/elastic" | 6 | "github.com/olivere/elastic" |
| 8 | "github.com/tal-tech/go-stash/stash/config" | 7 | "github.com/tal-tech/go-stash/stash/config" |
| 9 | "github.com/tal-tech/go-zero/core/executors" | 8 | "github.com/tal-tech/go-zero/core/executors" |
| @@ -14,7 +13,6 @@ type ( | @@ -14,7 +13,6 @@ type ( | ||
| 14 | Writer struct { | 13 | Writer struct { |
| 15 | docType string | 14 | docType string |
| 16 | client *elastic.Client | 15 | client *elastic.Client |
| 17 | - indexer *Index | ||
| 18 | inserter *executors.ChunkExecutor | 16 | inserter *executors.ChunkExecutor |
| 19 | } | 17 | } |
| 20 | 18 | ||
| @@ -24,7 +22,7 @@ type ( | @@ -24,7 +22,7 @@ type ( | ||
| 24 | } | 22 | } |
| 25 | ) | 23 | ) |
| 26 | 24 | ||
| 27 | -func NewWriter(c config.ElasticSearchConf, indexer *Index) (*Writer, error) { | 25 | +func NewWriter(c config.ElasticSearchConf) (*Writer, error) { |
| 28 | client, err := elastic.NewClient( | 26 | client, err := elastic.NewClient( |
| 29 | elastic.SetSniff(false), | 27 | elastic.SetSniff(false), |
| 30 | elastic.SetURL(c.Hosts...), | 28 | elastic.SetURL(c.Hosts...), |
| @@ -37,20 +35,12 @@ func NewWriter(c config.ElasticSearchConf, indexer *Index) (*Writer, error) { | @@ -37,20 +35,12 @@ func NewWriter(c config.ElasticSearchConf, indexer *Index) (*Writer, error) { | ||
| 37 | writer := Writer{ | 35 | writer := Writer{ |
| 38 | docType: c.DocType, | 36 | docType: c.DocType, |
| 39 | client: client, | 37 | client: client, |
| 40 | - indexer: indexer, | ||
| 41 | } | 38 | } |
| 42 | writer.inserter = executors.NewChunkExecutor(writer.execute, executors.WithChunkBytes(c.MaxChunkBytes)) | 39 | writer.inserter = executors.NewChunkExecutor(writer.execute, executors.WithChunkBytes(c.MaxChunkBytes)) |
| 43 | return &writer, nil | 40 | return &writer, nil |
| 44 | } | 41 | } |
| 45 | 42 | ||
| 46 | -func (w *Writer) Write(m map[string]interface{}) error { | ||
| 47 | - bs, err := jsoniter.Marshal(m) | ||
| 48 | - if err != nil { | ||
| 49 | - return err | ||
| 50 | - } | ||
| 51 | - | ||
| 52 | - index := w.indexer.GetIndex(m) | ||
| 53 | - val := string(bs) | 43 | +func (w *Writer) Write(index, val string) error { |
| 54 | return w.inserter.Add(valueWithIndex{ | 44 | return w.inserter.Add(valueWithIndex{ |
| 55 | index: index, | 45 | index: index, |
| 56 | val: val, | 46 | val: val, |
| @@ -8,12 +8,14 @@ import ( | @@ -8,12 +8,14 @@ import ( | ||
| 8 | 8 | ||
| 9 | type MessageHandler struct { | 9 | type MessageHandler struct { |
| 10 | writer *es.Writer | 10 | writer *es.Writer |
| 11 | + indexer *es.Index | ||
| 11 | filters []filter.FilterFunc | 12 | filters []filter.FilterFunc |
| 12 | } | 13 | } |
| 13 | 14 | ||
| 14 | -func NewHandler(writer *es.Writer) *MessageHandler { | 15 | +func NewHandler(writer *es.Writer, indexer *es.Index) *MessageHandler { |
| 15 | return &MessageHandler{ | 16 | return &MessageHandler{ |
| 16 | - writer: writer, | 17 | + writer: writer, |
| 18 | + indexer: indexer, | ||
| 17 | } | 19 | } |
| 18 | } | 20 | } |
| 19 | 21 | ||
| @@ -29,11 +31,17 @@ func (mh *MessageHandler) Consume(_, val string) error { | @@ -29,11 +31,17 @@ func (mh *MessageHandler) Consume(_, val string) error { | ||
| 29 | return err | 31 | return err |
| 30 | } | 32 | } |
| 31 | 33 | ||
| 34 | + index := mh.indexer.GetIndex(m) | ||
| 32 | for _, proc := range mh.filters { | 35 | for _, proc := range mh.filters { |
| 33 | if m = proc(m); m == nil { | 36 | if m = proc(m); m == nil { |
| 34 | return nil | 37 | return nil |
| 35 | } | 38 | } |
| 36 | } | 39 | } |
| 37 | 40 | ||
| 38 | - return mh.writer.Write(m) | 41 | + bs, err := jsoniter.Marshal(m) |
| 42 | + if err != nil { | ||
| 43 | + return err | ||
| 44 | + } | ||
| 45 | + | ||
| 46 | + return mh.writer.Write(index, string(bs)) | ||
| 39 | } | 47 | } |
| @@ -35,6 +35,10 @@ func main() { | @@ -35,6 +35,10 @@ func main() { | ||
| 35 | ) | 35 | ) |
| 36 | logx.Must(err) | 36 | logx.Must(err) |
| 37 | 37 | ||
| 38 | + filters := filter.CreateFilters(processor) | ||
| 39 | + writer, err := es.NewWriter(processor.Output.ElasticSearch) | ||
| 40 | + logx.Must(err) | ||
| 41 | + | ||
| 38 | var loc *time.Location | 42 | var loc *time.Location |
| 39 | if len(processor.Output.ElasticSearch.TimeZone) > 0 { | 43 | if len(processor.Output.ElasticSearch.TimeZone) > 0 { |
| 40 | loc, err = time.LoadLocation(processor.Output.ElasticSearch.TimeZone) | 44 | loc, err = time.LoadLocation(processor.Output.ElasticSearch.TimeZone) |
| @@ -43,11 +47,7 @@ func main() { | @@ -43,11 +47,7 @@ func main() { | ||
| 43 | loc = time.Local | 47 | loc = time.Local |
| 44 | } | 48 | } |
| 45 | indexer := es.NewIndex(client, processor.Output.ElasticSearch.Index, loc) | 49 | indexer := es.NewIndex(client, processor.Output.ElasticSearch.Index, loc) |
| 46 | - filters := filter.CreateFilters(processor) | ||
| 47 | - writer, err := es.NewWriter(processor.Output.ElasticSearch, indexer) | ||
| 48 | - logx.Must(err) | ||
| 49 | - | ||
| 50 | - handle := handler.NewHandler(writer) | 50 | + handle := handler.NewHandler(writer, indexer) |
| 51 | handle.AddFilters(filters...) | 51 | handle.AddFilters(filters...) |
| 52 | handle.AddFilters(filter.AddUriFieldFilter("url", "uri")) | 52 | handle.AddFilters(filter.AddUriFieldFilter("url", "uri")) |
| 53 | group.Add(kq.MustNewQueue(processor.Input.Kafka, handle)) | 53 | group.Add(kq.MustNewQueue(processor.Input.Kafka, handle)) |
-
请 注册 或 登录 后发表评论