writer.go
1.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
package es
import (
"context"
"github.com/olivere/elastic"
"github.com/tal-tech/go-stash/stash/config"
"github.com/tal-tech/go-zero/core/executors"
"github.com/tal-tech/go-zero/core/logx"
)
type (
Writer struct {
docType string
client *elastic.Client
inserter *executors.ChunkExecutor
}
valueWithIndex struct {
index string
val string
}
)
func NewWriter(c config.ElasticSearchConf) (*Writer, error) {
client, err := elastic.NewClient(
elastic.SetSniff(false),
elastic.SetURL(c.Hosts...),
elastic.SetGzip(c.Compress),
)
if err != nil {
return nil, err
}
writer := Writer{
docType: c.DocType,
client: client,
}
writer.inserter = executors.NewChunkExecutor(writer.execute, executors.WithChunkBytes(c.MaxChunkBytes))
return &writer, nil
}
func (w *Writer) Write(index, val string) error {
return w.inserter.Add(valueWithIndex{
index: index,
val: val,
}, len(val))
}
func (w *Writer) execute(vals []interface{}) {
var bulk = w.client.Bulk()
for _, val := range vals {
pair := val.(valueWithIndex)
req := elastic.NewBulkIndexRequest().Index(pair.index).Type(w.docType).Doc(pair.val)
bulk.Add(req)
}
_, err := bulk.Do(context.Background())
if err != nil {
logx.Error(err)
}
}