writer.go
1.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
package es
import (
"context"
"time"
"github.com/olivere/elastic"
"github.com/tal-tech/go-stash/stash/config"
"github.com/tal-tech/go-zero/core/executors"
"github.com/tal-tech/go-zero/core/logx"
)
const docType = "doc"
type (
Writer struct {
client *elastic.Client
indexer *Index
inserter *executors.ChunkExecutor
}
valueWithTime struct {
t time.Time
val string
}
)
func NewWriter(c config.ElasticSearchConf, indexer *Index) (*Writer, error) {
client, err := elastic.NewClient(
elastic.SetSniff(false),
elastic.SetURL(c.Hosts...),
elastic.SetGzip(c.Compress),
)
if err != nil {
return nil, err
}
writer := Writer{
client: client,
indexer: indexer,
}
writer.inserter = executors.NewChunkExecutor(writer.execute, executors.WithChunkBytes(c.MaxChunkBytes))
return &writer, nil
}
func (w *Writer) Write(t time.Time, val string) error {
return w.inserter.Add(valueWithTime{
t: t,
val: val,
}, len(val))
}
func (w *Writer) execute(vals []interface{}) {
var bulk = w.client.Bulk()
for _, val := range vals {
pair := val.(valueWithTime)
req := elastic.NewBulkIndexRequest().Index(w.indexer.GetIndex(pair.t)).Type(docType).Doc(pair.val)
bulk.Add(req)
}
_, err := bulk.Do(context.Background())
if err != nil {
logx.Error(err)
}
}