|
@@ -3,7 +3,6 @@ package consumer |
|
@@ -3,7 +3,6 @@ package consumer |
|
3
|
import (
|
3
|
import (
|
|
4
|
"context"
|
4
|
"context"
|
|
5
|
"errors"
|
5
|
"errors"
|
|
6
|
- "sync"
|
|
|
|
7
|
|
6
|
|
|
8
|
"gitlab.fjmaimaimai.com/mmm-go/partnermg/pkg/port/consumer/configs"
|
7
|
"gitlab.fjmaimaimai.com/mmm-go/partnermg/pkg/port/consumer/configs"
|
|
9
|
|
8
|
|
|
@@ -13,26 +12,26 @@ import ( |
|
@@ -13,26 +12,26 @@ import ( |
|
13
|
|
12
|
|
|
14
|
//MessageConsumer 消息消费者
|
13
|
//MessageConsumer 消息消费者
|
|
15
|
type MessageConsumer struct {
|
14
|
type MessageConsumer struct {
|
|
16
|
- ready chan bool
|
15
|
+ ready chan struct{}
|
|
17
|
kafkaHosts []string
|
16
|
kafkaHosts []string
|
|
18
|
groupId string
|
17
|
groupId string
|
|
19
|
topics []string
|
18
|
topics []string
|
|
20
|
topicsHandles map[string]TopicHandle
|
19
|
topicsHandles map[string]TopicHandle
|
|
21
|
}
|
20
|
}
|
|
22
|
|
21
|
|
|
23
|
-func NewMessageConsumer() *MessageConsumer {
|
|
|
|
24
|
- topics := []string{}
|
|
|
|
25
|
- for key := range TopicHandleRouters {
|
|
|
|
26
|
- topics = append(topics, key)
|
|
|
|
27
|
- }
|
|
|
|
28
|
- return &MessageConsumer{
|
|
|
|
29
|
- ready: make(chan bool),
|
|
|
|
30
|
- kafkaHosts: configs.Cfg.Servers,
|
|
|
|
31
|
- groupId: configs.Cfg.ConsumerId,
|
|
|
|
32
|
- topicsHandles: TopicHandleRouters,
|
|
|
|
33
|
- topics: topics,
|
|
|
|
34
|
- }
|
|
|
|
35
|
-}
|
22
|
+// func NewMessageConsumer() *MessageConsumer {
|
|
|
|
23
|
+// topics := []string{}
|
|
|
|
24
|
+// for key := range TopicHandleRouters {
|
|
|
|
25
|
+// topics = append(topics, key)
|
|
|
|
26
|
+// }
|
|
|
|
27
|
+// return &MessageConsumer{
|
|
|
|
28
|
+// ready: make(chan bool),
|
|
|
|
29
|
+// kafkaHosts: configs.Cfg.Servers,
|
|
|
|
30
|
+// groupId: configs.Cfg.ConsumerId,
|
|
|
|
31
|
+// topicsHandles: TopicHandleRouters,
|
|
|
|
32
|
+// topics: topics,
|
|
|
|
33
|
+// }
|
|
|
|
34
|
+// }
|
|
36
|
|
35
|
|
|
37
|
//实现对应的接口
|
36
|
//实现对应的接口
|
|
38
|
var _ sarama.ConsumerGroupHandler = (*MessageConsumer)(nil)
|
37
|
var _ sarama.ConsumerGroupHandler = (*MessageConsumer)(nil)
|
|
@@ -72,41 +71,95 @@ func (c *MessageConsumer) FindTopichandle(topic string) (TopicHandle, error) { |
|
@@ -72,41 +71,95 @@ func (c *MessageConsumer) FindTopichandle(topic string) (TopicHandle, error) { |
|
72
|
return nil, errors.New("TopicHandle not found")
|
71
|
return nil, errors.New("TopicHandle not found")
|
|
73
|
}
|
72
|
}
|
|
74
|
|
73
|
|
|
75
|
-//StartConsumer 启动
|
|
|
|
76
|
-//返回 Consumer关闭方法 和 error
|
|
|
|
77
|
-func StartConsumer(ctx context.Context) (func(), error) {
|
|
|
|
78
|
- consumer := NewMessageConsumer()
|
74
|
+type Runer struct {
|
|
|
|
75
|
+ msgConsumer *MessageConsumer
|
|
|
|
76
|
+ consumerGroup sarama.ConsumerGroup
|
|
|
|
77
|
+}
|
|
|
|
78
|
+
|
|
|
|
79
|
+func NewRuner() *Runer {
|
|
|
|
80
|
+ topics := []string{}
|
|
|
|
81
|
+ for key := range TopicHandleRouters {
|
|
|
|
82
|
+ topics = append(topics, key)
|
|
|
|
83
|
+ }
|
|
|
|
84
|
+
|
|
|
|
85
|
+ return &Runer{
|
|
|
|
86
|
+ msgConsumer: &MessageConsumer{
|
|
|
|
87
|
+ ready: make(chan struct{}),
|
|
|
|
88
|
+ kafkaHosts: configs.Cfg.Servers,
|
|
|
|
89
|
+ groupId: configs.Cfg.ConsumerId,
|
|
|
|
90
|
+ topicsHandles: TopicHandleRouters,
|
|
|
|
91
|
+ topics: topics,
|
|
|
|
92
|
+ },
|
|
|
|
93
|
+ }
|
|
|
|
94
|
+}
|
|
|
|
95
|
+
|
|
|
|
96
|
+func (r *Runer) InitConsumer() error {
|
|
79
|
config := sarama.NewConfig()
|
97
|
config := sarama.NewConfig()
|
|
80
|
config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRoundRobin
|
98
|
config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRoundRobin
|
|
81
|
config.Consumer.Offsets.Initial = sarama.OffsetNewest
|
99
|
config.Consumer.Offsets.Initial = sarama.OffsetNewest
|
|
82
|
config.Version = sarama.V0_11_0_2
|
100
|
config.Version = sarama.V0_11_0_2
|
|
83
|
- consumerGroup, err := sarama.NewConsumerGroup(consumer.kafkaHosts, consumer.groupId, config)
|
101
|
+ consumerGroup, err := sarama.NewConsumerGroup(r.msgConsumer.kafkaHosts, r.msgConsumer.groupId, config)
|
|
84
|
if err != nil {
|
102
|
if err != nil {
|
|
85
|
- return func() {}, err
|
103
|
+ return err
|
|
86
|
}
|
104
|
}
|
|
87
|
- wg := &sync.WaitGroup{}
|
|
|
|
88
|
- wg.Add(1)
|
|
|
|
89
|
- go func() {
|
|
|
|
90
|
- defer wg.Done()
|
105
|
+ r.consumerGroup = consumerGroup
|
|
|
|
106
|
+ return nil
|
|
|
|
107
|
+}
|
|
|
|
108
|
+
|
|
|
|
109
|
+func (r *Runer) Start(ctx context.Context) {
|
|
91
|
for {
|
110
|
for {
|
|
92
|
- if err := ctx.Err(); err != nil {
|
|
|
|
93
|
- logs.Error("ctx err:%s \n", err)
|
111
|
+ select {
|
|
|
|
112
|
+ case <-ctx.Done():
|
|
|
|
113
|
+ r.consumerGroup.Close()
|
|
|
|
114
|
+ logs.Warning("ctx cancel;consumerGroup.Close()")
|
|
94
|
return
|
115
|
return
|
|
95
|
- }
|
|
|
|
96
|
- if err := consumerGroup.Consume(ctx, consumer.topics, consumer); err != nil {
|
116
|
+ default:
|
|
|
|
117
|
+ if err := r.consumerGroup.Consume(ctx, r.msgConsumer.topics, r.msgConsumer); err != nil {
|
|
97
|
logs.Error("consumerGroup err:%s \n", err)
|
118
|
logs.Error("consumerGroup err:%s \n", err)
|
|
98
|
}
|
119
|
}
|
|
99
|
- consumer.ready = make(chan bool)
|
120
|
+ r.msgConsumer.ready = make(chan struct{})
|
|
100
|
}
|
121
|
}
|
|
101
|
- }()
|
|
|
|
102
|
- //等待 consumerGroup 设置完成
|
|
|
|
103
|
- <-consumer.ready
|
|
|
|
104
|
- logs.Info("Sarama consumer up and running!...")
|
|
|
|
105
|
- return func() {
|
|
|
|
106
|
- wg.Wait()
|
|
|
|
107
|
- if err := consumerGroup.Close(); err != nil {
|
|
|
|
108
|
- logs.Error("consumerGroup.Close err %s", err)
|
|
|
|
109
|
}
|
122
|
}
|
|
110
|
- logs.Info("consumerGroup.Close")
|
|
|
|
111
|
- }, nil
|
|
|
|
112
|
}
|
123
|
}
|
|
|
|
124
|
+func (r *Runer) IsReady() <-chan struct{} {
|
|
|
|
125
|
+ return r.msgConsumer.ready
|
|
|
|
126
|
+}
|
|
|
|
127
|
+
|
|
|
|
128
|
+//StartConsumer 启动
|
|
|
|
129
|
+//返回 Consumer关闭方法 和 error
|
|
|
|
130
|
+// func StartConsumer(ctx context.Context) (func(), error) {
|
|
|
|
131
|
+// consumer := NewMessageConsumer()
|
|
|
|
132
|
+// config := sarama.NewConfig()
|
|
|
|
133
|
+// config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRoundRobin
|
|
|
|
134
|
+// config.Consumer.Offsets.Initial = sarama.OffsetNewest
|
|
|
|
135
|
+// config.Version = sarama.V0_11_0_2
|
|
|
|
136
|
+// consumerGroup, err := sarama.NewConsumerGroup(consumer.kafkaHosts, consumer.groupId, config)
|
|
|
|
137
|
+// if err != nil {
|
|
|
|
138
|
+// return func() {}, err
|
|
|
|
139
|
+// }
|
|
|
|
140
|
+// wg := &sync.WaitGroup{}
|
|
|
|
141
|
+// wg.Add(1)
|
|
|
|
142
|
+// go func() {
|
|
|
|
143
|
+// defer wg.Done()
|
|
|
|
144
|
+// for {
|
|
|
|
145
|
+// if err := ctx.Err(); err != nil {
|
|
|
|
146
|
+// logs.Error("ctx err:%s \n", err)
|
|
|
|
147
|
+// return
|
|
|
|
148
|
+// }
|
|
|
|
149
|
+// if err := consumerGroup.Consume(ctx, consumer.topics, consumer); err != nil {
|
|
|
|
150
|
+// logs.Error("consumerGroup err:%s \n", err)
|
|
|
|
151
|
+// }
|
|
|
|
152
|
+// consumer.ready = make(chan bool)
|
|
|
|
153
|
+// }
|
|
|
|
154
|
+// }()
|
|
|
|
155
|
+// //等待 consumerGroup 设置完成
|
|
|
|
156
|
+// <-consumer.ready
|
|
|
|
157
|
+// logs.Info("Sarama consumer up and running!...")
|
|
|
|
158
|
+// return func() {
|
|
|
|
159
|
+// wg.Wait()
|
|
|
|
160
|
+// if err := consumerGroup.Close(); err != nil {
|
|
|
|
161
|
+// logs.Error("consumerGroup.Close err %s", err)
|
|
|
|
162
|
+// }
|
|
|
|
163
|
+// logs.Info("consumerGroup.Close")
|
|
|
|
164
|
+// }, nil
|
|
|
|
165
|
+// } |