作者 唐旭辉

更新vendor依赖

正在显示 36 个修改的文件 包含 4170 行增加123 行删除

要显示太多修改。

为保证性能只显示 36 of 36+ 个文件。

@@ -5,7 +5,7 @@ go 1.14 @@ -5,7 +5,7 @@ go 1.14
5 require ( 5 require (
6 github.com/GeeTeam/gt3-golang-sdk v0.0.0-20200116043922-446ca8a507d2 6 github.com/GeeTeam/gt3-golang-sdk v0.0.0-20200116043922-446ca8a507d2
7 github.com/ajg/form v1.5.1 // indirect 7 github.com/ajg/form v1.5.1 // indirect
8 - github.com/astaxie/beego v1.12.1 8 + github.com/astaxie/beego v1.12.2
9 github.com/dgrijalva/jwt-go v3.2.0+incompatible 9 github.com/dgrijalva/jwt-go v3.2.0+incompatible
10 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072 // indirect 10 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072 // indirect
11 github.com/fatih/structs v1.1.0 // indirect 11 github.com/fatih/structs v1.1.0 // indirect
@@ -21,7 +21,6 @@ require ( @@ -21,7 +21,6 @@ require (
21 github.com/onsi/ginkgo v1.13.0 21 github.com/onsi/ginkgo v1.13.0
22 github.com/onsi/gomega v1.10.1 22 github.com/onsi/gomega v1.10.1
23 github.com/sergi/go-diff v1.1.0 // indirect 23 github.com/sergi/go-diff v1.1.0 // indirect
24 - github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect  
25 github.com/shopspring/decimal v1.2.0 24 github.com/shopspring/decimal v1.2.0
26 github.com/smartystreets/goconvey v1.6.4 // indirect 25 github.com/smartystreets/goconvey v1.6.4 // indirect
27 github.com/valyala/fasthttp v1.14.0 // indirect 26 github.com/valyala/fasthttp v1.14.0 // indirect
1 package domain 1 package domain
2 2
3 -import "time" 3 +import (
  4 + "fmt"
  5 + "time"
  6 +)
4 7
5 // 状态(1:启用或者0:禁用) 8 // 状态(1:启用或者0:禁用)
6 const ( 9 const (
@@ -8,16 +11,24 @@ const ( @@ -8,16 +11,24 @@ const (
8 PARTNER_STATUS_YES int = 1 11 PARTNER_STATUS_YES int = 1
9 ) 12 )
10 13
11 -//合伙类别 (1.事业合伙人 2.业务合伙人 3.研发合伙人) 14 +//合伙类别 (1.事业合伙人 2.业务合伙人 4.研发合伙人)
  15 +//按二进制位区分:001 = 1; 010 = 2; 100=4
12 const ( 16 const (
13 PARTNER_CATEGORY_1 int = 1 17 PARTNER_CATEGORY_1 int = 1
14 PARTNER_CATEGORY_2 int = 2 18 PARTNER_CATEGORY_2 int = 2
15 - PARTNER_CATEGORY_3 int = 3 19 + PARTNER_CATEGORY_3 int = 4
16 ) 20 )
17 21
  22 +//partnerCategoryMap 合伙类别键值对 (只读,请勿在运行时修改)
  23 +var partnerCategoryMap = map[int]string{
  24 + PARTNER_CATEGORY_1: "事业合伙人",
  25 + PARTNER_CATEGORY_2: "业务合伙人",
  26 + PARTNER_CATEGORY_3: "研发合伙人",
  27 +}
  28 +
18 type PartnerInfo struct { 29 type PartnerInfo struct {
19 Partner Partner `json:"partner"` 30 Partner Partner `json:"partner"`
20 - //合伙类别 (1.事业合伙人 2.业务合伙人 3.研发合伙人) 31 + //合伙类别
21 PartnerCategory int `json:"partnerCategory"` 32 PartnerCategory int `json:"partnerCategory"`
22 // 登录密码 33 // 登录密码
23 Password string `json:"password"` 34 Password string `json:"password"`
@@ -35,6 +46,29 @@ type PartnerInfo struct { @@ -35,6 +46,29 @@ type PartnerInfo struct {
35 Salesman []Salesman `json:"salesman"` 46 Salesman []Salesman `json:"salesman"`
36 } 47 }
37 48
  49 +func (p *PartnerInfo) GetPartnerCategory() map[int]string {
  50 + categoryMap := map[int]string{}
  51 + for k, v := range partnerCategoryMap {
  52 + //合伙类别 按二进制位区分
  53 + if (p.PartnerCategory & k) > 0 {
  54 + categoryMap[k] = v
  55 + }
  56 + }
  57 + return categoryMap
  58 +}
  59 +
  60 +func (p *PartnerInfo) SetPartnerCategory(category []int) error {
  61 + n := 0
  62 + for _, v := range category {
  63 + if _, ok := partnerCategoryMap[v]; !ok {
  64 + return fmt.Errorf("未知的合伙人类型:%d", v)
  65 + }
  66 + n += v
  67 + }
  68 + p.PartnerCategory = n
  69 + return nil
  70 +}
  71 +
38 type PartnerFindOneQuery struct { 72 type PartnerFindOneQuery struct {
39 UserId int64 73 UserId int64
40 AccountEqual string 74 AccountEqual string
  1 +package service_gateway
  2 +
  3 +import (
  4 + "encoding/json"
  5 + "fmt"
  6 + "strconv"
  7 + "strings"
  8 + "time"
  9 +
  10 + "github.com/astaxie/beego/httplib"
  11 +)
  12 +
  13 +type httplibBaseServiceGateway struct {
  14 + baseURL string
  15 + connectTimeout time.Duration
  16 + readWriteTimeout time.Duration
  17 +}
  18 +
  19 +func (serviceGateway *httplibBaseServiceGateway) createRequest(url string, method string) *httplib.BeegoHTTPRequest {
  20 + var request *httplib.BeegoHTTPRequest
  21 + switch method {
  22 + case "get":
  23 + request = httplib.Get(url)
  24 + break
  25 + case "post":
  26 + request = httplib.Post(url)
  27 + break
  28 + case "put":
  29 + request = httplib.Put(url)
  30 + break
  31 + case "delete":
  32 + request = httplib.Delete(url)
  33 + break
  34 + case "head":
  35 + request = httplib.Head(url)
  36 + break
  37 + default:
  38 + request = httplib.Get(url)
  39 + }
  40 + return request.SetTimeout(serviceGateway.connectTimeout, serviceGateway.readWriteTimeout)
  41 +}
  42 +
  43 +func (serviceGateway *httplibBaseServiceGateway) responseHandle(response map[string]interface{}) (map[string]interface{}, error) {
  44 + data := make(map[string]interface{})
  45 + var err error
  46 + if code, ok := response["code"]; ok {
  47 + code := code.(float64)
  48 + if code == 0 {
  49 + data = response["data"].(map[string]interface{})
  50 + } else {
  51 + msg := response["msg"].(string)
  52 + err = fmt.Errorf(strings.Join([]string{strconv.FormatFloat(code, 'f', -1, 64), msg}, " "))
  53 + }
  54 + } else {
  55 + jsonBytes, marshalErr := json.Marshal(response)
  56 + if marshalErr != nil {
  57 + err = marshalErr
  58 + }
  59 + err = fmt.Errorf("无法解析的网关服务数据返回格式:%s", string(jsonBytes))
  60 + }
  61 + return data, err
  62 +}
@@ -15,12 +15,12 @@ var ( @@ -15,12 +15,12 @@ var (
15 //MyToken ... 15 //MyToken ...
16 type MyToken struct { 16 type MyToken struct {
17 jwt.StandardClaims 17 jwt.StandardClaims
18 - UID int64 `json:"uid"` //管理员的id  
19 - 18 + UID int64 `json:"uid"` //管理员的id
  19 + CompanyId int64 `json:"companyId"` //公司id
20 } 20 }
21 21
22 -func NewMyToken(id int64) *MyToken {  
23 - return &MyToken{UID: id} 22 +func NewMyToken(id int64, companyId int64) *MyToken {
  23 + return &MyToken{UID: id, CompanyId: companyId}
24 } 24 }
25 25
26 //CreateJWTToken ... 26 //CreateJWTToken ...
@@ -68,7 +68,8 @@ func (c *AdminLoginController) Login() { @@ -68,7 +68,8 @@ func (c *AdminLoginController) Login() {
68 if !adminuser.IsUsable { 68 if !adminuser.IsUsable {
69 c.ResponseError(errors.New("用户被禁用")) 69 c.ResponseError(errors.New("用户被禁用"))
70 } 70 }
71 - newJwt := lib.NewMyToken(adminuser.Id) 71 + //TODO
  72 + newJwt := lib.NewMyToken(adminuser.Id, 0)
72 newToken, err := newJwt.CreateJWTToken() 73 newToken, err := newJwt.CreateJWTToken()
73 if err != nil { 74 if err != nil {
74 logs.Error("生成jwt数据失败:%s", err) 75 logs.Error("生成jwt数据失败:%s", err)
@@ -36,8 +36,7 @@ install: @@ -36,8 +36,7 @@ install:
36 - go get github.com/beego/goyaml2 36 - go get github.com/beego/goyaml2
37 - go get gopkg.in/yaml.v2 37 - go get gopkg.in/yaml.v2
38 - go get github.com/belogik/goes 38 - go get github.com/belogik/goes
39 - - go get github.com/siddontang/ledisdb/config  
40 - - go get github.com/siddontang/ledisdb/ledis 39 + - go get github.com/ledisdb/ledisdb
41 - go get github.com/ssdb/gossdb/ssdb 40 - go get github.com/ssdb/gossdb/ssdb
42 - go get github.com/cloudflare/golz4 41 - go get github.com/cloudflare/golz4
43 - go get github.com/gogo/protobuf/proto 42 - go get github.com/gogo/protobuf/proto
@@ -49,7 +48,7 @@ install: @@ -49,7 +48,7 @@ install:
49 - go get -u honnef.co/go/tools/cmd/staticcheck 48 - go get -u honnef.co/go/tools/cmd/staticcheck
50 - go get -u github.com/mdempsky/unconvert 49 - go get -u github.com/mdempsky/unconvert
51 - go get -u github.com/gordonklaus/ineffassign 50 - go get -u github.com/gordonklaus/ineffassign
52 - - go get -u github.com/golang/lint/golint 51 + - go get -u golang.org/x/lint/golint
53 - go get -u github.com/go-redis/redis 52 - go get -u github.com/go-redis/redis
54 before_script: 53 before_script:
55 - psql --version 54 - psql --version
@@ -33,6 +33,8 @@ Congratulations! You've just built your first **beego** app. @@ -33,6 +33,8 @@ Congratulations! You've just built your first **beego** app.
33 33
34 ###### Please see [Documentation](http://beego.me/docs) for more. 34 ###### Please see [Documentation](http://beego.me/docs) for more.
35 35
  36 +###### [beego-example](https://github.com/beego-dev/beego-example)
  37 +
36 ## Features 38 ## Features
37 39
38 * RESTful support 40 * RESTful support
@@ -24,6 +24,8 @@ import ( @@ -24,6 +24,8 @@ import (
24 "text/template" 24 "text/template"
25 "time" 25 "time"
26 26
  27 + "github.com/prometheus/client_golang/prometheus/promhttp"
  28 +
27 "github.com/astaxie/beego/grace" 29 "github.com/astaxie/beego/grace"
28 "github.com/astaxie/beego/logs" 30 "github.com/astaxie/beego/logs"
29 "github.com/astaxie/beego/toolbox" 31 "github.com/astaxie/beego/toolbox"
@@ -55,12 +57,14 @@ func init() { @@ -55,12 +57,14 @@ func init() {
55 beeAdminApp = &adminApp{ 57 beeAdminApp = &adminApp{
56 routers: make(map[string]http.HandlerFunc), 58 routers: make(map[string]http.HandlerFunc),
57 } 59 }
  60 + // keep in mind that all data should be html escaped to avoid XSS attack
58 beeAdminApp.Route("/", adminIndex) 61 beeAdminApp.Route("/", adminIndex)
59 beeAdminApp.Route("/qps", qpsIndex) 62 beeAdminApp.Route("/qps", qpsIndex)
60 beeAdminApp.Route("/prof", profIndex) 63 beeAdminApp.Route("/prof", profIndex)
61 beeAdminApp.Route("/healthcheck", healthcheck) 64 beeAdminApp.Route("/healthcheck", healthcheck)
62 beeAdminApp.Route("/task", taskStatus) 65 beeAdminApp.Route("/task", taskStatus)
63 beeAdminApp.Route("/listconf", listConf) 66 beeAdminApp.Route("/listconf", listConf)
  67 + beeAdminApp.Route("/metrics", promhttp.Handler().ServeHTTP)
64 FilterMonitorFunc = func(string, string, time.Duration, string, int) bool { return true } 68 FilterMonitorFunc = func(string, string, time.Duration, string, int) bool { return true }
65 } 69 }
66 70
@@ -105,8 +109,8 @@ func listConf(rw http.ResponseWriter, r *http.Request) { @@ -105,8 +109,8 @@ func listConf(rw http.ResponseWriter, r *http.Request) {
105 case "conf": 109 case "conf":
106 m := make(M) 110 m := make(M)
107 list("BConfig", BConfig, m) 111 list("BConfig", BConfig, m)
108 - m["AppConfigPath"] = appConfigPath  
109 - m["AppConfigProvider"] = appConfigProvider 112 + m["AppConfigPath"] = template.HTMLEscapeString(appConfigPath)
  113 + m["AppConfigProvider"] = template.HTMLEscapeString(appConfigProvider)
110 tmpl := template.Must(template.New("dashboard").Parse(dashboardTpl)) 114 tmpl := template.Must(template.New("dashboard").Parse(dashboardTpl))
111 tmpl = template.Must(tmpl.Parse(configTpl)) 115 tmpl = template.Must(tmpl.Parse(configTpl))
112 tmpl = template.Must(tmpl.Parse(defaultScriptsTpl)) 116 tmpl = template.Must(tmpl.Parse(defaultScriptsTpl))
@@ -151,8 +155,9 @@ func listConf(rw http.ResponseWriter, r *http.Request) { @@ -151,8 +155,9 @@ func listConf(rw http.ResponseWriter, r *http.Request) {
151 resultList := new([][]string) 155 resultList := new([][]string)
152 for _, f := range bf { 156 for _, f := range bf {
153 var result = []string{ 157 var result = []string{
154 - f.pattern,  
155 - utils.GetFuncName(f.filterFunc), 158 + // void xss
  159 + template.HTMLEscapeString(f.pattern),
  160 + template.HTMLEscapeString(utils.GetFuncName(f.filterFunc)),
156 } 161 }
157 *resultList = append(*resultList, result) 162 *resultList = append(*resultList, result)
158 } 163 }
@@ -207,8 +212,8 @@ func PrintTree() M { @@ -207,8 +212,8 @@ func PrintTree() M {
207 212
208 printTree(resultList, t) 213 printTree(resultList, t)
209 214
210 - methods = append(methods, method)  
211 - methodsData[method] = resultList 215 + methods = append(methods, template.HTMLEscapeString(method))
  216 + methodsData[template.HTMLEscapeString(method)] = resultList
212 } 217 }
213 218
214 content["Data"] = methodsData 219 content["Data"] = methodsData
@@ -227,21 +232,21 @@ func printTree(resultList *[][]string, t *Tree) { @@ -227,21 +232,21 @@ func printTree(resultList *[][]string, t *Tree) {
227 if v, ok := l.runObject.(*ControllerInfo); ok { 232 if v, ok := l.runObject.(*ControllerInfo); ok {
228 if v.routerType == routerTypeBeego { 233 if v.routerType == routerTypeBeego {
229 var result = []string{ 234 var result = []string{
230 - v.pattern,  
231 - fmt.Sprintf("%s", v.methods),  
232 - v.controllerType.String(), 235 + template.HTMLEscapeString(v.pattern),
  236 + template.HTMLEscapeString(fmt.Sprintf("%s", v.methods)),
  237 + template.HTMLEscapeString(v.controllerType.String()),
233 } 238 }
234 *resultList = append(*resultList, result) 239 *resultList = append(*resultList, result)
235 } else if v.routerType == routerTypeRESTFul { 240 } else if v.routerType == routerTypeRESTFul {
236 var result = []string{ 241 var result = []string{
237 - v.pattern,  
238 - fmt.Sprintf("%s", v.methods), 242 + template.HTMLEscapeString(v.pattern),
  243 + template.HTMLEscapeString(fmt.Sprintf("%s", v.methods)),
239 "", 244 "",
240 } 245 }
241 *resultList = append(*resultList, result) 246 *resultList = append(*resultList, result)
242 } else if v.routerType == routerTypeHandler { 247 } else if v.routerType == routerTypeHandler {
243 var result = []string{ 248 var result = []string{
244 - v.pattern, 249 + template.HTMLEscapeString(v.pattern),
245 "", 250 "",
246 "", 251 "",
247 } 252 }
@@ -266,7 +271,7 @@ func profIndex(rw http.ResponseWriter, r *http.Request) { @@ -266,7 +271,7 @@ func profIndex(rw http.ResponseWriter, r *http.Request) {
266 result bytes.Buffer 271 result bytes.Buffer
267 ) 272 )
268 toolbox.ProcessInput(command, &result) 273 toolbox.ProcessInput(command, &result)
269 - data["Content"] = result.String() 274 + data["Content"] = template.HTMLEscapeString(result.String())
270 275
271 if format == "json" && command == "gc summary" { 276 if format == "json" && command == "gc summary" {
272 dataJSON, err := json.Marshal(data) 277 dataJSON, err := json.Marshal(data)
@@ -280,7 +285,7 @@ func profIndex(rw http.ResponseWriter, r *http.Request) { @@ -280,7 +285,7 @@ func profIndex(rw http.ResponseWriter, r *http.Request) {
280 return 285 return
281 } 286 }
282 287
283 - data["Title"] = command 288 + data["Title"] = template.HTMLEscapeString(command)
284 defaultTpl := defaultScriptsTpl 289 defaultTpl := defaultScriptsTpl
285 if command == "gc summary" { 290 if command == "gc summary" {
286 defaultTpl = gcAjaxTpl 291 defaultTpl = gcAjaxTpl
@@ -304,13 +309,13 @@ func healthcheck(rw http.ResponseWriter, _ *http.Request) { @@ -304,13 +309,13 @@ func healthcheck(rw http.ResponseWriter, _ *http.Request) {
304 if err := h.Check(); err != nil { 309 if err := h.Check(); err != nil {
305 result = []string{ 310 result = []string{
306 "error", 311 "error",
307 - name,  
308 - err.Error(), 312 + template.HTMLEscapeString(name),
  313 + template.HTMLEscapeString(err.Error()),
309 } 314 }
310 } else { 315 } else {
311 result = []string{ 316 result = []string{
312 "success", 317 "success",
313 - name, 318 + template.HTMLEscapeString(name),
314 "OK", 319 "OK",
315 } 320 }
316 } 321 }
@@ -334,11 +339,11 @@ func taskStatus(rw http.ResponseWriter, req *http.Request) { @@ -334,11 +339,11 @@ func taskStatus(rw http.ResponseWriter, req *http.Request) {
334 if taskname != "" { 339 if taskname != "" {
335 if t, ok := toolbox.AdminTaskList[taskname]; ok { 340 if t, ok := toolbox.AdminTaskList[taskname]; ok {
336 if err := t.Run(); err != nil { 341 if err := t.Run(); err != nil {
337 - data["Message"] = []string{"error", fmt.Sprintf("%s", err)} 342 + data["Message"] = []string{"error", template.HTMLEscapeString(fmt.Sprintf("%s", err))}
338 } 343 }
339 - data["Message"] = []string{"success", fmt.Sprintf("%s run success,Now the Status is <br>%s", taskname, t.GetStatus())} 344 + data["Message"] = []string{"success", template.HTMLEscapeString(fmt.Sprintf("%s run success,Now the Status is <br>%s", taskname, t.GetStatus()))}
340 } else { 345 } else {
341 - data["Message"] = []string{"warning", fmt.Sprintf("there's no task which named: %s", taskname)} 346 + data["Message"] = []string{"warning", template.HTMLEscapeString(fmt.Sprintf("there's no task which named: %s", taskname))}
342 } 347 }
343 } 348 }
344 349
@@ -354,10 +359,10 @@ func taskStatus(rw http.ResponseWriter, req *http.Request) { @@ -354,10 +359,10 @@ func taskStatus(rw http.ResponseWriter, req *http.Request) {
354 } 359 }
355 for tname, tk := range toolbox.AdminTaskList { 360 for tname, tk := range toolbox.AdminTaskList {
356 result := []string{ 361 result := []string{
357 - tname,  
358 - tk.GetSpec(),  
359 - tk.GetStatus(),  
360 - tk.GetPrev().String(), 362 + template.HTMLEscapeString(tname),
  363 + template.HTMLEscapeString(tk.GetSpec()),
  364 + template.HTMLEscapeString(tk.GetStatus()),
  365 + template.HTMLEscapeString(tk.GetPrev().String()),
361 } 366 }
362 *resultList = append(*resultList, result) 367 *resultList = append(*resultList, result)
363 } 368 }
@@ -123,14 +123,13 @@ func (app *App) Run(mws ...MiddleWare) { @@ -123,14 +123,13 @@ func (app *App) Run(mws ...MiddleWare) {
123 httpsAddr = fmt.Sprintf("%s:%d", BConfig.Listen.HTTPSAddr, BConfig.Listen.HTTPSPort) 123 httpsAddr = fmt.Sprintf("%s:%d", BConfig.Listen.HTTPSAddr, BConfig.Listen.HTTPSPort)
124 app.Server.Addr = httpsAddr 124 app.Server.Addr = httpsAddr
125 } 125 }
126 - server := grace.NewServer(httpsAddr, app.Handlers) 126 + server := grace.NewServer(httpsAddr, app.Server.Handler)
127 server.Server.ReadTimeout = app.Server.ReadTimeout 127 server.Server.ReadTimeout = app.Server.ReadTimeout
128 server.Server.WriteTimeout = app.Server.WriteTimeout 128 server.Server.WriteTimeout = app.Server.WriteTimeout
129 if BConfig.Listen.EnableMutualHTTPS { 129 if BConfig.Listen.EnableMutualHTTPS {
130 if err := server.ListenAndServeMutualTLS(BConfig.Listen.HTTPSCertFile, BConfig.Listen.HTTPSKeyFile, BConfig.Listen.TrustCaFile); err != nil { 130 if err := server.ListenAndServeMutualTLS(BConfig.Listen.HTTPSCertFile, BConfig.Listen.HTTPSKeyFile, BConfig.Listen.TrustCaFile); err != nil {
131 logs.Critical("ListenAndServeTLS: ", err, fmt.Sprintf("%d", os.Getpid())) 131 logs.Critical("ListenAndServeTLS: ", err, fmt.Sprintf("%d", os.Getpid()))
132 time.Sleep(100 * time.Microsecond) 132 time.Sleep(100 * time.Microsecond)
133 - endRunning <- true  
134 } 133 }
135 } else { 134 } else {
136 if BConfig.Listen.AutoTLS { 135 if BConfig.Listen.AutoTLS {
@@ -145,14 +144,14 @@ func (app *App) Run(mws ...MiddleWare) { @@ -145,14 +144,14 @@ func (app *App) Run(mws ...MiddleWare) {
145 if err := server.ListenAndServeTLS(BConfig.Listen.HTTPSCertFile, BConfig.Listen.HTTPSKeyFile); err != nil { 144 if err := server.ListenAndServeTLS(BConfig.Listen.HTTPSCertFile, BConfig.Listen.HTTPSKeyFile); err != nil {
146 logs.Critical("ListenAndServeTLS: ", err, fmt.Sprintf("%d", os.Getpid())) 145 logs.Critical("ListenAndServeTLS: ", err, fmt.Sprintf("%d", os.Getpid()))
147 time.Sleep(100 * time.Microsecond) 146 time.Sleep(100 * time.Microsecond)
148 - endRunning <- true  
149 } 147 }
150 } 148 }
  149 + endRunning <- true
151 }() 150 }()
152 } 151 }
153 if BConfig.Listen.EnableHTTP { 152 if BConfig.Listen.EnableHTTP {
154 go func() { 153 go func() {
155 - server := grace.NewServer(addr, app.Handlers) 154 + server := grace.NewServer(addr, app.Server.Handler)
156 server.Server.ReadTimeout = app.Server.ReadTimeout 155 server.Server.ReadTimeout = app.Server.ReadTimeout
157 server.Server.WriteTimeout = app.Server.WriteTimeout 156 server.Server.WriteTimeout = app.Server.WriteTimeout
158 if BConfig.Listen.ListenTCP4 { 157 if BConfig.Listen.ListenTCP4 {
@@ -161,8 +160,8 @@ func (app *App) Run(mws ...MiddleWare) { @@ -161,8 +160,8 @@ func (app *App) Run(mws ...MiddleWare) {
161 if err := server.ListenAndServe(); err != nil { 160 if err := server.ListenAndServe(); err != nil {
162 logs.Critical("ListenAndServe: ", err, fmt.Sprintf("%d", os.Getpid())) 161 logs.Critical("ListenAndServe: ", err, fmt.Sprintf("%d", os.Getpid()))
163 time.Sleep(100 * time.Microsecond) 162 time.Sleep(100 * time.Microsecond)
164 - endRunning <- true  
165 } 163 }
  164 + endRunning <- true
166 }() 165 }()
167 } 166 }
168 <-endRunning 167 <-endRunning
@@ -23,7 +23,7 @@ import ( @@ -23,7 +23,7 @@ import (
23 23
24 const ( 24 const (
25 // VERSION represent beego web framework version. 25 // VERSION represent beego web framework version.
26 - VERSION = "1.12.1" 26 + VERSION = "1.12.2"
27 27
28 // DEV is for develop 28 // DEV is for develop
29 DEV = "dev" 29 DEV = "dev"
  1 +// Copyright 2020 astaxie
  2 +//
  3 +// Licensed under the Apache License, Version 2.0 (the "License");
  4 +// you may not use this file except in compliance with the License.
  5 +// You may obtain a copy of the License at
  6 +//
  7 +// http://www.apache.org/licenses/LICENSE-2.0
  8 +//
  9 +// Unless required by applicable law or agreed to in writing, software
  10 +// distributed under the License is distributed on an "AS IS" BASIS,
  11 +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 +// See the License for the specific language governing permissions and
  13 +// limitations under the License.
  14 +
  15 +package beego
  16 +
  17 +var (
  18 + BuildVersion string
  19 + BuildGitRevision string
  20 + BuildStatus string
  21 + BuildTag string
  22 + BuildTime string
  23 +
  24 + GoVersion string
  25 +
  26 + GitBranch string
  27 +)
@@ -81,6 +81,8 @@ type WebConfig struct { @@ -81,6 +81,8 @@ type WebConfig struct {
81 DirectoryIndex bool 81 DirectoryIndex bool
82 StaticDir map[string]string 82 StaticDir map[string]string
83 StaticExtensionsToGzip []string 83 StaticExtensionsToGzip []string
  84 + StaticCacheFileSize int
  85 + StaticCacheFileNum int
84 TemplateLeft string 86 TemplateLeft string
85 TemplateRight string 87 TemplateRight string
86 ViewsPath string 88 ViewsPath string
@@ -129,6 +131,8 @@ var ( @@ -129,6 +131,8 @@ var (
129 appConfigPath string 131 appConfigPath string
130 // appConfigProvider is the provider for the config, default is ini 132 // appConfigProvider is the provider for the config, default is ini
131 appConfigProvider = "ini" 133 appConfigProvider = "ini"
  134 + // WorkPath is the absolute path to project root directory
  135 + WorkPath string
132 ) 136 )
133 137
134 func init() { 138 func init() {
@@ -137,7 +141,7 @@ func init() { @@ -137,7 +141,7 @@ func init() {
137 if AppPath, err = filepath.Abs(filepath.Dir(os.Args[0])); err != nil { 141 if AppPath, err = filepath.Abs(filepath.Dir(os.Args[0])); err != nil {
138 panic(err) 142 panic(err)
139 } 143 }
140 - workPath, err := os.Getwd() 144 + WorkPath, err = os.Getwd()
141 if err != nil { 145 if err != nil {
142 panic(err) 146 panic(err)
143 } 147 }
@@ -145,7 +149,7 @@ func init() { @@ -145,7 +149,7 @@ func init() {
145 if os.Getenv("BEEGO_RUNMODE") != "" { 149 if os.Getenv("BEEGO_RUNMODE") != "" {
146 filename = os.Getenv("BEEGO_RUNMODE") + ".app.conf" 150 filename = os.Getenv("BEEGO_RUNMODE") + ".app.conf"
147 } 151 }
148 - appConfigPath = filepath.Join(workPath, "conf", filename) 152 + appConfigPath = filepath.Join(WorkPath, "conf", filename)
149 if !utils.FileExists(appConfigPath) { 153 if !utils.FileExists(appConfigPath) {
150 appConfigPath = filepath.Join(AppPath, "conf", filename) 154 appConfigPath = filepath.Join(AppPath, "conf", filename)
151 if !utils.FileExists(appConfigPath) { 155 if !utils.FileExists(appConfigPath) {
@@ -236,6 +240,8 @@ func newBConfig() *Config { @@ -236,6 +240,8 @@ func newBConfig() *Config {
236 DirectoryIndex: false, 240 DirectoryIndex: false,
237 StaticDir: map[string]string{"/static": "static"}, 241 StaticDir: map[string]string{"/static": "static"},
238 StaticExtensionsToGzip: []string{".css", ".js"}, 242 StaticExtensionsToGzip: []string{".css", ".js"},
  243 + StaticCacheFileSize: 1024 * 100,
  244 + StaticCacheFileNum: 1000,
239 TemplateLeft: "{{", 245 TemplateLeft: "{{",
240 TemplateRight: "}}", 246 TemplateRight: "}}",
241 ViewsPath: "views", 247 ViewsPath: "views",
@@ -317,6 +323,14 @@ func assignConfig(ac config.Configer) error { @@ -317,6 +323,14 @@ func assignConfig(ac config.Configer) error {
317 } 323 }
318 } 324 }
319 325
  326 + if sfs, err := ac.Int("StaticCacheFileSize"); err == nil {
  327 + BConfig.WebConfig.StaticCacheFileSize = sfs
  328 + }
  329 +
  330 + if sfn, err := ac.Int("StaticCacheFileNum"); err == nil {
  331 + BConfig.WebConfig.StaticCacheFileNum = sfn
  332 + }
  333 +
320 if lo := ac.String("LogOutputs"); lo != "" { 334 if lo := ac.String("LogOutputs"); lo != "" {
321 // if lo is not nil or empty 335 // if lo is not nil or empty
322 // means user has set his own LogOutputs 336 // means user has set his own LogOutputs
@@ -408,9 +422,9 @@ func newAppConfig(appConfigProvider, appConfigPath string) (*beegoAppConfig, err @@ -408,9 +422,9 @@ func newAppConfig(appConfigProvider, appConfigPath string) (*beegoAppConfig, err
408 422
409 func (b *beegoAppConfig) Set(key, val string) error { 423 func (b *beegoAppConfig) Set(key, val string) error {
410 if err := b.innerConfig.Set(BConfig.RunMode+"::"+key, val); err != nil { 424 if err := b.innerConfig.Set(BConfig.RunMode+"::"+key, val); err != nil {
411 - return err 425 + return b.innerConfig.Set(key, val)
412 } 426 }
413 - return b.innerConfig.Set(key, val) 427 + return nil
414 } 428 }
415 429
416 func (b *beegoAppConfig) String(key string) string { 430 func (b *beegoAppConfig) String(key string) string {
@@ -20,6 +20,7 @@ import ( @@ -20,6 +20,7 @@ import (
20 "fmt" 20 "fmt"
21 "io/ioutil" 21 "io/ioutil"
22 "os" 22 "os"
  23 + "strconv"
23 "strings" 24 "strings"
24 "sync" 25 "sync"
25 ) 26 )
@@ -94,8 +95,10 @@ func (c *JSONConfigContainer) Int(key string) (int, error) { @@ -94,8 +95,10 @@ func (c *JSONConfigContainer) Int(key string) (int, error) {
94 if val != nil { 95 if val != nil {
95 if v, ok := val.(float64); ok { 96 if v, ok := val.(float64); ok {
96 return int(v), nil 97 return int(v), nil
  98 + } else if v, ok := val.(string); ok {
  99 + return strconv.Atoi(v)
97 } 100 }
98 - return 0, errors.New("not int value") 101 + return 0, errors.New("not valid value")
99 } 102 }
100 return 0, errors.New("not exist key:" + key) 103 return 0, errors.New("not exist key:" + key)
101 } 104 }
@@ -71,7 +71,9 @@ func (input *BeegoInput) Reset(ctx *Context) { @@ -71,7 +71,9 @@ func (input *BeegoInput) Reset(ctx *Context) {
71 input.CruSession = nil 71 input.CruSession = nil
72 input.pnames = input.pnames[:0] 72 input.pnames = input.pnames[:0]
73 input.pvalues = input.pvalues[:0] 73 input.pvalues = input.pvalues[:0]
  74 + input.dataLock.Lock()
74 input.data = nil 75 input.data = nil
  76 + input.dataLock.Unlock()
75 input.RequestBody = []byte{} 77 input.RequestBody = []byte{}
76 } 78 }
77 79
@@ -87,7 +89,7 @@ func (input *BeegoInput) URI() string { @@ -87,7 +89,7 @@ func (input *BeegoInput) URI() string {
87 89
88 // URL returns request url path (without query string, fragment). 90 // URL returns request url path (without query string, fragment).
89 func (input *BeegoInput) URL() string { 91 func (input *BeegoInput) URL() string {
90 - return input.Context.Request.URL.Path 92 + return input.Context.Request.URL.EscapedPath()
91 } 93 }
92 94
93 // Site returns base site url as scheme://domain type. 95 // Site returns base site url as scheme://domain type.
@@ -282,6 +284,11 @@ func (input *BeegoInput) ParamsLen() int { @@ -282,6 +284,11 @@ func (input *BeegoInput) ParamsLen() int {
282 func (input *BeegoInput) Param(key string) string { 284 func (input *BeegoInput) Param(key string) string {
283 for i, v := range input.pnames { 285 for i, v := range input.pnames {
284 if v == key && i <= len(input.pvalues) { 286 if v == key && i <= len(input.pvalues) {
  287 + // we cannot use url.PathEscape(input.pvalues[i])
  288 + // for example, if the value is /a/b
  289 + // after url.PathEscape(input.pvalues[i]), the value is %2Fa%2Fb
  290 + // However, the value is used in ControllerRegister.ServeHTTP
  291 + // and split by "/", so function crash...
285 return input.pvalues[i] 292 return input.pvalues[i]
286 } 293 }
287 } 294 }
@@ -2,37 +2,35 @@ module github.com/astaxie/beego @@ -2,37 +2,35 @@ module github.com/astaxie/beego
2 2
3 require ( 3 require (
4 github.com/Knetic/govaluate v3.0.0+incompatible // indirect 4 github.com/Knetic/govaluate v3.0.0+incompatible // indirect
5 - github.com/OwnLocal/goes v1.0.0  
6 github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd 5 github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd
7 github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542 6 github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542
8 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 7 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737
9 github.com/casbin/casbin v1.7.0 8 github.com/casbin/casbin v1.7.0
10 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 9 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58
11 - github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb  
12 - github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c // indirect 10 + github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d
  11 + github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808 // indirect
13 github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a // indirect 12 github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a // indirect
14 - github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 // indirect  
15 - github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect 13 + github.com/elastic/go-elasticsearch/v6 v6.8.5
16 github.com/elazarl/go-bindata-assetfs v1.0.0 14 github.com/elazarl/go-bindata-assetfs v1.0.0
17 github.com/go-redis/redis v6.14.2+incompatible 15 github.com/go-redis/redis v6.14.2+incompatible
18 - github.com/go-sql-driver/mysql v1.4.1 16 + github.com/go-sql-driver/mysql v1.5.0
19 github.com/gogo/protobuf v1.1.1 17 github.com/gogo/protobuf v1.1.1
20 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect 18 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
21 github.com/gomodule/redigo v2.0.0+incompatible 19 github.com/gomodule/redigo v2.0.0+incompatible
  20 + github.com/hashicorp/golang-lru v0.5.4
  21 + github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6
22 github.com/lib/pq v1.0.0 22 github.com/lib/pq v1.0.0
23 - github.com/mattn/go-sqlite3 v1.10.0 23 + github.com/mattn/go-sqlite3 v2.0.3+incompatible
24 github.com/pelletier/go-toml v1.2.0 // indirect 24 github.com/pelletier/go-toml v1.2.0 // indirect
25 - github.com/pkg/errors v0.8.0 // indirect 25 + github.com/prometheus/client_golang v1.7.0
26 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 26 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644
27 - github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 // indirect  
28 - github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373  
29 - github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d // indirect  
30 github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec 27 github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec
  28 + github.com/stretchr/testify v1.4.0
31 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c // indirect 29 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c // indirect
32 github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b // indirect 30 github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b // indirect
33 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 31 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
34 golang.org/x/tools v0.0.0-20200117065230-39095c1d176c 32 golang.org/x/tools v0.0.0-20200117065230-39095c1d176c
35 - gopkg.in/yaml.v2 v2.2.1 33 + gopkg.in/yaml.v2 v2.2.8
36 ) 34 )
37 35
38 replace golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85 => github.com/golang/crypto v0.0.0-20181127143415-eb0de9b17e85 36 replace golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85 => github.com/golang/crypto v0.0.0-20181127143415-eb0de9b17e85
@@ -46,7 +46,10 @@ func (srv *Server) Serve() (err error) { @@ -46,7 +46,10 @@ func (srv *Server) Serve() (err error) {
46 46
47 log.Println(syscall.Getpid(), srv.ln.Addr(), "Listener closed.") 47 log.Println(syscall.Getpid(), srv.ln.Addr(), "Listener closed.")
48 // wait for Shutdown to return 48 // wait for Shutdown to return
49 - return <-srv.terminalChan 49 + if shutdownErr := <-srv.terminalChan; shutdownErr != nil {
  50 + return shutdownErr
  51 + }
  52 + return
50 } 53 }
51 54
52 // ListenAndServe listens on the TCP network address srv.Addr and then calls Serve 55 // ListenAndServe listens on the TCP network address srv.Addr and then calls Serve
@@ -180,7 +183,7 @@ func (srv *Server) ListenAndServeMutualTLS(certFile, keyFile, trustFile string) @@ -180,7 +183,7 @@ func (srv *Server) ListenAndServeMutualTLS(certFile, keyFile, trustFile string)
180 log.Println(err) 183 log.Println(err)
181 return err 184 return err
182 } 185 }
183 - err = process.Kill() 186 + err = process.Signal(syscall.SIGTERM)
184 if err != nil { 187 if err != nil {
185 return err 188 return err
186 } 189 }
  1 +# httplib
  2 +httplib is an libs help you to curl remote url.
  3 +
  4 +# How to use?
  5 +
  6 +## GET
  7 +you can use Get to crawl data.
  8 +
  9 + import "github.com/astaxie/beego/httplib"
  10 +
  11 + str, err := httplib.Get("http://beego.me/").String()
  12 + if err != nil {
  13 + // error
  14 + }
  15 + fmt.Println(str)
  16 +
  17 +## POST
  18 +POST data to remote url
  19 +
  20 + req := httplib.Post("http://beego.me/")
  21 + req.Param("username","astaxie")
  22 + req.Param("password","123456")
  23 + str, err := req.String()
  24 + if err != nil {
  25 + // error
  26 + }
  27 + fmt.Println(str)
  28 +
  29 +## Set timeout
  30 +
  31 +The default timeout is `60` seconds, function prototype:
  32 +
  33 + SetTimeout(connectTimeout, readWriteTimeout time.Duration)
  34 +
  35 +Example:
  36 +
  37 + // GET
  38 + httplib.Get("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second)
  39 +
  40 + // POST
  41 + httplib.Post("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second)
  42 +
  43 +
  44 +## Debug
  45 +
  46 +If you want to debug the request info, set the debug on
  47 +
  48 + httplib.Get("http://beego.me/").Debug(true)
  49 +
  50 +## Set HTTP Basic Auth
  51 +
  52 + str, err := Get("http://beego.me/").SetBasicAuth("user", "passwd").String()
  53 + if err != nil {
  54 + // error
  55 + }
  56 + fmt.Println(str)
  57 +
  58 +## Set HTTPS
  59 +
  60 +If request url is https, You can set the client support TSL:
  61 +
  62 + httplib.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
  63 +
  64 +More info about the `tls.Config` please visit http://golang.org/pkg/crypto/tls/#Config
  65 +
  66 +## Set HTTP Version
  67 +
  68 +some servers need to specify the protocol version of HTTP
  69 +
  70 + httplib.Get("http://beego.me/").SetProtocolVersion("HTTP/1.1")
  71 +
  72 +## Set Cookie
  73 +
  74 +some http request need setcookie. So set it like this:
  75 +
  76 + cookie := &http.Cookie{}
  77 + cookie.Name = "username"
  78 + cookie.Value = "astaxie"
  79 + httplib.Get("http://beego.me/").SetCookie(cookie)
  80 +
  81 +## Upload file
  82 +
  83 +httplib support mutil file upload, use `req.PostFile()`
  84 +
  85 + req := httplib.Post("http://beego.me/")
  86 + req.Param("username","astaxie")
  87 + req.PostFile("uploadfile1", "httplib.pdf")
  88 + str, err := req.String()
  89 + if err != nil {
  90 + // error
  91 + }
  92 + fmt.Println(str)
  93 +
  94 +
  95 +See godoc for further documentation and examples.
  96 +
  97 +* [godoc.org/github.com/astaxie/beego/httplib](https://godoc.org/github.com/astaxie/beego/httplib)
  1 +// Copyright 2014 beego Author. All Rights Reserved.
  2 +//
  3 +// Licensed under the Apache License, Version 2.0 (the "License");
  4 +// you may not use this file except in compliance with the License.
  5 +// You may obtain a copy of the License at
  6 +//
  7 +// http://www.apache.org/licenses/LICENSE-2.0
  8 +//
  9 +// Unless required by applicable law or agreed to in writing, software
  10 +// distributed under the License is distributed on an "AS IS" BASIS,
  11 +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 +// See the License for the specific language governing permissions and
  13 +// limitations under the License.
  14 +
  15 +// Package httplib is used as http.Client
  16 +// Usage:
  17 +//
  18 +// import "github.com/astaxie/beego/httplib"
  19 +//
  20 +// b := httplib.Post("http://beego.me/")
  21 +// b.Param("username","astaxie")
  22 +// b.Param("password","123456")
  23 +// b.PostFile("uploadfile1", "httplib.pdf")
  24 +// b.PostFile("uploadfile2", "httplib.txt")
  25 +// str, err := b.String()
  26 +// if err != nil {
  27 +// t.Fatal(err)
  28 +// }
  29 +// fmt.Println(str)
  30 +//
  31 +// more docs http://beego.me/docs/module/httplib.md
  32 +package httplib
  33 +
  34 +import (
  35 + "bytes"
  36 + "compress/gzip"
  37 + "crypto/tls"
  38 + "encoding/json"
  39 + "encoding/xml"
  40 + "io"
  41 + "io/ioutil"
  42 + "log"
  43 + "mime/multipart"
  44 + "net"
  45 + "net/http"
  46 + "net/http/cookiejar"
  47 + "net/http/httputil"
  48 + "net/url"
  49 + "os"
  50 + "path"
  51 + "strings"
  52 + "sync"
  53 + "time"
  54 +
  55 + "gopkg.in/yaml.v2"
  56 +)
  57 +
  58 +var defaultSetting = BeegoHTTPSettings{
  59 + UserAgent: "beegoServer",
  60 + ConnectTimeout: 60 * time.Second,
  61 + ReadWriteTimeout: 60 * time.Second,
  62 + Gzip: true,
  63 + DumpBody: true,
  64 +}
  65 +
  66 +var defaultCookieJar http.CookieJar
  67 +var settingMutex sync.Mutex
  68 +
  69 +// createDefaultCookie creates a global cookiejar to store cookies.
  70 +func createDefaultCookie() {
  71 + settingMutex.Lock()
  72 + defer settingMutex.Unlock()
  73 + defaultCookieJar, _ = cookiejar.New(nil)
  74 +}
  75 +
  76 +// SetDefaultSetting Overwrite default settings
  77 +func SetDefaultSetting(setting BeegoHTTPSettings) {
  78 + settingMutex.Lock()
  79 + defer settingMutex.Unlock()
  80 + defaultSetting = setting
  81 +}
  82 +
  83 +// NewBeegoRequest return *BeegoHttpRequest with specific method
  84 +func NewBeegoRequest(rawurl, method string) *BeegoHTTPRequest {
  85 + var resp http.Response
  86 + u, err := url.Parse(rawurl)
  87 + if err != nil {
  88 + log.Println("Httplib:", err)
  89 + }
  90 + req := http.Request{
  91 + URL: u,
  92 + Method: method,
  93 + Header: make(http.Header),
  94 + Proto: "HTTP/1.1",
  95 + ProtoMajor: 1,
  96 + ProtoMinor: 1,
  97 + }
  98 + return &BeegoHTTPRequest{
  99 + url: rawurl,
  100 + req: &req,
  101 + params: map[string][]string{},
  102 + files: map[string]string{},
  103 + setting: defaultSetting,
  104 + resp: &resp,
  105 + }
  106 +}
  107 +
  108 +// Get returns *BeegoHttpRequest with GET method.
  109 +func Get(url string) *BeegoHTTPRequest {
  110 + return NewBeegoRequest(url, "GET")
  111 +}
  112 +
  113 +// Post returns *BeegoHttpRequest with POST method.
  114 +func Post(url string) *BeegoHTTPRequest {
  115 + return NewBeegoRequest(url, "POST")
  116 +}
  117 +
  118 +// Put returns *BeegoHttpRequest with PUT method.
  119 +func Put(url string) *BeegoHTTPRequest {
  120 + return NewBeegoRequest(url, "PUT")
  121 +}
  122 +
  123 +// Delete returns *BeegoHttpRequest DELETE method.
  124 +func Delete(url string) *BeegoHTTPRequest {
  125 + return NewBeegoRequest(url, "DELETE")
  126 +}
  127 +
  128 +// Head returns *BeegoHttpRequest with HEAD method.
  129 +func Head(url string) *BeegoHTTPRequest {
  130 + return NewBeegoRequest(url, "HEAD")
  131 +}
  132 +
  133 +// BeegoHTTPSettings is the http.Client setting
  134 +type BeegoHTTPSettings struct {
  135 + ShowDebug bool
  136 + UserAgent string
  137 + ConnectTimeout time.Duration
  138 + ReadWriteTimeout time.Duration
  139 + TLSClientConfig *tls.Config
  140 + Proxy func(*http.Request) (*url.URL, error)
  141 + Transport http.RoundTripper
  142 + CheckRedirect func(req *http.Request, via []*http.Request) error
  143 + EnableCookie bool
  144 + Gzip bool
  145 + DumpBody bool
  146 + Retries int // if set to -1 means will retry forever
  147 +}
  148 +
  149 +// BeegoHTTPRequest provides more useful methods for requesting one url than http.Request.
  150 +type BeegoHTTPRequest struct {
  151 + url string
  152 + req *http.Request
  153 + params map[string][]string
  154 + files map[string]string
  155 + setting BeegoHTTPSettings
  156 + resp *http.Response
  157 + body []byte
  158 + dump []byte
  159 +}
  160 +
  161 +// GetRequest return the request object
  162 +func (b *BeegoHTTPRequest) GetRequest() *http.Request {
  163 + return b.req
  164 +}
  165 +
  166 +// Setting Change request settings
  167 +func (b *BeegoHTTPRequest) Setting(setting BeegoHTTPSettings) *BeegoHTTPRequest {
  168 + b.setting = setting
  169 + return b
  170 +}
  171 +
  172 +// SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password.
  173 +func (b *BeegoHTTPRequest) SetBasicAuth(username, password string) *BeegoHTTPRequest {
  174 + b.req.SetBasicAuth(username, password)
  175 + return b
  176 +}
  177 +
  178 +// SetEnableCookie sets enable/disable cookiejar
  179 +func (b *BeegoHTTPRequest) SetEnableCookie(enable bool) *BeegoHTTPRequest {
  180 + b.setting.EnableCookie = enable
  181 + return b
  182 +}
  183 +
  184 +// SetUserAgent sets User-Agent header field
  185 +func (b *BeegoHTTPRequest) SetUserAgent(useragent string) *BeegoHTTPRequest {
  186 + b.setting.UserAgent = useragent
  187 + return b
  188 +}
  189 +
  190 +// Debug sets show debug or not when executing request.
  191 +func (b *BeegoHTTPRequest) Debug(isdebug bool) *BeegoHTTPRequest {
  192 + b.setting.ShowDebug = isdebug
  193 + return b
  194 +}
  195 +
  196 +// Retries sets Retries times.
  197 +// default is 0 means no retried.
  198 +// -1 means retried forever.
  199 +// others means retried times.
  200 +func (b *BeegoHTTPRequest) Retries(times int) *BeegoHTTPRequest {
  201 + b.setting.Retries = times
  202 + return b
  203 +}
  204 +
  205 +// DumpBody setting whether need to Dump the Body.
  206 +func (b *BeegoHTTPRequest) DumpBody(isdump bool) *BeegoHTTPRequest {
  207 + b.setting.DumpBody = isdump
  208 + return b
  209 +}
  210 +
  211 +// DumpRequest return the DumpRequest
  212 +func (b *BeegoHTTPRequest) DumpRequest() []byte {
  213 + return b.dump
  214 +}
  215 +
  216 +// SetTimeout sets connect time out and read-write time out for BeegoRequest.
  217 +func (b *BeegoHTTPRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHTTPRequest {
  218 + b.setting.ConnectTimeout = connectTimeout
  219 + b.setting.ReadWriteTimeout = readWriteTimeout
  220 + return b
  221 +}
  222 +
  223 +// SetTLSClientConfig sets tls connection configurations if visiting https url.
  224 +func (b *BeegoHTTPRequest) SetTLSClientConfig(config *tls.Config) *BeegoHTTPRequest {
  225 + b.setting.TLSClientConfig = config
  226 + return b
  227 +}
  228 +
  229 +// Header add header item string in request.
  230 +func (b *BeegoHTTPRequest) Header(key, value string) *BeegoHTTPRequest {
  231 + b.req.Header.Set(key, value)
  232 + return b
  233 +}
  234 +
  235 +// SetHost set the request host
  236 +func (b *BeegoHTTPRequest) SetHost(host string) *BeegoHTTPRequest {
  237 + b.req.Host = host
  238 + return b
  239 +}
  240 +
  241 +// SetProtocolVersion Set the protocol version for incoming requests.
  242 +// Client requests always use HTTP/1.1.
  243 +func (b *BeegoHTTPRequest) SetProtocolVersion(vers string) *BeegoHTTPRequest {
  244 + if len(vers) == 0 {
  245 + vers = "HTTP/1.1"
  246 + }
  247 +
  248 + major, minor, ok := http.ParseHTTPVersion(vers)
  249 + if ok {
  250 + b.req.Proto = vers
  251 + b.req.ProtoMajor = major
  252 + b.req.ProtoMinor = minor
  253 + }
  254 +
  255 + return b
  256 +}
  257 +
  258 +// SetCookie add cookie into request.
  259 +func (b *BeegoHTTPRequest) SetCookie(cookie *http.Cookie) *BeegoHTTPRequest {
  260 + b.req.Header.Add("Cookie", cookie.String())
  261 + return b
  262 +}
  263 +
  264 +// SetTransport set the setting transport
  265 +func (b *BeegoHTTPRequest) SetTransport(transport http.RoundTripper) *BeegoHTTPRequest {
  266 + b.setting.Transport = transport
  267 + return b
  268 +}
  269 +
  270 +// SetProxy set the http proxy
  271 +// example:
  272 +//
  273 +// func(req *http.Request) (*url.URL, error) {
  274 +// u, _ := url.ParseRequestURI("http://127.0.0.1:8118")
  275 +// return u, nil
  276 +// }
  277 +func (b *BeegoHTTPRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHTTPRequest {
  278 + b.setting.Proxy = proxy
  279 + return b
  280 +}
  281 +
  282 +// SetCheckRedirect specifies the policy for handling redirects.
  283 +//
  284 +// If CheckRedirect is nil, the Client uses its default policy,
  285 +// which is to stop after 10 consecutive requests.
  286 +func (b *BeegoHTTPRequest) SetCheckRedirect(redirect func(req *http.Request, via []*http.Request) error) *BeegoHTTPRequest {
  287 + b.setting.CheckRedirect = redirect
  288 + return b
  289 +}
  290 +
  291 +// Param adds query param in to request.
  292 +// params build query string as ?key1=value1&key2=value2...
  293 +func (b *BeegoHTTPRequest) Param(key, value string) *BeegoHTTPRequest {
  294 + if param, ok := b.params[key]; ok {
  295 + b.params[key] = append(param, value)
  296 + } else {
  297 + b.params[key] = []string{value}
  298 + }
  299 + return b
  300 +}
  301 +
  302 +// PostFile add a post file to the request
  303 +func (b *BeegoHTTPRequest) PostFile(formname, filename string) *BeegoHTTPRequest {
  304 + b.files[formname] = filename
  305 + return b
  306 +}
  307 +
  308 +// Body adds request raw body.
  309 +// it supports string and []byte.
  310 +func (b *BeegoHTTPRequest) Body(data interface{}) *BeegoHTTPRequest {
  311 + switch t := data.(type) {
  312 + case string:
  313 + bf := bytes.NewBufferString(t)
  314 + b.req.Body = ioutil.NopCloser(bf)
  315 + b.req.ContentLength = int64(len(t))
  316 + case []byte:
  317 + bf := bytes.NewBuffer(t)
  318 + b.req.Body = ioutil.NopCloser(bf)
  319 + b.req.ContentLength = int64(len(t))
  320 + }
  321 + return b
  322 +}
  323 +
  324 +// XMLBody adds request raw body encoding by XML.
  325 +func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) {
  326 + if b.req.Body == nil && obj != nil {
  327 + byts, err := xml.Marshal(obj)
  328 + if err != nil {
  329 + return b, err
  330 + }
  331 + b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
  332 + b.req.ContentLength = int64(len(byts))
  333 + b.req.Header.Set("Content-Type", "application/xml")
  334 + }
  335 + return b, nil
  336 +}
  337 +
  338 +// YAMLBody adds request raw body encoding by YAML.
  339 +func (b *BeegoHTTPRequest) YAMLBody(obj interface{}) (*BeegoHTTPRequest, error) {
  340 + if b.req.Body == nil && obj != nil {
  341 + byts, err := yaml.Marshal(obj)
  342 + if err != nil {
  343 + return b, err
  344 + }
  345 + b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
  346 + b.req.ContentLength = int64(len(byts))
  347 + b.req.Header.Set("Content-Type", "application/x+yaml")
  348 + }
  349 + return b, nil
  350 +}
  351 +
  352 +// JSONBody adds request raw body encoding by JSON.
  353 +func (b *BeegoHTTPRequest) JSONBody(obj interface{}) (*BeegoHTTPRequest, error) {
  354 + if b.req.Body == nil && obj != nil {
  355 + byts, err := json.Marshal(obj)
  356 + if err != nil {
  357 + return b, err
  358 + }
  359 + b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
  360 + b.req.ContentLength = int64(len(byts))
  361 + b.req.Header.Set("Content-Type", "application/json")
  362 + }
  363 + return b, nil
  364 +}
  365 +
  366 +func (b *BeegoHTTPRequest) buildURL(paramBody string) {
  367 + // build GET url with query string
  368 + if b.req.Method == "GET" && len(paramBody) > 0 {
  369 + if strings.Contains(b.url, "?") {
  370 + b.url += "&" + paramBody
  371 + } else {
  372 + b.url = b.url + "?" + paramBody
  373 + }
  374 + return
  375 + }
  376 +
  377 + // build POST/PUT/PATCH url and body
  378 + if (b.req.Method == "POST" || b.req.Method == "PUT" || b.req.Method == "PATCH" || b.req.Method == "DELETE") && b.req.Body == nil {
  379 + // with files
  380 + if len(b.files) > 0 {
  381 + pr, pw := io.Pipe()
  382 + bodyWriter := multipart.NewWriter(pw)
  383 + go func() {
  384 + for formname, filename := range b.files {
  385 + fileWriter, err := bodyWriter.CreateFormFile(formname, filename)
  386 + if err != nil {
  387 + log.Println("Httplib:", err)
  388 + }
  389 + fh, err := os.Open(filename)
  390 + if err != nil {
  391 + log.Println("Httplib:", err)
  392 + }
  393 + //iocopy
  394 + _, err = io.Copy(fileWriter, fh)
  395 + fh.Close()
  396 + if err != nil {
  397 + log.Println("Httplib:", err)
  398 + }
  399 + }
  400 + for k, v := range b.params {
  401 + for _, vv := range v {
  402 + bodyWriter.WriteField(k, vv)
  403 + }
  404 + }
  405 + bodyWriter.Close()
  406 + pw.Close()
  407 + }()
  408 + b.Header("Content-Type", bodyWriter.FormDataContentType())
  409 + b.req.Body = ioutil.NopCloser(pr)
  410 + b.Header("Transfer-Encoding", "chunked")
  411 + return
  412 + }
  413 +
  414 + // with params
  415 + if len(paramBody) > 0 {
  416 + b.Header("Content-Type", "application/x-www-form-urlencoded")
  417 + b.Body(paramBody)
  418 + }
  419 + }
  420 +}
  421 +
  422 +func (b *BeegoHTTPRequest) getResponse() (*http.Response, error) {
  423 + if b.resp.StatusCode != 0 {
  424 + return b.resp, nil
  425 + }
  426 + resp, err := b.DoRequest()
  427 + if err != nil {
  428 + return nil, err
  429 + }
  430 + b.resp = resp
  431 + return resp, nil
  432 +}
  433 +
  434 +// DoRequest will do the client.Do
  435 +func (b *BeegoHTTPRequest) DoRequest() (resp *http.Response, err error) {
  436 + var paramBody string
  437 + if len(b.params) > 0 {
  438 + var buf bytes.Buffer
  439 + for k, v := range b.params {
  440 + for _, vv := range v {
  441 + buf.WriteString(url.QueryEscape(k))
  442 + buf.WriteByte('=')
  443 + buf.WriteString(url.QueryEscape(vv))
  444 + buf.WriteByte('&')
  445 + }
  446 + }
  447 + paramBody = buf.String()
  448 + paramBody = paramBody[0 : len(paramBody)-1]
  449 + }
  450 +
  451 + b.buildURL(paramBody)
  452 + urlParsed, err := url.Parse(b.url)
  453 + if err != nil {
  454 + return nil, err
  455 + }
  456 +
  457 + b.req.URL = urlParsed
  458 +
  459 + trans := b.setting.Transport
  460 +
  461 + if trans == nil {
  462 + // create default transport
  463 + trans = &http.Transport{
  464 + TLSClientConfig: b.setting.TLSClientConfig,
  465 + Proxy: b.setting.Proxy,
  466 + Dial: TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
  467 + MaxIdleConnsPerHost: 100,
  468 + }
  469 + } else {
  470 + // if b.transport is *http.Transport then set the settings.
  471 + if t, ok := trans.(*http.Transport); ok {
  472 + if t.TLSClientConfig == nil {
  473 + t.TLSClientConfig = b.setting.TLSClientConfig
  474 + }
  475 + if t.Proxy == nil {
  476 + t.Proxy = b.setting.Proxy
  477 + }
  478 + if t.Dial == nil {
  479 + t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout)
  480 + }
  481 + }
  482 + }
  483 +
  484 + var jar http.CookieJar
  485 + if b.setting.EnableCookie {
  486 + if defaultCookieJar == nil {
  487 + createDefaultCookie()
  488 + }
  489 + jar = defaultCookieJar
  490 + }
  491 +
  492 + client := &http.Client{
  493 + Transport: trans,
  494 + Jar: jar,
  495 + }
  496 +
  497 + if b.setting.UserAgent != "" && b.req.Header.Get("User-Agent") == "" {
  498 + b.req.Header.Set("User-Agent", b.setting.UserAgent)
  499 + }
  500 +
  501 + if b.setting.CheckRedirect != nil {
  502 + client.CheckRedirect = b.setting.CheckRedirect
  503 + }
  504 +
  505 + if b.setting.ShowDebug {
  506 + dump, err := httputil.DumpRequest(b.req, b.setting.DumpBody)
  507 + if err != nil {
  508 + log.Println(err.Error())
  509 + }
  510 + b.dump = dump
  511 + }
  512 + // retries default value is 0, it will run once.
  513 + // retries equal to -1, it will run forever until success
  514 + // retries is setted, it will retries fixed times.
  515 + for i := 0; b.setting.Retries == -1 || i <= b.setting.Retries; i++ {
  516 + resp, err = client.Do(b.req)
  517 + if err == nil {
  518 + break
  519 + }
  520 + }
  521 + return resp, err
  522 +}
  523 +
  524 +// String returns the body string in response.
  525 +// it calls Response inner.
  526 +func (b *BeegoHTTPRequest) String() (string, error) {
  527 + data, err := b.Bytes()
  528 + if err != nil {
  529 + return "", err
  530 + }
  531 +
  532 + return string(data), nil
  533 +}
  534 +
  535 +// Bytes returns the body []byte in response.
  536 +// it calls Response inner.
  537 +func (b *BeegoHTTPRequest) Bytes() ([]byte, error) {
  538 + if b.body != nil {
  539 + return b.body, nil
  540 + }
  541 + resp, err := b.getResponse()
  542 + if err != nil {
  543 + return nil, err
  544 + }
  545 + if resp.Body == nil {
  546 + return nil, nil
  547 + }
  548 + defer resp.Body.Close()
  549 + if b.setting.Gzip && resp.Header.Get("Content-Encoding") == "gzip" {
  550 + reader, err := gzip.NewReader(resp.Body)
  551 + if err != nil {
  552 + return nil, err
  553 + }
  554 + b.body, err = ioutil.ReadAll(reader)
  555 + return b.body, err
  556 + }
  557 + b.body, err = ioutil.ReadAll(resp.Body)
  558 + return b.body, err
  559 +}
  560 +
  561 +// ToFile saves the body data in response to one file.
  562 +// it calls Response inner.
  563 +func (b *BeegoHTTPRequest) ToFile(filename string) error {
  564 + resp, err := b.getResponse()
  565 + if err != nil {
  566 + return err
  567 + }
  568 + if resp.Body == nil {
  569 + return nil
  570 + }
  571 + defer resp.Body.Close()
  572 + err = pathExistAndMkdir(filename)
  573 + if err != nil {
  574 + return err
  575 + }
  576 + f, err := os.Create(filename)
  577 + if err != nil {
  578 + return err
  579 + }
  580 + defer f.Close()
  581 + _, err = io.Copy(f, resp.Body)
  582 + return err
  583 +}
  584 +
  585 +//Check that the file directory exists, there is no automatically created
  586 +func pathExistAndMkdir(filename string) (err error) {
  587 + filename = path.Dir(filename)
  588 + _, err = os.Stat(filename)
  589 + if err == nil {
  590 + return nil
  591 + }
  592 + if os.IsNotExist(err) {
  593 + err = os.MkdirAll(filename, os.ModePerm)
  594 + if err == nil {
  595 + return nil
  596 + }
  597 + }
  598 + return err
  599 +}
  600 +
  601 +// ToJSON returns the map that marshals from the body bytes as json in response .
  602 +// it calls Response inner.
  603 +func (b *BeegoHTTPRequest) ToJSON(v interface{}) error {
  604 + data, err := b.Bytes()
  605 + if err != nil {
  606 + return err
  607 + }
  608 + return json.Unmarshal(data, v)
  609 +}
  610 +
  611 +// ToXML returns the map that marshals from the body bytes as xml in response .
  612 +// it calls Response inner.
  613 +func (b *BeegoHTTPRequest) ToXML(v interface{}) error {
  614 + data, err := b.Bytes()
  615 + if err != nil {
  616 + return err
  617 + }
  618 + return xml.Unmarshal(data, v)
  619 +}
  620 +
  621 +// ToYAML returns the map that marshals from the body bytes as yaml in response .
  622 +// it calls Response inner.
  623 +func (b *BeegoHTTPRequest) ToYAML(v interface{}) error {
  624 + data, err := b.Bytes()
  625 + if err != nil {
  626 + return err
  627 + }
  628 + return yaml.Unmarshal(data, v)
  629 +}
  630 +
  631 +// Response executes request client gets response mannually.
  632 +func (b *BeegoHTTPRequest) Response() (*http.Response, error) {
  633 + return b.getResponse()
  634 +}
  635 +
  636 +// TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field.
  637 +func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
  638 + return func(netw, addr string) (net.Conn, error) {
  639 + conn, err := net.DialTimeout(netw, addr, cTimeout)
  640 + if err != nil {
  641 + return nil, err
  642 + }
  643 + err = conn.SetDeadline(time.Now().Add(rwTimeout))
  644 + return conn, err
  645 + }
  646 +}
@@ -359,6 +359,10 @@ RESTART_LOGGER: @@ -359,6 +359,10 @@ RESTART_LOGGER:
359 359
360 func (w *fileLogWriter) deleteOldLog() { 360 func (w *fileLogWriter) deleteOldLog() {
361 dir := filepath.Dir(w.Filename) 361 dir := filepath.Dir(w.Filename)
  362 + absolutePath, err := filepath.EvalSymlinks(w.Filename)
  363 + if err == nil {
  364 + dir = filepath.Dir(absolutePath)
  365 + }
362 filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) { 366 filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) {
363 defer func() { 367 defer func() {
364 if r := recover(); r != nil { 368 if r := recover(); r != nil {
@@ -295,7 +295,11 @@ func (bl *BeeLogger) writeMsg(logLevel int, msg string, v ...interface{}) error @@ -295,7 +295,11 @@ func (bl *BeeLogger) writeMsg(logLevel int, msg string, v ...interface{}) error
295 lm.level = logLevel 295 lm.level = logLevel
296 lm.msg = msg 296 lm.msg = msg
297 lm.when = when 297 lm.when = when
298 - bl.msgChan <- lm 298 + if bl.outputs != nil {
  299 + bl.msgChan <- lm
  300 + } else {
  301 + logMsgPool.Put(lm)
  302 + }
299 } else { 303 } else {
300 bl.writeToLoggers(when, msg, logLevel) 304 bl.writeToLoggers(when, msg, logLevel)
301 } 305 }
@@ -500,7 +500,7 @@ func genRouterCode(pkgRealpath string) { @@ -500,7 +500,7 @@ func genRouterCode(pkgRealpath string) {
500 beego.GlobalControllerRouter["` + k + `"] = append(beego.GlobalControllerRouter["` + k + `"], 500 beego.GlobalControllerRouter["` + k + `"] = append(beego.GlobalControllerRouter["` + k + `"],
501 beego.ControllerComments{ 501 beego.ControllerComments{
502 Method: "` + strings.TrimSpace(c.Method) + `", 502 Method: "` + strings.TrimSpace(c.Method) + `",
503 - ` + "Router: `" + c.Router + "`" + `, 503 + ` + `Router: "` + c.Router + `"` + `,
504 AllowHTTPMethods: ` + allmethod + `, 504 AllowHTTPMethods: ` + allmethod + `,
505 MethodParams: ` + methodParams + `, 505 MethodParams: ` + methodParams + `,
506 Filters: ` + filters + `, 506 Filters: ` + filters + `,
@@ -18,6 +18,7 @@ import ( @@ -18,6 +18,7 @@ import (
18 "errors" 18 "errors"
19 "fmt" 19 "fmt"
20 "net/http" 20 "net/http"
  21 + "os"
21 "path" 22 "path"
22 "path/filepath" 23 "path/filepath"
23 "reflect" 24 "reflect"
@@ -121,6 +122,10 @@ type ControllerInfo struct { @@ -121,6 +122,10 @@ type ControllerInfo struct {
121 methodParams []*param.MethodParam 122 methodParams []*param.MethodParam
122 } 123 }
123 124
  125 +func (c *ControllerInfo) GetPattern() string {
  126 + return c.pattern
  127 +}
  128 +
124 // ControllerRegister containers registered router rules, controller handlers and filters. 129 // ControllerRegister containers registered router rules, controller handlers and filters.
125 type ControllerRegister struct { 130 type ControllerRegister struct {
126 routers map[string]*Tree 131 routers map[string]*Tree
@@ -249,25 +254,39 @@ func (p *ControllerRegister) addToRouter(method, pattern string, r *ControllerIn @@ -249,25 +254,39 @@ func (p *ControllerRegister) addToRouter(method, pattern string, r *ControllerIn
249 func (p *ControllerRegister) Include(cList ...ControllerInterface) { 254 func (p *ControllerRegister) Include(cList ...ControllerInterface) {
250 if BConfig.RunMode == DEV { 255 if BConfig.RunMode == DEV {
251 skip := make(map[string]bool, 10) 256 skip := make(map[string]bool, 10)
  257 + wgopath := utils.GetGOPATHs()
  258 + go111module := os.Getenv(`GO111MODULE`)
252 for _, c := range cList { 259 for _, c := range cList {
253 reflectVal := reflect.ValueOf(c) 260 reflectVal := reflect.ValueOf(c)
254 t := reflect.Indirect(reflectVal).Type() 261 t := reflect.Indirect(reflectVal).Type()
255 - wgopath := utils.GetGOPATHs()  
256 - if len(wgopath) == 0 {  
257 - panic("you are in dev mode. So please set gopath")  
258 - }  
259 - pkgpath := ""  
260 - for _, wg := range wgopath {  
261 - wg, _ = filepath.EvalSymlinks(filepath.Join(wg, "src", t.PkgPath()))  
262 - if utils.FileExists(wg) {  
263 - pkgpath = wg  
264 - break 262 + // for go modules
  263 + if go111module == `on` {
  264 + pkgpath := filepath.Join(WorkPath, "..", t.PkgPath())
  265 + if utils.FileExists(pkgpath) {
  266 + if pkgpath != "" {
  267 + if _, ok := skip[pkgpath]; !ok {
  268 + skip[pkgpath] = true
  269 + parserPkg(pkgpath, t.PkgPath())
  270 + }
  271 + }
265 } 272 }
266 - }  
267 - if pkgpath != "" {  
268 - if _, ok := skip[pkgpath]; !ok {  
269 - skip[pkgpath] = true  
270 - parserPkg(pkgpath, t.PkgPath()) 273 + } else {
  274 + if len(wgopath) == 0 {
  275 + panic("you are in dev mode. So please set gopath")
  276 + }
  277 + pkgpath := ""
  278 + for _, wg := range wgopath {
  279 + wg, _ = filepath.EvalSymlinks(filepath.Join(wg, "src", t.PkgPath()))
  280 + if utils.FileExists(wg) {
  281 + pkgpath = wg
  282 + break
  283 + }
  284 + }
  285 + if pkgpath != "" {
  286 + if _, ok := skip[pkgpath]; !ok {
  287 + skip[pkgpath] = true
  288 + parserPkg(pkgpath, t.PkgPath())
  289 + }
271 } 290 }
272 } 291 }
273 } 292 }
@@ -288,6 +307,21 @@ func (p *ControllerRegister) Include(cList ...ControllerInterface) { @@ -288,6 +307,21 @@ func (p *ControllerRegister) Include(cList ...ControllerInterface) {
288 } 307 }
289 } 308 }
290 309
  310 +// GetContext returns a context from pool, so usually you should remember to call Reset function to clean the context
  311 +// And don't forget to give back context to pool
  312 +// example:
  313 +// ctx := p.GetContext()
  314 +// ctx.Reset(w, q)
  315 +// defer p.GiveBackContext(ctx)
  316 +func (p *ControllerRegister) GetContext() *beecontext.Context {
  317 + return p.pool.Get().(*beecontext.Context)
  318 +}
  319 +
  320 +// GiveBackContext put the ctx into pool so that it could be reuse
  321 +func (p *ControllerRegister) GiveBackContext(ctx *beecontext.Context) {
  322 + p.pool.Put(ctx)
  323 +}
  324 +
291 // Get add get method 325 // Get add get method
292 // usage: 326 // usage:
293 // Get("/", func(ctx *context.Context){ 327 // Get("/", func(ctx *context.Context){
@@ -667,10 +701,11 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -667,10 +701,11 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
667 routerInfo *ControllerInfo 701 routerInfo *ControllerInfo
668 isRunnable bool 702 isRunnable bool
669 ) 703 )
670 - context := p.pool.Get().(*beecontext.Context) 704 + context := p.GetContext()
  705 +
671 context.Reset(rw, r) 706 context.Reset(rw, r)
672 707
673 - defer p.pool.Put(context) 708 + defer p.GiveBackContext(context)
674 if BConfig.RecoverFunc != nil { 709 if BConfig.RecoverFunc != nil {
675 defer BConfig.RecoverFunc(context) 710 defer BConfig.RecoverFunc(context)
676 } 711 }
@@ -739,7 +774,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -739,7 +774,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
739 routerInfo, findRouter = p.FindRouter(context) 774 routerInfo, findRouter = p.FindRouter(context)
740 } 775 }
741 776
742 - //if no matches to url, throw a not found exception 777 + // if no matches to url, throw a not found exception
743 if !findRouter { 778 if !findRouter {
744 exception("404", context) 779 exception("404", context)
745 goto Admin 780 goto Admin
@@ -750,19 +785,22 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -750,19 +785,22 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
750 } 785 }
751 } 786 }
752 787
753 - //execute middleware filters 788 + if routerInfo != nil {
  789 + // store router pattern into context
  790 + context.Input.SetData("RouterPattern", routerInfo.pattern)
  791 + }
  792 +
  793 + // execute middleware filters
754 if len(p.filters[BeforeExec]) > 0 && p.execFilter(context, urlPath, BeforeExec) { 794 if len(p.filters[BeforeExec]) > 0 && p.execFilter(context, urlPath, BeforeExec) {
755 goto Admin 795 goto Admin
756 } 796 }
757 797
758 - //check policies 798 + // check policies
759 if p.execPolicy(context, urlPath) { 799 if p.execPolicy(context, urlPath) {
760 goto Admin 800 goto Admin
761 } 801 }
762 802
763 if routerInfo != nil { 803 if routerInfo != nil {
764 - //store router pattern into context  
765 - context.Input.SetData("RouterPattern", routerInfo.pattern)  
766 if routerInfo.routerType == routerTypeRESTFul { 804 if routerInfo.routerType == routerTypeRESTFul {
767 if _, ok := routerInfo.methods[r.Method]; ok { 805 if _, ok := routerInfo.methods[r.Method]; ok {
768 isRunnable = true 806 isRunnable = true
@@ -796,7 +834,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -796,7 +834,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
796 834
797 // also defined runRouter & runMethod from filter 835 // also defined runRouter & runMethod from filter
798 if !isRunnable { 836 if !isRunnable {
799 - //Invoke the request handler 837 + // Invoke the request handler
800 var execController ControllerInterface 838 var execController ControllerInterface
801 if routerInfo != nil && routerInfo.initialize != nil { 839 if routerInfo != nil && routerInfo.initialize != nil {
802 execController = routerInfo.initialize() 840 execController = routerInfo.initialize()
@@ -809,13 +847,13 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -809,13 +847,13 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
809 } 847 }
810 } 848 }
811 849
812 - //call the controller init function 850 + // call the controller init function
813 execController.Init(context, runRouter.Name(), runMethod, execController) 851 execController.Init(context, runRouter.Name(), runMethod, execController)
814 852
815 - //call prepare function 853 + // call prepare function
816 execController.Prepare() 854 execController.Prepare()
817 855
818 - //if XSRF is Enable then check cookie where there has any cookie in the request's cookie _csrf 856 + // if XSRF is Enable then check cookie where there has any cookie in the request's cookie _csrf
819 if BConfig.WebConfig.EnableXSRF { 857 if BConfig.WebConfig.EnableXSRF {
820 execController.XSRFToken() 858 execController.XSRFToken()
821 if r.Method == http.MethodPost || r.Method == http.MethodDelete || r.Method == http.MethodPut || 859 if r.Method == http.MethodPost || r.Method == http.MethodDelete || r.Method == http.MethodPut ||
@@ -827,7 +865,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -827,7 +865,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
827 execController.URLMapping() 865 execController.URLMapping()
828 866
829 if !context.ResponseWriter.Started { 867 if !context.ResponseWriter.Started {
830 - //exec main logic 868 + // exec main logic
831 switch runMethod { 869 switch runMethod {
832 case http.MethodGet: 870 case http.MethodGet:
833 execController.Get() 871 execController.Get()
@@ -852,14 +890,14 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -852,14 +890,14 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
852 in := param.ConvertParams(methodParams, method.Type(), context) 890 in := param.ConvertParams(methodParams, method.Type(), context)
853 out := method.Call(in) 891 out := method.Call(in)
854 892
855 - //For backward compatibility we only handle response if we had incoming methodParams 893 + // For backward compatibility we only handle response if we had incoming methodParams
856 if methodParams != nil { 894 if methodParams != nil {
857 p.handleParamResponse(context, execController, out) 895 p.handleParamResponse(context, execController, out)
858 } 896 }
859 } 897 }
860 } 898 }
861 899
862 - //render template 900 + // render template
863 if !context.ResponseWriter.Started && context.Output.Status == 0 { 901 if !context.ResponseWriter.Started && context.Output.Status == 0 {
864 if BConfig.WebConfig.AutoRender { 902 if BConfig.WebConfig.AutoRender {
865 if err := execController.Render(); err != nil { 903 if err := execController.Render(); err != nil {
@@ -873,7 +911,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -873,7 +911,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
873 execController.Finish() 911 execController.Finish()
874 } 912 }
875 913
876 - //execute middleware filters 914 + // execute middleware filters
877 if len(p.filters[AfterExec]) > 0 && p.execFilter(context, urlPath, AfterExec) { 915 if len(p.filters[AfterExec]) > 0 && p.execFilter(context, urlPath, AfterExec) {
878 goto Admin 916 goto Admin
879 } 917 }
@@ -883,7 +921,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) @@ -883,7 +921,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request)
883 } 921 }
884 922
885 Admin: 923 Admin:
886 - //admin module record QPS 924 + // admin module record QPS
887 925
888 statusCode := context.ResponseWriter.Status 926 statusCode := context.ResponseWriter.Status
889 if statusCode == 0 { 927 if statusCode == 0 {
@@ -931,7 +969,7 @@ Admin: @@ -931,7 +969,7 @@ Admin:
931 } 969 }
932 970
933 func (p *ControllerRegister) handleParamResponse(context *beecontext.Context, execController ControllerInterface, results []reflect.Value) { 971 func (p *ControllerRegister) handleParamResponse(context *beecontext.Context, execController ControllerInterface, results []reflect.Value) {
934 - //looping in reverse order for the case when both error and value are returned and error sets the response status code 972 + // looping in reverse order for the case when both error and value are returned and error sets the response status code
935 for i := len(results) - 1; i >= 0; i-- { 973 for i := len(results) - 1; i >= 0; i-- {
936 result := results[i] 974 result := results[i]
937 if result.Kind() != reflect.Interface || !result.IsNil() { 975 if result.Kind() != reflect.Interface || !result.IsNil() {
@@ -973,11 +1011,11 @@ func toURL(params map[string]string) string { @@ -973,11 +1011,11 @@ func toURL(params map[string]string) string {
973 1011
974 // LogAccess logging info HTTP Access 1012 // LogAccess logging info HTTP Access
975 func LogAccess(ctx *beecontext.Context, startTime *time.Time, statusCode int) { 1013 func LogAccess(ctx *beecontext.Context, startTime *time.Time, statusCode int) {
976 - //Skip logging if AccessLogs config is false 1014 + // Skip logging if AccessLogs config is false
977 if !BConfig.Log.AccessLogs { 1015 if !BConfig.Log.AccessLogs {
978 return 1016 return
979 } 1017 }
980 - //Skip logging static requests unless EnableStaticLogs config is true 1018 + // Skip logging static requests unless EnableStaticLogs config is true
981 if !BConfig.Log.EnableStaticLogs && DefaultAccessLogFilter.Filter(ctx) { 1019 if !BConfig.Log.EnableStaticLogs && DefaultAccessLogFilter.Filter(ctx) {
982 return 1020 return
983 } 1021 }
@@ -1002,7 +1040,7 @@ func LogAccess(ctx *beecontext.Context, startTime *time.Time, statusCode int) { @@ -1002,7 +1040,7 @@ func LogAccess(ctx *beecontext.Context, startTime *time.Time, statusCode int) {
1002 HTTPReferrer: r.Header.Get("Referer"), 1040 HTTPReferrer: r.Header.Get("Referer"),
1003 HTTPUserAgent: r.Header.Get("User-Agent"), 1041 HTTPUserAgent: r.Header.Get("User-Agent"),
1004 RemoteUser: r.Header.Get("Remote-User"), 1042 RemoteUser: r.Header.Get("Remote-User"),
1005 - BodyBytesSent: 0, //@todo this one is missing! 1043 + BodyBytesSent: 0, // @todo this one is missing!
1006 } 1044 }
1007 logs.AccessLog(record, BConfig.Log.AccessLogsFormat) 1045 logs.AccessLog(record, BConfig.Log.AccessLogsFormat)
1008 } 1046 }
@@ -74,7 +74,9 @@ func (st *CookieSessionStore) SessionID() string { @@ -74,7 +74,9 @@ func (st *CookieSessionStore) SessionID() string {
74 74
75 // SessionRelease Write cookie session to http response cookie 75 // SessionRelease Write cookie session to http response cookie
76 func (st *CookieSessionStore) SessionRelease(w http.ResponseWriter) { 76 func (st *CookieSessionStore) SessionRelease(w http.ResponseWriter) {
  77 + st.lock.Lock()
77 encodedCookie, err := encodeCookie(cookiepder.block, cookiepder.config.SecurityKey, cookiepder.config.SecurityName, st.values) 78 encodedCookie, err := encodeCookie(cookiepder.block, cookiepder.config.SecurityKey, cookiepder.config.SecurityName, st.values)
  79 + st.lock.Unlock()
78 if err == nil { 80 if err == nil {
79 cookie := &http.Cookie{Name: cookiepder.config.CookieName, 81 cookie := &http.Cookie{Name: cookiepder.config.CookieName,
80 Value: url.QueryEscape(encodedCookie), 82 Value: url.QueryEscape(encodedCookie),
@@ -129,8 +129,9 @@ func (fp *FileProvider) SessionInit(maxlifetime int64, savePath string) error { @@ -129,8 +129,9 @@ func (fp *FileProvider) SessionInit(maxlifetime int64, savePath string) error {
129 // if file is not exist, create it. 129 // if file is not exist, create it.
130 // the file path is generated from sid string. 130 // the file path is generated from sid string.
131 func (fp *FileProvider) SessionRead(sid string) (Store, error) { 131 func (fp *FileProvider) SessionRead(sid string) (Store, error) {
132 - if strings.ContainsAny(sid, "./") {  
133 - return nil, nil 132 + invalidChars := "./"
  133 + if strings.ContainsAny(sid, invalidChars) {
  134 + return nil, errors.New("the sid shouldn't have following characters: " + invalidChars)
134 } 135 }
135 if len(sid) < 2 { 136 if len(sid) < 2 {
136 return nil, errors.New("length of the sid is less than 2") 137 return nil, errors.New("length of the sid is less than 2")
@@ -138,7 +139,7 @@ func (fp *FileProvider) SessionRead(sid string) (Store, error) { @@ -138,7 +139,7 @@ func (fp *FileProvider) SessionRead(sid string) (Store, error) {
138 filepder.lock.Lock() 139 filepder.lock.Lock()
139 defer filepder.lock.Unlock() 140 defer filepder.lock.Unlock()
140 141
141 - err := os.MkdirAll(path.Join(fp.savePath, string(sid[0]), string(sid[1])), 0777) 142 + err := os.MkdirAll(path.Join(fp.savePath, string(sid[0]), string(sid[1])), 0755)
142 if err != nil { 143 if err != nil {
143 SLogger.Println(err.Error()) 144 SLogger.Println(err.Error())
144 } 145 }
@@ -231,7 +232,7 @@ func (fp *FileProvider) SessionRegenerate(oldsid, sid string) (Store, error) { @@ -231,7 +232,7 @@ func (fp *FileProvider) SessionRegenerate(oldsid, sid string) (Store, error) {
231 return nil, fmt.Errorf("newsid %s exist", newSidFile) 232 return nil, fmt.Errorf("newsid %s exist", newSidFile)
232 } 233 }
233 234
234 - err = os.MkdirAll(newPath, 0777) 235 + err = os.MkdirAll(newPath, 0755)
235 if err != nil { 236 if err != nil {
236 SLogger.Println(err.Error()) 237 SLogger.Println(err.Error())
237 } 238 }
@@ -28,6 +28,7 @@ import ( @@ -28,6 +28,7 @@ import (
28 28
29 "github.com/astaxie/beego/context" 29 "github.com/astaxie/beego/context"
30 "github.com/astaxie/beego/logs" 30 "github.com/astaxie/beego/logs"
  31 + "github.com/hashicorp/golang-lru"
31 ) 32 )
32 33
33 var errNotStaticRequest = errors.New("request not a static file request") 34 var errNotStaticRequest = errors.New("request not a static file request")
@@ -67,6 +68,10 @@ func serverStaticRouter(ctx *context.Context) { @@ -67,6 +68,10 @@ func serverStaticRouter(ctx *context.Context) {
67 http.ServeFile(ctx.ResponseWriter, ctx.Request, filePath) 68 http.ServeFile(ctx.ResponseWriter, ctx.Request, filePath)
68 } 69 }
69 return 70 return
  71 + } else if fileInfo.Size() > int64(BConfig.WebConfig.StaticCacheFileSize) {
  72 + //over size file serve with http module
  73 + http.ServeFile(ctx.ResponseWriter, ctx.Request, filePath)
  74 + return
70 } 75 }
71 76
72 var enableCompress = BConfig.EnableGzip && isStaticCompress(filePath) 77 var enableCompress = BConfig.EnableGzip && isStaticCompress(filePath)
@@ -93,10 +98,11 @@ func serverStaticRouter(ctx *context.Context) { @@ -93,10 +98,11 @@ func serverStaticRouter(ctx *context.Context) {
93 } 98 }
94 99
95 type serveContentHolder struct { 100 type serveContentHolder struct {
96 - data []byte  
97 - modTime time.Time  
98 - size int64  
99 - encoding string 101 + data []byte
  102 + modTime time.Time
  103 + size int64
  104 + originSize int64 //original file size:to judge file changed
  105 + encoding string
100 } 106 }
101 107
102 type serveContentReader struct { 108 type serveContentReader struct {
@@ -104,22 +110,36 @@ type serveContentReader struct { @@ -104,22 +110,36 @@ type serveContentReader struct {
104 } 110 }
105 111
106 var ( 112 var (
107 - staticFileMap = make(map[string]*serveContentHolder)  
108 - mapLock sync.RWMutex 113 + staticFileLruCache *lru.Cache
  114 + lruLock sync.RWMutex
109 ) 115 )
110 116
111 func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, *serveContentReader, error) { 117 func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, *serveContentReader, error) {
  118 + if staticFileLruCache == nil {
  119 + //avoid lru cache error
  120 + if BConfig.WebConfig.StaticCacheFileNum >= 1 {
  121 + staticFileLruCache, _ = lru.New(BConfig.WebConfig.StaticCacheFileNum)
  122 + } else {
  123 + staticFileLruCache, _ = lru.New(1)
  124 + }
  125 + }
112 mapKey := acceptEncoding + ":" + filePath 126 mapKey := acceptEncoding + ":" + filePath
113 - mapLock.RLock()  
114 - mapFile := staticFileMap[mapKey]  
115 - mapLock.RUnlock() 127 + lruLock.RLock()
  128 + var mapFile *serveContentHolder
  129 + if cacheItem, ok := staticFileLruCache.Get(mapKey); ok {
  130 + mapFile = cacheItem.(*serveContentHolder)
  131 + }
  132 + lruLock.RUnlock()
116 if isOk(mapFile, fi) { 133 if isOk(mapFile, fi) {
117 reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)} 134 reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)}
118 return mapFile.encoding != "", mapFile.encoding, mapFile, reader, nil 135 return mapFile.encoding != "", mapFile.encoding, mapFile, reader, nil
119 } 136 }
120 - mapLock.Lock()  
121 - defer mapLock.Unlock()  
122 - if mapFile = staticFileMap[mapKey]; !isOk(mapFile, fi) { 137 + lruLock.Lock()
  138 + defer lruLock.Unlock()
  139 + if cacheItem, ok := staticFileLruCache.Get(mapKey); ok {
  140 + mapFile = cacheItem.(*serveContentHolder)
  141 + }
  142 + if !isOk(mapFile, fi) {
123 file, err := os.Open(filePath) 143 file, err := os.Open(filePath)
124 if err != nil { 144 if err != nil {
125 return false, "", nil, nil, err 145 return false, "", nil, nil, err
@@ -130,8 +150,10 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str @@ -130,8 +150,10 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str
130 if err != nil { 150 if err != nil {
131 return false, "", nil, nil, err 151 return false, "", nil, nil, err
132 } 152 }
133 - mapFile = &serveContentHolder{data: bufferWriter.Bytes(), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n}  
134 - staticFileMap[mapKey] = mapFile 153 + mapFile = &serveContentHolder{data: bufferWriter.Bytes(), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), originSize: fi.Size(), encoding: n}
  154 + if isOk(mapFile, fi) {
  155 + staticFileLruCache.Add(mapKey, mapFile)
  156 + }
135 } 157 }
136 158
137 reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)} 159 reader := &serveContentReader{Reader: bytes.NewReader(mapFile.data)}
@@ -141,8 +163,10 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str @@ -141,8 +163,10 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str
141 func isOk(s *serveContentHolder, fi os.FileInfo) bool { 163 func isOk(s *serveContentHolder, fi os.FileInfo) bool {
142 if s == nil { 164 if s == nil {
143 return false 165 return false
  166 + } else if s.size > int64(BConfig.WebConfig.StaticCacheFileSize) {
  167 + return false
144 } 168 }
145 - return s.modTime == fi.ModTime() && s.size == fi.Size() 169 + return s.modTime == fi.ModTime() && s.originSize == fi.Size()
146 } 170 }
147 171
148 // isStaticCompress detect static files 172 // isStaticCompress detect static files
@@ -117,8 +117,8 @@ func (m *URLMap) GetMap() map[string]interface{} { @@ -117,8 +117,8 @@ func (m *URLMap) GetMap() map[string]interface{} {
117 117
118 // GetMapData return all mapdata 118 // GetMapData return all mapdata
119 func (m *URLMap) GetMapData() []map[string]interface{} { 119 func (m *URLMap) GetMapData() []map[string]interface{} {
120 - m.lock.Lock()  
121 - defer m.lock.Unlock() 120 + m.lock.RLock()
  121 + defer m.lock.RUnlock()
122 122
123 var resultLists []map[string]interface{} 123 var resultLists []map[string]interface{}
124 124
@@ -33,7 +33,7 @@ type bounds struct { @@ -33,7 +33,7 @@ type bounds struct {
33 // The bounds for each field. 33 // The bounds for each field.
34 var ( 34 var (
35 AdminTaskList map[string]Tasker 35 AdminTaskList map[string]Tasker
36 - taskLock sync.Mutex 36 + taskLock sync.RWMutex
37 stop chan bool 37 stop chan bool
38 changed chan bool 38 changed chan bool
39 isstart bool 39 isstart bool
@@ -408,7 +408,10 @@ func run() { @@ -408,7 +408,10 @@ func run() {
408 } 408 }
409 409
410 for { 410 for {
  411 + // we only use RLock here because NewMapSorter copy the reference, do not change any thing
  412 + taskLock.RLock()
411 sortList := NewMapSorter(AdminTaskList) 413 sortList := NewMapSorter(AdminTaskList)
  414 + taskLock.RUnlock()
412 sortList.Sort() 415 sortList.Sort()
413 var effective time.Time 416 var effective time.Time
414 if len(AdminTaskList) == 0 || sortList.Vals[0].GetNext().IsZero() { 417 if len(AdminTaskList) == 0 || sortList.Vals[0].GetNext().IsZero() {
@@ -432,9 +435,11 @@ func run() { @@ -432,9 +435,11 @@ func run() {
432 continue 435 continue
433 case <-changed: 436 case <-changed:
434 now = time.Now().Local() 437 now = time.Now().Local()
  438 + taskLock.Lock()
435 for _, t := range AdminTaskList { 439 for _, t := range AdminTaskList {
436 t.SetNext(now) 440 t.SetNext(now)
437 } 441 }
  442 + taskLock.Unlock()
438 continue 443 continue
439 case <-stop: 444 case <-stop:
440 return 445 return
  1 +Copyright (C) 2013 Blake Mizerany
  2 +
  3 +Permission is hereby granted, free of charge, to any person obtaining
  4 +a copy of this software and associated documentation files (the
  5 +"Software"), to deal in the Software without restriction, including
  6 +without limitation the rights to use, copy, modify, merge, publish,
  7 +distribute, sublicense, and/or sell copies of the Software, and to
  8 +permit persons to whom the Software is furnished to do so, subject to
  9 +the following conditions:
  10 +
  11 +The above copyright notice and this permission notice shall be
  12 +included in all copies or substantial portions of the Software.
  13 +
  14 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  15 +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  16 +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  17 +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  18 +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  19 +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  20 +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  1 +8
  2 +5
  3 +26
  4 +12
  5 +5
  6 +235
  7 +13
  8 +6
  9 +28
  10 +30
  11 +3
  12 +3
  13 +3
  14 +3
  15 +5
  16 +2
  17 +33
  18 +7
  19 +2
  20 +4
  21 +7
  22 +12
  23 +14
  24 +5
  25 +8
  26 +3
  27 +10
  28 +4
  29 +5
  30 +3
  31 +6
  32 +6
  33 +209
  34 +20
  35 +3
  36 +10
  37 +14
  38 +3
  39 +4
  40 +6
  41 +8
  42 +5
  43 +11
  44 +7
  45 +3
  46 +2
  47 +3
  48 +3
  49 +212
  50 +5
  51 +222
  52 +4
  53 +10
  54 +10
  55 +5
  56 +6
  57 +3
  58 +8
  59 +3
  60 +10
  61 +254
  62 +220
  63 +2
  64 +3
  65 +5
  66 +24
  67 +5
  68 +4
  69 +222
  70 +7
  71 +3
  72 +3
  73 +223
  74 +8
  75 +15
  76 +12
  77 +14
  78 +14
  79 +3
  80 +2
  81 +2
  82 +3
  83 +13
  84 +3
  85 +11
  86 +4
  87 +4
  88 +6
  89 +5
  90 +7
  91 +13
  92 +5
  93 +3
  94 +5
  95 +2
  96 +5
  97 +3
  98 +5
  99 +2
  100 +7
  101 +15
  102 +17
  103 +14
  104 +3
  105 +6
  106 +6
  107 +3
  108 +17
  109 +5
  110 +4
  111 +7
  112 +6
  113 +4
  114 +4
  115 +8
  116 +6
  117 +8
  118 +3
  119 +9
  120 +3
  121 +6
  122 +3
  123 +4
  124 +5
  125 +3
  126 +3
  127 +660
  128 +4
  129 +6
  130 +10
  131 +3
  132 +6
  133 +3
  134 +2
  135 +5
  136 +13
  137 +2
  138 +4
  139 +4
  140 +10
  141 +4
  142 +8
  143 +4
  144 +3
  145 +7
  146 +9
  147 +9
  148 +3
  149 +10
  150 +37
  151 +3
  152 +13
  153 +4
  154 +12
  155 +3
  156 +6
  157 +10
  158 +8
  159 +5
  160 +21
  161 +2
  162 +3
  163 +8
  164 +3
  165 +2
  166 +3
  167 +3
  168 +4
  169 +12
  170 +2
  171 +4
  172 +8
  173 +8
  174 +4
  175 +3
  176 +2
  177 +20
  178 +1
  179 +6
  180 +32
  181 +2
  182 +11
  183 +6
  184 +18
  185 +3
  186 +8
  187 +11
  188 +3
  189 +212
  190 +3
  191 +4
  192 +2
  193 +6
  194 +7
  195 +12
  196 +11
  197 +3
  198 +2
  199 +16
  200 +10
  201 +6
  202 +4
  203 +6
  204 +3
  205 +2
  206 +7
  207 +3
  208 +2
  209 +2
  210 +2
  211 +2
  212 +5
  213 +6
  214 +4
  215 +3
  216 +10
  217 +3
  218 +4
  219 +6
  220 +5
  221 +3
  222 +4
  223 +4
  224 +5
  225 +6
  226 +4
  227 +3
  228 +4
  229 +4
  230 +5
  231 +7
  232 +5
  233 +5
  234 +3
  235 +2
  236 +7
  237 +2
  238 +4
  239 +12
  240 +4
  241 +5
  242 +6
  243 +2
  244 +4
  245 +4
  246 +8
  247 +4
  248 +15
  249 +13
  250 +7
  251 +16
  252 +5
  253 +3
  254 +23
  255 +5
  256 +5
  257 +7
  258 +3
  259 +2
  260 +9
  261 +8
  262 +7
  263 +5
  264 +8
  265 +11
  266 +4
  267 +10
  268 +76
  269 +4
  270 +47
  271 +4
  272 +3
  273 +2
  274 +7
  275 +4
  276 +2
  277 +3
  278 +37
  279 +10
  280 +4
  281 +2
  282 +20
  283 +5
  284 +4
  285 +4
  286 +10
  287 +10
  288 +4
  289 +3
  290 +7
  291 +23
  292 +240
  293 +7
  294 +13
  295 +5
  296 +5
  297 +3
  298 +3
  299 +2
  300 +5
  301 +4
  302 +2
  303 +8
  304 +7
  305 +19
  306 +2
  307 +23
  308 +8
  309 +7
  310 +2
  311 +5
  312 +3
  313 +8
  314 +3
  315 +8
  316 +13
  317 +5
  318 +5
  319 +5
  320 +2
  321 +3
  322 +23
  323 +4
  324 +9
  325 +8
  326 +4
  327 +3
  328 +3
  329 +5
  330 +220
  331 +2
  332 +3
  333 +4
  334 +6
  335 +14
  336 +3
  337 +53
  338 +6
  339 +2
  340 +5
  341 +18
  342 +6
  343 +3
  344 +219
  345 +6
  346 +5
  347 +2
  348 +5
  349 +3
  350 +6
  351 +5
  352 +15
  353 +4
  354 +3
  355 +17
  356 +3
  357 +2
  358 +4
  359 +7
  360 +2
  361 +3
  362 +3
  363 +4
  364 +4
  365 +3
  366 +2
  367 +664
  368 +6
  369 +3
  370 +23
  371 +5
  372 +5
  373 +16
  374 +5
  375 +8
  376 +2
  377 +4
  378 +2
  379 +24
  380 +12
  381 +3
  382 +2
  383 +3
  384 +5
  385 +8
  386 +3
  387 +5
  388 +4
  389 +3
  390 +14
  391 +3
  392 +5
  393 +8
  394 +2
  395 +3
  396 +7
  397 +9
  398 +4
  399 +2
  400 +3
  401 +6
  402 +8
  403 +4
  404 +3
  405 +4
  406 +6
  407 +5
  408 +3
  409 +3
  410 +6
  411 +3
  412 +19
  413 +4
  414 +4
  415 +6
  416 +3
  417 +6
  418 +3
  419 +5
  420 +22
  421 +5
  422 +4
  423 +4
  424 +3
  425 +8
  426 +11
  427 +4
  428 +9
  429 +7
  430 +6
  431 +13
  432 +4
  433 +4
  434 +4
  435 +6
  436 +17
  437 +9
  438 +3
  439 +3
  440 +3
  441 +4
  442 +3
  443 +221
  444 +5
  445 +11
  446 +3
  447 +4
  448 +2
  449 +12
  450 +6
  451 +3
  452 +5
  453 +7
  454 +5
  455 +7
  456 +4
  457 +9
  458 +7
  459 +14
  460 +37
  461 +19
  462 +217
  463 +16
  464 +3
  465 +5
  466 +2
  467 +2
  468 +7
  469 +19
  470 +7
  471 +6
  472 +7
  473 +4
  474 +24
  475 +5
  476 +11
  477 +4
  478 +7
  479 +7
  480 +9
  481 +13
  482 +3
  483 +4
  484 +3
  485 +6
  486 +28
  487 +4
  488 +4
  489 +5
  490 +5
  491 +2
  492 +5
  493 +6
  494 +4
  495 +4
  496 +6
  497 +10
  498 +5
  499 +4
  500 +3
  501 +2
  502 +3
  503 +3
  504 +6
  505 +5
  506 +5
  507 +4
  508 +3
  509 +2
  510 +3
  511 +7
  512 +4
  513 +6
  514 +18
  515 +16
  516 +8
  517 +16
  518 +4
  519 +5
  520 +8
  521 +6
  522 +9
  523 +13
  524 +1545
  525 +6
  526 +215
  527 +6
  528 +5
  529 +6
  530 +3
  531 +45
  532 +31
  533 +5
  534 +2
  535 +2
  536 +4
  537 +3
  538 +3
  539 +2
  540 +5
  541 +4
  542 +3
  543 +5
  544 +7
  545 +7
  546 +4
  547 +5
  548 +8
  549 +5
  550 +4
  551 +749
  552 +2
  553 +31
  554 +9
  555 +11
  556 +2
  557 +11
  558 +5
  559 +4
  560 +4
  561 +7
  562 +9
  563 +11
  564 +4
  565 +5
  566 +4
  567 +7
  568 +3
  569 +4
  570 +6
  571 +2
  572 +15
  573 +3
  574 +4
  575 +3
  576 +4
  577 +3
  578 +5
  579 +2
  580 +13
  581 +5
  582 +5
  583 +3
  584 +3
  585 +23
  586 +4
  587 +4
  588 +5
  589 +7
  590 +4
  591 +13
  592 +2
  593 +4
  594 +3
  595 +4
  596 +2
  597 +6
  598 +2
  599 +7
  600 +3
  601 +5
  602 +5
  603 +3
  604 +29
  605 +5
  606 +4
  607 +4
  608 +3
  609 +10
  610 +2
  611 +3
  612 +79
  613 +16
  614 +6
  615 +6
  616 +7
  617 +7
  618 +3
  619 +5
  620 +5
  621 +7
  622 +4
  623 +3
  624 +7
  625 +9
  626 +5
  627 +6
  628 +5
  629 +9
  630 +6
  631 +3
  632 +6
  633 +4
  634 +17
  635 +2
  636 +10
  637 +9
  638 +3
  639 +6
  640 +2
  641 +3
  642 +21
  643 +22
  644 +5
  645 +11
  646 +4
  647 +2
  648 +17
  649 +2
  650 +224
  651 +2
  652 +14
  653 +3
  654 +4
  655 +4
  656 +2
  657 +4
  658 +4
  659 +4
  660 +4
  661 +5
  662 +3
  663 +4
  664 +4
  665 +10
  666 +2
  667 +6
  668 +3
  669 +3
  670 +5
  671 +7
  672 +2
  673 +7
  674 +5
  675 +6
  676 +3
  677 +218
  678 +2
  679 +2
  680 +5
  681 +2
  682 +6
  683 +3
  684 +5
  685 +222
  686 +14
  687 +6
  688 +33
  689 +3
  690 +2
  691 +5
  692 +3
  693 +3
  694 +3
  695 +9
  696 +5
  697 +3
  698 +3
  699 +2
  700 +7
  701 +4
  702 +3
  703 +4
  704 +3
  705 +5
  706 +6
  707 +5
  708 +26
  709 +4
  710 +13
  711 +9
  712 +7
  713 +3
  714 +221
  715 +3
  716 +3
  717 +4
  718 +4
  719 +4
  720 +4
  721 +2
  722 +18
  723 +5
  724 +3
  725 +7
  726 +9
  727 +6
  728 +8
  729 +3
  730 +10
  731 +3
  732 +11
  733 +9
  734 +5
  735 +4
  736 +17
  737 +5
  738 +5
  739 +6
  740 +6
  741 +3
  742 +2
  743 +4
  744 +12
  745 +17
  746 +6
  747 +7
  748 +218
  749 +4
  750 +2
  751 +4
  752 +10
  753 +3
  754 +5
  755 +15
  756 +3
  757 +9
  758 +4
  759 +3
  760 +3
  761 +6
  762 +29
  763 +3
  764 +3
  765 +4
  766 +5
  767 +5
  768 +3
  769 +8
  770 +5
  771 +6
  772 +6
  773 +7
  774 +5
  775 +3
  776 +5
  777 +3
  778 +29
  779 +2
  780 +31
  781 +5
  782 +15
  783 +24
  784 +16
  785 +5
  786 +207
  787 +4
  788 +3
  789 +3
  790 +2
  791 +15
  792 +4
  793 +4
  794 +13
  795 +5
  796 +5
  797 +4
  798 +6
  799 +10
  800 +2
  801 +7
  802 +8
  803 +4
  804 +6
  805 +20
  806 +5
  807 +3
  808 +4
  809 +3
  810 +12
  811 +12
  812 +5
  813 +17
  814 +7
  815 +3
  816 +3
  817 +3
  818 +6
  819 +10
  820 +3
  821 +5
  822 +25
  823 +80
  824 +4
  825 +9
  826 +3
  827 +2
  828 +11
  829 +3
  830 +3
  831 +2
  832 +3
  833 +8
  834 +7
  835 +5
  836 +5
  837 +19
  838 +5
  839 +3
  840 +3
  841 +12
  842 +11
  843 +2
  844 +6
  845 +5
  846 +5
  847 +5
  848 +3
  849 +3
  850 +3
  851 +4
  852 +209
  853 +14
  854 +3
  855 +2
  856 +5
  857 +19
  858 +4
  859 +4
  860 +3
  861 +4
  862 +14
  863 +5
  864 +6
  865 +4
  866 +13
  867 +9
  868 +7
  869 +4
  870 +7
  871 +10
  872 +2
  873 +9
  874 +5
  875 +7
  876 +2
  877 +8
  878 +4
  879 +6
  880 +5
  881 +5
  882 +222
  883 +8
  884 +7
  885 +12
  886 +5
  887 +216
  888 +3
  889 +4
  890 +4
  891 +6
  892 +3
  893 +14
  894 +8
  895 +7
  896 +13
  897 +4
  898 +3
  899 +3
  900 +3
  901 +3
  902 +17
  903 +5
  904 +4
  905 +3
  906 +33
  907 +6
  908 +6
  909 +33
  910 +7
  911 +5
  912 +3
  913 +8
  914 +7
  915 +5
  916 +2
  917 +9
  918 +4
  919 +2
  920 +233
  921 +24
  922 +7
  923 +4
  924 +8
  925 +10
  926 +3
  927 +4
  928 +15
  929 +2
  930 +16
  931 +3
  932 +3
  933 +13
  934 +12
  935 +7
  936 +5
  937 +4
  938 +207
  939 +4
  940 +2
  941 +4
  942 +27
  943 +15
  944 +2
  945 +5
  946 +2
  947 +25
  948 +6
  949 +5
  950 +5
  951 +6
  952 +13
  953 +6
  954 +18
  955 +6
  956 +4
  957 +12
  958 +225
  959 +10
  960 +7
  961 +5
  962 +2
  963 +2
  964 +11
  965 +4
  966 +14
  967 +21
  968 +8
  969 +10
  970 +3
  971 +5
  972 +4
  973 +232
  974 +2
  975 +5
  976 +5
  977 +3
  978 +7
  979 +17
  980 +11
  981 +6
  982 +6
  983 +23
  984 +4
  985 +6
  986 +3
  987 +5
  988 +4
  989 +2
  990 +17
  991 +3
  992 +6
  993 +5
  994 +8
  995 +3
  996 +2
  997 +2
  998 +14
  999 +9
  1000 +4
  1001 +4
  1002 +2
  1003 +5
  1004 +5
  1005 +3
  1006 +7
  1007 +6
  1008 +12
  1009 +6
  1010 +10
  1011 +3
  1012 +6
  1013 +2
  1014 +2
  1015 +19
  1016 +5
  1017 +4
  1018 +4
  1019 +9
  1020 +2
  1021 +4
  1022 +13
  1023 +3
  1024 +5
  1025 +6
  1026 +3
  1027 +6
  1028 +5
  1029 +4
  1030 +9
  1031 +6
  1032 +3
  1033 +5
  1034 +7
  1035 +3
  1036 +6
  1037 +6
  1038 +4
  1039 +3
  1040 +10
  1041 +6
  1042 +3
  1043 +221
  1044 +3
  1045 +5
  1046 +3
  1047 +6
  1048 +4
  1049 +8
  1050 +5
  1051 +3
  1052 +6
  1053 +4
  1054 +4
  1055 +2
  1056 +54
  1057 +5
  1058 +6
  1059 +11
  1060 +3
  1061 +3
  1062 +4
  1063 +4
  1064 +4
  1065 +3
  1066 +7
  1067 +3
  1068 +11
  1069 +11
  1070 +7
  1071 +10
  1072 +6
  1073 +13
  1074 +223
  1075 +213
  1076 +15
  1077 +231
  1078 +7
  1079 +3
  1080 +7
  1081 +228
  1082 +2
  1083 +3
  1084 +4
  1085 +4
  1086 +5
  1087 +6
  1088 +7
  1089 +4
  1090 +13
  1091 +3
  1092 +4
  1093 +5
  1094 +3
  1095 +6
  1096 +4
  1097 +6
  1098 +7
  1099 +2
  1100 +4
  1101 +3
  1102 +4
  1103 +3
  1104 +3
  1105 +6
  1106 +3
  1107 +7
  1108 +3
  1109 +5
  1110 +18
  1111 +5
  1112 +6
  1113 +8
  1114 +10
  1115 +3
  1116 +3
  1117 +3
  1118 +2
  1119 +4
  1120 +2
  1121 +4
  1122 +4
  1123 +5
  1124 +6
  1125 +6
  1126 +4
  1127 +10
  1128 +13
  1129 +3
  1130 +12
  1131 +5
  1132 +12
  1133 +16
  1134 +8
  1135 +4
  1136 +19
  1137 +11
  1138 +2
  1139 +4
  1140 +5
  1141 +6
  1142 +8
  1143 +5
  1144 +6
  1145 +4
  1146 +18
  1147 +10
  1148 +4
  1149 +2
  1150 +216
  1151 +6
  1152 +6
  1153 +6
  1154 +2
  1155 +4
  1156 +12
  1157 +8
  1158 +3
  1159 +11
  1160 +5
  1161 +6
  1162 +14
  1163 +5
  1164 +3
  1165 +13
  1166 +4
  1167 +5
  1168 +4
  1169 +5
  1170 +3
  1171 +28
  1172 +6
  1173 +3
  1174 +7
  1175 +219
  1176 +3
  1177 +9
  1178 +7
  1179 +3
  1180 +10
  1181 +6
  1182 +3
  1183 +4
  1184 +19
  1185 +5
  1186 +7
  1187 +11
  1188 +6
  1189 +15
  1190 +19
  1191 +4
  1192 +13
  1193 +11
  1194 +3
  1195 +7
  1196 +5
  1197 +10
  1198 +2
  1199 +8
  1200 +11
  1201 +2
  1202 +6
  1203 +4
  1204 +6
  1205 +24
  1206 +6
  1207 +3
  1208 +3
  1209 +3
  1210 +3
  1211 +6
  1212 +18
  1213 +4
  1214 +11
  1215 +4
  1216 +2
  1217 +5
  1218 +10
  1219 +8
  1220 +3
  1221 +9
  1222 +5
  1223 +3
  1224 +4
  1225 +5
  1226 +6
  1227 +2
  1228 +5
  1229 +7
  1230 +4
  1231 +4
  1232 +14
  1233 +6
  1234 +4
  1235 +4
  1236 +5
  1237 +5
  1238 +7
  1239 +2
  1240 +4
  1241 +3
  1242 +7
  1243 +3
  1244 +3
  1245 +6
  1246 +4
  1247 +5
  1248 +4
  1249 +4
  1250 +4
  1251 +3
  1252 +3
  1253 +3
  1254 +3
  1255 +8
  1256 +14
  1257 +2
  1258 +3
  1259 +5
  1260 +3
  1261 +2
  1262 +4
  1263 +5
  1264 +3
  1265 +7
  1266 +3
  1267 +3
  1268 +18
  1269 +3
  1270 +4
  1271 +4
  1272 +5
  1273 +7
  1274 +3
  1275 +3
  1276 +3
  1277 +13
  1278 +5
  1279 +4
  1280 +8
  1281 +211
  1282 +5
  1283 +5
  1284 +3
  1285 +5
  1286 +2
  1287 +5
  1288 +4
  1289 +2
  1290 +655
  1291 +6
  1292 +3
  1293 +5
  1294 +11
  1295 +2
  1296 +5
  1297 +3
  1298 +12
  1299 +9
  1300 +15
  1301 +11
  1302 +5
  1303 +12
  1304 +217
  1305 +2
  1306 +6
  1307 +17
  1308 +3
  1309 +3
  1310 +207
  1311 +5
  1312 +5
  1313 +4
  1314 +5
  1315 +9
  1316 +3
  1317 +2
  1318 +8
  1319 +5
  1320 +4
  1321 +3
  1322 +2
  1323 +5
  1324 +12
  1325 +4
  1326 +14
  1327 +5
  1328 +4
  1329 +2
  1330 +13
  1331 +5
  1332 +8
  1333 +4
  1334 +225
  1335 +4
  1336 +3
  1337 +4
  1338 +5
  1339 +4
  1340 +3
  1341 +3
  1342 +6
  1343 +23
  1344 +9
  1345 +2
  1346 +6
  1347 +7
  1348 +233
  1349 +4
  1350 +4
  1351 +6
  1352 +18
  1353 +3
  1354 +4
  1355 +6
  1356 +3
  1357 +4
  1358 +4
  1359 +2
  1360 +3
  1361 +7
  1362 +4
  1363 +13
  1364 +227
  1365 +4
  1366 +3
  1367 +5
  1368 +4
  1369 +2
  1370 +12
  1371 +9
  1372 +17
  1373 +3
  1374 +7
  1375 +14
  1376 +6
  1377 +4
  1378 +5
  1379 +21
  1380 +4
  1381 +8
  1382 +9
  1383 +2
  1384 +9
  1385 +25
  1386 +16
  1387 +3
  1388 +6
  1389 +4
  1390 +7
  1391 +8
  1392 +5
  1393 +2
  1394 +3
  1395 +5
  1396 +4
  1397 +3
  1398 +3
  1399 +5
  1400 +3
  1401 +3
  1402 +3
  1403 +2
  1404 +3
  1405 +19
  1406 +2
  1407 +4
  1408 +3
  1409 +4
  1410 +2
  1411 +3
  1412 +4
  1413 +4
  1414 +2
  1415 +4
  1416 +3
  1417 +3
  1418 +3
  1419 +2
  1420 +6
  1421 +3
  1422 +17
  1423 +5
  1424 +6
  1425 +4
  1426 +3
  1427 +13
  1428 +5
  1429 +3
  1430 +3
  1431 +3
  1432 +4
  1433 +9
  1434 +4
  1435 +2
  1436 +14
  1437 +12
  1438 +4
  1439 +5
  1440 +24
  1441 +4
  1442 +3
  1443 +37
  1444 +12
  1445 +11
  1446 +21
  1447 +3
  1448 +4
  1449 +3
  1450 +13
  1451 +4
  1452 +2
  1453 +3
  1454 +15
  1455 +4
  1456 +11
  1457 +4
  1458 +4
  1459 +3
  1460 +8
  1461 +3
  1462 +4
  1463 +4
  1464 +12
  1465 +8
  1466 +5
  1467 +3
  1468 +3
  1469 +4
  1470 +2
  1471 +220
  1472 +3
  1473 +5
  1474 +223
  1475 +3
  1476 +3
  1477 +3
  1478 +10
  1479 +3
  1480 +15
  1481 +4
  1482 +241
  1483 +9
  1484 +7
  1485 +3
  1486 +6
  1487 +6
  1488 +23
  1489 +4
  1490 +13
  1491 +7
  1492 +3
  1493 +4
  1494 +7
  1495 +4
  1496 +9
  1497 +3
  1498 +3
  1499 +4
  1500 +10
  1501 +5
  1502 +5
  1503 +1
  1504 +5
  1505 +24
  1506 +2
  1507 +4
  1508 +5
  1509 +5
  1510 +6
  1511 +14
  1512 +3
  1513 +8
  1514 +2
  1515 +3
  1516 +5
  1517 +13
  1518 +13
  1519 +3
  1520 +5
  1521 +2
  1522 +3
  1523 +15
  1524 +3
  1525 +4
  1526 +2
  1527 +10
  1528 +4
  1529 +4
  1530 +4
  1531 +5
  1532 +5
  1533 +3
  1534 +5
  1535 +3
  1536 +4
  1537 +7
  1538 +4
  1539 +27
  1540 +3
  1541 +6
  1542 +4
  1543 +15
  1544 +3
  1545 +5
  1546 +6
  1547 +6
  1548 +5
  1549 +4
  1550 +8
  1551 +3
  1552 +9
  1553 +2
  1554 +6
  1555 +3
  1556 +4
  1557 +3
  1558 +7
  1559 +4
  1560 +18
  1561 +3
  1562 +11
  1563 +3
  1564 +3
  1565 +8
  1566 +9
  1567 +7
  1568 +24
  1569 +3
  1570 +219
  1571 +7
  1572 +10
  1573 +4
  1574 +5
  1575 +9
  1576 +12
  1577 +2
  1578 +5
  1579 +4
  1580 +4
  1581 +4
  1582 +3
  1583 +3
  1584 +19
  1585 +5
  1586 +8
  1587 +16
  1588 +8
  1589 +6
  1590 +22
  1591 +3
  1592 +23
  1593 +3
  1594 +242
  1595 +9
  1596 +4
  1597 +3
  1598 +3
  1599 +5
  1600 +7
  1601 +3
  1602 +3
  1603 +5
  1604 +8
  1605 +3
  1606 +7
  1607 +5
  1608 +14
  1609 +8
  1610 +10
  1611 +3
  1612 +4
  1613 +3
  1614 +7
  1615 +4
  1616 +6
  1617 +7
  1618 +4
  1619 +10
  1620 +4
  1621 +3
  1622 +11
  1623 +3
  1624 +7
  1625 +10
  1626 +3
  1627 +13
  1628 +6
  1629 +8
  1630 +12
  1631 +10
  1632 +5
  1633 +7
  1634 +9
  1635 +3
  1636 +4
  1637 +7
  1638 +7
  1639 +10
  1640 +8
  1641 +30
  1642 +9
  1643 +19
  1644 +4
  1645 +3
  1646 +19
  1647 +15
  1648 +4
  1649 +13
  1650 +3
  1651 +215
  1652 +223
  1653 +4
  1654 +7
  1655 +4
  1656 +8
  1657 +17
  1658 +16
  1659 +3
  1660 +7
  1661 +6
  1662 +5
  1663 +5
  1664 +4
  1665 +12
  1666 +3
  1667 +7
  1668 +4
  1669 +4
  1670 +13
  1671 +4
  1672 +5
  1673 +2
  1674 +5
  1675 +6
  1676 +5
  1677 +6
  1678 +6
  1679 +7
  1680 +10
  1681 +18
  1682 +23
  1683 +9
  1684 +3
  1685 +3
  1686 +6
  1687 +5
  1688 +2
  1689 +4
  1690 +2
  1691 +7
  1692 +3
  1693 +3
  1694 +2
  1695 +5
  1696 +5
  1697 +14
  1698 +10
  1699 +224
  1700 +6
  1701 +3
  1702 +4
  1703 +3
  1704 +7
  1705 +5
  1706 +9
  1707 +3
  1708 +6
  1709 +4
  1710 +2
  1711 +5
  1712 +11
  1713 +4
  1714 +3
  1715 +3
  1716 +2
  1717 +8
  1718 +4
  1719 +7
  1720 +4
  1721 +10
  1722 +7
  1723 +3
  1724 +3
  1725 +18
  1726 +18
  1727 +17
  1728 +3
  1729 +3
  1730 +3
  1731 +4
  1732 +5
  1733 +3
  1734 +3
  1735 +4
  1736 +12
  1737 +7
  1738 +3
  1739 +11
  1740 +13
  1741 +5
  1742 +4
  1743 +7
  1744 +13
  1745 +5
  1746 +4
  1747 +11
  1748 +3
  1749 +12
  1750 +3
  1751 +6
  1752 +4
  1753 +4
  1754 +21
  1755 +4
  1756 +6
  1757 +9
  1758 +5
  1759 +3
  1760 +10
  1761 +8
  1762 +4
  1763 +6
  1764 +4
  1765 +4
  1766 +6
  1767 +5
  1768 +4
  1769 +8
  1770 +6
  1771 +4
  1772 +6
  1773 +4
  1774 +4
  1775 +5
  1776 +9
  1777 +6
  1778 +3
  1779 +4
  1780 +2
  1781 +9
  1782 +3
  1783 +18
  1784 +2
  1785 +4
  1786 +3
  1787 +13
  1788 +3
  1789 +6
  1790 +6
  1791 +8
  1792 +7
  1793 +9
  1794 +3
  1795 +2
  1796 +16
  1797 +3
  1798 +4
  1799 +6
  1800 +3
  1801 +2
  1802 +33
  1803 +22
  1804 +14
  1805 +4
  1806 +9
  1807 +12
  1808 +4
  1809 +5
  1810 +6
  1811 +3
  1812 +23
  1813 +9
  1814 +4
  1815 +3
  1816 +5
  1817 +5
  1818 +3
  1819 +4
  1820 +5
  1821 +3
  1822 +5
  1823 +3
  1824 +10
  1825 +4
  1826 +5
  1827 +5
  1828 +8
  1829 +4
  1830 +4
  1831 +6
  1832 +8
  1833 +5
  1834 +4
  1835 +3
  1836 +4
  1837 +6
  1838 +3
  1839 +3
  1840 +3
  1841 +5
  1842 +9
  1843 +12
  1844 +6
  1845 +5
  1846 +9
  1847 +3
  1848 +5
  1849 +3
  1850 +2
  1851 +2
  1852 +2
  1853 +18
  1854 +3
  1855 +2
  1856 +21
  1857 +2
  1858 +5
  1859 +4
  1860 +6
  1861 +4
  1862 +5
  1863 +10
  1864 +3
  1865 +9
  1866 +3
  1867 +2
  1868 +10
  1869 +7
  1870 +3
  1871 +6
  1872 +6
  1873 +4
  1874 +4
  1875 +8
  1876 +12
  1877 +7
  1878 +3
  1879 +7
  1880 +3
  1881 +3
  1882 +9
  1883 +3
  1884 +4
  1885 +5
  1886 +4
  1887 +4
  1888 +5
  1889 +5
  1890 +10
  1891 +15
  1892 +4
  1893 +4
  1894 +14
  1895 +6
  1896 +227
  1897 +3
  1898 +14
  1899 +5
  1900 +216
  1901 +22
  1902 +5
  1903 +4
  1904 +2
  1905 +2
  1906 +6
  1907 +3
  1908 +4
  1909 +2
  1910 +9
  1911 +9
  1912 +4
  1913 +3
  1914 +28
  1915 +13
  1916 +11
  1917 +4
  1918 +5
  1919 +3
  1920 +3
  1921 +2
  1922 +3
  1923 +3
  1924 +5
  1925 +3
  1926 +4
  1927 +3
  1928 +5
  1929 +23
  1930 +26
  1931 +3
  1932 +4
  1933 +5
  1934 +6
  1935 +4
  1936 +6
  1937 +3
  1938 +5
  1939 +5
  1940 +3
  1941 +4
  1942 +3
  1943 +2
  1944 +2
  1945 +2
  1946 +7
  1947 +14
  1948 +3
  1949 +6
  1950 +7
  1951 +17
  1952 +2
  1953 +2
  1954 +15
  1955 +14
  1956 +16
  1957 +4
  1958 +6
  1959 +7
  1960 +13
  1961 +6
  1962 +4
  1963 +5
  1964 +6
  1965 +16
  1966 +3
  1967 +3
  1968 +28
  1969 +3
  1970 +6
  1971 +15
  1972 +3
  1973 +9
  1974 +2
  1975 +4
  1976 +6
  1977 +3
  1978 +3
  1979 +22
  1980 +4
  1981 +12
  1982 +6
  1983 +7
  1984 +2
  1985 +5
  1986 +4
  1987 +10
  1988 +3
  1989 +16
  1990 +6
  1991 +9
  1992 +2
  1993 +5
  1994 +12
  1995 +7
  1996 +5
  1997 +5
  1998 +5
  1999 +5
  2000 +2
  2001 +11
  2002 +9
  2003 +17
  2004 +4
  2005 +3
  2006 +11
  2007 +7
  2008 +3
  2009 +5
  2010 +15
  2011 +4
  2012 +3
  2013 +4
  2014 +211
  2015 +8
  2016 +7
  2017 +5
  2018 +4
  2019 +7
  2020 +6
  2021 +7
  2022 +6
  2023 +3
  2024 +6
  2025 +5
  2026 +6
  2027 +5
  2028 +3
  2029 +4
  2030 +4
  2031 +26
  2032 +4
  2033 +6
  2034 +10
  2035 +4
  2036 +4
  2037 +3
  2038 +2
  2039 +3
  2040 +3
  2041 +4
  2042 +5
  2043 +9
  2044 +3
  2045 +9
  2046 +4
  2047 +4
  2048 +5
  2049 +5
  2050 +8
  2051 +2
  2052 +4
  2053 +2
  2054 +3
  2055 +8
  2056 +4
  2057 +11
  2058 +19
  2059 +5
  2060 +8
  2061 +6
  2062 +3
  2063 +5
  2064 +6
  2065 +12
  2066 +3
  2067 +2
  2068 +4
  2069 +16
  2070 +12
  2071 +3
  2072 +4
  2073 +4
  2074 +8
  2075 +6
  2076 +5
  2077 +6
  2078 +6
  2079 +219
  2080 +8
  2081 +222
  2082 +6
  2083 +16
  2084 +3
  2085 +13
  2086 +19
  2087 +5
  2088 +4
  2089 +3
  2090 +11
  2091 +6
  2092 +10
  2093 +4
  2094 +7
  2095 +7
  2096 +12
  2097 +5
  2098 +3
  2099 +3
  2100 +5
  2101 +6
  2102 +10
  2103 +3
  2104 +8
  2105 +2
  2106 +5
  2107 +4
  2108 +7
  2109 +2
  2110 +4
  2111 +4
  2112 +2
  2113 +12
  2114 +9
  2115 +6
  2116 +4
  2117 +2
  2118 +40
  2119 +2
  2120 +4
  2121 +10
  2122 +4
  2123 +223
  2124 +4
  2125 +2
  2126 +20
  2127 +6
  2128 +7
  2129 +24
  2130 +5
  2131 +4
  2132 +5
  2133 +2
  2134 +20
  2135 +16
  2136 +6
  2137 +5
  2138 +13
  2139 +2
  2140 +3
  2141 +3
  2142 +19
  2143 +3
  2144 +2
  2145 +4
  2146 +5
  2147 +6
  2148 +7
  2149 +11
  2150 +12
  2151 +5
  2152 +6
  2153 +7
  2154 +7
  2155 +3
  2156 +5
  2157 +3
  2158 +5
  2159 +3
  2160 +14
  2161 +3
  2162 +4
  2163 +4
  2164 +2
  2165 +11
  2166 +1
  2167 +7
  2168 +3
  2169 +9
  2170 +6
  2171 +11
  2172 +12
  2173 +5
  2174 +8
  2175 +6
  2176 +221
  2177 +4
  2178 +2
  2179 +12
  2180 +4
  2181 +3
  2182 +15
  2183 +4
  2184 +5
  2185 +226
  2186 +7
  2187 +218
  2188 +7
  2189 +5
  2190 +4
  2191 +5
  2192 +18
  2193 +4
  2194 +5
  2195 +9
  2196 +4
  2197 +4
  2198 +2
  2199 +9
  2200 +18
  2201 +18
  2202 +9
  2203 +5
  2204 +6
  2205 +6
  2206 +3
  2207 +3
  2208 +7
  2209 +3
  2210 +5
  2211 +4
  2212 +4
  2213 +4
  2214 +12
  2215 +3
  2216 +6
  2217 +31
  2218 +5
  2219 +4
  2220 +7
  2221 +3
  2222 +6
  2223 +5
  2224 +6
  2225 +5
  2226 +11
  2227 +2
  2228 +2
  2229 +11
  2230 +11
  2231 +6
  2232 +7
  2233 +5
  2234 +8
  2235 +7
  2236 +10
  2237 +5
  2238 +23
  2239 +7
  2240 +4
  2241 +3
  2242 +5
  2243 +34
  2244 +2
  2245 +5
  2246 +23
  2247 +7
  2248 +3
  2249 +6
  2250 +8
  2251 +4
  2252 +4
  2253 +4
  2254 +2
  2255 +5
  2256 +3
  2257 +8
  2258 +5
  2259 +4
  2260 +8
  2261 +25
  2262 +2
  2263 +3
  2264 +17
  2265 +8
  2266 +3
  2267 +4
  2268 +8
  2269 +7
  2270 +3
  2271 +15
  2272 +6
  2273 +5
  2274 +7
  2275 +21
  2276 +9
  2277 +5
  2278 +6
  2279 +6
  2280 +5
  2281 +3
  2282 +2
  2283 +3
  2284 +10
  2285 +3
  2286 +6
  2287 +3
  2288 +14
  2289 +7
  2290 +4
  2291 +4
  2292 +8
  2293 +7
  2294 +8
  2295 +2
  2296 +6
  2297 +12
  2298 +4
  2299 +213
  2300 +6
  2301 +5
  2302 +21
  2303 +8
  2304 +2
  2305 +5
  2306 +23
  2307 +3
  2308 +11
  2309 +2
  2310 +3
  2311 +6
  2312 +25
  2313 +2
  2314 +3
  2315 +6
  2316 +7
  2317 +6
  2318 +6
  2319 +4
  2320 +4
  2321 +6
  2322 +3
  2323 +17
  2324 +9
  2325 +7
  2326 +6
  2327 +4
  2328 +3
  2329 +10
  2330 +7
  2331 +2
  2332 +3
  2333 +3
  2334 +3
  2335 +11
  2336 +8
  2337 +3
  2338 +7
  2339 +6
  2340 +4
  2341 +14
  2342 +36
  2343 +3
  2344 +4
  2345 +3
  2346 +3
  2347 +22
  2348 +13
  2349 +21
  2350 +4
  2351 +2
  2352 +7
  2353 +4
  2354 +4
  2355 +17
  2356 +15
  2357 +3
  2358 +7
  2359 +11
  2360 +2
  2361 +4
  2362 +7
  2363 +6
  2364 +209
  2365 +6
  2366 +3
  2367 +2
  2368 +2
  2369 +24
  2370 +4
  2371 +9
  2372 +4
  2373 +3
  2374 +3
  2375 +3
  2376 +29
  2377 +2
  2378 +2
  2379 +4
  2380 +3
  2381 +3
  2382 +5
  2383 +4
  2384 +6
  2385 +3
  2386 +3
  2387 +2
  2388 +4
  1 +// Package quantile computes approximate quantiles over an unbounded data
  2 +// stream within low memory and CPU bounds.
  3 +//
  4 +// A small amount of accuracy is traded to achieve the above properties.
  5 +//
  6 +// Multiple streams can be merged before calling Query to generate a single set
  7 +// of results. This is meaningful when the streams represent the same type of
  8 +// data. See Merge and Samples.
  9 +//
  10 +// For more detailed information about the algorithm used, see:
  11 +//
  12 +// Effective Computation of Biased Quantiles over Data Streams
  13 +//
  14 +// http://www.cs.rutgers.edu/~muthu/bquant.pdf
  15 +package quantile
  16 +
  17 +import (
  18 + "math"
  19 + "sort"
  20 +)
  21 +
  22 +// Sample holds an observed value and meta information for compression. JSON
  23 +// tags have been added for convenience.
  24 +type Sample struct {
  25 + Value float64 `json:",string"`
  26 + Width float64 `json:",string"`
  27 + Delta float64 `json:",string"`
  28 +}
  29 +
  30 +// Samples represents a slice of samples. It implements sort.Interface.
  31 +type Samples []Sample
  32 +
  33 +func (a Samples) Len() int { return len(a) }
  34 +func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
  35 +func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
  36 +
  37 +type invariant func(s *stream, r float64) float64
  38 +
  39 +// NewLowBiased returns an initialized Stream for low-biased quantiles
  40 +// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
  41 +// error guarantees can still be given even for the lower ranks of the data
  42 +// distribution.
  43 +//
  44 +// The provided epsilon is a relative error, i.e. the true quantile of a value
  45 +// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
  46 +//
  47 +// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
  48 +// properties.
  49 +func NewLowBiased(epsilon float64) *Stream {
  50 + ƒ := func(s *stream, r float64) float64 {
  51 + return 2 * epsilon * r
  52 + }
  53 + return newStream(ƒ)
  54 +}
  55 +
  56 +// NewHighBiased returns an initialized Stream for high-biased quantiles
  57 +// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
  58 +// error guarantees can still be given even for the higher ranks of the data
  59 +// distribution.
  60 +//
  61 +// The provided epsilon is a relative error, i.e. the true quantile of a value
  62 +// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
  63 +//
  64 +// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
  65 +// properties.
  66 +func NewHighBiased(epsilon float64) *Stream {
  67 + ƒ := func(s *stream, r float64) float64 {
  68 + return 2 * epsilon * (s.n - r)
  69 + }
  70 + return newStream(ƒ)
  71 +}
  72 +
  73 +// NewTargeted returns an initialized Stream concerned with a particular set of
  74 +// quantile values that are supplied a priori. Knowing these a priori reduces
  75 +// space and computation time. The targets map maps the desired quantiles to
  76 +// their absolute errors, i.e. the true quantile of a value returned by a query
  77 +// is guaranteed to be within (Quantile±Epsilon).
  78 +//
  79 +// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
  80 +func NewTargeted(targetMap map[float64]float64) *Stream {
  81 + // Convert map to slice to avoid slow iterations on a map.
  82 + // ƒ is called on the hot path, so converting the map to a slice
  83 + // beforehand results in significant CPU savings.
  84 + targets := targetMapToSlice(targetMap)
  85 +
  86 + ƒ := func(s *stream, r float64) float64 {
  87 + var m = math.MaxFloat64
  88 + var f float64
  89 + for _, t := range targets {
  90 + if t.quantile*s.n <= r {
  91 + f = (2 * t.epsilon * r) / t.quantile
  92 + } else {
  93 + f = (2 * t.epsilon * (s.n - r)) / (1 - t.quantile)
  94 + }
  95 + if f < m {
  96 + m = f
  97 + }
  98 + }
  99 + return m
  100 + }
  101 + return newStream(ƒ)
  102 +}
  103 +
  104 +type target struct {
  105 + quantile float64
  106 + epsilon float64
  107 +}
  108 +
  109 +func targetMapToSlice(targetMap map[float64]float64) []target {
  110 + targets := make([]target, 0, len(targetMap))
  111 +
  112 + for quantile, epsilon := range targetMap {
  113 + t := target{
  114 + quantile: quantile,
  115 + epsilon: epsilon,
  116 + }
  117 + targets = append(targets, t)
  118 + }
  119 +
  120 + return targets
  121 +}
  122 +
  123 +// Stream computes quantiles for a stream of float64s. It is not thread-safe by
  124 +// design. Take care when using across multiple goroutines.
  125 +type Stream struct {
  126 + *stream
  127 + b Samples
  128 + sorted bool
  129 +}
  130 +
  131 +func newStream(ƒ invariant) *Stream {
  132 + x := &stream{ƒ: ƒ}
  133 + return &Stream{x, make(Samples, 0, 500), true}
  134 +}
  135 +
  136 +// Insert inserts v into the stream.
  137 +func (s *Stream) Insert(v float64) {
  138 + s.insert(Sample{Value: v, Width: 1})
  139 +}
  140 +
  141 +func (s *Stream) insert(sample Sample) {
  142 + s.b = append(s.b, sample)
  143 + s.sorted = false
  144 + if len(s.b) == cap(s.b) {
  145 + s.flush()
  146 + }
  147 +}
  148 +
  149 +// Query returns the computed qth percentiles value. If s was created with
  150 +// NewTargeted, and q is not in the set of quantiles provided a priori, Query
  151 +// will return an unspecified result.
  152 +func (s *Stream) Query(q float64) float64 {
  153 + if !s.flushed() {
  154 + // Fast path when there hasn't been enough data for a flush;
  155 + // this also yields better accuracy for small sets of data.
  156 + l := len(s.b)
  157 + if l == 0 {
  158 + return 0
  159 + }
  160 + i := int(math.Ceil(float64(l) * q))
  161 + if i > 0 {
  162 + i -= 1
  163 + }
  164 + s.maybeSort()
  165 + return s.b[i].Value
  166 + }
  167 + s.flush()
  168 + return s.stream.query(q)
  169 +}
  170 +
  171 +// Merge merges samples into the underlying streams samples. This is handy when
  172 +// merging multiple streams from separate threads, database shards, etc.
  173 +//
  174 +// ATTENTION: This method is broken and does not yield correct results. The
  175 +// underlying algorithm is not capable of merging streams correctly.
  176 +func (s *Stream) Merge(samples Samples) {
  177 + sort.Sort(samples)
  178 + s.stream.merge(samples)
  179 +}
  180 +
  181 +// Reset reinitializes and clears the list reusing the samples buffer memory.
  182 +func (s *Stream) Reset() {
  183 + s.stream.reset()
  184 + s.b = s.b[:0]
  185 +}
  186 +
  187 +// Samples returns stream samples held by s.
  188 +func (s *Stream) Samples() Samples {
  189 + if !s.flushed() {
  190 + return s.b
  191 + }
  192 + s.flush()
  193 + return s.stream.samples()
  194 +}
  195 +
  196 +// Count returns the total number of samples observed in the stream
  197 +// since initialization.
  198 +func (s *Stream) Count() int {
  199 + return len(s.b) + s.stream.count()
  200 +}
  201 +
  202 +func (s *Stream) flush() {
  203 + s.maybeSort()
  204 + s.stream.merge(s.b)
  205 + s.b = s.b[:0]
  206 +}
  207 +
  208 +func (s *Stream) maybeSort() {
  209 + if !s.sorted {
  210 + s.sorted = true
  211 + sort.Sort(s.b)
  212 + }
  213 +}
  214 +
  215 +func (s *Stream) flushed() bool {
  216 + return len(s.stream.l) > 0
  217 +}
  218 +
  219 +type stream struct {
  220 + n float64
  221 + l []Sample
  222 + ƒ invariant
  223 +}
  224 +
  225 +func (s *stream) reset() {
  226 + s.l = s.l[:0]
  227 + s.n = 0
  228 +}
  229 +
  230 +func (s *stream) insert(v float64) {
  231 + s.merge(Samples{{v, 1, 0}})
  232 +}
  233 +
  234 +func (s *stream) merge(samples Samples) {
  235 + // TODO(beorn7): This tries to merge not only individual samples, but
  236 + // whole summaries. The paper doesn't mention merging summaries at
  237 + // all. Unittests show that the merging is inaccurate. Find out how to
  238 + // do merges properly.
  239 + var r float64
  240 + i := 0
  241 + for _, sample := range samples {
  242 + for ; i < len(s.l); i++ {
  243 + c := s.l[i]
  244 + if c.Value > sample.Value {
  245 + // Insert at position i.
  246 + s.l = append(s.l, Sample{})
  247 + copy(s.l[i+1:], s.l[i:])
  248 + s.l[i] = Sample{
  249 + sample.Value,
  250 + sample.Width,
  251 + math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
  252 + // TODO(beorn7): How to calculate delta correctly?
  253 + }
  254 + i++
  255 + goto inserted
  256 + }
  257 + r += c.Width
  258 + }
  259 + s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
  260 + i++
  261 + inserted:
  262 + s.n += sample.Width
  263 + r += sample.Width
  264 + }
  265 + s.compress()
  266 +}
  267 +
  268 +func (s *stream) count() int {
  269 + return int(s.n)
  270 +}
  271 +
  272 +func (s *stream) query(q float64) float64 {
  273 + t := math.Ceil(q * s.n)
  274 + t += math.Ceil(s.ƒ(s, t) / 2)
  275 + p := s.l[0]
  276 + var r float64
  277 + for _, c := range s.l[1:] {
  278 + r += p.Width
  279 + if r+c.Width+c.Delta > t {
  280 + return p.Value
  281 + }
  282 + p = c
  283 + }
  284 + return p.Value
  285 +}
  286 +
  287 +func (s *stream) compress() {
  288 + if len(s.l) < 2 {
  289 + return
  290 + }
  291 + x := s.l[len(s.l)-1]
  292 + xi := len(s.l) - 1
  293 + r := s.n - 1 - x.Width
  294 +
  295 + for i := len(s.l) - 2; i >= 0; i-- {
  296 + c := s.l[i]
  297 + if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
  298 + x.Width += c.Width
  299 + s.l[xi] = x
  300 + // Remove element at i.
  301 + copy(s.l[i:], s.l[i+1:])
  302 + s.l = s.l[:len(s.l)-1]
  303 + xi -= 1
  304 + } else {
  305 + x = c
  306 + xi = i
  307 + }
  308 + r -= c.Width
  309 + }
  310 +}
  311 +
  312 +func (s *stream) samples() Samples {
  313 + samples := make(Samples, len(s.l))
  314 + copy(samples, s.l)
  315 + return samples
  316 +}
  1 +language: go
  2 +go:
  3 + - "1.x"
  4 + - master
  5 +env:
  6 + - TAGS=""
  7 + - TAGS="-tags purego"
  8 +script: go test $TAGS -v ./...
  1 +Copyright (c) 2016 Caleb Spare
  2 +
  3 +MIT License
  4 +
  5 +Permission is hereby granted, free of charge, to any person obtaining
  6 +a copy of this software and associated documentation files (the
  7 +"Software"), to deal in the Software without restriction, including
  8 +without limitation the rights to use, copy, modify, merge, publish,
  9 +distribute, sublicense, and/or sell copies of the Software, and to
  10 +permit persons to whom the Software is furnished to do so, subject to
  11 +the following conditions:
  12 +
  13 +The above copyright notice and this permission notice shall be
  14 +included in all copies or substantial portions of the Software.
  15 +
  16 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  17 +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18 +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  1 +# xxhash
  2 +
  3 +[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash)
  4 +[![Build Status](https://travis-ci.org/cespare/xxhash.svg?branch=master)](https://travis-ci.org/cespare/xxhash)
  5 +
  6 +xxhash is a Go implementation of the 64-bit
  7 +[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
  8 +high-quality hashing algorithm that is much faster than anything in the Go
  9 +standard library.
  10 +
  11 +This package provides a straightforward API:
  12 +
  13 +```
  14 +func Sum64(b []byte) uint64
  15 +func Sum64String(s string) uint64
  16 +type Digest struct{ ... }
  17 + func New() *Digest
  18 +```
  19 +
  20 +The `Digest` type implements hash.Hash64. Its key methods are:
  21 +
  22 +```
  23 +func (*Digest) Write([]byte) (int, error)
  24 +func (*Digest) WriteString(string) (int, error)
  25 +func (*Digest) Sum64() uint64
  26 +```
  27 +
  28 +This implementation provides a fast pure-Go implementation and an even faster
  29 +assembly implementation for amd64.
  30 +
  31 +## Compatibility
  32 +
  33 +This package is in a module and the latest code is in version 2 of the module.
  34 +You need a version of Go with at least "minimal module compatibility" to use
  35 +github.com/cespare/xxhash/v2:
  36 +
  37 +* 1.9.7+ for Go 1.9
  38 +* 1.10.3+ for Go 1.10
  39 +* Go 1.11 or later
  40 +
  41 +I recommend using the latest release of Go.
  42 +
  43 +## Benchmarks
  44 +
  45 +Here are some quick benchmarks comparing the pure-Go and assembly
  46 +implementations of Sum64.
  47 +
  48 +| input size | purego | asm |
  49 +| --- | --- | --- |
  50 +| 5 B | 979.66 MB/s | 1291.17 MB/s |
  51 +| 100 B | 7475.26 MB/s | 7973.40 MB/s |
  52 +| 4 KB | 17573.46 MB/s | 17602.65 MB/s |
  53 +| 10 MB | 17131.46 MB/s | 17142.16 MB/s |
  54 +
  55 +These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using
  56 +the following commands under Go 1.11.2:
  57 +
  58 +```
  59 +$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes'
  60 +$ go test -benchtime 10s -bench '/xxhash,direct,bytes'
  61 +```
  62 +
  63 +## Projects using this package
  64 +
  65 +- [InfluxDB](https://github.com/influxdata/influxdb)
  66 +- [Prometheus](https://github.com/prometheus/prometheus)
  67 +- [FreeCache](https://github.com/coocood/freecache)
  1 +module github.com/cespare/xxhash/v2
  2 +
  3 +go 1.11
  1 +// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
  2 +// at http://cyan4973.github.io/xxHash/.
  3 +package xxhash
  4 +
  5 +import (
  6 + "encoding/binary"
  7 + "errors"
  8 + "math/bits"
  9 +)
  10 +
  11 +const (
  12 + prime1 uint64 = 11400714785074694791
  13 + prime2 uint64 = 14029467366897019727
  14 + prime3 uint64 = 1609587929392839161
  15 + prime4 uint64 = 9650029242287828579
  16 + prime5 uint64 = 2870177450012600261
  17 +)
  18 +
  19 +// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
  20 +// possible in the Go code is worth a small (but measurable) performance boost
  21 +// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
  22 +// convenience in the Go code in a few places where we need to intentionally
  23 +// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
  24 +// result overflows a uint64).
  25 +var (
  26 + prime1v = prime1
  27 + prime2v = prime2
  28 + prime3v = prime3
  29 + prime4v = prime4
  30 + prime5v = prime5
  31 +)
  32 +
  33 +// Digest implements hash.Hash64.
  34 +type Digest struct {
  35 + v1 uint64
  36 + v2 uint64
  37 + v3 uint64
  38 + v4 uint64
  39 + total uint64
  40 + mem [32]byte
  41 + n int // how much of mem is used
  42 +}
  43 +
  44 +// New creates a new Digest that computes the 64-bit xxHash algorithm.
  45 +func New() *Digest {
  46 + var d Digest
  47 + d.Reset()
  48 + return &d
  49 +}
  50 +
  51 +// Reset clears the Digest's state so that it can be reused.
  52 +func (d *Digest) Reset() {
  53 + d.v1 = prime1v + prime2
  54 + d.v2 = prime2
  55 + d.v3 = 0
  56 + d.v4 = -prime1v
  57 + d.total = 0
  58 + d.n = 0
  59 +}
  60 +
  61 +// Size always returns 8 bytes.
  62 +func (d *Digest) Size() int { return 8 }
  63 +
  64 +// BlockSize always returns 32 bytes.
  65 +func (d *Digest) BlockSize() int { return 32 }
  66 +
  67 +// Write adds more data to d. It always returns len(b), nil.
  68 +func (d *Digest) Write(b []byte) (n int, err error) {
  69 + n = len(b)
  70 + d.total += uint64(n)
  71 +
  72 + if d.n+n < 32 {
  73 + // This new data doesn't even fill the current block.
  74 + copy(d.mem[d.n:], b)
  75 + d.n += n
  76 + return
  77 + }
  78 +
  79 + if d.n > 0 {
  80 + // Finish off the partial block.
  81 + copy(d.mem[d.n:], b)
  82 + d.v1 = round(d.v1, u64(d.mem[0:8]))
  83 + d.v2 = round(d.v2, u64(d.mem[8:16]))
  84 + d.v3 = round(d.v3, u64(d.mem[16:24]))
  85 + d.v4 = round(d.v4, u64(d.mem[24:32]))
  86 + b = b[32-d.n:]
  87 + d.n = 0
  88 + }
  89 +
  90 + if len(b) >= 32 {
  91 + // One or more full blocks left.
  92 + nw := writeBlocks(d, b)
  93 + b = b[nw:]
  94 + }
  95 +
  96 + // Store any remaining partial block.
  97 + copy(d.mem[:], b)
  98 + d.n = len(b)
  99 +
  100 + return
  101 +}
  102 +
  103 +// Sum appends the current hash to b and returns the resulting slice.
  104 +func (d *Digest) Sum(b []byte) []byte {
  105 + s := d.Sum64()
  106 + return append(
  107 + b,
  108 + byte(s>>56),
  109 + byte(s>>48),
  110 + byte(s>>40),
  111 + byte(s>>32),
  112 + byte(s>>24),
  113 + byte(s>>16),
  114 + byte(s>>8),
  115 + byte(s),
  116 + )
  117 +}
  118 +
  119 +// Sum64 returns the current hash.
  120 +func (d *Digest) Sum64() uint64 {
  121 + var h uint64
  122 +
  123 + if d.total >= 32 {
  124 + v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4
  125 + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
  126 + h = mergeRound(h, v1)
  127 + h = mergeRound(h, v2)
  128 + h = mergeRound(h, v3)
  129 + h = mergeRound(h, v4)
  130 + } else {
  131 + h = d.v3 + prime5
  132 + }
  133 +
  134 + h += d.total
  135 +
  136 + i, end := 0, d.n
  137 + for ; i+8 <= end; i += 8 {
  138 + k1 := round(0, u64(d.mem[i:i+8]))
  139 + h ^= k1
  140 + h = rol27(h)*prime1 + prime4
  141 + }
  142 + if i+4 <= end {
  143 + h ^= uint64(u32(d.mem[i:i+4])) * prime1
  144 + h = rol23(h)*prime2 + prime3
  145 + i += 4
  146 + }
  147 + for i < end {
  148 + h ^= uint64(d.mem[i]) * prime5
  149 + h = rol11(h) * prime1
  150 + i++
  151 + }
  152 +
  153 + h ^= h >> 33
  154 + h *= prime2
  155 + h ^= h >> 29
  156 + h *= prime3
  157 + h ^= h >> 32
  158 +
  159 + return h
  160 +}
  161 +
  162 +const (
  163 + magic = "xxh\x06"
  164 + marshaledSize = len(magic) + 8*5 + 32
  165 +)
  166 +
  167 +// MarshalBinary implements the encoding.BinaryMarshaler interface.
  168 +func (d *Digest) MarshalBinary() ([]byte, error) {
  169 + b := make([]byte, 0, marshaledSize)
  170 + b = append(b, magic...)
  171 + b = appendUint64(b, d.v1)
  172 + b = appendUint64(b, d.v2)
  173 + b = appendUint64(b, d.v3)
  174 + b = appendUint64(b, d.v4)
  175 + b = appendUint64(b, d.total)
  176 + b = append(b, d.mem[:d.n]...)
  177 + b = b[:len(b)+len(d.mem)-d.n]
  178 + return b, nil
  179 +}
  180 +
  181 +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
  182 +func (d *Digest) UnmarshalBinary(b []byte) error {
  183 + if len(b) < len(magic) || string(b[:len(magic)]) != magic {
  184 + return errors.New("xxhash: invalid hash state identifier")
  185 + }
  186 + if len(b) != marshaledSize {
  187 + return errors.New("xxhash: invalid hash state size")
  188 + }
  189 + b = b[len(magic):]
  190 + b, d.v1 = consumeUint64(b)
  191 + b, d.v2 = consumeUint64(b)
  192 + b, d.v3 = consumeUint64(b)
  193 + b, d.v4 = consumeUint64(b)
  194 + b, d.total = consumeUint64(b)
  195 + copy(d.mem[:], b)
  196 + b = b[len(d.mem):]
  197 + d.n = int(d.total % uint64(len(d.mem)))
  198 + return nil
  199 +}
  200 +
  201 +func appendUint64(b []byte, x uint64) []byte {
  202 + var a [8]byte
  203 + binary.LittleEndian.PutUint64(a[:], x)
  204 + return append(b, a[:]...)
  205 +}
  206 +
  207 +func consumeUint64(b []byte) ([]byte, uint64) {
  208 + x := u64(b)
  209 + return b[8:], x
  210 +}
  211 +
  212 +func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }
  213 +func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
  214 +
  215 +func round(acc, input uint64) uint64 {
  216 + acc += input * prime2
  217 + acc = rol31(acc)
  218 + acc *= prime1
  219 + return acc
  220 +}
  221 +
  222 +func mergeRound(acc, val uint64) uint64 {
  223 + val = round(0, val)
  224 + acc ^= val
  225 + acc = acc*prime1 + prime4
  226 + return acc
  227 +}
  228 +
  229 +func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) }
  230 +func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) }
  231 +func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) }
  232 +func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) }
  233 +func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) }
  234 +func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) }
  235 +func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) }
  236 +func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) }
  1 +// +build !appengine
  2 +// +build gc
  3 +// +build !purego
  4 +
  5 +package xxhash
  6 +
  7 +// Sum64 computes the 64-bit xxHash digest of b.
  8 +//
  9 +//go:noescape
  10 +func Sum64(b []byte) uint64
  11 +
  12 +//go:noescape
  13 +func writeBlocks(d *Digest, b []byte) int