正在显示
23 个修改的文件
包含
4552 行增加
和
3 行删除
@@ -10,6 +10,6 @@ RUN ["ln","-sf","/usr/share/zoneinfo/Asia/Shanghai","/etc/localtime"] | @@ -10,6 +10,6 @@ RUN ["ln","-sf","/usr/share/zoneinfo/Asia/Shanghai","/etc/localtime"] | ||
10 | ENV GO111MODULE on | 10 | ENV GO111MODULE on |
11 | ENV GOPROXY https://goproxy.cn | 11 | ENV GOPROXY https://goproxy.cn |
12 | RUN ["go","mod","tidy"] | 12 | RUN ["go","mod","tidy"] |
13 | -RUN ["go","build"] | 13 | +RUN ["go","build","-mod=vendor"] |
14 | EXPOSE 8082 | 14 | EXPOSE 8082 |
15 | ENTRYPOINT ["./partnermg"] | 15 | ENTRYPOINT ["./partnermg"] |
@@ -160,7 +160,7 @@ type postPurposeOrderDetail struct { | @@ -160,7 +160,7 @@ type postPurposeOrderDetail struct { | ||
160 | //订单编号 | 160 | //订单编号 |
161 | OrderId string `json:"orderId"` | 161 | OrderId string `json:"orderId"` |
162 | //买家姓名 | 162 | //买家姓名 |
163 | - BuyerName string `json:"buyerName"` | 163 | + BuyerName string `json:"buyer"` |
164 | //对应合伙人 id | 164 | //对应合伙人 id |
165 | PartnerId int64 `json:"partnerID"` | 165 | PartnerId int64 `json:"partnerID"` |
166 | PartnerName string `json:"partnerName"` | 166 | PartnerName string `json:"partnerName"` |
1 | +package geetest | ||
2 | + | ||
3 | +import ( | ||
4 | + "crypto/md5" | ||
5 | + "encoding/hex" | ||
6 | + "encoding/json" | ||
7 | + "errors" | ||
8 | + "io/ioutil" | ||
9 | + "net/http" | ||
10 | + "net/url" | ||
11 | + "strings" | ||
12 | + "time" | ||
13 | +) | ||
14 | + | ||
15 | +type GeetestLib struct { | ||
16 | + CaptchaID string | ||
17 | + PrivateKey string | ||
18 | + Client *http.Client | ||
19 | +} | ||
20 | + | ||
21 | +type FailbackRegisterRespnse struct { | ||
22 | + Success int `json:"success"` | ||
23 | + GT string `json:"gt"` | ||
24 | + Challenge string `json:"challenge"` | ||
25 | + NewCaptcha int `json:"new_captcha"` | ||
26 | +} | ||
27 | + | ||
28 | +const ( | ||
29 | + geetestHost = "http://api.geetest.com" | ||
30 | + registerURL = geetestHost + "/register.php" | ||
31 | + validateURL = geetestHost + "/validate.php" | ||
32 | +) | ||
33 | + | ||
34 | +func MD5Encode(input string) string { | ||
35 | + md5Instant := md5.New() | ||
36 | + md5Instant.Write([]byte(input)) | ||
37 | + return hex.EncodeToString(md5Instant.Sum(nil)) | ||
38 | +} | ||
39 | + | ||
40 | +// 初始化 GeetestLib | ||
41 | +func NewGeetestLib(capthcaID string, privateKey string, timeOut time.Duration) (geetest GeetestLib){ | ||
42 | + client := &http.Client{Timeout: timeOut} | ||
43 | + geetest = GeetestLib{capthcaID, privateKey, client} | ||
44 | + return | ||
45 | +} | ||
46 | + | ||
47 | +func (g *GeetestLib) getFailBackRegisterResponse(success int, challenge string) []byte { | ||
48 | + if challenge == "" { | ||
49 | + challenge = hex.EncodeToString(md5.New().Sum(nil)) | ||
50 | + } | ||
51 | + | ||
52 | + response := FailbackRegisterRespnse{ | ||
53 | + success, | ||
54 | + g.CaptchaID, | ||
55 | + challenge, | ||
56 | + 1, | ||
57 | + } | ||
58 | + res, _ := json.Marshal(response) | ||
59 | + return res | ||
60 | +} | ||
61 | + | ||
62 | +func (g *GeetestLib) do(req *http.Request) (body []byte, err error) { | ||
63 | + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") | ||
64 | + var resp *http.Response | ||
65 | + if resp, err = g.Client.Do(req); err != nil { | ||
66 | + return | ||
67 | + } | ||
68 | + defer resp.Body.Close() | ||
69 | + if resp.StatusCode >= http.StatusInternalServerError { | ||
70 | + err = errors.New("http status code 5xx") | ||
71 | + return | ||
72 | + } | ||
73 | + | ||
74 | + if body, err = ioutil.ReadAll(resp.Body); err != nil { | ||
75 | + return | ||
76 | + } | ||
77 | + return | ||
78 | +} | ||
79 | + | ||
80 | +func (g *GeetestLib) PreProcess(userID string, userIP string) (int8, []byte) { | ||
81 | + params := url.Values{} | ||
82 | + params.Add("gt", g.CaptchaID) | ||
83 | + params.Add("new_captcha", "1") | ||
84 | + if userID != "" { | ||
85 | + params.Add("user_id", userID) | ||
86 | + } | ||
87 | + if userIP != "" { | ||
88 | + params.Add("ip_adress", userIP) | ||
89 | + } | ||
90 | + req, _ := http.NewRequest("GET", registerURL+"?"+params.Encode(), nil) | ||
91 | + body, err := g.do(req) | ||
92 | + if err != nil { | ||
93 | + return 0, g.getFailBackRegisterResponse(0, "") | ||
94 | + } | ||
95 | + challenge := string(body) | ||
96 | + if len(challenge) != 32 { | ||
97 | + return 0, g.getFailBackRegisterResponse(0, "") | ||
98 | + } else { | ||
99 | + challenge = MD5Encode(challenge + g.PrivateKey) | ||
100 | + return 1, g.getFailBackRegisterResponse(1, challenge) | ||
101 | + } | ||
102 | +} | ||
103 | + | ||
104 | +func (g *GeetestLib) checkParas(challenge string, validate string, seccode string) bool { | ||
105 | + if challenge == "" || validate == "" || seccode == "" { | ||
106 | + return false | ||
107 | + } | ||
108 | + return true | ||
109 | +} | ||
110 | + | ||
111 | +func (g *GeetestLib) checkSuccessRes(challenge string, validate string) bool { | ||
112 | + return MD5Encode(g.PrivateKey+"geetest"+challenge) == validate | ||
113 | +} | ||
114 | + | ||
115 | +func (g *GeetestLib) checkFailbackRes(challenge string, validate string) bool { | ||
116 | + return MD5Encode(challenge) == validate | ||
117 | +} | ||
118 | + | ||
119 | +func (g *GeetestLib) SuccessValidate(challenge string, validate string, seccode string, userID string, userIP string) bool { | ||
120 | + if !g.checkParas(challenge, validate, seccode) { | ||
121 | + return false | ||
122 | + } | ||
123 | + if !g.checkSuccessRes(challenge, validate) { | ||
124 | + return false | ||
125 | + } | ||
126 | + params := url.Values{} | ||
127 | + params.Add("seccode", seccode) | ||
128 | + params.Add("challenge", challenge) | ||
129 | + params.Add("captchaid", g.CaptchaID) | ||
130 | + params.Add("sdk", "golang_v1.0.0") | ||
131 | + if userID != "" { | ||
132 | + params.Add("user_id", userID) | ||
133 | + } | ||
134 | + if userIP != "" { | ||
135 | + params.Add("ip_adress", userIP) | ||
136 | + } | ||
137 | + req, _ := http.NewRequest("POST", validateURL, strings.NewReader(params.Encode())) | ||
138 | + body, err := g.do(req) | ||
139 | + if err != nil { | ||
140 | + return false | ||
141 | + } | ||
142 | + res := string(body) | ||
143 | + return res == MD5Encode(seccode) | ||
144 | +} | ||
145 | + | ||
146 | +func (g *GeetestLib) FailbackValidate(challenge string, validate string, seccode string) bool { | ||
147 | + if !g.checkParas(challenge, validate, seccode) { | ||
148 | + return false | ||
149 | + } | ||
150 | + if !g.checkFailbackRes(challenge, validate) { | ||
151 | + return false | ||
152 | + } | ||
153 | + return true | ||
154 | +} |
vendor/github.com/ajg/form/.travis.yml
0 → 100644
1 | +## Copyright 2014 Alvaro J. Genial. All rights reserved. | ||
2 | +## Use of this source code is governed by a BSD-style | ||
3 | +## license that can be found in the LICENSE file. | ||
4 | + | ||
5 | +language: go | ||
6 | + | ||
7 | +go: | ||
8 | + - tip | ||
9 | + - 1.6 | ||
10 | + - 1.5 | ||
11 | + - 1.4 | ||
12 | + - 1.3 | ||
13 | + # 1.2 | ||
14 | + | ||
15 | +before_install: | ||
16 | + # - go get -v golang.org/x/tools/cmd/cover | ||
17 | + # - go get -v golang.org/x/tools/cmd/vet | ||
18 | + # - go get -v golang.org/x/lint/golint | ||
19 | + - export PATH=$PATH:/home/travis/gopath/bin | ||
20 | + | ||
21 | +script: | ||
22 | + - go build -v ./... | ||
23 | + - go test -v -cover ./... | ||
24 | + - go vet ./... | ||
25 | + # - golint . |
vendor/github.com/ajg/form/LICENSE
0 → 100644
1 | +Copyright (c) 2014 Alvaro J. Genial. All rights reserved. | ||
2 | + | ||
3 | +Redistribution and use in source and binary forms, with or without | ||
4 | +modification, are permitted provided that the following conditions are | ||
5 | +met: | ||
6 | + | ||
7 | + * Redistributions of source code must retain the above copyright | ||
8 | +notice, this list of conditions and the following disclaimer. | ||
9 | + * Redistributions in binary form must reproduce the above | ||
10 | +copyright notice, this list of conditions and the following disclaimer | ||
11 | +in the documentation and/or other materials provided with the | ||
12 | +distribution. | ||
13 | + * Neither the name of Google Inc. nor the names of its | ||
14 | +contributors may be used to endorse or promote products derived from | ||
15 | +this software without specific prior written permission. | ||
16 | + | ||
17 | +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | ||
18 | +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | ||
19 | +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | ||
20 | +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | ||
21 | +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||
22 | +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | ||
23 | +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | ||
24 | +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | ||
25 | +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | ||
26 | +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | ||
27 | +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
vendor/github.com/ajg/form/README.md
0 → 100644
1 | +form | ||
2 | +==== | ||
3 | + | ||
4 | +A Form Encoding & Decoding Package for Go, written by [Alvaro J. Genial](http://alva.ro). | ||
5 | + | ||
6 | +[![Build Status](https://travis-ci.org/ajg/form.png?branch=master)](https://travis-ci.org/ajg/form) | ||
7 | +[![GoDoc](https://godoc.org/github.com/ajg/form?status.png)](https://godoc.org/github.com/ajg/form) | ||
8 | + | ||
9 | +Synopsis | ||
10 | +-------- | ||
11 | + | ||
12 | +This library is designed to allow seamless, high-fidelity encoding and decoding of arbitrary data in `application/x-www-form-urlencoded` format and as [`url.Values`](http://golang.org/pkg/net/url/#Values). It is intended to be useful primarily in dealing with web forms and URI query strings, both of which natively employ said format. | ||
13 | + | ||
14 | +Unsurprisingly, `form` is modeled after other Go [`encoding`](http://golang.org/pkg/encoding/) packages, in particular [`encoding/json`](http://golang.org/pkg/encoding/json/), and follows the same conventions (see below for more.) It aims to automatically handle any kind of concrete Go [data value](#values) (i.e., not functions, channels, etc.) while providing mechanisms for custom behavior. | ||
15 | + | ||
16 | +Status | ||
17 | +------ | ||
18 | + | ||
19 | +The implementation is in usable shape and is fairly well tested with its accompanying test suite. The API is unlikely to change much, but still may. Lastly, the code has not yet undergone a security review to ensure it is free of vulnerabilities. Please file an issue or send a pull request for fixes & improvements. | ||
20 | + | ||
21 | +Dependencies | ||
22 | +------------ | ||
23 | + | ||
24 | +The only requirement is [Go 1.2](http://golang.org/doc/go1.2) or later. | ||
25 | + | ||
26 | +Usage | ||
27 | +----- | ||
28 | + | ||
29 | +```go | ||
30 | +import "github.com/ajg/form" | ||
31 | +// or: "gopkg.in/ajg/form.v1" | ||
32 | +``` | ||
33 | + | ||
34 | +Given a type like the following... | ||
35 | + | ||
36 | +```go | ||
37 | +type User struct { | ||
38 | + Name string `form:"name"` | ||
39 | + Email string `form:"email"` | ||
40 | + Joined time.Time `form:"joined,omitempty"` | ||
41 | + Posts []int `form:"posts"` | ||
42 | + Preferences map[string]string `form:"prefs"` | ||
43 | + Avatar []byte `form:"avatar"` | ||
44 | + PasswordHash int64 `form:"-"` | ||
45 | +} | ||
46 | +``` | ||
47 | + | ||
48 | +...it is easy to encode data of that type... | ||
49 | + | ||
50 | + | ||
51 | +```go | ||
52 | +func PostUser(url string, u User) error { | ||
53 | + var c http.Client | ||
54 | + _, err := c.PostForm(url, form.EncodeToValues(u)) | ||
55 | + return err | ||
56 | +} | ||
57 | +``` | ||
58 | + | ||
59 | +...as well as decode it... | ||
60 | + | ||
61 | + | ||
62 | +```go | ||
63 | +func Handler(w http.ResponseWriter, r *http.Request) { | ||
64 | + var u User | ||
65 | + | ||
66 | + d := form.NewDecoder(r.Body) | ||
67 | + if err := d.Decode(&u); err != nil { | ||
68 | + http.Error(w, "Form could not be decoded", http.StatusBadRequest) | ||
69 | + return | ||
70 | + } | ||
71 | + | ||
72 | + fmt.Fprintf(w, "Decoded: %#v", u) | ||
73 | +} | ||
74 | +``` | ||
75 | + | ||
76 | +...without having to do any grunt work. | ||
77 | + | ||
78 | +Field Tags | ||
79 | +---------- | ||
80 | + | ||
81 | +Like other encoding packages, `form` supports the following options for fields: | ||
82 | + | ||
83 | + - `` `form:"-"` ``: Causes the field to be ignored during encoding and decoding. | ||
84 | + - `` `form:"<name>"` ``: Overrides the field's name; useful especially when dealing with external identifiers in camelCase, as are commonly found on the web. | ||
85 | + - `` `form:",omitempty"` ``: Elides the field during encoding if it is empty (typically meaning equal to the type's zero value.) | ||
86 | + - `` `form:"<name>,omitempty"` ``: The way to combine the two options above. | ||
87 | + | ||
88 | +Values | ||
89 | +------ | ||
90 | + | ||
91 | +### Simple Values | ||
92 | + | ||
93 | +Values of the following types are all considered simple: | ||
94 | + | ||
95 | + - `bool` | ||
96 | + - `int`, `int8`, `int16`, `int32`, `int64`, `rune` | ||
97 | + - `uint`, `uint8`, `uint16`, `uint32`, `uint64`, `byte` | ||
98 | + - `float32`, `float64` | ||
99 | + - `complex64`, `complex128` | ||
100 | + - `string` | ||
101 | + - `[]byte` (see note) | ||
102 | + - [`time.Time`](http://golang.org/pkg/time/#Time) | ||
103 | + - [`url.URL`](http://golang.org/pkg/net/url/#URL) | ||
104 | + - An alias of any of the above | ||
105 | + - A pointer to any of the above | ||
106 | + | ||
107 | +### Composite Values | ||
108 | + | ||
109 | +A composite value is one that can contain other values. Values of the following kinds... | ||
110 | + | ||
111 | + - Maps | ||
112 | + - Slices; except `[]byte` (see note) | ||
113 | + - Structs; except [`time.Time`](http://golang.org/pkg/time/#Time) and [`url.URL`](http://golang.org/pkg/net/url/#URL) | ||
114 | + - Arrays | ||
115 | + - An alias of any of the above | ||
116 | + - A pointer to any of the above | ||
117 | + | ||
118 | +...are considered composites in general, unless they implement custom marshaling/unmarshaling. Composite values are encoded as a flat mapping of paths to values, where the paths are constructed by joining the parent and child paths with a period (`.`). | ||
119 | + | ||
120 | +(Note: a byte slice is treated as a `string` by default because it's more efficient, but can also be decoded as a slice—i.e., with indexes.) | ||
121 | + | ||
122 | +### Untyped Values | ||
123 | + | ||
124 | +While encouraged, it is not necessary to define a type (e.g. a `struct`) in order to use `form`, since it is able to encode and decode untyped data generically using the following rules: | ||
125 | + | ||
126 | + - Simple values will be treated as a `string`. | ||
127 | + - Composite values will be treated as a `map[string]interface{}`, itself able to contain nested values (both scalar and compound) ad infinitum. | ||
128 | + - However, if there is a value (of any supported type) already present in a map for a given key, then it will be used when possible, rather than being replaced with a generic value as specified above; this makes it possible to handle partially typed, dynamic or schema-less values. | ||
129 | + | ||
130 | +### Zero Values | ||
131 | + | ||
132 | +By default, and without custom marshaling, zero values (also known as empty/default values) are encoded as the empty string. To disable this behavior, meaning to keep zero values in their literal form (e.g. `0` for integral types), `Encoder` offers a `KeepZeros` setter method, which will do just that when set to `true`. | ||
133 | + | ||
134 | +### Unsupported Values | ||
135 | + | ||
136 | +Values of the following kinds aren't supported and, if present, must be ignored. | ||
137 | + | ||
138 | + - Channel | ||
139 | + - Function | ||
140 | + - Unsafe pointer | ||
141 | + - An alias of any of the above | ||
142 | + - A pointer to any of the above | ||
143 | + | ||
144 | +Custom Marshaling | ||
145 | +----------------- | ||
146 | + | ||
147 | +There is a default (generally lossless) marshaling & unmarshaling scheme for any concrete data value in Go, which is good enough in most cases. However, it is possible to override it and use a custom scheme. For instance, a "binary" field could be marshaled more efficiently using [base64](http://golang.org/pkg/encoding/base64/) to prevent it from being percent-escaped during serialization to `application/x-www-form-urlencoded` format. | ||
148 | + | ||
149 | +Because `form` provides support for [`encoding.TextMarshaler`](http://golang.org/pkg/encoding/#TextMarshaler) and [`encoding.TextUnmarshaler`](http://golang.org/pkg/encoding/#TextUnmarshaler) it is easy to do that; for instance, like this: | ||
150 | + | ||
151 | +```go | ||
152 | +import "encoding" | ||
153 | + | ||
154 | +type Binary []byte | ||
155 | + | ||
156 | +var ( | ||
157 | + _ encoding.TextMarshaler = &Binary{} | ||
158 | + _ encoding.TextUnmarshaler = &Binary{} | ||
159 | +) | ||
160 | + | ||
161 | +func (b Binary) MarshalText() ([]byte, error) { | ||
162 | + return []byte(base64.URLEncoding.EncodeToString([]byte(b))), nil | ||
163 | +} | ||
164 | + | ||
165 | +func (b *Binary) UnmarshalText(text []byte) error { | ||
166 | + bs, err := base64.URLEncoding.DecodeString(string(text)) | ||
167 | + if err == nil { | ||
168 | + *b = Binary(bs) | ||
169 | + } | ||
170 | + return err | ||
171 | +} | ||
172 | +``` | ||
173 | + | ||
174 | +Now any value with type `Binary` will automatically be encoded using the [URL](http://golang.org/pkg/encoding/base64/#URLEncoding) variant of base64. It is left as an exercise to the reader to improve upon this scheme by eliminating the need for padding (which, besides being superfluous, uses `=`, a character that will end up percent-escaped.) | ||
175 | + | ||
176 | +Keys | ||
177 | +---- | ||
178 | + | ||
179 | +In theory any value can be a key as long as it has a string representation. However, by default, periods have special meaning to `form`, and thus, under the hood (i.e. in encoded form) they are transparently escaped using a preceding backslash (`\`). Backslashes within keys, themselves, are also escaped in this manner (e.g. as `\\`) in order to permit representing `\.` itself (as `\\\.`). | ||
180 | + | ||
181 | +(Note: it is normally unnecessary to deal with this issue unless keys are being constructed manually—e.g. literally embedded in HTML or in a URI.) | ||
182 | + | ||
183 | +The default delimiter and escape characters used for encoding and decoding composite keys can be changed using the `DelimitWith` and `EscapeWith` setter methods of `Encoder` and `Decoder`, respectively. For example... | ||
184 | + | ||
185 | +```go | ||
186 | +package main | ||
187 | + | ||
188 | +import ( | ||
189 | + "os" | ||
190 | + | ||
191 | + "github.com/ajg/form" | ||
192 | +) | ||
193 | + | ||
194 | +func main() { | ||
195 | + type B struct { | ||
196 | + Qux string `form:"qux"` | ||
197 | + } | ||
198 | + type A struct { | ||
199 | + FooBar B `form:"foo.bar"` | ||
200 | + } | ||
201 | + a := A{FooBar: B{"XYZ"}} | ||
202 | + os.Stdout.WriteString("Default: ") | ||
203 | + form.NewEncoder(os.Stdout).Encode(a) | ||
204 | + os.Stdout.WriteString("\nCustom: ") | ||
205 | + form.NewEncoder(os.Stdout).DelimitWith('/').Encode(a) | ||
206 | + os.Stdout.WriteString("\n") | ||
207 | +} | ||
208 | + | ||
209 | +``` | ||
210 | + | ||
211 | +...will produce... | ||
212 | + | ||
213 | +``` | ||
214 | +Default: foo%5C.bar.qux=XYZ | ||
215 | +Custom: foo.bar%2Fqux=XYZ | ||
216 | +``` | ||
217 | + | ||
218 | +(`%5C` and `%2F` represent `\` and `/`, respectively.) | ||
219 | + | ||
220 | +Limitations | ||
221 | +----------- | ||
222 | + | ||
223 | + - Circular (self-referential) values are untested. | ||
224 | + | ||
225 | +Future Work | ||
226 | +----------- | ||
227 | + | ||
228 | +The following items would be nice to have in the future—though they are not being worked on yet: | ||
229 | + | ||
230 | + - An option to treat all values as if they had been tagged with `omitempty`. | ||
231 | + - An option to automatically treat all field names in `camelCase` or `underscore_case`. | ||
232 | + - Built-in support for the types in [`math/big`](http://golang.org/pkg/math/big/). | ||
233 | + - Built-in support for the types in [`image/color`](http://golang.org/pkg/image/color/). | ||
234 | + - Improve encoding/decoding by reading/writing directly from/to the `io.Reader`/`io.Writer` when possible, rather than going through an intermediate representation (i.e. `node`) which requires more memory. | ||
235 | + | ||
236 | +(Feel free to implement any of these and then send a pull request.) | ||
237 | + | ||
238 | +Related Work | ||
239 | +------------ | ||
240 | + | ||
241 | + - Package [gorilla/schema](https://github.com/gorilla/schema), which only implements decoding. | ||
242 | + - Package [google/go-querystring](https://github.com/google/go-querystring), which only implements encoding. | ||
243 | + | ||
244 | +License | ||
245 | +------- | ||
246 | + | ||
247 | +This library is distributed under a BSD-style [LICENSE](./LICENSE). |
vendor/github.com/ajg/form/TODO.md
0 → 100644
vendor/github.com/ajg/form/decode.go
0 → 100644
1 | +// Copyright 2014 Alvaro J. Genial. All rights reserved. | ||
2 | +// Use of this source code is governed by a BSD-style | ||
3 | +// license that can be found in the LICENSE file. | ||
4 | + | ||
5 | +package form | ||
6 | + | ||
7 | +import ( | ||
8 | + "fmt" | ||
9 | + "io" | ||
10 | + "io/ioutil" | ||
11 | + "net/url" | ||
12 | + "reflect" | ||
13 | + "strconv" | ||
14 | + "time" | ||
15 | +) | ||
16 | + | ||
17 | +// NewDecoder returns a new form Decoder. | ||
18 | +func NewDecoder(r io.Reader) *Decoder { | ||
19 | + return &Decoder{r, defaultDelimiter, defaultEscape, false, false} | ||
20 | +} | ||
21 | + | ||
22 | +// Decoder decodes data from a form (application/x-www-form-urlencoded). | ||
23 | +type Decoder struct { | ||
24 | + r io.Reader | ||
25 | + d rune | ||
26 | + e rune | ||
27 | + ignoreUnknown bool | ||
28 | + ignoreCase bool | ||
29 | +} | ||
30 | + | ||
31 | +// DelimitWith sets r as the delimiter used for composite keys by Decoder d and returns the latter; it is '.' by default. | ||
32 | +func (d *Decoder) DelimitWith(r rune) *Decoder { | ||
33 | + d.d = r | ||
34 | + return d | ||
35 | +} | ||
36 | + | ||
37 | +// EscapeWith sets r as the escape used for delimiters (and to escape itself) by Decoder d and returns the latter; it is '\\' by default. | ||
38 | +func (d *Decoder) EscapeWith(r rune) *Decoder { | ||
39 | + d.e = r | ||
40 | + return d | ||
41 | +} | ||
42 | + | ||
43 | +// Decode reads in and decodes form-encoded data into dst. | ||
44 | +func (d Decoder) Decode(dst interface{}) error { | ||
45 | + bs, err := ioutil.ReadAll(d.r) | ||
46 | + if err != nil { | ||
47 | + return err | ||
48 | + } | ||
49 | + vs, err := url.ParseQuery(string(bs)) | ||
50 | + if err != nil { | ||
51 | + return err | ||
52 | + } | ||
53 | + v := reflect.ValueOf(dst) | ||
54 | + return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v))) | ||
55 | +} | ||
56 | + | ||
57 | +// IgnoreUnknownKeys if set to true it will make the Decoder ignore values | ||
58 | +// that are not found in the destination object instead of returning an error. | ||
59 | +func (d *Decoder) IgnoreUnknownKeys(ignoreUnknown bool) { | ||
60 | + d.ignoreUnknown = ignoreUnknown | ||
61 | +} | ||
62 | + | ||
63 | +// IgnoreCase if set to true it will make the Decoder try to set values in the | ||
64 | +// destination object even if the case does not match. | ||
65 | +func (d *Decoder) IgnoreCase(ignoreCase bool) { | ||
66 | + d.ignoreCase = ignoreCase | ||
67 | +} | ||
68 | + | ||
69 | +// DecodeString decodes src into dst. | ||
70 | +func (d Decoder) DecodeString(dst interface{}, src string) error { | ||
71 | + vs, err := url.ParseQuery(src) | ||
72 | + if err != nil { | ||
73 | + return err | ||
74 | + } | ||
75 | + v := reflect.ValueOf(dst) | ||
76 | + return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v))) | ||
77 | +} | ||
78 | + | ||
79 | +// DecodeValues decodes vs into dst. | ||
80 | +func (d Decoder) DecodeValues(dst interface{}, vs url.Values) error { | ||
81 | + v := reflect.ValueOf(dst) | ||
82 | + return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v))) | ||
83 | +} | ||
84 | + | ||
85 | +// DecodeString decodes src into dst. | ||
86 | +func DecodeString(dst interface{}, src string) error { | ||
87 | + return NewDecoder(nil).DecodeString(dst, src) | ||
88 | +} | ||
89 | + | ||
90 | +// DecodeValues decodes vs into dst. | ||
91 | +func DecodeValues(dst interface{}, vs url.Values) error { | ||
92 | + return NewDecoder(nil).DecodeValues(dst, vs) | ||
93 | +} | ||
94 | + | ||
95 | +func (d Decoder) decodeNode(v reflect.Value, n node) (err error) { | ||
96 | + defer func() { | ||
97 | + if e := recover(); e != nil { | ||
98 | + err = fmt.Errorf("%v", e) | ||
99 | + } | ||
100 | + }() | ||
101 | + | ||
102 | + if v.Kind() == reflect.Slice { | ||
103 | + return fmt.Errorf("could not decode directly into slice; use pointer to slice") | ||
104 | + } | ||
105 | + d.decodeValue(v, n) | ||
106 | + return nil | ||
107 | +} | ||
108 | + | ||
109 | +func (d Decoder) decodeValue(v reflect.Value, x interface{}) { | ||
110 | + t := v.Type() | ||
111 | + k := v.Kind() | ||
112 | + | ||
113 | + if k == reflect.Ptr && v.IsNil() { | ||
114 | + v.Set(reflect.New(t.Elem())) | ||
115 | + } | ||
116 | + | ||
117 | + if unmarshalValue(v, x) { | ||
118 | + return | ||
119 | + } | ||
120 | + | ||
121 | + empty := isEmpty(x) | ||
122 | + | ||
123 | + switch k { | ||
124 | + case reflect.Ptr: | ||
125 | + d.decodeValue(v.Elem(), x) | ||
126 | + return | ||
127 | + case reflect.Interface: | ||
128 | + if !v.IsNil() { | ||
129 | + d.decodeValue(v.Elem(), x) | ||
130 | + return | ||
131 | + | ||
132 | + } else if empty { | ||
133 | + return // Allow nil interfaces only if empty. | ||
134 | + } else { | ||
135 | + panic("form: cannot decode non-empty value into into nil interface") | ||
136 | + } | ||
137 | + } | ||
138 | + | ||
139 | + if empty { | ||
140 | + v.Set(reflect.Zero(t)) // Treat the empty string as the zero value. | ||
141 | + return | ||
142 | + } | ||
143 | + | ||
144 | + switch k { | ||
145 | + case reflect.Struct: | ||
146 | + if t.ConvertibleTo(timeType) { | ||
147 | + d.decodeTime(v, x) | ||
148 | + } else if t.ConvertibleTo(urlType) { | ||
149 | + d.decodeURL(v, x) | ||
150 | + } else { | ||
151 | + d.decodeStruct(v, x) | ||
152 | + } | ||
153 | + case reflect.Slice: | ||
154 | + d.decodeSlice(v, x) | ||
155 | + case reflect.Array: | ||
156 | + d.decodeArray(v, x) | ||
157 | + case reflect.Map: | ||
158 | + d.decodeMap(v, x) | ||
159 | + case reflect.Invalid, reflect.Uintptr, reflect.UnsafePointer, reflect.Chan, reflect.Func: | ||
160 | + panic(t.String() + " has unsupported kind " + k.String()) | ||
161 | + default: | ||
162 | + d.decodeBasic(v, x) | ||
163 | + } | ||
164 | +} | ||
165 | + | ||
166 | +func (d Decoder) decodeStruct(v reflect.Value, x interface{}) { | ||
167 | + t := v.Type() | ||
168 | + for k, c := range getNode(x) { | ||
169 | + if f, ok := findField(v, k, d.ignoreCase); !ok && k == "" { | ||
170 | + panic(getString(x) + " cannot be decoded as " + t.String()) | ||
171 | + } else if !ok { | ||
172 | + if !d.ignoreUnknown { | ||
173 | + panic(k + " doesn't exist in " + t.String()) | ||
174 | + } | ||
175 | + } else if !f.CanSet() { | ||
176 | + panic(k + " cannot be set in " + t.String()) | ||
177 | + } else { | ||
178 | + d.decodeValue(f, c) | ||
179 | + } | ||
180 | + } | ||
181 | +} | ||
182 | + | ||
183 | +func (d Decoder) decodeMap(v reflect.Value, x interface{}) { | ||
184 | + t := v.Type() | ||
185 | + if v.IsNil() { | ||
186 | + v.Set(reflect.MakeMap(t)) | ||
187 | + } | ||
188 | + for k, c := range getNode(x) { | ||
189 | + i := reflect.New(t.Key()).Elem() | ||
190 | + d.decodeValue(i, k) | ||
191 | + | ||
192 | + w := v.MapIndex(i) | ||
193 | + if w.IsValid() { // We have an actual element value to decode into. | ||
194 | + if w.Kind() == reflect.Interface { | ||
195 | + w = w.Elem() | ||
196 | + } | ||
197 | + w = reflect.New(w.Type()).Elem() | ||
198 | + } else if t.Elem().Kind() != reflect.Interface { // The map's element type is concrete. | ||
199 | + w = reflect.New(t.Elem()).Elem() | ||
200 | + } else { | ||
201 | + // The best we can do here is to decode as either a string (for scalars) or a map[string]interface {} (for the rest). | ||
202 | + // We could try to guess the type based on the string (e.g. true/false => bool) but that'll get ugly fast, | ||
203 | + // especially if we have to guess the kind (slice vs. array vs. map) and index type (e.g. string, int, etc.) | ||
204 | + switch c.(type) { | ||
205 | + case node: | ||
206 | + w = reflect.MakeMap(stringMapType) | ||
207 | + case string: | ||
208 | + w = reflect.New(stringType).Elem() | ||
209 | + default: | ||
210 | + panic("value is neither node nor string") | ||
211 | + } | ||
212 | + } | ||
213 | + | ||
214 | + d.decodeValue(w, c) | ||
215 | + v.SetMapIndex(i, w) | ||
216 | + } | ||
217 | +} | ||
218 | + | ||
219 | +func (d Decoder) decodeArray(v reflect.Value, x interface{}) { | ||
220 | + t := v.Type() | ||
221 | + for k, c := range getNode(x) { | ||
222 | + i, err := strconv.Atoi(k) | ||
223 | + if err != nil { | ||
224 | + panic(k + " is not a valid index for type " + t.String()) | ||
225 | + } | ||
226 | + if l := v.Len(); i >= l { | ||
227 | + panic("index is above array size") | ||
228 | + } | ||
229 | + d.decodeValue(v.Index(i), c) | ||
230 | + } | ||
231 | +} | ||
232 | + | ||
233 | +func (d Decoder) decodeSlice(v reflect.Value, x interface{}) { | ||
234 | + t := v.Type() | ||
235 | + if t.Elem().Kind() == reflect.Uint8 { | ||
236 | + // Allow, but don't require, byte slices to be encoded as a single string. | ||
237 | + if s, ok := x.(string); ok { | ||
238 | + v.SetBytes([]byte(s)) | ||
239 | + return | ||
240 | + } | ||
241 | + } | ||
242 | + | ||
243 | + // NOTE: Implicit indexing is currently done at the parseValues level, | ||
244 | + // so if if an implicitKey reaches here it will always replace the last. | ||
245 | + implicit := 0 | ||
246 | + for k, c := range getNode(x) { | ||
247 | + var i int | ||
248 | + if k == implicitKey { | ||
249 | + i = implicit | ||
250 | + implicit++ | ||
251 | + } else { | ||
252 | + explicit, err := strconv.Atoi(k) | ||
253 | + if err != nil { | ||
254 | + panic(k + " is not a valid index for type " + t.String()) | ||
255 | + } | ||
256 | + i = explicit | ||
257 | + implicit = explicit + 1 | ||
258 | + } | ||
259 | + // "Extend" the slice if it's too short. | ||
260 | + if l := v.Len(); i >= l { | ||
261 | + delta := i - l + 1 | ||
262 | + v.Set(reflect.AppendSlice(v, reflect.MakeSlice(t, delta, delta))) | ||
263 | + } | ||
264 | + d.decodeValue(v.Index(i), c) | ||
265 | + } | ||
266 | +} | ||
267 | + | ||
268 | +func (d Decoder) decodeBasic(v reflect.Value, x interface{}) { | ||
269 | + t := v.Type() | ||
270 | + switch k, s := t.Kind(), getString(x); k { | ||
271 | + case reflect.Bool: | ||
272 | + if b, e := strconv.ParseBool(s); e == nil { | ||
273 | + v.SetBool(b) | ||
274 | + } else { | ||
275 | + panic("could not parse bool from " + strconv.Quote(s)) | ||
276 | + } | ||
277 | + case reflect.Int, | ||
278 | + reflect.Int8, | ||
279 | + reflect.Int16, | ||
280 | + reflect.Int32, | ||
281 | + reflect.Int64: | ||
282 | + if i, e := strconv.ParseInt(s, 10, 64); e == nil { | ||
283 | + v.SetInt(i) | ||
284 | + } else { | ||
285 | + panic("could not parse int from " + strconv.Quote(s)) | ||
286 | + } | ||
287 | + case reflect.Uint, | ||
288 | + reflect.Uint8, | ||
289 | + reflect.Uint16, | ||
290 | + reflect.Uint32, | ||
291 | + reflect.Uint64: | ||
292 | + if u, e := strconv.ParseUint(s, 10, 64); e == nil { | ||
293 | + v.SetUint(u) | ||
294 | + } else { | ||
295 | + panic("could not parse uint from " + strconv.Quote(s)) | ||
296 | + } | ||
297 | + case reflect.Float32, | ||
298 | + reflect.Float64: | ||
299 | + if f, e := strconv.ParseFloat(s, 64); e == nil { | ||
300 | + v.SetFloat(f) | ||
301 | + } else { | ||
302 | + panic("could not parse float from " + strconv.Quote(s)) | ||
303 | + } | ||
304 | + case reflect.Complex64, | ||
305 | + reflect.Complex128: | ||
306 | + var c complex128 | ||
307 | + if n, err := fmt.Sscanf(s, "%g", &c); n == 1 && err == nil { | ||
308 | + v.SetComplex(c) | ||
309 | + } else { | ||
310 | + panic("could not parse complex from " + strconv.Quote(s)) | ||
311 | + } | ||
312 | + case reflect.String: | ||
313 | + v.SetString(s) | ||
314 | + default: | ||
315 | + panic(t.String() + " has unsupported kind " + k.String()) | ||
316 | + } | ||
317 | +} | ||
318 | + | ||
319 | +func (d Decoder) decodeTime(v reflect.Value, x interface{}) { | ||
320 | + t := v.Type() | ||
321 | + s := getString(x) | ||
322 | + // TODO: Find a more efficient way to do this. | ||
323 | + for _, f := range allowedTimeFormats { | ||
324 | + if p, err := time.Parse(f, s); err == nil { | ||
325 | + v.Set(reflect.ValueOf(p).Convert(v.Type())) | ||
326 | + return | ||
327 | + } | ||
328 | + } | ||
329 | + panic("cannot decode string `" + s + "` as " + t.String()) | ||
330 | +} | ||
331 | + | ||
332 | +func (d Decoder) decodeURL(v reflect.Value, x interface{}) { | ||
333 | + t := v.Type() | ||
334 | + s := getString(x) | ||
335 | + if u, err := url.Parse(s); err == nil { | ||
336 | + v.Set(reflect.ValueOf(*u).Convert(v.Type())) | ||
337 | + return | ||
338 | + } | ||
339 | + panic("cannot decode string `" + s + "` as " + t.String()) | ||
340 | +} | ||
341 | + | ||
342 | +var allowedTimeFormats = []string{ | ||
343 | + "2006-01-02T15:04:05.999999999Z07:00", | ||
344 | + "2006-01-02T15:04:05.999999999Z07", | ||
345 | + "2006-01-02T15:04:05.999999999Z", | ||
346 | + "2006-01-02T15:04:05.999999999", | ||
347 | + "2006-01-02T15:04:05Z07:00", | ||
348 | + "2006-01-02T15:04:05Z07", | ||
349 | + "2006-01-02T15:04:05Z", | ||
350 | + "2006-01-02T15:04:05", | ||
351 | + "2006-01-02T15:04Z", | ||
352 | + "2006-01-02T15:04", | ||
353 | + "2006-01-02T15Z", | ||
354 | + "2006-01-02T15", | ||
355 | + "2006-01-02", | ||
356 | + "2006-01", | ||
357 | + "2006", | ||
358 | + "15:04:05.999999999Z07:00", | ||
359 | + "15:04:05.999999999Z07", | ||
360 | + "15:04:05.999999999Z", | ||
361 | + "15:04:05.999999999", | ||
362 | + "15:04:05Z07:00", | ||
363 | + "15:04:05Z07", | ||
364 | + "15:04:05Z", | ||
365 | + "15:04:05", | ||
366 | + "15:04Z", | ||
367 | + "15:04", | ||
368 | + "15Z", | ||
369 | + "15", | ||
370 | +} |
vendor/github.com/ajg/form/encode.go
0 → 100644
1 | +// Copyright 2014 Alvaro J. Genial. All rights reserved. | ||
2 | +// Use of this source code is governed by a BSD-style | ||
3 | +// license that can be found in the LICENSE file. | ||
4 | + | ||
5 | +package form | ||
6 | + | ||
7 | +import ( | ||
8 | + "encoding" | ||
9 | + "errors" | ||
10 | + "fmt" | ||
11 | + "io" | ||
12 | + "net/url" | ||
13 | + "reflect" | ||
14 | + "strconv" | ||
15 | + "strings" | ||
16 | + "time" | ||
17 | +) | ||
18 | + | ||
19 | +// NewEncoder returns a new form Encoder. | ||
20 | +func NewEncoder(w io.Writer) *Encoder { | ||
21 | + return &Encoder{w, defaultDelimiter, defaultEscape, false} | ||
22 | +} | ||
23 | + | ||
24 | +// Encoder provides a way to encode to a Writer. | ||
25 | +type Encoder struct { | ||
26 | + w io.Writer | ||
27 | + d rune | ||
28 | + e rune | ||
29 | + z bool | ||
30 | +} | ||
31 | + | ||
32 | +// DelimitWith sets r as the delimiter used for composite keys by Encoder e and returns the latter; it is '.' by default. | ||
33 | +func (e *Encoder) DelimitWith(r rune) *Encoder { | ||
34 | + e.d = r | ||
35 | + return e | ||
36 | +} | ||
37 | + | ||
38 | +// EscapeWith sets r as the escape used for delimiters (and to escape itself) by Encoder e and returns the latter; it is '\\' by default. | ||
39 | +func (e *Encoder) EscapeWith(r rune) *Encoder { | ||
40 | + e.e = r | ||
41 | + return e | ||
42 | +} | ||
43 | + | ||
44 | +// KeepZeros sets whether Encoder e should keep zero (default) values in their literal form when encoding, and returns the former; by default zero values are not kept, but are rather encoded as the empty string. | ||
45 | +func (e *Encoder) KeepZeros(z bool) *Encoder { | ||
46 | + e.z = z | ||
47 | + return e | ||
48 | +} | ||
49 | + | ||
50 | +// Encode encodes dst as form and writes it out using the Encoder's Writer. | ||
51 | +func (e Encoder) Encode(dst interface{}) error { | ||
52 | + v := reflect.ValueOf(dst) | ||
53 | + n, err := encodeToNode(v, e.z) | ||
54 | + if err != nil { | ||
55 | + return err | ||
56 | + } | ||
57 | + s := n.values(e.d, e.e).Encode() | ||
58 | + l, err := io.WriteString(e.w, s) | ||
59 | + switch { | ||
60 | + case err != nil: | ||
61 | + return err | ||
62 | + case l != len(s): | ||
63 | + return errors.New("could not write data completely") | ||
64 | + } | ||
65 | + return nil | ||
66 | +} | ||
67 | + | ||
68 | +// EncodeToString encodes dst as a form and returns it as a string. | ||
69 | +func EncodeToString(dst interface{}) (string, error) { | ||
70 | + v := reflect.ValueOf(dst) | ||
71 | + n, err := encodeToNode(v, false) | ||
72 | + if err != nil { | ||
73 | + return "", err | ||
74 | + } | ||
75 | + vs := n.values(defaultDelimiter, defaultEscape) | ||
76 | + return vs.Encode(), nil | ||
77 | +} | ||
78 | + | ||
79 | +// EncodeToValues encodes dst as a form and returns it as Values. | ||
80 | +func EncodeToValues(dst interface{}) (url.Values, error) { | ||
81 | + v := reflect.ValueOf(dst) | ||
82 | + n, err := encodeToNode(v, false) | ||
83 | + if err != nil { | ||
84 | + return nil, err | ||
85 | + } | ||
86 | + vs := n.values(defaultDelimiter, defaultEscape) | ||
87 | + return vs, nil | ||
88 | +} | ||
89 | + | ||
90 | +func encodeToNode(v reflect.Value, z bool) (n node, err error) { | ||
91 | + defer func() { | ||
92 | + if e := recover(); e != nil { | ||
93 | + err = fmt.Errorf("%v", e) | ||
94 | + } | ||
95 | + }() | ||
96 | + return getNode(encodeValue(v, z)), nil | ||
97 | +} | ||
98 | + | ||
99 | +func encodeValue(v reflect.Value, z bool) interface{} { | ||
100 | + t := v.Type() | ||
101 | + k := v.Kind() | ||
102 | + | ||
103 | + if s, ok := marshalValue(v); ok { | ||
104 | + return s | ||
105 | + } else if !z && isEmptyValue(v) { | ||
106 | + return "" // Treat the zero value as the empty string. | ||
107 | + } | ||
108 | + | ||
109 | + switch k { | ||
110 | + case reflect.Ptr, reflect.Interface: | ||
111 | + return encodeValue(v.Elem(), z) | ||
112 | + case reflect.Struct: | ||
113 | + if t.ConvertibleTo(timeType) { | ||
114 | + return encodeTime(v) | ||
115 | + } else if t.ConvertibleTo(urlType) { | ||
116 | + return encodeURL(v) | ||
117 | + } | ||
118 | + return encodeStruct(v, z) | ||
119 | + case reflect.Slice: | ||
120 | + return encodeSlice(v, z) | ||
121 | + case reflect.Array: | ||
122 | + return encodeArray(v, z) | ||
123 | + case reflect.Map: | ||
124 | + return encodeMap(v, z) | ||
125 | + case reflect.Invalid, reflect.Uintptr, reflect.UnsafePointer, reflect.Chan, reflect.Func: | ||
126 | + panic(t.String() + " has unsupported kind " + t.Kind().String()) | ||
127 | + default: | ||
128 | + return encodeBasic(v) | ||
129 | + } | ||
130 | +} | ||
131 | + | ||
132 | +func encodeStruct(v reflect.Value, z bool) interface{} { | ||
133 | + t := v.Type() | ||
134 | + n := node{} | ||
135 | + for i := 0; i < t.NumField(); i++ { | ||
136 | + f := t.Field(i) | ||
137 | + k, oe := fieldInfo(f) | ||
138 | + | ||
139 | + if k == "-" { | ||
140 | + continue | ||
141 | + } else if fv := v.Field(i); oe && isEmptyValue(fv) { | ||
142 | + delete(n, k) | ||
143 | + } else { | ||
144 | + n[k] = encodeValue(fv, z) | ||
145 | + } | ||
146 | + } | ||
147 | + return n | ||
148 | +} | ||
149 | + | ||
150 | +func encodeMap(v reflect.Value, z bool) interface{} { | ||
151 | + n := node{} | ||
152 | + for _, i := range v.MapKeys() { | ||
153 | + k := getString(encodeValue(i, z)) | ||
154 | + n[k] = encodeValue(v.MapIndex(i), z) | ||
155 | + } | ||
156 | + return n | ||
157 | +} | ||
158 | + | ||
159 | +func encodeArray(v reflect.Value, z bool) interface{} { | ||
160 | + n := node{} | ||
161 | + for i := 0; i < v.Len(); i++ { | ||
162 | + n[strconv.Itoa(i)] = encodeValue(v.Index(i), z) | ||
163 | + } | ||
164 | + return n | ||
165 | +} | ||
166 | + | ||
167 | +func encodeSlice(v reflect.Value, z bool) interface{} { | ||
168 | + t := v.Type() | ||
169 | + if t.Elem().Kind() == reflect.Uint8 { | ||
170 | + return string(v.Bytes()) // Encode byte slices as a single string by default. | ||
171 | + } | ||
172 | + n := node{} | ||
173 | + for i := 0; i < v.Len(); i++ { | ||
174 | + n[strconv.Itoa(i)] = encodeValue(v.Index(i), z) | ||
175 | + } | ||
176 | + return n | ||
177 | +} | ||
178 | + | ||
179 | +func encodeTime(v reflect.Value) string { | ||
180 | + t := v.Convert(timeType).Interface().(time.Time) | ||
181 | + if t.Year() == 0 && (t.Month() == 0 || t.Month() == 1) && (t.Day() == 0 || t.Day() == 1) { | ||
182 | + return t.Format("15:04:05.999999999Z07:00") | ||
183 | + } else if t.Hour() == 0 && t.Minute() == 0 && t.Second() == 0 && t.Nanosecond() == 0 { | ||
184 | + return t.Format("2006-01-02") | ||
185 | + } | ||
186 | + return t.Format("2006-01-02T15:04:05.999999999Z07:00") | ||
187 | +} | ||
188 | + | ||
189 | +func encodeURL(v reflect.Value) string { | ||
190 | + u := v.Convert(urlType).Interface().(url.URL) | ||
191 | + return u.String() | ||
192 | +} | ||
193 | + | ||
194 | +func encodeBasic(v reflect.Value) string { | ||
195 | + t := v.Type() | ||
196 | + switch k := t.Kind(); k { | ||
197 | + case reflect.Bool: | ||
198 | + return strconv.FormatBool(v.Bool()) | ||
199 | + case reflect.Int, | ||
200 | + reflect.Int8, | ||
201 | + reflect.Int16, | ||
202 | + reflect.Int32, | ||
203 | + reflect.Int64: | ||
204 | + return strconv.FormatInt(v.Int(), 10) | ||
205 | + case reflect.Uint, | ||
206 | + reflect.Uint8, | ||
207 | + reflect.Uint16, | ||
208 | + reflect.Uint32, | ||
209 | + reflect.Uint64: | ||
210 | + return strconv.FormatUint(v.Uint(), 10) | ||
211 | + case reflect.Float32: | ||
212 | + return strconv.FormatFloat(v.Float(), 'g', -1, 32) | ||
213 | + case reflect.Float64: | ||
214 | + return strconv.FormatFloat(v.Float(), 'g', -1, 64) | ||
215 | + case reflect.Complex64, reflect.Complex128: | ||
216 | + s := fmt.Sprintf("%g", v.Complex()) | ||
217 | + return strings.TrimSuffix(strings.TrimPrefix(s, "("), ")") | ||
218 | + case reflect.String: | ||
219 | + return v.String() | ||
220 | + } | ||
221 | + panic(t.String() + " has unsupported kind " + t.Kind().String()) | ||
222 | +} | ||
223 | + | ||
224 | +func isEmptyValue(v reflect.Value) bool { | ||
225 | + switch t := v.Type(); v.Kind() { | ||
226 | + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: | ||
227 | + return v.Len() == 0 | ||
228 | + case reflect.Bool: | ||
229 | + return !v.Bool() | ||
230 | + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: | ||
231 | + return v.Int() == 0 | ||
232 | + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: | ||
233 | + return v.Uint() == 0 | ||
234 | + case reflect.Float32, reflect.Float64: | ||
235 | + return v.Float() == 0 | ||
236 | + case reflect.Complex64, reflect.Complex128: | ||
237 | + return v.Complex() == 0 | ||
238 | + case reflect.Interface, reflect.Ptr: | ||
239 | + return v.IsNil() | ||
240 | + case reflect.Struct: | ||
241 | + if t.ConvertibleTo(timeType) { | ||
242 | + return v.Convert(timeType).Interface().(time.Time).IsZero() | ||
243 | + } | ||
244 | + return reflect.DeepEqual(v, reflect.Zero(t)) | ||
245 | + } | ||
246 | + return false | ||
247 | +} | ||
248 | + | ||
249 | +// canIndexOrdinally returns whether a value contains an ordered sequence of elements. | ||
250 | +func canIndexOrdinally(v reflect.Value) bool { | ||
251 | + if !v.IsValid() { | ||
252 | + return false | ||
253 | + } | ||
254 | + switch t := v.Type(); t.Kind() { | ||
255 | + case reflect.Ptr, reflect.Interface: | ||
256 | + return canIndexOrdinally(v.Elem()) | ||
257 | + case reflect.Slice, reflect.Array: | ||
258 | + return true | ||
259 | + } | ||
260 | + return false | ||
261 | +} | ||
262 | + | ||
263 | +func fieldInfo(f reflect.StructField) (k string, oe bool) { | ||
264 | + if f.PkgPath != "" { // Skip private fields. | ||
265 | + return omittedKey, oe | ||
266 | + } | ||
267 | + | ||
268 | + k = f.Name | ||
269 | + tag := f.Tag.Get("form") | ||
270 | + if tag == "" { | ||
271 | + return k, oe | ||
272 | + } | ||
273 | + | ||
274 | + ps := strings.SplitN(tag, ",", 2) | ||
275 | + if ps[0] != "" { | ||
276 | + k = ps[0] | ||
277 | + } | ||
278 | + if len(ps) == 2 { | ||
279 | + oe = ps[1] == "omitempty" | ||
280 | + } | ||
281 | + return k, oe | ||
282 | +} | ||
283 | + | ||
284 | +func findField(v reflect.Value, n string, ignoreCase bool) (reflect.Value, bool) { | ||
285 | + t := v.Type() | ||
286 | + l := v.NumField() | ||
287 | + | ||
288 | + var lowerN string | ||
289 | + caseInsensitiveMatch := -1 | ||
290 | + if ignoreCase { | ||
291 | + lowerN = strings.ToLower(n) | ||
292 | + } | ||
293 | + | ||
294 | + // First try named fields. | ||
295 | + for i := 0; i < l; i++ { | ||
296 | + f := t.Field(i) | ||
297 | + k, _ := fieldInfo(f) | ||
298 | + if k == omittedKey { | ||
299 | + continue | ||
300 | + } else if n == k { | ||
301 | + return v.Field(i), true | ||
302 | + } else if ignoreCase && lowerN == strings.ToLower(k) { | ||
303 | + caseInsensitiveMatch = i | ||
304 | + } | ||
305 | + } | ||
306 | + | ||
307 | + // If no exact match was found try case insensitive match. | ||
308 | + if caseInsensitiveMatch != -1 { | ||
309 | + return v.Field(caseInsensitiveMatch), true | ||
310 | + } | ||
311 | + | ||
312 | + // Then try anonymous (embedded) fields. | ||
313 | + for i := 0; i < l; i++ { | ||
314 | + f := t.Field(i) | ||
315 | + k, _ := fieldInfo(f) | ||
316 | + if k == omittedKey || !f.Anonymous { // || k != "" ? | ||
317 | + continue | ||
318 | + } | ||
319 | + fv := v.Field(i) | ||
320 | + fk := fv.Kind() | ||
321 | + for fk == reflect.Ptr || fk == reflect.Interface { | ||
322 | + fv = fv.Elem() | ||
323 | + fk = fv.Kind() | ||
324 | + } | ||
325 | + | ||
326 | + if fk != reflect.Struct { | ||
327 | + continue | ||
328 | + } | ||
329 | + if ev, ok := findField(fv, n, ignoreCase); ok { | ||
330 | + return ev, true | ||
331 | + } | ||
332 | + } | ||
333 | + | ||
334 | + return reflect.Value{}, false | ||
335 | +} | ||
336 | + | ||
337 | +var ( | ||
338 | + stringType = reflect.TypeOf(string("")) | ||
339 | + stringMapType = reflect.TypeOf(map[string]interface{}{}) | ||
340 | + timeType = reflect.TypeOf(time.Time{}) | ||
341 | + timePtrType = reflect.TypeOf(&time.Time{}) | ||
342 | + urlType = reflect.TypeOf(url.URL{}) | ||
343 | +) | ||
344 | + | ||
345 | +func skipTextMarshalling(t reflect.Type) bool { | ||
346 | + /*// Skip time.Time because its text unmarshaling is overly rigid: | ||
347 | + return t == timeType || t == timePtrType*/ | ||
348 | + // Skip time.Time & convertibles because its text unmarshaling is overly rigid: | ||
349 | + return t.ConvertibleTo(timeType) || t.ConvertibleTo(timePtrType) | ||
350 | +} | ||
351 | + | ||
352 | +func unmarshalValue(v reflect.Value, x interface{}) bool { | ||
353 | + if skipTextMarshalling(v.Type()) { | ||
354 | + return false | ||
355 | + } | ||
356 | + | ||
357 | + tu, ok := v.Interface().(encoding.TextUnmarshaler) | ||
358 | + if !ok && !v.CanAddr() { | ||
359 | + return false | ||
360 | + } else if !ok { | ||
361 | + return unmarshalValue(v.Addr(), x) | ||
362 | + } | ||
363 | + | ||
364 | + s := getString(x) | ||
365 | + if err := tu.UnmarshalText([]byte(s)); err != nil { | ||
366 | + panic(err) | ||
367 | + } | ||
368 | + return true | ||
369 | +} | ||
370 | + | ||
371 | +func marshalValue(v reflect.Value) (string, bool) { | ||
372 | + if skipTextMarshalling(v.Type()) { | ||
373 | + return "", false | ||
374 | + } | ||
375 | + | ||
376 | + tm, ok := v.Interface().(encoding.TextMarshaler) | ||
377 | + if !ok && !v.CanAddr() { | ||
378 | + return "", false | ||
379 | + } else if !ok { | ||
380 | + return marshalValue(v.Addr()) | ||
381 | + } | ||
382 | + | ||
383 | + bs, err := tm.MarshalText() | ||
384 | + if err != nil { | ||
385 | + panic(err) | ||
386 | + } | ||
387 | + return string(bs), true | ||
388 | +} |
vendor/github.com/ajg/form/form.go
0 → 100644
1 | +// Copyright 2014 Alvaro J. Genial. All rights reserved. | ||
2 | +// Use of this source code is governed by a BSD-style | ||
3 | +// license that can be found in the LICENSE file. | ||
4 | + | ||
5 | +// Package form implements encoding and decoding of application/x-www-form-urlencoded data. | ||
6 | +package form | ||
7 | + | ||
8 | +const ( | ||
9 | + implicitKey = "_" | ||
10 | + omittedKey = "-" | ||
11 | + | ||
12 | + defaultDelimiter = '.' | ||
13 | + defaultEscape = '\\' | ||
14 | +) |
vendor/github.com/ajg/form/node.go
0 → 100644
1 | +// Copyright 2014 Alvaro J. Genial. All rights reserved. | ||
2 | +// Use of this source code is governed by a BSD-style | ||
3 | +// license that can be found in the LICENSE file. | ||
4 | + | ||
5 | +package form | ||
6 | + | ||
7 | +import ( | ||
8 | + "net/url" | ||
9 | + "strconv" | ||
10 | + "strings" | ||
11 | +) | ||
12 | + | ||
13 | +type node map[string]interface{} | ||
14 | + | ||
15 | +func (n node) values(d, e rune) url.Values { | ||
16 | + vs := url.Values{} | ||
17 | + n.merge(d, e, "", &vs) | ||
18 | + return vs | ||
19 | +} | ||
20 | + | ||
21 | +func (n node) merge(d, e rune, p string, vs *url.Values) { | ||
22 | + for k, x := range n { | ||
23 | + switch y := x.(type) { | ||
24 | + case string: | ||
25 | + vs.Add(p+escape(d, e, k), y) | ||
26 | + case node: | ||
27 | + y.merge(d, e, p+escape(d, e, k)+string(d), vs) | ||
28 | + default: | ||
29 | + panic("value is neither string nor node") | ||
30 | + } | ||
31 | + } | ||
32 | +} | ||
33 | + | ||
34 | +// TODO: Add tests for implicit indexing. | ||
35 | +func parseValues(d, e rune, vs url.Values, canIndexFirstLevelOrdinally bool) node { | ||
36 | + // NOTE: Because of the flattening of potentially multiple strings to one key, implicit indexing works: | ||
37 | + // i. At the first level; e.g. Foo.Bar=A&Foo.Bar=B becomes 0.Foo.Bar=A&1.Foo.Bar=B | ||
38 | + // ii. At the last level; e.g. Foo.Bar._=A&Foo.Bar._=B becomes Foo.Bar.0=A&Foo.Bar.1=B | ||
39 | + // TODO: At in-between levels; e.g. Foo._.Bar=A&Foo._.Bar=B becomes Foo.0.Bar=A&Foo.1.Bar=B | ||
40 | + // (This last one requires that there only be one placeholder in order for it to be unambiguous.) | ||
41 | + | ||
42 | + m := map[string]string{} | ||
43 | + for k, ss := range vs { | ||
44 | + indexLastLevelOrdinally := strings.HasSuffix(k, string(d)+implicitKey) | ||
45 | + | ||
46 | + for i, s := range ss { | ||
47 | + if canIndexFirstLevelOrdinally { | ||
48 | + k = strconv.Itoa(i) + string(d) + k | ||
49 | + } else if indexLastLevelOrdinally { | ||
50 | + k = strings.TrimSuffix(k, implicitKey) + strconv.Itoa(i) | ||
51 | + } | ||
52 | + | ||
53 | + m[k] = s | ||
54 | + } | ||
55 | + } | ||
56 | + | ||
57 | + n := node{} | ||
58 | + for k, s := range m { | ||
59 | + n = n.split(d, e, k, s) | ||
60 | + } | ||
61 | + return n | ||
62 | +} | ||
63 | + | ||
64 | +func splitPath(d, e rune, path string) (k, rest string) { | ||
65 | + esc := false | ||
66 | + for i, r := range path { | ||
67 | + switch { | ||
68 | + case !esc && r == e: | ||
69 | + esc = true | ||
70 | + case !esc && r == d: | ||
71 | + return unescape(d, e, path[:i]), path[i+1:] | ||
72 | + default: | ||
73 | + esc = false | ||
74 | + } | ||
75 | + } | ||
76 | + return unescape(d, e, path), "" | ||
77 | +} | ||
78 | + | ||
79 | +func (n node) split(d, e rune, path, s string) node { | ||
80 | + k, rest := splitPath(d, e, path) | ||
81 | + if rest == "" { | ||
82 | + return add(n, k, s) | ||
83 | + } | ||
84 | + if _, ok := n[k]; !ok { | ||
85 | + n[k] = node{} | ||
86 | + } | ||
87 | + | ||
88 | + c := getNode(n[k]) | ||
89 | + n[k] = c.split(d, e, rest, s) | ||
90 | + return n | ||
91 | +} | ||
92 | + | ||
93 | +func add(n node, k, s string) node { | ||
94 | + if n == nil { | ||
95 | + return node{k: s} | ||
96 | + } | ||
97 | + | ||
98 | + if _, ok := n[k]; ok { | ||
99 | + panic("key " + k + " already set") | ||
100 | + } | ||
101 | + | ||
102 | + n[k] = s | ||
103 | + return n | ||
104 | +} | ||
105 | + | ||
106 | +func isEmpty(x interface{}) bool { | ||
107 | + switch y := x.(type) { | ||
108 | + case string: | ||
109 | + return y == "" | ||
110 | + case node: | ||
111 | + if s, ok := y[""].(string); ok { | ||
112 | + return s == "" | ||
113 | + } | ||
114 | + return false | ||
115 | + } | ||
116 | + panic("value is neither string nor node") | ||
117 | +} | ||
118 | + | ||
119 | +func getNode(x interface{}) node { | ||
120 | + switch y := x.(type) { | ||
121 | + case string: | ||
122 | + return node{"": y} | ||
123 | + case node: | ||
124 | + return y | ||
125 | + } | ||
126 | + panic("value is neither string nor node") | ||
127 | +} | ||
128 | + | ||
129 | +func getString(x interface{}) string { | ||
130 | + switch y := x.(type) { | ||
131 | + case string: | ||
132 | + return y | ||
133 | + case node: | ||
134 | + if s, ok := y[""].(string); ok { | ||
135 | + return s | ||
136 | + } | ||
137 | + return "" | ||
138 | + } | ||
139 | + panic("value is neither string nor node") | ||
140 | +} | ||
141 | + | ||
142 | +func escape(d, e rune, s string) string { | ||
143 | + s = strings.Replace(s, string(e), string(e)+string(e), -1) // Escape the escape (\ => \\) | ||
144 | + s = strings.Replace(s, string(d), string(e)+string(d), -1) // Escape the delimiter (. => \.) | ||
145 | + return s | ||
146 | +} | ||
147 | + | ||
148 | +func unescape(d, e rune, s string) string { | ||
149 | + s = strings.Replace(s, string(e)+string(d), string(d), -1) // Unescape the delimiter (\. => .) | ||
150 | + s = strings.Replace(s, string(e)+string(e), string(e), -1) // Unescape the escape (\\ => \) | ||
151 | + return s | ||
152 | +} |
vendor/github.com/ajg/form/pre-commit.sh
0 → 100644
1 | +#!/bin/bash -eu | ||
2 | + | ||
3 | +# TODO: Only colorize messages given a suitable terminal. | ||
4 | +# FIXME: Handle case in which no stash entry is created due to no changes. | ||
5 | + | ||
6 | +printf "\e[30m=== PRE-COMMIT STARTING ===\e[m\n" | ||
7 | +git stash save --quiet --keep-index --include-untracked | ||
8 | + | ||
9 | +if go build -v ./... && go test -v -cover ./... && go vet ./... && golint . && travis-lint; then | ||
10 | + result=$? | ||
11 | + printf "\e[32m=== PRE-COMMIT SUCCEEDED ===\e[m\n" | ||
12 | +else | ||
13 | + result=$? | ||
14 | + printf "\e[31m=== PRE-COMMIT FAILED ===\e[m\n" | ||
15 | +fi | ||
16 | + | ||
17 | +git stash pop --quiet | ||
18 | +exit $result |
vendor/github.com/andybalholm/brotli/LICENSE
0 → 100644
1 | +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. | ||
2 | + | ||
3 | +Permission is hereby granted, free of charge, to any person obtaining a copy | ||
4 | +of this software and associated documentation files (the "Software"), to deal | ||
5 | +in the Software without restriction, including without limitation the rights | ||
6 | +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
7 | +copies of the Software, and to permit persons to whom the Software is | ||
8 | +furnished to do so, subject to the following conditions: | ||
9 | + | ||
10 | +The above copyright notice and this permission notice shall be included in | ||
11 | +all copies or substantial portions of the Software. | ||
12 | + | ||
13 | +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
14 | +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
15 | +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
16 | +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
17 | +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
18 | +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | ||
19 | +THE SOFTWARE. |
1 | +This package is a brotli compressor and decompressor implemented in Go. | ||
2 | +It was translated from the reference implementation (https://github.com/google/brotli) | ||
3 | +with the `c2go` tool at https://github.com/andybalholm/c2go. | ||
4 | + | ||
5 | +I am using it in production with https://github.com/andybalholm/redwood. |
1 | +package brotli | ||
2 | + | ||
3 | +/* Copyright 2013 Google Inc. All Rights Reserved. | ||
4 | + | ||
5 | + Distributed under MIT license. | ||
6 | + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | ||
7 | +*/ | ||
8 | + | ||
9 | +/* Function to find backward reference copies. */ | ||
10 | + | ||
11 | +func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint { | ||
12 | + if distance <= max_distance { | ||
13 | + var distance_plus_3 uint = distance + 3 | ||
14 | + var offset0 uint = distance_plus_3 - uint(dist_cache[0]) | ||
15 | + var offset1 uint = distance_plus_3 - uint(dist_cache[1]) | ||
16 | + if distance == uint(dist_cache[0]) { | ||
17 | + return 0 | ||
18 | + } else if distance == uint(dist_cache[1]) { | ||
19 | + return 1 | ||
20 | + } else if offset0 < 7 { | ||
21 | + return (0x9750468 >> (4 * offset0)) & 0xF | ||
22 | + } else if offset1 < 7 { | ||
23 | + return (0xFDB1ACE >> (4 * offset1)) & 0xF | ||
24 | + } else if distance == uint(dist_cache[2]) { | ||
25 | + return 2 | ||
26 | + } else if distance == uint(dist_cache[3]) { | ||
27 | + return 3 | ||
28 | + } | ||
29 | + } | ||
30 | + | ||
31 | + return distance + numDistanceShortCodes - 1 | ||
32 | +} | ||
33 | + | ||
34 | +/* "commands" points to the next output command to write to, "*num_commands" is | ||
35 | + initially the total amount of commands output by previous | ||
36 | + CreateBackwardReferences calls, and must be incremented by the amount written | ||
37 | + by this call. */ | ||
38 | +func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { | ||
39 | + var max_backward_limit uint = maxBackwardLimit(params.lgwin) | ||
40 | + var orig_commands []command = commands | ||
41 | + var insert_length uint = *last_insert_len | ||
42 | + var pos_end uint = position + num_bytes | ||
43 | + var store_end uint | ||
44 | + if num_bytes >= hasher.StoreLookahead() { | ||
45 | + store_end = position + num_bytes - hasher.StoreLookahead() + 1 | ||
46 | + } else { | ||
47 | + store_end = position | ||
48 | + } | ||
49 | + var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params) | ||
50 | + var apply_random_heuristics uint = position + random_heuristics_window_size | ||
51 | + var gap uint = 0 | ||
52 | + /* Set maximum distance, see section 9.1. of the spec. */ | ||
53 | + | ||
54 | + const kMinScore uint = scoreBase + 100 | ||
55 | + | ||
56 | + /* For speed up heuristics for random data. */ | ||
57 | + | ||
58 | + /* Minimum score to accept a backward reference. */ | ||
59 | + hasher.PrepareDistanceCache(dist_cache) | ||
60 | + var sr2 hasherSearchResult | ||
61 | + var sr hasherSearchResult | ||
62 | + | ||
63 | + for position+hasher.HashTypeLength() < pos_end { | ||
64 | + var max_length uint = pos_end - position | ||
65 | + var max_distance uint = brotli_min_size_t(position, max_backward_limit) | ||
66 | + sr.len = 0 | ||
67 | + sr.len_code_delta = 0 | ||
68 | + sr.distance = 0 | ||
69 | + sr.score = kMinScore | ||
70 | + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, &sr) | ||
71 | + if sr.score > kMinScore { | ||
72 | + /* Found a match. Let's look for something even better ahead. */ | ||
73 | + var delayed_backward_references_in_row int = 0 | ||
74 | + max_length-- | ||
75 | + for ; ; max_length-- { | ||
76 | + var cost_diff_lazy uint = 175 | ||
77 | + if params.quality < minQualityForExtensiveReferenceSearch { | ||
78 | + sr2.len = brotli_min_size_t(sr.len-1, max_length) | ||
79 | + } else { | ||
80 | + sr2.len = 0 | ||
81 | + } | ||
82 | + sr2.len_code_delta = 0 | ||
83 | + sr2.distance = 0 | ||
84 | + sr2.score = kMinScore | ||
85 | + max_distance = brotli_min_size_t(position+1, max_backward_limit) | ||
86 | + hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, &sr2) | ||
87 | + if sr2.score >= sr.score+cost_diff_lazy { | ||
88 | + /* Ok, let's just write one byte for now and start a match from the | ||
89 | + next byte. */ | ||
90 | + position++ | ||
91 | + | ||
92 | + insert_length++ | ||
93 | + sr = sr2 | ||
94 | + delayed_backward_references_in_row++ | ||
95 | + if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end { | ||
96 | + continue | ||
97 | + } | ||
98 | + } | ||
99 | + | ||
100 | + break | ||
101 | + } | ||
102 | + | ||
103 | + apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size | ||
104 | + max_distance = brotli_min_size_t(position, max_backward_limit) | ||
105 | + { | ||
106 | + /* The first 16 codes are special short-codes, | ||
107 | + and the minimum offset is 1. */ | ||
108 | + var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache) | ||
109 | + if (sr.distance <= (max_distance + gap)) && distance_code > 0 { | ||
110 | + dist_cache[3] = dist_cache[2] | ||
111 | + dist_cache[2] = dist_cache[1] | ||
112 | + dist_cache[1] = dist_cache[0] | ||
113 | + dist_cache[0] = int(sr.distance) | ||
114 | + hasher.PrepareDistanceCache(dist_cache) | ||
115 | + } | ||
116 | + | ||
117 | + initCommand(&commands[0], ¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code) | ||
118 | + commands = commands[1:] | ||
119 | + } | ||
120 | + | ||
121 | + *num_literals += insert_length | ||
122 | + insert_length = 0 | ||
123 | + /* Put the hash keys into the table, if there are enough bytes left. | ||
124 | + Depending on the hasher implementation, it can push all positions | ||
125 | + in the given range or only a subset of them. | ||
126 | + Avoid hash poisoning with RLE data. */ | ||
127 | + { | ||
128 | + var range_start uint = position + 2 | ||
129 | + var range_end uint = brotli_min_size_t(position+sr.len, store_end) | ||
130 | + if sr.distance < sr.len>>2 { | ||
131 | + range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2))) | ||
132 | + } | ||
133 | + | ||
134 | + hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end) | ||
135 | + } | ||
136 | + | ||
137 | + position += sr.len | ||
138 | + } else { | ||
139 | + insert_length++ | ||
140 | + position++ | ||
141 | + | ||
142 | + /* If we have not seen matches for a long time, we can skip some | ||
143 | + match lookups. Unsuccessful match lookups are very very expensive | ||
144 | + and this kind of a heuristic speeds up compression quite | ||
145 | + a lot. */ | ||
146 | + if position > apply_random_heuristics { | ||
147 | + /* Going through uncompressible data, jump. */ | ||
148 | + if position > apply_random_heuristics+4*random_heuristics_window_size { | ||
149 | + var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4) | ||
150 | + /* It is quite a long time since we saw a copy, so we assume | ||
151 | + that this data is not compressible, and store hashes less | ||
152 | + often. Hashes of non compressible data are less likely to | ||
153 | + turn out to be useful in the future, too, so we store less of | ||
154 | + them to not to flood out the hash table of good compressible | ||
155 | + data. */ | ||
156 | + | ||
157 | + var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin) | ||
158 | + for ; position < pos_jump; position += 4 { | ||
159 | + hasher.Store(ringbuffer, ringbuffer_mask, position) | ||
160 | + insert_length += 4 | ||
161 | + } | ||
162 | + } else { | ||
163 | + var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2) | ||
164 | + var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin) | ||
165 | + for ; position < pos_jump; position += 2 { | ||
166 | + hasher.Store(ringbuffer, ringbuffer_mask, position) | ||
167 | + insert_length += 2 | ||
168 | + } | ||
169 | + } | ||
170 | + } | ||
171 | + } | ||
172 | + } | ||
173 | + | ||
174 | + insert_length += pos_end - position | ||
175 | + *last_insert_len = insert_length | ||
176 | + *num_commands += uint(-cap(commands) + cap(orig_commands)) | ||
177 | +} |
1 | +package brotli | ||
2 | + | ||
3 | +import "math" | ||
4 | + | ||
5 | +type zopfliNode struct { | ||
6 | + length uint32 | ||
7 | + distance uint32 | ||
8 | + dcode_insert_length uint32 | ||
9 | + u struct { | ||
10 | + cost float32 | ||
11 | + next uint32 | ||
12 | + shortcut uint32 | ||
13 | + } | ||
14 | +} | ||
15 | + | ||
16 | +const maxEffectiveDistanceAlphabetSize = 544 | ||
17 | + | ||
18 | +const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */ | ||
19 | + | ||
20 | +var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1} | ||
21 | + | ||
22 | +var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3} | ||
23 | + | ||
24 | +func initZopfliNodes(array []zopfliNode, length uint) { | ||
25 | + var stub zopfliNode | ||
26 | + var i uint | ||
27 | + stub.length = 1 | ||
28 | + stub.distance = 0 | ||
29 | + stub.dcode_insert_length = 0 | ||
30 | + stub.u.cost = kInfinity | ||
31 | + for i = 0; i < length; i++ { | ||
32 | + array[i] = stub | ||
33 | + } | ||
34 | +} | ||
35 | + | ||
36 | +func zopfliNodeCopyLength(self *zopfliNode) uint32 { | ||
37 | + return self.length & 0x1FFFFFF | ||
38 | +} | ||
39 | + | ||
40 | +func zopfliNodeLengthCode(self *zopfliNode) uint32 { | ||
41 | + var modifier uint32 = self.length >> 25 | ||
42 | + return zopfliNodeCopyLength(self) + 9 - modifier | ||
43 | +} | ||
44 | + | ||
45 | +func zopfliNodeCopyDistance(self *zopfliNode) uint32 { | ||
46 | + return self.distance | ||
47 | +} | ||
48 | + | ||
49 | +func zopfliNodeDistanceCode(self *zopfliNode) uint32 { | ||
50 | + var short_code uint32 = self.dcode_insert_length >> 27 | ||
51 | + if short_code == 0 { | ||
52 | + return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1 | ||
53 | + } else { | ||
54 | + return short_code - 1 | ||
55 | + } | ||
56 | +} | ||
57 | + | ||
58 | +func zopfliNodeCommandLength(self *zopfliNode) uint32 { | ||
59 | + return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF) | ||
60 | +} | ||
61 | + | ||
62 | +/* Histogram based cost model for zopflification. */ | ||
63 | +type zopfliCostModel struct { | ||
64 | + cost_cmd_ [numCommandSymbols]float32 | ||
65 | + cost_dist_ []float32 | ||
66 | + distance_histogram_size uint32 | ||
67 | + literal_costs_ []float32 | ||
68 | + min_cost_cmd_ float32 | ||
69 | + num_bytes_ uint | ||
70 | +} | ||
71 | + | ||
72 | +func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) { | ||
73 | + var distance_histogram_size uint32 = dist.alphabet_size | ||
74 | + if distance_histogram_size > maxEffectiveDistanceAlphabetSize { | ||
75 | + distance_histogram_size = maxEffectiveDistanceAlphabetSize | ||
76 | + } | ||
77 | + | ||
78 | + self.num_bytes_ = num_bytes | ||
79 | + self.literal_costs_ = make([]float32, (num_bytes + 2)) | ||
80 | + self.cost_dist_ = make([]float32, (dist.alphabet_size)) | ||
81 | + self.distance_histogram_size = distance_histogram_size | ||
82 | +} | ||
83 | + | ||
84 | +func cleanupZopfliCostModel(self *zopfliCostModel) { | ||
85 | + self.literal_costs_ = nil | ||
86 | + self.cost_dist_ = nil | ||
87 | +} | ||
88 | + | ||
89 | +func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) { | ||
90 | + var sum uint = 0 | ||
91 | + var missing_symbol_sum uint | ||
92 | + var log2sum float32 | ||
93 | + var missing_symbol_cost float32 | ||
94 | + var i uint | ||
95 | + for i = 0; i < histogram_size; i++ { | ||
96 | + sum += uint(histogram[i]) | ||
97 | + } | ||
98 | + | ||
99 | + log2sum = float32(fastLog2(sum)) | ||
100 | + missing_symbol_sum = sum | ||
101 | + if !literal_histogram { | ||
102 | + for i = 0; i < histogram_size; i++ { | ||
103 | + if histogram[i] == 0 { | ||
104 | + missing_symbol_sum++ | ||
105 | + } | ||
106 | + } | ||
107 | + } | ||
108 | + | ||
109 | + missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2 | ||
110 | + for i = 0; i < histogram_size; i++ { | ||
111 | + if histogram[i] == 0 { | ||
112 | + cost[i] = missing_symbol_cost | ||
113 | + continue | ||
114 | + } | ||
115 | + | ||
116 | + /* Shannon bits for this symbol. */ | ||
117 | + cost[i] = log2sum - float32(fastLog2(uint(histogram[i]))) | ||
118 | + | ||
119 | + /* Cannot be coded with less than 1 bit */ | ||
120 | + if cost[i] < 1 { | ||
121 | + cost[i] = 1 | ||
122 | + } | ||
123 | + } | ||
124 | +} | ||
125 | + | ||
126 | +func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) { | ||
127 | + var histogram_literal [numLiteralSymbols]uint32 | ||
128 | + var histogram_cmd [numCommandSymbols]uint32 | ||
129 | + var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32 | ||
130 | + var cost_literal [numLiteralSymbols]float32 | ||
131 | + var pos uint = position - last_insert_len | ||
132 | + var min_cost_cmd float32 = kInfinity | ||
133 | + var i uint | ||
134 | + var cost_cmd []float32 = self.cost_cmd_[:] | ||
135 | + var literal_costs []float32 | ||
136 | + | ||
137 | + histogram_literal = [numLiteralSymbols]uint32{} | ||
138 | + histogram_cmd = [numCommandSymbols]uint32{} | ||
139 | + histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{} | ||
140 | + | ||
141 | + for i = 0; i < num_commands; i++ { | ||
142 | + var inslength uint = uint(commands[i].insert_len_) | ||
143 | + var copylength uint = uint(commandCopyLen(&commands[i])) | ||
144 | + var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF | ||
145 | + var cmdcode uint = uint(commands[i].cmd_prefix_) | ||
146 | + var j uint | ||
147 | + | ||
148 | + histogram_cmd[cmdcode]++ | ||
149 | + if cmdcode >= 128 { | ||
150 | + histogram_dist[distcode]++ | ||
151 | + } | ||
152 | + | ||
153 | + for j = 0; j < inslength; j++ { | ||
154 | + histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++ | ||
155 | + } | ||
156 | + | ||
157 | + pos += inslength + copylength | ||
158 | + } | ||
159 | + | ||
160 | + setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:]) | ||
161 | + setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd) | ||
162 | + setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_) | ||
163 | + | ||
164 | + for i = 0; i < numCommandSymbols; i++ { | ||
165 | + min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i]) | ||
166 | + } | ||
167 | + | ||
168 | + self.min_cost_cmd_ = min_cost_cmd | ||
169 | + { | ||
170 | + literal_costs = self.literal_costs_ | ||
171 | + var literal_carry float32 = 0.0 | ||
172 | + var num_bytes uint = self.num_bytes_ | ||
173 | + literal_costs[0] = 0.0 | ||
174 | + for i = 0; i < num_bytes; i++ { | ||
175 | + literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]] | ||
176 | + literal_costs[i+1] = literal_costs[i] + literal_carry | ||
177 | + literal_carry -= literal_costs[i+1] - literal_costs[i] | ||
178 | + } | ||
179 | + } | ||
180 | +} | ||
181 | + | ||
182 | +func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) { | ||
183 | + var literal_costs []float32 = self.literal_costs_ | ||
184 | + var literal_carry float32 = 0.0 | ||
185 | + var cost_dist []float32 = self.cost_dist_ | ||
186 | + var cost_cmd []float32 = self.cost_cmd_[:] | ||
187 | + var num_bytes uint = self.num_bytes_ | ||
188 | + var i uint | ||
189 | + estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:]) | ||
190 | + literal_costs[0] = 0.0 | ||
191 | + for i = 0; i < num_bytes; i++ { | ||
192 | + literal_carry += literal_costs[i+1] | ||
193 | + literal_costs[i+1] = literal_costs[i] + literal_carry | ||
194 | + literal_carry -= literal_costs[i+1] - literal_costs[i] | ||
195 | + } | ||
196 | + | ||
197 | + for i = 0; i < numCommandSymbols; i++ { | ||
198 | + cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i)))) | ||
199 | + } | ||
200 | + | ||
201 | + for i = 0; uint32(i) < self.distance_histogram_size; i++ { | ||
202 | + cost_dist[i] = float32(fastLog2(uint(20 + uint32(i)))) | ||
203 | + } | ||
204 | + | ||
205 | + self.min_cost_cmd_ = float32(fastLog2(11)) | ||
206 | +} | ||
207 | + | ||
208 | +func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 { | ||
209 | + return self.cost_cmd_[cmdcode] | ||
210 | +} | ||
211 | + | ||
212 | +func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 { | ||
213 | + return self.cost_dist_[distcode] | ||
214 | +} | ||
215 | + | ||
216 | +func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 { | ||
217 | + return self.literal_costs_[to] - self.literal_costs_[from] | ||
218 | +} | ||
219 | + | ||
220 | +func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 { | ||
221 | + return self.min_cost_cmd_ | ||
222 | +} | ||
223 | + | ||
224 | +/* REQUIRES: len >= 2, start_pos <= pos */ | ||
225 | +/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */ | ||
226 | +/* Maintains the "ZopfliNode array invariant". */ | ||
227 | +func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) { | ||
228 | + var next *zopfliNode = &nodes[pos+len] | ||
229 | + next.length = uint32(len | (len+9-len_code)<<25) | ||
230 | + next.distance = uint32(dist) | ||
231 | + next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos)) | ||
232 | + next.u.cost = cost | ||
233 | +} | ||
234 | + | ||
235 | +type posData struct { | ||
236 | + pos uint | ||
237 | + distance_cache [4]int | ||
238 | + costdiff float32 | ||
239 | + cost float32 | ||
240 | +} | ||
241 | + | ||
242 | +/* Maintains the smallest 8 cost difference together with their positions */ | ||
243 | +type startPosQueue struct { | ||
244 | + q_ [8]posData | ||
245 | + idx_ uint | ||
246 | +} | ||
247 | + | ||
248 | +func initStartPosQueue(self *startPosQueue) { | ||
249 | + self.idx_ = 0 | ||
250 | +} | ||
251 | + | ||
252 | +func startPosQueueSize(self *startPosQueue) uint { | ||
253 | + return brotli_min_size_t(self.idx_, 8) | ||
254 | +} | ||
255 | + | ||
256 | +func startPosQueuePush(self *startPosQueue, posdata *posData) { | ||
257 | + var offset uint = ^(self.idx_) & 7 | ||
258 | + self.idx_++ | ||
259 | + var len uint = startPosQueueSize(self) | ||
260 | + var i uint | ||
261 | + var q []posData = self.q_[:] | ||
262 | + q[offset] = *posdata | ||
263 | + | ||
264 | + /* Restore the sorted order. In the list of |len| items at most |len - 1| | ||
265 | + adjacent element comparisons / swaps are required. */ | ||
266 | + for i = 1; i < len; i++ { | ||
267 | + if q[offset&7].costdiff > q[(offset+1)&7].costdiff { | ||
268 | + var tmp posData = q[offset&7] | ||
269 | + q[offset&7] = q[(offset+1)&7] | ||
270 | + q[(offset+1)&7] = tmp | ||
271 | + } | ||
272 | + | ||
273 | + offset++ | ||
274 | + } | ||
275 | +} | ||
276 | + | ||
277 | +func startPosQueueAt(self *startPosQueue, k uint) *posData { | ||
278 | + return &self.q_[(k-self.idx_)&7] | ||
279 | +} | ||
280 | + | ||
281 | +/* Returns the minimum possible copy length that can improve the cost of any */ | ||
282 | +/* future position. */ | ||
283 | +func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint { | ||
284 | + var min_cost float32 = start_cost | ||
285 | + var len uint = 2 | ||
286 | + var next_len_bucket uint = 4 | ||
287 | + /* Compute the minimum possible cost of reaching any future position. */ | ||
288 | + | ||
289 | + var next_len_offset uint = 10 | ||
290 | + for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost { | ||
291 | + /* We already reached (pos + len) with no more cost than the minimum | ||
292 | + possible cost of reaching anything from this pos, so there is no point in | ||
293 | + looking for lengths <= len. */ | ||
294 | + len++ | ||
295 | + | ||
296 | + if len == next_len_offset { | ||
297 | + /* We reached the next copy length code bucket, so we add one more | ||
298 | + extra bit to the minimum cost. */ | ||
299 | + min_cost += 1.0 | ||
300 | + | ||
301 | + next_len_offset += next_len_bucket | ||
302 | + next_len_bucket *= 2 | ||
303 | + } | ||
304 | + } | ||
305 | + | ||
306 | + return uint(len) | ||
307 | +} | ||
308 | + | ||
309 | +/* REQUIRES: nodes[pos].cost < kInfinity | ||
310 | + REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ | ||
311 | +func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 { | ||
312 | + var clen uint = uint(zopfliNodeCopyLength(&nodes[pos])) | ||
313 | + var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF) | ||
314 | + var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos])) | ||
315 | + | ||
316 | + /* Since |block_start + pos| is the end position of the command, the copy part | ||
317 | + starts from |block_start + pos - clen|. Distances that are greater than | ||
318 | + this or greater than |max_backward_limit| + |gap| are static dictionary | ||
319 | + references, and do not update the last distances. | ||
320 | + Also distance code 0 (last distance) does not update the last distances. */ | ||
321 | + if pos == 0 { | ||
322 | + return 0 | ||
323 | + } else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 { | ||
324 | + return uint32(pos) | ||
325 | + } else { | ||
326 | + return nodes[pos-clen-ilen].u.shortcut | ||
327 | + } | ||
328 | +} | ||
329 | + | ||
330 | +/* Fills in dist_cache[0..3] with the last four distances (as defined by | ||
331 | + Section 4. of the Spec) that would be used at (block_start + pos) if we | ||
332 | + used the shortest path of commands from block_start, computed from | ||
333 | + nodes[0..pos]. The last four distances at block_start are in | ||
334 | + starting_dist_cache[0..3]. | ||
335 | + REQUIRES: nodes[pos].cost < kInfinity | ||
336 | + REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ | ||
337 | +func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) { | ||
338 | + var idx int = 0 | ||
339 | + var p uint = uint(nodes[pos].u.shortcut) | ||
340 | + for idx < 4 && p > 0 { | ||
341 | + var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF) | ||
342 | + var clen uint = uint(zopfliNodeCopyLength(&nodes[p])) | ||
343 | + var dist uint = uint(zopfliNodeCopyDistance(&nodes[p])) | ||
344 | + dist_cache[idx] = int(dist) | ||
345 | + idx++ | ||
346 | + | ||
347 | + /* Because of prerequisite, p >= clen + ilen >= 2. */ | ||
348 | + p = uint(nodes[p-clen-ilen].u.shortcut) | ||
349 | + } | ||
350 | + | ||
351 | + for ; idx < 4; idx++ { | ||
352 | + dist_cache[idx] = starting_dist_cache[0] | ||
353 | + starting_dist_cache = starting_dist_cache[1:] | ||
354 | + } | ||
355 | +} | ||
356 | + | ||
357 | +/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it | ||
358 | + is eligible. */ | ||
359 | +func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) { | ||
360 | + /* Save cost, because ComputeDistanceCache invalidates it. */ | ||
361 | + var node_cost float32 = nodes[pos].u.cost | ||
362 | + nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes) | ||
363 | + if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) { | ||
364 | + var posdata posData | ||
365 | + posdata.pos = pos | ||
366 | + posdata.cost = node_cost | ||
367 | + posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos) | ||
368 | + computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:]) | ||
369 | + startPosQueuePush(queue, &posdata) | ||
370 | + } | ||
371 | +} | ||
372 | + | ||
373 | +/* Returns longest copy length. */ | ||
374 | +func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint { | ||
375 | + var cur_ix uint = block_start + pos | ||
376 | + var cur_ix_masked uint = cur_ix & ringbuffer_mask | ||
377 | + var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit) | ||
378 | + var max_len uint = num_bytes - pos | ||
379 | + var max_zopfli_len uint = maxZopfliLen(params) | ||
380 | + var max_iters uint = maxZopfliCandidates(params) | ||
381 | + var min_len uint | ||
382 | + var result uint = 0 | ||
383 | + var k uint | ||
384 | + var gap uint = 0 | ||
385 | + | ||
386 | + evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes) | ||
387 | + { | ||
388 | + var posdata *posData = startPosQueueAt(queue, 0) | ||
389 | + var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos)) | ||
390 | + min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos) | ||
391 | + } | ||
392 | + | ||
393 | + /* Go over the command starting positions in order of increasing cost | ||
394 | + difference. */ | ||
395 | + for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ { | ||
396 | + var posdata *posData = startPosQueueAt(queue, k) | ||
397 | + var start uint = posdata.pos | ||
398 | + var inscode uint16 = getInsertLengthCode(pos - start) | ||
399 | + var start_costdiff float32 = posdata.costdiff | ||
400 | + var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos) | ||
401 | + var best_len uint = min_len - 1 | ||
402 | + var j uint = 0 | ||
403 | + /* Look for last distance matches using the distance cache from this | ||
404 | + starting position. */ | ||
405 | + for ; j < numDistanceShortCodes && best_len < max_len; j++ { | ||
406 | + var idx uint = uint(kDistanceCacheIndex[j]) | ||
407 | + var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j]) | ||
408 | + var prev_ix uint = cur_ix - backward | ||
409 | + var len uint = 0 | ||
410 | + var continuation byte = ringbuffer[cur_ix_masked+best_len] | ||
411 | + if cur_ix_masked+best_len > ringbuffer_mask { | ||
412 | + break | ||
413 | + } | ||
414 | + | ||
415 | + if backward > max_distance+gap { | ||
416 | + /* Word dictionary -> ignore. */ | ||
417 | + continue | ||
418 | + } | ||
419 | + | ||
420 | + if backward <= max_distance { | ||
421 | + /* Regular backward reference. */ | ||
422 | + if prev_ix >= cur_ix { | ||
423 | + continue | ||
424 | + } | ||
425 | + | ||
426 | + prev_ix &= ringbuffer_mask | ||
427 | + if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] { | ||
428 | + continue | ||
429 | + } | ||
430 | + | ||
431 | + len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len) | ||
432 | + } else { | ||
433 | + continue | ||
434 | + } | ||
435 | + { | ||
436 | + var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j) | ||
437 | + var l uint | ||
438 | + for l = best_len + 1; l <= len; l++ { | ||
439 | + var copycode uint16 = getCopyLengthCode(l) | ||
440 | + var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0) | ||
441 | + var tmp float32 | ||
442 | + if cmdcode < 128 { | ||
443 | + tmp = base_cost | ||
444 | + } else { | ||
445 | + tmp = dist_cost | ||
446 | + } | ||
447 | + var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) | ||
448 | + if cost < nodes[pos+l].u.cost { | ||
449 | + updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost) | ||
450 | + result = brotli_max_size_t(result, l) | ||
451 | + } | ||
452 | + | ||
453 | + best_len = l | ||
454 | + } | ||
455 | + } | ||
456 | + } | ||
457 | + | ||
458 | + /* At higher iterations look only for new last distance matches, since | ||
459 | + looking only for new command start positions with the same distances | ||
460 | + does not help much. */ | ||
461 | + if k >= 2 { | ||
462 | + continue | ||
463 | + } | ||
464 | + { | ||
465 | + /* Loop through all possible copy lengths at this position. */ | ||
466 | + var len uint = min_len | ||
467 | + for j = 0; j < num_matches; j++ { | ||
468 | + var match backwardMatch = matches[j] | ||
469 | + var dist uint = uint(match.distance) | ||
470 | + var is_dictionary_match bool = (dist > max_distance+gap) | ||
471 | + var dist_code uint = dist + numDistanceShortCodes - 1 | ||
472 | + var dist_symbol uint16 | ||
473 | + var distextra uint32 | ||
474 | + var distnumextra uint32 | ||
475 | + var dist_cost float32 | ||
476 | + var max_match_len uint | ||
477 | + /* We already tried all possible last distance matches, so we can use | ||
478 | + normal distance code here. */ | ||
479 | + prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra) | ||
480 | + | ||
481 | + distnumextra = uint32(dist_symbol) >> 10 | ||
482 | + dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF) | ||
483 | + | ||
484 | + /* Try all copy lengths up until the maximum copy length corresponding | ||
485 | + to this distance. If the distance refers to the static dictionary, or | ||
486 | + the maximum length is long enough, try only one maximum length. */ | ||
487 | + max_match_len = backwardMatchLength(&match) | ||
488 | + | ||
489 | + if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) { | ||
490 | + len = max_match_len | ||
491 | + } | ||
492 | + | ||
493 | + for ; len <= max_match_len; len++ { | ||
494 | + var len_code uint | ||
495 | + if is_dictionary_match { | ||
496 | + len_code = backwardMatchLengthCode(&match) | ||
497 | + } else { | ||
498 | + len_code = len | ||
499 | + } | ||
500 | + var copycode uint16 = getCopyLengthCode(len_code) | ||
501 | + var cmdcode uint16 = combineLengthCodes(inscode, copycode, false) | ||
502 | + var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) | ||
503 | + if cost < nodes[pos+len].u.cost { | ||
504 | + updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost) | ||
505 | + result = brotli_max_size_t(result, uint(len)) | ||
506 | + } | ||
507 | + } | ||
508 | + } | ||
509 | + } | ||
510 | + } | ||
511 | + | ||
512 | + return result | ||
513 | +} | ||
514 | + | ||
515 | +func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint { | ||
516 | + var index uint = num_bytes | ||
517 | + var num_commands uint = 0 | ||
518 | + for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 { | ||
519 | + index-- | ||
520 | + } | ||
521 | + nodes[index].u.next = math.MaxUint32 | ||
522 | + for index != 0 { | ||
523 | + var len uint = uint(zopfliNodeCommandLength(&nodes[index])) | ||
524 | + index -= uint(len) | ||
525 | + nodes[index].u.next = uint32(len) | ||
526 | + num_commands++ | ||
527 | + } | ||
528 | + | ||
529 | + return num_commands | ||
530 | +} | ||
531 | + | ||
532 | +/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ | ||
533 | +func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) { | ||
534 | + var max_backward_limit uint = maxBackwardLimit(params.lgwin) | ||
535 | + var pos uint = 0 | ||
536 | + var offset uint32 = nodes[0].u.next | ||
537 | + var i uint | ||
538 | + var gap uint = 0 | ||
539 | + for i = 0; offset != math.MaxUint32; i++ { | ||
540 | + var next *zopfliNode = &nodes[uint32(pos)+offset] | ||
541 | + var copy_length uint = uint(zopfliNodeCopyLength(next)) | ||
542 | + var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF) | ||
543 | + pos += insert_length | ||
544 | + offset = next.u.next | ||
545 | + if i == 0 { | ||
546 | + insert_length += *last_insert_len | ||
547 | + *last_insert_len = 0 | ||
548 | + } | ||
549 | + { | ||
550 | + var distance uint = uint(zopfliNodeCopyDistance(next)) | ||
551 | + var len_code uint = uint(zopfliNodeLengthCode(next)) | ||
552 | + var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit) | ||
553 | + var is_dictionary bool = (distance > max_distance+gap) | ||
554 | + var dist_code uint = uint(zopfliNodeDistanceCode(next)) | ||
555 | + initCommand(&commands[i], ¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code) | ||
556 | + | ||
557 | + if !is_dictionary && dist_code > 0 { | ||
558 | + dist_cache[3] = dist_cache[2] | ||
559 | + dist_cache[2] = dist_cache[1] | ||
560 | + dist_cache[1] = dist_cache[0] | ||
561 | + dist_cache[0] = int(distance) | ||
562 | + } | ||
563 | + } | ||
564 | + | ||
565 | + *num_literals += insert_length | ||
566 | + pos += copy_length | ||
567 | + } | ||
568 | + | ||
569 | + *last_insert_len += num_bytes - pos | ||
570 | +} | ||
571 | + | ||
572 | +func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint { | ||
573 | + var max_backward_limit uint = maxBackwardLimit(params.lgwin) | ||
574 | + var max_zopfli_len uint = maxZopfliLen(params) | ||
575 | + var queue startPosQueue | ||
576 | + var cur_match_pos uint = 0 | ||
577 | + var i uint | ||
578 | + nodes[0].length = 0 | ||
579 | + nodes[0].u.cost = 0 | ||
580 | + initStartPosQueue(&queue) | ||
581 | + for i = 0; i+3 < num_bytes; i++ { | ||
582 | + var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes) | ||
583 | + if skip < longCopyQuickStep { | ||
584 | + skip = 0 | ||
585 | + } | ||
586 | + cur_match_pos += uint(num_matches[i]) | ||
587 | + if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len { | ||
588 | + skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip) | ||
589 | + } | ||
590 | + | ||
591 | + if skip > 1 { | ||
592 | + skip-- | ||
593 | + for skip != 0 { | ||
594 | + i++ | ||
595 | + if i+3 >= num_bytes { | ||
596 | + break | ||
597 | + } | ||
598 | + evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes) | ||
599 | + cur_match_pos += uint(num_matches[i]) | ||
600 | + skip-- | ||
601 | + } | ||
602 | + } | ||
603 | + } | ||
604 | + | ||
605 | + return computeShortestPathFromNodes(num_bytes, nodes) | ||
606 | +} | ||
607 | + | ||
608 | +/* Computes the shortest path of commands from position to at most | ||
609 | + position + num_bytes. | ||
610 | + | ||
611 | + On return, path->size() is the number of commands found and path[i] is the | ||
612 | + length of the i-th command (copy length plus insert length). | ||
613 | + Note that the sum of the lengths of all commands can be less than num_bytes. | ||
614 | + | ||
615 | + On return, the nodes[0..num_bytes] array will have the following | ||
616 | + "ZopfliNode array invariant": | ||
617 | + For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then | ||
618 | + (1) nodes[i].copy_length() >= 2 | ||
619 | + (2) nodes[i].command_length() <= i and | ||
620 | + (3) nodes[i - nodes[i].command_length()].cost < kInfinity | ||
621 | + | ||
622 | + REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */ | ||
623 | +func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint { | ||
624 | + var max_backward_limit uint = maxBackwardLimit(params.lgwin) | ||
625 | + var max_zopfli_len uint = maxZopfliLen(params) | ||
626 | + var model zopfliCostModel | ||
627 | + var queue startPosQueue | ||
628 | + var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch | ||
629 | + var store_end uint | ||
630 | + if num_bytes >= hasher.StoreLookahead() { | ||
631 | + store_end = position + num_bytes - hasher.StoreLookahead() + 1 | ||
632 | + } else { | ||
633 | + store_end = position | ||
634 | + } | ||
635 | + var i uint | ||
636 | + var gap uint = 0 | ||
637 | + var lz_matches_offset uint = 0 | ||
638 | + nodes[0].length = 0 | ||
639 | + nodes[0].u.cost = 0 | ||
640 | + initZopfliCostModel(&model, ¶ms.dist, num_bytes) | ||
641 | + zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) | ||
642 | + initStartPosQueue(&queue) | ||
643 | + for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ { | ||
644 | + var pos uint = position + i | ||
645 | + var max_distance uint = brotli_min_size_t(pos, max_backward_limit) | ||
646 | + var skip uint | ||
647 | + var num_matches uint | ||
648 | + num_matches = findAllMatchesH10(hasher, ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:]) | ||
649 | + if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len { | ||
650 | + matches[0] = matches[num_matches-1] | ||
651 | + num_matches = 1 | ||
652 | + } | ||
653 | + | ||
654 | + skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes) | ||
655 | + if skip < longCopyQuickStep { | ||
656 | + skip = 0 | ||
657 | + } | ||
658 | + if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len { | ||
659 | + skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip) | ||
660 | + } | ||
661 | + | ||
662 | + if skip > 1 { | ||
663 | + /* Add the tail of the copy to the hasher. */ | ||
664 | + hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end)) | ||
665 | + | ||
666 | + skip-- | ||
667 | + for skip != 0 { | ||
668 | + i++ | ||
669 | + if i+hasher.HashTypeLength()-1 >= num_bytes { | ||
670 | + break | ||
671 | + } | ||
672 | + evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes) | ||
673 | + skip-- | ||
674 | + } | ||
675 | + } | ||
676 | + } | ||
677 | + | ||
678 | + cleanupZopfliCostModel(&model) | ||
679 | + return computeShortestPathFromNodes(num_bytes, nodes) | ||
680 | +} | ||
681 | + | ||
682 | +func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { | ||
683 | + var nodes []zopfliNode | ||
684 | + nodes = make([]zopfliNode, (num_bytes + 1)) | ||
685 | + initZopfliNodes(nodes, num_bytes+1) | ||
686 | + *num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) | ||
687 | + zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) | ||
688 | + nodes = nil | ||
689 | +} | ||
690 | + | ||
691 | +func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { | ||
692 | + var max_backward_limit uint = maxBackwardLimit(params.lgwin) | ||
693 | + var num_matches []uint32 = make([]uint32, num_bytes) | ||
694 | + var matches_size uint = 4 * num_bytes | ||
695 | + var store_end uint | ||
696 | + if num_bytes >= hasher.StoreLookahead() { | ||
697 | + store_end = position + num_bytes - hasher.StoreLookahead() + 1 | ||
698 | + } else { | ||
699 | + store_end = position | ||
700 | + } | ||
701 | + var cur_match_pos uint = 0 | ||
702 | + var i uint | ||
703 | + var orig_num_literals uint | ||
704 | + var orig_last_insert_len uint | ||
705 | + var orig_dist_cache [4]int | ||
706 | + var orig_num_commands uint | ||
707 | + var model zopfliCostModel | ||
708 | + var nodes []zopfliNode | ||
709 | + var matches []backwardMatch = make([]backwardMatch, matches_size) | ||
710 | + var gap uint = 0 | ||
711 | + var shadow_matches uint = 0 | ||
712 | + var new_array []backwardMatch | ||
713 | + for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ { | ||
714 | + var pos uint = position + i | ||
715 | + var max_distance uint = brotli_min_size_t(pos, max_backward_limit) | ||
716 | + var max_length uint = num_bytes - i | ||
717 | + var num_found_matches uint | ||
718 | + var cur_match_end uint | ||
719 | + var j uint | ||
720 | + | ||
721 | + /* Ensure that we have enough free slots. */ | ||
722 | + if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches { | ||
723 | + var new_size uint = matches_size | ||
724 | + if new_size == 0 { | ||
725 | + new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches | ||
726 | + } | ||
727 | + | ||
728 | + for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches { | ||
729 | + new_size *= 2 | ||
730 | + } | ||
731 | + | ||
732 | + new_array = make([]backwardMatch, new_size) | ||
733 | + if matches_size != 0 { | ||
734 | + copy(new_array, matches[:matches_size]) | ||
735 | + } | ||
736 | + | ||
737 | + matches = new_array | ||
738 | + matches_size = new_size | ||
739 | + } | ||
740 | + | ||
741 | + num_found_matches = findAllMatchesH10(hasher.(*h10), ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:]) | ||
742 | + cur_match_end = cur_match_pos + num_found_matches | ||
743 | + for j = cur_match_pos; j+1 < cur_match_end; j++ { | ||
744 | + assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1])) | ||
745 | + } | ||
746 | + | ||
747 | + num_matches[i] = uint32(num_found_matches) | ||
748 | + if num_found_matches > 0 { | ||
749 | + var match_len uint = backwardMatchLength(&matches[cur_match_end-1]) | ||
750 | + if match_len > maxZopfliLenQuality11 { | ||
751 | + var skip uint = match_len - 1 | ||
752 | + matches[cur_match_pos] = matches[cur_match_end-1] | ||
753 | + cur_match_pos++ | ||
754 | + num_matches[i] = 1 | ||
755 | + | ||
756 | + /* Add the tail of the copy to the hasher. */ | ||
757 | + hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end)) | ||
758 | + var pos uint = i | ||
759 | + for i := 0; i < int(skip); i++ { | ||
760 | + num_matches[pos+1:][i] = 0 | ||
761 | + } | ||
762 | + i += skip | ||
763 | + } else { | ||
764 | + cur_match_pos = cur_match_end | ||
765 | + } | ||
766 | + } | ||
767 | + } | ||
768 | + | ||
769 | + orig_num_literals = *num_literals | ||
770 | + orig_last_insert_len = *last_insert_len | ||
771 | + copy(orig_dist_cache[:], dist_cache[:4]) | ||
772 | + orig_num_commands = *num_commands | ||
773 | + nodes = make([]zopfliNode, (num_bytes + 1)) | ||
774 | + initZopfliCostModel(&model, ¶ms.dist, num_bytes) | ||
775 | + for i = 0; i < 2; i++ { | ||
776 | + initZopfliNodes(nodes, num_bytes+1) | ||
777 | + if i == 0 { | ||
778 | + zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) | ||
779 | + } else { | ||
780 | + zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len) | ||
781 | + } | ||
782 | + | ||
783 | + *num_commands = orig_num_commands | ||
784 | + *num_literals = orig_num_literals | ||
785 | + *last_insert_len = orig_last_insert_len | ||
786 | + copy(dist_cache, orig_dist_cache[:4]) | ||
787 | + *num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) | ||
788 | + zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) | ||
789 | + } | ||
790 | + | ||
791 | + cleanupZopfliCostModel(&model) | ||
792 | + nodes = nil | ||
793 | + matches = nil | ||
794 | + num_matches = nil | ||
795 | +} |
1 | +package brotli | ||
2 | + | ||
3 | +/* Copyright 2013 Google Inc. All Rights Reserved. | ||
4 | + | ||
5 | + Distributed under MIT license. | ||
6 | + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | ||
7 | +*/ | ||
8 | + | ||
9 | +/* Functions to estimate the bit cost of Huffman trees. */ | ||
10 | +func shannonEntropy(population []uint32, size uint, total *uint) float64 { | ||
11 | + var sum uint = 0 | ||
12 | + var retval float64 = 0 | ||
13 | + var population_end []uint32 = population[size:] | ||
14 | + var p uint | ||
15 | + for -cap(population) < -cap(population_end) { | ||
16 | + p = uint(population[0]) | ||
17 | + population = population[1:] | ||
18 | + sum += p | ||
19 | + retval -= float64(p) * fastLog2(p) | ||
20 | + } | ||
21 | + | ||
22 | + if sum != 0 { | ||
23 | + retval += float64(sum) * fastLog2(sum) | ||
24 | + } | ||
25 | + *total = sum | ||
26 | + return retval | ||
27 | +} | ||
28 | + | ||
29 | +func bitsEntropy(population []uint32, size uint) float64 { | ||
30 | + var sum uint | ||
31 | + var retval float64 = shannonEntropy(population, size, &sum) | ||
32 | + if retval < float64(sum) { | ||
33 | + /* At least one bit per literal is needed. */ | ||
34 | + retval = float64(sum) | ||
35 | + } | ||
36 | + | ||
37 | + return retval | ||
38 | +} | ||
39 | + | ||
40 | +const kOneSymbolHistogramCost float64 = 12 | ||
41 | +const kTwoSymbolHistogramCost float64 = 20 | ||
42 | +const kThreeSymbolHistogramCost float64 = 28 | ||
43 | +const kFourSymbolHistogramCost float64 = 37 | ||
44 | + | ||
45 | +func populationCostLiteral(histogram *histogramLiteral) float64 { | ||
46 | + var data_size uint = histogramDataSizeLiteral() | ||
47 | + var count int = 0 | ||
48 | + var s [5]uint | ||
49 | + var bits float64 = 0.0 | ||
50 | + var i uint | ||
51 | + if histogram.total_count_ == 0 { | ||
52 | + return kOneSymbolHistogramCost | ||
53 | + } | ||
54 | + | ||
55 | + for i = 0; i < data_size; i++ { | ||
56 | + if histogram.data_[i] > 0 { | ||
57 | + s[count] = i | ||
58 | + count++ | ||
59 | + if count > 4 { | ||
60 | + break | ||
61 | + } | ||
62 | + } | ||
63 | + } | ||
64 | + | ||
65 | + if count == 1 { | ||
66 | + return kOneSymbolHistogramCost | ||
67 | + } | ||
68 | + | ||
69 | + if count == 2 { | ||
70 | + return kTwoSymbolHistogramCost + float64(histogram.total_count_) | ||
71 | + } | ||
72 | + | ||
73 | + if count == 3 { | ||
74 | + var histo0 uint32 = histogram.data_[s[0]] | ||
75 | + var histo1 uint32 = histogram.data_[s[1]] | ||
76 | + var histo2 uint32 = histogram.data_[s[2]] | ||
77 | + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) | ||
78 | + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) | ||
79 | + } | ||
80 | + | ||
81 | + if count == 4 { | ||
82 | + var histo [4]uint32 | ||
83 | + var h23 uint32 | ||
84 | + var histomax uint32 | ||
85 | + for i = 0; i < 4; i++ { | ||
86 | + histo[i] = histogram.data_[s[i]] | ||
87 | + } | ||
88 | + | ||
89 | + /* Sort */ | ||
90 | + for i = 0; i < 4; i++ { | ||
91 | + var j uint | ||
92 | + for j = i + 1; j < 4; j++ { | ||
93 | + if histo[j] > histo[i] { | ||
94 | + var tmp uint32 = histo[j] | ||
95 | + histo[j] = histo[i] | ||
96 | + histo[i] = tmp | ||
97 | + } | ||
98 | + } | ||
99 | + } | ||
100 | + | ||
101 | + h23 = histo[2] + histo[3] | ||
102 | + histomax = brotli_max_uint32_t(h23, histo[0]) | ||
103 | + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) | ||
104 | + } | ||
105 | + { | ||
106 | + var max_depth uint = 1 | ||
107 | + var depth_histo = [codeLengthCodes]uint32{0} | ||
108 | + /* In this loop we compute the entropy of the histogram and simultaneously | ||
109 | + build a simplified histogram of the code length codes where we use the | ||
110 | + zero repeat code 17, but we don't use the non-zero repeat code 16. */ | ||
111 | + | ||
112 | + var log2total float64 = fastLog2(histogram.total_count_) | ||
113 | + for i = 0; i < data_size; { | ||
114 | + if histogram.data_[i] > 0 { | ||
115 | + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) | ||
116 | + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = | ||
117 | + = log2(total_count) - log2(count(symbol)) */ | ||
118 | + | ||
119 | + var depth uint = uint(log2p + 0.5) | ||
120 | + /* Approximate the bit depth by round(-log2(P(symbol))) */ | ||
121 | + bits += float64(histogram.data_[i]) * log2p | ||
122 | + | ||
123 | + if depth > 15 { | ||
124 | + depth = 15 | ||
125 | + } | ||
126 | + | ||
127 | + if depth > max_depth { | ||
128 | + max_depth = depth | ||
129 | + } | ||
130 | + | ||
131 | + depth_histo[depth]++ | ||
132 | + i++ | ||
133 | + } else { | ||
134 | + var reps uint32 = 1 | ||
135 | + /* Compute the run length of zeros and add the appropriate number of 0 | ||
136 | + and 17 code length codes to the code length code histogram. */ | ||
137 | + | ||
138 | + var k uint | ||
139 | + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { | ||
140 | + reps++ | ||
141 | + } | ||
142 | + | ||
143 | + i += uint(reps) | ||
144 | + if i == data_size { | ||
145 | + /* Don't add any cost for the last zero run, since these are encoded | ||
146 | + only implicitly. */ | ||
147 | + break | ||
148 | + } | ||
149 | + | ||
150 | + if reps < 3 { | ||
151 | + depth_histo[0] += reps | ||
152 | + } else { | ||
153 | + reps -= 2 | ||
154 | + for reps > 0 { | ||
155 | + depth_histo[repeatZeroCodeLength]++ | ||
156 | + | ||
157 | + /* Add the 3 extra bits for the 17 code length code. */ | ||
158 | + bits += 3 | ||
159 | + | ||
160 | + reps >>= 3 | ||
161 | + } | ||
162 | + } | ||
163 | + } | ||
164 | + } | ||
165 | + | ||
166 | + /* Add the estimated encoding cost of the code length code histogram. */ | ||
167 | + bits += float64(18 + 2*max_depth) | ||
168 | + | ||
169 | + /* Add the entropy of the code length code histogram. */ | ||
170 | + bits += bitsEntropy(depth_histo[:], codeLengthCodes) | ||
171 | + } | ||
172 | + | ||
173 | + return bits | ||
174 | +} | ||
175 | + | ||
176 | +func populationCostCommand(histogram *histogramCommand) float64 { | ||
177 | + var data_size uint = histogramDataSizeCommand() | ||
178 | + var count int = 0 | ||
179 | + var s [5]uint | ||
180 | + var bits float64 = 0.0 | ||
181 | + var i uint | ||
182 | + if histogram.total_count_ == 0 { | ||
183 | + return kOneSymbolHistogramCost | ||
184 | + } | ||
185 | + | ||
186 | + for i = 0; i < data_size; i++ { | ||
187 | + if histogram.data_[i] > 0 { | ||
188 | + s[count] = i | ||
189 | + count++ | ||
190 | + if count > 4 { | ||
191 | + break | ||
192 | + } | ||
193 | + } | ||
194 | + } | ||
195 | + | ||
196 | + if count == 1 { | ||
197 | + return kOneSymbolHistogramCost | ||
198 | + } | ||
199 | + | ||
200 | + if count == 2 { | ||
201 | + return kTwoSymbolHistogramCost + float64(histogram.total_count_) | ||
202 | + } | ||
203 | + | ||
204 | + if count == 3 { | ||
205 | + var histo0 uint32 = histogram.data_[s[0]] | ||
206 | + var histo1 uint32 = histogram.data_[s[1]] | ||
207 | + var histo2 uint32 = histogram.data_[s[2]] | ||
208 | + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) | ||
209 | + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) | ||
210 | + } | ||
211 | + | ||
212 | + if count == 4 { | ||
213 | + var histo [4]uint32 | ||
214 | + var h23 uint32 | ||
215 | + var histomax uint32 | ||
216 | + for i = 0; i < 4; i++ { | ||
217 | + histo[i] = histogram.data_[s[i]] | ||
218 | + } | ||
219 | + | ||
220 | + /* Sort */ | ||
221 | + for i = 0; i < 4; i++ { | ||
222 | + var j uint | ||
223 | + for j = i + 1; j < 4; j++ { | ||
224 | + if histo[j] > histo[i] { | ||
225 | + var tmp uint32 = histo[j] | ||
226 | + histo[j] = histo[i] | ||
227 | + histo[i] = tmp | ||
228 | + } | ||
229 | + } | ||
230 | + } | ||
231 | + | ||
232 | + h23 = histo[2] + histo[3] | ||
233 | + histomax = brotli_max_uint32_t(h23, histo[0]) | ||
234 | + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) | ||
235 | + } | ||
236 | + { | ||
237 | + var max_depth uint = 1 | ||
238 | + var depth_histo = [codeLengthCodes]uint32{0} | ||
239 | + /* In this loop we compute the entropy of the histogram and simultaneously | ||
240 | + build a simplified histogram of the code length codes where we use the | ||
241 | + zero repeat code 17, but we don't use the non-zero repeat code 16. */ | ||
242 | + | ||
243 | + var log2total float64 = fastLog2(histogram.total_count_) | ||
244 | + for i = 0; i < data_size; { | ||
245 | + if histogram.data_[i] > 0 { | ||
246 | + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) | ||
247 | + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = | ||
248 | + = log2(total_count) - log2(count(symbol)) */ | ||
249 | + | ||
250 | + var depth uint = uint(log2p + 0.5) | ||
251 | + /* Approximate the bit depth by round(-log2(P(symbol))) */ | ||
252 | + bits += float64(histogram.data_[i]) * log2p | ||
253 | + | ||
254 | + if depth > 15 { | ||
255 | + depth = 15 | ||
256 | + } | ||
257 | + | ||
258 | + if depth > max_depth { | ||
259 | + max_depth = depth | ||
260 | + } | ||
261 | + | ||
262 | + depth_histo[depth]++ | ||
263 | + i++ | ||
264 | + } else { | ||
265 | + var reps uint32 = 1 | ||
266 | + /* Compute the run length of zeros and add the appropriate number of 0 | ||
267 | + and 17 code length codes to the code length code histogram. */ | ||
268 | + | ||
269 | + var k uint | ||
270 | + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { | ||
271 | + reps++ | ||
272 | + } | ||
273 | + | ||
274 | + i += uint(reps) | ||
275 | + if i == data_size { | ||
276 | + /* Don't add any cost for the last zero run, since these are encoded | ||
277 | + only implicitly. */ | ||
278 | + break | ||
279 | + } | ||
280 | + | ||
281 | + if reps < 3 { | ||
282 | + depth_histo[0] += reps | ||
283 | + } else { | ||
284 | + reps -= 2 | ||
285 | + for reps > 0 { | ||
286 | + depth_histo[repeatZeroCodeLength]++ | ||
287 | + | ||
288 | + /* Add the 3 extra bits for the 17 code length code. */ | ||
289 | + bits += 3 | ||
290 | + | ||
291 | + reps >>= 3 | ||
292 | + } | ||
293 | + } | ||
294 | + } | ||
295 | + } | ||
296 | + | ||
297 | + /* Add the estimated encoding cost of the code length code histogram. */ | ||
298 | + bits += float64(18 + 2*max_depth) | ||
299 | + | ||
300 | + /* Add the entropy of the code length code histogram. */ | ||
301 | + bits += bitsEntropy(depth_histo[:], codeLengthCodes) | ||
302 | + } | ||
303 | + | ||
304 | + return bits | ||
305 | +} | ||
306 | + | ||
307 | +func populationCostDistance(histogram *histogramDistance) float64 { | ||
308 | + var data_size uint = histogramDataSizeDistance() | ||
309 | + var count int = 0 | ||
310 | + var s [5]uint | ||
311 | + var bits float64 = 0.0 | ||
312 | + var i uint | ||
313 | + if histogram.total_count_ == 0 { | ||
314 | + return kOneSymbolHistogramCost | ||
315 | + } | ||
316 | + | ||
317 | + for i = 0; i < data_size; i++ { | ||
318 | + if histogram.data_[i] > 0 { | ||
319 | + s[count] = i | ||
320 | + count++ | ||
321 | + if count > 4 { | ||
322 | + break | ||
323 | + } | ||
324 | + } | ||
325 | + } | ||
326 | + | ||
327 | + if count == 1 { | ||
328 | + return kOneSymbolHistogramCost | ||
329 | + } | ||
330 | + | ||
331 | + if count == 2 { | ||
332 | + return kTwoSymbolHistogramCost + float64(histogram.total_count_) | ||
333 | + } | ||
334 | + | ||
335 | + if count == 3 { | ||
336 | + var histo0 uint32 = histogram.data_[s[0]] | ||
337 | + var histo1 uint32 = histogram.data_[s[1]] | ||
338 | + var histo2 uint32 = histogram.data_[s[2]] | ||
339 | + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) | ||
340 | + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) | ||
341 | + } | ||
342 | + | ||
343 | + if count == 4 { | ||
344 | + var histo [4]uint32 | ||
345 | + var h23 uint32 | ||
346 | + var histomax uint32 | ||
347 | + for i = 0; i < 4; i++ { | ||
348 | + histo[i] = histogram.data_[s[i]] | ||
349 | + } | ||
350 | + | ||
351 | + /* Sort */ | ||
352 | + for i = 0; i < 4; i++ { | ||
353 | + var j uint | ||
354 | + for j = i + 1; j < 4; j++ { | ||
355 | + if histo[j] > histo[i] { | ||
356 | + var tmp uint32 = histo[j] | ||
357 | + histo[j] = histo[i] | ||
358 | + histo[i] = tmp | ||
359 | + } | ||
360 | + } | ||
361 | + } | ||
362 | + | ||
363 | + h23 = histo[2] + histo[3] | ||
364 | + histomax = brotli_max_uint32_t(h23, histo[0]) | ||
365 | + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) | ||
366 | + } | ||
367 | + { | ||
368 | + var max_depth uint = 1 | ||
369 | + var depth_histo = [codeLengthCodes]uint32{0} | ||
370 | + /* In this loop we compute the entropy of the histogram and simultaneously | ||
371 | + build a simplified histogram of the code length codes where we use the | ||
372 | + zero repeat code 17, but we don't use the non-zero repeat code 16. */ | ||
373 | + | ||
374 | + var log2total float64 = fastLog2(histogram.total_count_) | ||
375 | + for i = 0; i < data_size; { | ||
376 | + if histogram.data_[i] > 0 { | ||
377 | + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) | ||
378 | + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = | ||
379 | + = log2(total_count) - log2(count(symbol)) */ | ||
380 | + | ||
381 | + var depth uint = uint(log2p + 0.5) | ||
382 | + /* Approximate the bit depth by round(-log2(P(symbol))) */ | ||
383 | + bits += float64(histogram.data_[i]) * log2p | ||
384 | + | ||
385 | + if depth > 15 { | ||
386 | + depth = 15 | ||
387 | + } | ||
388 | + | ||
389 | + if depth > max_depth { | ||
390 | + max_depth = depth | ||
391 | + } | ||
392 | + | ||
393 | + depth_histo[depth]++ | ||
394 | + i++ | ||
395 | + } else { | ||
396 | + var reps uint32 = 1 | ||
397 | + /* Compute the run length of zeros and add the appropriate number of 0 | ||
398 | + and 17 code length codes to the code length code histogram. */ | ||
399 | + | ||
400 | + var k uint | ||
401 | + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { | ||
402 | + reps++ | ||
403 | + } | ||
404 | + | ||
405 | + i += uint(reps) | ||
406 | + if i == data_size { | ||
407 | + /* Don't add any cost for the last zero run, since these are encoded | ||
408 | + only implicitly. */ | ||
409 | + break | ||
410 | + } | ||
411 | + | ||
412 | + if reps < 3 { | ||
413 | + depth_histo[0] += reps | ||
414 | + } else { | ||
415 | + reps -= 2 | ||
416 | + for reps > 0 { | ||
417 | + depth_histo[repeatZeroCodeLength]++ | ||
418 | + | ||
419 | + /* Add the 3 extra bits for the 17 code length code. */ | ||
420 | + bits += 3 | ||
421 | + | ||
422 | + reps >>= 3 | ||
423 | + } | ||
424 | + } | ||
425 | + } | ||
426 | + } | ||
427 | + | ||
428 | + /* Add the estimated encoding cost of the code length code histogram. */ | ||
429 | + bits += float64(18 + 2*max_depth) | ||
430 | + | ||
431 | + /* Add the entropy of the code length code histogram. */ | ||
432 | + bits += bitsEntropy(depth_histo[:], codeLengthCodes) | ||
433 | + } | ||
434 | + | ||
435 | + return bits | ||
436 | +} |
1 | +package brotli | ||
2 | + | ||
3 | +import "encoding/binary" | ||
4 | + | ||
5 | +/* Copyright 2013 Google Inc. All Rights Reserved. | ||
6 | + | ||
7 | + Distributed under MIT license. | ||
8 | + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | ||
9 | +*/ | ||
10 | + | ||
11 | +/* Bit reading helpers */ | ||
12 | + | ||
13 | +const shortFillBitWindowRead = (8 >> 1) | ||
14 | + | ||
15 | +var kBitMask = [33]uint32{ | ||
16 | + 0x00000000, | ||
17 | + 0x00000001, | ||
18 | + 0x00000003, | ||
19 | + 0x00000007, | ||
20 | + 0x0000000F, | ||
21 | + 0x0000001F, | ||
22 | + 0x0000003F, | ||
23 | + 0x0000007F, | ||
24 | + 0x000000FF, | ||
25 | + 0x000001FF, | ||
26 | + 0x000003FF, | ||
27 | + 0x000007FF, | ||
28 | + 0x00000FFF, | ||
29 | + 0x00001FFF, | ||
30 | + 0x00003FFF, | ||
31 | + 0x00007FFF, | ||
32 | + 0x0000FFFF, | ||
33 | + 0x0001FFFF, | ||
34 | + 0x0003FFFF, | ||
35 | + 0x0007FFFF, | ||
36 | + 0x000FFFFF, | ||
37 | + 0x001FFFFF, | ||
38 | + 0x003FFFFF, | ||
39 | + 0x007FFFFF, | ||
40 | + 0x00FFFFFF, | ||
41 | + 0x01FFFFFF, | ||
42 | + 0x03FFFFFF, | ||
43 | + 0x07FFFFFF, | ||
44 | + 0x0FFFFFFF, | ||
45 | + 0x1FFFFFFF, | ||
46 | + 0x3FFFFFFF, | ||
47 | + 0x7FFFFFFF, | ||
48 | + 0xFFFFFFFF, | ||
49 | +} | ||
50 | + | ||
51 | +func bitMask(n uint32) uint32 { | ||
52 | + return kBitMask[n] | ||
53 | +} | ||
54 | + | ||
55 | +type bitReader struct { | ||
56 | + val_ uint64 | ||
57 | + bit_pos_ uint32 | ||
58 | + input []byte | ||
59 | + input_len uint | ||
60 | + byte_pos uint | ||
61 | +} | ||
62 | + | ||
63 | +type bitReaderState struct { | ||
64 | + val_ uint64 | ||
65 | + bit_pos_ uint32 | ||
66 | + input []byte | ||
67 | + input_len uint | ||
68 | + byte_pos uint | ||
69 | +} | ||
70 | + | ||
71 | +/* Initializes the BrotliBitReader fields. */ | ||
72 | + | ||
73 | +/* Ensures that accumulator is not empty. | ||
74 | + May consume up to sizeof(brotli_reg_t) - 1 bytes of input. | ||
75 | + Returns false if data is required but there is no input available. | ||
76 | + For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned | ||
77 | + reading. */ | ||
78 | +func bitReaderSaveState(from *bitReader, to *bitReaderState) { | ||
79 | + to.val_ = from.val_ | ||
80 | + to.bit_pos_ = from.bit_pos_ | ||
81 | + to.input = from.input | ||
82 | + to.input_len = from.input_len | ||
83 | + to.byte_pos = from.byte_pos | ||
84 | +} | ||
85 | + | ||
86 | +func bitReaderRestoreState(to *bitReader, from *bitReaderState) { | ||
87 | + to.val_ = from.val_ | ||
88 | + to.bit_pos_ = from.bit_pos_ | ||
89 | + to.input = from.input | ||
90 | + to.input_len = from.input_len | ||
91 | + to.byte_pos = from.byte_pos | ||
92 | +} | ||
93 | + | ||
94 | +func getAvailableBits(br *bitReader) uint32 { | ||
95 | + return 64 - br.bit_pos_ | ||
96 | +} | ||
97 | + | ||
98 | +/* Returns amount of unread bytes the bit reader still has buffered from the | ||
99 | + BrotliInput, including whole bytes in br->val_. */ | ||
100 | +func getRemainingBytes(br *bitReader) uint { | ||
101 | + return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3)) | ||
102 | +} | ||
103 | + | ||
104 | +/* Checks if there is at least |num| bytes left in the input ring-buffer | ||
105 | + (excluding the bits remaining in br->val_). */ | ||
106 | +func checkInputAmount(br *bitReader, num uint) bool { | ||
107 | + return br.input_len-br.byte_pos >= num | ||
108 | +} | ||
109 | + | ||
110 | +/* Guarantees that there are at least |n_bits| + 1 bits in accumulator. | ||
111 | + Precondition: accumulator contains at least 1 bit. | ||
112 | + |n_bits| should be in the range [1..24] for regular build. For portable | ||
113 | + non-64-bit little-endian build only 16 bits are safe to request. */ | ||
114 | +func fillBitWindow(br *bitReader, n_bits uint32) { | ||
115 | + if br.bit_pos_ >= 32 { | ||
116 | + br.val_ >>= 32 | ||
117 | + br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */ | ||
118 | + br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32 | ||
119 | + br.byte_pos += 4 | ||
120 | + } | ||
121 | +} | ||
122 | + | ||
123 | +/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no | ||
124 | + more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */ | ||
125 | +func fillBitWindow16(br *bitReader) { | ||
126 | + fillBitWindow(br, 17) | ||
127 | +} | ||
128 | + | ||
129 | +/* Tries to pull one byte of input to accumulator. | ||
130 | + Returns false if there is no input available. */ | ||
131 | +func pullByte(br *bitReader) bool { | ||
132 | + if br.byte_pos == br.input_len { | ||
133 | + return false | ||
134 | + } | ||
135 | + | ||
136 | + br.val_ >>= 8 | ||
137 | + br.val_ |= (uint64(br.input[br.byte_pos])) << 56 | ||
138 | + br.bit_pos_ -= 8 | ||
139 | + br.byte_pos++ | ||
140 | + return true | ||
141 | +} | ||
142 | + | ||
143 | +/* Returns currently available bits. | ||
144 | + The number of valid bits could be calculated by BrotliGetAvailableBits. */ | ||
145 | +func getBitsUnmasked(br *bitReader) uint64 { | ||
146 | + return br.val_ >> br.bit_pos_ | ||
147 | +} | ||
148 | + | ||
149 | +/* Like BrotliGetBits, but does not mask the result. | ||
150 | + The result contains at least 16 valid bits. */ | ||
151 | +func get16BitsUnmasked(br *bitReader) uint32 { | ||
152 | + fillBitWindow(br, 16) | ||
153 | + return uint32(getBitsUnmasked(br)) | ||
154 | +} | ||
155 | + | ||
156 | +/* Returns the specified number of bits from |br| without advancing bit | ||
157 | + position. */ | ||
158 | +func getBits(br *bitReader, n_bits uint32) uint32 { | ||
159 | + fillBitWindow(br, n_bits) | ||
160 | + return uint32(getBitsUnmasked(br)) & bitMask(n_bits) | ||
161 | +} | ||
162 | + | ||
163 | +/* Tries to peek the specified amount of bits. Returns false, if there | ||
164 | + is not enough input. */ | ||
165 | +func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool { | ||
166 | + for getAvailableBits(br) < n_bits { | ||
167 | + if !pullByte(br) { | ||
168 | + return false | ||
169 | + } | ||
170 | + } | ||
171 | + | ||
172 | + *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) | ||
173 | + return true | ||
174 | +} | ||
175 | + | ||
176 | +/* Advances the bit pos by |n_bits|. */ | ||
177 | +func dropBits(br *bitReader, n_bits uint32) { | ||
178 | + br.bit_pos_ += n_bits | ||
179 | +} | ||
180 | + | ||
181 | +func bitReaderUnload(br *bitReader) { | ||
182 | + var unused_bytes uint32 = getAvailableBits(br) >> 3 | ||
183 | + var unused_bits uint32 = unused_bytes << 3 | ||
184 | + br.byte_pos -= uint(unused_bytes) | ||
185 | + if unused_bits == 64 { | ||
186 | + br.val_ = 0 | ||
187 | + } else { | ||
188 | + br.val_ <<= unused_bits | ||
189 | + } | ||
190 | + | ||
191 | + br.bit_pos_ += unused_bits | ||
192 | +} | ||
193 | + | ||
194 | +/* Reads the specified number of bits from |br| and advances the bit pos. | ||
195 | + Precondition: accumulator MUST contain at least |n_bits|. */ | ||
196 | +func takeBits(br *bitReader, n_bits uint32, val *uint32) { | ||
197 | + *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) | ||
198 | + dropBits(br, n_bits) | ||
199 | +} | ||
200 | + | ||
201 | +/* Reads the specified number of bits from |br| and advances the bit pos. | ||
202 | + Assumes that there is enough input to perform BrotliFillBitWindow. */ | ||
203 | +func readBits(br *bitReader, n_bits uint32) uint32 { | ||
204 | + var val uint32 | ||
205 | + fillBitWindow(br, n_bits) | ||
206 | + takeBits(br, n_bits, &val) | ||
207 | + return val | ||
208 | +} | ||
209 | + | ||
210 | +/* Tries to read the specified amount of bits. Returns false, if there | ||
211 | + is not enough input. |n_bits| MUST be positive. */ | ||
212 | +func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool { | ||
213 | + for getAvailableBits(br) < n_bits { | ||
214 | + if !pullByte(br) { | ||
215 | + return false | ||
216 | + } | ||
217 | + } | ||
218 | + | ||
219 | + takeBits(br, n_bits, val) | ||
220 | + return true | ||
221 | +} | ||
222 | + | ||
223 | +/* Advances the bit reader position to the next byte boundary and verifies | ||
224 | + that any skipped bits are set to zero. */ | ||
225 | +func bitReaderJumpToByteBoundary(br *bitReader) bool { | ||
226 | + var pad_bits_count uint32 = getAvailableBits(br) & 0x7 | ||
227 | + var pad_bits uint32 = 0 | ||
228 | + if pad_bits_count != 0 { | ||
229 | + takeBits(br, pad_bits_count, &pad_bits) | ||
230 | + } | ||
231 | + | ||
232 | + return pad_bits == 0 | ||
233 | +} | ||
234 | + | ||
235 | +/* Copies remaining input bytes stored in the bit reader to the output. Value | ||
236 | + |num| may not be larger than BrotliGetRemainingBytes. The bit reader must be | ||
237 | + warmed up again after this. */ | ||
238 | +func copyBytes(dest []byte, br *bitReader, num uint) { | ||
239 | + for getAvailableBits(br) >= 8 && num > 0 { | ||
240 | + dest[0] = byte(getBitsUnmasked(br)) | ||
241 | + dropBits(br, 8) | ||
242 | + dest = dest[1:] | ||
243 | + num-- | ||
244 | + } | ||
245 | + | ||
246 | + copy(dest, br.input[br.byte_pos:][:num]) | ||
247 | + br.byte_pos += num | ||
248 | +} | ||
249 | + | ||
250 | +func initBitReader(br *bitReader) { | ||
251 | + br.val_ = 0 | ||
252 | + br.bit_pos_ = 64 | ||
253 | +} | ||
254 | + | ||
255 | +func warmupBitReader(br *bitReader) bool { | ||
256 | + /* Fixing alignment after unaligned BrotliFillWindow would result accumulator | ||
257 | + overflow. If unalignment is caused by BrotliSafeReadBits, then there is | ||
258 | + enough space in accumulator to fix alignment. */ | ||
259 | + if getAvailableBits(br) == 0 { | ||
260 | + if !pullByte(br) { | ||
261 | + return false | ||
262 | + } | ||
263 | + } | ||
264 | + | ||
265 | + return true | ||
266 | +} |
1 | +package brotli | ||
2 | + | ||
3 | +/* Copyright 2013 Google Inc. All Rights Reserved. | ||
4 | + | ||
5 | + Distributed under MIT license. | ||
6 | + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | ||
7 | +*/ | ||
8 | + | ||
9 | +/* Block split point selection utilities. */ | ||
10 | + | ||
11 | +type blockSplit struct { | ||
12 | + num_types uint | ||
13 | + num_blocks uint | ||
14 | + types []byte | ||
15 | + lengths []uint32 | ||
16 | + types_alloc_size uint | ||
17 | + lengths_alloc_size uint | ||
18 | +} | ||
19 | + | ||
20 | +const ( | ||
21 | + kMaxLiteralHistograms uint = 100 | ||
22 | + kMaxCommandHistograms uint = 50 | ||
23 | + kLiteralBlockSwitchCost float64 = 28.1 | ||
24 | + kCommandBlockSwitchCost float64 = 13.5 | ||
25 | + kDistanceBlockSwitchCost float64 = 14.6 | ||
26 | + kLiteralStrideLength uint = 70 | ||
27 | + kCommandStrideLength uint = 40 | ||
28 | + kSymbolsPerLiteralHistogram uint = 544 | ||
29 | + kSymbolsPerCommandHistogram uint = 530 | ||
30 | + kSymbolsPerDistanceHistogram uint = 544 | ||
31 | + kMinLengthForBlockSplitting uint = 128 | ||
32 | + kIterMulForRefining uint = 2 | ||
33 | + kMinItersForRefining uint = 100 | ||
34 | +) | ||
35 | + | ||
36 | +func countLiterals(cmds []command, num_commands uint) uint { | ||
37 | + var total_length uint = 0 | ||
38 | + /* Count how many we have. */ | ||
39 | + | ||
40 | + var i uint | ||
41 | + for i = 0; i < num_commands; i++ { | ||
42 | + total_length += uint(cmds[i].insert_len_) | ||
43 | + } | ||
44 | + | ||
45 | + return total_length | ||
46 | +} | ||
47 | + | ||
48 | +func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) { | ||
49 | + var pos uint = 0 | ||
50 | + var from_pos uint = offset & mask | ||
51 | + var i uint | ||
52 | + for i = 0; i < num_commands; i++ { | ||
53 | + var insert_len uint = uint(cmds[i].insert_len_) | ||
54 | + if from_pos+insert_len > mask { | ||
55 | + var head_size uint = mask + 1 - from_pos | ||
56 | + copy(literals[pos:], data[from_pos:][:head_size]) | ||
57 | + from_pos = 0 | ||
58 | + pos += head_size | ||
59 | + insert_len -= head_size | ||
60 | + } | ||
61 | + | ||
62 | + if insert_len > 0 { | ||
63 | + copy(literals[pos:], data[from_pos:][:insert_len]) | ||
64 | + pos += insert_len | ||
65 | + } | ||
66 | + | ||
67 | + from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask)) | ||
68 | + } | ||
69 | +} | ||
70 | + | ||
71 | +func myRand(seed *uint32) uint32 { | ||
72 | + /* Initial seed should be 7. In this case, loop length is (1 << 29). */ | ||
73 | + *seed *= 16807 | ||
74 | + | ||
75 | + return *seed | ||
76 | +} | ||
77 | + | ||
78 | +func bitCost(count uint) float64 { | ||
79 | + if count == 0 { | ||
80 | + return -2.0 | ||
81 | + } else { | ||
82 | + return fastLog2(count) | ||
83 | + } | ||
84 | +} | ||
85 | + | ||
86 | +const histogramsPerBatch = 64 | ||
87 | + | ||
88 | +const clustersPerBatch = 16 | ||
89 | + | ||
90 | +func initBlockSplit(self *blockSplit) { | ||
91 | + self.num_types = 0 | ||
92 | + self.num_blocks = 0 | ||
93 | + self.types = nil | ||
94 | + self.lengths = nil | ||
95 | + self.types_alloc_size = 0 | ||
96 | + self.lengths_alloc_size = 0 | ||
97 | +} | ||
98 | + | ||
99 | +func destroyBlockSplit(self *blockSplit) { | ||
100 | + self.types = nil | ||
101 | + self.lengths = nil | ||
102 | +} | ||
103 | + | ||
104 | +func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { | ||
105 | + { | ||
106 | + var literals_count uint = countLiterals(cmds, num_commands) | ||
107 | + var literals []byte = make([]byte, literals_count) | ||
108 | + | ||
109 | + /* Create a continuous array of literals. */ | ||
110 | + copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals) | ||
111 | + | ||
112 | + /* Create the block split on the array of literals. | ||
113 | + Literal histograms have alphabet size 256. */ | ||
114 | + splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split) | ||
115 | + | ||
116 | + literals = nil | ||
117 | + } | ||
118 | + { | ||
119 | + var insert_and_copy_codes []uint16 = make([]uint16, num_commands) | ||
120 | + /* Compute prefix codes for commands. */ | ||
121 | + | ||
122 | + var i uint | ||
123 | + for i = 0; i < num_commands; i++ { | ||
124 | + insert_and_copy_codes[i] = cmds[i].cmd_prefix_ | ||
125 | + } | ||
126 | + | ||
127 | + /* Create the block split on the array of command prefixes. */ | ||
128 | + splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) | ||
129 | + | ||
130 | + /* TODO: reuse for distances? */ | ||
131 | + | ||
132 | + insert_and_copy_codes = nil | ||
133 | + } | ||
134 | + { | ||
135 | + var distance_prefixes []uint16 = make([]uint16, num_commands) | ||
136 | + var j uint = 0 | ||
137 | + /* Create a continuous array of distance prefixes. */ | ||
138 | + | ||
139 | + var i uint | ||
140 | + for i = 0; i < num_commands; i++ { | ||
141 | + var cmd *command = &cmds[i] | ||
142 | + if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { | ||
143 | + distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF | ||
144 | + j++ | ||
145 | + } | ||
146 | + } | ||
147 | + | ||
148 | + /* Create the block split on the array of distance prefixes. */ | ||
149 | + splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split) | ||
150 | + | ||
151 | + distance_prefixes = nil | ||
152 | + } | ||
153 | +} |
1 | +package brotli | ||
2 | + | ||
3 | +import "math" | ||
4 | + | ||
5 | +/* Copyright 2013 Google Inc. All Rights Reserved. | ||
6 | + | ||
7 | + Distributed under MIT license. | ||
8 | + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | ||
9 | +*/ | ||
10 | + | ||
11 | +func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { | ||
12 | + var seed uint32 = 7 | ||
13 | + var block_length uint = length / num_histograms | ||
14 | + var i uint | ||
15 | + clearHistogramsCommand(histograms, num_histograms) | ||
16 | + for i = 0; i < num_histograms; i++ { | ||
17 | + var pos uint = length * i / num_histograms | ||
18 | + if i != 0 { | ||
19 | + pos += uint(myRand(&seed) % uint32(block_length)) | ||
20 | + } | ||
21 | + | ||
22 | + if pos+stride >= length { | ||
23 | + pos = length - stride - 1 | ||
24 | + } | ||
25 | + | ||
26 | + histogramAddVectorCommand(&histograms[i], data[pos:], stride) | ||
27 | + } | ||
28 | +} | ||
29 | + | ||
30 | +func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) { | ||
31 | + var pos uint = 0 | ||
32 | + if stride >= length { | ||
33 | + stride = length | ||
34 | + } else { | ||
35 | + pos = uint(myRand(seed) % uint32(length-stride+1)) | ||
36 | + } | ||
37 | + | ||
38 | + histogramAddVectorCommand(sample, data[pos:], stride) | ||
39 | +} | ||
40 | + | ||
41 | +func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { | ||
42 | + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining | ||
43 | + var seed uint32 = 7 | ||
44 | + var iter uint | ||
45 | + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms | ||
46 | + for iter = 0; iter < iters; iter++ { | ||
47 | + var sample histogramCommand | ||
48 | + histogramClearCommand(&sample) | ||
49 | + randomSampleCommand(&seed, data, length, stride, &sample) | ||
50 | + histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample) | ||
51 | + } | ||
52 | +} | ||
53 | + | ||
54 | +/* Assigns a block id from the range [0, num_histograms) to each data element | ||
55 | + in data[0..length) and fills in block_id[0..length) with the assigned values. | ||
56 | + Returns the number of blocks, i.e. one plus the number of block switches. */ | ||
57 | +func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { | ||
58 | + var data_size uint = histogramDataSizeCommand() | ||
59 | + var bitmaplen uint = (num_histograms + 7) >> 3 | ||
60 | + var num_blocks uint = 1 | ||
61 | + var i uint | ||
62 | + var j uint | ||
63 | + assert(num_histograms <= 256) | ||
64 | + if num_histograms <= 1 { | ||
65 | + for i = 0; i < length; i++ { | ||
66 | + block_id[i] = 0 | ||
67 | + } | ||
68 | + | ||
69 | + return 1 | ||
70 | + } | ||
71 | + | ||
72 | + for i := 0; i < int(data_size*num_histograms); i++ { | ||
73 | + insert_cost[i] = 0 | ||
74 | + } | ||
75 | + for i = 0; i < num_histograms; i++ { | ||
76 | + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) | ||
77 | + } | ||
78 | + | ||
79 | + for i = data_size; i != 0; { | ||
80 | + i-- | ||
81 | + for j = 0; j < num_histograms; j++ { | ||
82 | + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) | ||
83 | + } | ||
84 | + } | ||
85 | + | ||
86 | + for i := 0; i < int(num_histograms); i++ { | ||
87 | + cost[i] = 0 | ||
88 | + } | ||
89 | + for i := 0; i < int(length*bitmaplen); i++ { | ||
90 | + switch_signal[i] = 0 | ||
91 | + } | ||
92 | + | ||
93 | + /* After each iteration of this loop, cost[k] will contain the difference | ||
94 | + between the minimum cost of arriving at the current byte position using | ||
95 | + entropy code k, and the minimum cost of arriving at the current byte | ||
96 | + position. This difference is capped at the block switch cost, and if it | ||
97 | + reaches block switch cost, it means that when we trace back from the last | ||
98 | + position, we need to switch here. */ | ||
99 | + for i = 0; i < length; i++ { | ||
100 | + var byte_ix uint = i | ||
101 | + var ix uint = byte_ix * bitmaplen | ||
102 | + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms | ||
103 | + var min_cost float64 = 1e99 | ||
104 | + var block_switch_cost float64 = block_switch_bitcost | ||
105 | + var k uint | ||
106 | + for k = 0; k < num_histograms; k++ { | ||
107 | + /* We are coding the symbol in data[byte_ix] with entropy code k. */ | ||
108 | + cost[k] += insert_cost[insert_cost_ix+k] | ||
109 | + | ||
110 | + if cost[k] < min_cost { | ||
111 | + min_cost = cost[k] | ||
112 | + block_id[byte_ix] = byte(k) | ||
113 | + } | ||
114 | + } | ||
115 | + | ||
116 | + /* More blocks for the beginning. */ | ||
117 | + if byte_ix < 2000 { | ||
118 | + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 | ||
119 | + } | ||
120 | + | ||
121 | + for k = 0; k < num_histograms; k++ { | ||
122 | + cost[k] -= min_cost | ||
123 | + if cost[k] >= block_switch_cost { | ||
124 | + var mask byte = byte(1 << (k & 7)) | ||
125 | + cost[k] = block_switch_cost | ||
126 | + assert(k>>3 < bitmaplen) | ||
127 | + switch_signal[ix+(k>>3)] |= mask | ||
128 | + /* Trace back from the last position and switch at the marked places. */ | ||
129 | + } | ||
130 | + } | ||
131 | + } | ||
132 | + { | ||
133 | + var byte_ix uint = length - 1 | ||
134 | + var ix uint = byte_ix * bitmaplen | ||
135 | + var cur_id byte = block_id[byte_ix] | ||
136 | + for byte_ix > 0 { | ||
137 | + var mask byte = byte(1 << (cur_id & 7)) | ||
138 | + assert(uint(cur_id)>>3 < bitmaplen) | ||
139 | + byte_ix-- | ||
140 | + ix -= bitmaplen | ||
141 | + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { | ||
142 | + if cur_id != block_id[byte_ix] { | ||
143 | + cur_id = block_id[byte_ix] | ||
144 | + num_blocks++ | ||
145 | + } | ||
146 | + } | ||
147 | + | ||
148 | + block_id[byte_ix] = cur_id | ||
149 | + } | ||
150 | + } | ||
151 | + | ||
152 | + return num_blocks | ||
153 | +} | ||
154 | + | ||
155 | +var remapBlockIdsCommand_kInvalidId uint16 = 256 | ||
156 | + | ||
157 | +func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { | ||
158 | + var next_id uint16 = 0 | ||
159 | + var i uint | ||
160 | + for i = 0; i < num_histograms; i++ { | ||
161 | + new_id[i] = remapBlockIdsCommand_kInvalidId | ||
162 | + } | ||
163 | + | ||
164 | + for i = 0; i < length; i++ { | ||
165 | + assert(uint(block_ids[i]) < num_histograms) | ||
166 | + if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId { | ||
167 | + new_id[block_ids[i]] = next_id | ||
168 | + next_id++ | ||
169 | + } | ||
170 | + } | ||
171 | + | ||
172 | + for i = 0; i < length; i++ { | ||
173 | + block_ids[i] = byte(new_id[block_ids[i]]) | ||
174 | + assert(uint(block_ids[i]) < num_histograms) | ||
175 | + } | ||
176 | + | ||
177 | + assert(uint(next_id) <= num_histograms) | ||
178 | + return uint(next_id) | ||
179 | +} | ||
180 | + | ||
181 | +func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) { | ||
182 | + var i uint | ||
183 | + clearHistogramsCommand(histograms, num_histograms) | ||
184 | + for i = 0; i < length; i++ { | ||
185 | + histogramAddCommand(&histograms[block_ids[i]], uint(data[i])) | ||
186 | + } | ||
187 | +} | ||
188 | + | ||
189 | +var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32 | ||
190 | + | ||
191 | +func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { | ||
192 | + var histogram_symbols []uint32 = make([]uint32, num_blocks) | ||
193 | + var block_lengths []uint32 = make([]uint32, num_blocks) | ||
194 | + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch | ||
195 | + var all_histograms_size uint = 0 | ||
196 | + var all_histograms_capacity uint = expected_num_clusters | ||
197 | + var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity) | ||
198 | + var cluster_size_size uint = 0 | ||
199 | + var cluster_size_capacity uint = expected_num_clusters | ||
200 | + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) | ||
201 | + var num_clusters uint = 0 | ||
202 | + var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch)) | ||
203 | + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 | ||
204 | + var pairs_capacity uint = max_num_pairs + 1 | ||
205 | + var pairs []histogramPair = make([]histogramPair, pairs_capacity) | ||
206 | + var pos uint = 0 | ||
207 | + var clusters []uint32 | ||
208 | + var num_final_clusters uint | ||
209 | + var new_index []uint32 | ||
210 | + var i uint | ||
211 | + var sizes = [histogramsPerBatch]uint32{0} | ||
212 | + var new_clusters = [histogramsPerBatch]uint32{0} | ||
213 | + var symbols = [histogramsPerBatch]uint32{0} | ||
214 | + var remap = [histogramsPerBatch]uint32{0} | ||
215 | + | ||
216 | + for i := 0; i < int(num_blocks); i++ { | ||
217 | + block_lengths[i] = 0 | ||
218 | + } | ||
219 | + { | ||
220 | + var block_idx uint = 0 | ||
221 | + for i = 0; i < length; i++ { | ||
222 | + assert(block_idx < num_blocks) | ||
223 | + block_lengths[block_idx]++ | ||
224 | + if i+1 == length || block_ids[i] != block_ids[i+1] { | ||
225 | + block_idx++ | ||
226 | + } | ||
227 | + } | ||
228 | + | ||
229 | + assert(block_idx == num_blocks) | ||
230 | + } | ||
231 | + | ||
232 | + for i = 0; i < num_blocks; i += histogramsPerBatch { | ||
233 | + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) | ||
234 | + var num_new_clusters uint | ||
235 | + var j uint | ||
236 | + for j = 0; j < num_to_combine; j++ { | ||
237 | + var k uint | ||
238 | + histogramClearCommand(&histograms[j]) | ||
239 | + for k = 0; uint32(k) < block_lengths[i+j]; k++ { | ||
240 | + histogramAddCommand(&histograms[j], uint(data[pos])) | ||
241 | + pos++ | ||
242 | + } | ||
243 | + | ||
244 | + histograms[j].bit_cost_ = populationCostCommand(&histograms[j]) | ||
245 | + new_clusters[j] = uint32(j) | ||
246 | + symbols[j] = uint32(j) | ||
247 | + sizes[j] = 1 | ||
248 | + } | ||
249 | + | ||
250 | + num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) | ||
251 | + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { | ||
252 | + var _new_size uint | ||
253 | + if all_histograms_capacity == 0 { | ||
254 | + _new_size = all_histograms_size + num_new_clusters | ||
255 | + } else { | ||
256 | + _new_size = all_histograms_capacity | ||
257 | + } | ||
258 | + var new_array []histogramCommand | ||
259 | + for _new_size < (all_histograms_size + num_new_clusters) { | ||
260 | + _new_size *= 2 | ||
261 | + } | ||
262 | + new_array = make([]histogramCommand, _new_size) | ||
263 | + if all_histograms_capacity != 0 { | ||
264 | + copy(new_array, all_histograms[:all_histograms_capacity]) | ||
265 | + } | ||
266 | + | ||
267 | + all_histograms = new_array | ||
268 | + all_histograms_capacity = _new_size | ||
269 | + } | ||
270 | + | ||
271 | + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) | ||
272 | + for j = 0; j < num_new_clusters; j++ { | ||
273 | + all_histograms[all_histograms_size] = histograms[new_clusters[j]] | ||
274 | + all_histograms_size++ | ||
275 | + cluster_size[cluster_size_size] = sizes[new_clusters[j]] | ||
276 | + cluster_size_size++ | ||
277 | + remap[new_clusters[j]] = uint32(j) | ||
278 | + } | ||
279 | + | ||
280 | + for j = 0; j < num_to_combine; j++ { | ||
281 | + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] | ||
282 | + } | ||
283 | + | ||
284 | + num_clusters += num_new_clusters | ||
285 | + assert(num_clusters == cluster_size_size) | ||
286 | + assert(num_clusters == all_histograms_size) | ||
287 | + } | ||
288 | + | ||
289 | + histograms = nil | ||
290 | + | ||
291 | + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) | ||
292 | + if pairs_capacity < max_num_pairs+1 { | ||
293 | + pairs = nil | ||
294 | + pairs = make([]histogramPair, (max_num_pairs + 1)) | ||
295 | + } | ||
296 | + | ||
297 | + clusters = make([]uint32, num_clusters) | ||
298 | + for i = 0; i < num_clusters; i++ { | ||
299 | + clusters[i] = uint32(i) | ||
300 | + } | ||
301 | + | ||
302 | + num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) | ||
303 | + pairs = nil | ||
304 | + cluster_size = nil | ||
305 | + | ||
306 | + new_index = make([]uint32, num_clusters) | ||
307 | + for i = 0; i < num_clusters; i++ { | ||
308 | + new_index[i] = clusterBlocksCommand_kInvalidIndex | ||
309 | + } | ||
310 | + pos = 0 | ||
311 | + { | ||
312 | + var next_index uint32 = 0 | ||
313 | + for i = 0; i < num_blocks; i++ { | ||
314 | + var histo histogramCommand | ||
315 | + var j uint | ||
316 | + var best_out uint32 | ||
317 | + var best_bits float64 | ||
318 | + histogramClearCommand(&histo) | ||
319 | + for j = 0; uint32(j) < block_lengths[i]; j++ { | ||
320 | + histogramAddCommand(&histo, uint(data[pos])) | ||
321 | + pos++ | ||
322 | + } | ||
323 | + | ||
324 | + if i == 0 { | ||
325 | + best_out = histogram_symbols[0] | ||
326 | + } else { | ||
327 | + best_out = histogram_symbols[i-1] | ||
328 | + } | ||
329 | + best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out]) | ||
330 | + for j = 0; j < num_final_clusters; j++ { | ||
331 | + var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]]) | ||
332 | + if cur_bits < best_bits { | ||
333 | + best_bits = cur_bits | ||
334 | + best_out = clusters[j] | ||
335 | + } | ||
336 | + } | ||
337 | + | ||
338 | + histogram_symbols[i] = best_out | ||
339 | + if new_index[best_out] == clusterBlocksCommand_kInvalidIndex { | ||
340 | + new_index[best_out] = next_index | ||
341 | + next_index++ | ||
342 | + } | ||
343 | + } | ||
344 | + } | ||
345 | + | ||
346 | + clusters = nil | ||
347 | + all_histograms = nil | ||
348 | + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) | ||
349 | + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) | ||
350 | + { | ||
351 | + var cur_length uint32 = 0 | ||
352 | + var block_idx uint = 0 | ||
353 | + var max_type byte = 0 | ||
354 | + for i = 0; i < num_blocks; i++ { | ||
355 | + cur_length += block_lengths[i] | ||
356 | + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { | ||
357 | + var id byte = byte(new_index[histogram_symbols[i]]) | ||
358 | + split.types[block_idx] = id | ||
359 | + split.lengths[block_idx] = cur_length | ||
360 | + max_type = brotli_max_uint8_t(max_type, id) | ||
361 | + cur_length = 0 | ||
362 | + block_idx++ | ||
363 | + } | ||
364 | + } | ||
365 | + | ||
366 | + split.num_blocks = block_idx | ||
367 | + split.num_types = uint(max_type) + 1 | ||
368 | + } | ||
369 | + | ||
370 | + new_index = nil | ||
371 | + block_lengths = nil | ||
372 | + histogram_symbols = nil | ||
373 | +} | ||
374 | + | ||
375 | +func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { | ||
376 | + var data_size uint = histogramDataSizeCommand() | ||
377 | + var num_histograms uint = length/literals_per_histogram + 1 | ||
378 | + var histograms []histogramCommand | ||
379 | + if num_histograms > max_histograms { | ||
380 | + num_histograms = max_histograms | ||
381 | + } | ||
382 | + | ||
383 | + if length == 0 { | ||
384 | + split.num_types = 1 | ||
385 | + return | ||
386 | + } else if length < kMinLengthForBlockSplitting { | ||
387 | + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) | ||
388 | + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) | ||
389 | + split.num_types = 1 | ||
390 | + split.types[split.num_blocks] = 0 | ||
391 | + split.lengths[split.num_blocks] = uint32(length) | ||
392 | + split.num_blocks++ | ||
393 | + return | ||
394 | + } | ||
395 | + | ||
396 | + histograms = make([]histogramCommand, num_histograms) | ||
397 | + | ||
398 | + /* Find good entropy codes. */ | ||
399 | + initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms) | ||
400 | + | ||
401 | + refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms) | ||
402 | + { | ||
403 | + var block_ids []byte = make([]byte, length) | ||
404 | + var num_blocks uint = 0 | ||
405 | + var bitmaplen uint = (num_histograms + 7) >> 3 | ||
406 | + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) | ||
407 | + var cost []float64 = make([]float64, num_histograms) | ||
408 | + var switch_signal []byte = make([]byte, (length * bitmaplen)) | ||
409 | + var new_id []uint16 = make([]uint16, num_histograms) | ||
410 | + var iters uint | ||
411 | + if params.quality < hqZopflificationQuality { | ||
412 | + iters = 3 | ||
413 | + } else { | ||
414 | + iters = 10 | ||
415 | + } | ||
416 | + /* Find a good path through literals with the good entropy codes. */ | ||
417 | + | ||
418 | + var i uint | ||
419 | + for i = 0; i < iters; i++ { | ||
420 | + num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) | ||
421 | + num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms) | ||
422 | + buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms) | ||
423 | + } | ||
424 | + | ||
425 | + insert_cost = nil | ||
426 | + cost = nil | ||
427 | + switch_signal = nil | ||
428 | + new_id = nil | ||
429 | + histograms = nil | ||
430 | + clusterBlocksCommand(data, length, num_blocks, block_ids, split) | ||
431 | + block_ids = nil | ||
432 | + } | ||
433 | +} |
1 | +package brotli | ||
2 | + | ||
3 | +import "math" | ||
4 | + | ||
5 | +/* Copyright 2013 Google Inc. All Rights Reserved. | ||
6 | + | ||
7 | + Distributed under MIT license. | ||
8 | + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | ||
9 | +*/ | ||
10 | + | ||
11 | +func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { | ||
12 | + var seed uint32 = 7 | ||
13 | + var block_length uint = length / num_histograms | ||
14 | + var i uint | ||
15 | + clearHistogramsDistance(histograms, num_histograms) | ||
16 | + for i = 0; i < num_histograms; i++ { | ||
17 | + var pos uint = length * i / num_histograms | ||
18 | + if i != 0 { | ||
19 | + pos += uint(myRand(&seed) % uint32(block_length)) | ||
20 | + } | ||
21 | + | ||
22 | + if pos+stride >= length { | ||
23 | + pos = length - stride - 1 | ||
24 | + } | ||
25 | + | ||
26 | + histogramAddVectorDistance(&histograms[i], data[pos:], stride) | ||
27 | + } | ||
28 | +} | ||
29 | + | ||
30 | +func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) { | ||
31 | + var pos uint = 0 | ||
32 | + if stride >= length { | ||
33 | + stride = length | ||
34 | + } else { | ||
35 | + pos = uint(myRand(seed) % uint32(length-stride+1)) | ||
36 | + } | ||
37 | + | ||
38 | + histogramAddVectorDistance(sample, data[pos:], stride) | ||
39 | +} | ||
40 | + | ||
41 | +func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { | ||
42 | + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining | ||
43 | + var seed uint32 = 7 | ||
44 | + var iter uint | ||
45 | + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms | ||
46 | + for iter = 0; iter < iters; iter++ { | ||
47 | + var sample histogramDistance | ||
48 | + histogramClearDistance(&sample) | ||
49 | + randomSampleDistance(&seed, data, length, stride, &sample) | ||
50 | + histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample) | ||
51 | + } | ||
52 | +} | ||
53 | + | ||
54 | +/* Assigns a block id from the range [0, num_histograms) to each data element | ||
55 | + in data[0..length) and fills in block_id[0..length) with the assigned values. | ||
56 | + Returns the number of blocks, i.e. one plus the number of block switches. */ | ||
57 | +func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { | ||
58 | + var data_size uint = histogramDataSizeDistance() | ||
59 | + var bitmaplen uint = (num_histograms + 7) >> 3 | ||
60 | + var num_blocks uint = 1 | ||
61 | + var i uint | ||
62 | + var j uint | ||
63 | + assert(num_histograms <= 256) | ||
64 | + if num_histograms <= 1 { | ||
65 | + for i = 0; i < length; i++ { | ||
66 | + block_id[i] = 0 | ||
67 | + } | ||
68 | + | ||
69 | + return 1 | ||
70 | + } | ||
71 | + | ||
72 | + for i := 0; i < int(data_size*num_histograms); i++ { | ||
73 | + insert_cost[i] = 0 | ||
74 | + } | ||
75 | + for i = 0; i < num_histograms; i++ { | ||
76 | + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) | ||
77 | + } | ||
78 | + | ||
79 | + for i = data_size; i != 0; { | ||
80 | + i-- | ||
81 | + for j = 0; j < num_histograms; j++ { | ||
82 | + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) | ||
83 | + } | ||
84 | + } | ||
85 | + | ||
86 | + for i := 0; i < int(num_histograms); i++ { | ||
87 | + cost[i] = 0 | ||
88 | + } | ||
89 | + for i := 0; i < int(length*bitmaplen); i++ { | ||
90 | + switch_signal[i] = 0 | ||
91 | + } | ||
92 | + | ||
93 | + /* After each iteration of this loop, cost[k] will contain the difference | ||
94 | + between the minimum cost of arriving at the current byte position using | ||
95 | + entropy code k, and the minimum cost of arriving at the current byte | ||
96 | + position. This difference is capped at the block switch cost, and if it | ||
97 | + reaches block switch cost, it means that when we trace back from the last | ||
98 | + position, we need to switch here. */ | ||
99 | + for i = 0; i < length; i++ { | ||
100 | + var byte_ix uint = i | ||
101 | + var ix uint = byte_ix * bitmaplen | ||
102 | + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms | ||
103 | + var min_cost float64 = 1e99 | ||
104 | + var block_switch_cost float64 = block_switch_bitcost | ||
105 | + var k uint | ||
106 | + for k = 0; k < num_histograms; k++ { | ||
107 | + /* We are coding the symbol in data[byte_ix] with entropy code k. */ | ||
108 | + cost[k] += insert_cost[insert_cost_ix+k] | ||
109 | + | ||
110 | + if cost[k] < min_cost { | ||
111 | + min_cost = cost[k] | ||
112 | + block_id[byte_ix] = byte(k) | ||
113 | + } | ||
114 | + } | ||
115 | + | ||
116 | + /* More blocks for the beginning. */ | ||
117 | + if byte_ix < 2000 { | ||
118 | + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 | ||
119 | + } | ||
120 | + | ||
121 | + for k = 0; k < num_histograms; k++ { | ||
122 | + cost[k] -= min_cost | ||
123 | + if cost[k] >= block_switch_cost { | ||
124 | + var mask byte = byte(1 << (k & 7)) | ||
125 | + cost[k] = block_switch_cost | ||
126 | + assert(k>>3 < bitmaplen) | ||
127 | + switch_signal[ix+(k>>3)] |= mask | ||
128 | + /* Trace back from the last position and switch at the marked places. */ | ||
129 | + } | ||
130 | + } | ||
131 | + } | ||
132 | + { | ||
133 | + var byte_ix uint = length - 1 | ||
134 | + var ix uint = byte_ix * bitmaplen | ||
135 | + var cur_id byte = block_id[byte_ix] | ||
136 | + for byte_ix > 0 { | ||
137 | + var mask byte = byte(1 << (cur_id & 7)) | ||
138 | + assert(uint(cur_id)>>3 < bitmaplen) | ||
139 | + byte_ix-- | ||
140 | + ix -= bitmaplen | ||
141 | + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { | ||
142 | + if cur_id != block_id[byte_ix] { | ||
143 | + cur_id = block_id[byte_ix] | ||
144 | + num_blocks++ | ||
145 | + } | ||
146 | + } | ||
147 | + | ||
148 | + block_id[byte_ix] = cur_id | ||
149 | + } | ||
150 | + } | ||
151 | + | ||
152 | + return num_blocks | ||
153 | +} | ||
154 | + | ||
155 | +var remapBlockIdsDistance_kInvalidId uint16 = 256 | ||
156 | + | ||
157 | +func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { | ||
158 | + var next_id uint16 = 0 | ||
159 | + var i uint | ||
160 | + for i = 0; i < num_histograms; i++ { | ||
161 | + new_id[i] = remapBlockIdsDistance_kInvalidId | ||
162 | + } | ||
163 | + | ||
164 | + for i = 0; i < length; i++ { | ||
165 | + assert(uint(block_ids[i]) < num_histograms) | ||
166 | + if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId { | ||
167 | + new_id[block_ids[i]] = next_id | ||
168 | + next_id++ | ||
169 | + } | ||
170 | + } | ||
171 | + | ||
172 | + for i = 0; i < length; i++ { | ||
173 | + block_ids[i] = byte(new_id[block_ids[i]]) | ||
174 | + assert(uint(block_ids[i]) < num_histograms) | ||
175 | + } | ||
176 | + | ||
177 | + assert(uint(next_id) <= num_histograms) | ||
178 | + return uint(next_id) | ||
179 | +} | ||
180 | + | ||
181 | +func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) { | ||
182 | + var i uint | ||
183 | + clearHistogramsDistance(histograms, num_histograms) | ||
184 | + for i = 0; i < length; i++ { | ||
185 | + histogramAddDistance(&histograms[block_ids[i]], uint(data[i])) | ||
186 | + } | ||
187 | +} | ||
188 | + | ||
189 | +var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32 | ||
190 | + | ||
191 | +func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { | ||
192 | + var histogram_symbols []uint32 = make([]uint32, num_blocks) | ||
193 | + var block_lengths []uint32 = make([]uint32, num_blocks) | ||
194 | + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch | ||
195 | + var all_histograms_size uint = 0 | ||
196 | + var all_histograms_capacity uint = expected_num_clusters | ||
197 | + var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity) | ||
198 | + var cluster_size_size uint = 0 | ||
199 | + var cluster_size_capacity uint = expected_num_clusters | ||
200 | + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) | ||
201 | + var num_clusters uint = 0 | ||
202 | + var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch)) | ||
203 | + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 | ||
204 | + var pairs_capacity uint = max_num_pairs + 1 | ||
205 | + var pairs []histogramPair = make([]histogramPair, pairs_capacity) | ||
206 | + var pos uint = 0 | ||
207 | + var clusters []uint32 | ||
208 | + var num_final_clusters uint | ||
209 | + var new_index []uint32 | ||
210 | + var i uint | ||
211 | + var sizes = [histogramsPerBatch]uint32{0} | ||
212 | + var new_clusters = [histogramsPerBatch]uint32{0} | ||
213 | + var symbols = [histogramsPerBatch]uint32{0} | ||
214 | + var remap = [histogramsPerBatch]uint32{0} | ||
215 | + | ||
216 | + for i := 0; i < int(num_blocks); i++ { | ||
217 | + block_lengths[i] = 0 | ||
218 | + } | ||
219 | + { | ||
220 | + var block_idx uint = 0 | ||
221 | + for i = 0; i < length; i++ { | ||
222 | + assert(block_idx < num_blocks) | ||
223 | + block_lengths[block_idx]++ | ||
224 | + if i+1 == length || block_ids[i] != block_ids[i+1] { | ||
225 | + block_idx++ | ||
226 | + } | ||
227 | + } | ||
228 | + | ||
229 | + assert(block_idx == num_blocks) | ||
230 | + } | ||
231 | + | ||
232 | + for i = 0; i < num_blocks; i += histogramsPerBatch { | ||
233 | + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) | ||
234 | + var num_new_clusters uint | ||
235 | + var j uint | ||
236 | + for j = 0; j < num_to_combine; j++ { | ||
237 | + var k uint | ||
238 | + histogramClearDistance(&histograms[j]) | ||
239 | + for k = 0; uint32(k) < block_lengths[i+j]; k++ { | ||
240 | + histogramAddDistance(&histograms[j], uint(data[pos])) | ||
241 | + pos++ | ||
242 | + } | ||
243 | + | ||
244 | + histograms[j].bit_cost_ = populationCostDistance(&histograms[j]) | ||
245 | + new_clusters[j] = uint32(j) | ||
246 | + symbols[j] = uint32(j) | ||
247 | + sizes[j] = 1 | ||
248 | + } | ||
249 | + | ||
250 | + num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) | ||
251 | + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { | ||
252 | + var _new_size uint | ||
253 | + if all_histograms_capacity == 0 { | ||
254 | + _new_size = all_histograms_size + num_new_clusters | ||
255 | + } else { | ||
256 | + _new_size = all_histograms_capacity | ||
257 | + } | ||
258 | + var new_array []histogramDistance | ||
259 | + for _new_size < (all_histograms_size + num_new_clusters) { | ||
260 | + _new_size *= 2 | ||
261 | + } | ||
262 | + new_array = make([]histogramDistance, _new_size) | ||
263 | + if all_histograms_capacity != 0 { | ||
264 | + copy(new_array, all_histograms[:all_histograms_capacity]) | ||
265 | + } | ||
266 | + | ||
267 | + all_histograms = new_array | ||
268 | + all_histograms_capacity = _new_size | ||
269 | + } | ||
270 | + | ||
271 | + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) | ||
272 | + for j = 0; j < num_new_clusters; j++ { | ||
273 | + all_histograms[all_histograms_size] = histograms[new_clusters[j]] | ||
274 | + all_histograms_size++ | ||
275 | + cluster_size[cluster_size_size] = sizes[new_clusters[j]] | ||
276 | + cluster_size_size++ | ||
277 | + remap[new_clusters[j]] = uint32(j) | ||
278 | + } | ||
279 | + | ||
280 | + for j = 0; j < num_to_combine; j++ { | ||
281 | + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] | ||
282 | + } | ||
283 | + | ||
284 | + num_clusters += num_new_clusters | ||
285 | + assert(num_clusters == cluster_size_size) | ||
286 | + assert(num_clusters == all_histograms_size) | ||
287 | + } | ||
288 | + | ||
289 | + histograms = nil | ||
290 | + | ||
291 | + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) | ||
292 | + if pairs_capacity < max_num_pairs+1 { | ||
293 | + pairs = nil | ||
294 | + pairs = make([]histogramPair, (max_num_pairs + 1)) | ||
295 | + } | ||
296 | + | ||
297 | + clusters = make([]uint32, num_clusters) | ||
298 | + for i = 0; i < num_clusters; i++ { | ||
299 | + clusters[i] = uint32(i) | ||
300 | + } | ||
301 | + | ||
302 | + num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) | ||
303 | + pairs = nil | ||
304 | + cluster_size = nil | ||
305 | + | ||
306 | + new_index = make([]uint32, num_clusters) | ||
307 | + for i = 0; i < num_clusters; i++ { | ||
308 | + new_index[i] = clusterBlocksDistance_kInvalidIndex | ||
309 | + } | ||
310 | + pos = 0 | ||
311 | + { | ||
312 | + var next_index uint32 = 0 | ||
313 | + for i = 0; i < num_blocks; i++ { | ||
314 | + var histo histogramDistance | ||
315 | + var j uint | ||
316 | + var best_out uint32 | ||
317 | + var best_bits float64 | ||
318 | + histogramClearDistance(&histo) | ||
319 | + for j = 0; uint32(j) < block_lengths[i]; j++ { | ||
320 | + histogramAddDistance(&histo, uint(data[pos])) | ||
321 | + pos++ | ||
322 | + } | ||
323 | + | ||
324 | + if i == 0 { | ||
325 | + best_out = histogram_symbols[0] | ||
326 | + } else { | ||
327 | + best_out = histogram_symbols[i-1] | ||
328 | + } | ||
329 | + best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out]) | ||
330 | + for j = 0; j < num_final_clusters; j++ { | ||
331 | + var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]]) | ||
332 | + if cur_bits < best_bits { | ||
333 | + best_bits = cur_bits | ||
334 | + best_out = clusters[j] | ||
335 | + } | ||
336 | + } | ||
337 | + | ||
338 | + histogram_symbols[i] = best_out | ||
339 | + if new_index[best_out] == clusterBlocksDistance_kInvalidIndex { | ||
340 | + new_index[best_out] = next_index | ||
341 | + next_index++ | ||
342 | + } | ||
343 | + } | ||
344 | + } | ||
345 | + | ||
346 | + clusters = nil | ||
347 | + all_histograms = nil | ||
348 | + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) | ||
349 | + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) | ||
350 | + { | ||
351 | + var cur_length uint32 = 0 | ||
352 | + var block_idx uint = 0 | ||
353 | + var max_type byte = 0 | ||
354 | + for i = 0; i < num_blocks; i++ { | ||
355 | + cur_length += block_lengths[i] | ||
356 | + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { | ||
357 | + var id byte = byte(new_index[histogram_symbols[i]]) | ||
358 | + split.types[block_idx] = id | ||
359 | + split.lengths[block_idx] = cur_length | ||
360 | + max_type = brotli_max_uint8_t(max_type, id) | ||
361 | + cur_length = 0 | ||
362 | + block_idx++ | ||
363 | + } | ||
364 | + } | ||
365 | + | ||
366 | + split.num_blocks = block_idx | ||
367 | + split.num_types = uint(max_type) + 1 | ||
368 | + } | ||
369 | + | ||
370 | + new_index = nil | ||
371 | + block_lengths = nil | ||
372 | + histogram_symbols = nil | ||
373 | +} | ||
374 | + | ||
375 | +func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { | ||
376 | + var data_size uint = histogramDataSizeDistance() | ||
377 | + var num_histograms uint = length/literals_per_histogram + 1 | ||
378 | + var histograms []histogramDistance | ||
379 | + if num_histograms > max_histograms { | ||
380 | + num_histograms = max_histograms | ||
381 | + } | ||
382 | + | ||
383 | + if length == 0 { | ||
384 | + split.num_types = 1 | ||
385 | + return | ||
386 | + } else if length < kMinLengthForBlockSplitting { | ||
387 | + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) | ||
388 | + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) | ||
389 | + split.num_types = 1 | ||
390 | + split.types[split.num_blocks] = 0 | ||
391 | + split.lengths[split.num_blocks] = uint32(length) | ||
392 | + split.num_blocks++ | ||
393 | + return | ||
394 | + } | ||
395 | + | ||
396 | + histograms = make([]histogramDistance, num_histograms) | ||
397 | + | ||
398 | + /* Find good entropy codes. */ | ||
399 | + initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms) | ||
400 | + | ||
401 | + refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms) | ||
402 | + { | ||
403 | + var block_ids []byte = make([]byte, length) | ||
404 | + var num_blocks uint = 0 | ||
405 | + var bitmaplen uint = (num_histograms + 7) >> 3 | ||
406 | + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) | ||
407 | + var cost []float64 = make([]float64, num_histograms) | ||
408 | + var switch_signal []byte = make([]byte, (length * bitmaplen)) | ||
409 | + var new_id []uint16 = make([]uint16, num_histograms) | ||
410 | + var iters uint | ||
411 | + if params.quality < hqZopflificationQuality { | ||
412 | + iters = 3 | ||
413 | + } else { | ||
414 | + iters = 10 | ||
415 | + } | ||
416 | + /* Find a good path through literals with the good entropy codes. */ | ||
417 | + | ||
418 | + var i uint | ||
419 | + for i = 0; i < iters; i++ { | ||
420 | + num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) | ||
421 | + num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms) | ||
422 | + buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms) | ||
423 | + } | ||
424 | + | ||
425 | + insert_cost = nil | ||
426 | + cost = nil | ||
427 | + switch_signal = nil | ||
428 | + new_id = nil | ||
429 | + histograms = nil | ||
430 | + clusterBlocksDistance(data, length, num_blocks, block_ids, split) | ||
431 | + block_ids = nil | ||
432 | + } | ||
433 | +} |
1 | +package brotli | ||
2 | + | ||
3 | +import "math" | ||
4 | + | ||
5 | +/* Copyright 2013 Google Inc. All Rights Reserved. | ||
6 | + | ||
7 | + Distributed under MIT license. | ||
8 | + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT | ||
9 | +*/ | ||
10 | + | ||
11 | +func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { | ||
12 | + var seed uint32 = 7 | ||
13 | + var block_length uint = length / num_histograms | ||
14 | + var i uint | ||
15 | + clearHistogramsLiteral(histograms, num_histograms) | ||
16 | + for i = 0; i < num_histograms; i++ { | ||
17 | + var pos uint = length * i / num_histograms | ||
18 | + if i != 0 { | ||
19 | + pos += uint(myRand(&seed) % uint32(block_length)) | ||
20 | + } | ||
21 | + | ||
22 | + if pos+stride >= length { | ||
23 | + pos = length - stride - 1 | ||
24 | + } | ||
25 | + | ||
26 | + histogramAddVectorLiteral(&histograms[i], data[pos:], stride) | ||
27 | + } | ||
28 | +} | ||
29 | + | ||
30 | +func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) { | ||
31 | + var pos uint = 0 | ||
32 | + if stride >= length { | ||
33 | + stride = length | ||
34 | + } else { | ||
35 | + pos = uint(myRand(seed) % uint32(length-stride+1)) | ||
36 | + } | ||
37 | + | ||
38 | + histogramAddVectorLiteral(sample, data[pos:], stride) | ||
39 | +} | ||
40 | + | ||
41 | +func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { | ||
42 | + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining | ||
43 | + var seed uint32 = 7 | ||
44 | + var iter uint | ||
45 | + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms | ||
46 | + for iter = 0; iter < iters; iter++ { | ||
47 | + var sample histogramLiteral | ||
48 | + histogramClearLiteral(&sample) | ||
49 | + randomSampleLiteral(&seed, data, length, stride, &sample) | ||
50 | + histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample) | ||
51 | + } | ||
52 | +} | ||
53 | + | ||
54 | +/* Assigns a block id from the range [0, num_histograms) to each data element | ||
55 | + in data[0..length) and fills in block_id[0..length) with the assigned values. | ||
56 | + Returns the number of blocks, i.e. one plus the number of block switches. */ | ||
57 | +func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { | ||
58 | + var data_size uint = histogramDataSizeLiteral() | ||
59 | + var bitmaplen uint = (num_histograms + 7) >> 3 | ||
60 | + var num_blocks uint = 1 | ||
61 | + var i uint | ||
62 | + var j uint | ||
63 | + assert(num_histograms <= 256) | ||
64 | + if num_histograms <= 1 { | ||
65 | + for i = 0; i < length; i++ { | ||
66 | + block_id[i] = 0 | ||
67 | + } | ||
68 | + | ||
69 | + return 1 | ||
70 | + } | ||
71 | + | ||
72 | + for i := 0; i < int(data_size*num_histograms); i++ { | ||
73 | + insert_cost[i] = 0 | ||
74 | + } | ||
75 | + for i = 0; i < num_histograms; i++ { | ||
76 | + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) | ||
77 | + } | ||
78 | + | ||
79 | + for i = data_size; i != 0; { | ||
80 | + i-- | ||
81 | + for j = 0; j < num_histograms; j++ { | ||
82 | + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) | ||
83 | + } | ||
84 | + } | ||
85 | + | ||
86 | + for i := 0; i < int(num_histograms); i++ { | ||
87 | + cost[i] = 0 | ||
88 | + } | ||
89 | + for i := 0; i < int(length*bitmaplen); i++ { | ||
90 | + switch_signal[i] = 0 | ||
91 | + } | ||
92 | + | ||
93 | + /* After each iteration of this loop, cost[k] will contain the difference | ||
94 | + between the minimum cost of arriving at the current byte position using | ||
95 | + entropy code k, and the minimum cost of arriving at the current byte | ||
96 | + position. This difference is capped at the block switch cost, and if it | ||
97 | + reaches block switch cost, it means that when we trace back from the last | ||
98 | + position, we need to switch here. */ | ||
99 | + for i = 0; i < length; i++ { | ||
100 | + var byte_ix uint = i | ||
101 | + var ix uint = byte_ix * bitmaplen | ||
102 | + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms | ||
103 | + var min_cost float64 = 1e99 | ||
104 | + var block_switch_cost float64 = block_switch_bitcost | ||
105 | + var k uint | ||
106 | + for k = 0; k < num_histograms; k++ { | ||
107 | + /* We are coding the symbol in data[byte_ix] with entropy code k. */ | ||
108 | + cost[k] += insert_cost[insert_cost_ix+k] | ||
109 | + | ||
110 | + if cost[k] < min_cost { | ||
111 | + min_cost = cost[k] | ||
112 | + block_id[byte_ix] = byte(k) | ||
113 | + } | ||
114 | + } | ||
115 | + | ||
116 | + /* More blocks for the beginning. */ | ||
117 | + if byte_ix < 2000 { | ||
118 | + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 | ||
119 | + } | ||
120 | + | ||
121 | + for k = 0; k < num_histograms; k++ { | ||
122 | + cost[k] -= min_cost | ||
123 | + if cost[k] >= block_switch_cost { | ||
124 | + var mask byte = byte(1 << (k & 7)) | ||
125 | + cost[k] = block_switch_cost | ||
126 | + assert(k>>3 < bitmaplen) | ||
127 | + switch_signal[ix+(k>>3)] |= mask | ||
128 | + /* Trace back from the last position and switch at the marked places. */ | ||
129 | + } | ||
130 | + } | ||
131 | + } | ||
132 | + { | ||
133 | + var byte_ix uint = length - 1 | ||
134 | + var ix uint = byte_ix * bitmaplen | ||
135 | + var cur_id byte = block_id[byte_ix] | ||
136 | + for byte_ix > 0 { | ||
137 | + var mask byte = byte(1 << (cur_id & 7)) | ||
138 | + assert(uint(cur_id)>>3 < bitmaplen) | ||
139 | + byte_ix-- | ||
140 | + ix -= bitmaplen | ||
141 | + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { | ||
142 | + if cur_id != block_id[byte_ix] { | ||
143 | + cur_id = block_id[byte_ix] | ||
144 | + num_blocks++ | ||
145 | + } | ||
146 | + } | ||
147 | + | ||
148 | + block_id[byte_ix] = cur_id | ||
149 | + } | ||
150 | + } | ||
151 | + | ||
152 | + return num_blocks | ||
153 | +} | ||
154 | + | ||
155 | +var remapBlockIdsLiteral_kInvalidId uint16 = 256 | ||
156 | + | ||
157 | +func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { | ||
158 | + var next_id uint16 = 0 | ||
159 | + var i uint | ||
160 | + for i = 0; i < num_histograms; i++ { | ||
161 | + new_id[i] = remapBlockIdsLiteral_kInvalidId | ||
162 | + } | ||
163 | + | ||
164 | + for i = 0; i < length; i++ { | ||
165 | + assert(uint(block_ids[i]) < num_histograms) | ||
166 | + if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId { | ||
167 | + new_id[block_ids[i]] = next_id | ||
168 | + next_id++ | ||
169 | + } | ||
170 | + } | ||
171 | + | ||
172 | + for i = 0; i < length; i++ { | ||
173 | + block_ids[i] = byte(new_id[block_ids[i]]) | ||
174 | + assert(uint(block_ids[i]) < num_histograms) | ||
175 | + } | ||
176 | + | ||
177 | + assert(uint(next_id) <= num_histograms) | ||
178 | + return uint(next_id) | ||
179 | +} | ||
180 | + | ||
181 | +func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) { | ||
182 | + var i uint | ||
183 | + clearHistogramsLiteral(histograms, num_histograms) | ||
184 | + for i = 0; i < length; i++ { | ||
185 | + histogramAddLiteral(&histograms[block_ids[i]], uint(data[i])) | ||
186 | + } | ||
187 | +} | ||
188 | + | ||
189 | +var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32 | ||
190 | + | ||
191 | +func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { | ||
192 | + var histogram_symbols []uint32 = make([]uint32, num_blocks) | ||
193 | + var block_lengths []uint32 = make([]uint32, num_blocks) | ||
194 | + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch | ||
195 | + var all_histograms_size uint = 0 | ||
196 | + var all_histograms_capacity uint = expected_num_clusters | ||
197 | + var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity) | ||
198 | + var cluster_size_size uint = 0 | ||
199 | + var cluster_size_capacity uint = expected_num_clusters | ||
200 | + var cluster_size []uint32 = make([]uint32, cluster_size_capacity) | ||
201 | + var num_clusters uint = 0 | ||
202 | + var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch)) | ||
203 | + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 | ||
204 | + var pairs_capacity uint = max_num_pairs + 1 | ||
205 | + var pairs []histogramPair = make([]histogramPair, pairs_capacity) | ||
206 | + var pos uint = 0 | ||
207 | + var clusters []uint32 | ||
208 | + var num_final_clusters uint | ||
209 | + var new_index []uint32 | ||
210 | + var i uint | ||
211 | + var sizes = [histogramsPerBatch]uint32{0} | ||
212 | + var new_clusters = [histogramsPerBatch]uint32{0} | ||
213 | + var symbols = [histogramsPerBatch]uint32{0} | ||
214 | + var remap = [histogramsPerBatch]uint32{0} | ||
215 | + | ||
216 | + for i := 0; i < int(num_blocks); i++ { | ||
217 | + block_lengths[i] = 0 | ||
218 | + } | ||
219 | + { | ||
220 | + var block_idx uint = 0 | ||
221 | + for i = 0; i < length; i++ { | ||
222 | + assert(block_idx < num_blocks) | ||
223 | + block_lengths[block_idx]++ | ||
224 | + if i+1 == length || block_ids[i] != block_ids[i+1] { | ||
225 | + block_idx++ | ||
226 | + } | ||
227 | + } | ||
228 | + | ||
229 | + assert(block_idx == num_blocks) | ||
230 | + } | ||
231 | + | ||
232 | + for i = 0; i < num_blocks; i += histogramsPerBatch { | ||
233 | + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) | ||
234 | + var num_new_clusters uint | ||
235 | + var j uint | ||
236 | + for j = 0; j < num_to_combine; j++ { | ||
237 | + var k uint | ||
238 | + histogramClearLiteral(&histograms[j]) | ||
239 | + for k = 0; uint32(k) < block_lengths[i+j]; k++ { | ||
240 | + histogramAddLiteral(&histograms[j], uint(data[pos])) | ||
241 | + pos++ | ||
242 | + } | ||
243 | + | ||
244 | + histograms[j].bit_cost_ = populationCostLiteral(&histograms[j]) | ||
245 | + new_clusters[j] = uint32(j) | ||
246 | + symbols[j] = uint32(j) | ||
247 | + sizes[j] = 1 | ||
248 | + } | ||
249 | + | ||
250 | + num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) | ||
251 | + if all_histograms_capacity < (all_histograms_size + num_new_clusters) { | ||
252 | + var _new_size uint | ||
253 | + if all_histograms_capacity == 0 { | ||
254 | + _new_size = all_histograms_size + num_new_clusters | ||
255 | + } else { | ||
256 | + _new_size = all_histograms_capacity | ||
257 | + } | ||
258 | + var new_array []histogramLiteral | ||
259 | + for _new_size < (all_histograms_size + num_new_clusters) { | ||
260 | + _new_size *= 2 | ||
261 | + } | ||
262 | + new_array = make([]histogramLiteral, _new_size) | ||
263 | + if all_histograms_capacity != 0 { | ||
264 | + copy(new_array, all_histograms[:all_histograms_capacity]) | ||
265 | + } | ||
266 | + | ||
267 | + all_histograms = new_array | ||
268 | + all_histograms_capacity = _new_size | ||
269 | + } | ||
270 | + | ||
271 | + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) | ||
272 | + for j = 0; j < num_new_clusters; j++ { | ||
273 | + all_histograms[all_histograms_size] = histograms[new_clusters[j]] | ||
274 | + all_histograms_size++ | ||
275 | + cluster_size[cluster_size_size] = sizes[new_clusters[j]] | ||
276 | + cluster_size_size++ | ||
277 | + remap[new_clusters[j]] = uint32(j) | ||
278 | + } | ||
279 | + | ||
280 | + for j = 0; j < num_to_combine; j++ { | ||
281 | + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] | ||
282 | + } | ||
283 | + | ||
284 | + num_clusters += num_new_clusters | ||
285 | + assert(num_clusters == cluster_size_size) | ||
286 | + assert(num_clusters == all_histograms_size) | ||
287 | + } | ||
288 | + | ||
289 | + histograms = nil | ||
290 | + | ||
291 | + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) | ||
292 | + if pairs_capacity < max_num_pairs+1 { | ||
293 | + pairs = nil | ||
294 | + pairs = make([]histogramPair, (max_num_pairs + 1)) | ||
295 | + } | ||
296 | + | ||
297 | + clusters = make([]uint32, num_clusters) | ||
298 | + for i = 0; i < num_clusters; i++ { | ||
299 | + clusters[i] = uint32(i) | ||
300 | + } | ||
301 | + | ||
302 | + num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) | ||
303 | + pairs = nil | ||
304 | + cluster_size = nil | ||
305 | + | ||
306 | + new_index = make([]uint32, num_clusters) | ||
307 | + for i = 0; i < num_clusters; i++ { | ||
308 | + new_index[i] = clusterBlocksLiteral_kInvalidIndex | ||
309 | + } | ||
310 | + pos = 0 | ||
311 | + { | ||
312 | + var next_index uint32 = 0 | ||
313 | + for i = 0; i < num_blocks; i++ { | ||
314 | + var histo histogramLiteral | ||
315 | + var j uint | ||
316 | + var best_out uint32 | ||
317 | + var best_bits float64 | ||
318 | + histogramClearLiteral(&histo) | ||
319 | + for j = 0; uint32(j) < block_lengths[i]; j++ { | ||
320 | + histogramAddLiteral(&histo, uint(data[pos])) | ||
321 | + pos++ | ||
322 | + } | ||
323 | + | ||
324 | + if i == 0 { | ||
325 | + best_out = histogram_symbols[0] | ||
326 | + } else { | ||
327 | + best_out = histogram_symbols[i-1] | ||
328 | + } | ||
329 | + best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out]) | ||
330 | + for j = 0; j < num_final_clusters; j++ { | ||
331 | + var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]]) | ||
332 | + if cur_bits < best_bits { | ||
333 | + best_bits = cur_bits | ||
334 | + best_out = clusters[j] | ||
335 | + } | ||
336 | + } | ||
337 | + | ||
338 | + histogram_symbols[i] = best_out | ||
339 | + if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex { | ||
340 | + new_index[best_out] = next_index | ||
341 | + next_index++ | ||
342 | + } | ||
343 | + } | ||
344 | + } | ||
345 | + | ||
346 | + clusters = nil | ||
347 | + all_histograms = nil | ||
348 | + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) | ||
349 | + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) | ||
350 | + { | ||
351 | + var cur_length uint32 = 0 | ||
352 | + var block_idx uint = 0 | ||
353 | + var max_type byte = 0 | ||
354 | + for i = 0; i < num_blocks; i++ { | ||
355 | + cur_length += block_lengths[i] | ||
356 | + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { | ||
357 | + var id byte = byte(new_index[histogram_symbols[i]]) | ||
358 | + split.types[block_idx] = id | ||
359 | + split.lengths[block_idx] = cur_length | ||
360 | + max_type = brotli_max_uint8_t(max_type, id) | ||
361 | + cur_length = 0 | ||
362 | + block_idx++ | ||
363 | + } | ||
364 | + } | ||
365 | + | ||
366 | + split.num_blocks = block_idx | ||
367 | + split.num_types = uint(max_type) + 1 | ||
368 | + } | ||
369 | + | ||
370 | + new_index = nil | ||
371 | + block_lengths = nil | ||
372 | + histogram_symbols = nil | ||
373 | +} | ||
374 | + | ||
375 | +func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { | ||
376 | + var data_size uint = histogramDataSizeLiteral() | ||
377 | + var num_histograms uint = length/literals_per_histogram + 1 | ||
378 | + var histograms []histogramLiteral | ||
379 | + if num_histograms > max_histograms { | ||
380 | + num_histograms = max_histograms | ||
381 | + } | ||
382 | + | ||
383 | + if length == 0 { | ||
384 | + split.num_types = 1 | ||
385 | + return | ||
386 | + } else if length < kMinLengthForBlockSplitting { | ||
387 | + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) | ||
388 | + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) | ||
389 | + split.num_types = 1 | ||
390 | + split.types[split.num_blocks] = 0 | ||
391 | + split.lengths[split.num_blocks] = uint32(length) | ||
392 | + split.num_blocks++ | ||
393 | + return | ||
394 | + } | ||
395 | + | ||
396 | + histograms = make([]histogramLiteral, num_histograms) | ||
397 | + | ||
398 | + /* Find good entropy codes. */ | ||
399 | + initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms) | ||
400 | + | ||
401 | + refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms) | ||
402 | + { | ||
403 | + var block_ids []byte = make([]byte, length) | ||
404 | + var num_blocks uint = 0 | ||
405 | + var bitmaplen uint = (num_histograms + 7) >> 3 | ||
406 | + var insert_cost []float64 = make([]float64, (data_size * num_histograms)) | ||
407 | + var cost []float64 = make([]float64, num_histograms) | ||
408 | + var switch_signal []byte = make([]byte, (length * bitmaplen)) | ||
409 | + var new_id []uint16 = make([]uint16, num_histograms) | ||
410 | + var iters uint | ||
411 | + if params.quality < hqZopflificationQuality { | ||
412 | + iters = 3 | ||
413 | + } else { | ||
414 | + iters = 10 | ||
415 | + } | ||
416 | + /* Find a good path through literals with the good entropy codes. */ | ||
417 | + | ||
418 | + var i uint | ||
419 | + for i = 0; i < iters; i++ { | ||
420 | + num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) | ||
421 | + num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms) | ||
422 | + buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms) | ||
423 | + } | ||
424 | + | ||
425 | + insert_cost = nil | ||
426 | + cost = nil | ||
427 | + switch_signal = nil | ||
428 | + new_id = nil | ||
429 | + histograms = nil | ||
430 | + clusterBlocksLiteral(data, length, num_blocks, block_ids, split) | ||
431 | + block_ids = nil | ||
432 | + } | ||
433 | +} |
-
请 注册 或 登录 后发表评论