作者 tangxvhui

更新

要显示太多修改。

为保证性能只显示 23 of 23+ 个文件。

@@ -22,6 +22,6 @@ @@ -22,6 +22,6 @@
22 *.sum 22 *.sum
23 23
24 24
25 -/vendor 25 +
26 /*.exe~ 26 /*.exe~
27 /logs 27 /logs
@@ -10,6 +10,6 @@ RUN ["ln","-sf","/usr/share/zoneinfo/Asia/Shanghai","/etc/localtime"] @@ -10,6 +10,6 @@ RUN ["ln","-sf","/usr/share/zoneinfo/Asia/Shanghai","/etc/localtime"]
10 ENV GO111MODULE on 10 ENV GO111MODULE on
11 ENV GOPROXY https://goproxy.cn 11 ENV GOPROXY https://goproxy.cn
12 RUN ["go","mod","tidy"] 12 RUN ["go","mod","tidy"]
13 -RUN ["go","build"] 13 +RUN ["go","build","-mod=vendor"]
14 EXPOSE 8082 14 EXPOSE 8082
15 ENTRYPOINT ["./partnermg"] 15 ENTRYPOINT ["./partnermg"]
@@ -160,7 +160,7 @@ type postPurposeOrderDetail struct { @@ -160,7 +160,7 @@ type postPurposeOrderDetail struct {
160 //订单编号 160 //订单编号
161 OrderId string `json:"orderId"` 161 OrderId string `json:"orderId"`
162 //买家姓名 162 //买家姓名
163 - BuyerName string `json:"buyerName"` 163 + BuyerName string `json:"buyer"`
164 //对应合伙人 id 164 //对应合伙人 id
165 PartnerId int64 `json:"partnerID"` 165 PartnerId int64 `json:"partnerID"`
166 PartnerName string `json:"partnerName"` 166 PartnerName string `json:"partnerName"`
  1 +package geetest
  2 +
  3 +import (
  4 + "crypto/md5"
  5 + "encoding/hex"
  6 + "encoding/json"
  7 + "errors"
  8 + "io/ioutil"
  9 + "net/http"
  10 + "net/url"
  11 + "strings"
  12 + "time"
  13 +)
  14 +
  15 +type GeetestLib struct {
  16 + CaptchaID string
  17 + PrivateKey string
  18 + Client *http.Client
  19 +}
  20 +
  21 +type FailbackRegisterRespnse struct {
  22 + Success int `json:"success"`
  23 + GT string `json:"gt"`
  24 + Challenge string `json:"challenge"`
  25 + NewCaptcha int `json:"new_captcha"`
  26 +}
  27 +
  28 +const (
  29 + geetestHost = "http://api.geetest.com"
  30 + registerURL = geetestHost + "/register.php"
  31 + validateURL = geetestHost + "/validate.php"
  32 +)
  33 +
  34 +func MD5Encode(input string) string {
  35 + md5Instant := md5.New()
  36 + md5Instant.Write([]byte(input))
  37 + return hex.EncodeToString(md5Instant.Sum(nil))
  38 +}
  39 +
  40 +// 初始化 GeetestLib
  41 +func NewGeetestLib(capthcaID string, privateKey string, timeOut time.Duration) (geetest GeetestLib){
  42 + client := &http.Client{Timeout: timeOut}
  43 + geetest = GeetestLib{capthcaID, privateKey, client}
  44 + return
  45 +}
  46 +
  47 +func (g *GeetestLib) getFailBackRegisterResponse(success int, challenge string) []byte {
  48 + if challenge == "" {
  49 + challenge = hex.EncodeToString(md5.New().Sum(nil))
  50 + }
  51 +
  52 + response := FailbackRegisterRespnse{
  53 + success,
  54 + g.CaptchaID,
  55 + challenge,
  56 + 1,
  57 + }
  58 + res, _ := json.Marshal(response)
  59 + return res
  60 +}
  61 +
  62 +func (g *GeetestLib) do(req *http.Request) (body []byte, err error) {
  63 + req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
  64 + var resp *http.Response
  65 + if resp, err = g.Client.Do(req); err != nil {
  66 + return
  67 + }
  68 + defer resp.Body.Close()
  69 + if resp.StatusCode >= http.StatusInternalServerError {
  70 + err = errors.New("http status code 5xx")
  71 + return
  72 + }
  73 +
  74 + if body, err = ioutil.ReadAll(resp.Body); err != nil {
  75 + return
  76 + }
  77 + return
  78 +}
  79 +
  80 +func (g *GeetestLib) PreProcess(userID string, userIP string) (int8, []byte) {
  81 + params := url.Values{}
  82 + params.Add("gt", g.CaptchaID)
  83 + params.Add("new_captcha", "1")
  84 + if userID != "" {
  85 + params.Add("user_id", userID)
  86 + }
  87 + if userIP != "" {
  88 + params.Add("ip_adress", userIP)
  89 + }
  90 + req, _ := http.NewRequest("GET", registerURL+"?"+params.Encode(), nil)
  91 + body, err := g.do(req)
  92 + if err != nil {
  93 + return 0, g.getFailBackRegisterResponse(0, "")
  94 + }
  95 + challenge := string(body)
  96 + if len(challenge) != 32 {
  97 + return 0, g.getFailBackRegisterResponse(0, "")
  98 + } else {
  99 + challenge = MD5Encode(challenge + g.PrivateKey)
  100 + return 1, g.getFailBackRegisterResponse(1, challenge)
  101 + }
  102 +}
  103 +
  104 +func (g *GeetestLib) checkParas(challenge string, validate string, seccode string) bool {
  105 + if challenge == "" || validate == "" || seccode == "" {
  106 + return false
  107 + }
  108 + return true
  109 +}
  110 +
  111 +func (g *GeetestLib) checkSuccessRes(challenge string, validate string) bool {
  112 + return MD5Encode(g.PrivateKey+"geetest"+challenge) == validate
  113 +}
  114 +
  115 +func (g *GeetestLib) checkFailbackRes(challenge string, validate string) bool {
  116 + return MD5Encode(challenge) == validate
  117 +}
  118 +
  119 +func (g *GeetestLib) SuccessValidate(challenge string, validate string, seccode string, userID string, userIP string) bool {
  120 + if !g.checkParas(challenge, validate, seccode) {
  121 + return false
  122 + }
  123 + if !g.checkSuccessRes(challenge, validate) {
  124 + return false
  125 + }
  126 + params := url.Values{}
  127 + params.Add("seccode", seccode)
  128 + params.Add("challenge", challenge)
  129 + params.Add("captchaid", g.CaptchaID)
  130 + params.Add("sdk", "golang_v1.0.0")
  131 + if userID != "" {
  132 + params.Add("user_id", userID)
  133 + }
  134 + if userIP != "" {
  135 + params.Add("ip_adress", userIP)
  136 + }
  137 + req, _ := http.NewRequest("POST", validateURL, strings.NewReader(params.Encode()))
  138 + body, err := g.do(req)
  139 + if err != nil {
  140 + return false
  141 + }
  142 + res := string(body)
  143 + return res == MD5Encode(seccode)
  144 +}
  145 +
  146 +func (g *GeetestLib) FailbackValidate(challenge string, validate string, seccode string) bool {
  147 + if !g.checkParas(challenge, validate, seccode) {
  148 + return false
  149 + }
  150 + if !g.checkFailbackRes(challenge, validate) {
  151 + return false
  152 + }
  153 + return true
  154 +}
  1 +## Copyright 2014 Alvaro J. Genial. All rights reserved.
  2 +## Use of this source code is governed by a BSD-style
  3 +## license that can be found in the LICENSE file.
  4 +
  5 +language: go
  6 +
  7 +go:
  8 + - tip
  9 + - 1.6
  10 + - 1.5
  11 + - 1.4
  12 + - 1.3
  13 + # 1.2
  14 +
  15 +before_install:
  16 + # - go get -v golang.org/x/tools/cmd/cover
  17 + # - go get -v golang.org/x/tools/cmd/vet
  18 + # - go get -v golang.org/x/lint/golint
  19 + - export PATH=$PATH:/home/travis/gopath/bin
  20 +
  21 +script:
  22 + - go build -v ./...
  23 + - go test -v -cover ./...
  24 + - go vet ./...
  25 + # - golint .
  1 +Copyright (c) 2014 Alvaro J. Genial. All rights reserved.
  2 +
  3 +Redistribution and use in source and binary forms, with or without
  4 +modification, are permitted provided that the following conditions are
  5 +met:
  6 +
  7 + * Redistributions of source code must retain the above copyright
  8 +notice, this list of conditions and the following disclaimer.
  9 + * Redistributions in binary form must reproduce the above
  10 +copyright notice, this list of conditions and the following disclaimer
  11 +in the documentation and/or other materials provided with the
  12 +distribution.
  13 + * Neither the name of Google Inc. nor the names of its
  14 +contributors may be used to endorse or promote products derived from
  15 +this software without specific prior written permission.
  16 +
  17 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  1 +form
  2 +====
  3 +
  4 +A Form Encoding & Decoding Package for Go, written by [Alvaro J. Genial](http://alva.ro).
  5 +
  6 +[![Build Status](https://travis-ci.org/ajg/form.png?branch=master)](https://travis-ci.org/ajg/form)
  7 +[![GoDoc](https://godoc.org/github.com/ajg/form?status.png)](https://godoc.org/github.com/ajg/form)
  8 +
  9 +Synopsis
  10 +--------
  11 +
  12 +This library is designed to allow seamless, high-fidelity encoding and decoding of arbitrary data in `application/x-www-form-urlencoded` format and as [`url.Values`](http://golang.org/pkg/net/url/#Values). It is intended to be useful primarily in dealing with web forms and URI query strings, both of which natively employ said format.
  13 +
  14 +Unsurprisingly, `form` is modeled after other Go [`encoding`](http://golang.org/pkg/encoding/) packages, in particular [`encoding/json`](http://golang.org/pkg/encoding/json/), and follows the same conventions (see below for more.) It aims to automatically handle any kind of concrete Go [data value](#values) (i.e., not functions, channels, etc.) while providing mechanisms for custom behavior.
  15 +
  16 +Status
  17 +------
  18 +
  19 +The implementation is in usable shape and is fairly well tested with its accompanying test suite. The API is unlikely to change much, but still may. Lastly, the code has not yet undergone a security review to ensure it is free of vulnerabilities. Please file an issue or send a pull request for fixes & improvements.
  20 +
  21 +Dependencies
  22 +------------
  23 +
  24 +The only requirement is [Go 1.2](http://golang.org/doc/go1.2) or later.
  25 +
  26 +Usage
  27 +-----
  28 +
  29 +```go
  30 +import "github.com/ajg/form"
  31 +// or: "gopkg.in/ajg/form.v1"
  32 +```
  33 +
  34 +Given a type like the following...
  35 +
  36 +```go
  37 +type User struct {
  38 + Name string `form:"name"`
  39 + Email string `form:"email"`
  40 + Joined time.Time `form:"joined,omitempty"`
  41 + Posts []int `form:"posts"`
  42 + Preferences map[string]string `form:"prefs"`
  43 + Avatar []byte `form:"avatar"`
  44 + PasswordHash int64 `form:"-"`
  45 +}
  46 +```
  47 +
  48 +...it is easy to encode data of that type...
  49 +
  50 +
  51 +```go
  52 +func PostUser(url string, u User) error {
  53 + var c http.Client
  54 + _, err := c.PostForm(url, form.EncodeToValues(u))
  55 + return err
  56 +}
  57 +```
  58 +
  59 +...as well as decode it...
  60 +
  61 +
  62 +```go
  63 +func Handler(w http.ResponseWriter, r *http.Request) {
  64 + var u User
  65 +
  66 + d := form.NewDecoder(r.Body)
  67 + if err := d.Decode(&u); err != nil {
  68 + http.Error(w, "Form could not be decoded", http.StatusBadRequest)
  69 + return
  70 + }
  71 +
  72 + fmt.Fprintf(w, "Decoded: %#v", u)
  73 +}
  74 +```
  75 +
  76 +...without having to do any grunt work.
  77 +
  78 +Field Tags
  79 +----------
  80 +
  81 +Like other encoding packages, `form` supports the following options for fields:
  82 +
  83 + - `` `form:"-"` ``: Causes the field to be ignored during encoding and decoding.
  84 + - `` `form:"<name>"` ``: Overrides the field's name; useful especially when dealing with external identifiers in camelCase, as are commonly found on the web.
  85 + - `` `form:",omitempty"` ``: Elides the field during encoding if it is empty (typically meaning equal to the type's zero value.)
  86 + - `` `form:"<name>,omitempty"` ``: The way to combine the two options above.
  87 +
  88 +Values
  89 +------
  90 +
  91 +### Simple Values
  92 +
  93 +Values of the following types are all considered simple:
  94 +
  95 + - `bool`
  96 + - `int`, `int8`, `int16`, `int32`, `int64`, `rune`
  97 + - `uint`, `uint8`, `uint16`, `uint32`, `uint64`, `byte`
  98 + - `float32`, `float64`
  99 + - `complex64`, `complex128`
  100 + - `string`
  101 + - `[]byte` (see note)
  102 + - [`time.Time`](http://golang.org/pkg/time/#Time)
  103 + - [`url.URL`](http://golang.org/pkg/net/url/#URL)
  104 + - An alias of any of the above
  105 + - A pointer to any of the above
  106 +
  107 +### Composite Values
  108 +
  109 +A composite value is one that can contain other values. Values of the following kinds...
  110 +
  111 + - Maps
  112 + - Slices; except `[]byte` (see note)
  113 + - Structs; except [`time.Time`](http://golang.org/pkg/time/#Time) and [`url.URL`](http://golang.org/pkg/net/url/#URL)
  114 + - Arrays
  115 + - An alias of any of the above
  116 + - A pointer to any of the above
  117 +
  118 +...are considered composites in general, unless they implement custom marshaling/unmarshaling. Composite values are encoded as a flat mapping of paths to values, where the paths are constructed by joining the parent and child paths with a period (`.`).
  119 +
  120 +(Note: a byte slice is treated as a `string` by default because it's more efficient, but can also be decoded as a slice—i.e., with indexes.)
  121 +
  122 +### Untyped Values
  123 +
  124 +While encouraged, it is not necessary to define a type (e.g. a `struct`) in order to use `form`, since it is able to encode and decode untyped data generically using the following rules:
  125 +
  126 + - Simple values will be treated as a `string`.
  127 + - Composite values will be treated as a `map[string]interface{}`, itself able to contain nested values (both scalar and compound) ad infinitum.
  128 + - However, if there is a value (of any supported type) already present in a map for a given key, then it will be used when possible, rather than being replaced with a generic value as specified above; this makes it possible to handle partially typed, dynamic or schema-less values.
  129 +
  130 +### Zero Values
  131 +
  132 +By default, and without custom marshaling, zero values (also known as empty/default values) are encoded as the empty string. To disable this behavior, meaning to keep zero values in their literal form (e.g. `0` for integral types), `Encoder` offers a `KeepZeros` setter method, which will do just that when set to `true`.
  133 +
  134 +### Unsupported Values
  135 +
  136 +Values of the following kinds aren't supported and, if present, must be ignored.
  137 +
  138 + - Channel
  139 + - Function
  140 + - Unsafe pointer
  141 + - An alias of any of the above
  142 + - A pointer to any of the above
  143 +
  144 +Custom Marshaling
  145 +-----------------
  146 +
  147 +There is a default (generally lossless) marshaling & unmarshaling scheme for any concrete data value in Go, which is good enough in most cases. However, it is possible to override it and use a custom scheme. For instance, a "binary" field could be marshaled more efficiently using [base64](http://golang.org/pkg/encoding/base64/) to prevent it from being percent-escaped during serialization to `application/x-www-form-urlencoded` format.
  148 +
  149 +Because `form` provides support for [`encoding.TextMarshaler`](http://golang.org/pkg/encoding/#TextMarshaler) and [`encoding.TextUnmarshaler`](http://golang.org/pkg/encoding/#TextUnmarshaler) it is easy to do that; for instance, like this:
  150 +
  151 +```go
  152 +import "encoding"
  153 +
  154 +type Binary []byte
  155 +
  156 +var (
  157 + _ encoding.TextMarshaler = &Binary{}
  158 + _ encoding.TextUnmarshaler = &Binary{}
  159 +)
  160 +
  161 +func (b Binary) MarshalText() ([]byte, error) {
  162 + return []byte(base64.URLEncoding.EncodeToString([]byte(b))), nil
  163 +}
  164 +
  165 +func (b *Binary) UnmarshalText(text []byte) error {
  166 + bs, err := base64.URLEncoding.DecodeString(string(text))
  167 + if err == nil {
  168 + *b = Binary(bs)
  169 + }
  170 + return err
  171 +}
  172 +```
  173 +
  174 +Now any value with type `Binary` will automatically be encoded using the [URL](http://golang.org/pkg/encoding/base64/#URLEncoding) variant of base64. It is left as an exercise to the reader to improve upon this scheme by eliminating the need for padding (which, besides being superfluous, uses `=`, a character that will end up percent-escaped.)
  175 +
  176 +Keys
  177 +----
  178 +
  179 +In theory any value can be a key as long as it has a string representation. However, by default, periods have special meaning to `form`, and thus, under the hood (i.e. in encoded form) they are transparently escaped using a preceding backslash (`\`). Backslashes within keys, themselves, are also escaped in this manner (e.g. as `\\`) in order to permit representing `\.` itself (as `\\\.`).
  180 +
  181 +(Note: it is normally unnecessary to deal with this issue unless keys are being constructed manually—e.g. literally embedded in HTML or in a URI.)
  182 +
  183 +The default delimiter and escape characters used for encoding and decoding composite keys can be changed using the `DelimitWith` and `EscapeWith` setter methods of `Encoder` and `Decoder`, respectively. For example...
  184 +
  185 +```go
  186 +package main
  187 +
  188 +import (
  189 + "os"
  190 +
  191 + "github.com/ajg/form"
  192 +)
  193 +
  194 +func main() {
  195 + type B struct {
  196 + Qux string `form:"qux"`
  197 + }
  198 + type A struct {
  199 + FooBar B `form:"foo.bar"`
  200 + }
  201 + a := A{FooBar: B{"XYZ"}}
  202 + os.Stdout.WriteString("Default: ")
  203 + form.NewEncoder(os.Stdout).Encode(a)
  204 + os.Stdout.WriteString("\nCustom: ")
  205 + form.NewEncoder(os.Stdout).DelimitWith('/').Encode(a)
  206 + os.Stdout.WriteString("\n")
  207 +}
  208 +
  209 +```
  210 +
  211 +...will produce...
  212 +
  213 +```
  214 +Default: foo%5C.bar.qux=XYZ
  215 +Custom: foo.bar%2Fqux=XYZ
  216 +```
  217 +
  218 +(`%5C` and `%2F` represent `\` and `/`, respectively.)
  219 +
  220 +Limitations
  221 +-----------
  222 +
  223 + - Circular (self-referential) values are untested.
  224 +
  225 +Future Work
  226 +-----------
  227 +
  228 +The following items would be nice to have in the future—though they are not being worked on yet:
  229 +
  230 + - An option to treat all values as if they had been tagged with `omitempty`.
  231 + - An option to automatically treat all field names in `camelCase` or `underscore_case`.
  232 + - Built-in support for the types in [`math/big`](http://golang.org/pkg/math/big/).
  233 + - Built-in support for the types in [`image/color`](http://golang.org/pkg/image/color/).
  234 + - Improve encoding/decoding by reading/writing directly from/to the `io.Reader`/`io.Writer` when possible, rather than going through an intermediate representation (i.e. `node`) which requires more memory.
  235 +
  236 +(Feel free to implement any of these and then send a pull request.)
  237 +
  238 +Related Work
  239 +------------
  240 +
  241 + - Package [gorilla/schema](https://github.com/gorilla/schema), which only implements decoding.
  242 + - Package [google/go-querystring](https://github.com/google/go-querystring), which only implements encoding.
  243 +
  244 +License
  245 +-------
  246 +
  247 +This library is distributed under a BSD-style [LICENSE](./LICENSE).
  1 +TODO
  2 +====
  3 +
  4 + - Document IgnoreCase and IgnoreUnknownKeys in README.
  1 +// Copyright 2014 Alvaro J. Genial. All rights reserved.
  2 +// Use of this source code is governed by a BSD-style
  3 +// license that can be found in the LICENSE file.
  4 +
  5 +package form
  6 +
  7 +import (
  8 + "fmt"
  9 + "io"
  10 + "io/ioutil"
  11 + "net/url"
  12 + "reflect"
  13 + "strconv"
  14 + "time"
  15 +)
  16 +
  17 +// NewDecoder returns a new form Decoder.
  18 +func NewDecoder(r io.Reader) *Decoder {
  19 + return &Decoder{r, defaultDelimiter, defaultEscape, false, false}
  20 +}
  21 +
  22 +// Decoder decodes data from a form (application/x-www-form-urlencoded).
  23 +type Decoder struct {
  24 + r io.Reader
  25 + d rune
  26 + e rune
  27 + ignoreUnknown bool
  28 + ignoreCase bool
  29 +}
  30 +
  31 +// DelimitWith sets r as the delimiter used for composite keys by Decoder d and returns the latter; it is '.' by default.
  32 +func (d *Decoder) DelimitWith(r rune) *Decoder {
  33 + d.d = r
  34 + return d
  35 +}
  36 +
  37 +// EscapeWith sets r as the escape used for delimiters (and to escape itself) by Decoder d and returns the latter; it is '\\' by default.
  38 +func (d *Decoder) EscapeWith(r rune) *Decoder {
  39 + d.e = r
  40 + return d
  41 +}
  42 +
  43 +// Decode reads in and decodes form-encoded data into dst.
  44 +func (d Decoder) Decode(dst interface{}) error {
  45 + bs, err := ioutil.ReadAll(d.r)
  46 + if err != nil {
  47 + return err
  48 + }
  49 + vs, err := url.ParseQuery(string(bs))
  50 + if err != nil {
  51 + return err
  52 + }
  53 + v := reflect.ValueOf(dst)
  54 + return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v)))
  55 +}
  56 +
  57 +// IgnoreUnknownKeys if set to true it will make the Decoder ignore values
  58 +// that are not found in the destination object instead of returning an error.
  59 +func (d *Decoder) IgnoreUnknownKeys(ignoreUnknown bool) {
  60 + d.ignoreUnknown = ignoreUnknown
  61 +}
  62 +
  63 +// IgnoreCase if set to true it will make the Decoder try to set values in the
  64 +// destination object even if the case does not match.
  65 +func (d *Decoder) IgnoreCase(ignoreCase bool) {
  66 + d.ignoreCase = ignoreCase
  67 +}
  68 +
  69 +// DecodeString decodes src into dst.
  70 +func (d Decoder) DecodeString(dst interface{}, src string) error {
  71 + vs, err := url.ParseQuery(src)
  72 + if err != nil {
  73 + return err
  74 + }
  75 + v := reflect.ValueOf(dst)
  76 + return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v)))
  77 +}
  78 +
  79 +// DecodeValues decodes vs into dst.
  80 +func (d Decoder) DecodeValues(dst interface{}, vs url.Values) error {
  81 + v := reflect.ValueOf(dst)
  82 + return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v)))
  83 +}
  84 +
  85 +// DecodeString decodes src into dst.
  86 +func DecodeString(dst interface{}, src string) error {
  87 + return NewDecoder(nil).DecodeString(dst, src)
  88 +}
  89 +
  90 +// DecodeValues decodes vs into dst.
  91 +func DecodeValues(dst interface{}, vs url.Values) error {
  92 + return NewDecoder(nil).DecodeValues(dst, vs)
  93 +}
  94 +
  95 +func (d Decoder) decodeNode(v reflect.Value, n node) (err error) {
  96 + defer func() {
  97 + if e := recover(); e != nil {
  98 + err = fmt.Errorf("%v", e)
  99 + }
  100 + }()
  101 +
  102 + if v.Kind() == reflect.Slice {
  103 + return fmt.Errorf("could not decode directly into slice; use pointer to slice")
  104 + }
  105 + d.decodeValue(v, n)
  106 + return nil
  107 +}
  108 +
  109 +func (d Decoder) decodeValue(v reflect.Value, x interface{}) {
  110 + t := v.Type()
  111 + k := v.Kind()
  112 +
  113 + if k == reflect.Ptr && v.IsNil() {
  114 + v.Set(reflect.New(t.Elem()))
  115 + }
  116 +
  117 + if unmarshalValue(v, x) {
  118 + return
  119 + }
  120 +
  121 + empty := isEmpty(x)
  122 +
  123 + switch k {
  124 + case reflect.Ptr:
  125 + d.decodeValue(v.Elem(), x)
  126 + return
  127 + case reflect.Interface:
  128 + if !v.IsNil() {
  129 + d.decodeValue(v.Elem(), x)
  130 + return
  131 +
  132 + } else if empty {
  133 + return // Allow nil interfaces only if empty.
  134 + } else {
  135 + panic("form: cannot decode non-empty value into into nil interface")
  136 + }
  137 + }
  138 +
  139 + if empty {
  140 + v.Set(reflect.Zero(t)) // Treat the empty string as the zero value.
  141 + return
  142 + }
  143 +
  144 + switch k {
  145 + case reflect.Struct:
  146 + if t.ConvertibleTo(timeType) {
  147 + d.decodeTime(v, x)
  148 + } else if t.ConvertibleTo(urlType) {
  149 + d.decodeURL(v, x)
  150 + } else {
  151 + d.decodeStruct(v, x)
  152 + }
  153 + case reflect.Slice:
  154 + d.decodeSlice(v, x)
  155 + case reflect.Array:
  156 + d.decodeArray(v, x)
  157 + case reflect.Map:
  158 + d.decodeMap(v, x)
  159 + case reflect.Invalid, reflect.Uintptr, reflect.UnsafePointer, reflect.Chan, reflect.Func:
  160 + panic(t.String() + " has unsupported kind " + k.String())
  161 + default:
  162 + d.decodeBasic(v, x)
  163 + }
  164 +}
  165 +
  166 +func (d Decoder) decodeStruct(v reflect.Value, x interface{}) {
  167 + t := v.Type()
  168 + for k, c := range getNode(x) {
  169 + if f, ok := findField(v, k, d.ignoreCase); !ok && k == "" {
  170 + panic(getString(x) + " cannot be decoded as " + t.String())
  171 + } else if !ok {
  172 + if !d.ignoreUnknown {
  173 + panic(k + " doesn't exist in " + t.String())
  174 + }
  175 + } else if !f.CanSet() {
  176 + panic(k + " cannot be set in " + t.String())
  177 + } else {
  178 + d.decodeValue(f, c)
  179 + }
  180 + }
  181 +}
  182 +
  183 +func (d Decoder) decodeMap(v reflect.Value, x interface{}) {
  184 + t := v.Type()
  185 + if v.IsNil() {
  186 + v.Set(reflect.MakeMap(t))
  187 + }
  188 + for k, c := range getNode(x) {
  189 + i := reflect.New(t.Key()).Elem()
  190 + d.decodeValue(i, k)
  191 +
  192 + w := v.MapIndex(i)
  193 + if w.IsValid() { // We have an actual element value to decode into.
  194 + if w.Kind() == reflect.Interface {
  195 + w = w.Elem()
  196 + }
  197 + w = reflect.New(w.Type()).Elem()
  198 + } else if t.Elem().Kind() != reflect.Interface { // The map's element type is concrete.
  199 + w = reflect.New(t.Elem()).Elem()
  200 + } else {
  201 + // The best we can do here is to decode as either a string (for scalars) or a map[string]interface {} (for the rest).
  202 + // We could try to guess the type based on the string (e.g. true/false => bool) but that'll get ugly fast,
  203 + // especially if we have to guess the kind (slice vs. array vs. map) and index type (e.g. string, int, etc.)
  204 + switch c.(type) {
  205 + case node:
  206 + w = reflect.MakeMap(stringMapType)
  207 + case string:
  208 + w = reflect.New(stringType).Elem()
  209 + default:
  210 + panic("value is neither node nor string")
  211 + }
  212 + }
  213 +
  214 + d.decodeValue(w, c)
  215 + v.SetMapIndex(i, w)
  216 + }
  217 +}
  218 +
  219 +func (d Decoder) decodeArray(v reflect.Value, x interface{}) {
  220 + t := v.Type()
  221 + for k, c := range getNode(x) {
  222 + i, err := strconv.Atoi(k)
  223 + if err != nil {
  224 + panic(k + " is not a valid index for type " + t.String())
  225 + }
  226 + if l := v.Len(); i >= l {
  227 + panic("index is above array size")
  228 + }
  229 + d.decodeValue(v.Index(i), c)
  230 + }
  231 +}
  232 +
  233 +func (d Decoder) decodeSlice(v reflect.Value, x interface{}) {
  234 + t := v.Type()
  235 + if t.Elem().Kind() == reflect.Uint8 {
  236 + // Allow, but don't require, byte slices to be encoded as a single string.
  237 + if s, ok := x.(string); ok {
  238 + v.SetBytes([]byte(s))
  239 + return
  240 + }
  241 + }
  242 +
  243 + // NOTE: Implicit indexing is currently done at the parseValues level,
  244 + // so if if an implicitKey reaches here it will always replace the last.
  245 + implicit := 0
  246 + for k, c := range getNode(x) {
  247 + var i int
  248 + if k == implicitKey {
  249 + i = implicit
  250 + implicit++
  251 + } else {
  252 + explicit, err := strconv.Atoi(k)
  253 + if err != nil {
  254 + panic(k + " is not a valid index for type " + t.String())
  255 + }
  256 + i = explicit
  257 + implicit = explicit + 1
  258 + }
  259 + // "Extend" the slice if it's too short.
  260 + if l := v.Len(); i >= l {
  261 + delta := i - l + 1
  262 + v.Set(reflect.AppendSlice(v, reflect.MakeSlice(t, delta, delta)))
  263 + }
  264 + d.decodeValue(v.Index(i), c)
  265 + }
  266 +}
  267 +
  268 +func (d Decoder) decodeBasic(v reflect.Value, x interface{}) {
  269 + t := v.Type()
  270 + switch k, s := t.Kind(), getString(x); k {
  271 + case reflect.Bool:
  272 + if b, e := strconv.ParseBool(s); e == nil {
  273 + v.SetBool(b)
  274 + } else {
  275 + panic("could not parse bool from " + strconv.Quote(s))
  276 + }
  277 + case reflect.Int,
  278 + reflect.Int8,
  279 + reflect.Int16,
  280 + reflect.Int32,
  281 + reflect.Int64:
  282 + if i, e := strconv.ParseInt(s, 10, 64); e == nil {
  283 + v.SetInt(i)
  284 + } else {
  285 + panic("could not parse int from " + strconv.Quote(s))
  286 + }
  287 + case reflect.Uint,
  288 + reflect.Uint8,
  289 + reflect.Uint16,
  290 + reflect.Uint32,
  291 + reflect.Uint64:
  292 + if u, e := strconv.ParseUint(s, 10, 64); e == nil {
  293 + v.SetUint(u)
  294 + } else {
  295 + panic("could not parse uint from " + strconv.Quote(s))
  296 + }
  297 + case reflect.Float32,
  298 + reflect.Float64:
  299 + if f, e := strconv.ParseFloat(s, 64); e == nil {
  300 + v.SetFloat(f)
  301 + } else {
  302 + panic("could not parse float from " + strconv.Quote(s))
  303 + }
  304 + case reflect.Complex64,
  305 + reflect.Complex128:
  306 + var c complex128
  307 + if n, err := fmt.Sscanf(s, "%g", &c); n == 1 && err == nil {
  308 + v.SetComplex(c)
  309 + } else {
  310 + panic("could not parse complex from " + strconv.Quote(s))
  311 + }
  312 + case reflect.String:
  313 + v.SetString(s)
  314 + default:
  315 + panic(t.String() + " has unsupported kind " + k.String())
  316 + }
  317 +}
  318 +
  319 +func (d Decoder) decodeTime(v reflect.Value, x interface{}) {
  320 + t := v.Type()
  321 + s := getString(x)
  322 + // TODO: Find a more efficient way to do this.
  323 + for _, f := range allowedTimeFormats {
  324 + if p, err := time.Parse(f, s); err == nil {
  325 + v.Set(reflect.ValueOf(p).Convert(v.Type()))
  326 + return
  327 + }
  328 + }
  329 + panic("cannot decode string `" + s + "` as " + t.String())
  330 +}
  331 +
  332 +func (d Decoder) decodeURL(v reflect.Value, x interface{}) {
  333 + t := v.Type()
  334 + s := getString(x)
  335 + if u, err := url.Parse(s); err == nil {
  336 + v.Set(reflect.ValueOf(*u).Convert(v.Type()))
  337 + return
  338 + }
  339 + panic("cannot decode string `" + s + "` as " + t.String())
  340 +}
  341 +
  342 +var allowedTimeFormats = []string{
  343 + "2006-01-02T15:04:05.999999999Z07:00",
  344 + "2006-01-02T15:04:05.999999999Z07",
  345 + "2006-01-02T15:04:05.999999999Z",
  346 + "2006-01-02T15:04:05.999999999",
  347 + "2006-01-02T15:04:05Z07:00",
  348 + "2006-01-02T15:04:05Z07",
  349 + "2006-01-02T15:04:05Z",
  350 + "2006-01-02T15:04:05",
  351 + "2006-01-02T15:04Z",
  352 + "2006-01-02T15:04",
  353 + "2006-01-02T15Z",
  354 + "2006-01-02T15",
  355 + "2006-01-02",
  356 + "2006-01",
  357 + "2006",
  358 + "15:04:05.999999999Z07:00",
  359 + "15:04:05.999999999Z07",
  360 + "15:04:05.999999999Z",
  361 + "15:04:05.999999999",
  362 + "15:04:05Z07:00",
  363 + "15:04:05Z07",
  364 + "15:04:05Z",
  365 + "15:04:05",
  366 + "15:04Z",
  367 + "15:04",
  368 + "15Z",
  369 + "15",
  370 +}
  1 +// Copyright 2014 Alvaro J. Genial. All rights reserved.
  2 +// Use of this source code is governed by a BSD-style
  3 +// license that can be found in the LICENSE file.
  4 +
  5 +package form
  6 +
  7 +import (
  8 + "encoding"
  9 + "errors"
  10 + "fmt"
  11 + "io"
  12 + "net/url"
  13 + "reflect"
  14 + "strconv"
  15 + "strings"
  16 + "time"
  17 +)
  18 +
  19 +// NewEncoder returns a new form Encoder.
  20 +func NewEncoder(w io.Writer) *Encoder {
  21 + return &Encoder{w, defaultDelimiter, defaultEscape, false}
  22 +}
  23 +
  24 +// Encoder provides a way to encode to a Writer.
  25 +type Encoder struct {
  26 + w io.Writer
  27 + d rune
  28 + e rune
  29 + z bool
  30 +}
  31 +
  32 +// DelimitWith sets r as the delimiter used for composite keys by Encoder e and returns the latter; it is '.' by default.
  33 +func (e *Encoder) DelimitWith(r rune) *Encoder {
  34 + e.d = r
  35 + return e
  36 +}
  37 +
  38 +// EscapeWith sets r as the escape used for delimiters (and to escape itself) by Encoder e and returns the latter; it is '\\' by default.
  39 +func (e *Encoder) EscapeWith(r rune) *Encoder {
  40 + e.e = r
  41 + return e
  42 +}
  43 +
  44 +// KeepZeros sets whether Encoder e should keep zero (default) values in their literal form when encoding, and returns the former; by default zero values are not kept, but are rather encoded as the empty string.
  45 +func (e *Encoder) KeepZeros(z bool) *Encoder {
  46 + e.z = z
  47 + return e
  48 +}
  49 +
  50 +// Encode encodes dst as form and writes it out using the Encoder's Writer.
  51 +func (e Encoder) Encode(dst interface{}) error {
  52 + v := reflect.ValueOf(dst)
  53 + n, err := encodeToNode(v, e.z)
  54 + if err != nil {
  55 + return err
  56 + }
  57 + s := n.values(e.d, e.e).Encode()
  58 + l, err := io.WriteString(e.w, s)
  59 + switch {
  60 + case err != nil:
  61 + return err
  62 + case l != len(s):
  63 + return errors.New("could not write data completely")
  64 + }
  65 + return nil
  66 +}
  67 +
  68 +// EncodeToString encodes dst as a form and returns it as a string.
  69 +func EncodeToString(dst interface{}) (string, error) {
  70 + v := reflect.ValueOf(dst)
  71 + n, err := encodeToNode(v, false)
  72 + if err != nil {
  73 + return "", err
  74 + }
  75 + vs := n.values(defaultDelimiter, defaultEscape)
  76 + return vs.Encode(), nil
  77 +}
  78 +
  79 +// EncodeToValues encodes dst as a form and returns it as Values.
  80 +func EncodeToValues(dst interface{}) (url.Values, error) {
  81 + v := reflect.ValueOf(dst)
  82 + n, err := encodeToNode(v, false)
  83 + if err != nil {
  84 + return nil, err
  85 + }
  86 + vs := n.values(defaultDelimiter, defaultEscape)
  87 + return vs, nil
  88 +}
  89 +
  90 +func encodeToNode(v reflect.Value, z bool) (n node, err error) {
  91 + defer func() {
  92 + if e := recover(); e != nil {
  93 + err = fmt.Errorf("%v", e)
  94 + }
  95 + }()
  96 + return getNode(encodeValue(v, z)), nil
  97 +}
  98 +
  99 +func encodeValue(v reflect.Value, z bool) interface{} {
  100 + t := v.Type()
  101 + k := v.Kind()
  102 +
  103 + if s, ok := marshalValue(v); ok {
  104 + return s
  105 + } else if !z && isEmptyValue(v) {
  106 + return "" // Treat the zero value as the empty string.
  107 + }
  108 +
  109 + switch k {
  110 + case reflect.Ptr, reflect.Interface:
  111 + return encodeValue(v.Elem(), z)
  112 + case reflect.Struct:
  113 + if t.ConvertibleTo(timeType) {
  114 + return encodeTime(v)
  115 + } else if t.ConvertibleTo(urlType) {
  116 + return encodeURL(v)
  117 + }
  118 + return encodeStruct(v, z)
  119 + case reflect.Slice:
  120 + return encodeSlice(v, z)
  121 + case reflect.Array:
  122 + return encodeArray(v, z)
  123 + case reflect.Map:
  124 + return encodeMap(v, z)
  125 + case reflect.Invalid, reflect.Uintptr, reflect.UnsafePointer, reflect.Chan, reflect.Func:
  126 + panic(t.String() + " has unsupported kind " + t.Kind().String())
  127 + default:
  128 + return encodeBasic(v)
  129 + }
  130 +}
  131 +
  132 +func encodeStruct(v reflect.Value, z bool) interface{} {
  133 + t := v.Type()
  134 + n := node{}
  135 + for i := 0; i < t.NumField(); i++ {
  136 + f := t.Field(i)
  137 + k, oe := fieldInfo(f)
  138 +
  139 + if k == "-" {
  140 + continue
  141 + } else if fv := v.Field(i); oe && isEmptyValue(fv) {
  142 + delete(n, k)
  143 + } else {
  144 + n[k] = encodeValue(fv, z)
  145 + }
  146 + }
  147 + return n
  148 +}
  149 +
  150 +func encodeMap(v reflect.Value, z bool) interface{} {
  151 + n := node{}
  152 + for _, i := range v.MapKeys() {
  153 + k := getString(encodeValue(i, z))
  154 + n[k] = encodeValue(v.MapIndex(i), z)
  155 + }
  156 + return n
  157 +}
  158 +
  159 +func encodeArray(v reflect.Value, z bool) interface{} {
  160 + n := node{}
  161 + for i := 0; i < v.Len(); i++ {
  162 + n[strconv.Itoa(i)] = encodeValue(v.Index(i), z)
  163 + }
  164 + return n
  165 +}
  166 +
  167 +func encodeSlice(v reflect.Value, z bool) interface{} {
  168 + t := v.Type()
  169 + if t.Elem().Kind() == reflect.Uint8 {
  170 + return string(v.Bytes()) // Encode byte slices as a single string by default.
  171 + }
  172 + n := node{}
  173 + for i := 0; i < v.Len(); i++ {
  174 + n[strconv.Itoa(i)] = encodeValue(v.Index(i), z)
  175 + }
  176 + return n
  177 +}
  178 +
  179 +func encodeTime(v reflect.Value) string {
  180 + t := v.Convert(timeType).Interface().(time.Time)
  181 + if t.Year() == 0 && (t.Month() == 0 || t.Month() == 1) && (t.Day() == 0 || t.Day() == 1) {
  182 + return t.Format("15:04:05.999999999Z07:00")
  183 + } else if t.Hour() == 0 && t.Minute() == 0 && t.Second() == 0 && t.Nanosecond() == 0 {
  184 + return t.Format("2006-01-02")
  185 + }
  186 + return t.Format("2006-01-02T15:04:05.999999999Z07:00")
  187 +}
  188 +
  189 +func encodeURL(v reflect.Value) string {
  190 + u := v.Convert(urlType).Interface().(url.URL)
  191 + return u.String()
  192 +}
  193 +
  194 +func encodeBasic(v reflect.Value) string {
  195 + t := v.Type()
  196 + switch k := t.Kind(); k {
  197 + case reflect.Bool:
  198 + return strconv.FormatBool(v.Bool())
  199 + case reflect.Int,
  200 + reflect.Int8,
  201 + reflect.Int16,
  202 + reflect.Int32,
  203 + reflect.Int64:
  204 + return strconv.FormatInt(v.Int(), 10)
  205 + case reflect.Uint,
  206 + reflect.Uint8,
  207 + reflect.Uint16,
  208 + reflect.Uint32,
  209 + reflect.Uint64:
  210 + return strconv.FormatUint(v.Uint(), 10)
  211 + case reflect.Float32:
  212 + return strconv.FormatFloat(v.Float(), 'g', -1, 32)
  213 + case reflect.Float64:
  214 + return strconv.FormatFloat(v.Float(), 'g', -1, 64)
  215 + case reflect.Complex64, reflect.Complex128:
  216 + s := fmt.Sprintf("%g", v.Complex())
  217 + return strings.TrimSuffix(strings.TrimPrefix(s, "("), ")")
  218 + case reflect.String:
  219 + return v.String()
  220 + }
  221 + panic(t.String() + " has unsupported kind " + t.Kind().String())
  222 +}
  223 +
  224 +func isEmptyValue(v reflect.Value) bool {
  225 + switch t := v.Type(); v.Kind() {
  226 + case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
  227 + return v.Len() == 0
  228 + case reflect.Bool:
  229 + return !v.Bool()
  230 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
  231 + return v.Int() == 0
  232 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
  233 + return v.Uint() == 0
  234 + case reflect.Float32, reflect.Float64:
  235 + return v.Float() == 0
  236 + case reflect.Complex64, reflect.Complex128:
  237 + return v.Complex() == 0
  238 + case reflect.Interface, reflect.Ptr:
  239 + return v.IsNil()
  240 + case reflect.Struct:
  241 + if t.ConvertibleTo(timeType) {
  242 + return v.Convert(timeType).Interface().(time.Time).IsZero()
  243 + }
  244 + return reflect.DeepEqual(v, reflect.Zero(t))
  245 + }
  246 + return false
  247 +}
  248 +
  249 +// canIndexOrdinally returns whether a value contains an ordered sequence of elements.
  250 +func canIndexOrdinally(v reflect.Value) bool {
  251 + if !v.IsValid() {
  252 + return false
  253 + }
  254 + switch t := v.Type(); t.Kind() {
  255 + case reflect.Ptr, reflect.Interface:
  256 + return canIndexOrdinally(v.Elem())
  257 + case reflect.Slice, reflect.Array:
  258 + return true
  259 + }
  260 + return false
  261 +}
  262 +
  263 +func fieldInfo(f reflect.StructField) (k string, oe bool) {
  264 + if f.PkgPath != "" { // Skip private fields.
  265 + return omittedKey, oe
  266 + }
  267 +
  268 + k = f.Name
  269 + tag := f.Tag.Get("form")
  270 + if tag == "" {
  271 + return k, oe
  272 + }
  273 +
  274 + ps := strings.SplitN(tag, ",", 2)
  275 + if ps[0] != "" {
  276 + k = ps[0]
  277 + }
  278 + if len(ps) == 2 {
  279 + oe = ps[1] == "omitempty"
  280 + }
  281 + return k, oe
  282 +}
  283 +
  284 +func findField(v reflect.Value, n string, ignoreCase bool) (reflect.Value, bool) {
  285 + t := v.Type()
  286 + l := v.NumField()
  287 +
  288 + var lowerN string
  289 + caseInsensitiveMatch := -1
  290 + if ignoreCase {
  291 + lowerN = strings.ToLower(n)
  292 + }
  293 +
  294 + // First try named fields.
  295 + for i := 0; i < l; i++ {
  296 + f := t.Field(i)
  297 + k, _ := fieldInfo(f)
  298 + if k == omittedKey {
  299 + continue
  300 + } else if n == k {
  301 + return v.Field(i), true
  302 + } else if ignoreCase && lowerN == strings.ToLower(k) {
  303 + caseInsensitiveMatch = i
  304 + }
  305 + }
  306 +
  307 + // If no exact match was found try case insensitive match.
  308 + if caseInsensitiveMatch != -1 {
  309 + return v.Field(caseInsensitiveMatch), true
  310 + }
  311 +
  312 + // Then try anonymous (embedded) fields.
  313 + for i := 0; i < l; i++ {
  314 + f := t.Field(i)
  315 + k, _ := fieldInfo(f)
  316 + if k == omittedKey || !f.Anonymous { // || k != "" ?
  317 + continue
  318 + }
  319 + fv := v.Field(i)
  320 + fk := fv.Kind()
  321 + for fk == reflect.Ptr || fk == reflect.Interface {
  322 + fv = fv.Elem()
  323 + fk = fv.Kind()
  324 + }
  325 +
  326 + if fk != reflect.Struct {
  327 + continue
  328 + }
  329 + if ev, ok := findField(fv, n, ignoreCase); ok {
  330 + return ev, true
  331 + }
  332 + }
  333 +
  334 + return reflect.Value{}, false
  335 +}
  336 +
  337 +var (
  338 + stringType = reflect.TypeOf(string(""))
  339 + stringMapType = reflect.TypeOf(map[string]interface{}{})
  340 + timeType = reflect.TypeOf(time.Time{})
  341 + timePtrType = reflect.TypeOf(&time.Time{})
  342 + urlType = reflect.TypeOf(url.URL{})
  343 +)
  344 +
  345 +func skipTextMarshalling(t reflect.Type) bool {
  346 + /*// Skip time.Time because its text unmarshaling is overly rigid:
  347 + return t == timeType || t == timePtrType*/
  348 + // Skip time.Time & convertibles because its text unmarshaling is overly rigid:
  349 + return t.ConvertibleTo(timeType) || t.ConvertibleTo(timePtrType)
  350 +}
  351 +
  352 +func unmarshalValue(v reflect.Value, x interface{}) bool {
  353 + if skipTextMarshalling(v.Type()) {
  354 + return false
  355 + }
  356 +
  357 + tu, ok := v.Interface().(encoding.TextUnmarshaler)
  358 + if !ok && !v.CanAddr() {
  359 + return false
  360 + } else if !ok {
  361 + return unmarshalValue(v.Addr(), x)
  362 + }
  363 +
  364 + s := getString(x)
  365 + if err := tu.UnmarshalText([]byte(s)); err != nil {
  366 + panic(err)
  367 + }
  368 + return true
  369 +}
  370 +
  371 +func marshalValue(v reflect.Value) (string, bool) {
  372 + if skipTextMarshalling(v.Type()) {
  373 + return "", false
  374 + }
  375 +
  376 + tm, ok := v.Interface().(encoding.TextMarshaler)
  377 + if !ok && !v.CanAddr() {
  378 + return "", false
  379 + } else if !ok {
  380 + return marshalValue(v.Addr())
  381 + }
  382 +
  383 + bs, err := tm.MarshalText()
  384 + if err != nil {
  385 + panic(err)
  386 + }
  387 + return string(bs), true
  388 +}
  1 +// Copyright 2014 Alvaro J. Genial. All rights reserved.
  2 +// Use of this source code is governed by a BSD-style
  3 +// license that can be found in the LICENSE file.
  4 +
  5 +// Package form implements encoding and decoding of application/x-www-form-urlencoded data.
  6 +package form
  7 +
  8 +const (
  9 + implicitKey = "_"
  10 + omittedKey = "-"
  11 +
  12 + defaultDelimiter = '.'
  13 + defaultEscape = '\\'
  14 +)
  1 +// Copyright 2014 Alvaro J. Genial. All rights reserved.
  2 +// Use of this source code is governed by a BSD-style
  3 +// license that can be found in the LICENSE file.
  4 +
  5 +package form
  6 +
  7 +import (
  8 + "net/url"
  9 + "strconv"
  10 + "strings"
  11 +)
  12 +
  13 +type node map[string]interface{}
  14 +
  15 +func (n node) values(d, e rune) url.Values {
  16 + vs := url.Values{}
  17 + n.merge(d, e, "", &vs)
  18 + return vs
  19 +}
  20 +
  21 +func (n node) merge(d, e rune, p string, vs *url.Values) {
  22 + for k, x := range n {
  23 + switch y := x.(type) {
  24 + case string:
  25 + vs.Add(p+escape(d, e, k), y)
  26 + case node:
  27 + y.merge(d, e, p+escape(d, e, k)+string(d), vs)
  28 + default:
  29 + panic("value is neither string nor node")
  30 + }
  31 + }
  32 +}
  33 +
  34 +// TODO: Add tests for implicit indexing.
  35 +func parseValues(d, e rune, vs url.Values, canIndexFirstLevelOrdinally bool) node {
  36 + // NOTE: Because of the flattening of potentially multiple strings to one key, implicit indexing works:
  37 + // i. At the first level; e.g. Foo.Bar=A&Foo.Bar=B becomes 0.Foo.Bar=A&1.Foo.Bar=B
  38 + // ii. At the last level; e.g. Foo.Bar._=A&Foo.Bar._=B becomes Foo.Bar.0=A&Foo.Bar.1=B
  39 + // TODO: At in-between levels; e.g. Foo._.Bar=A&Foo._.Bar=B becomes Foo.0.Bar=A&Foo.1.Bar=B
  40 + // (This last one requires that there only be one placeholder in order for it to be unambiguous.)
  41 +
  42 + m := map[string]string{}
  43 + for k, ss := range vs {
  44 + indexLastLevelOrdinally := strings.HasSuffix(k, string(d)+implicitKey)
  45 +
  46 + for i, s := range ss {
  47 + if canIndexFirstLevelOrdinally {
  48 + k = strconv.Itoa(i) + string(d) + k
  49 + } else if indexLastLevelOrdinally {
  50 + k = strings.TrimSuffix(k, implicitKey) + strconv.Itoa(i)
  51 + }
  52 +
  53 + m[k] = s
  54 + }
  55 + }
  56 +
  57 + n := node{}
  58 + for k, s := range m {
  59 + n = n.split(d, e, k, s)
  60 + }
  61 + return n
  62 +}
  63 +
  64 +func splitPath(d, e rune, path string) (k, rest string) {
  65 + esc := false
  66 + for i, r := range path {
  67 + switch {
  68 + case !esc && r == e:
  69 + esc = true
  70 + case !esc && r == d:
  71 + return unescape(d, e, path[:i]), path[i+1:]
  72 + default:
  73 + esc = false
  74 + }
  75 + }
  76 + return unescape(d, e, path), ""
  77 +}
  78 +
  79 +func (n node) split(d, e rune, path, s string) node {
  80 + k, rest := splitPath(d, e, path)
  81 + if rest == "" {
  82 + return add(n, k, s)
  83 + }
  84 + if _, ok := n[k]; !ok {
  85 + n[k] = node{}
  86 + }
  87 +
  88 + c := getNode(n[k])
  89 + n[k] = c.split(d, e, rest, s)
  90 + return n
  91 +}
  92 +
  93 +func add(n node, k, s string) node {
  94 + if n == nil {
  95 + return node{k: s}
  96 + }
  97 +
  98 + if _, ok := n[k]; ok {
  99 + panic("key " + k + " already set")
  100 + }
  101 +
  102 + n[k] = s
  103 + return n
  104 +}
  105 +
  106 +func isEmpty(x interface{}) bool {
  107 + switch y := x.(type) {
  108 + case string:
  109 + return y == ""
  110 + case node:
  111 + if s, ok := y[""].(string); ok {
  112 + return s == ""
  113 + }
  114 + return false
  115 + }
  116 + panic("value is neither string nor node")
  117 +}
  118 +
  119 +func getNode(x interface{}) node {
  120 + switch y := x.(type) {
  121 + case string:
  122 + return node{"": y}
  123 + case node:
  124 + return y
  125 + }
  126 + panic("value is neither string nor node")
  127 +}
  128 +
  129 +func getString(x interface{}) string {
  130 + switch y := x.(type) {
  131 + case string:
  132 + return y
  133 + case node:
  134 + if s, ok := y[""].(string); ok {
  135 + return s
  136 + }
  137 + return ""
  138 + }
  139 + panic("value is neither string nor node")
  140 +}
  141 +
  142 +func escape(d, e rune, s string) string {
  143 + s = strings.Replace(s, string(e), string(e)+string(e), -1) // Escape the escape (\ => \\)
  144 + s = strings.Replace(s, string(d), string(e)+string(d), -1) // Escape the delimiter (. => \.)
  145 + return s
  146 +}
  147 +
  148 +func unescape(d, e rune, s string) string {
  149 + s = strings.Replace(s, string(e)+string(d), string(d), -1) // Unescape the delimiter (\. => .)
  150 + s = strings.Replace(s, string(e)+string(e), string(e), -1) // Unescape the escape (\\ => \)
  151 + return s
  152 +}
  1 +#!/bin/bash -eu
  2 +
  3 +# TODO: Only colorize messages given a suitable terminal.
  4 +# FIXME: Handle case in which no stash entry is created due to no changes.
  5 +
  6 +printf "\e[30m=== PRE-COMMIT STARTING ===\e[m\n"
  7 +git stash save --quiet --keep-index --include-untracked
  8 +
  9 +if go build -v ./... && go test -v -cover ./... && go vet ./... && golint . && travis-lint; then
  10 + result=$?
  11 + printf "\e[32m=== PRE-COMMIT SUCCEEDED ===\e[m\n"
  12 +else
  13 + result=$?
  14 + printf "\e[31m=== PRE-COMMIT FAILED ===\e[m\n"
  15 +fi
  16 +
  17 +git stash pop --quiet
  18 +exit $result
  1 +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
  2 +
  3 +Permission is hereby granted, free of charge, to any person obtaining a copy
  4 +of this software and associated documentation files (the "Software"), to deal
  5 +in the Software without restriction, including without limitation the rights
  6 +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  7 +copies of the Software, and to permit persons to whom the Software is
  8 +furnished to do so, subject to the following conditions:
  9 +
  10 +The above copyright notice and this permission notice shall be included in
  11 +all copies or substantial portions of the Software.
  12 +
  13 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14 +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15 +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16 +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17 +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18 +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  19 +THE SOFTWARE.
  1 +This package is a brotli compressor and decompressor implemented in Go.
  2 +It was translated from the reference implementation (https://github.com/google/brotli)
  3 +with the `c2go` tool at https://github.com/andybalholm/c2go.
  4 +
  5 +I am using it in production with https://github.com/andybalholm/redwood.
  1 +package brotli
  2 +
  3 +/* Copyright 2013 Google Inc. All Rights Reserved.
  4 +
  5 + Distributed under MIT license.
  6 + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  7 +*/
  8 +
  9 +/* Function to find backward reference copies. */
  10 +
  11 +func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint {
  12 + if distance <= max_distance {
  13 + var distance_plus_3 uint = distance + 3
  14 + var offset0 uint = distance_plus_3 - uint(dist_cache[0])
  15 + var offset1 uint = distance_plus_3 - uint(dist_cache[1])
  16 + if distance == uint(dist_cache[0]) {
  17 + return 0
  18 + } else if distance == uint(dist_cache[1]) {
  19 + return 1
  20 + } else if offset0 < 7 {
  21 + return (0x9750468 >> (4 * offset0)) & 0xF
  22 + } else if offset1 < 7 {
  23 + return (0xFDB1ACE >> (4 * offset1)) & 0xF
  24 + } else if distance == uint(dist_cache[2]) {
  25 + return 2
  26 + } else if distance == uint(dist_cache[3]) {
  27 + return 3
  28 + }
  29 + }
  30 +
  31 + return distance + numDistanceShortCodes - 1
  32 +}
  33 +
  34 +/* "commands" points to the next output command to write to, "*num_commands" is
  35 + initially the total amount of commands output by previous
  36 + CreateBackwardReferences calls, and must be incremented by the amount written
  37 + by this call. */
  38 +func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
  39 + var max_backward_limit uint = maxBackwardLimit(params.lgwin)
  40 + var orig_commands []command = commands
  41 + var insert_length uint = *last_insert_len
  42 + var pos_end uint = position + num_bytes
  43 + var store_end uint
  44 + if num_bytes >= hasher.StoreLookahead() {
  45 + store_end = position + num_bytes - hasher.StoreLookahead() + 1
  46 + } else {
  47 + store_end = position
  48 + }
  49 + var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params)
  50 + var apply_random_heuristics uint = position + random_heuristics_window_size
  51 + var gap uint = 0
  52 + /* Set maximum distance, see section 9.1. of the spec. */
  53 +
  54 + const kMinScore uint = scoreBase + 100
  55 +
  56 + /* For speed up heuristics for random data. */
  57 +
  58 + /* Minimum score to accept a backward reference. */
  59 + hasher.PrepareDistanceCache(dist_cache)
  60 + var sr2 hasherSearchResult
  61 + var sr hasherSearchResult
  62 +
  63 + for position+hasher.HashTypeLength() < pos_end {
  64 + var max_length uint = pos_end - position
  65 + var max_distance uint = brotli_min_size_t(position, max_backward_limit)
  66 + sr.len = 0
  67 + sr.len_code_delta = 0
  68 + sr.distance = 0
  69 + sr.score = kMinScore
  70 + hasher.FindLongestMatch(&params.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, &sr)
  71 + if sr.score > kMinScore {
  72 + /* Found a match. Let's look for something even better ahead. */
  73 + var delayed_backward_references_in_row int = 0
  74 + max_length--
  75 + for ; ; max_length-- {
  76 + var cost_diff_lazy uint = 175
  77 + if params.quality < minQualityForExtensiveReferenceSearch {
  78 + sr2.len = brotli_min_size_t(sr.len-1, max_length)
  79 + } else {
  80 + sr2.len = 0
  81 + }
  82 + sr2.len_code_delta = 0
  83 + sr2.distance = 0
  84 + sr2.score = kMinScore
  85 + max_distance = brotli_min_size_t(position+1, max_backward_limit)
  86 + hasher.FindLongestMatch(&params.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, &sr2)
  87 + if sr2.score >= sr.score+cost_diff_lazy {
  88 + /* Ok, let's just write one byte for now and start a match from the
  89 + next byte. */
  90 + position++
  91 +
  92 + insert_length++
  93 + sr = sr2
  94 + delayed_backward_references_in_row++
  95 + if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end {
  96 + continue
  97 + }
  98 + }
  99 +
  100 + break
  101 + }
  102 +
  103 + apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size
  104 + max_distance = brotli_min_size_t(position, max_backward_limit)
  105 + {
  106 + /* The first 16 codes are special short-codes,
  107 + and the minimum offset is 1. */
  108 + var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache)
  109 + if (sr.distance <= (max_distance + gap)) && distance_code > 0 {
  110 + dist_cache[3] = dist_cache[2]
  111 + dist_cache[2] = dist_cache[1]
  112 + dist_cache[1] = dist_cache[0]
  113 + dist_cache[0] = int(sr.distance)
  114 + hasher.PrepareDistanceCache(dist_cache)
  115 + }
  116 +
  117 + initCommand(&commands[0], &params.dist, insert_length, sr.len, sr.len_code_delta, distance_code)
  118 + commands = commands[1:]
  119 + }
  120 +
  121 + *num_literals += insert_length
  122 + insert_length = 0
  123 + /* Put the hash keys into the table, if there are enough bytes left.
  124 + Depending on the hasher implementation, it can push all positions
  125 + in the given range or only a subset of them.
  126 + Avoid hash poisoning with RLE data. */
  127 + {
  128 + var range_start uint = position + 2
  129 + var range_end uint = brotli_min_size_t(position+sr.len, store_end)
  130 + if sr.distance < sr.len>>2 {
  131 + range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2)))
  132 + }
  133 +
  134 + hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end)
  135 + }
  136 +
  137 + position += sr.len
  138 + } else {
  139 + insert_length++
  140 + position++
  141 +
  142 + /* If we have not seen matches for a long time, we can skip some
  143 + match lookups. Unsuccessful match lookups are very very expensive
  144 + and this kind of a heuristic speeds up compression quite
  145 + a lot. */
  146 + if position > apply_random_heuristics {
  147 + /* Going through uncompressible data, jump. */
  148 + if position > apply_random_heuristics+4*random_heuristics_window_size {
  149 + var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4)
  150 + /* It is quite a long time since we saw a copy, so we assume
  151 + that this data is not compressible, and store hashes less
  152 + often. Hashes of non compressible data are less likely to
  153 + turn out to be useful in the future, too, so we store less of
  154 + them to not to flood out the hash table of good compressible
  155 + data. */
  156 +
  157 + var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin)
  158 + for ; position < pos_jump; position += 4 {
  159 + hasher.Store(ringbuffer, ringbuffer_mask, position)
  160 + insert_length += 4
  161 + }
  162 + } else {
  163 + var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2)
  164 + var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin)
  165 + for ; position < pos_jump; position += 2 {
  166 + hasher.Store(ringbuffer, ringbuffer_mask, position)
  167 + insert_length += 2
  168 + }
  169 + }
  170 + }
  171 + }
  172 + }
  173 +
  174 + insert_length += pos_end - position
  175 + *last_insert_len = insert_length
  176 + *num_commands += uint(-cap(commands) + cap(orig_commands))
  177 +}
  1 +package brotli
  2 +
  3 +import "math"
  4 +
  5 +type zopfliNode struct {
  6 + length uint32
  7 + distance uint32
  8 + dcode_insert_length uint32
  9 + u struct {
  10 + cost float32
  11 + next uint32
  12 + shortcut uint32
  13 + }
  14 +}
  15 +
  16 +const maxEffectiveDistanceAlphabetSize = 544
  17 +
  18 +const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
  19 +
  20 +var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1}
  21 +
  22 +var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3}
  23 +
  24 +func initZopfliNodes(array []zopfliNode, length uint) {
  25 + var stub zopfliNode
  26 + var i uint
  27 + stub.length = 1
  28 + stub.distance = 0
  29 + stub.dcode_insert_length = 0
  30 + stub.u.cost = kInfinity
  31 + for i = 0; i < length; i++ {
  32 + array[i] = stub
  33 + }
  34 +}
  35 +
  36 +func zopfliNodeCopyLength(self *zopfliNode) uint32 {
  37 + return self.length & 0x1FFFFFF
  38 +}
  39 +
  40 +func zopfliNodeLengthCode(self *zopfliNode) uint32 {
  41 + var modifier uint32 = self.length >> 25
  42 + return zopfliNodeCopyLength(self) + 9 - modifier
  43 +}
  44 +
  45 +func zopfliNodeCopyDistance(self *zopfliNode) uint32 {
  46 + return self.distance
  47 +}
  48 +
  49 +func zopfliNodeDistanceCode(self *zopfliNode) uint32 {
  50 + var short_code uint32 = self.dcode_insert_length >> 27
  51 + if short_code == 0 {
  52 + return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1
  53 + } else {
  54 + return short_code - 1
  55 + }
  56 +}
  57 +
  58 +func zopfliNodeCommandLength(self *zopfliNode) uint32 {
  59 + return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF)
  60 +}
  61 +
  62 +/* Histogram based cost model for zopflification. */
  63 +type zopfliCostModel struct {
  64 + cost_cmd_ [numCommandSymbols]float32
  65 + cost_dist_ []float32
  66 + distance_histogram_size uint32
  67 + literal_costs_ []float32
  68 + min_cost_cmd_ float32
  69 + num_bytes_ uint
  70 +}
  71 +
  72 +func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) {
  73 + var distance_histogram_size uint32 = dist.alphabet_size
  74 + if distance_histogram_size > maxEffectiveDistanceAlphabetSize {
  75 + distance_histogram_size = maxEffectiveDistanceAlphabetSize
  76 + }
  77 +
  78 + self.num_bytes_ = num_bytes
  79 + self.literal_costs_ = make([]float32, (num_bytes + 2))
  80 + self.cost_dist_ = make([]float32, (dist.alphabet_size))
  81 + self.distance_histogram_size = distance_histogram_size
  82 +}
  83 +
  84 +func cleanupZopfliCostModel(self *zopfliCostModel) {
  85 + self.literal_costs_ = nil
  86 + self.cost_dist_ = nil
  87 +}
  88 +
  89 +func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) {
  90 + var sum uint = 0
  91 + var missing_symbol_sum uint
  92 + var log2sum float32
  93 + var missing_symbol_cost float32
  94 + var i uint
  95 + for i = 0; i < histogram_size; i++ {
  96 + sum += uint(histogram[i])
  97 + }
  98 +
  99 + log2sum = float32(fastLog2(sum))
  100 + missing_symbol_sum = sum
  101 + if !literal_histogram {
  102 + for i = 0; i < histogram_size; i++ {
  103 + if histogram[i] == 0 {
  104 + missing_symbol_sum++
  105 + }
  106 + }
  107 + }
  108 +
  109 + missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2
  110 + for i = 0; i < histogram_size; i++ {
  111 + if histogram[i] == 0 {
  112 + cost[i] = missing_symbol_cost
  113 + continue
  114 + }
  115 +
  116 + /* Shannon bits for this symbol. */
  117 + cost[i] = log2sum - float32(fastLog2(uint(histogram[i])))
  118 +
  119 + /* Cannot be coded with less than 1 bit */
  120 + if cost[i] < 1 {
  121 + cost[i] = 1
  122 + }
  123 + }
  124 +}
  125 +
  126 +func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) {
  127 + var histogram_literal [numLiteralSymbols]uint32
  128 + var histogram_cmd [numCommandSymbols]uint32
  129 + var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
  130 + var cost_literal [numLiteralSymbols]float32
  131 + var pos uint = position - last_insert_len
  132 + var min_cost_cmd float32 = kInfinity
  133 + var i uint
  134 + var cost_cmd []float32 = self.cost_cmd_[:]
  135 + var literal_costs []float32
  136 +
  137 + histogram_literal = [numLiteralSymbols]uint32{}
  138 + histogram_cmd = [numCommandSymbols]uint32{}
  139 + histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
  140 +
  141 + for i = 0; i < num_commands; i++ {
  142 + var inslength uint = uint(commands[i].insert_len_)
  143 + var copylength uint = uint(commandCopyLen(&commands[i]))
  144 + var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
  145 + var cmdcode uint = uint(commands[i].cmd_prefix_)
  146 + var j uint
  147 +
  148 + histogram_cmd[cmdcode]++
  149 + if cmdcode >= 128 {
  150 + histogram_dist[distcode]++
  151 + }
  152 +
  153 + for j = 0; j < inslength; j++ {
  154 + histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++
  155 + }
  156 +
  157 + pos += inslength + copylength
  158 + }
  159 +
  160 + setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:])
  161 + setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
  162 + setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
  163 +
  164 + for i = 0; i < numCommandSymbols; i++ {
  165 + min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
  166 + }
  167 +
  168 + self.min_cost_cmd_ = min_cost_cmd
  169 + {
  170 + literal_costs = self.literal_costs_
  171 + var literal_carry float32 = 0.0
  172 + var num_bytes uint = self.num_bytes_
  173 + literal_costs[0] = 0.0
  174 + for i = 0; i < num_bytes; i++ {
  175 + literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]]
  176 + literal_costs[i+1] = literal_costs[i] + literal_carry
  177 + literal_carry -= literal_costs[i+1] - literal_costs[i]
  178 + }
  179 + }
  180 +}
  181 +
  182 +func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) {
  183 + var literal_costs []float32 = self.literal_costs_
  184 + var literal_carry float32 = 0.0
  185 + var cost_dist []float32 = self.cost_dist_
  186 + var cost_cmd []float32 = self.cost_cmd_[:]
  187 + var num_bytes uint = self.num_bytes_
  188 + var i uint
  189 + estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:])
  190 + literal_costs[0] = 0.0
  191 + for i = 0; i < num_bytes; i++ {
  192 + literal_carry += literal_costs[i+1]
  193 + literal_costs[i+1] = literal_costs[i] + literal_carry
  194 + literal_carry -= literal_costs[i+1] - literal_costs[i]
  195 + }
  196 +
  197 + for i = 0; i < numCommandSymbols; i++ {
  198 + cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i))))
  199 + }
  200 +
  201 + for i = 0; uint32(i) < self.distance_histogram_size; i++ {
  202 + cost_dist[i] = float32(fastLog2(uint(20 + uint32(i))))
  203 + }
  204 +
  205 + self.min_cost_cmd_ = float32(fastLog2(11))
  206 +}
  207 +
  208 +func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 {
  209 + return self.cost_cmd_[cmdcode]
  210 +}
  211 +
  212 +func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 {
  213 + return self.cost_dist_[distcode]
  214 +}
  215 +
  216 +func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 {
  217 + return self.literal_costs_[to] - self.literal_costs_[from]
  218 +}
  219 +
  220 +func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 {
  221 + return self.min_cost_cmd_
  222 +}
  223 +
  224 +/* REQUIRES: len >= 2, start_pos <= pos */
  225 +/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */
  226 +/* Maintains the "ZopfliNode array invariant". */
  227 +func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) {
  228 + var next *zopfliNode = &nodes[pos+len]
  229 + next.length = uint32(len | (len+9-len_code)<<25)
  230 + next.distance = uint32(dist)
  231 + next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos))
  232 + next.u.cost = cost
  233 +}
  234 +
  235 +type posData struct {
  236 + pos uint
  237 + distance_cache [4]int
  238 + costdiff float32
  239 + cost float32
  240 +}
  241 +
  242 +/* Maintains the smallest 8 cost difference together with their positions */
  243 +type startPosQueue struct {
  244 + q_ [8]posData
  245 + idx_ uint
  246 +}
  247 +
  248 +func initStartPosQueue(self *startPosQueue) {
  249 + self.idx_ = 0
  250 +}
  251 +
  252 +func startPosQueueSize(self *startPosQueue) uint {
  253 + return brotli_min_size_t(self.idx_, 8)
  254 +}
  255 +
  256 +func startPosQueuePush(self *startPosQueue, posdata *posData) {
  257 + var offset uint = ^(self.idx_) & 7
  258 + self.idx_++
  259 + var len uint = startPosQueueSize(self)
  260 + var i uint
  261 + var q []posData = self.q_[:]
  262 + q[offset] = *posdata
  263 +
  264 + /* Restore the sorted order. In the list of |len| items at most |len - 1|
  265 + adjacent element comparisons / swaps are required. */
  266 + for i = 1; i < len; i++ {
  267 + if q[offset&7].costdiff > q[(offset+1)&7].costdiff {
  268 + var tmp posData = q[offset&7]
  269 + q[offset&7] = q[(offset+1)&7]
  270 + q[(offset+1)&7] = tmp
  271 + }
  272 +
  273 + offset++
  274 + }
  275 +}
  276 +
  277 +func startPosQueueAt(self *startPosQueue, k uint) *posData {
  278 + return &self.q_[(k-self.idx_)&7]
  279 +}
  280 +
  281 +/* Returns the minimum possible copy length that can improve the cost of any */
  282 +/* future position. */
  283 +func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint {
  284 + var min_cost float32 = start_cost
  285 + var len uint = 2
  286 + var next_len_bucket uint = 4
  287 + /* Compute the minimum possible cost of reaching any future position. */
  288 +
  289 + var next_len_offset uint = 10
  290 + for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost {
  291 + /* We already reached (pos + len) with no more cost than the minimum
  292 + possible cost of reaching anything from this pos, so there is no point in
  293 + looking for lengths <= len. */
  294 + len++
  295 +
  296 + if len == next_len_offset {
  297 + /* We reached the next copy length code bucket, so we add one more
  298 + extra bit to the minimum cost. */
  299 + min_cost += 1.0
  300 +
  301 + next_len_offset += next_len_bucket
  302 + next_len_bucket *= 2
  303 + }
  304 + }
  305 +
  306 + return uint(len)
  307 +}
  308 +
  309 +/* REQUIRES: nodes[pos].cost < kInfinity
  310 + REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
  311 +func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 {
  312 + var clen uint = uint(zopfliNodeCopyLength(&nodes[pos]))
  313 + var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF)
  314 + var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos]))
  315 +
  316 + /* Since |block_start + pos| is the end position of the command, the copy part
  317 + starts from |block_start + pos - clen|. Distances that are greater than
  318 + this or greater than |max_backward_limit| + |gap| are static dictionary
  319 + references, and do not update the last distances.
  320 + Also distance code 0 (last distance) does not update the last distances. */
  321 + if pos == 0 {
  322 + return 0
  323 + } else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 {
  324 + return uint32(pos)
  325 + } else {
  326 + return nodes[pos-clen-ilen].u.shortcut
  327 + }
  328 +}
  329 +
  330 +/* Fills in dist_cache[0..3] with the last four distances (as defined by
  331 + Section 4. of the Spec) that would be used at (block_start + pos) if we
  332 + used the shortest path of commands from block_start, computed from
  333 + nodes[0..pos]. The last four distances at block_start are in
  334 + starting_dist_cache[0..3].
  335 + REQUIRES: nodes[pos].cost < kInfinity
  336 + REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
  337 +func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) {
  338 + var idx int = 0
  339 + var p uint = uint(nodes[pos].u.shortcut)
  340 + for idx < 4 && p > 0 {
  341 + var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF)
  342 + var clen uint = uint(zopfliNodeCopyLength(&nodes[p]))
  343 + var dist uint = uint(zopfliNodeCopyDistance(&nodes[p]))
  344 + dist_cache[idx] = int(dist)
  345 + idx++
  346 +
  347 + /* Because of prerequisite, p >= clen + ilen >= 2. */
  348 + p = uint(nodes[p-clen-ilen].u.shortcut)
  349 + }
  350 +
  351 + for ; idx < 4; idx++ {
  352 + dist_cache[idx] = starting_dist_cache[0]
  353 + starting_dist_cache = starting_dist_cache[1:]
  354 + }
  355 +}
  356 +
  357 +/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
  358 + is eligible. */
  359 +func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) {
  360 + /* Save cost, because ComputeDistanceCache invalidates it. */
  361 + var node_cost float32 = nodes[pos].u.cost
  362 + nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes)
  363 + if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) {
  364 + var posdata posData
  365 + posdata.pos = pos
  366 + posdata.cost = node_cost
  367 + posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos)
  368 + computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:])
  369 + startPosQueuePush(queue, &posdata)
  370 + }
  371 +}
  372 +
  373 +/* Returns longest copy length. */
  374 +func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint {
  375 + var cur_ix uint = block_start + pos
  376 + var cur_ix_masked uint = cur_ix & ringbuffer_mask
  377 + var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit)
  378 + var max_len uint = num_bytes - pos
  379 + var max_zopfli_len uint = maxZopfliLen(params)
  380 + var max_iters uint = maxZopfliCandidates(params)
  381 + var min_len uint
  382 + var result uint = 0
  383 + var k uint
  384 + var gap uint = 0
  385 +
  386 + evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes)
  387 + {
  388 + var posdata *posData = startPosQueueAt(queue, 0)
  389 + var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos))
  390 + min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos)
  391 + }
  392 +
  393 + /* Go over the command starting positions in order of increasing cost
  394 + difference. */
  395 + for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ {
  396 + var posdata *posData = startPosQueueAt(queue, k)
  397 + var start uint = posdata.pos
  398 + var inscode uint16 = getInsertLengthCode(pos - start)
  399 + var start_costdiff float32 = posdata.costdiff
  400 + var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos)
  401 + var best_len uint = min_len - 1
  402 + var j uint = 0
  403 + /* Look for last distance matches using the distance cache from this
  404 + starting position. */
  405 + for ; j < numDistanceShortCodes && best_len < max_len; j++ {
  406 + var idx uint = uint(kDistanceCacheIndex[j])
  407 + var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j])
  408 + var prev_ix uint = cur_ix - backward
  409 + var len uint = 0
  410 + var continuation byte = ringbuffer[cur_ix_masked+best_len]
  411 + if cur_ix_masked+best_len > ringbuffer_mask {
  412 + break
  413 + }
  414 +
  415 + if backward > max_distance+gap {
  416 + /* Word dictionary -> ignore. */
  417 + continue
  418 + }
  419 +
  420 + if backward <= max_distance {
  421 + /* Regular backward reference. */
  422 + if prev_ix >= cur_ix {
  423 + continue
  424 + }
  425 +
  426 + prev_ix &= ringbuffer_mask
  427 + if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] {
  428 + continue
  429 + }
  430 +
  431 + len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len)
  432 + } else {
  433 + continue
  434 + }
  435 + {
  436 + var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j)
  437 + var l uint
  438 + for l = best_len + 1; l <= len; l++ {
  439 + var copycode uint16 = getCopyLengthCode(l)
  440 + var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0)
  441 + var tmp float32
  442 + if cmdcode < 128 {
  443 + tmp = base_cost
  444 + } else {
  445 + tmp = dist_cost
  446 + }
  447 + var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
  448 + if cost < nodes[pos+l].u.cost {
  449 + updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost)
  450 + result = brotli_max_size_t(result, l)
  451 + }
  452 +
  453 + best_len = l
  454 + }
  455 + }
  456 + }
  457 +
  458 + /* At higher iterations look only for new last distance matches, since
  459 + looking only for new command start positions with the same distances
  460 + does not help much. */
  461 + if k >= 2 {
  462 + continue
  463 + }
  464 + {
  465 + /* Loop through all possible copy lengths at this position. */
  466 + var len uint = min_len
  467 + for j = 0; j < num_matches; j++ {
  468 + var match backwardMatch = matches[j]
  469 + var dist uint = uint(match.distance)
  470 + var is_dictionary_match bool = (dist > max_distance+gap)
  471 + var dist_code uint = dist + numDistanceShortCodes - 1
  472 + var dist_symbol uint16
  473 + var distextra uint32
  474 + var distnumextra uint32
  475 + var dist_cost float32
  476 + var max_match_len uint
  477 + /* We already tried all possible last distance matches, so we can use
  478 + normal distance code here. */
  479 + prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra)
  480 +
  481 + distnumextra = uint32(dist_symbol) >> 10
  482 + dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF)
  483 +
  484 + /* Try all copy lengths up until the maximum copy length corresponding
  485 + to this distance. If the distance refers to the static dictionary, or
  486 + the maximum length is long enough, try only one maximum length. */
  487 + max_match_len = backwardMatchLength(&match)
  488 +
  489 + if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) {
  490 + len = max_match_len
  491 + }
  492 +
  493 + for ; len <= max_match_len; len++ {
  494 + var len_code uint
  495 + if is_dictionary_match {
  496 + len_code = backwardMatchLengthCode(&match)
  497 + } else {
  498 + len_code = len
  499 + }
  500 + var copycode uint16 = getCopyLengthCode(len_code)
  501 + var cmdcode uint16 = combineLengthCodes(inscode, copycode, false)
  502 + var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
  503 + if cost < nodes[pos+len].u.cost {
  504 + updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost)
  505 + result = brotli_max_size_t(result, uint(len))
  506 + }
  507 + }
  508 + }
  509 + }
  510 + }
  511 +
  512 + return result
  513 +}
  514 +
  515 +func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
  516 + var index uint = num_bytes
  517 + var num_commands uint = 0
  518 + for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 {
  519 + index--
  520 + }
  521 + nodes[index].u.next = math.MaxUint32
  522 + for index != 0 {
  523 + var len uint = uint(zopfliNodeCommandLength(&nodes[index]))
  524 + index -= uint(len)
  525 + nodes[index].u.next = uint32(len)
  526 + num_commands++
  527 + }
  528 +
  529 + return num_commands
  530 +}
  531 +
  532 +/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
  533 +func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) {
  534 + var max_backward_limit uint = maxBackwardLimit(params.lgwin)
  535 + var pos uint = 0
  536 + var offset uint32 = nodes[0].u.next
  537 + var i uint
  538 + var gap uint = 0
  539 + for i = 0; offset != math.MaxUint32; i++ {
  540 + var next *zopfliNode = &nodes[uint32(pos)+offset]
  541 + var copy_length uint = uint(zopfliNodeCopyLength(next))
  542 + var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF)
  543 + pos += insert_length
  544 + offset = next.u.next
  545 + if i == 0 {
  546 + insert_length += *last_insert_len
  547 + *last_insert_len = 0
  548 + }
  549 + {
  550 + var distance uint = uint(zopfliNodeCopyDistance(next))
  551 + var len_code uint = uint(zopfliNodeLengthCode(next))
  552 + var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
  553 + var is_dictionary bool = (distance > max_distance+gap)
  554 + var dist_code uint = uint(zopfliNodeDistanceCode(next))
  555 + initCommand(&commands[i], &params.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)
  556 +
  557 + if !is_dictionary && dist_code > 0 {
  558 + dist_cache[3] = dist_cache[2]
  559 + dist_cache[2] = dist_cache[1]
  560 + dist_cache[1] = dist_cache[0]
  561 + dist_cache[0] = int(distance)
  562 + }
  563 + }
  564 +
  565 + *num_literals += insert_length
  566 + pos += copy_length
  567 + }
  568 +
  569 + *last_insert_len += num_bytes - pos
  570 +}
  571 +
  572 +func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint {
  573 + var max_backward_limit uint = maxBackwardLimit(params.lgwin)
  574 + var max_zopfli_len uint = maxZopfliLen(params)
  575 + var queue startPosQueue
  576 + var cur_match_pos uint = 0
  577 + var i uint
  578 + nodes[0].length = 0
  579 + nodes[0].u.cost = 0
  580 + initStartPosQueue(&queue)
  581 + for i = 0; i+3 < num_bytes; i++ {
  582 + var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes)
  583 + if skip < longCopyQuickStep {
  584 + skip = 0
  585 + }
  586 + cur_match_pos += uint(num_matches[i])
  587 + if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len {
  588 + skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip)
  589 + }
  590 +
  591 + if skip > 1 {
  592 + skip--
  593 + for skip != 0 {
  594 + i++
  595 + if i+3 >= num_bytes {
  596 + break
  597 + }
  598 + evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes)
  599 + cur_match_pos += uint(num_matches[i])
  600 + skip--
  601 + }
  602 + }
  603 + }
  604 +
  605 + return computeShortestPathFromNodes(num_bytes, nodes)
  606 +}
  607 +
  608 +/* Computes the shortest path of commands from position to at most
  609 + position + num_bytes.
  610 +
  611 + On return, path->size() is the number of commands found and path[i] is the
  612 + length of the i-th command (copy length plus insert length).
  613 + Note that the sum of the lengths of all commands can be less than num_bytes.
  614 +
  615 + On return, the nodes[0..num_bytes] array will have the following
  616 + "ZopfliNode array invariant":
  617 + For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
  618 + (1) nodes[i].copy_length() >= 2
  619 + (2) nodes[i].command_length() <= i and
  620 + (3) nodes[i - nodes[i].command_length()].cost < kInfinity
  621 +
  622 + REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */
  623 +func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
  624 + var max_backward_limit uint = maxBackwardLimit(params.lgwin)
  625 + var max_zopfli_len uint = maxZopfliLen(params)
  626 + var model zopfliCostModel
  627 + var queue startPosQueue
  628 + var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch
  629 + var store_end uint
  630 + if num_bytes >= hasher.StoreLookahead() {
  631 + store_end = position + num_bytes - hasher.StoreLookahead() + 1
  632 + } else {
  633 + store_end = position
  634 + }
  635 + var i uint
  636 + var gap uint = 0
  637 + var lz_matches_offset uint = 0
  638 + nodes[0].length = 0
  639 + nodes[0].u.cost = 0
  640 + initZopfliCostModel(&model, &params.dist, num_bytes)
  641 + zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
  642 + initStartPosQueue(&queue)
  643 + for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
  644 + var pos uint = position + i
  645 + var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
  646 + var skip uint
  647 + var num_matches uint
  648 + num_matches = findAllMatchesH10(hasher, &params.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:])
  649 + if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len {
  650 + matches[0] = matches[num_matches-1]
  651 + num_matches = 1
  652 + }
  653 +
  654 + skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes)
  655 + if skip < longCopyQuickStep {
  656 + skip = 0
  657 + }
  658 + if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len {
  659 + skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip)
  660 + }
  661 +
  662 + if skip > 1 {
  663 + /* Add the tail of the copy to the hasher. */
  664 + hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end))
  665 +
  666 + skip--
  667 + for skip != 0 {
  668 + i++
  669 + if i+hasher.HashTypeLength()-1 >= num_bytes {
  670 + break
  671 + }
  672 + evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes)
  673 + skip--
  674 + }
  675 + }
  676 + }
  677 +
  678 + cleanupZopfliCostModel(&model)
  679 + return computeShortestPathFromNodes(num_bytes, nodes)
  680 +}
  681 +
  682 +func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
  683 + var nodes []zopfliNode
  684 + nodes = make([]zopfliNode, (num_bytes + 1))
  685 + initZopfliNodes(nodes, num_bytes+1)
  686 + *num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
  687 + zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
  688 + nodes = nil
  689 +}
  690 +
  691 +func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
  692 + var max_backward_limit uint = maxBackwardLimit(params.lgwin)
  693 + var num_matches []uint32 = make([]uint32, num_bytes)
  694 + var matches_size uint = 4 * num_bytes
  695 + var store_end uint
  696 + if num_bytes >= hasher.StoreLookahead() {
  697 + store_end = position + num_bytes - hasher.StoreLookahead() + 1
  698 + } else {
  699 + store_end = position
  700 + }
  701 + var cur_match_pos uint = 0
  702 + var i uint
  703 + var orig_num_literals uint
  704 + var orig_last_insert_len uint
  705 + var orig_dist_cache [4]int
  706 + var orig_num_commands uint
  707 + var model zopfliCostModel
  708 + var nodes []zopfliNode
  709 + var matches []backwardMatch = make([]backwardMatch, matches_size)
  710 + var gap uint = 0
  711 + var shadow_matches uint = 0
  712 + var new_array []backwardMatch
  713 + for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
  714 + var pos uint = position + i
  715 + var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
  716 + var max_length uint = num_bytes - i
  717 + var num_found_matches uint
  718 + var cur_match_end uint
  719 + var j uint
  720 +
  721 + /* Ensure that we have enough free slots. */
  722 + if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
  723 + var new_size uint = matches_size
  724 + if new_size == 0 {
  725 + new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches
  726 + }
  727 +
  728 + for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
  729 + new_size *= 2
  730 + }
  731 +
  732 + new_array = make([]backwardMatch, new_size)
  733 + if matches_size != 0 {
  734 + copy(new_array, matches[:matches_size])
  735 + }
  736 +
  737 + matches = new_array
  738 + matches_size = new_size
  739 + }
  740 +
  741 + num_found_matches = findAllMatchesH10(hasher.(*h10), &params.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:])
  742 + cur_match_end = cur_match_pos + num_found_matches
  743 + for j = cur_match_pos; j+1 < cur_match_end; j++ {
  744 + assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1]))
  745 + }
  746 +
  747 + num_matches[i] = uint32(num_found_matches)
  748 + if num_found_matches > 0 {
  749 + var match_len uint = backwardMatchLength(&matches[cur_match_end-1])
  750 + if match_len > maxZopfliLenQuality11 {
  751 + var skip uint = match_len - 1
  752 + matches[cur_match_pos] = matches[cur_match_end-1]
  753 + cur_match_pos++
  754 + num_matches[i] = 1
  755 +
  756 + /* Add the tail of the copy to the hasher. */
  757 + hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end))
  758 + var pos uint = i
  759 + for i := 0; i < int(skip); i++ {
  760 + num_matches[pos+1:][i] = 0
  761 + }
  762 + i += skip
  763 + } else {
  764 + cur_match_pos = cur_match_end
  765 + }
  766 + }
  767 + }
  768 +
  769 + orig_num_literals = *num_literals
  770 + orig_last_insert_len = *last_insert_len
  771 + copy(orig_dist_cache[:], dist_cache[:4])
  772 + orig_num_commands = *num_commands
  773 + nodes = make([]zopfliNode, (num_bytes + 1))
  774 + initZopfliCostModel(&model, &params.dist, num_bytes)
  775 + for i = 0; i < 2; i++ {
  776 + initZopfliNodes(nodes, num_bytes+1)
  777 + if i == 0 {
  778 + zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
  779 + } else {
  780 + zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len)
  781 + }
  782 +
  783 + *num_commands = orig_num_commands
  784 + *num_literals = orig_num_literals
  785 + *last_insert_len = orig_last_insert_len
  786 + copy(dist_cache, orig_dist_cache[:4])
  787 + *num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
  788 + zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
  789 + }
  790 +
  791 + cleanupZopfliCostModel(&model)
  792 + nodes = nil
  793 + matches = nil
  794 + num_matches = nil
  795 +}
  1 +package brotli
  2 +
  3 +/* Copyright 2013 Google Inc. All Rights Reserved.
  4 +
  5 + Distributed under MIT license.
  6 + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  7 +*/
  8 +
  9 +/* Functions to estimate the bit cost of Huffman trees. */
  10 +func shannonEntropy(population []uint32, size uint, total *uint) float64 {
  11 + var sum uint = 0
  12 + var retval float64 = 0
  13 + var population_end []uint32 = population[size:]
  14 + var p uint
  15 + for -cap(population) < -cap(population_end) {
  16 + p = uint(population[0])
  17 + population = population[1:]
  18 + sum += p
  19 + retval -= float64(p) * fastLog2(p)
  20 + }
  21 +
  22 + if sum != 0 {
  23 + retval += float64(sum) * fastLog2(sum)
  24 + }
  25 + *total = sum
  26 + return retval
  27 +}
  28 +
  29 +func bitsEntropy(population []uint32, size uint) float64 {
  30 + var sum uint
  31 + var retval float64 = shannonEntropy(population, size, &sum)
  32 + if retval < float64(sum) {
  33 + /* At least one bit per literal is needed. */
  34 + retval = float64(sum)
  35 + }
  36 +
  37 + return retval
  38 +}
  39 +
  40 +const kOneSymbolHistogramCost float64 = 12
  41 +const kTwoSymbolHistogramCost float64 = 20
  42 +const kThreeSymbolHistogramCost float64 = 28
  43 +const kFourSymbolHistogramCost float64 = 37
  44 +
  45 +func populationCostLiteral(histogram *histogramLiteral) float64 {
  46 + var data_size uint = histogramDataSizeLiteral()
  47 + var count int = 0
  48 + var s [5]uint
  49 + var bits float64 = 0.0
  50 + var i uint
  51 + if histogram.total_count_ == 0 {
  52 + return kOneSymbolHistogramCost
  53 + }
  54 +
  55 + for i = 0; i < data_size; i++ {
  56 + if histogram.data_[i] > 0 {
  57 + s[count] = i
  58 + count++
  59 + if count > 4 {
  60 + break
  61 + }
  62 + }
  63 + }
  64 +
  65 + if count == 1 {
  66 + return kOneSymbolHistogramCost
  67 + }
  68 +
  69 + if count == 2 {
  70 + return kTwoSymbolHistogramCost + float64(histogram.total_count_)
  71 + }
  72 +
  73 + if count == 3 {
  74 + var histo0 uint32 = histogram.data_[s[0]]
  75 + var histo1 uint32 = histogram.data_[s[1]]
  76 + var histo2 uint32 = histogram.data_[s[2]]
  77 + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
  78 + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
  79 + }
  80 +
  81 + if count == 4 {
  82 + var histo [4]uint32
  83 + var h23 uint32
  84 + var histomax uint32
  85 + for i = 0; i < 4; i++ {
  86 + histo[i] = histogram.data_[s[i]]
  87 + }
  88 +
  89 + /* Sort */
  90 + for i = 0; i < 4; i++ {
  91 + var j uint
  92 + for j = i + 1; j < 4; j++ {
  93 + if histo[j] > histo[i] {
  94 + var tmp uint32 = histo[j]
  95 + histo[j] = histo[i]
  96 + histo[i] = tmp
  97 + }
  98 + }
  99 + }
  100 +
  101 + h23 = histo[2] + histo[3]
  102 + histomax = brotli_max_uint32_t(h23, histo[0])
  103 + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
  104 + }
  105 + {
  106 + var max_depth uint = 1
  107 + var depth_histo = [codeLengthCodes]uint32{0}
  108 + /* In this loop we compute the entropy of the histogram and simultaneously
  109 + build a simplified histogram of the code length codes where we use the
  110 + zero repeat code 17, but we don't use the non-zero repeat code 16. */
  111 +
  112 + var log2total float64 = fastLog2(histogram.total_count_)
  113 + for i = 0; i < data_size; {
  114 + if histogram.data_[i] > 0 {
  115 + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
  116 + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
  117 + = log2(total_count) - log2(count(symbol)) */
  118 +
  119 + var depth uint = uint(log2p + 0.5)
  120 + /* Approximate the bit depth by round(-log2(P(symbol))) */
  121 + bits += float64(histogram.data_[i]) * log2p
  122 +
  123 + if depth > 15 {
  124 + depth = 15
  125 + }
  126 +
  127 + if depth > max_depth {
  128 + max_depth = depth
  129 + }
  130 +
  131 + depth_histo[depth]++
  132 + i++
  133 + } else {
  134 + var reps uint32 = 1
  135 + /* Compute the run length of zeros and add the appropriate number of 0
  136 + and 17 code length codes to the code length code histogram. */
  137 +
  138 + var k uint
  139 + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
  140 + reps++
  141 + }
  142 +
  143 + i += uint(reps)
  144 + if i == data_size {
  145 + /* Don't add any cost for the last zero run, since these are encoded
  146 + only implicitly. */
  147 + break
  148 + }
  149 +
  150 + if reps < 3 {
  151 + depth_histo[0] += reps
  152 + } else {
  153 + reps -= 2
  154 + for reps > 0 {
  155 + depth_histo[repeatZeroCodeLength]++
  156 +
  157 + /* Add the 3 extra bits for the 17 code length code. */
  158 + bits += 3
  159 +
  160 + reps >>= 3
  161 + }
  162 + }
  163 + }
  164 + }
  165 +
  166 + /* Add the estimated encoding cost of the code length code histogram. */
  167 + bits += float64(18 + 2*max_depth)
  168 +
  169 + /* Add the entropy of the code length code histogram. */
  170 + bits += bitsEntropy(depth_histo[:], codeLengthCodes)
  171 + }
  172 +
  173 + return bits
  174 +}
  175 +
  176 +func populationCostCommand(histogram *histogramCommand) float64 {
  177 + var data_size uint = histogramDataSizeCommand()
  178 + var count int = 0
  179 + var s [5]uint
  180 + var bits float64 = 0.0
  181 + var i uint
  182 + if histogram.total_count_ == 0 {
  183 + return kOneSymbolHistogramCost
  184 + }
  185 +
  186 + for i = 0; i < data_size; i++ {
  187 + if histogram.data_[i] > 0 {
  188 + s[count] = i
  189 + count++
  190 + if count > 4 {
  191 + break
  192 + }
  193 + }
  194 + }
  195 +
  196 + if count == 1 {
  197 + return kOneSymbolHistogramCost
  198 + }
  199 +
  200 + if count == 2 {
  201 + return kTwoSymbolHistogramCost + float64(histogram.total_count_)
  202 + }
  203 +
  204 + if count == 3 {
  205 + var histo0 uint32 = histogram.data_[s[0]]
  206 + var histo1 uint32 = histogram.data_[s[1]]
  207 + var histo2 uint32 = histogram.data_[s[2]]
  208 + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
  209 + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
  210 + }
  211 +
  212 + if count == 4 {
  213 + var histo [4]uint32
  214 + var h23 uint32
  215 + var histomax uint32
  216 + for i = 0; i < 4; i++ {
  217 + histo[i] = histogram.data_[s[i]]
  218 + }
  219 +
  220 + /* Sort */
  221 + for i = 0; i < 4; i++ {
  222 + var j uint
  223 + for j = i + 1; j < 4; j++ {
  224 + if histo[j] > histo[i] {
  225 + var tmp uint32 = histo[j]
  226 + histo[j] = histo[i]
  227 + histo[i] = tmp
  228 + }
  229 + }
  230 + }
  231 +
  232 + h23 = histo[2] + histo[3]
  233 + histomax = brotli_max_uint32_t(h23, histo[0])
  234 + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
  235 + }
  236 + {
  237 + var max_depth uint = 1
  238 + var depth_histo = [codeLengthCodes]uint32{0}
  239 + /* In this loop we compute the entropy of the histogram and simultaneously
  240 + build a simplified histogram of the code length codes where we use the
  241 + zero repeat code 17, but we don't use the non-zero repeat code 16. */
  242 +
  243 + var log2total float64 = fastLog2(histogram.total_count_)
  244 + for i = 0; i < data_size; {
  245 + if histogram.data_[i] > 0 {
  246 + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
  247 + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
  248 + = log2(total_count) - log2(count(symbol)) */
  249 +
  250 + var depth uint = uint(log2p + 0.5)
  251 + /* Approximate the bit depth by round(-log2(P(symbol))) */
  252 + bits += float64(histogram.data_[i]) * log2p
  253 +
  254 + if depth > 15 {
  255 + depth = 15
  256 + }
  257 +
  258 + if depth > max_depth {
  259 + max_depth = depth
  260 + }
  261 +
  262 + depth_histo[depth]++
  263 + i++
  264 + } else {
  265 + var reps uint32 = 1
  266 + /* Compute the run length of zeros and add the appropriate number of 0
  267 + and 17 code length codes to the code length code histogram. */
  268 +
  269 + var k uint
  270 + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
  271 + reps++
  272 + }
  273 +
  274 + i += uint(reps)
  275 + if i == data_size {
  276 + /* Don't add any cost for the last zero run, since these are encoded
  277 + only implicitly. */
  278 + break
  279 + }
  280 +
  281 + if reps < 3 {
  282 + depth_histo[0] += reps
  283 + } else {
  284 + reps -= 2
  285 + for reps > 0 {
  286 + depth_histo[repeatZeroCodeLength]++
  287 +
  288 + /* Add the 3 extra bits for the 17 code length code. */
  289 + bits += 3
  290 +
  291 + reps >>= 3
  292 + }
  293 + }
  294 + }
  295 + }
  296 +
  297 + /* Add the estimated encoding cost of the code length code histogram. */
  298 + bits += float64(18 + 2*max_depth)
  299 +
  300 + /* Add the entropy of the code length code histogram. */
  301 + bits += bitsEntropy(depth_histo[:], codeLengthCodes)
  302 + }
  303 +
  304 + return bits
  305 +}
  306 +
  307 +func populationCostDistance(histogram *histogramDistance) float64 {
  308 + var data_size uint = histogramDataSizeDistance()
  309 + var count int = 0
  310 + var s [5]uint
  311 + var bits float64 = 0.0
  312 + var i uint
  313 + if histogram.total_count_ == 0 {
  314 + return kOneSymbolHistogramCost
  315 + }
  316 +
  317 + for i = 0; i < data_size; i++ {
  318 + if histogram.data_[i] > 0 {
  319 + s[count] = i
  320 + count++
  321 + if count > 4 {
  322 + break
  323 + }
  324 + }
  325 + }
  326 +
  327 + if count == 1 {
  328 + return kOneSymbolHistogramCost
  329 + }
  330 +
  331 + if count == 2 {
  332 + return kTwoSymbolHistogramCost + float64(histogram.total_count_)
  333 + }
  334 +
  335 + if count == 3 {
  336 + var histo0 uint32 = histogram.data_[s[0]]
  337 + var histo1 uint32 = histogram.data_[s[1]]
  338 + var histo2 uint32 = histogram.data_[s[2]]
  339 + var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
  340 + return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
  341 + }
  342 +
  343 + if count == 4 {
  344 + var histo [4]uint32
  345 + var h23 uint32
  346 + var histomax uint32
  347 + for i = 0; i < 4; i++ {
  348 + histo[i] = histogram.data_[s[i]]
  349 + }
  350 +
  351 + /* Sort */
  352 + for i = 0; i < 4; i++ {
  353 + var j uint
  354 + for j = i + 1; j < 4; j++ {
  355 + if histo[j] > histo[i] {
  356 + var tmp uint32 = histo[j]
  357 + histo[j] = histo[i]
  358 + histo[i] = tmp
  359 + }
  360 + }
  361 + }
  362 +
  363 + h23 = histo[2] + histo[3]
  364 + histomax = brotli_max_uint32_t(h23, histo[0])
  365 + return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
  366 + }
  367 + {
  368 + var max_depth uint = 1
  369 + var depth_histo = [codeLengthCodes]uint32{0}
  370 + /* In this loop we compute the entropy of the histogram and simultaneously
  371 + build a simplified histogram of the code length codes where we use the
  372 + zero repeat code 17, but we don't use the non-zero repeat code 16. */
  373 +
  374 + var log2total float64 = fastLog2(histogram.total_count_)
  375 + for i = 0; i < data_size; {
  376 + if histogram.data_[i] > 0 {
  377 + var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
  378 + /* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
  379 + = log2(total_count) - log2(count(symbol)) */
  380 +
  381 + var depth uint = uint(log2p + 0.5)
  382 + /* Approximate the bit depth by round(-log2(P(symbol))) */
  383 + bits += float64(histogram.data_[i]) * log2p
  384 +
  385 + if depth > 15 {
  386 + depth = 15
  387 + }
  388 +
  389 + if depth > max_depth {
  390 + max_depth = depth
  391 + }
  392 +
  393 + depth_histo[depth]++
  394 + i++
  395 + } else {
  396 + var reps uint32 = 1
  397 + /* Compute the run length of zeros and add the appropriate number of 0
  398 + and 17 code length codes to the code length code histogram. */
  399 +
  400 + var k uint
  401 + for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
  402 + reps++
  403 + }
  404 +
  405 + i += uint(reps)
  406 + if i == data_size {
  407 + /* Don't add any cost for the last zero run, since these are encoded
  408 + only implicitly. */
  409 + break
  410 + }
  411 +
  412 + if reps < 3 {
  413 + depth_histo[0] += reps
  414 + } else {
  415 + reps -= 2
  416 + for reps > 0 {
  417 + depth_histo[repeatZeroCodeLength]++
  418 +
  419 + /* Add the 3 extra bits for the 17 code length code. */
  420 + bits += 3
  421 +
  422 + reps >>= 3
  423 + }
  424 + }
  425 + }
  426 + }
  427 +
  428 + /* Add the estimated encoding cost of the code length code histogram. */
  429 + bits += float64(18 + 2*max_depth)
  430 +
  431 + /* Add the entropy of the code length code histogram. */
  432 + bits += bitsEntropy(depth_histo[:], codeLengthCodes)
  433 + }
  434 +
  435 + return bits
  436 +}
  1 +package brotli
  2 +
  3 +import "encoding/binary"
  4 +
  5 +/* Copyright 2013 Google Inc. All Rights Reserved.
  6 +
  7 + Distributed under MIT license.
  8 + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  9 +*/
  10 +
  11 +/* Bit reading helpers */
  12 +
  13 +const shortFillBitWindowRead = (8 >> 1)
  14 +
  15 +var kBitMask = [33]uint32{
  16 + 0x00000000,
  17 + 0x00000001,
  18 + 0x00000003,
  19 + 0x00000007,
  20 + 0x0000000F,
  21 + 0x0000001F,
  22 + 0x0000003F,
  23 + 0x0000007F,
  24 + 0x000000FF,
  25 + 0x000001FF,
  26 + 0x000003FF,
  27 + 0x000007FF,
  28 + 0x00000FFF,
  29 + 0x00001FFF,
  30 + 0x00003FFF,
  31 + 0x00007FFF,
  32 + 0x0000FFFF,
  33 + 0x0001FFFF,
  34 + 0x0003FFFF,
  35 + 0x0007FFFF,
  36 + 0x000FFFFF,
  37 + 0x001FFFFF,
  38 + 0x003FFFFF,
  39 + 0x007FFFFF,
  40 + 0x00FFFFFF,
  41 + 0x01FFFFFF,
  42 + 0x03FFFFFF,
  43 + 0x07FFFFFF,
  44 + 0x0FFFFFFF,
  45 + 0x1FFFFFFF,
  46 + 0x3FFFFFFF,
  47 + 0x7FFFFFFF,
  48 + 0xFFFFFFFF,
  49 +}
  50 +
  51 +func bitMask(n uint32) uint32 {
  52 + return kBitMask[n]
  53 +}
  54 +
  55 +type bitReader struct {
  56 + val_ uint64
  57 + bit_pos_ uint32
  58 + input []byte
  59 + input_len uint
  60 + byte_pos uint
  61 +}
  62 +
  63 +type bitReaderState struct {
  64 + val_ uint64
  65 + bit_pos_ uint32
  66 + input []byte
  67 + input_len uint
  68 + byte_pos uint
  69 +}
  70 +
  71 +/* Initializes the BrotliBitReader fields. */
  72 +
  73 +/* Ensures that accumulator is not empty.
  74 + May consume up to sizeof(brotli_reg_t) - 1 bytes of input.
  75 + Returns false if data is required but there is no input available.
  76 + For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
  77 + reading. */
  78 +func bitReaderSaveState(from *bitReader, to *bitReaderState) {
  79 + to.val_ = from.val_
  80 + to.bit_pos_ = from.bit_pos_
  81 + to.input = from.input
  82 + to.input_len = from.input_len
  83 + to.byte_pos = from.byte_pos
  84 +}
  85 +
  86 +func bitReaderRestoreState(to *bitReader, from *bitReaderState) {
  87 + to.val_ = from.val_
  88 + to.bit_pos_ = from.bit_pos_
  89 + to.input = from.input
  90 + to.input_len = from.input_len
  91 + to.byte_pos = from.byte_pos
  92 +}
  93 +
  94 +func getAvailableBits(br *bitReader) uint32 {
  95 + return 64 - br.bit_pos_
  96 +}
  97 +
  98 +/* Returns amount of unread bytes the bit reader still has buffered from the
  99 + BrotliInput, including whole bytes in br->val_. */
  100 +func getRemainingBytes(br *bitReader) uint {
  101 + return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3))
  102 +}
  103 +
  104 +/* Checks if there is at least |num| bytes left in the input ring-buffer
  105 + (excluding the bits remaining in br->val_). */
  106 +func checkInputAmount(br *bitReader, num uint) bool {
  107 + return br.input_len-br.byte_pos >= num
  108 +}
  109 +
  110 +/* Guarantees that there are at least |n_bits| + 1 bits in accumulator.
  111 + Precondition: accumulator contains at least 1 bit.
  112 + |n_bits| should be in the range [1..24] for regular build. For portable
  113 + non-64-bit little-endian build only 16 bits are safe to request. */
  114 +func fillBitWindow(br *bitReader, n_bits uint32) {
  115 + if br.bit_pos_ >= 32 {
  116 + br.val_ >>= 32
  117 + br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */
  118 + br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32
  119 + br.byte_pos += 4
  120 + }
  121 +}
  122 +
  123 +/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
  124 + more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
  125 +func fillBitWindow16(br *bitReader) {
  126 + fillBitWindow(br, 17)
  127 +}
  128 +
  129 +/* Tries to pull one byte of input to accumulator.
  130 + Returns false if there is no input available. */
  131 +func pullByte(br *bitReader) bool {
  132 + if br.byte_pos == br.input_len {
  133 + return false
  134 + }
  135 +
  136 + br.val_ >>= 8
  137 + br.val_ |= (uint64(br.input[br.byte_pos])) << 56
  138 + br.bit_pos_ -= 8
  139 + br.byte_pos++
  140 + return true
  141 +}
  142 +
  143 +/* Returns currently available bits.
  144 + The number of valid bits could be calculated by BrotliGetAvailableBits. */
  145 +func getBitsUnmasked(br *bitReader) uint64 {
  146 + return br.val_ >> br.bit_pos_
  147 +}
  148 +
  149 +/* Like BrotliGetBits, but does not mask the result.
  150 + The result contains at least 16 valid bits. */
  151 +func get16BitsUnmasked(br *bitReader) uint32 {
  152 + fillBitWindow(br, 16)
  153 + return uint32(getBitsUnmasked(br))
  154 +}
  155 +
  156 +/* Returns the specified number of bits from |br| without advancing bit
  157 + position. */
  158 +func getBits(br *bitReader, n_bits uint32) uint32 {
  159 + fillBitWindow(br, n_bits)
  160 + return uint32(getBitsUnmasked(br)) & bitMask(n_bits)
  161 +}
  162 +
  163 +/* Tries to peek the specified amount of bits. Returns false, if there
  164 + is not enough input. */
  165 +func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool {
  166 + for getAvailableBits(br) < n_bits {
  167 + if !pullByte(br) {
  168 + return false
  169 + }
  170 + }
  171 +
  172 + *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
  173 + return true
  174 +}
  175 +
  176 +/* Advances the bit pos by |n_bits|. */
  177 +func dropBits(br *bitReader, n_bits uint32) {
  178 + br.bit_pos_ += n_bits
  179 +}
  180 +
  181 +func bitReaderUnload(br *bitReader) {
  182 + var unused_bytes uint32 = getAvailableBits(br) >> 3
  183 + var unused_bits uint32 = unused_bytes << 3
  184 + br.byte_pos -= uint(unused_bytes)
  185 + if unused_bits == 64 {
  186 + br.val_ = 0
  187 + } else {
  188 + br.val_ <<= unused_bits
  189 + }
  190 +
  191 + br.bit_pos_ += unused_bits
  192 +}
  193 +
  194 +/* Reads the specified number of bits from |br| and advances the bit pos.
  195 + Precondition: accumulator MUST contain at least |n_bits|. */
  196 +func takeBits(br *bitReader, n_bits uint32, val *uint32) {
  197 + *val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
  198 + dropBits(br, n_bits)
  199 +}
  200 +
  201 +/* Reads the specified number of bits from |br| and advances the bit pos.
  202 + Assumes that there is enough input to perform BrotliFillBitWindow. */
  203 +func readBits(br *bitReader, n_bits uint32) uint32 {
  204 + var val uint32
  205 + fillBitWindow(br, n_bits)
  206 + takeBits(br, n_bits, &val)
  207 + return val
  208 +}
  209 +
  210 +/* Tries to read the specified amount of bits. Returns false, if there
  211 + is not enough input. |n_bits| MUST be positive. */
  212 +func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool {
  213 + for getAvailableBits(br) < n_bits {
  214 + if !pullByte(br) {
  215 + return false
  216 + }
  217 + }
  218 +
  219 + takeBits(br, n_bits, val)
  220 + return true
  221 +}
  222 +
  223 +/* Advances the bit reader position to the next byte boundary and verifies
  224 + that any skipped bits are set to zero. */
  225 +func bitReaderJumpToByteBoundary(br *bitReader) bool {
  226 + var pad_bits_count uint32 = getAvailableBits(br) & 0x7
  227 + var pad_bits uint32 = 0
  228 + if pad_bits_count != 0 {
  229 + takeBits(br, pad_bits_count, &pad_bits)
  230 + }
  231 +
  232 + return pad_bits == 0
  233 +}
  234 +
  235 +/* Copies remaining input bytes stored in the bit reader to the output. Value
  236 + |num| may not be larger than BrotliGetRemainingBytes. The bit reader must be
  237 + warmed up again after this. */
  238 +func copyBytes(dest []byte, br *bitReader, num uint) {
  239 + for getAvailableBits(br) >= 8 && num > 0 {
  240 + dest[0] = byte(getBitsUnmasked(br))
  241 + dropBits(br, 8)
  242 + dest = dest[1:]
  243 + num--
  244 + }
  245 +
  246 + copy(dest, br.input[br.byte_pos:][:num])
  247 + br.byte_pos += num
  248 +}
  249 +
  250 +func initBitReader(br *bitReader) {
  251 + br.val_ = 0
  252 + br.bit_pos_ = 64
  253 +}
  254 +
  255 +func warmupBitReader(br *bitReader) bool {
  256 + /* Fixing alignment after unaligned BrotliFillWindow would result accumulator
  257 + overflow. If unalignment is caused by BrotliSafeReadBits, then there is
  258 + enough space in accumulator to fix alignment. */
  259 + if getAvailableBits(br) == 0 {
  260 + if !pullByte(br) {
  261 + return false
  262 + }
  263 + }
  264 +
  265 + return true
  266 +}
  1 +package brotli
  2 +
  3 +/* Copyright 2013 Google Inc. All Rights Reserved.
  4 +
  5 + Distributed under MIT license.
  6 + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  7 +*/
  8 +
  9 +/* Block split point selection utilities. */
  10 +
  11 +type blockSplit struct {
  12 + num_types uint
  13 + num_blocks uint
  14 + types []byte
  15 + lengths []uint32
  16 + types_alloc_size uint
  17 + lengths_alloc_size uint
  18 +}
  19 +
  20 +const (
  21 + kMaxLiteralHistograms uint = 100
  22 + kMaxCommandHistograms uint = 50
  23 + kLiteralBlockSwitchCost float64 = 28.1
  24 + kCommandBlockSwitchCost float64 = 13.5
  25 + kDistanceBlockSwitchCost float64 = 14.6
  26 + kLiteralStrideLength uint = 70
  27 + kCommandStrideLength uint = 40
  28 + kSymbolsPerLiteralHistogram uint = 544
  29 + kSymbolsPerCommandHistogram uint = 530
  30 + kSymbolsPerDistanceHistogram uint = 544
  31 + kMinLengthForBlockSplitting uint = 128
  32 + kIterMulForRefining uint = 2
  33 + kMinItersForRefining uint = 100
  34 +)
  35 +
  36 +func countLiterals(cmds []command, num_commands uint) uint {
  37 + var total_length uint = 0
  38 + /* Count how many we have. */
  39 +
  40 + var i uint
  41 + for i = 0; i < num_commands; i++ {
  42 + total_length += uint(cmds[i].insert_len_)
  43 + }
  44 +
  45 + return total_length
  46 +}
  47 +
  48 +func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) {
  49 + var pos uint = 0
  50 + var from_pos uint = offset & mask
  51 + var i uint
  52 + for i = 0; i < num_commands; i++ {
  53 + var insert_len uint = uint(cmds[i].insert_len_)
  54 + if from_pos+insert_len > mask {
  55 + var head_size uint = mask + 1 - from_pos
  56 + copy(literals[pos:], data[from_pos:][:head_size])
  57 + from_pos = 0
  58 + pos += head_size
  59 + insert_len -= head_size
  60 + }
  61 +
  62 + if insert_len > 0 {
  63 + copy(literals[pos:], data[from_pos:][:insert_len])
  64 + pos += insert_len
  65 + }
  66 +
  67 + from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask))
  68 + }
  69 +}
  70 +
  71 +func myRand(seed *uint32) uint32 {
  72 + /* Initial seed should be 7. In this case, loop length is (1 << 29). */
  73 + *seed *= 16807
  74 +
  75 + return *seed
  76 +}
  77 +
  78 +func bitCost(count uint) float64 {
  79 + if count == 0 {
  80 + return -2.0
  81 + } else {
  82 + return fastLog2(count)
  83 + }
  84 +}
  85 +
  86 +const histogramsPerBatch = 64
  87 +
  88 +const clustersPerBatch = 16
  89 +
  90 +func initBlockSplit(self *blockSplit) {
  91 + self.num_types = 0
  92 + self.num_blocks = 0
  93 + self.types = nil
  94 + self.lengths = nil
  95 + self.types_alloc_size = 0
  96 + self.lengths_alloc_size = 0
  97 +}
  98 +
  99 +func destroyBlockSplit(self *blockSplit) {
  100 + self.types = nil
  101 + self.lengths = nil
  102 +}
  103 +
  104 +func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
  105 + {
  106 + var literals_count uint = countLiterals(cmds, num_commands)
  107 + var literals []byte = make([]byte, literals_count)
  108 +
  109 + /* Create a continuous array of literals. */
  110 + copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals)
  111 +
  112 + /* Create the block split on the array of literals.
  113 + Literal histograms have alphabet size 256. */
  114 + splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split)
  115 +
  116 + literals = nil
  117 + }
  118 + {
  119 + var insert_and_copy_codes []uint16 = make([]uint16, num_commands)
  120 + /* Compute prefix codes for commands. */
  121 +
  122 + var i uint
  123 + for i = 0; i < num_commands; i++ {
  124 + insert_and_copy_codes[i] = cmds[i].cmd_prefix_
  125 + }
  126 +
  127 + /* Create the block split on the array of command prefixes. */
  128 + splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
  129 +
  130 + /* TODO: reuse for distances? */
  131 +
  132 + insert_and_copy_codes = nil
  133 + }
  134 + {
  135 + var distance_prefixes []uint16 = make([]uint16, num_commands)
  136 + var j uint = 0
  137 + /* Create a continuous array of distance prefixes. */
  138 +
  139 + var i uint
  140 + for i = 0; i < num_commands; i++ {
  141 + var cmd *command = &cmds[i]
  142 + if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
  143 + distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF
  144 + j++
  145 + }
  146 + }
  147 +
  148 + /* Create the block split on the array of distance prefixes. */
  149 + splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split)
  150 +
  151 + distance_prefixes = nil
  152 + }
  153 +}
  1 +package brotli
  2 +
  3 +import "math"
  4 +
  5 +/* Copyright 2013 Google Inc. All Rights Reserved.
  6 +
  7 + Distributed under MIT license.
  8 + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  9 +*/
  10 +
  11 +func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
  12 + var seed uint32 = 7
  13 + var block_length uint = length / num_histograms
  14 + var i uint
  15 + clearHistogramsCommand(histograms, num_histograms)
  16 + for i = 0; i < num_histograms; i++ {
  17 + var pos uint = length * i / num_histograms
  18 + if i != 0 {
  19 + pos += uint(myRand(&seed) % uint32(block_length))
  20 + }
  21 +
  22 + if pos+stride >= length {
  23 + pos = length - stride - 1
  24 + }
  25 +
  26 + histogramAddVectorCommand(&histograms[i], data[pos:], stride)
  27 + }
  28 +}
  29 +
  30 +func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) {
  31 + var pos uint = 0
  32 + if stride >= length {
  33 + stride = length
  34 + } else {
  35 + pos = uint(myRand(seed) % uint32(length-stride+1))
  36 + }
  37 +
  38 + histogramAddVectorCommand(sample, data[pos:], stride)
  39 +}
  40 +
  41 +func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
  42 + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
  43 + var seed uint32 = 7
  44 + var iter uint
  45 + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
  46 + for iter = 0; iter < iters; iter++ {
  47 + var sample histogramCommand
  48 + histogramClearCommand(&sample)
  49 + randomSampleCommand(&seed, data, length, stride, &sample)
  50 + histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample)
  51 + }
  52 +}
  53 +
  54 +/* Assigns a block id from the range [0, num_histograms) to each data element
  55 + in data[0..length) and fills in block_id[0..length) with the assigned values.
  56 + Returns the number of blocks, i.e. one plus the number of block switches. */
  57 +func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
  58 + var data_size uint = histogramDataSizeCommand()
  59 + var bitmaplen uint = (num_histograms + 7) >> 3
  60 + var num_blocks uint = 1
  61 + var i uint
  62 + var j uint
  63 + assert(num_histograms <= 256)
  64 + if num_histograms <= 1 {
  65 + for i = 0; i < length; i++ {
  66 + block_id[i] = 0
  67 + }
  68 +
  69 + return 1
  70 + }
  71 +
  72 + for i := 0; i < int(data_size*num_histograms); i++ {
  73 + insert_cost[i] = 0
  74 + }
  75 + for i = 0; i < num_histograms; i++ {
  76 + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
  77 + }
  78 +
  79 + for i = data_size; i != 0; {
  80 + i--
  81 + for j = 0; j < num_histograms; j++ {
  82 + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
  83 + }
  84 + }
  85 +
  86 + for i := 0; i < int(num_histograms); i++ {
  87 + cost[i] = 0
  88 + }
  89 + for i := 0; i < int(length*bitmaplen); i++ {
  90 + switch_signal[i] = 0
  91 + }
  92 +
  93 + /* After each iteration of this loop, cost[k] will contain the difference
  94 + between the minimum cost of arriving at the current byte position using
  95 + entropy code k, and the minimum cost of arriving at the current byte
  96 + position. This difference is capped at the block switch cost, and if it
  97 + reaches block switch cost, it means that when we trace back from the last
  98 + position, we need to switch here. */
  99 + for i = 0; i < length; i++ {
  100 + var byte_ix uint = i
  101 + var ix uint = byte_ix * bitmaplen
  102 + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
  103 + var min_cost float64 = 1e99
  104 + var block_switch_cost float64 = block_switch_bitcost
  105 + var k uint
  106 + for k = 0; k < num_histograms; k++ {
  107 + /* We are coding the symbol in data[byte_ix] with entropy code k. */
  108 + cost[k] += insert_cost[insert_cost_ix+k]
  109 +
  110 + if cost[k] < min_cost {
  111 + min_cost = cost[k]
  112 + block_id[byte_ix] = byte(k)
  113 + }
  114 + }
  115 +
  116 + /* More blocks for the beginning. */
  117 + if byte_ix < 2000 {
  118 + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
  119 + }
  120 +
  121 + for k = 0; k < num_histograms; k++ {
  122 + cost[k] -= min_cost
  123 + if cost[k] >= block_switch_cost {
  124 + var mask byte = byte(1 << (k & 7))
  125 + cost[k] = block_switch_cost
  126 + assert(k>>3 < bitmaplen)
  127 + switch_signal[ix+(k>>3)] |= mask
  128 + /* Trace back from the last position and switch at the marked places. */
  129 + }
  130 + }
  131 + }
  132 + {
  133 + var byte_ix uint = length - 1
  134 + var ix uint = byte_ix * bitmaplen
  135 + var cur_id byte = block_id[byte_ix]
  136 + for byte_ix > 0 {
  137 + var mask byte = byte(1 << (cur_id & 7))
  138 + assert(uint(cur_id)>>3 < bitmaplen)
  139 + byte_ix--
  140 + ix -= bitmaplen
  141 + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
  142 + if cur_id != block_id[byte_ix] {
  143 + cur_id = block_id[byte_ix]
  144 + num_blocks++
  145 + }
  146 + }
  147 +
  148 + block_id[byte_ix] = cur_id
  149 + }
  150 + }
  151 +
  152 + return num_blocks
  153 +}
  154 +
  155 +var remapBlockIdsCommand_kInvalidId uint16 = 256
  156 +
  157 +func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
  158 + var next_id uint16 = 0
  159 + var i uint
  160 + for i = 0; i < num_histograms; i++ {
  161 + new_id[i] = remapBlockIdsCommand_kInvalidId
  162 + }
  163 +
  164 + for i = 0; i < length; i++ {
  165 + assert(uint(block_ids[i]) < num_histograms)
  166 + if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId {
  167 + new_id[block_ids[i]] = next_id
  168 + next_id++
  169 + }
  170 + }
  171 +
  172 + for i = 0; i < length; i++ {
  173 + block_ids[i] = byte(new_id[block_ids[i]])
  174 + assert(uint(block_ids[i]) < num_histograms)
  175 + }
  176 +
  177 + assert(uint(next_id) <= num_histograms)
  178 + return uint(next_id)
  179 +}
  180 +
  181 +func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) {
  182 + var i uint
  183 + clearHistogramsCommand(histograms, num_histograms)
  184 + for i = 0; i < length; i++ {
  185 + histogramAddCommand(&histograms[block_ids[i]], uint(data[i]))
  186 + }
  187 +}
  188 +
  189 +var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32
  190 +
  191 +func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
  192 + var histogram_symbols []uint32 = make([]uint32, num_blocks)
  193 + var block_lengths []uint32 = make([]uint32, num_blocks)
  194 + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
  195 + var all_histograms_size uint = 0
  196 + var all_histograms_capacity uint = expected_num_clusters
  197 + var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity)
  198 + var cluster_size_size uint = 0
  199 + var cluster_size_capacity uint = expected_num_clusters
  200 + var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
  201 + var num_clusters uint = 0
  202 + var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch))
  203 + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
  204 + var pairs_capacity uint = max_num_pairs + 1
  205 + var pairs []histogramPair = make([]histogramPair, pairs_capacity)
  206 + var pos uint = 0
  207 + var clusters []uint32
  208 + var num_final_clusters uint
  209 + var new_index []uint32
  210 + var i uint
  211 + var sizes = [histogramsPerBatch]uint32{0}
  212 + var new_clusters = [histogramsPerBatch]uint32{0}
  213 + var symbols = [histogramsPerBatch]uint32{0}
  214 + var remap = [histogramsPerBatch]uint32{0}
  215 +
  216 + for i := 0; i < int(num_blocks); i++ {
  217 + block_lengths[i] = 0
  218 + }
  219 + {
  220 + var block_idx uint = 0
  221 + for i = 0; i < length; i++ {
  222 + assert(block_idx < num_blocks)
  223 + block_lengths[block_idx]++
  224 + if i+1 == length || block_ids[i] != block_ids[i+1] {
  225 + block_idx++
  226 + }
  227 + }
  228 +
  229 + assert(block_idx == num_blocks)
  230 + }
  231 +
  232 + for i = 0; i < num_blocks; i += histogramsPerBatch {
  233 + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
  234 + var num_new_clusters uint
  235 + var j uint
  236 + for j = 0; j < num_to_combine; j++ {
  237 + var k uint
  238 + histogramClearCommand(&histograms[j])
  239 + for k = 0; uint32(k) < block_lengths[i+j]; k++ {
  240 + histogramAddCommand(&histograms[j], uint(data[pos]))
  241 + pos++
  242 + }
  243 +
  244 + histograms[j].bit_cost_ = populationCostCommand(&histograms[j])
  245 + new_clusters[j] = uint32(j)
  246 + symbols[j] = uint32(j)
  247 + sizes[j] = 1
  248 + }
  249 +
  250 + num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
  251 + if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
  252 + var _new_size uint
  253 + if all_histograms_capacity == 0 {
  254 + _new_size = all_histograms_size + num_new_clusters
  255 + } else {
  256 + _new_size = all_histograms_capacity
  257 + }
  258 + var new_array []histogramCommand
  259 + for _new_size < (all_histograms_size + num_new_clusters) {
  260 + _new_size *= 2
  261 + }
  262 + new_array = make([]histogramCommand, _new_size)
  263 + if all_histograms_capacity != 0 {
  264 + copy(new_array, all_histograms[:all_histograms_capacity])
  265 + }
  266 +
  267 + all_histograms = new_array
  268 + all_histograms_capacity = _new_size
  269 + }
  270 +
  271 + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
  272 + for j = 0; j < num_new_clusters; j++ {
  273 + all_histograms[all_histograms_size] = histograms[new_clusters[j]]
  274 + all_histograms_size++
  275 + cluster_size[cluster_size_size] = sizes[new_clusters[j]]
  276 + cluster_size_size++
  277 + remap[new_clusters[j]] = uint32(j)
  278 + }
  279 +
  280 + for j = 0; j < num_to_combine; j++ {
  281 + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
  282 + }
  283 +
  284 + num_clusters += num_new_clusters
  285 + assert(num_clusters == cluster_size_size)
  286 + assert(num_clusters == all_histograms_size)
  287 + }
  288 +
  289 + histograms = nil
  290 +
  291 + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
  292 + if pairs_capacity < max_num_pairs+1 {
  293 + pairs = nil
  294 + pairs = make([]histogramPair, (max_num_pairs + 1))
  295 + }
  296 +
  297 + clusters = make([]uint32, num_clusters)
  298 + for i = 0; i < num_clusters; i++ {
  299 + clusters[i] = uint32(i)
  300 + }
  301 +
  302 + num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
  303 + pairs = nil
  304 + cluster_size = nil
  305 +
  306 + new_index = make([]uint32, num_clusters)
  307 + for i = 0; i < num_clusters; i++ {
  308 + new_index[i] = clusterBlocksCommand_kInvalidIndex
  309 + }
  310 + pos = 0
  311 + {
  312 + var next_index uint32 = 0
  313 + for i = 0; i < num_blocks; i++ {
  314 + var histo histogramCommand
  315 + var j uint
  316 + var best_out uint32
  317 + var best_bits float64
  318 + histogramClearCommand(&histo)
  319 + for j = 0; uint32(j) < block_lengths[i]; j++ {
  320 + histogramAddCommand(&histo, uint(data[pos]))
  321 + pos++
  322 + }
  323 +
  324 + if i == 0 {
  325 + best_out = histogram_symbols[0]
  326 + } else {
  327 + best_out = histogram_symbols[i-1]
  328 + }
  329 + best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out])
  330 + for j = 0; j < num_final_clusters; j++ {
  331 + var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]])
  332 + if cur_bits < best_bits {
  333 + best_bits = cur_bits
  334 + best_out = clusters[j]
  335 + }
  336 + }
  337 +
  338 + histogram_symbols[i] = best_out
  339 + if new_index[best_out] == clusterBlocksCommand_kInvalidIndex {
  340 + new_index[best_out] = next_index
  341 + next_index++
  342 + }
  343 + }
  344 + }
  345 +
  346 + clusters = nil
  347 + all_histograms = nil
  348 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
  349 + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
  350 + {
  351 + var cur_length uint32 = 0
  352 + var block_idx uint = 0
  353 + var max_type byte = 0
  354 + for i = 0; i < num_blocks; i++ {
  355 + cur_length += block_lengths[i]
  356 + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
  357 + var id byte = byte(new_index[histogram_symbols[i]])
  358 + split.types[block_idx] = id
  359 + split.lengths[block_idx] = cur_length
  360 + max_type = brotli_max_uint8_t(max_type, id)
  361 + cur_length = 0
  362 + block_idx++
  363 + }
  364 + }
  365 +
  366 + split.num_blocks = block_idx
  367 + split.num_types = uint(max_type) + 1
  368 + }
  369 +
  370 + new_index = nil
  371 + block_lengths = nil
  372 + histogram_symbols = nil
  373 +}
  374 +
  375 +func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
  376 + var data_size uint = histogramDataSizeCommand()
  377 + var num_histograms uint = length/literals_per_histogram + 1
  378 + var histograms []histogramCommand
  379 + if num_histograms > max_histograms {
  380 + num_histograms = max_histograms
  381 + }
  382 +
  383 + if length == 0 {
  384 + split.num_types = 1
  385 + return
  386 + } else if length < kMinLengthForBlockSplitting {
  387 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
  388 + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
  389 + split.num_types = 1
  390 + split.types[split.num_blocks] = 0
  391 + split.lengths[split.num_blocks] = uint32(length)
  392 + split.num_blocks++
  393 + return
  394 + }
  395 +
  396 + histograms = make([]histogramCommand, num_histograms)
  397 +
  398 + /* Find good entropy codes. */
  399 + initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
  400 +
  401 + refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
  402 + {
  403 + var block_ids []byte = make([]byte, length)
  404 + var num_blocks uint = 0
  405 + var bitmaplen uint = (num_histograms + 7) >> 3
  406 + var insert_cost []float64 = make([]float64, (data_size * num_histograms))
  407 + var cost []float64 = make([]float64, num_histograms)
  408 + var switch_signal []byte = make([]byte, (length * bitmaplen))
  409 + var new_id []uint16 = make([]uint16, num_histograms)
  410 + var iters uint
  411 + if params.quality < hqZopflificationQuality {
  412 + iters = 3
  413 + } else {
  414 + iters = 10
  415 + }
  416 + /* Find a good path through literals with the good entropy codes. */
  417 +
  418 + var i uint
  419 + for i = 0; i < iters; i++ {
  420 + num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
  421 + num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms)
  422 + buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms)
  423 + }
  424 +
  425 + insert_cost = nil
  426 + cost = nil
  427 + switch_signal = nil
  428 + new_id = nil
  429 + histograms = nil
  430 + clusterBlocksCommand(data, length, num_blocks, block_ids, split)
  431 + block_ids = nil
  432 + }
  433 +}
  1 +package brotli
  2 +
  3 +import "math"
  4 +
  5 +/* Copyright 2013 Google Inc. All Rights Reserved.
  6 +
  7 + Distributed under MIT license.
  8 + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  9 +*/
  10 +
  11 +func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
  12 + var seed uint32 = 7
  13 + var block_length uint = length / num_histograms
  14 + var i uint
  15 + clearHistogramsDistance(histograms, num_histograms)
  16 + for i = 0; i < num_histograms; i++ {
  17 + var pos uint = length * i / num_histograms
  18 + if i != 0 {
  19 + pos += uint(myRand(&seed) % uint32(block_length))
  20 + }
  21 +
  22 + if pos+stride >= length {
  23 + pos = length - stride - 1
  24 + }
  25 +
  26 + histogramAddVectorDistance(&histograms[i], data[pos:], stride)
  27 + }
  28 +}
  29 +
  30 +func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) {
  31 + var pos uint = 0
  32 + if stride >= length {
  33 + stride = length
  34 + } else {
  35 + pos = uint(myRand(seed) % uint32(length-stride+1))
  36 + }
  37 +
  38 + histogramAddVectorDistance(sample, data[pos:], stride)
  39 +}
  40 +
  41 +func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
  42 + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
  43 + var seed uint32 = 7
  44 + var iter uint
  45 + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
  46 + for iter = 0; iter < iters; iter++ {
  47 + var sample histogramDistance
  48 + histogramClearDistance(&sample)
  49 + randomSampleDistance(&seed, data, length, stride, &sample)
  50 + histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample)
  51 + }
  52 +}
  53 +
  54 +/* Assigns a block id from the range [0, num_histograms) to each data element
  55 + in data[0..length) and fills in block_id[0..length) with the assigned values.
  56 + Returns the number of blocks, i.e. one plus the number of block switches. */
  57 +func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
  58 + var data_size uint = histogramDataSizeDistance()
  59 + var bitmaplen uint = (num_histograms + 7) >> 3
  60 + var num_blocks uint = 1
  61 + var i uint
  62 + var j uint
  63 + assert(num_histograms <= 256)
  64 + if num_histograms <= 1 {
  65 + for i = 0; i < length; i++ {
  66 + block_id[i] = 0
  67 + }
  68 +
  69 + return 1
  70 + }
  71 +
  72 + for i := 0; i < int(data_size*num_histograms); i++ {
  73 + insert_cost[i] = 0
  74 + }
  75 + for i = 0; i < num_histograms; i++ {
  76 + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
  77 + }
  78 +
  79 + for i = data_size; i != 0; {
  80 + i--
  81 + for j = 0; j < num_histograms; j++ {
  82 + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
  83 + }
  84 + }
  85 +
  86 + for i := 0; i < int(num_histograms); i++ {
  87 + cost[i] = 0
  88 + }
  89 + for i := 0; i < int(length*bitmaplen); i++ {
  90 + switch_signal[i] = 0
  91 + }
  92 +
  93 + /* After each iteration of this loop, cost[k] will contain the difference
  94 + between the minimum cost of arriving at the current byte position using
  95 + entropy code k, and the minimum cost of arriving at the current byte
  96 + position. This difference is capped at the block switch cost, and if it
  97 + reaches block switch cost, it means that when we trace back from the last
  98 + position, we need to switch here. */
  99 + for i = 0; i < length; i++ {
  100 + var byte_ix uint = i
  101 + var ix uint = byte_ix * bitmaplen
  102 + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
  103 + var min_cost float64 = 1e99
  104 + var block_switch_cost float64 = block_switch_bitcost
  105 + var k uint
  106 + for k = 0; k < num_histograms; k++ {
  107 + /* We are coding the symbol in data[byte_ix] with entropy code k. */
  108 + cost[k] += insert_cost[insert_cost_ix+k]
  109 +
  110 + if cost[k] < min_cost {
  111 + min_cost = cost[k]
  112 + block_id[byte_ix] = byte(k)
  113 + }
  114 + }
  115 +
  116 + /* More blocks for the beginning. */
  117 + if byte_ix < 2000 {
  118 + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
  119 + }
  120 +
  121 + for k = 0; k < num_histograms; k++ {
  122 + cost[k] -= min_cost
  123 + if cost[k] >= block_switch_cost {
  124 + var mask byte = byte(1 << (k & 7))
  125 + cost[k] = block_switch_cost
  126 + assert(k>>3 < bitmaplen)
  127 + switch_signal[ix+(k>>3)] |= mask
  128 + /* Trace back from the last position and switch at the marked places. */
  129 + }
  130 + }
  131 + }
  132 + {
  133 + var byte_ix uint = length - 1
  134 + var ix uint = byte_ix * bitmaplen
  135 + var cur_id byte = block_id[byte_ix]
  136 + for byte_ix > 0 {
  137 + var mask byte = byte(1 << (cur_id & 7))
  138 + assert(uint(cur_id)>>3 < bitmaplen)
  139 + byte_ix--
  140 + ix -= bitmaplen
  141 + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
  142 + if cur_id != block_id[byte_ix] {
  143 + cur_id = block_id[byte_ix]
  144 + num_blocks++
  145 + }
  146 + }
  147 +
  148 + block_id[byte_ix] = cur_id
  149 + }
  150 + }
  151 +
  152 + return num_blocks
  153 +}
  154 +
  155 +var remapBlockIdsDistance_kInvalidId uint16 = 256
  156 +
  157 +func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
  158 + var next_id uint16 = 0
  159 + var i uint
  160 + for i = 0; i < num_histograms; i++ {
  161 + new_id[i] = remapBlockIdsDistance_kInvalidId
  162 + }
  163 +
  164 + for i = 0; i < length; i++ {
  165 + assert(uint(block_ids[i]) < num_histograms)
  166 + if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId {
  167 + new_id[block_ids[i]] = next_id
  168 + next_id++
  169 + }
  170 + }
  171 +
  172 + for i = 0; i < length; i++ {
  173 + block_ids[i] = byte(new_id[block_ids[i]])
  174 + assert(uint(block_ids[i]) < num_histograms)
  175 + }
  176 +
  177 + assert(uint(next_id) <= num_histograms)
  178 + return uint(next_id)
  179 +}
  180 +
  181 +func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) {
  182 + var i uint
  183 + clearHistogramsDistance(histograms, num_histograms)
  184 + for i = 0; i < length; i++ {
  185 + histogramAddDistance(&histograms[block_ids[i]], uint(data[i]))
  186 + }
  187 +}
  188 +
  189 +var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32
  190 +
  191 +func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
  192 + var histogram_symbols []uint32 = make([]uint32, num_blocks)
  193 + var block_lengths []uint32 = make([]uint32, num_blocks)
  194 + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
  195 + var all_histograms_size uint = 0
  196 + var all_histograms_capacity uint = expected_num_clusters
  197 + var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity)
  198 + var cluster_size_size uint = 0
  199 + var cluster_size_capacity uint = expected_num_clusters
  200 + var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
  201 + var num_clusters uint = 0
  202 + var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch))
  203 + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
  204 + var pairs_capacity uint = max_num_pairs + 1
  205 + var pairs []histogramPair = make([]histogramPair, pairs_capacity)
  206 + var pos uint = 0
  207 + var clusters []uint32
  208 + var num_final_clusters uint
  209 + var new_index []uint32
  210 + var i uint
  211 + var sizes = [histogramsPerBatch]uint32{0}
  212 + var new_clusters = [histogramsPerBatch]uint32{0}
  213 + var symbols = [histogramsPerBatch]uint32{0}
  214 + var remap = [histogramsPerBatch]uint32{0}
  215 +
  216 + for i := 0; i < int(num_blocks); i++ {
  217 + block_lengths[i] = 0
  218 + }
  219 + {
  220 + var block_idx uint = 0
  221 + for i = 0; i < length; i++ {
  222 + assert(block_idx < num_blocks)
  223 + block_lengths[block_idx]++
  224 + if i+1 == length || block_ids[i] != block_ids[i+1] {
  225 + block_idx++
  226 + }
  227 + }
  228 +
  229 + assert(block_idx == num_blocks)
  230 + }
  231 +
  232 + for i = 0; i < num_blocks; i += histogramsPerBatch {
  233 + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
  234 + var num_new_clusters uint
  235 + var j uint
  236 + for j = 0; j < num_to_combine; j++ {
  237 + var k uint
  238 + histogramClearDistance(&histograms[j])
  239 + for k = 0; uint32(k) < block_lengths[i+j]; k++ {
  240 + histogramAddDistance(&histograms[j], uint(data[pos]))
  241 + pos++
  242 + }
  243 +
  244 + histograms[j].bit_cost_ = populationCostDistance(&histograms[j])
  245 + new_clusters[j] = uint32(j)
  246 + symbols[j] = uint32(j)
  247 + sizes[j] = 1
  248 + }
  249 +
  250 + num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
  251 + if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
  252 + var _new_size uint
  253 + if all_histograms_capacity == 0 {
  254 + _new_size = all_histograms_size + num_new_clusters
  255 + } else {
  256 + _new_size = all_histograms_capacity
  257 + }
  258 + var new_array []histogramDistance
  259 + for _new_size < (all_histograms_size + num_new_clusters) {
  260 + _new_size *= 2
  261 + }
  262 + new_array = make([]histogramDistance, _new_size)
  263 + if all_histograms_capacity != 0 {
  264 + copy(new_array, all_histograms[:all_histograms_capacity])
  265 + }
  266 +
  267 + all_histograms = new_array
  268 + all_histograms_capacity = _new_size
  269 + }
  270 +
  271 + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
  272 + for j = 0; j < num_new_clusters; j++ {
  273 + all_histograms[all_histograms_size] = histograms[new_clusters[j]]
  274 + all_histograms_size++
  275 + cluster_size[cluster_size_size] = sizes[new_clusters[j]]
  276 + cluster_size_size++
  277 + remap[new_clusters[j]] = uint32(j)
  278 + }
  279 +
  280 + for j = 0; j < num_to_combine; j++ {
  281 + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
  282 + }
  283 +
  284 + num_clusters += num_new_clusters
  285 + assert(num_clusters == cluster_size_size)
  286 + assert(num_clusters == all_histograms_size)
  287 + }
  288 +
  289 + histograms = nil
  290 +
  291 + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
  292 + if pairs_capacity < max_num_pairs+1 {
  293 + pairs = nil
  294 + pairs = make([]histogramPair, (max_num_pairs + 1))
  295 + }
  296 +
  297 + clusters = make([]uint32, num_clusters)
  298 + for i = 0; i < num_clusters; i++ {
  299 + clusters[i] = uint32(i)
  300 + }
  301 +
  302 + num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
  303 + pairs = nil
  304 + cluster_size = nil
  305 +
  306 + new_index = make([]uint32, num_clusters)
  307 + for i = 0; i < num_clusters; i++ {
  308 + new_index[i] = clusterBlocksDistance_kInvalidIndex
  309 + }
  310 + pos = 0
  311 + {
  312 + var next_index uint32 = 0
  313 + for i = 0; i < num_blocks; i++ {
  314 + var histo histogramDistance
  315 + var j uint
  316 + var best_out uint32
  317 + var best_bits float64
  318 + histogramClearDistance(&histo)
  319 + for j = 0; uint32(j) < block_lengths[i]; j++ {
  320 + histogramAddDistance(&histo, uint(data[pos]))
  321 + pos++
  322 + }
  323 +
  324 + if i == 0 {
  325 + best_out = histogram_symbols[0]
  326 + } else {
  327 + best_out = histogram_symbols[i-1]
  328 + }
  329 + best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out])
  330 + for j = 0; j < num_final_clusters; j++ {
  331 + var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]])
  332 + if cur_bits < best_bits {
  333 + best_bits = cur_bits
  334 + best_out = clusters[j]
  335 + }
  336 + }
  337 +
  338 + histogram_symbols[i] = best_out
  339 + if new_index[best_out] == clusterBlocksDistance_kInvalidIndex {
  340 + new_index[best_out] = next_index
  341 + next_index++
  342 + }
  343 + }
  344 + }
  345 +
  346 + clusters = nil
  347 + all_histograms = nil
  348 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
  349 + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
  350 + {
  351 + var cur_length uint32 = 0
  352 + var block_idx uint = 0
  353 + var max_type byte = 0
  354 + for i = 0; i < num_blocks; i++ {
  355 + cur_length += block_lengths[i]
  356 + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
  357 + var id byte = byte(new_index[histogram_symbols[i]])
  358 + split.types[block_idx] = id
  359 + split.lengths[block_idx] = cur_length
  360 + max_type = brotli_max_uint8_t(max_type, id)
  361 + cur_length = 0
  362 + block_idx++
  363 + }
  364 + }
  365 +
  366 + split.num_blocks = block_idx
  367 + split.num_types = uint(max_type) + 1
  368 + }
  369 +
  370 + new_index = nil
  371 + block_lengths = nil
  372 + histogram_symbols = nil
  373 +}
  374 +
  375 +func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
  376 + var data_size uint = histogramDataSizeDistance()
  377 + var num_histograms uint = length/literals_per_histogram + 1
  378 + var histograms []histogramDistance
  379 + if num_histograms > max_histograms {
  380 + num_histograms = max_histograms
  381 + }
  382 +
  383 + if length == 0 {
  384 + split.num_types = 1
  385 + return
  386 + } else if length < kMinLengthForBlockSplitting {
  387 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
  388 + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
  389 + split.num_types = 1
  390 + split.types[split.num_blocks] = 0
  391 + split.lengths[split.num_blocks] = uint32(length)
  392 + split.num_blocks++
  393 + return
  394 + }
  395 +
  396 + histograms = make([]histogramDistance, num_histograms)
  397 +
  398 + /* Find good entropy codes. */
  399 + initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
  400 +
  401 + refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
  402 + {
  403 + var block_ids []byte = make([]byte, length)
  404 + var num_blocks uint = 0
  405 + var bitmaplen uint = (num_histograms + 7) >> 3
  406 + var insert_cost []float64 = make([]float64, (data_size * num_histograms))
  407 + var cost []float64 = make([]float64, num_histograms)
  408 + var switch_signal []byte = make([]byte, (length * bitmaplen))
  409 + var new_id []uint16 = make([]uint16, num_histograms)
  410 + var iters uint
  411 + if params.quality < hqZopflificationQuality {
  412 + iters = 3
  413 + } else {
  414 + iters = 10
  415 + }
  416 + /* Find a good path through literals with the good entropy codes. */
  417 +
  418 + var i uint
  419 + for i = 0; i < iters; i++ {
  420 + num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
  421 + num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms)
  422 + buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms)
  423 + }
  424 +
  425 + insert_cost = nil
  426 + cost = nil
  427 + switch_signal = nil
  428 + new_id = nil
  429 + histograms = nil
  430 + clusterBlocksDistance(data, length, num_blocks, block_ids, split)
  431 + block_ids = nil
  432 + }
  433 +}
  1 +package brotli
  2 +
  3 +import "math"
  4 +
  5 +/* Copyright 2013 Google Inc. All Rights Reserved.
  6 +
  7 + Distributed under MIT license.
  8 + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  9 +*/
  10 +
  11 +func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
  12 + var seed uint32 = 7
  13 + var block_length uint = length / num_histograms
  14 + var i uint
  15 + clearHistogramsLiteral(histograms, num_histograms)
  16 + for i = 0; i < num_histograms; i++ {
  17 + var pos uint = length * i / num_histograms
  18 + if i != 0 {
  19 + pos += uint(myRand(&seed) % uint32(block_length))
  20 + }
  21 +
  22 + if pos+stride >= length {
  23 + pos = length - stride - 1
  24 + }
  25 +
  26 + histogramAddVectorLiteral(&histograms[i], data[pos:], stride)
  27 + }
  28 +}
  29 +
  30 +func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) {
  31 + var pos uint = 0
  32 + if stride >= length {
  33 + stride = length
  34 + } else {
  35 + pos = uint(myRand(seed) % uint32(length-stride+1))
  36 + }
  37 +
  38 + histogramAddVectorLiteral(sample, data[pos:], stride)
  39 +}
  40 +
  41 +func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
  42 + var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
  43 + var seed uint32 = 7
  44 + var iter uint
  45 + iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
  46 + for iter = 0; iter < iters; iter++ {
  47 + var sample histogramLiteral
  48 + histogramClearLiteral(&sample)
  49 + randomSampleLiteral(&seed, data, length, stride, &sample)
  50 + histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample)
  51 + }
  52 +}
  53 +
  54 +/* Assigns a block id from the range [0, num_histograms) to each data element
  55 + in data[0..length) and fills in block_id[0..length) with the assigned values.
  56 + Returns the number of blocks, i.e. one plus the number of block switches. */
  57 +func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
  58 + var data_size uint = histogramDataSizeLiteral()
  59 + var bitmaplen uint = (num_histograms + 7) >> 3
  60 + var num_blocks uint = 1
  61 + var i uint
  62 + var j uint
  63 + assert(num_histograms <= 256)
  64 + if num_histograms <= 1 {
  65 + for i = 0; i < length; i++ {
  66 + block_id[i] = 0
  67 + }
  68 +
  69 + return 1
  70 + }
  71 +
  72 + for i := 0; i < int(data_size*num_histograms); i++ {
  73 + insert_cost[i] = 0
  74 + }
  75 + for i = 0; i < num_histograms; i++ {
  76 + insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
  77 + }
  78 +
  79 + for i = data_size; i != 0; {
  80 + i--
  81 + for j = 0; j < num_histograms; j++ {
  82 + insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
  83 + }
  84 + }
  85 +
  86 + for i := 0; i < int(num_histograms); i++ {
  87 + cost[i] = 0
  88 + }
  89 + for i := 0; i < int(length*bitmaplen); i++ {
  90 + switch_signal[i] = 0
  91 + }
  92 +
  93 + /* After each iteration of this loop, cost[k] will contain the difference
  94 + between the minimum cost of arriving at the current byte position using
  95 + entropy code k, and the minimum cost of arriving at the current byte
  96 + position. This difference is capped at the block switch cost, and if it
  97 + reaches block switch cost, it means that when we trace back from the last
  98 + position, we need to switch here. */
  99 + for i = 0; i < length; i++ {
  100 + var byte_ix uint = i
  101 + var ix uint = byte_ix * bitmaplen
  102 + var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
  103 + var min_cost float64 = 1e99
  104 + var block_switch_cost float64 = block_switch_bitcost
  105 + var k uint
  106 + for k = 0; k < num_histograms; k++ {
  107 + /* We are coding the symbol in data[byte_ix] with entropy code k. */
  108 + cost[k] += insert_cost[insert_cost_ix+k]
  109 +
  110 + if cost[k] < min_cost {
  111 + min_cost = cost[k]
  112 + block_id[byte_ix] = byte(k)
  113 + }
  114 + }
  115 +
  116 + /* More blocks for the beginning. */
  117 + if byte_ix < 2000 {
  118 + block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
  119 + }
  120 +
  121 + for k = 0; k < num_histograms; k++ {
  122 + cost[k] -= min_cost
  123 + if cost[k] >= block_switch_cost {
  124 + var mask byte = byte(1 << (k & 7))
  125 + cost[k] = block_switch_cost
  126 + assert(k>>3 < bitmaplen)
  127 + switch_signal[ix+(k>>3)] |= mask
  128 + /* Trace back from the last position and switch at the marked places. */
  129 + }
  130 + }
  131 + }
  132 + {
  133 + var byte_ix uint = length - 1
  134 + var ix uint = byte_ix * bitmaplen
  135 + var cur_id byte = block_id[byte_ix]
  136 + for byte_ix > 0 {
  137 + var mask byte = byte(1 << (cur_id & 7))
  138 + assert(uint(cur_id)>>3 < bitmaplen)
  139 + byte_ix--
  140 + ix -= bitmaplen
  141 + if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
  142 + if cur_id != block_id[byte_ix] {
  143 + cur_id = block_id[byte_ix]
  144 + num_blocks++
  145 + }
  146 + }
  147 +
  148 + block_id[byte_ix] = cur_id
  149 + }
  150 + }
  151 +
  152 + return num_blocks
  153 +}
  154 +
  155 +var remapBlockIdsLiteral_kInvalidId uint16 = 256
  156 +
  157 +func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
  158 + var next_id uint16 = 0
  159 + var i uint
  160 + for i = 0; i < num_histograms; i++ {
  161 + new_id[i] = remapBlockIdsLiteral_kInvalidId
  162 + }
  163 +
  164 + for i = 0; i < length; i++ {
  165 + assert(uint(block_ids[i]) < num_histograms)
  166 + if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId {
  167 + new_id[block_ids[i]] = next_id
  168 + next_id++
  169 + }
  170 + }
  171 +
  172 + for i = 0; i < length; i++ {
  173 + block_ids[i] = byte(new_id[block_ids[i]])
  174 + assert(uint(block_ids[i]) < num_histograms)
  175 + }
  176 +
  177 + assert(uint(next_id) <= num_histograms)
  178 + return uint(next_id)
  179 +}
  180 +
  181 +func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) {
  182 + var i uint
  183 + clearHistogramsLiteral(histograms, num_histograms)
  184 + for i = 0; i < length; i++ {
  185 + histogramAddLiteral(&histograms[block_ids[i]], uint(data[i]))
  186 + }
  187 +}
  188 +
  189 +var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32
  190 +
  191 +func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
  192 + var histogram_symbols []uint32 = make([]uint32, num_blocks)
  193 + var block_lengths []uint32 = make([]uint32, num_blocks)
  194 + var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
  195 + var all_histograms_size uint = 0
  196 + var all_histograms_capacity uint = expected_num_clusters
  197 + var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity)
  198 + var cluster_size_size uint = 0
  199 + var cluster_size_capacity uint = expected_num_clusters
  200 + var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
  201 + var num_clusters uint = 0
  202 + var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch))
  203 + var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
  204 + var pairs_capacity uint = max_num_pairs + 1
  205 + var pairs []histogramPair = make([]histogramPair, pairs_capacity)
  206 + var pos uint = 0
  207 + var clusters []uint32
  208 + var num_final_clusters uint
  209 + var new_index []uint32
  210 + var i uint
  211 + var sizes = [histogramsPerBatch]uint32{0}
  212 + var new_clusters = [histogramsPerBatch]uint32{0}
  213 + var symbols = [histogramsPerBatch]uint32{0}
  214 + var remap = [histogramsPerBatch]uint32{0}
  215 +
  216 + for i := 0; i < int(num_blocks); i++ {
  217 + block_lengths[i] = 0
  218 + }
  219 + {
  220 + var block_idx uint = 0
  221 + for i = 0; i < length; i++ {
  222 + assert(block_idx < num_blocks)
  223 + block_lengths[block_idx]++
  224 + if i+1 == length || block_ids[i] != block_ids[i+1] {
  225 + block_idx++
  226 + }
  227 + }
  228 +
  229 + assert(block_idx == num_blocks)
  230 + }
  231 +
  232 + for i = 0; i < num_blocks; i += histogramsPerBatch {
  233 + var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
  234 + var num_new_clusters uint
  235 + var j uint
  236 + for j = 0; j < num_to_combine; j++ {
  237 + var k uint
  238 + histogramClearLiteral(&histograms[j])
  239 + for k = 0; uint32(k) < block_lengths[i+j]; k++ {
  240 + histogramAddLiteral(&histograms[j], uint(data[pos]))
  241 + pos++
  242 + }
  243 +
  244 + histograms[j].bit_cost_ = populationCostLiteral(&histograms[j])
  245 + new_clusters[j] = uint32(j)
  246 + symbols[j] = uint32(j)
  247 + sizes[j] = 1
  248 + }
  249 +
  250 + num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
  251 + if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
  252 + var _new_size uint
  253 + if all_histograms_capacity == 0 {
  254 + _new_size = all_histograms_size + num_new_clusters
  255 + } else {
  256 + _new_size = all_histograms_capacity
  257 + }
  258 + var new_array []histogramLiteral
  259 + for _new_size < (all_histograms_size + num_new_clusters) {
  260 + _new_size *= 2
  261 + }
  262 + new_array = make([]histogramLiteral, _new_size)
  263 + if all_histograms_capacity != 0 {
  264 + copy(new_array, all_histograms[:all_histograms_capacity])
  265 + }
  266 +
  267 + all_histograms = new_array
  268 + all_histograms_capacity = _new_size
  269 + }
  270 +
  271 + brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
  272 + for j = 0; j < num_new_clusters; j++ {
  273 + all_histograms[all_histograms_size] = histograms[new_clusters[j]]
  274 + all_histograms_size++
  275 + cluster_size[cluster_size_size] = sizes[new_clusters[j]]
  276 + cluster_size_size++
  277 + remap[new_clusters[j]] = uint32(j)
  278 + }
  279 +
  280 + for j = 0; j < num_to_combine; j++ {
  281 + histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
  282 + }
  283 +
  284 + num_clusters += num_new_clusters
  285 + assert(num_clusters == cluster_size_size)
  286 + assert(num_clusters == all_histograms_size)
  287 + }
  288 +
  289 + histograms = nil
  290 +
  291 + max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
  292 + if pairs_capacity < max_num_pairs+1 {
  293 + pairs = nil
  294 + pairs = make([]histogramPair, (max_num_pairs + 1))
  295 + }
  296 +
  297 + clusters = make([]uint32, num_clusters)
  298 + for i = 0; i < num_clusters; i++ {
  299 + clusters[i] = uint32(i)
  300 + }
  301 +
  302 + num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
  303 + pairs = nil
  304 + cluster_size = nil
  305 +
  306 + new_index = make([]uint32, num_clusters)
  307 + for i = 0; i < num_clusters; i++ {
  308 + new_index[i] = clusterBlocksLiteral_kInvalidIndex
  309 + }
  310 + pos = 0
  311 + {
  312 + var next_index uint32 = 0
  313 + for i = 0; i < num_blocks; i++ {
  314 + var histo histogramLiteral
  315 + var j uint
  316 + var best_out uint32
  317 + var best_bits float64
  318 + histogramClearLiteral(&histo)
  319 + for j = 0; uint32(j) < block_lengths[i]; j++ {
  320 + histogramAddLiteral(&histo, uint(data[pos]))
  321 + pos++
  322 + }
  323 +
  324 + if i == 0 {
  325 + best_out = histogram_symbols[0]
  326 + } else {
  327 + best_out = histogram_symbols[i-1]
  328 + }
  329 + best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out])
  330 + for j = 0; j < num_final_clusters; j++ {
  331 + var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]])
  332 + if cur_bits < best_bits {
  333 + best_bits = cur_bits
  334 + best_out = clusters[j]
  335 + }
  336 + }
  337 +
  338 + histogram_symbols[i] = best_out
  339 + if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex {
  340 + new_index[best_out] = next_index
  341 + next_index++
  342 + }
  343 + }
  344 + }
  345 +
  346 + clusters = nil
  347 + all_histograms = nil
  348 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
  349 + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
  350 + {
  351 + var cur_length uint32 = 0
  352 + var block_idx uint = 0
  353 + var max_type byte = 0
  354 + for i = 0; i < num_blocks; i++ {
  355 + cur_length += block_lengths[i]
  356 + if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
  357 + var id byte = byte(new_index[histogram_symbols[i]])
  358 + split.types[block_idx] = id
  359 + split.lengths[block_idx] = cur_length
  360 + max_type = brotli_max_uint8_t(max_type, id)
  361 + cur_length = 0
  362 + block_idx++
  363 + }
  364 + }
  365 +
  366 + split.num_blocks = block_idx
  367 + split.num_types = uint(max_type) + 1
  368 + }
  369 +
  370 + new_index = nil
  371 + block_lengths = nil
  372 + histogram_symbols = nil
  373 +}
  374 +
  375 +func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
  376 + var data_size uint = histogramDataSizeLiteral()
  377 + var num_histograms uint = length/literals_per_histogram + 1
  378 + var histograms []histogramLiteral
  379 + if num_histograms > max_histograms {
  380 + num_histograms = max_histograms
  381 + }
  382 +
  383 + if length == 0 {
  384 + split.num_types = 1
  385 + return
  386 + } else if length < kMinLengthForBlockSplitting {
  387 + brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
  388 + brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
  389 + split.num_types = 1
  390 + split.types[split.num_blocks] = 0
  391 + split.lengths[split.num_blocks] = uint32(length)
  392 + split.num_blocks++
  393 + return
  394 + }
  395 +
  396 + histograms = make([]histogramLiteral, num_histograms)
  397 +
  398 + /* Find good entropy codes. */
  399 + initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
  400 +
  401 + refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
  402 + {
  403 + var block_ids []byte = make([]byte, length)
  404 + var num_blocks uint = 0
  405 + var bitmaplen uint = (num_histograms + 7) >> 3
  406 + var insert_cost []float64 = make([]float64, (data_size * num_histograms))
  407 + var cost []float64 = make([]float64, num_histograms)
  408 + var switch_signal []byte = make([]byte, (length * bitmaplen))
  409 + var new_id []uint16 = make([]uint16, num_histograms)
  410 + var iters uint
  411 + if params.quality < hqZopflificationQuality {
  412 + iters = 3
  413 + } else {
  414 + iters = 10
  415 + }
  416 + /* Find a good path through literals with the good entropy codes. */
  417 +
  418 + var i uint
  419 + for i = 0; i < iters; i++ {
  420 + num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
  421 + num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms)
  422 + buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms)
  423 + }
  424 +
  425 + insert_cost = nil
  426 + cost = nil
  427 + switch_signal = nil
  428 + new_id = nil
  429 + histograms = nil
  430 + clusterBlocksLiteral(data, length, num_blocks, block_ids, split)
  431 + block_ids = nil
  432 + }
  433 +}