作者 tangxvhui

更新

要显示太多修改。

为保证性能只显示 23 of 23+ 个文件。

... ... @@ -22,6 +22,6 @@
*.sum
/vendor
/*.exe~
/logs
... ...
... ... @@ -10,6 +10,6 @@ RUN ["ln","-sf","/usr/share/zoneinfo/Asia/Shanghai","/etc/localtime"]
ENV GO111MODULE on
ENV GOPROXY https://goproxy.cn
RUN ["go","mod","tidy"]
RUN ["go","build"]
RUN ["go","build","-mod=vendor"]
EXPOSE 8082
ENTRYPOINT ["./partnermg"]
\ No newline at end of file
... ...
... ... @@ -160,7 +160,7 @@ type postPurposeOrderDetail struct {
//订单编号
OrderId string `json:"orderId"`
//买家姓名
BuyerName string `json:"buyerName"`
BuyerName string `json:"buyer"`
//对应合伙人 id
PartnerId int64 `json:"partnerID"`
PartnerName string `json:"partnerName"`
... ...
package geetest
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
)
type GeetestLib struct {
CaptchaID string
PrivateKey string
Client *http.Client
}
type FailbackRegisterRespnse struct {
Success int `json:"success"`
GT string `json:"gt"`
Challenge string `json:"challenge"`
NewCaptcha int `json:"new_captcha"`
}
const (
geetestHost = "http://api.geetest.com"
registerURL = geetestHost + "/register.php"
validateURL = geetestHost + "/validate.php"
)
func MD5Encode(input string) string {
md5Instant := md5.New()
md5Instant.Write([]byte(input))
return hex.EncodeToString(md5Instant.Sum(nil))
}
// 初始化 GeetestLib
func NewGeetestLib(capthcaID string, privateKey string, timeOut time.Duration) (geetest GeetestLib){
client := &http.Client{Timeout: timeOut}
geetest = GeetestLib{capthcaID, privateKey, client}
return
}
func (g *GeetestLib) getFailBackRegisterResponse(success int, challenge string) []byte {
if challenge == "" {
challenge = hex.EncodeToString(md5.New().Sum(nil))
}
response := FailbackRegisterRespnse{
success,
g.CaptchaID,
challenge,
1,
}
res, _ := json.Marshal(response)
return res
}
func (g *GeetestLib) do(req *http.Request) (body []byte, err error) {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
var resp *http.Response
if resp, err = g.Client.Do(req); err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode >= http.StatusInternalServerError {
err = errors.New("http status code 5xx")
return
}
if body, err = ioutil.ReadAll(resp.Body); err != nil {
return
}
return
}
func (g *GeetestLib) PreProcess(userID string, userIP string) (int8, []byte) {
params := url.Values{}
params.Add("gt", g.CaptchaID)
params.Add("new_captcha", "1")
if userID != "" {
params.Add("user_id", userID)
}
if userIP != "" {
params.Add("ip_adress", userIP)
}
req, _ := http.NewRequest("GET", registerURL+"?"+params.Encode(), nil)
body, err := g.do(req)
if err != nil {
return 0, g.getFailBackRegisterResponse(0, "")
}
challenge := string(body)
if len(challenge) != 32 {
return 0, g.getFailBackRegisterResponse(0, "")
} else {
challenge = MD5Encode(challenge + g.PrivateKey)
return 1, g.getFailBackRegisterResponse(1, challenge)
}
}
func (g *GeetestLib) checkParas(challenge string, validate string, seccode string) bool {
if challenge == "" || validate == "" || seccode == "" {
return false
}
return true
}
func (g *GeetestLib) checkSuccessRes(challenge string, validate string) bool {
return MD5Encode(g.PrivateKey+"geetest"+challenge) == validate
}
func (g *GeetestLib) checkFailbackRes(challenge string, validate string) bool {
return MD5Encode(challenge) == validate
}
func (g *GeetestLib) SuccessValidate(challenge string, validate string, seccode string, userID string, userIP string) bool {
if !g.checkParas(challenge, validate, seccode) {
return false
}
if !g.checkSuccessRes(challenge, validate) {
return false
}
params := url.Values{}
params.Add("seccode", seccode)
params.Add("challenge", challenge)
params.Add("captchaid", g.CaptchaID)
params.Add("sdk", "golang_v1.0.0")
if userID != "" {
params.Add("user_id", userID)
}
if userIP != "" {
params.Add("ip_adress", userIP)
}
req, _ := http.NewRequest("POST", validateURL, strings.NewReader(params.Encode()))
body, err := g.do(req)
if err != nil {
return false
}
res := string(body)
return res == MD5Encode(seccode)
}
func (g *GeetestLib) FailbackValidate(challenge string, validate string, seccode string) bool {
if !g.checkParas(challenge, validate, seccode) {
return false
}
if !g.checkFailbackRes(challenge, validate) {
return false
}
return true
}
... ...
## Copyright 2014 Alvaro J. Genial. All rights reserved.
## Use of this source code is governed by a BSD-style
## license that can be found in the LICENSE file.
language: go
go:
- tip
- 1.6
- 1.5
- 1.4
- 1.3
# 1.2
before_install:
# - go get -v golang.org/x/tools/cmd/cover
# - go get -v golang.org/x/tools/cmd/vet
# - go get -v golang.org/x/lint/golint
- export PATH=$PATH:/home/travis/gopath/bin
script:
- go build -v ./...
- go test -v -cover ./...
- go vet ./...
# - golint .
... ...
Copyright (c) 2014 Alvaro J. Genial. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
... ...
form
====
A Form Encoding & Decoding Package for Go, written by [Alvaro J. Genial](http://alva.ro).
[![Build Status](https://travis-ci.org/ajg/form.png?branch=master)](https://travis-ci.org/ajg/form)
[![GoDoc](https://godoc.org/github.com/ajg/form?status.png)](https://godoc.org/github.com/ajg/form)
Synopsis
--------
This library is designed to allow seamless, high-fidelity encoding and decoding of arbitrary data in `application/x-www-form-urlencoded` format and as [`url.Values`](http://golang.org/pkg/net/url/#Values). It is intended to be useful primarily in dealing with web forms and URI query strings, both of which natively employ said format.
Unsurprisingly, `form` is modeled after other Go [`encoding`](http://golang.org/pkg/encoding/) packages, in particular [`encoding/json`](http://golang.org/pkg/encoding/json/), and follows the same conventions (see below for more.) It aims to automatically handle any kind of concrete Go [data value](#values) (i.e., not functions, channels, etc.) while providing mechanisms for custom behavior.
Status
------
The implementation is in usable shape and is fairly well tested with its accompanying test suite. The API is unlikely to change much, but still may. Lastly, the code has not yet undergone a security review to ensure it is free of vulnerabilities. Please file an issue or send a pull request for fixes & improvements.
Dependencies
------------
The only requirement is [Go 1.2](http://golang.org/doc/go1.2) or later.
Usage
-----
```go
import "github.com/ajg/form"
// or: "gopkg.in/ajg/form.v1"
```
Given a type like the following...
```go
type User struct {
Name string `form:"name"`
Email string `form:"email"`
Joined time.Time `form:"joined,omitempty"`
Posts []int `form:"posts"`
Preferences map[string]string `form:"prefs"`
Avatar []byte `form:"avatar"`
PasswordHash int64 `form:"-"`
}
```
...it is easy to encode data of that type...
```go
func PostUser(url string, u User) error {
var c http.Client
_, err := c.PostForm(url, form.EncodeToValues(u))
return err
}
```
...as well as decode it...
```go
func Handler(w http.ResponseWriter, r *http.Request) {
var u User
d := form.NewDecoder(r.Body)
if err := d.Decode(&u); err != nil {
http.Error(w, "Form could not be decoded", http.StatusBadRequest)
return
}
fmt.Fprintf(w, "Decoded: %#v", u)
}
```
...without having to do any grunt work.
Field Tags
----------
Like other encoding packages, `form` supports the following options for fields:
- `` `form:"-"` ``: Causes the field to be ignored during encoding and decoding.
- `` `form:"<name>"` ``: Overrides the field's name; useful especially when dealing with external identifiers in camelCase, as are commonly found on the web.
- `` `form:",omitempty"` ``: Elides the field during encoding if it is empty (typically meaning equal to the type's zero value.)
- `` `form:"<name>,omitempty"` ``: The way to combine the two options above.
Values
------
### Simple Values
Values of the following types are all considered simple:
- `bool`
- `int`, `int8`, `int16`, `int32`, `int64`, `rune`
- `uint`, `uint8`, `uint16`, `uint32`, `uint64`, `byte`
- `float32`, `float64`
- `complex64`, `complex128`
- `string`
- `[]byte` (see note)
- [`time.Time`](http://golang.org/pkg/time/#Time)
- [`url.URL`](http://golang.org/pkg/net/url/#URL)
- An alias of any of the above
- A pointer to any of the above
### Composite Values
A composite value is one that can contain other values. Values of the following kinds...
- Maps
- Slices; except `[]byte` (see note)
- Structs; except [`time.Time`](http://golang.org/pkg/time/#Time) and [`url.URL`](http://golang.org/pkg/net/url/#URL)
- Arrays
- An alias of any of the above
- A pointer to any of the above
...are considered composites in general, unless they implement custom marshaling/unmarshaling. Composite values are encoded as a flat mapping of paths to values, where the paths are constructed by joining the parent and child paths with a period (`.`).
(Note: a byte slice is treated as a `string` by default because it's more efficient, but can also be decoded as a slice—i.e., with indexes.)
### Untyped Values
While encouraged, it is not necessary to define a type (e.g. a `struct`) in order to use `form`, since it is able to encode and decode untyped data generically using the following rules:
- Simple values will be treated as a `string`.
- Composite values will be treated as a `map[string]interface{}`, itself able to contain nested values (both scalar and compound) ad infinitum.
- However, if there is a value (of any supported type) already present in a map for a given key, then it will be used when possible, rather than being replaced with a generic value as specified above; this makes it possible to handle partially typed, dynamic or schema-less values.
### Zero Values
By default, and without custom marshaling, zero values (also known as empty/default values) are encoded as the empty string. To disable this behavior, meaning to keep zero values in their literal form (e.g. `0` for integral types), `Encoder` offers a `KeepZeros` setter method, which will do just that when set to `true`.
### Unsupported Values
Values of the following kinds aren't supported and, if present, must be ignored.
- Channel
- Function
- Unsafe pointer
- An alias of any of the above
- A pointer to any of the above
Custom Marshaling
-----------------
There is a default (generally lossless) marshaling & unmarshaling scheme for any concrete data value in Go, which is good enough in most cases. However, it is possible to override it and use a custom scheme. For instance, a "binary" field could be marshaled more efficiently using [base64](http://golang.org/pkg/encoding/base64/) to prevent it from being percent-escaped during serialization to `application/x-www-form-urlencoded` format.
Because `form` provides support for [`encoding.TextMarshaler`](http://golang.org/pkg/encoding/#TextMarshaler) and [`encoding.TextUnmarshaler`](http://golang.org/pkg/encoding/#TextUnmarshaler) it is easy to do that; for instance, like this:
```go
import "encoding"
type Binary []byte
var (
_ encoding.TextMarshaler = &Binary{}
_ encoding.TextUnmarshaler = &Binary{}
)
func (b Binary) MarshalText() ([]byte, error) {
return []byte(base64.URLEncoding.EncodeToString([]byte(b))), nil
}
func (b *Binary) UnmarshalText(text []byte) error {
bs, err := base64.URLEncoding.DecodeString(string(text))
if err == nil {
*b = Binary(bs)
}
return err
}
```
Now any value with type `Binary` will automatically be encoded using the [URL](http://golang.org/pkg/encoding/base64/#URLEncoding) variant of base64. It is left as an exercise to the reader to improve upon this scheme by eliminating the need for padding (which, besides being superfluous, uses `=`, a character that will end up percent-escaped.)
Keys
----
In theory any value can be a key as long as it has a string representation. However, by default, periods have special meaning to `form`, and thus, under the hood (i.e. in encoded form) they are transparently escaped using a preceding backslash (`\`). Backslashes within keys, themselves, are also escaped in this manner (e.g. as `\\`) in order to permit representing `\.` itself (as `\\\.`).
(Note: it is normally unnecessary to deal with this issue unless keys are being constructed manually—e.g. literally embedded in HTML or in a URI.)
The default delimiter and escape characters used for encoding and decoding composite keys can be changed using the `DelimitWith` and `EscapeWith` setter methods of `Encoder` and `Decoder`, respectively. For example...
```go
package main
import (
"os"
"github.com/ajg/form"
)
func main() {
type B struct {
Qux string `form:"qux"`
}
type A struct {
FooBar B `form:"foo.bar"`
}
a := A{FooBar: B{"XYZ"}}
os.Stdout.WriteString("Default: ")
form.NewEncoder(os.Stdout).Encode(a)
os.Stdout.WriteString("\nCustom: ")
form.NewEncoder(os.Stdout).DelimitWith('/').Encode(a)
os.Stdout.WriteString("\n")
}
```
...will produce...
```
Default: foo%5C.bar.qux=XYZ
Custom: foo.bar%2Fqux=XYZ
```
(`%5C` and `%2F` represent `\` and `/`, respectively.)
Limitations
-----------
- Circular (self-referential) values are untested.
Future Work
-----------
The following items would be nice to have in the future—though they are not being worked on yet:
- An option to treat all values as if they had been tagged with `omitempty`.
- An option to automatically treat all field names in `camelCase` or `underscore_case`.
- Built-in support for the types in [`math/big`](http://golang.org/pkg/math/big/).
- Built-in support for the types in [`image/color`](http://golang.org/pkg/image/color/).
- Improve encoding/decoding by reading/writing directly from/to the `io.Reader`/`io.Writer` when possible, rather than going through an intermediate representation (i.e. `node`) which requires more memory.
(Feel free to implement any of these and then send a pull request.)
Related Work
------------
- Package [gorilla/schema](https://github.com/gorilla/schema), which only implements decoding.
- Package [google/go-querystring](https://github.com/google/go-querystring), which only implements encoding.
License
-------
This library is distributed under a BSD-style [LICENSE](./LICENSE).
... ...
TODO
====
- Document IgnoreCase and IgnoreUnknownKeys in README.
... ...
// Copyright 2014 Alvaro J. Genial. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package form
import (
"fmt"
"io"
"io/ioutil"
"net/url"
"reflect"
"strconv"
"time"
)
// NewDecoder returns a new form Decoder.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r, defaultDelimiter, defaultEscape, false, false}
}
// Decoder decodes data from a form (application/x-www-form-urlencoded).
type Decoder struct {
r io.Reader
d rune
e rune
ignoreUnknown bool
ignoreCase bool
}
// DelimitWith sets r as the delimiter used for composite keys by Decoder d and returns the latter; it is '.' by default.
func (d *Decoder) DelimitWith(r rune) *Decoder {
d.d = r
return d
}
// EscapeWith sets r as the escape used for delimiters (and to escape itself) by Decoder d and returns the latter; it is '\\' by default.
func (d *Decoder) EscapeWith(r rune) *Decoder {
d.e = r
return d
}
// Decode reads in and decodes form-encoded data into dst.
func (d Decoder) Decode(dst interface{}) error {
bs, err := ioutil.ReadAll(d.r)
if err != nil {
return err
}
vs, err := url.ParseQuery(string(bs))
if err != nil {
return err
}
v := reflect.ValueOf(dst)
return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v)))
}
// IgnoreUnknownKeys if set to true it will make the Decoder ignore values
// that are not found in the destination object instead of returning an error.
func (d *Decoder) IgnoreUnknownKeys(ignoreUnknown bool) {
d.ignoreUnknown = ignoreUnknown
}
// IgnoreCase if set to true it will make the Decoder try to set values in the
// destination object even if the case does not match.
func (d *Decoder) IgnoreCase(ignoreCase bool) {
d.ignoreCase = ignoreCase
}
// DecodeString decodes src into dst.
func (d Decoder) DecodeString(dst interface{}, src string) error {
vs, err := url.ParseQuery(src)
if err != nil {
return err
}
v := reflect.ValueOf(dst)
return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v)))
}
// DecodeValues decodes vs into dst.
func (d Decoder) DecodeValues(dst interface{}, vs url.Values) error {
v := reflect.ValueOf(dst)
return d.decodeNode(v, parseValues(d.d, d.e, vs, canIndexOrdinally(v)))
}
// DecodeString decodes src into dst.
func DecodeString(dst interface{}, src string) error {
return NewDecoder(nil).DecodeString(dst, src)
}
// DecodeValues decodes vs into dst.
func DecodeValues(dst interface{}, vs url.Values) error {
return NewDecoder(nil).DecodeValues(dst, vs)
}
func (d Decoder) decodeNode(v reflect.Value, n node) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%v", e)
}
}()
if v.Kind() == reflect.Slice {
return fmt.Errorf("could not decode directly into slice; use pointer to slice")
}
d.decodeValue(v, n)
return nil
}
func (d Decoder) decodeValue(v reflect.Value, x interface{}) {
t := v.Type()
k := v.Kind()
if k == reflect.Ptr && v.IsNil() {
v.Set(reflect.New(t.Elem()))
}
if unmarshalValue(v, x) {
return
}
empty := isEmpty(x)
switch k {
case reflect.Ptr:
d.decodeValue(v.Elem(), x)
return
case reflect.Interface:
if !v.IsNil() {
d.decodeValue(v.Elem(), x)
return
} else if empty {
return // Allow nil interfaces only if empty.
} else {
panic("form: cannot decode non-empty value into into nil interface")
}
}
if empty {
v.Set(reflect.Zero(t)) // Treat the empty string as the zero value.
return
}
switch k {
case reflect.Struct:
if t.ConvertibleTo(timeType) {
d.decodeTime(v, x)
} else if t.ConvertibleTo(urlType) {
d.decodeURL(v, x)
} else {
d.decodeStruct(v, x)
}
case reflect.Slice:
d.decodeSlice(v, x)
case reflect.Array:
d.decodeArray(v, x)
case reflect.Map:
d.decodeMap(v, x)
case reflect.Invalid, reflect.Uintptr, reflect.UnsafePointer, reflect.Chan, reflect.Func:
panic(t.String() + " has unsupported kind " + k.String())
default:
d.decodeBasic(v, x)
}
}
func (d Decoder) decodeStruct(v reflect.Value, x interface{}) {
t := v.Type()
for k, c := range getNode(x) {
if f, ok := findField(v, k, d.ignoreCase); !ok && k == "" {
panic(getString(x) + " cannot be decoded as " + t.String())
} else if !ok {
if !d.ignoreUnknown {
panic(k + " doesn't exist in " + t.String())
}
} else if !f.CanSet() {
panic(k + " cannot be set in " + t.String())
} else {
d.decodeValue(f, c)
}
}
}
func (d Decoder) decodeMap(v reflect.Value, x interface{}) {
t := v.Type()
if v.IsNil() {
v.Set(reflect.MakeMap(t))
}
for k, c := range getNode(x) {
i := reflect.New(t.Key()).Elem()
d.decodeValue(i, k)
w := v.MapIndex(i)
if w.IsValid() { // We have an actual element value to decode into.
if w.Kind() == reflect.Interface {
w = w.Elem()
}
w = reflect.New(w.Type()).Elem()
} else if t.Elem().Kind() != reflect.Interface { // The map's element type is concrete.
w = reflect.New(t.Elem()).Elem()
} else {
// The best we can do here is to decode as either a string (for scalars) or a map[string]interface {} (for the rest).
// We could try to guess the type based on the string (e.g. true/false => bool) but that'll get ugly fast,
// especially if we have to guess the kind (slice vs. array vs. map) and index type (e.g. string, int, etc.)
switch c.(type) {
case node:
w = reflect.MakeMap(stringMapType)
case string:
w = reflect.New(stringType).Elem()
default:
panic("value is neither node nor string")
}
}
d.decodeValue(w, c)
v.SetMapIndex(i, w)
}
}
func (d Decoder) decodeArray(v reflect.Value, x interface{}) {
t := v.Type()
for k, c := range getNode(x) {
i, err := strconv.Atoi(k)
if err != nil {
panic(k + " is not a valid index for type " + t.String())
}
if l := v.Len(); i >= l {
panic("index is above array size")
}
d.decodeValue(v.Index(i), c)
}
}
func (d Decoder) decodeSlice(v reflect.Value, x interface{}) {
t := v.Type()
if t.Elem().Kind() == reflect.Uint8 {
// Allow, but don't require, byte slices to be encoded as a single string.
if s, ok := x.(string); ok {
v.SetBytes([]byte(s))
return
}
}
// NOTE: Implicit indexing is currently done at the parseValues level,
// so if if an implicitKey reaches here it will always replace the last.
implicit := 0
for k, c := range getNode(x) {
var i int
if k == implicitKey {
i = implicit
implicit++
} else {
explicit, err := strconv.Atoi(k)
if err != nil {
panic(k + " is not a valid index for type " + t.String())
}
i = explicit
implicit = explicit + 1
}
// "Extend" the slice if it's too short.
if l := v.Len(); i >= l {
delta := i - l + 1
v.Set(reflect.AppendSlice(v, reflect.MakeSlice(t, delta, delta)))
}
d.decodeValue(v.Index(i), c)
}
}
func (d Decoder) decodeBasic(v reflect.Value, x interface{}) {
t := v.Type()
switch k, s := t.Kind(), getString(x); k {
case reflect.Bool:
if b, e := strconv.ParseBool(s); e == nil {
v.SetBool(b)
} else {
panic("could not parse bool from " + strconv.Quote(s))
}
case reflect.Int,
reflect.Int8,
reflect.Int16,
reflect.Int32,
reflect.Int64:
if i, e := strconv.ParseInt(s, 10, 64); e == nil {
v.SetInt(i)
} else {
panic("could not parse int from " + strconv.Quote(s))
}
case reflect.Uint,
reflect.Uint8,
reflect.Uint16,
reflect.Uint32,
reflect.Uint64:
if u, e := strconv.ParseUint(s, 10, 64); e == nil {
v.SetUint(u)
} else {
panic("could not parse uint from " + strconv.Quote(s))
}
case reflect.Float32,
reflect.Float64:
if f, e := strconv.ParseFloat(s, 64); e == nil {
v.SetFloat(f)
} else {
panic("could not parse float from " + strconv.Quote(s))
}
case reflect.Complex64,
reflect.Complex128:
var c complex128
if n, err := fmt.Sscanf(s, "%g", &c); n == 1 && err == nil {
v.SetComplex(c)
} else {
panic("could not parse complex from " + strconv.Quote(s))
}
case reflect.String:
v.SetString(s)
default:
panic(t.String() + " has unsupported kind " + k.String())
}
}
func (d Decoder) decodeTime(v reflect.Value, x interface{}) {
t := v.Type()
s := getString(x)
// TODO: Find a more efficient way to do this.
for _, f := range allowedTimeFormats {
if p, err := time.Parse(f, s); err == nil {
v.Set(reflect.ValueOf(p).Convert(v.Type()))
return
}
}
panic("cannot decode string `" + s + "` as " + t.String())
}
func (d Decoder) decodeURL(v reflect.Value, x interface{}) {
t := v.Type()
s := getString(x)
if u, err := url.Parse(s); err == nil {
v.Set(reflect.ValueOf(*u).Convert(v.Type()))
return
}
panic("cannot decode string `" + s + "` as " + t.String())
}
var allowedTimeFormats = []string{
"2006-01-02T15:04:05.999999999Z07:00",
"2006-01-02T15:04:05.999999999Z07",
"2006-01-02T15:04:05.999999999Z",
"2006-01-02T15:04:05.999999999",
"2006-01-02T15:04:05Z07:00",
"2006-01-02T15:04:05Z07",
"2006-01-02T15:04:05Z",
"2006-01-02T15:04:05",
"2006-01-02T15:04Z",
"2006-01-02T15:04",
"2006-01-02T15Z",
"2006-01-02T15",
"2006-01-02",
"2006-01",
"2006",
"15:04:05.999999999Z07:00",
"15:04:05.999999999Z07",
"15:04:05.999999999Z",
"15:04:05.999999999",
"15:04:05Z07:00",
"15:04:05Z07",
"15:04:05Z",
"15:04:05",
"15:04Z",
"15:04",
"15Z",
"15",
}
... ...
// Copyright 2014 Alvaro J. Genial. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package form
import (
"encoding"
"errors"
"fmt"
"io"
"net/url"
"reflect"
"strconv"
"strings"
"time"
)
// NewEncoder returns a new form Encoder.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{w, defaultDelimiter, defaultEscape, false}
}
// Encoder provides a way to encode to a Writer.
type Encoder struct {
w io.Writer
d rune
e rune
z bool
}
// DelimitWith sets r as the delimiter used for composite keys by Encoder e and returns the latter; it is '.' by default.
func (e *Encoder) DelimitWith(r rune) *Encoder {
e.d = r
return e
}
// EscapeWith sets r as the escape used for delimiters (and to escape itself) by Encoder e and returns the latter; it is '\\' by default.
func (e *Encoder) EscapeWith(r rune) *Encoder {
e.e = r
return e
}
// KeepZeros sets whether Encoder e should keep zero (default) values in their literal form when encoding, and returns the former; by default zero values are not kept, but are rather encoded as the empty string.
func (e *Encoder) KeepZeros(z bool) *Encoder {
e.z = z
return e
}
// Encode encodes dst as form and writes it out using the Encoder's Writer.
func (e Encoder) Encode(dst interface{}) error {
v := reflect.ValueOf(dst)
n, err := encodeToNode(v, e.z)
if err != nil {
return err
}
s := n.values(e.d, e.e).Encode()
l, err := io.WriteString(e.w, s)
switch {
case err != nil:
return err
case l != len(s):
return errors.New("could not write data completely")
}
return nil
}
// EncodeToString encodes dst as a form and returns it as a string.
func EncodeToString(dst interface{}) (string, error) {
v := reflect.ValueOf(dst)
n, err := encodeToNode(v, false)
if err != nil {
return "", err
}
vs := n.values(defaultDelimiter, defaultEscape)
return vs.Encode(), nil
}
// EncodeToValues encodes dst as a form and returns it as Values.
func EncodeToValues(dst interface{}) (url.Values, error) {
v := reflect.ValueOf(dst)
n, err := encodeToNode(v, false)
if err != nil {
return nil, err
}
vs := n.values(defaultDelimiter, defaultEscape)
return vs, nil
}
func encodeToNode(v reflect.Value, z bool) (n node, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%v", e)
}
}()
return getNode(encodeValue(v, z)), nil
}
func encodeValue(v reflect.Value, z bool) interface{} {
t := v.Type()
k := v.Kind()
if s, ok := marshalValue(v); ok {
return s
} else if !z && isEmptyValue(v) {
return "" // Treat the zero value as the empty string.
}
switch k {
case reflect.Ptr, reflect.Interface:
return encodeValue(v.Elem(), z)
case reflect.Struct:
if t.ConvertibleTo(timeType) {
return encodeTime(v)
} else if t.ConvertibleTo(urlType) {
return encodeURL(v)
}
return encodeStruct(v, z)
case reflect.Slice:
return encodeSlice(v, z)
case reflect.Array:
return encodeArray(v, z)
case reflect.Map:
return encodeMap(v, z)
case reflect.Invalid, reflect.Uintptr, reflect.UnsafePointer, reflect.Chan, reflect.Func:
panic(t.String() + " has unsupported kind " + t.Kind().String())
default:
return encodeBasic(v)
}
}
func encodeStruct(v reflect.Value, z bool) interface{} {
t := v.Type()
n := node{}
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
k, oe := fieldInfo(f)
if k == "-" {
continue
} else if fv := v.Field(i); oe && isEmptyValue(fv) {
delete(n, k)
} else {
n[k] = encodeValue(fv, z)
}
}
return n
}
func encodeMap(v reflect.Value, z bool) interface{} {
n := node{}
for _, i := range v.MapKeys() {
k := getString(encodeValue(i, z))
n[k] = encodeValue(v.MapIndex(i), z)
}
return n
}
func encodeArray(v reflect.Value, z bool) interface{} {
n := node{}
for i := 0; i < v.Len(); i++ {
n[strconv.Itoa(i)] = encodeValue(v.Index(i), z)
}
return n
}
func encodeSlice(v reflect.Value, z bool) interface{} {
t := v.Type()
if t.Elem().Kind() == reflect.Uint8 {
return string(v.Bytes()) // Encode byte slices as a single string by default.
}
n := node{}
for i := 0; i < v.Len(); i++ {
n[strconv.Itoa(i)] = encodeValue(v.Index(i), z)
}
return n
}
func encodeTime(v reflect.Value) string {
t := v.Convert(timeType).Interface().(time.Time)
if t.Year() == 0 && (t.Month() == 0 || t.Month() == 1) && (t.Day() == 0 || t.Day() == 1) {
return t.Format("15:04:05.999999999Z07:00")
} else if t.Hour() == 0 && t.Minute() == 0 && t.Second() == 0 && t.Nanosecond() == 0 {
return t.Format("2006-01-02")
}
return t.Format("2006-01-02T15:04:05.999999999Z07:00")
}
func encodeURL(v reflect.Value) string {
u := v.Convert(urlType).Interface().(url.URL)
return u.String()
}
func encodeBasic(v reflect.Value) string {
t := v.Type()
switch k := t.Kind(); k {
case reflect.Bool:
return strconv.FormatBool(v.Bool())
case reflect.Int,
reflect.Int8,
reflect.Int16,
reflect.Int32,
reflect.Int64:
return strconv.FormatInt(v.Int(), 10)
case reflect.Uint,
reflect.Uint8,
reflect.Uint16,
reflect.Uint32,
reflect.Uint64:
return strconv.FormatUint(v.Uint(), 10)
case reflect.Float32:
return strconv.FormatFloat(v.Float(), 'g', -1, 32)
case reflect.Float64:
return strconv.FormatFloat(v.Float(), 'g', -1, 64)
case reflect.Complex64, reflect.Complex128:
s := fmt.Sprintf("%g", v.Complex())
return strings.TrimSuffix(strings.TrimPrefix(s, "("), ")")
case reflect.String:
return v.String()
}
panic(t.String() + " has unsupported kind " + t.Kind().String())
}
func isEmptyValue(v reflect.Value) bool {
switch t := v.Type(); v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Complex64, reflect.Complex128:
return v.Complex() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
case reflect.Struct:
if t.ConvertibleTo(timeType) {
return v.Convert(timeType).Interface().(time.Time).IsZero()
}
return reflect.DeepEqual(v, reflect.Zero(t))
}
return false
}
// canIndexOrdinally returns whether a value contains an ordered sequence of elements.
func canIndexOrdinally(v reflect.Value) bool {
if !v.IsValid() {
return false
}
switch t := v.Type(); t.Kind() {
case reflect.Ptr, reflect.Interface:
return canIndexOrdinally(v.Elem())
case reflect.Slice, reflect.Array:
return true
}
return false
}
func fieldInfo(f reflect.StructField) (k string, oe bool) {
if f.PkgPath != "" { // Skip private fields.
return omittedKey, oe
}
k = f.Name
tag := f.Tag.Get("form")
if tag == "" {
return k, oe
}
ps := strings.SplitN(tag, ",", 2)
if ps[0] != "" {
k = ps[0]
}
if len(ps) == 2 {
oe = ps[1] == "omitempty"
}
return k, oe
}
func findField(v reflect.Value, n string, ignoreCase bool) (reflect.Value, bool) {
t := v.Type()
l := v.NumField()
var lowerN string
caseInsensitiveMatch := -1
if ignoreCase {
lowerN = strings.ToLower(n)
}
// First try named fields.
for i := 0; i < l; i++ {
f := t.Field(i)
k, _ := fieldInfo(f)
if k == omittedKey {
continue
} else if n == k {
return v.Field(i), true
} else if ignoreCase && lowerN == strings.ToLower(k) {
caseInsensitiveMatch = i
}
}
// If no exact match was found try case insensitive match.
if caseInsensitiveMatch != -1 {
return v.Field(caseInsensitiveMatch), true
}
// Then try anonymous (embedded) fields.
for i := 0; i < l; i++ {
f := t.Field(i)
k, _ := fieldInfo(f)
if k == omittedKey || !f.Anonymous { // || k != "" ?
continue
}
fv := v.Field(i)
fk := fv.Kind()
for fk == reflect.Ptr || fk == reflect.Interface {
fv = fv.Elem()
fk = fv.Kind()
}
if fk != reflect.Struct {
continue
}
if ev, ok := findField(fv, n, ignoreCase); ok {
return ev, true
}
}
return reflect.Value{}, false
}
var (
stringType = reflect.TypeOf(string(""))
stringMapType = reflect.TypeOf(map[string]interface{}{})
timeType = reflect.TypeOf(time.Time{})
timePtrType = reflect.TypeOf(&time.Time{})
urlType = reflect.TypeOf(url.URL{})
)
func skipTextMarshalling(t reflect.Type) bool {
/*// Skip time.Time because its text unmarshaling is overly rigid:
return t == timeType || t == timePtrType*/
// Skip time.Time & convertibles because its text unmarshaling is overly rigid:
return t.ConvertibleTo(timeType) || t.ConvertibleTo(timePtrType)
}
func unmarshalValue(v reflect.Value, x interface{}) bool {
if skipTextMarshalling(v.Type()) {
return false
}
tu, ok := v.Interface().(encoding.TextUnmarshaler)
if !ok && !v.CanAddr() {
return false
} else if !ok {
return unmarshalValue(v.Addr(), x)
}
s := getString(x)
if err := tu.UnmarshalText([]byte(s)); err != nil {
panic(err)
}
return true
}
func marshalValue(v reflect.Value) (string, bool) {
if skipTextMarshalling(v.Type()) {
return "", false
}
tm, ok := v.Interface().(encoding.TextMarshaler)
if !ok && !v.CanAddr() {
return "", false
} else if !ok {
return marshalValue(v.Addr())
}
bs, err := tm.MarshalText()
if err != nil {
panic(err)
}
return string(bs), true
}
... ...
// Copyright 2014 Alvaro J. Genial. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package form implements encoding and decoding of application/x-www-form-urlencoded data.
package form
const (
implicitKey = "_"
omittedKey = "-"
defaultDelimiter = '.'
defaultEscape = '\\'
)
... ...
// Copyright 2014 Alvaro J. Genial. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package form
import (
"net/url"
"strconv"
"strings"
)
type node map[string]interface{}
func (n node) values(d, e rune) url.Values {
vs := url.Values{}
n.merge(d, e, "", &vs)
return vs
}
func (n node) merge(d, e rune, p string, vs *url.Values) {
for k, x := range n {
switch y := x.(type) {
case string:
vs.Add(p+escape(d, e, k), y)
case node:
y.merge(d, e, p+escape(d, e, k)+string(d), vs)
default:
panic("value is neither string nor node")
}
}
}
// TODO: Add tests for implicit indexing.
func parseValues(d, e rune, vs url.Values, canIndexFirstLevelOrdinally bool) node {
// NOTE: Because of the flattening of potentially multiple strings to one key, implicit indexing works:
// i. At the first level; e.g. Foo.Bar=A&Foo.Bar=B becomes 0.Foo.Bar=A&1.Foo.Bar=B
// ii. At the last level; e.g. Foo.Bar._=A&Foo.Bar._=B becomes Foo.Bar.0=A&Foo.Bar.1=B
// TODO: At in-between levels; e.g. Foo._.Bar=A&Foo._.Bar=B becomes Foo.0.Bar=A&Foo.1.Bar=B
// (This last one requires that there only be one placeholder in order for it to be unambiguous.)
m := map[string]string{}
for k, ss := range vs {
indexLastLevelOrdinally := strings.HasSuffix(k, string(d)+implicitKey)
for i, s := range ss {
if canIndexFirstLevelOrdinally {
k = strconv.Itoa(i) + string(d) + k
} else if indexLastLevelOrdinally {
k = strings.TrimSuffix(k, implicitKey) + strconv.Itoa(i)
}
m[k] = s
}
}
n := node{}
for k, s := range m {
n = n.split(d, e, k, s)
}
return n
}
func splitPath(d, e rune, path string) (k, rest string) {
esc := false
for i, r := range path {
switch {
case !esc && r == e:
esc = true
case !esc && r == d:
return unescape(d, e, path[:i]), path[i+1:]
default:
esc = false
}
}
return unescape(d, e, path), ""
}
func (n node) split(d, e rune, path, s string) node {
k, rest := splitPath(d, e, path)
if rest == "" {
return add(n, k, s)
}
if _, ok := n[k]; !ok {
n[k] = node{}
}
c := getNode(n[k])
n[k] = c.split(d, e, rest, s)
return n
}
func add(n node, k, s string) node {
if n == nil {
return node{k: s}
}
if _, ok := n[k]; ok {
panic("key " + k + " already set")
}
n[k] = s
return n
}
func isEmpty(x interface{}) bool {
switch y := x.(type) {
case string:
return y == ""
case node:
if s, ok := y[""].(string); ok {
return s == ""
}
return false
}
panic("value is neither string nor node")
}
func getNode(x interface{}) node {
switch y := x.(type) {
case string:
return node{"": y}
case node:
return y
}
panic("value is neither string nor node")
}
func getString(x interface{}) string {
switch y := x.(type) {
case string:
return y
case node:
if s, ok := y[""].(string); ok {
return s
}
return ""
}
panic("value is neither string nor node")
}
func escape(d, e rune, s string) string {
s = strings.Replace(s, string(e), string(e)+string(e), -1) // Escape the escape (\ => \\)
s = strings.Replace(s, string(d), string(e)+string(d), -1) // Escape the delimiter (. => \.)
return s
}
func unescape(d, e rune, s string) string {
s = strings.Replace(s, string(e)+string(d), string(d), -1) // Unescape the delimiter (\. => .)
s = strings.Replace(s, string(e)+string(e), string(e), -1) // Unescape the escape (\\ => \)
return s
}
... ...
#!/bin/bash -eu
# TODO: Only colorize messages given a suitable terminal.
# FIXME: Handle case in which no stash entry is created due to no changes.
printf "\e[30m=== PRE-COMMIT STARTING ===\e[m\n"
git stash save --quiet --keep-index --include-untracked
if go build -v ./... && go test -v -cover ./... && go vet ./... && golint . && travis-lint; then
result=$?
printf "\e[32m=== PRE-COMMIT SUCCEEDED ===\e[m\n"
else
result=$?
printf "\e[31m=== PRE-COMMIT FAILED ===\e[m\n"
fi
git stash pop --quiet
exit $result
... ...
Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
... ...
This package is a brotli compressor and decompressor implemented in Go.
It was translated from the reference implementation (https://github.com/google/brotli)
with the `c2go` tool at https://github.com/andybalholm/c2go.
I am using it in production with https://github.com/andybalholm/redwood.
... ...
package brotli
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint {
if distance <= max_distance {
var distance_plus_3 uint = distance + 3
var offset0 uint = distance_plus_3 - uint(dist_cache[0])
var offset1 uint = distance_plus_3 - uint(dist_cache[1])
if distance == uint(dist_cache[0]) {
return 0
} else if distance == uint(dist_cache[1]) {
return 1
} else if offset0 < 7 {
return (0x9750468 >> (4 * offset0)) & 0xF
} else if offset1 < 7 {
return (0xFDB1ACE >> (4 * offset1)) & 0xF
} else if distance == uint(dist_cache[2]) {
return 2
} else if distance == uint(dist_cache[3]) {
return 3
}
}
return distance + numDistanceShortCodes - 1
}
/* "commands" points to the next output command to write to, "*num_commands" is
initially the total amount of commands output by previous
CreateBackwardReferences calls, and must be incremented by the amount written
by this call. */
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var orig_commands []command = commands
var insert_length uint = *last_insert_len
var pos_end uint = position + num_bytes
var store_end uint
if num_bytes >= hasher.StoreLookahead() {
store_end = position + num_bytes - hasher.StoreLookahead() + 1
} else {
store_end = position
}
var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params)
var apply_random_heuristics uint = position + random_heuristics_window_size
var gap uint = 0
/* Set maximum distance, see section 9.1. of the spec. */
const kMinScore uint = scoreBase + 100
/* For speed up heuristics for random data. */
/* Minimum score to accept a backward reference. */
hasher.PrepareDistanceCache(dist_cache)
var sr2 hasherSearchResult
var sr hasherSearchResult
for position+hasher.HashTypeLength() < pos_end {
var max_length uint = pos_end - position
var max_distance uint = brotli_min_size_t(position, max_backward_limit)
sr.len = 0
sr.len_code_delta = 0
sr.distance = 0
sr.score = kMinScore
hasher.FindLongestMatch(&params.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, &sr)
if sr.score > kMinScore {
/* Found a match. Let's look for something even better ahead. */
var delayed_backward_references_in_row int = 0
max_length--
for ; ; max_length-- {
var cost_diff_lazy uint = 175
if params.quality < minQualityForExtensiveReferenceSearch {
sr2.len = brotli_min_size_t(sr.len-1, max_length)
} else {
sr2.len = 0
}
sr2.len_code_delta = 0
sr2.distance = 0
sr2.score = kMinScore
max_distance = brotli_min_size_t(position+1, max_backward_limit)
hasher.FindLongestMatch(&params.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, &sr2)
if sr2.score >= sr.score+cost_diff_lazy {
/* Ok, let's just write one byte for now and start a match from the
next byte. */
position++
insert_length++
sr = sr2
delayed_backward_references_in_row++
if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end {
continue
}
}
break
}
apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size
max_distance = brotli_min_size_t(position, max_backward_limit)
{
/* The first 16 codes are special short-codes,
and the minimum offset is 1. */
var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache)
if (sr.distance <= (max_distance + gap)) && distance_code > 0 {
dist_cache[3] = dist_cache[2]
dist_cache[2] = dist_cache[1]
dist_cache[1] = dist_cache[0]
dist_cache[0] = int(sr.distance)
hasher.PrepareDistanceCache(dist_cache)
}
initCommand(&commands[0], &params.dist, insert_length, sr.len, sr.len_code_delta, distance_code)
commands = commands[1:]
}
*num_literals += insert_length
insert_length = 0
/* Put the hash keys into the table, if there are enough bytes left.
Depending on the hasher implementation, it can push all positions
in the given range or only a subset of them.
Avoid hash poisoning with RLE data. */
{
var range_start uint = position + 2
var range_end uint = brotli_min_size_t(position+sr.len, store_end)
if sr.distance < sr.len>>2 {
range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2)))
}
hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end)
}
position += sr.len
} else {
insert_length++
position++
/* If we have not seen matches for a long time, we can skip some
match lookups. Unsuccessful match lookups are very very expensive
and this kind of a heuristic speeds up compression quite
a lot. */
if position > apply_random_heuristics {
/* Going through uncompressible data, jump. */
if position > apply_random_heuristics+4*random_heuristics_window_size {
var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4)
/* It is quite a long time since we saw a copy, so we assume
that this data is not compressible, and store hashes less
often. Hashes of non compressible data are less likely to
turn out to be useful in the future, too, so we store less of
them to not to flood out the hash table of good compressible
data. */
var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin)
for ; position < pos_jump; position += 4 {
hasher.Store(ringbuffer, ringbuffer_mask, position)
insert_length += 4
}
} else {
var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2)
var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin)
for ; position < pos_jump; position += 2 {
hasher.Store(ringbuffer, ringbuffer_mask, position)
insert_length += 2
}
}
}
}
}
insert_length += pos_end - position
*last_insert_len = insert_length
*num_commands += uint(-cap(commands) + cap(orig_commands))
}
... ...
package brotli
import "math"
type zopfliNode struct {
length uint32
distance uint32
dcode_insert_length uint32
u struct {
cost float32
next uint32
shortcut uint32
}
}
const maxEffectiveDistanceAlphabetSize = 544
const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1}
var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3}
func initZopfliNodes(array []zopfliNode, length uint) {
var stub zopfliNode
var i uint
stub.length = 1
stub.distance = 0
stub.dcode_insert_length = 0
stub.u.cost = kInfinity
for i = 0; i < length; i++ {
array[i] = stub
}
}
func zopfliNodeCopyLength(self *zopfliNode) uint32 {
return self.length & 0x1FFFFFF
}
func zopfliNodeLengthCode(self *zopfliNode) uint32 {
var modifier uint32 = self.length >> 25
return zopfliNodeCopyLength(self) + 9 - modifier
}
func zopfliNodeCopyDistance(self *zopfliNode) uint32 {
return self.distance
}
func zopfliNodeDistanceCode(self *zopfliNode) uint32 {
var short_code uint32 = self.dcode_insert_length >> 27
if short_code == 0 {
return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1
} else {
return short_code - 1
}
}
func zopfliNodeCommandLength(self *zopfliNode) uint32 {
return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF)
}
/* Histogram based cost model for zopflification. */
type zopfliCostModel struct {
cost_cmd_ [numCommandSymbols]float32
cost_dist_ []float32
distance_histogram_size uint32
literal_costs_ []float32
min_cost_cmd_ float32
num_bytes_ uint
}
func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) {
var distance_histogram_size uint32 = dist.alphabet_size
if distance_histogram_size > maxEffectiveDistanceAlphabetSize {
distance_histogram_size = maxEffectiveDistanceAlphabetSize
}
self.num_bytes_ = num_bytes
self.literal_costs_ = make([]float32, (num_bytes + 2))
self.cost_dist_ = make([]float32, (dist.alphabet_size))
self.distance_histogram_size = distance_histogram_size
}
func cleanupZopfliCostModel(self *zopfliCostModel) {
self.literal_costs_ = nil
self.cost_dist_ = nil
}
func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) {
var sum uint = 0
var missing_symbol_sum uint
var log2sum float32
var missing_symbol_cost float32
var i uint
for i = 0; i < histogram_size; i++ {
sum += uint(histogram[i])
}
log2sum = float32(fastLog2(sum))
missing_symbol_sum = sum
if !literal_histogram {
for i = 0; i < histogram_size; i++ {
if histogram[i] == 0 {
missing_symbol_sum++
}
}
}
missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2
for i = 0; i < histogram_size; i++ {
if histogram[i] == 0 {
cost[i] = missing_symbol_cost
continue
}
/* Shannon bits for this symbol. */
cost[i] = log2sum - float32(fastLog2(uint(histogram[i])))
/* Cannot be coded with less than 1 bit */
if cost[i] < 1 {
cost[i] = 1
}
}
}
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) {
var histogram_literal [numLiteralSymbols]uint32
var histogram_cmd [numCommandSymbols]uint32
var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
var cost_literal [numLiteralSymbols]float32
var pos uint = position - last_insert_len
var min_cost_cmd float32 = kInfinity
var i uint
var cost_cmd []float32 = self.cost_cmd_[:]
var literal_costs []float32
histogram_literal = [numLiteralSymbols]uint32{}
histogram_cmd = [numCommandSymbols]uint32{}
histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
for i = 0; i < num_commands; i++ {
var inslength uint = uint(commands[i].insert_len_)
var copylength uint = uint(commandCopyLen(&commands[i]))
var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
var cmdcode uint = uint(commands[i].cmd_prefix_)
var j uint
histogram_cmd[cmdcode]++
if cmdcode >= 128 {
histogram_dist[distcode]++
}
for j = 0; j < inslength; j++ {
histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++
}
pos += inslength + copylength
}
setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:])
setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
for i = 0; i < numCommandSymbols; i++ {
min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
}
self.min_cost_cmd_ = min_cost_cmd
{
literal_costs = self.literal_costs_
var literal_carry float32 = 0.0
var num_bytes uint = self.num_bytes_
literal_costs[0] = 0.0
for i = 0; i < num_bytes; i++ {
literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]]
literal_costs[i+1] = literal_costs[i] + literal_carry
literal_carry -= literal_costs[i+1] - literal_costs[i]
}
}
}
func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) {
var literal_costs []float32 = self.literal_costs_
var literal_carry float32 = 0.0
var cost_dist []float32 = self.cost_dist_
var cost_cmd []float32 = self.cost_cmd_[:]
var num_bytes uint = self.num_bytes_
var i uint
estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:])
literal_costs[0] = 0.0
for i = 0; i < num_bytes; i++ {
literal_carry += literal_costs[i+1]
literal_costs[i+1] = literal_costs[i] + literal_carry
literal_carry -= literal_costs[i+1] - literal_costs[i]
}
for i = 0; i < numCommandSymbols; i++ {
cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i))))
}
for i = 0; uint32(i) < self.distance_histogram_size; i++ {
cost_dist[i] = float32(fastLog2(uint(20 + uint32(i))))
}
self.min_cost_cmd_ = float32(fastLog2(11))
}
func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 {
return self.cost_cmd_[cmdcode]
}
func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 {
return self.cost_dist_[distcode]
}
func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 {
return self.literal_costs_[to] - self.literal_costs_[from]
}
func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 {
return self.min_cost_cmd_
}
/* REQUIRES: len >= 2, start_pos <= pos */
/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */
/* Maintains the "ZopfliNode array invariant". */
func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) {
var next *zopfliNode = &nodes[pos+len]
next.length = uint32(len | (len+9-len_code)<<25)
next.distance = uint32(dist)
next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos))
next.u.cost = cost
}
type posData struct {
pos uint
distance_cache [4]int
costdiff float32
cost float32
}
/* Maintains the smallest 8 cost difference together with their positions */
type startPosQueue struct {
q_ [8]posData
idx_ uint
}
func initStartPosQueue(self *startPosQueue) {
self.idx_ = 0
}
func startPosQueueSize(self *startPosQueue) uint {
return brotli_min_size_t(self.idx_, 8)
}
func startPosQueuePush(self *startPosQueue, posdata *posData) {
var offset uint = ^(self.idx_) & 7
self.idx_++
var len uint = startPosQueueSize(self)
var i uint
var q []posData = self.q_[:]
q[offset] = *posdata
/* Restore the sorted order. In the list of |len| items at most |len - 1|
adjacent element comparisons / swaps are required. */
for i = 1; i < len; i++ {
if q[offset&7].costdiff > q[(offset+1)&7].costdiff {
var tmp posData = q[offset&7]
q[offset&7] = q[(offset+1)&7]
q[(offset+1)&7] = tmp
}
offset++
}
}
func startPosQueueAt(self *startPosQueue, k uint) *posData {
return &self.q_[(k-self.idx_)&7]
}
/* Returns the minimum possible copy length that can improve the cost of any */
/* future position. */
func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint {
var min_cost float32 = start_cost
var len uint = 2
var next_len_bucket uint = 4
/* Compute the minimum possible cost of reaching any future position. */
var next_len_offset uint = 10
for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost {
/* We already reached (pos + len) with no more cost than the minimum
possible cost of reaching anything from this pos, so there is no point in
looking for lengths <= len. */
len++
if len == next_len_offset {
/* We reached the next copy length code bucket, so we add one more
extra bit to the minimum cost. */
min_cost += 1.0
next_len_offset += next_len_bucket
next_len_bucket *= 2
}
}
return uint(len)
}
/* REQUIRES: nodes[pos].cost < kInfinity
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 {
var clen uint = uint(zopfliNodeCopyLength(&nodes[pos]))
var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF)
var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos]))
/* Since |block_start + pos| is the end position of the command, the copy part
starts from |block_start + pos - clen|. Distances that are greater than
this or greater than |max_backward_limit| + |gap| are static dictionary
references, and do not update the last distances.
Also distance code 0 (last distance) does not update the last distances. */
if pos == 0 {
return 0
} else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 {
return uint32(pos)
} else {
return nodes[pos-clen-ilen].u.shortcut
}
}
/* Fills in dist_cache[0..3] with the last four distances (as defined by
Section 4. of the Spec) that would be used at (block_start + pos) if we
used the shortest path of commands from block_start, computed from
nodes[0..pos]. The last four distances at block_start are in
starting_dist_cache[0..3].
REQUIRES: nodes[pos].cost < kInfinity
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) {
var idx int = 0
var p uint = uint(nodes[pos].u.shortcut)
for idx < 4 && p > 0 {
var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF)
var clen uint = uint(zopfliNodeCopyLength(&nodes[p]))
var dist uint = uint(zopfliNodeCopyDistance(&nodes[p]))
dist_cache[idx] = int(dist)
idx++
/* Because of prerequisite, p >= clen + ilen >= 2. */
p = uint(nodes[p-clen-ilen].u.shortcut)
}
for ; idx < 4; idx++ {
dist_cache[idx] = starting_dist_cache[0]
starting_dist_cache = starting_dist_cache[1:]
}
}
/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it
is eligible. */
func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) {
/* Save cost, because ComputeDistanceCache invalidates it. */
var node_cost float32 = nodes[pos].u.cost
nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes)
if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) {
var posdata posData
posdata.pos = pos
posdata.cost = node_cost
posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos)
computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:])
startPosQueuePush(queue, &posdata)
}
}
/* Returns longest copy length. */
func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint {
var cur_ix uint = block_start + pos
var cur_ix_masked uint = cur_ix & ringbuffer_mask
var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit)
var max_len uint = num_bytes - pos
var max_zopfli_len uint = maxZopfliLen(params)
var max_iters uint = maxZopfliCandidates(params)
var min_len uint
var result uint = 0
var k uint
var gap uint = 0
evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes)
{
var posdata *posData = startPosQueueAt(queue, 0)
var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos))
min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos)
}
/* Go over the command starting positions in order of increasing cost
difference. */
for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ {
var posdata *posData = startPosQueueAt(queue, k)
var start uint = posdata.pos
var inscode uint16 = getInsertLengthCode(pos - start)
var start_costdiff float32 = posdata.costdiff
var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos)
var best_len uint = min_len - 1
var j uint = 0
/* Look for last distance matches using the distance cache from this
starting position. */
for ; j < numDistanceShortCodes && best_len < max_len; j++ {
var idx uint = uint(kDistanceCacheIndex[j])
var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j])
var prev_ix uint = cur_ix - backward
var len uint = 0
var continuation byte = ringbuffer[cur_ix_masked+best_len]
if cur_ix_masked+best_len > ringbuffer_mask {
break
}
if backward > max_distance+gap {
/* Word dictionary -> ignore. */
continue
}
if backward <= max_distance {
/* Regular backward reference. */
if prev_ix >= cur_ix {
continue
}
prev_ix &= ringbuffer_mask
if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] {
continue
}
len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len)
} else {
continue
}
{
var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j)
var l uint
for l = best_len + 1; l <= len; l++ {
var copycode uint16 = getCopyLengthCode(l)
var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0)
var tmp float32
if cmdcode < 128 {
tmp = base_cost
} else {
tmp = dist_cost
}
var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
if cost < nodes[pos+l].u.cost {
updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost)
result = brotli_max_size_t(result, l)
}
best_len = l
}
}
}
/* At higher iterations look only for new last distance matches, since
looking only for new command start positions with the same distances
does not help much. */
if k >= 2 {
continue
}
{
/* Loop through all possible copy lengths at this position. */
var len uint = min_len
for j = 0; j < num_matches; j++ {
var match backwardMatch = matches[j]
var dist uint = uint(match.distance)
var is_dictionary_match bool = (dist > max_distance+gap)
var dist_code uint = dist + numDistanceShortCodes - 1
var dist_symbol uint16
var distextra uint32
var distnumextra uint32
var dist_cost float32
var max_match_len uint
/* We already tried all possible last distance matches, so we can use
normal distance code here. */
prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra)
distnumextra = uint32(dist_symbol) >> 10
dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF)
/* Try all copy lengths up until the maximum copy length corresponding
to this distance. If the distance refers to the static dictionary, or
the maximum length is long enough, try only one maximum length. */
max_match_len = backwardMatchLength(&match)
if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) {
len = max_match_len
}
for ; len <= max_match_len; len++ {
var len_code uint
if is_dictionary_match {
len_code = backwardMatchLengthCode(&match)
} else {
len_code = len
}
var copycode uint16 = getCopyLengthCode(len_code)
var cmdcode uint16 = combineLengthCodes(inscode, copycode, false)
var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode)
if cost < nodes[pos+len].u.cost {
updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost)
result = brotli_max_size_t(result, uint(len))
}
}
}
}
}
return result
}
func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
var index uint = num_bytes
var num_commands uint = 0
for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 {
index--
}
nodes[index].u.next = math.MaxUint32
for index != 0 {
var len uint = uint(zopfliNodeCommandLength(&nodes[index]))
index -= uint(len)
nodes[index].u.next = uint32(len)
num_commands++
}
return num_commands
}
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var pos uint = 0
var offset uint32 = nodes[0].u.next
var i uint
var gap uint = 0
for i = 0; offset != math.MaxUint32; i++ {
var next *zopfliNode = &nodes[uint32(pos)+offset]
var copy_length uint = uint(zopfliNodeCopyLength(next))
var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF)
pos += insert_length
offset = next.u.next
if i == 0 {
insert_length += *last_insert_len
*last_insert_len = 0
}
{
var distance uint = uint(zopfliNodeCopyDistance(next))
var len_code uint = uint(zopfliNodeLengthCode(next))
var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
var is_dictionary bool = (distance > max_distance+gap)
var dist_code uint = uint(zopfliNodeDistanceCode(next))
initCommand(&commands[i], &params.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)
if !is_dictionary && dist_code > 0 {
dist_cache[3] = dist_cache[2]
dist_cache[2] = dist_cache[1]
dist_cache[1] = dist_cache[0]
dist_cache[0] = int(distance)
}
}
*num_literals += insert_length
pos += copy_length
}
*last_insert_len += num_bytes - pos
}
func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var max_zopfli_len uint = maxZopfliLen(params)
var queue startPosQueue
var cur_match_pos uint = 0
var i uint
nodes[0].length = 0
nodes[0].u.cost = 0
initStartPosQueue(&queue)
for i = 0; i+3 < num_bytes; i++ {
var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes)
if skip < longCopyQuickStep {
skip = 0
}
cur_match_pos += uint(num_matches[i])
if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len {
skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip)
}
if skip > 1 {
skip--
for skip != 0 {
i++
if i+3 >= num_bytes {
break
}
evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes)
cur_match_pos += uint(num_matches[i])
skip--
}
}
}
return computeShortestPathFromNodes(num_bytes, nodes)
}
/* Computes the shortest path of commands from position to at most
position + num_bytes.
On return, path->size() is the number of commands found and path[i] is the
length of the i-th command (copy length plus insert length).
Note that the sum of the lengths of all commands can be less than num_bytes.
On return, the nodes[0..num_bytes] array will have the following
"ZopfliNode array invariant":
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
(1) nodes[i].copy_length() >= 2
(2) nodes[i].command_length() <= i and
(3) nodes[i - nodes[i].command_length()].cost < kInfinity
REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var max_zopfli_len uint = maxZopfliLen(params)
var model zopfliCostModel
var queue startPosQueue
var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch
var store_end uint
if num_bytes >= hasher.StoreLookahead() {
store_end = position + num_bytes - hasher.StoreLookahead() + 1
} else {
store_end = position
}
var i uint
var gap uint = 0
var lz_matches_offset uint = 0
nodes[0].length = 0
nodes[0].u.cost = 0
initZopfliCostModel(&model, &params.dist, num_bytes)
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
initStartPosQueue(&queue)
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
var pos uint = position + i
var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
var skip uint
var num_matches uint
num_matches = findAllMatchesH10(hasher, &params.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:])
if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len {
matches[0] = matches[num_matches-1]
num_matches = 1
}
skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes)
if skip < longCopyQuickStep {
skip = 0
}
if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len {
skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip)
}
if skip > 1 {
/* Add the tail of the copy to the hasher. */
hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end))
skip--
for skip != 0 {
i++
if i+hasher.HashTypeLength()-1 >= num_bytes {
break
}
evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes)
skip--
}
}
}
cleanupZopfliCostModel(&model)
return computeShortestPathFromNodes(num_bytes, nodes)
}
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var nodes []zopfliNode
nodes = make([]zopfliNode, (num_bytes + 1))
initZopfliNodes(nodes, num_bytes+1)
*num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
nodes = nil
}
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var num_matches []uint32 = make([]uint32, num_bytes)
var matches_size uint = 4 * num_bytes
var store_end uint
if num_bytes >= hasher.StoreLookahead() {
store_end = position + num_bytes - hasher.StoreLookahead() + 1
} else {
store_end = position
}
var cur_match_pos uint = 0
var i uint
var orig_num_literals uint
var orig_last_insert_len uint
var orig_dist_cache [4]int
var orig_num_commands uint
var model zopfliCostModel
var nodes []zopfliNode
var matches []backwardMatch = make([]backwardMatch, matches_size)
var gap uint = 0
var shadow_matches uint = 0
var new_array []backwardMatch
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
var pos uint = position + i
var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
var max_length uint = num_bytes - i
var num_found_matches uint
var cur_match_end uint
var j uint
/* Ensure that we have enough free slots. */
if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
var new_size uint = matches_size
if new_size == 0 {
new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches
}
for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
new_size *= 2
}
new_array = make([]backwardMatch, new_size)
if matches_size != 0 {
copy(new_array, matches[:matches_size])
}
matches = new_array
matches_size = new_size
}
num_found_matches = findAllMatchesH10(hasher.(*h10), &params.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:])
cur_match_end = cur_match_pos + num_found_matches
for j = cur_match_pos; j+1 < cur_match_end; j++ {
assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1]))
}
num_matches[i] = uint32(num_found_matches)
if num_found_matches > 0 {
var match_len uint = backwardMatchLength(&matches[cur_match_end-1])
if match_len > maxZopfliLenQuality11 {
var skip uint = match_len - 1
matches[cur_match_pos] = matches[cur_match_end-1]
cur_match_pos++
num_matches[i] = 1
/* Add the tail of the copy to the hasher. */
hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end))
var pos uint = i
for i := 0; i < int(skip); i++ {
num_matches[pos+1:][i] = 0
}
i += skip
} else {
cur_match_pos = cur_match_end
}
}
}
orig_num_literals = *num_literals
orig_last_insert_len = *last_insert_len
copy(orig_dist_cache[:], dist_cache[:4])
orig_num_commands = *num_commands
nodes = make([]zopfliNode, (num_bytes + 1))
initZopfliCostModel(&model, &params.dist, num_bytes)
for i = 0; i < 2; i++ {
initZopfliNodes(nodes, num_bytes+1)
if i == 0 {
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
} else {
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len)
}
*num_commands = orig_num_commands
*num_literals = orig_num_literals
*last_insert_len = orig_last_insert_len
copy(dist_cache, orig_dist_cache[:4])
*num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
}
cleanupZopfliCostModel(&model)
nodes = nil
matches = nil
num_matches = nil
}
... ...
package brotli
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions to estimate the bit cost of Huffman trees. */
func shannonEntropy(population []uint32, size uint, total *uint) float64 {
var sum uint = 0
var retval float64 = 0
var population_end []uint32 = population[size:]
var p uint
for -cap(population) < -cap(population_end) {
p = uint(population[0])
population = population[1:]
sum += p
retval -= float64(p) * fastLog2(p)
}
if sum != 0 {
retval += float64(sum) * fastLog2(sum)
}
*total = sum
return retval
}
func bitsEntropy(population []uint32, size uint) float64 {
var sum uint
var retval float64 = shannonEntropy(population, size, &sum)
if retval < float64(sum) {
/* At least one bit per literal is needed. */
retval = float64(sum)
}
return retval
}
const kOneSymbolHistogramCost float64 = 12
const kTwoSymbolHistogramCost float64 = 20
const kThreeSymbolHistogramCost float64 = 28
const kFourSymbolHistogramCost float64 = 37
func populationCostLiteral(histogram *histogramLiteral) float64 {
var data_size uint = histogramDataSizeLiteral()
var count int = 0
var s [5]uint
var bits float64 = 0.0
var i uint
if histogram.total_count_ == 0 {
return kOneSymbolHistogramCost
}
for i = 0; i < data_size; i++ {
if histogram.data_[i] > 0 {
s[count] = i
count++
if count > 4 {
break
}
}
}
if count == 1 {
return kOneSymbolHistogramCost
}
if count == 2 {
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
}
if count == 3 {
var histo0 uint32 = histogram.data_[s[0]]
var histo1 uint32 = histogram.data_[s[1]]
var histo2 uint32 = histogram.data_[s[2]]
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
}
if count == 4 {
var histo [4]uint32
var h23 uint32
var histomax uint32
for i = 0; i < 4; i++ {
histo[i] = histogram.data_[s[i]]
}
/* Sort */
for i = 0; i < 4; i++ {
var j uint
for j = i + 1; j < 4; j++ {
if histo[j] > histo[i] {
var tmp uint32 = histo[j]
histo[j] = histo[i]
histo[i] = tmp
}
}
}
h23 = histo[2] + histo[3]
histomax = brotli_max_uint32_t(h23, histo[0])
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
}
{
var max_depth uint = 1
var depth_histo = [codeLengthCodes]uint32{0}
/* In this loop we compute the entropy of the histogram and simultaneously
build a simplified histogram of the code length codes where we use the
zero repeat code 17, but we don't use the non-zero repeat code 16. */
var log2total float64 = fastLog2(histogram.total_count_)
for i = 0; i < data_size; {
if histogram.data_[i] > 0 {
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
= log2(total_count) - log2(count(symbol)) */
var depth uint = uint(log2p + 0.5)
/* Approximate the bit depth by round(-log2(P(symbol))) */
bits += float64(histogram.data_[i]) * log2p
if depth > 15 {
depth = 15
}
if depth > max_depth {
max_depth = depth
}
depth_histo[depth]++
i++
} else {
var reps uint32 = 1
/* Compute the run length of zeros and add the appropriate number of 0
and 17 code length codes to the code length code histogram. */
var k uint
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
reps++
}
i += uint(reps)
if i == data_size {
/* Don't add any cost for the last zero run, since these are encoded
only implicitly. */
break
}
if reps < 3 {
depth_histo[0] += reps
} else {
reps -= 2
for reps > 0 {
depth_histo[repeatZeroCodeLength]++
/* Add the 3 extra bits for the 17 code length code. */
bits += 3
reps >>= 3
}
}
}
}
/* Add the estimated encoding cost of the code length code histogram. */
bits += float64(18 + 2*max_depth)
/* Add the entropy of the code length code histogram. */
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
}
return bits
}
func populationCostCommand(histogram *histogramCommand) float64 {
var data_size uint = histogramDataSizeCommand()
var count int = 0
var s [5]uint
var bits float64 = 0.0
var i uint
if histogram.total_count_ == 0 {
return kOneSymbolHistogramCost
}
for i = 0; i < data_size; i++ {
if histogram.data_[i] > 0 {
s[count] = i
count++
if count > 4 {
break
}
}
}
if count == 1 {
return kOneSymbolHistogramCost
}
if count == 2 {
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
}
if count == 3 {
var histo0 uint32 = histogram.data_[s[0]]
var histo1 uint32 = histogram.data_[s[1]]
var histo2 uint32 = histogram.data_[s[2]]
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
}
if count == 4 {
var histo [4]uint32
var h23 uint32
var histomax uint32
for i = 0; i < 4; i++ {
histo[i] = histogram.data_[s[i]]
}
/* Sort */
for i = 0; i < 4; i++ {
var j uint
for j = i + 1; j < 4; j++ {
if histo[j] > histo[i] {
var tmp uint32 = histo[j]
histo[j] = histo[i]
histo[i] = tmp
}
}
}
h23 = histo[2] + histo[3]
histomax = brotli_max_uint32_t(h23, histo[0])
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
}
{
var max_depth uint = 1
var depth_histo = [codeLengthCodes]uint32{0}
/* In this loop we compute the entropy of the histogram and simultaneously
build a simplified histogram of the code length codes where we use the
zero repeat code 17, but we don't use the non-zero repeat code 16. */
var log2total float64 = fastLog2(histogram.total_count_)
for i = 0; i < data_size; {
if histogram.data_[i] > 0 {
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
= log2(total_count) - log2(count(symbol)) */
var depth uint = uint(log2p + 0.5)
/* Approximate the bit depth by round(-log2(P(symbol))) */
bits += float64(histogram.data_[i]) * log2p
if depth > 15 {
depth = 15
}
if depth > max_depth {
max_depth = depth
}
depth_histo[depth]++
i++
} else {
var reps uint32 = 1
/* Compute the run length of zeros and add the appropriate number of 0
and 17 code length codes to the code length code histogram. */
var k uint
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
reps++
}
i += uint(reps)
if i == data_size {
/* Don't add any cost for the last zero run, since these are encoded
only implicitly. */
break
}
if reps < 3 {
depth_histo[0] += reps
} else {
reps -= 2
for reps > 0 {
depth_histo[repeatZeroCodeLength]++
/* Add the 3 extra bits for the 17 code length code. */
bits += 3
reps >>= 3
}
}
}
}
/* Add the estimated encoding cost of the code length code histogram. */
bits += float64(18 + 2*max_depth)
/* Add the entropy of the code length code histogram. */
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
}
return bits
}
func populationCostDistance(histogram *histogramDistance) float64 {
var data_size uint = histogramDataSizeDistance()
var count int = 0
var s [5]uint
var bits float64 = 0.0
var i uint
if histogram.total_count_ == 0 {
return kOneSymbolHistogramCost
}
for i = 0; i < data_size; i++ {
if histogram.data_[i] > 0 {
s[count] = i
count++
if count > 4 {
break
}
}
}
if count == 1 {
return kOneSymbolHistogramCost
}
if count == 2 {
return kTwoSymbolHistogramCost + float64(histogram.total_count_)
}
if count == 3 {
var histo0 uint32 = histogram.data_[s[0]]
var histo1 uint32 = histogram.data_[s[1]]
var histo2 uint32 = histogram.data_[s[2]]
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2))
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax)
}
if count == 4 {
var histo [4]uint32
var h23 uint32
var histomax uint32
for i = 0; i < 4; i++ {
histo[i] = histogram.data_[s[i]]
}
/* Sort */
for i = 0; i < 4; i++ {
var j uint
for j = i + 1; j < 4; j++ {
if histo[j] > histo[i] {
var tmp uint32 = histo[j]
histo[j] = histo[i]
histo[i] = tmp
}
}
}
h23 = histo[2] + histo[3]
histomax = brotli_max_uint32_t(h23, histo[0])
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax)
}
{
var max_depth uint = 1
var depth_histo = [codeLengthCodes]uint32{0}
/* In this loop we compute the entropy of the histogram and simultaneously
build a simplified histogram of the code length codes where we use the
zero repeat code 17, but we don't use the non-zero repeat code 16. */
var log2total float64 = fastLog2(histogram.total_count_)
for i = 0; i < data_size; {
if histogram.data_[i] > 0 {
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
= log2(total_count) - log2(count(symbol)) */
var depth uint = uint(log2p + 0.5)
/* Approximate the bit depth by round(-log2(P(symbol))) */
bits += float64(histogram.data_[i]) * log2p
if depth > 15 {
depth = 15
}
if depth > max_depth {
max_depth = depth
}
depth_histo[depth]++
i++
} else {
var reps uint32 = 1
/* Compute the run length of zeros and add the appropriate number of 0
and 17 code length codes to the code length code histogram. */
var k uint
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ {
reps++
}
i += uint(reps)
if i == data_size {
/* Don't add any cost for the last zero run, since these are encoded
only implicitly. */
break
}
if reps < 3 {
depth_histo[0] += reps
} else {
reps -= 2
for reps > 0 {
depth_histo[repeatZeroCodeLength]++
/* Add the 3 extra bits for the 17 code length code. */
bits += 3
reps >>= 3
}
}
}
}
/* Add the estimated encoding cost of the code length code histogram. */
bits += float64(18 + 2*max_depth)
/* Add the entropy of the code length code histogram. */
bits += bitsEntropy(depth_histo[:], codeLengthCodes)
}
return bits
}
... ...
package brotli
import "encoding/binary"
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Bit reading helpers */
const shortFillBitWindowRead = (8 >> 1)
var kBitMask = [33]uint32{
0x00000000,
0x00000001,
0x00000003,
0x00000007,
0x0000000F,
0x0000001F,
0x0000003F,
0x0000007F,
0x000000FF,
0x000001FF,
0x000003FF,
0x000007FF,
0x00000FFF,
0x00001FFF,
0x00003FFF,
0x00007FFF,
0x0000FFFF,
0x0001FFFF,
0x0003FFFF,
0x0007FFFF,
0x000FFFFF,
0x001FFFFF,
0x003FFFFF,
0x007FFFFF,
0x00FFFFFF,
0x01FFFFFF,
0x03FFFFFF,
0x07FFFFFF,
0x0FFFFFFF,
0x1FFFFFFF,
0x3FFFFFFF,
0x7FFFFFFF,
0xFFFFFFFF,
}
func bitMask(n uint32) uint32 {
return kBitMask[n]
}
type bitReader struct {
val_ uint64
bit_pos_ uint32
input []byte
input_len uint
byte_pos uint
}
type bitReaderState struct {
val_ uint64
bit_pos_ uint32
input []byte
input_len uint
byte_pos uint
}
/* Initializes the BrotliBitReader fields. */
/* Ensures that accumulator is not empty.
May consume up to sizeof(brotli_reg_t) - 1 bytes of input.
Returns false if data is required but there is no input available.
For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
reading. */
func bitReaderSaveState(from *bitReader, to *bitReaderState) {
to.val_ = from.val_
to.bit_pos_ = from.bit_pos_
to.input = from.input
to.input_len = from.input_len
to.byte_pos = from.byte_pos
}
func bitReaderRestoreState(to *bitReader, from *bitReaderState) {
to.val_ = from.val_
to.bit_pos_ = from.bit_pos_
to.input = from.input
to.input_len = from.input_len
to.byte_pos = from.byte_pos
}
func getAvailableBits(br *bitReader) uint32 {
return 64 - br.bit_pos_
}
/* Returns amount of unread bytes the bit reader still has buffered from the
BrotliInput, including whole bytes in br->val_. */
func getRemainingBytes(br *bitReader) uint {
return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3))
}
/* Checks if there is at least |num| bytes left in the input ring-buffer
(excluding the bits remaining in br->val_). */
func checkInputAmount(br *bitReader, num uint) bool {
return br.input_len-br.byte_pos >= num
}
/* Guarantees that there are at least |n_bits| + 1 bits in accumulator.
Precondition: accumulator contains at least 1 bit.
|n_bits| should be in the range [1..24] for regular build. For portable
non-64-bit little-endian build only 16 bits are safe to request. */
func fillBitWindow(br *bitReader, n_bits uint32) {
if br.bit_pos_ >= 32 {
br.val_ >>= 32
br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */
br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32
br.byte_pos += 4
}
}
/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
func fillBitWindow16(br *bitReader) {
fillBitWindow(br, 17)
}
/* Tries to pull one byte of input to accumulator.
Returns false if there is no input available. */
func pullByte(br *bitReader) bool {
if br.byte_pos == br.input_len {
return false
}
br.val_ >>= 8
br.val_ |= (uint64(br.input[br.byte_pos])) << 56
br.bit_pos_ -= 8
br.byte_pos++
return true
}
/* Returns currently available bits.
The number of valid bits could be calculated by BrotliGetAvailableBits. */
func getBitsUnmasked(br *bitReader) uint64 {
return br.val_ >> br.bit_pos_
}
/* Like BrotliGetBits, but does not mask the result.
The result contains at least 16 valid bits. */
func get16BitsUnmasked(br *bitReader) uint32 {
fillBitWindow(br, 16)
return uint32(getBitsUnmasked(br))
}
/* Returns the specified number of bits from |br| without advancing bit
position. */
func getBits(br *bitReader, n_bits uint32) uint32 {
fillBitWindow(br, n_bits)
return uint32(getBitsUnmasked(br)) & bitMask(n_bits)
}
/* Tries to peek the specified amount of bits. Returns false, if there
is not enough input. */
func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool {
for getAvailableBits(br) < n_bits {
if !pullByte(br) {
return false
}
}
*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
return true
}
/* Advances the bit pos by |n_bits|. */
func dropBits(br *bitReader, n_bits uint32) {
br.bit_pos_ += n_bits
}
func bitReaderUnload(br *bitReader) {
var unused_bytes uint32 = getAvailableBits(br) >> 3
var unused_bits uint32 = unused_bytes << 3
br.byte_pos -= uint(unused_bytes)
if unused_bits == 64 {
br.val_ = 0
} else {
br.val_ <<= unused_bits
}
br.bit_pos_ += unused_bits
}
/* Reads the specified number of bits from |br| and advances the bit pos.
Precondition: accumulator MUST contain at least |n_bits|. */
func takeBits(br *bitReader, n_bits uint32, val *uint32) {
*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits)
dropBits(br, n_bits)
}
/* Reads the specified number of bits from |br| and advances the bit pos.
Assumes that there is enough input to perform BrotliFillBitWindow. */
func readBits(br *bitReader, n_bits uint32) uint32 {
var val uint32
fillBitWindow(br, n_bits)
takeBits(br, n_bits, &val)
return val
}
/* Tries to read the specified amount of bits. Returns false, if there
is not enough input. |n_bits| MUST be positive. */
func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool {
for getAvailableBits(br) < n_bits {
if !pullByte(br) {
return false
}
}
takeBits(br, n_bits, val)
return true
}
/* Advances the bit reader position to the next byte boundary and verifies
that any skipped bits are set to zero. */
func bitReaderJumpToByteBoundary(br *bitReader) bool {
var pad_bits_count uint32 = getAvailableBits(br) & 0x7
var pad_bits uint32 = 0
if pad_bits_count != 0 {
takeBits(br, pad_bits_count, &pad_bits)
}
return pad_bits == 0
}
/* Copies remaining input bytes stored in the bit reader to the output. Value
|num| may not be larger than BrotliGetRemainingBytes. The bit reader must be
warmed up again after this. */
func copyBytes(dest []byte, br *bitReader, num uint) {
for getAvailableBits(br) >= 8 && num > 0 {
dest[0] = byte(getBitsUnmasked(br))
dropBits(br, 8)
dest = dest[1:]
num--
}
copy(dest, br.input[br.byte_pos:][:num])
br.byte_pos += num
}
func initBitReader(br *bitReader) {
br.val_ = 0
br.bit_pos_ = 64
}
func warmupBitReader(br *bitReader) bool {
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
overflow. If unalignment is caused by BrotliSafeReadBits, then there is
enough space in accumulator to fix alignment. */
if getAvailableBits(br) == 0 {
if !pullByte(br) {
return false
}
}
return true
}
... ...
package brotli
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Block split point selection utilities. */
type blockSplit struct {
num_types uint
num_blocks uint
types []byte
lengths []uint32
types_alloc_size uint
lengths_alloc_size uint
}
const (
kMaxLiteralHistograms uint = 100
kMaxCommandHistograms uint = 50
kLiteralBlockSwitchCost float64 = 28.1
kCommandBlockSwitchCost float64 = 13.5
kDistanceBlockSwitchCost float64 = 14.6
kLiteralStrideLength uint = 70
kCommandStrideLength uint = 40
kSymbolsPerLiteralHistogram uint = 544
kSymbolsPerCommandHistogram uint = 530
kSymbolsPerDistanceHistogram uint = 544
kMinLengthForBlockSplitting uint = 128
kIterMulForRefining uint = 2
kMinItersForRefining uint = 100
)
func countLiterals(cmds []command, num_commands uint) uint {
var total_length uint = 0
/* Count how many we have. */
var i uint
for i = 0; i < num_commands; i++ {
total_length += uint(cmds[i].insert_len_)
}
return total_length
}
func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) {
var pos uint = 0
var from_pos uint = offset & mask
var i uint
for i = 0; i < num_commands; i++ {
var insert_len uint = uint(cmds[i].insert_len_)
if from_pos+insert_len > mask {
var head_size uint = mask + 1 - from_pos
copy(literals[pos:], data[from_pos:][:head_size])
from_pos = 0
pos += head_size
insert_len -= head_size
}
if insert_len > 0 {
copy(literals[pos:], data[from_pos:][:insert_len])
pos += insert_len
}
from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask))
}
}
func myRand(seed *uint32) uint32 {
/* Initial seed should be 7. In this case, loop length is (1 << 29). */
*seed *= 16807
return *seed
}
func bitCost(count uint) float64 {
if count == 0 {
return -2.0
} else {
return fastLog2(count)
}
}
const histogramsPerBatch = 64
const clustersPerBatch = 16
func initBlockSplit(self *blockSplit) {
self.num_types = 0
self.num_blocks = 0
self.types = nil
self.lengths = nil
self.types_alloc_size = 0
self.lengths_alloc_size = 0
}
func destroyBlockSplit(self *blockSplit) {
self.types = nil
self.lengths = nil
}
func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
{
var literals_count uint = countLiterals(cmds, num_commands)
var literals []byte = make([]byte, literals_count)
/* Create a continuous array of literals. */
copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals)
/* Create the block split on the array of literals.
Literal histograms have alphabet size 256. */
splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split)
literals = nil
}
{
var insert_and_copy_codes []uint16 = make([]uint16, num_commands)
/* Compute prefix codes for commands. */
var i uint
for i = 0; i < num_commands; i++ {
insert_and_copy_codes[i] = cmds[i].cmd_prefix_
}
/* Create the block split on the array of command prefixes. */
splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
/* TODO: reuse for distances? */
insert_and_copy_codes = nil
}
{
var distance_prefixes []uint16 = make([]uint16, num_commands)
var j uint = 0
/* Create a continuous array of distance prefixes. */
var i uint
for i = 0; i < num_commands; i++ {
var cmd *command = &cmds[i]
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF
j++
}
}
/* Create the block split on the array of distance prefixes. */
splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split)
distance_prefixes = nil
}
}
... ...
package brotli
import "math"
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
var seed uint32 = 7
var block_length uint = length / num_histograms
var i uint
clearHistogramsCommand(histograms, num_histograms)
for i = 0; i < num_histograms; i++ {
var pos uint = length * i / num_histograms
if i != 0 {
pos += uint(myRand(&seed) % uint32(block_length))
}
if pos+stride >= length {
pos = length - stride - 1
}
histogramAddVectorCommand(&histograms[i], data[pos:], stride)
}
}
func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) {
var pos uint = 0
if stride >= length {
stride = length
} else {
pos = uint(myRand(seed) % uint32(length-stride+1))
}
histogramAddVectorCommand(sample, data[pos:], stride)
}
func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
var seed uint32 = 7
var iter uint
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
for iter = 0; iter < iters; iter++ {
var sample histogramCommand
histogramClearCommand(&sample)
randomSampleCommand(&seed, data, length, stride, &sample)
histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample)
}
}
/* Assigns a block id from the range [0, num_histograms) to each data element
in data[0..length) and fills in block_id[0..length) with the assigned values.
Returns the number of blocks, i.e. one plus the number of block switches. */
func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = histogramDataSizeCommand()
var bitmaplen uint = (num_histograms + 7) >> 3
var num_blocks uint = 1
var i uint
var j uint
assert(num_histograms <= 256)
if num_histograms <= 1 {
for i = 0; i < length; i++ {
block_id[i] = 0
}
return 1
}
for i := 0; i < int(data_size*num_histograms); i++ {
insert_cost[i] = 0
}
for i = 0; i < num_histograms; i++ {
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
}
for i = data_size; i != 0; {
i--
for j = 0; j < num_histograms; j++ {
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
}
}
for i := 0; i < int(num_histograms); i++ {
cost[i] = 0
}
for i := 0; i < int(length*bitmaplen); i++ {
switch_signal[i] = 0
}
/* After each iteration of this loop, cost[k] will contain the difference
between the minimum cost of arriving at the current byte position using
entropy code k, and the minimum cost of arriving at the current byte
position. This difference is capped at the block switch cost, and if it
reaches block switch cost, it means that when we trace back from the last
position, we need to switch here. */
for i = 0; i < length; i++ {
var byte_ix uint = i
var ix uint = byte_ix * bitmaplen
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
var min_cost float64 = 1e99
var block_switch_cost float64 = block_switch_bitcost
var k uint
for k = 0; k < num_histograms; k++ {
/* We are coding the symbol in data[byte_ix] with entropy code k. */
cost[k] += insert_cost[insert_cost_ix+k]
if cost[k] < min_cost {
min_cost = cost[k]
block_id[byte_ix] = byte(k)
}
}
/* More blocks for the beginning. */
if byte_ix < 2000 {
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
}
for k = 0; k < num_histograms; k++ {
cost[k] -= min_cost
if cost[k] >= block_switch_cost {
var mask byte = byte(1 << (k & 7))
cost[k] = block_switch_cost
assert(k>>3 < bitmaplen)
switch_signal[ix+(k>>3)] |= mask
/* Trace back from the last position and switch at the marked places. */
}
}
}
{
var byte_ix uint = length - 1
var ix uint = byte_ix * bitmaplen
var cur_id byte = block_id[byte_ix]
for byte_ix > 0 {
var mask byte = byte(1 << (cur_id & 7))
assert(uint(cur_id)>>3 < bitmaplen)
byte_ix--
ix -= bitmaplen
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
if cur_id != block_id[byte_ix] {
cur_id = block_id[byte_ix]
num_blocks++
}
}
block_id[byte_ix] = cur_id
}
}
return num_blocks
}
var remapBlockIdsCommand_kInvalidId uint16 = 256
func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
var next_id uint16 = 0
var i uint
for i = 0; i < num_histograms; i++ {
new_id[i] = remapBlockIdsCommand_kInvalidId
}
for i = 0; i < length; i++ {
assert(uint(block_ids[i]) < num_histograms)
if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId {
new_id[block_ids[i]] = next_id
next_id++
}
}
for i = 0; i < length; i++ {
block_ids[i] = byte(new_id[block_ids[i]])
assert(uint(block_ids[i]) < num_histograms)
}
assert(uint(next_id) <= num_histograms)
return uint(next_id)
}
func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) {
var i uint
clearHistogramsCommand(histograms, num_histograms)
for i = 0; i < length; i++ {
histogramAddCommand(&histograms[block_ids[i]], uint(data[i]))
}
}
var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32
func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
var histogram_symbols []uint32 = make([]uint32, num_blocks)
var block_lengths []uint32 = make([]uint32, num_blocks)
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
var all_histograms_size uint = 0
var all_histograms_capacity uint = expected_num_clusters
var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity)
var cluster_size_size uint = 0
var cluster_size_capacity uint = expected_num_clusters
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
var num_clusters uint = 0
var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch))
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
var pairs_capacity uint = max_num_pairs + 1
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
var pos uint = 0
var clusters []uint32
var num_final_clusters uint
var new_index []uint32
var i uint
var sizes = [histogramsPerBatch]uint32{0}
var new_clusters = [histogramsPerBatch]uint32{0}
var symbols = [histogramsPerBatch]uint32{0}
var remap = [histogramsPerBatch]uint32{0}
for i := 0; i < int(num_blocks); i++ {
block_lengths[i] = 0
}
{
var block_idx uint = 0
for i = 0; i < length; i++ {
assert(block_idx < num_blocks)
block_lengths[block_idx]++
if i+1 == length || block_ids[i] != block_ids[i+1] {
block_idx++
}
}
assert(block_idx == num_blocks)
}
for i = 0; i < num_blocks; i += histogramsPerBatch {
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
var num_new_clusters uint
var j uint
for j = 0; j < num_to_combine; j++ {
var k uint
histogramClearCommand(&histograms[j])
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
histogramAddCommand(&histograms[j], uint(data[pos]))
pos++
}
histograms[j].bit_cost_ = populationCostCommand(&histograms[j])
new_clusters[j] = uint32(j)
symbols[j] = uint32(j)
sizes[j] = 1
}
num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
var _new_size uint
if all_histograms_capacity == 0 {
_new_size = all_histograms_size + num_new_clusters
} else {
_new_size = all_histograms_capacity
}
var new_array []histogramCommand
for _new_size < (all_histograms_size + num_new_clusters) {
_new_size *= 2
}
new_array = make([]histogramCommand, _new_size)
if all_histograms_capacity != 0 {
copy(new_array, all_histograms[:all_histograms_capacity])
}
all_histograms = new_array
all_histograms_capacity = _new_size
}
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
for j = 0; j < num_new_clusters; j++ {
all_histograms[all_histograms_size] = histograms[new_clusters[j]]
all_histograms_size++
cluster_size[cluster_size_size] = sizes[new_clusters[j]]
cluster_size_size++
remap[new_clusters[j]] = uint32(j)
}
for j = 0; j < num_to_combine; j++ {
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
}
num_clusters += num_new_clusters
assert(num_clusters == cluster_size_size)
assert(num_clusters == all_histograms_size)
}
histograms = nil
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
if pairs_capacity < max_num_pairs+1 {
pairs = nil
pairs = make([]histogramPair, (max_num_pairs + 1))
}
clusters = make([]uint32, num_clusters)
for i = 0; i < num_clusters; i++ {
clusters[i] = uint32(i)
}
num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
pairs = nil
cluster_size = nil
new_index = make([]uint32, num_clusters)
for i = 0; i < num_clusters; i++ {
new_index[i] = clusterBlocksCommand_kInvalidIndex
}
pos = 0
{
var next_index uint32 = 0
for i = 0; i < num_blocks; i++ {
var histo histogramCommand
var j uint
var best_out uint32
var best_bits float64
histogramClearCommand(&histo)
for j = 0; uint32(j) < block_lengths[i]; j++ {
histogramAddCommand(&histo, uint(data[pos]))
pos++
}
if i == 0 {
best_out = histogram_symbols[0]
} else {
best_out = histogram_symbols[i-1]
}
best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out])
for j = 0; j < num_final_clusters; j++ {
var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]])
if cur_bits < best_bits {
best_bits = cur_bits
best_out = clusters[j]
}
}
histogram_symbols[i] = best_out
if new_index[best_out] == clusterBlocksCommand_kInvalidIndex {
new_index[best_out] = next_index
next_index++
}
}
}
clusters = nil
all_histograms = nil
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
{
var cur_length uint32 = 0
var block_idx uint = 0
var max_type byte = 0
for i = 0; i < num_blocks; i++ {
cur_length += block_lengths[i]
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
var id byte = byte(new_index[histogram_symbols[i]])
split.types[block_idx] = id
split.lengths[block_idx] = cur_length
max_type = brotli_max_uint8_t(max_type, id)
cur_length = 0
block_idx++
}
}
split.num_blocks = block_idx
split.num_types = uint(max_type) + 1
}
new_index = nil
block_lengths = nil
histogram_symbols = nil
}
func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
var data_size uint = histogramDataSizeCommand()
var num_histograms uint = length/literals_per_histogram + 1
var histograms []histogramCommand
if num_histograms > max_histograms {
num_histograms = max_histograms
}
if length == 0 {
split.num_types = 1
return
} else if length < kMinLengthForBlockSplitting {
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
split.num_types = 1
split.types[split.num_blocks] = 0
split.lengths[split.num_blocks] = uint32(length)
split.num_blocks++
return
}
histograms = make([]histogramCommand, num_histograms)
/* Find good entropy codes. */
initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)
{
var block_ids []byte = make([]byte, length)
var num_blocks uint = 0
var bitmaplen uint = (num_histograms + 7) >> 3
var insert_cost []float64 = make([]float64, (data_size * num_histograms))
var cost []float64 = make([]float64, num_histograms)
var switch_signal []byte = make([]byte, (length * bitmaplen))
var new_id []uint16 = make([]uint16, num_histograms)
var iters uint
if params.quality < hqZopflificationQuality {
iters = 3
} else {
iters = 10
}
/* Find a good path through literals with the good entropy codes. */
var i uint
for i = 0; i < iters; i++ {
num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms)
buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms)
}
insert_cost = nil
cost = nil
switch_signal = nil
new_id = nil
histograms = nil
clusterBlocksCommand(data, length, num_blocks, block_ids, split)
block_ids = nil
}
}
... ...
package brotli
import "math"
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
var seed uint32 = 7
var block_length uint = length / num_histograms
var i uint
clearHistogramsDistance(histograms, num_histograms)
for i = 0; i < num_histograms; i++ {
var pos uint = length * i / num_histograms
if i != 0 {
pos += uint(myRand(&seed) % uint32(block_length))
}
if pos+stride >= length {
pos = length - stride - 1
}
histogramAddVectorDistance(&histograms[i], data[pos:], stride)
}
}
func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) {
var pos uint = 0
if stride >= length {
stride = length
} else {
pos = uint(myRand(seed) % uint32(length-stride+1))
}
histogramAddVectorDistance(sample, data[pos:], stride)
}
func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
var seed uint32 = 7
var iter uint
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
for iter = 0; iter < iters; iter++ {
var sample histogramDistance
histogramClearDistance(&sample)
randomSampleDistance(&seed, data, length, stride, &sample)
histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample)
}
}
/* Assigns a block id from the range [0, num_histograms) to each data element
in data[0..length) and fills in block_id[0..length) with the assigned values.
Returns the number of blocks, i.e. one plus the number of block switches. */
func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = histogramDataSizeDistance()
var bitmaplen uint = (num_histograms + 7) >> 3
var num_blocks uint = 1
var i uint
var j uint
assert(num_histograms <= 256)
if num_histograms <= 1 {
for i = 0; i < length; i++ {
block_id[i] = 0
}
return 1
}
for i := 0; i < int(data_size*num_histograms); i++ {
insert_cost[i] = 0
}
for i = 0; i < num_histograms; i++ {
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
}
for i = data_size; i != 0; {
i--
for j = 0; j < num_histograms; j++ {
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
}
}
for i := 0; i < int(num_histograms); i++ {
cost[i] = 0
}
for i := 0; i < int(length*bitmaplen); i++ {
switch_signal[i] = 0
}
/* After each iteration of this loop, cost[k] will contain the difference
between the minimum cost of arriving at the current byte position using
entropy code k, and the minimum cost of arriving at the current byte
position. This difference is capped at the block switch cost, and if it
reaches block switch cost, it means that when we trace back from the last
position, we need to switch here. */
for i = 0; i < length; i++ {
var byte_ix uint = i
var ix uint = byte_ix * bitmaplen
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
var min_cost float64 = 1e99
var block_switch_cost float64 = block_switch_bitcost
var k uint
for k = 0; k < num_histograms; k++ {
/* We are coding the symbol in data[byte_ix] with entropy code k. */
cost[k] += insert_cost[insert_cost_ix+k]
if cost[k] < min_cost {
min_cost = cost[k]
block_id[byte_ix] = byte(k)
}
}
/* More blocks for the beginning. */
if byte_ix < 2000 {
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
}
for k = 0; k < num_histograms; k++ {
cost[k] -= min_cost
if cost[k] >= block_switch_cost {
var mask byte = byte(1 << (k & 7))
cost[k] = block_switch_cost
assert(k>>3 < bitmaplen)
switch_signal[ix+(k>>3)] |= mask
/* Trace back from the last position and switch at the marked places. */
}
}
}
{
var byte_ix uint = length - 1
var ix uint = byte_ix * bitmaplen
var cur_id byte = block_id[byte_ix]
for byte_ix > 0 {
var mask byte = byte(1 << (cur_id & 7))
assert(uint(cur_id)>>3 < bitmaplen)
byte_ix--
ix -= bitmaplen
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
if cur_id != block_id[byte_ix] {
cur_id = block_id[byte_ix]
num_blocks++
}
}
block_id[byte_ix] = cur_id
}
}
return num_blocks
}
var remapBlockIdsDistance_kInvalidId uint16 = 256
func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
var next_id uint16 = 0
var i uint
for i = 0; i < num_histograms; i++ {
new_id[i] = remapBlockIdsDistance_kInvalidId
}
for i = 0; i < length; i++ {
assert(uint(block_ids[i]) < num_histograms)
if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId {
new_id[block_ids[i]] = next_id
next_id++
}
}
for i = 0; i < length; i++ {
block_ids[i] = byte(new_id[block_ids[i]])
assert(uint(block_ids[i]) < num_histograms)
}
assert(uint(next_id) <= num_histograms)
return uint(next_id)
}
func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) {
var i uint
clearHistogramsDistance(histograms, num_histograms)
for i = 0; i < length; i++ {
histogramAddDistance(&histograms[block_ids[i]], uint(data[i]))
}
}
var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32
func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
var histogram_symbols []uint32 = make([]uint32, num_blocks)
var block_lengths []uint32 = make([]uint32, num_blocks)
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
var all_histograms_size uint = 0
var all_histograms_capacity uint = expected_num_clusters
var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity)
var cluster_size_size uint = 0
var cluster_size_capacity uint = expected_num_clusters
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
var num_clusters uint = 0
var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch))
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
var pairs_capacity uint = max_num_pairs + 1
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
var pos uint = 0
var clusters []uint32
var num_final_clusters uint
var new_index []uint32
var i uint
var sizes = [histogramsPerBatch]uint32{0}
var new_clusters = [histogramsPerBatch]uint32{0}
var symbols = [histogramsPerBatch]uint32{0}
var remap = [histogramsPerBatch]uint32{0}
for i := 0; i < int(num_blocks); i++ {
block_lengths[i] = 0
}
{
var block_idx uint = 0
for i = 0; i < length; i++ {
assert(block_idx < num_blocks)
block_lengths[block_idx]++
if i+1 == length || block_ids[i] != block_ids[i+1] {
block_idx++
}
}
assert(block_idx == num_blocks)
}
for i = 0; i < num_blocks; i += histogramsPerBatch {
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
var num_new_clusters uint
var j uint
for j = 0; j < num_to_combine; j++ {
var k uint
histogramClearDistance(&histograms[j])
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
histogramAddDistance(&histograms[j], uint(data[pos]))
pos++
}
histograms[j].bit_cost_ = populationCostDistance(&histograms[j])
new_clusters[j] = uint32(j)
symbols[j] = uint32(j)
sizes[j] = 1
}
num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
var _new_size uint
if all_histograms_capacity == 0 {
_new_size = all_histograms_size + num_new_clusters
} else {
_new_size = all_histograms_capacity
}
var new_array []histogramDistance
for _new_size < (all_histograms_size + num_new_clusters) {
_new_size *= 2
}
new_array = make([]histogramDistance, _new_size)
if all_histograms_capacity != 0 {
copy(new_array, all_histograms[:all_histograms_capacity])
}
all_histograms = new_array
all_histograms_capacity = _new_size
}
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
for j = 0; j < num_new_clusters; j++ {
all_histograms[all_histograms_size] = histograms[new_clusters[j]]
all_histograms_size++
cluster_size[cluster_size_size] = sizes[new_clusters[j]]
cluster_size_size++
remap[new_clusters[j]] = uint32(j)
}
for j = 0; j < num_to_combine; j++ {
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
}
num_clusters += num_new_clusters
assert(num_clusters == cluster_size_size)
assert(num_clusters == all_histograms_size)
}
histograms = nil
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
if pairs_capacity < max_num_pairs+1 {
pairs = nil
pairs = make([]histogramPair, (max_num_pairs + 1))
}
clusters = make([]uint32, num_clusters)
for i = 0; i < num_clusters; i++ {
clusters[i] = uint32(i)
}
num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
pairs = nil
cluster_size = nil
new_index = make([]uint32, num_clusters)
for i = 0; i < num_clusters; i++ {
new_index[i] = clusterBlocksDistance_kInvalidIndex
}
pos = 0
{
var next_index uint32 = 0
for i = 0; i < num_blocks; i++ {
var histo histogramDistance
var j uint
var best_out uint32
var best_bits float64
histogramClearDistance(&histo)
for j = 0; uint32(j) < block_lengths[i]; j++ {
histogramAddDistance(&histo, uint(data[pos]))
pos++
}
if i == 0 {
best_out = histogram_symbols[0]
} else {
best_out = histogram_symbols[i-1]
}
best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out])
for j = 0; j < num_final_clusters; j++ {
var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]])
if cur_bits < best_bits {
best_bits = cur_bits
best_out = clusters[j]
}
}
histogram_symbols[i] = best_out
if new_index[best_out] == clusterBlocksDistance_kInvalidIndex {
new_index[best_out] = next_index
next_index++
}
}
}
clusters = nil
all_histograms = nil
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
{
var cur_length uint32 = 0
var block_idx uint = 0
var max_type byte = 0
for i = 0; i < num_blocks; i++ {
cur_length += block_lengths[i]
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
var id byte = byte(new_index[histogram_symbols[i]])
split.types[block_idx] = id
split.lengths[block_idx] = cur_length
max_type = brotli_max_uint8_t(max_type, id)
cur_length = 0
block_idx++
}
}
split.num_blocks = block_idx
split.num_types = uint(max_type) + 1
}
new_index = nil
block_lengths = nil
histogram_symbols = nil
}
func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
var data_size uint = histogramDataSizeDistance()
var num_histograms uint = length/literals_per_histogram + 1
var histograms []histogramDistance
if num_histograms > max_histograms {
num_histograms = max_histograms
}
if length == 0 {
split.num_types = 1
return
} else if length < kMinLengthForBlockSplitting {
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
split.num_types = 1
split.types[split.num_blocks] = 0
split.lengths[split.num_blocks] = uint32(length)
split.num_blocks++
return
}
histograms = make([]histogramDistance, num_histograms)
/* Find good entropy codes. */
initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)
{
var block_ids []byte = make([]byte, length)
var num_blocks uint = 0
var bitmaplen uint = (num_histograms + 7) >> 3
var insert_cost []float64 = make([]float64, (data_size * num_histograms))
var cost []float64 = make([]float64, num_histograms)
var switch_signal []byte = make([]byte, (length * bitmaplen))
var new_id []uint16 = make([]uint16, num_histograms)
var iters uint
if params.quality < hqZopflificationQuality {
iters = 3
} else {
iters = 10
}
/* Find a good path through literals with the good entropy codes. */
var i uint
for i = 0; i < iters; i++ {
num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms)
buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms)
}
insert_cost = nil
cost = nil
switch_signal = nil
new_id = nil
histograms = nil
clusterBlocksDistance(data, length, num_blocks, block_ids, split)
block_ids = nil
}
}
... ...
package brotli
import "math"
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
var seed uint32 = 7
var block_length uint = length / num_histograms
var i uint
clearHistogramsLiteral(histograms, num_histograms)
for i = 0; i < num_histograms; i++ {
var pos uint = length * i / num_histograms
if i != 0 {
pos += uint(myRand(&seed) % uint32(block_length))
}
if pos+stride >= length {
pos = length - stride - 1
}
histogramAddVectorLiteral(&histograms[i], data[pos:], stride)
}
}
func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) {
var pos uint = 0
if stride >= length {
stride = length
} else {
pos = uint(myRand(seed) % uint32(length-stride+1))
}
histogramAddVectorLiteral(sample, data[pos:], stride)
}
func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
var seed uint32 = 7
var iter uint
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
for iter = 0; iter < iters; iter++ {
var sample histogramLiteral
histogramClearLiteral(&sample)
randomSampleLiteral(&seed, data, length, stride, &sample)
histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample)
}
}
/* Assigns a block id from the range [0, num_histograms) to each data element
in data[0..length) and fills in block_id[0..length) with the assigned values.
Returns the number of blocks, i.e. one plus the number of block switches. */
func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = histogramDataSizeLiteral()
var bitmaplen uint = (num_histograms + 7) >> 3
var num_blocks uint = 1
var i uint
var j uint
assert(num_histograms <= 256)
if num_histograms <= 1 {
for i = 0; i < length; i++ {
block_id[i] = 0
}
return 1
}
for i := 0; i < int(data_size*num_histograms); i++ {
insert_cost[i] = 0
}
for i = 0; i < num_histograms; i++ {
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
}
for i = data_size; i != 0; {
i--
for j = 0; j < num_histograms; j++ {
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i]))
}
}
for i := 0; i < int(num_histograms); i++ {
cost[i] = 0
}
for i := 0; i < int(length*bitmaplen); i++ {
switch_signal[i] = 0
}
/* After each iteration of this loop, cost[k] will contain the difference
between the minimum cost of arriving at the current byte position using
entropy code k, and the minimum cost of arriving at the current byte
position. This difference is capped at the block switch cost, and if it
reaches block switch cost, it means that when we trace back from the last
position, we need to switch here. */
for i = 0; i < length; i++ {
var byte_ix uint = i
var ix uint = byte_ix * bitmaplen
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms
var min_cost float64 = 1e99
var block_switch_cost float64 = block_switch_bitcost
var k uint
for k = 0; k < num_histograms; k++ {
/* We are coding the symbol in data[byte_ix] with entropy code k. */
cost[k] += insert_cost[insert_cost_ix+k]
if cost[k] < min_cost {
min_cost = cost[k]
block_id[byte_ix] = byte(k)
}
}
/* More blocks for the beginning. */
if byte_ix < 2000 {
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000
}
for k = 0; k < num_histograms; k++ {
cost[k] -= min_cost
if cost[k] >= block_switch_cost {
var mask byte = byte(1 << (k & 7))
cost[k] = block_switch_cost
assert(k>>3 < bitmaplen)
switch_signal[ix+(k>>3)] |= mask
/* Trace back from the last position and switch at the marked places. */
}
}
}
{
var byte_ix uint = length - 1
var ix uint = byte_ix * bitmaplen
var cur_id byte = block_id[byte_ix]
for byte_ix > 0 {
var mask byte = byte(1 << (cur_id & 7))
assert(uint(cur_id)>>3 < bitmaplen)
byte_ix--
ix -= bitmaplen
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 {
if cur_id != block_id[byte_ix] {
cur_id = block_id[byte_ix]
num_blocks++
}
}
block_id[byte_ix] = cur_id
}
}
return num_blocks
}
var remapBlockIdsLiteral_kInvalidId uint16 = 256
func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint {
var next_id uint16 = 0
var i uint
for i = 0; i < num_histograms; i++ {
new_id[i] = remapBlockIdsLiteral_kInvalidId
}
for i = 0; i < length; i++ {
assert(uint(block_ids[i]) < num_histograms)
if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId {
new_id[block_ids[i]] = next_id
next_id++
}
}
for i = 0; i < length; i++ {
block_ids[i] = byte(new_id[block_ids[i]])
assert(uint(block_ids[i]) < num_histograms)
}
assert(uint(next_id) <= num_histograms)
return uint(next_id)
}
func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) {
var i uint
clearHistogramsLiteral(histograms, num_histograms)
for i = 0; i < length; i++ {
histogramAddLiteral(&histograms[block_ids[i]], uint(data[i]))
}
}
var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32
func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) {
var histogram_symbols []uint32 = make([]uint32, num_blocks)
var block_lengths []uint32 = make([]uint32, num_blocks)
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
var all_histograms_size uint = 0
var all_histograms_capacity uint = expected_num_clusters
var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity)
var cluster_size_size uint = 0
var cluster_size_capacity uint = expected_num_clusters
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
var num_clusters uint = 0
var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch))
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
var pairs_capacity uint = max_num_pairs + 1
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
var pos uint = 0
var clusters []uint32
var num_final_clusters uint
var new_index []uint32
var i uint
var sizes = [histogramsPerBatch]uint32{0}
var new_clusters = [histogramsPerBatch]uint32{0}
var symbols = [histogramsPerBatch]uint32{0}
var remap = [histogramsPerBatch]uint32{0}
for i := 0; i < int(num_blocks); i++ {
block_lengths[i] = 0
}
{
var block_idx uint = 0
for i = 0; i < length; i++ {
assert(block_idx < num_blocks)
block_lengths[block_idx]++
if i+1 == length || block_ids[i] != block_ids[i+1] {
block_idx++
}
}
assert(block_idx == num_blocks)
}
for i = 0; i < num_blocks; i += histogramsPerBatch {
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch)
var num_new_clusters uint
var j uint
for j = 0; j < num_to_combine; j++ {
var k uint
histogramClearLiteral(&histograms[j])
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
histogramAddLiteral(&histograms[j], uint(data[pos]))
pos++
}
histograms[j].bit_cost_ = populationCostLiteral(&histograms[j])
new_clusters[j] = uint32(j)
symbols[j] = uint32(j)
sizes[j] = 1
}
num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs)
if all_histograms_capacity < (all_histograms_size + num_new_clusters) {
var _new_size uint
if all_histograms_capacity == 0 {
_new_size = all_histograms_size + num_new_clusters
} else {
_new_size = all_histograms_capacity
}
var new_array []histogramLiteral
for _new_size < (all_histograms_size + num_new_clusters) {
_new_size *= 2
}
new_array = make([]histogramLiteral, _new_size)
if all_histograms_capacity != 0 {
copy(new_array, all_histograms[:all_histograms_capacity])
}
all_histograms = new_array
all_histograms_capacity = _new_size
}
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters)
for j = 0; j < num_new_clusters; j++ {
all_histograms[all_histograms_size] = histograms[new_clusters[j]]
all_histograms_size++
cluster_size[cluster_size_size] = sizes[new_clusters[j]]
cluster_size_size++
remap[new_clusters[j]] = uint32(j)
}
for j = 0; j < num_to_combine; j++ {
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]]
}
num_clusters += num_new_clusters
assert(num_clusters == cluster_size_size)
assert(num_clusters == all_histograms_size)
}
histograms = nil
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters)
if pairs_capacity < max_num_pairs+1 {
pairs = nil
pairs = make([]histogramPair, (max_num_pairs + 1))
}
clusters = make([]uint32, num_clusters)
for i = 0; i < num_clusters; i++ {
clusters[i] = uint32(i)
}
num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs)
pairs = nil
cluster_size = nil
new_index = make([]uint32, num_clusters)
for i = 0; i < num_clusters; i++ {
new_index[i] = clusterBlocksLiteral_kInvalidIndex
}
pos = 0
{
var next_index uint32 = 0
for i = 0; i < num_blocks; i++ {
var histo histogramLiteral
var j uint
var best_out uint32
var best_bits float64
histogramClearLiteral(&histo)
for j = 0; uint32(j) < block_lengths[i]; j++ {
histogramAddLiteral(&histo, uint(data[pos]))
pos++
}
if i == 0 {
best_out = histogram_symbols[0]
} else {
best_out = histogram_symbols[i-1]
}
best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out])
for j = 0; j < num_final_clusters; j++ {
var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]])
if cur_bits < best_bits {
best_bits = cur_bits
best_out = clusters[j]
}
}
histogram_symbols[i] = best_out
if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex {
new_index[best_out] = next_index
next_index++
}
}
}
clusters = nil
all_histograms = nil
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks)
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks)
{
var cur_length uint32 = 0
var block_idx uint = 0
var max_type byte = 0
for i = 0; i < num_blocks; i++ {
cur_length += block_lengths[i]
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] {
var id byte = byte(new_index[histogram_symbols[i]])
split.types[block_idx] = id
split.lengths[block_idx] = cur_length
max_type = brotli_max_uint8_t(max_type, id)
cur_length = 0
block_idx++
}
}
split.num_blocks = block_idx
split.num_types = uint(max_type) + 1
}
new_index = nil
block_lengths = nil
histogram_symbols = nil
}
func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
var data_size uint = histogramDataSizeLiteral()
var num_histograms uint = length/literals_per_histogram + 1
var histograms []histogramLiteral
if num_histograms > max_histograms {
num_histograms = max_histograms
}
if length == 0 {
split.num_types = 1
return
} else if length < kMinLengthForBlockSplitting {
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1)
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1)
split.num_types = 1
split.types[split.num_blocks] = 0
split.lengths[split.num_blocks] = uint32(length)
split.num_blocks++
return
}
histograms = make([]histogramLiteral, num_histograms)
/* Find good entropy codes. */
initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)
{
var block_ids []byte = make([]byte, length)
var num_blocks uint = 0
var bitmaplen uint = (num_histograms + 7) >> 3
var insert_cost []float64 = make([]float64, (data_size * num_histograms))
var cost []float64 = make([]float64, num_histograms)
var switch_signal []byte = make([]byte, (length * bitmaplen))
var new_id []uint16 = make([]uint16, num_histograms)
var iters uint
if params.quality < hqZopflificationQuality {
iters = 3
} else {
iters = 10
}
/* Find a good path through literals with the good entropy codes. */
var i uint
for i = 0; i < iters; i++ {
num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids)
num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms)
buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms)
}
insert_cost = nil
cost = nil
switch_signal = nil
new_id = nil
histograms = nil
clusterBlocksLiteral(data, length, num_blocks, block_ids, split)
block_ids = nil
}
}
... ...