// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package flate import ( "bytes" "encoding/binary" "fmt" "io" "math" ) const ( // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused // 8 bits: xlength = length - MIN_MATCH_LENGTH // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal lengthShift = 22 offsetMask = 1<maxnumlit offHist [32]uint16 // offset codes litHist [256]uint16 // codes 0->255 n uint16 // Must be able to contain maxStoreBlockSize tokens [maxStoreBlockSize + 1]token } func (t *tokens) Reset() { if t.n == 0 { return } t.n = 0 t.nLits = 0 for i := range t.litHist[:] { t.litHist[i] = 0 } for i := range t.extraHist[:] { t.extraHist[i] = 0 } for i := range t.offHist[:] { t.offHist[i] = 0 } } func (t *tokens) Fill() { if t.n == 0 { return } for i, v := range t.litHist[:] { if v == 0 { t.litHist[i] = 1 t.nLits++ } } for i, v := range t.extraHist[:literalCount-256] { if v == 0 { t.nLits++ t.extraHist[i] = 1 } } for i, v := range t.offHist[:offsetCodeCount] { if v == 0 { t.offHist[i] = 1 } } } func indexTokens(in []token) tokens { var t tokens t.indexTokens(in) return t } func (t *tokens) indexTokens(in []token) { t.Reset() for _, tok := range in { if tok < matchType { t.tokens[t.n] = tok t.litHist[tok]++ t.n++ continue } t.AddMatch(uint32(tok.length()), tok.offset()) } } // emitLiteral writes a literal chunk and returns the number of bytes written. func emitLiteral(dst *tokens, lit []byte) { ol := int(dst.n) for i, v := range lit { dst.tokens[(i+ol)&maxStoreBlockSize] = token(v) dst.litHist[v]++ } dst.n += uint16(len(lit)) dst.nLits += len(lit) } func (t *tokens) AddLiteral(lit byte) { t.tokens[t.n] = token(lit) t.litHist[lit]++ t.n++ t.nLits++ } // EstimatedBits will return an minimum size estimated by an *optimal* // compression of the block. // The size of the block func (t *tokens) EstimatedBits() int { shannon := float64(0) bits := int(0) nMatches := 0 if t.nLits > 0 { invTotal := 1.0 / float64(t.nLits) for _, v := range t.litHist[:] { if v > 0 { n := float64(v) shannon += math.Ceil(-math.Log2(n*invTotal) * n) } } // Just add 15 for EOB shannon += 15 for _, v := range t.extraHist[1 : literalCount-256] { if v > 0 { n := float64(v) shannon += math.Ceil(-math.Log2(n*invTotal) * n) bits += int(lengthExtraBits[v&31]) * int(v) nMatches += int(v) } } } if nMatches > 0 { invTotal := 1.0 / float64(nMatches) for _, v := range t.offHist[:offsetCodeCount] { if v > 0 { n := float64(v) shannon += math.Ceil(-math.Log2(n*invTotal) * n) bits += int(offsetExtraBits[v&31]) * int(n) } } } return int(shannon) + bits } // AddMatch adds a match to the tokens. // This function is very sensitive to inlining and right on the border. func (t *tokens) AddMatch(xlength uint32, xoffset uint32) { if debugDecode { if xlength >= maxMatchLength+baseMatchLength { panic(fmt.Errorf("invalid length: %v", xlength)) } if xoffset >= maxMatchOffset+baseMatchOffset { panic(fmt.Errorf("invalid offset: %v", xoffset)) } } t.nLits++ lengthCode := lengthCodes1[uint8(xlength)] & 31 t.tokens[t.n] = token(matchType | xlength<= maxMatchOffset+baseMatchOffset { panic(fmt.Errorf("invalid offset: %v", xoffset)) } } oc := offsetCode(xoffset) & 31 for xlength > 0 { xl := xlength if xl > 258 { // We need to have at least baseMatchLength left over for next loop. xl = 258 - baseMatchLength } xlength -= xl xl -= 3 t.nLits++ lengthCode := lengthCodes1[uint8(xl)] & 31 t.tokens[t.n] = token(matchType | uint32(xl)<> lengthShift) } // The code is never more than 8 bits, but is returned as uint32 for convenience. func lengthCode(len uint8) uint32 { return uint32(lengthCodes[len]) } // Returns the offset code corresponding to a specific offset func offsetCode(off uint32) uint32 { if false { if off < uint32(len(offsetCodes)) { return offsetCodes[off&255] } else if off>>7 < uint32(len(offsetCodes)) { return offsetCodes[(off>>7)&255] + 14 } else { return offsetCodes[(off>>14)&255] + 28 } } if off < uint32(len(offsetCodes)) { return offsetCodes[uint8(off)] } return offsetCodes14[uint8(off>>7)] }