Dinesh Belwalkar | e63f7f9 | 2019-11-22 23:11:16 +0000 | [diff] [blame] | 1 | package zstd |
| 2 | |
| 3 | import ( |
Dinesh Belwalkar | 396b652 | 2020-02-06 22:11:53 +0000 | [diff] [blame] | 4 | "errors" |
Dinesh Belwalkar | e63f7f9 | 2019-11-22 23:11:16 +0000 | [diff] [blame] | 5 | "fmt" |
| 6 | "runtime" |
| 7 | "strings" |
| 8 | ) |
| 9 | |
| 10 | // EOption is an option for creating a encoder. |
| 11 | type EOption func(*encoderOptions) error |
| 12 | |
| 13 | // options retains accumulated state of multiple options. |
| 14 | type encoderOptions struct { |
| 15 | concurrent int |
| 16 | crc bool |
| 17 | single *bool |
| 18 | pad int |
| 19 | blockSize int |
| 20 | windowSize int |
| 21 | level EncoderLevel |
| 22 | fullZero bool |
Dinesh Belwalkar | 396b652 | 2020-02-06 22:11:53 +0000 | [diff] [blame] | 23 | noEntropy bool |
Dinesh Belwalkar | e63f7f9 | 2019-11-22 23:11:16 +0000 | [diff] [blame] | 24 | } |
| 25 | |
| 26 | func (o *encoderOptions) setDefault() { |
| 27 | *o = encoderOptions{ |
| 28 | // use less ram: true for now, but may change. |
| 29 | concurrent: runtime.GOMAXPROCS(0), |
| 30 | crc: true, |
| 31 | single: nil, |
| 32 | blockSize: 1 << 16, |
| 33 | windowSize: 1 << 22, |
| 34 | level: SpeedDefault, |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | // encoder returns an encoder with the selected options. |
| 39 | func (o encoderOptions) encoder() encoder { |
| 40 | switch o.level { |
| 41 | case SpeedDefault: |
| 42 | return &doubleFastEncoder{fastEncoder: fastEncoder{maxMatchOff: int32(o.windowSize)}} |
| 43 | case SpeedFastest: |
| 44 | return &fastEncoder{maxMatchOff: int32(o.windowSize)} |
| 45 | } |
| 46 | panic("unknown compression level") |
| 47 | } |
| 48 | |
| 49 | // WithEncoderCRC will add CRC value to output. |
| 50 | // Output will be 4 bytes larger. |
| 51 | func WithEncoderCRC(b bool) EOption { |
| 52 | return func(o *encoderOptions) error { o.crc = b; return nil } |
| 53 | } |
| 54 | |
| 55 | // WithEncoderConcurrency will set the concurrency, |
| 56 | // meaning the maximum number of decoders to run concurrently. |
| 57 | // The value supplied must be at least 1. |
| 58 | // By default this will be set to GOMAXPROCS. |
| 59 | func WithEncoderConcurrency(n int) EOption { |
| 60 | return func(o *encoderOptions) error { |
| 61 | if n <= 0 { |
| 62 | return fmt.Errorf("concurrency must be at least 1") |
| 63 | } |
| 64 | o.concurrent = n |
| 65 | return nil |
| 66 | } |
| 67 | } |
| 68 | |
Dinesh Belwalkar | 396b652 | 2020-02-06 22:11:53 +0000 | [diff] [blame] | 69 | // WithWindowSize will set the maximum allowed back-reference distance. |
| 70 | // The value must be a power of two between WindowSizeMin and WindowSizeMax. |
| 71 | // A larger value will enable better compression but allocate more memory and, |
| 72 | // for above-default values, take considerably longer. |
| 73 | // The default value is determined by the compression level. |
| 74 | func WithWindowSize(n int) EOption { |
| 75 | return func(o *encoderOptions) error { |
| 76 | switch { |
| 77 | case n < MinWindowSize: |
| 78 | return fmt.Errorf("window size must be at least %d", MinWindowSize) |
| 79 | case n > MaxWindowSize: |
| 80 | return fmt.Errorf("window size must be at most %d", MaxWindowSize) |
| 81 | case (n & (n - 1)) != 0: |
| 82 | return errors.New("window size must be a power of 2") |
| 83 | } |
| 84 | |
| 85 | o.windowSize = n |
| 86 | if o.blockSize > o.windowSize { |
| 87 | o.blockSize = o.windowSize |
| 88 | } |
| 89 | return nil |
| 90 | } |
| 91 | } |
| 92 | |
Dinesh Belwalkar | e63f7f9 | 2019-11-22 23:11:16 +0000 | [diff] [blame] | 93 | // WithEncoderPadding will add padding to all output so the size will be a multiple of n. |
| 94 | // This can be used to obfuscate the exact output size or make blocks of a certain size. |
| 95 | // The contents will be a skippable frame, so it will be invisible by the decoder. |
| 96 | // n must be > 0 and <= 1GB, 1<<30 bytes. |
| 97 | // The padded area will be filled with data from crypto/rand.Reader. |
| 98 | // If `EncodeAll` is used with data already in the destination, the total size will be multiple of this. |
| 99 | func WithEncoderPadding(n int) EOption { |
| 100 | return func(o *encoderOptions) error { |
| 101 | if n <= 0 { |
| 102 | return fmt.Errorf("padding must be at least 1") |
| 103 | } |
| 104 | // No need to waste our time. |
| 105 | if n == 1 { |
| 106 | o.pad = 0 |
| 107 | } |
| 108 | if n > 1<<30 { |
| 109 | return fmt.Errorf("padding must less than 1GB (1<<30 bytes) ") |
| 110 | } |
| 111 | o.pad = n |
| 112 | return nil |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | // EncoderLevel predefines encoder compression levels. |
| 117 | // Only use the constants made available, since the actual mapping |
| 118 | // of these values are very likely to change and your compression could change |
| 119 | // unpredictably when upgrading the library. |
| 120 | type EncoderLevel int |
| 121 | |
| 122 | const ( |
| 123 | speedNotSet EncoderLevel = iota |
| 124 | |
| 125 | // SpeedFastest will choose the fastest reasonable compression. |
| 126 | // This is roughly equivalent to the fastest Zstandard mode. |
| 127 | SpeedFastest |
| 128 | |
| 129 | // SpeedDefault is the default "pretty fast" compression option. |
| 130 | // This is roughly equivalent to the default Zstandard mode (level 3). |
| 131 | SpeedDefault |
| 132 | |
| 133 | // speedLast should be kept as the last actual compression option. |
| 134 | // The is not for external usage, but is used to keep track of the valid options. |
| 135 | speedLast |
| 136 | |
| 137 | // SpeedBetterCompression will (in the future) yield better compression than the default, |
| 138 | // but at approximately 4x the CPU usage of the default. |
| 139 | // For now this is not implemented. |
| 140 | SpeedBetterCompression = SpeedDefault |
| 141 | |
| 142 | // SpeedBestCompression will choose the best available compression option. |
| 143 | // For now this is not implemented. |
| 144 | SpeedBestCompression = SpeedDefault |
| 145 | ) |
| 146 | |
| 147 | // EncoderLevelFromString will convert a string representation of an encoding level back |
| 148 | // to a compression level. The compare is not case sensitive. |
| 149 | // If the string wasn't recognized, (false, SpeedDefault) will be returned. |
| 150 | func EncoderLevelFromString(s string) (bool, EncoderLevel) { |
| 151 | for l := EncoderLevel(speedNotSet + 1); l < speedLast; l++ { |
| 152 | if strings.EqualFold(s, l.String()) { |
| 153 | return true, l |
| 154 | } |
| 155 | } |
| 156 | return false, SpeedDefault |
| 157 | } |
| 158 | |
| 159 | // EncoderLevelFromZstd will return an encoder level that closest matches the compression |
| 160 | // ratio of a specific zstd compression level. |
| 161 | // Many input values will provide the same compression level. |
| 162 | func EncoderLevelFromZstd(level int) EncoderLevel { |
| 163 | switch { |
| 164 | case level < 3: |
| 165 | return SpeedFastest |
| 166 | case level >= 3: |
| 167 | return SpeedDefault |
| 168 | } |
| 169 | return SpeedDefault |
| 170 | } |
| 171 | |
| 172 | // String provides a string representation of the compression level. |
| 173 | func (e EncoderLevel) String() string { |
| 174 | switch e { |
| 175 | case SpeedFastest: |
| 176 | return "fastest" |
| 177 | case SpeedDefault: |
| 178 | return "default" |
| 179 | default: |
| 180 | return "invalid" |
| 181 | } |
| 182 | } |
| 183 | |
| 184 | // WithEncoderLevel specifies a predefined compression level. |
| 185 | func WithEncoderLevel(l EncoderLevel) EOption { |
| 186 | return func(o *encoderOptions) error { |
| 187 | switch { |
| 188 | case l <= speedNotSet || l >= speedLast: |
| 189 | return fmt.Errorf("unknown encoder level") |
| 190 | } |
| 191 | o.level = l |
| 192 | return nil |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | // WithZeroFrames will encode 0 length input as full frames. |
| 197 | // This can be needed for compatibility with zstandard usage, |
| 198 | // but is not needed for this package. |
| 199 | func WithZeroFrames(b bool) EOption { |
| 200 | return func(o *encoderOptions) error { |
| 201 | o.fullZero = b |
| 202 | return nil |
| 203 | } |
| 204 | } |
| 205 | |
Dinesh Belwalkar | 396b652 | 2020-02-06 22:11:53 +0000 | [diff] [blame] | 206 | // WithNoEntropyCompression will always skip entropy compression of literals. |
| 207 | // This can be useful if content has matches, but unlikely to benefit from entropy |
| 208 | // compression. Usually the slight speed improvement is not worth enabling this. |
| 209 | func WithNoEntropyCompression(b bool) EOption { |
| 210 | return func(o *encoderOptions) error { |
| 211 | o.noEntropy = b |
| 212 | return nil |
| 213 | } |
| 214 | } |
| 215 | |
Dinesh Belwalkar | e63f7f9 | 2019-11-22 23:11:16 +0000 | [diff] [blame] | 216 | // WithSingleSegment will set the "single segment" flag when EncodeAll is used. |
| 217 | // If this flag is set, data must be regenerated within a single continuous memory segment. |
| 218 | // In this case, Window_Descriptor byte is skipped, but Frame_Content_Size is necessarily present. |
| 219 | // As a consequence, the decoder must allocate a memory segment of size equal or larger than size of your content. |
| 220 | // In order to preserve the decoder from unreasonable memory requirements, |
| 221 | // a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range. |
| 222 | // For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB. |
| 223 | // This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations. |
| 224 | // If this is not specified, block encodes will automatically choose this based on the input size. |
| 225 | // This setting has no effect on streamed encodes. |
| 226 | func WithSingleSegment(b bool) EOption { |
| 227 | return func(o *encoderOptions) error { |
| 228 | o.single = &b |
| 229 | return nil |
| 230 | } |
| 231 | } |