Dinesh Belwalkar | e63f7f9 | 2019-11-22 23:11:16 +0000 | [diff] [blame] | 1 | package zstd |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "runtime" |
| 6 | "strings" |
| 7 | ) |
| 8 | |
| 9 | // EOption is an option for creating a encoder. |
| 10 | type EOption func(*encoderOptions) error |
| 11 | |
| 12 | // options retains accumulated state of multiple options. |
| 13 | type encoderOptions struct { |
| 14 | concurrent int |
| 15 | crc bool |
| 16 | single *bool |
| 17 | pad int |
| 18 | blockSize int |
| 19 | windowSize int |
| 20 | level EncoderLevel |
| 21 | fullZero bool |
| 22 | } |
| 23 | |
| 24 | func (o *encoderOptions) setDefault() { |
| 25 | *o = encoderOptions{ |
| 26 | // use less ram: true for now, but may change. |
| 27 | concurrent: runtime.GOMAXPROCS(0), |
| 28 | crc: true, |
| 29 | single: nil, |
| 30 | blockSize: 1 << 16, |
| 31 | windowSize: 1 << 22, |
| 32 | level: SpeedDefault, |
| 33 | } |
| 34 | } |
| 35 | |
| 36 | // encoder returns an encoder with the selected options. |
| 37 | func (o encoderOptions) encoder() encoder { |
| 38 | switch o.level { |
| 39 | case SpeedDefault: |
| 40 | return &doubleFastEncoder{fastEncoder: fastEncoder{maxMatchOff: int32(o.windowSize)}} |
| 41 | case SpeedFastest: |
| 42 | return &fastEncoder{maxMatchOff: int32(o.windowSize)} |
| 43 | } |
| 44 | panic("unknown compression level") |
| 45 | } |
| 46 | |
| 47 | // WithEncoderCRC will add CRC value to output. |
| 48 | // Output will be 4 bytes larger. |
| 49 | func WithEncoderCRC(b bool) EOption { |
| 50 | return func(o *encoderOptions) error { o.crc = b; return nil } |
| 51 | } |
| 52 | |
| 53 | // WithEncoderConcurrency will set the concurrency, |
| 54 | // meaning the maximum number of decoders to run concurrently. |
| 55 | // The value supplied must be at least 1. |
| 56 | // By default this will be set to GOMAXPROCS. |
| 57 | func WithEncoderConcurrency(n int) EOption { |
| 58 | return func(o *encoderOptions) error { |
| 59 | if n <= 0 { |
| 60 | return fmt.Errorf("concurrency must be at least 1") |
| 61 | } |
| 62 | o.concurrent = n |
| 63 | return nil |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | // WithEncoderPadding will add padding to all output so the size will be a multiple of n. |
| 68 | // This can be used to obfuscate the exact output size or make blocks of a certain size. |
| 69 | // The contents will be a skippable frame, so it will be invisible by the decoder. |
| 70 | // n must be > 0 and <= 1GB, 1<<30 bytes. |
| 71 | // The padded area will be filled with data from crypto/rand.Reader. |
| 72 | // If `EncodeAll` is used with data already in the destination, the total size will be multiple of this. |
| 73 | func WithEncoderPadding(n int) EOption { |
| 74 | return func(o *encoderOptions) error { |
| 75 | if n <= 0 { |
| 76 | return fmt.Errorf("padding must be at least 1") |
| 77 | } |
| 78 | // No need to waste our time. |
| 79 | if n == 1 { |
| 80 | o.pad = 0 |
| 81 | } |
| 82 | if n > 1<<30 { |
| 83 | return fmt.Errorf("padding must less than 1GB (1<<30 bytes) ") |
| 84 | } |
| 85 | o.pad = n |
| 86 | return nil |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | // EncoderLevel predefines encoder compression levels. |
| 91 | // Only use the constants made available, since the actual mapping |
| 92 | // of these values are very likely to change and your compression could change |
| 93 | // unpredictably when upgrading the library. |
| 94 | type EncoderLevel int |
| 95 | |
| 96 | const ( |
| 97 | speedNotSet EncoderLevel = iota |
| 98 | |
| 99 | // SpeedFastest will choose the fastest reasonable compression. |
| 100 | // This is roughly equivalent to the fastest Zstandard mode. |
| 101 | SpeedFastest |
| 102 | |
| 103 | // SpeedDefault is the default "pretty fast" compression option. |
| 104 | // This is roughly equivalent to the default Zstandard mode (level 3). |
| 105 | SpeedDefault |
| 106 | |
| 107 | // speedLast should be kept as the last actual compression option. |
| 108 | // The is not for external usage, but is used to keep track of the valid options. |
| 109 | speedLast |
| 110 | |
| 111 | // SpeedBetterCompression will (in the future) yield better compression than the default, |
| 112 | // but at approximately 4x the CPU usage of the default. |
| 113 | // For now this is not implemented. |
| 114 | SpeedBetterCompression = SpeedDefault |
| 115 | |
| 116 | // SpeedBestCompression will choose the best available compression option. |
| 117 | // For now this is not implemented. |
| 118 | SpeedBestCompression = SpeedDefault |
| 119 | ) |
| 120 | |
| 121 | // EncoderLevelFromString will convert a string representation of an encoding level back |
| 122 | // to a compression level. The compare is not case sensitive. |
| 123 | // If the string wasn't recognized, (false, SpeedDefault) will be returned. |
| 124 | func EncoderLevelFromString(s string) (bool, EncoderLevel) { |
| 125 | for l := EncoderLevel(speedNotSet + 1); l < speedLast; l++ { |
| 126 | if strings.EqualFold(s, l.String()) { |
| 127 | return true, l |
| 128 | } |
| 129 | } |
| 130 | return false, SpeedDefault |
| 131 | } |
| 132 | |
| 133 | // EncoderLevelFromZstd will return an encoder level that closest matches the compression |
| 134 | // ratio of a specific zstd compression level. |
| 135 | // Many input values will provide the same compression level. |
| 136 | func EncoderLevelFromZstd(level int) EncoderLevel { |
| 137 | switch { |
| 138 | case level < 3: |
| 139 | return SpeedFastest |
| 140 | case level >= 3: |
| 141 | return SpeedDefault |
| 142 | } |
| 143 | return SpeedDefault |
| 144 | } |
| 145 | |
| 146 | // String provides a string representation of the compression level. |
| 147 | func (e EncoderLevel) String() string { |
| 148 | switch e { |
| 149 | case SpeedFastest: |
| 150 | return "fastest" |
| 151 | case SpeedDefault: |
| 152 | return "default" |
| 153 | default: |
| 154 | return "invalid" |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | // WithEncoderLevel specifies a predefined compression level. |
| 159 | func WithEncoderLevel(l EncoderLevel) EOption { |
| 160 | return func(o *encoderOptions) error { |
| 161 | switch { |
| 162 | case l <= speedNotSet || l >= speedLast: |
| 163 | return fmt.Errorf("unknown encoder level") |
| 164 | } |
| 165 | o.level = l |
| 166 | return nil |
| 167 | } |
| 168 | } |
| 169 | |
| 170 | // WithZeroFrames will encode 0 length input as full frames. |
| 171 | // This can be needed for compatibility with zstandard usage, |
| 172 | // but is not needed for this package. |
| 173 | func WithZeroFrames(b bool) EOption { |
| 174 | return func(o *encoderOptions) error { |
| 175 | o.fullZero = b |
| 176 | return nil |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | // WithSingleSegment will set the "single segment" flag when EncodeAll is used. |
| 181 | // If this flag is set, data must be regenerated within a single continuous memory segment. |
| 182 | // In this case, Window_Descriptor byte is skipped, but Frame_Content_Size is necessarily present. |
| 183 | // As a consequence, the decoder must allocate a memory segment of size equal or larger than size of your content. |
| 184 | // In order to preserve the decoder from unreasonable memory requirements, |
| 185 | // a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range. |
| 186 | // For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB. |
| 187 | // This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations. |
| 188 | // If this is not specified, block encodes will automatically choose this based on the input size. |
| 189 | // This setting has no effect on streamed encodes. |
| 190 | func WithSingleSegment(b bool) EOption { |
| 191 | return func(o *encoderOptions) error { |
| 192 | o.single = &b |
| 193 | return nil |
| 194 | } |
| 195 | } |