blob: 40eb457331a689374d1ac240f194379112deeea6 [file] [log] [blame]
Dinesh Belwalkare63f7f92019-11-22 23:11:16 +00001package zstd
2
3import (
Dinesh Belwalkar396b6522020-02-06 22:11:53 +00004 "errors"
Dinesh Belwalkare63f7f92019-11-22 23:11:16 +00005 "fmt"
6 "runtime"
7 "strings"
8)
9
10// EOption is an option for creating a encoder.
11type EOption func(*encoderOptions) error
12
13// options retains accumulated state of multiple options.
14type encoderOptions struct {
15 concurrent int
16 crc bool
17 single *bool
18 pad int
19 blockSize int
20 windowSize int
21 level EncoderLevel
22 fullZero bool
Dinesh Belwalkar396b6522020-02-06 22:11:53 +000023 noEntropy bool
Dinesh Belwalkare63f7f92019-11-22 23:11:16 +000024}
25
26func (o *encoderOptions) setDefault() {
27 *o = encoderOptions{
28 // use less ram: true for now, but may change.
29 concurrent: runtime.GOMAXPROCS(0),
30 crc: true,
31 single: nil,
32 blockSize: 1 << 16,
33 windowSize: 1 << 22,
34 level: SpeedDefault,
35 }
36}
37
38// encoder returns an encoder with the selected options.
39func (o encoderOptions) encoder() encoder {
40 switch o.level {
41 case SpeedDefault:
42 return &doubleFastEncoder{fastEncoder: fastEncoder{maxMatchOff: int32(o.windowSize)}}
43 case SpeedFastest:
44 return &fastEncoder{maxMatchOff: int32(o.windowSize)}
45 }
46 panic("unknown compression level")
47}
48
49// WithEncoderCRC will add CRC value to output.
50// Output will be 4 bytes larger.
51func WithEncoderCRC(b bool) EOption {
52 return func(o *encoderOptions) error { o.crc = b; return nil }
53}
54
55// WithEncoderConcurrency will set the concurrency,
56// meaning the maximum number of decoders to run concurrently.
57// The value supplied must be at least 1.
58// By default this will be set to GOMAXPROCS.
59func WithEncoderConcurrency(n int) EOption {
60 return func(o *encoderOptions) error {
61 if n <= 0 {
62 return fmt.Errorf("concurrency must be at least 1")
63 }
64 o.concurrent = n
65 return nil
66 }
67}
68
Dinesh Belwalkar396b6522020-02-06 22:11:53 +000069// WithWindowSize will set the maximum allowed back-reference distance.
70// The value must be a power of two between WindowSizeMin and WindowSizeMax.
71// A larger value will enable better compression but allocate more memory and,
72// for above-default values, take considerably longer.
73// The default value is determined by the compression level.
74func WithWindowSize(n int) EOption {
75 return func(o *encoderOptions) error {
76 switch {
77 case n < MinWindowSize:
78 return fmt.Errorf("window size must be at least %d", MinWindowSize)
79 case n > MaxWindowSize:
80 return fmt.Errorf("window size must be at most %d", MaxWindowSize)
81 case (n & (n - 1)) != 0:
82 return errors.New("window size must be a power of 2")
83 }
84
85 o.windowSize = n
86 if o.blockSize > o.windowSize {
87 o.blockSize = o.windowSize
88 }
89 return nil
90 }
91}
92
Dinesh Belwalkare63f7f92019-11-22 23:11:16 +000093// WithEncoderPadding will add padding to all output so the size will be a multiple of n.
94// This can be used to obfuscate the exact output size or make blocks of a certain size.
95// The contents will be a skippable frame, so it will be invisible by the decoder.
96// n must be > 0 and <= 1GB, 1<<30 bytes.
97// The padded area will be filled with data from crypto/rand.Reader.
98// If `EncodeAll` is used with data already in the destination, the total size will be multiple of this.
99func WithEncoderPadding(n int) EOption {
100 return func(o *encoderOptions) error {
101 if n <= 0 {
102 return fmt.Errorf("padding must be at least 1")
103 }
104 // No need to waste our time.
105 if n == 1 {
106 o.pad = 0
107 }
108 if n > 1<<30 {
109 return fmt.Errorf("padding must less than 1GB (1<<30 bytes) ")
110 }
111 o.pad = n
112 return nil
113 }
114}
115
116// EncoderLevel predefines encoder compression levels.
117// Only use the constants made available, since the actual mapping
118// of these values are very likely to change and your compression could change
119// unpredictably when upgrading the library.
120type EncoderLevel int
121
122const (
123 speedNotSet EncoderLevel = iota
124
125 // SpeedFastest will choose the fastest reasonable compression.
126 // This is roughly equivalent to the fastest Zstandard mode.
127 SpeedFastest
128
129 // SpeedDefault is the default "pretty fast" compression option.
130 // This is roughly equivalent to the default Zstandard mode (level 3).
131 SpeedDefault
132
133 // speedLast should be kept as the last actual compression option.
134 // The is not for external usage, but is used to keep track of the valid options.
135 speedLast
136
137 // SpeedBetterCompression will (in the future) yield better compression than the default,
138 // but at approximately 4x the CPU usage of the default.
139 // For now this is not implemented.
140 SpeedBetterCompression = SpeedDefault
141
142 // SpeedBestCompression will choose the best available compression option.
143 // For now this is not implemented.
144 SpeedBestCompression = SpeedDefault
145)
146
147// EncoderLevelFromString will convert a string representation of an encoding level back
148// to a compression level. The compare is not case sensitive.
149// If the string wasn't recognized, (false, SpeedDefault) will be returned.
150func EncoderLevelFromString(s string) (bool, EncoderLevel) {
151 for l := EncoderLevel(speedNotSet + 1); l < speedLast; l++ {
152 if strings.EqualFold(s, l.String()) {
153 return true, l
154 }
155 }
156 return false, SpeedDefault
157}
158
159// EncoderLevelFromZstd will return an encoder level that closest matches the compression
160// ratio of a specific zstd compression level.
161// Many input values will provide the same compression level.
162func EncoderLevelFromZstd(level int) EncoderLevel {
163 switch {
164 case level < 3:
165 return SpeedFastest
166 case level >= 3:
167 return SpeedDefault
168 }
169 return SpeedDefault
170}
171
172// String provides a string representation of the compression level.
173func (e EncoderLevel) String() string {
174 switch e {
175 case SpeedFastest:
176 return "fastest"
177 case SpeedDefault:
178 return "default"
179 default:
180 return "invalid"
181 }
182}
183
184// WithEncoderLevel specifies a predefined compression level.
185func WithEncoderLevel(l EncoderLevel) EOption {
186 return func(o *encoderOptions) error {
187 switch {
188 case l <= speedNotSet || l >= speedLast:
189 return fmt.Errorf("unknown encoder level")
190 }
191 o.level = l
192 return nil
193 }
194}
195
196// WithZeroFrames will encode 0 length input as full frames.
197// This can be needed for compatibility with zstandard usage,
198// but is not needed for this package.
199func WithZeroFrames(b bool) EOption {
200 return func(o *encoderOptions) error {
201 o.fullZero = b
202 return nil
203 }
204}
205
Dinesh Belwalkar396b6522020-02-06 22:11:53 +0000206// WithNoEntropyCompression will always skip entropy compression of literals.
207// This can be useful if content has matches, but unlikely to benefit from entropy
208// compression. Usually the slight speed improvement is not worth enabling this.
209func WithNoEntropyCompression(b bool) EOption {
210 return func(o *encoderOptions) error {
211 o.noEntropy = b
212 return nil
213 }
214}
215
Dinesh Belwalkare63f7f92019-11-22 23:11:16 +0000216// WithSingleSegment will set the "single segment" flag when EncodeAll is used.
217// If this flag is set, data must be regenerated within a single continuous memory segment.
218// In this case, Window_Descriptor byte is skipped, but Frame_Content_Size is necessarily present.
219// As a consequence, the decoder must allocate a memory segment of size equal or larger than size of your content.
220// In order to preserve the decoder from unreasonable memory requirements,
221// a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range.
222// For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB.
223// This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations.
224// If this is not specified, block encodes will automatically choose this based on the input size.
225// This setting has no effect on streamed encodes.
226func WithSingleSegment(b bool) EOption {
227 return func(o *encoderOptions) error {
228 o.single = &b
229 return nil
230 }
231}