VOL-1921 - updated to use go mod
Change-Id: I8d5187fa91fa619494f972bc29d3bd61e5be3a82
diff --git a/vendor/github.com/DataDog/zstd/README.md b/vendor/github.com/DataDog/zstd/README.md
index a444e0b..b32c3e7 100644
--- a/vendor/github.com/DataDog/zstd/README.md
+++ b/vendor/github.com/DataDog/zstd/README.md
@@ -2,8 +2,8 @@
[C Zstd Homepage](https://github.com/Cyan4973/zstd)
-The current headers and C files are from *v1.4.0* (Commit
-[83b51e9](https://github.com/facebook/zstd/releases/tag/v1.4.0)).
+The current headers and C files are from *v1.4.1* (Commit
+[52181f8](https://github.com/facebook/zstd/releases/tag/v1.4.1)).
## Usage
diff --git a/vendor/github.com/DataDog/zstd/compiler.h b/vendor/github.com/DataDog/zstd/compiler.h
index 0836e3e..87bf51a 100644
--- a/vendor/github.com/DataDog/zstd/compiler.h
+++ b/vendor/github.com/DataDog/zstd/compiler.h
@@ -127,6 +127,13 @@
} \
}
+/* vectorization */
+#if !defined(__clang__) && defined(__GNUC__)
+# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
+#else
+# define DONT_VECTORIZE
+#endif
+
/* disable warnings */
#ifdef _MSC_VER /* Visual Studio */
# include <intrin.h> /* For Visual 2005 */
diff --git a/vendor/github.com/DataDog/zstd/cover.c b/vendor/github.com/DataDog/zstd/cover.c
index 21464ad..6219967 100644
--- a/vendor/github.com/DataDog/zstd/cover.c
+++ b/vendor/github.com/DataDog/zstd/cover.c
@@ -526,10 +526,10 @@
* Prepare a context for dictionary building.
* The context is only dependent on the parameter `d` and can used multiple
* times.
- * Returns 1 on success or zero on error.
+ * Returns 0 on success or error code on error.
* The context must be destroyed with `COVER_ctx_destroy()`.
*/
-static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
+static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer,
const size_t *samplesSizes, unsigned nbSamples,
unsigned d, double splitPoint) {
const BYTE *const samples = (const BYTE *)samplesBuffer;
@@ -544,17 +544,17 @@
totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) {
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
(unsigned)(totalSamplesSize>>20), (COVER_MAX_SAMPLES_SIZE >> 20));
- return 0;
+ return ERROR(srcSize_wrong);
}
/* Check if there are at least 5 training samples */
if (nbTrainSamples < 5) {
DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid.", nbTrainSamples);
- return 0;
+ return ERROR(srcSize_wrong);
}
/* Check if there's testing sample */
if (nbTestSamples < 1) {
DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.", nbTestSamples);
- return 0;
+ return ERROR(srcSize_wrong);
}
/* Zero the context */
memset(ctx, 0, sizeof(*ctx));
@@ -577,7 +577,7 @@
if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) {
DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n");
COVER_ctx_destroy(ctx);
- return 0;
+ return ERROR(memory_allocation);
}
ctx->freqs = NULL;
ctx->d = d;
@@ -624,7 +624,7 @@
(ctx->d <= 8 ? &COVER_cmp8 : &COVER_cmp), &COVER_group);
ctx->freqs = ctx->suffix;
ctx->suffix = NULL;
- return 1;
+ return 0;
}
void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel)
@@ -729,11 +729,11 @@
/* Checks */
if (!COVER_checkParameters(parameters, dictBufferCapacity)) {
DISPLAYLEVEL(1, "Cover parameters incorrect\n");
- return ERROR(GENERIC);
+ return ERROR(parameter_outOfBound);
}
if (nbSamples == 0) {
DISPLAYLEVEL(1, "Cover must have at least one input file\n");
- return ERROR(GENERIC);
+ return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
@@ -741,15 +741,18 @@
return ERROR(dstSize_tooSmall);
}
/* Initialize context and activeDmers */
- if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
- parameters.d, parameters.splitPoint)) {
- return ERROR(GENERIC);
+ {
+ size_t const initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
+ parameters.d, parameters.splitPoint);
+ if (ZSTD_isError(initVal)) {
+ return initVal;
+ }
}
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, g_displayLevel);
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
COVER_ctx_destroy(&ctx);
- return ERROR(GENERIC);
+ return ERROR(memory_allocation);
}
DISPLAYLEVEL(2, "Building dictionary\n");
@@ -810,7 +813,7 @@
cctx, dst, dstCapacity, samples + offsets[i],
samplesSizes[i], cdict);
if (ZSTD_isError(size)) {
- totalCompressedSize = ERROR(GENERIC);
+ totalCompressedSize = size;
goto _compressCleanup;
}
totalCompressedSize += size;
@@ -886,9 +889,11 @@
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
* If this dictionary is the best so far save it and its parameters.
*/
-void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
- ZDICT_cover_params_t parameters, void *dict,
- size_t dictSize) {
+void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters,
+ COVER_dictSelection_t selection) {
+ void* dict = selection.dictContent;
+ size_t compressedSize = selection.totalCompressedSize;
+ size_t dictSize = selection.dictSize;
if (!best) {
return;
}
@@ -914,6 +919,9 @@
}
}
/* Save the dictionary, parameters, and size */
+ if (!dict) {
+ return;
+ }
memcpy(best->dict, dict, dictSize);
best->dictSize = dictSize;
best->parameters = parameters;
@@ -926,6 +934,111 @@
}
}
+COVER_dictSelection_t COVER_dictSelectionError(size_t error) {
+ COVER_dictSelection_t selection = { NULL, 0, error };
+ return selection;
+}
+
+unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) {
+ return (ZSTD_isError(selection.totalCompressedSize) || !selection.dictContent);
+}
+
+void COVER_dictSelectionFree(COVER_dictSelection_t selection){
+ free(selection.dictContent);
+}
+
+COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
+ size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
+ size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
+
+ size_t largestDict = 0;
+ size_t largestCompressed = 0;
+ BYTE* customDictContentEnd = customDictContent + dictContentSize;
+
+ BYTE * largestDictbuffer = (BYTE *)malloc(dictContentSize);
+ BYTE * candidateDictBuffer = (BYTE *)malloc(dictContentSize);
+ double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
+
+ if (!largestDictbuffer || !candidateDictBuffer) {
+ free(largestDictbuffer);
+ free(candidateDictBuffer);
+ return COVER_dictSelectionError(dictContentSize);
+ }
+
+ /* Initial dictionary size and compressed size */
+ memcpy(largestDictbuffer, customDictContent, dictContentSize);
+ dictContentSize = ZDICT_finalizeDictionary(
+ largestDictbuffer, dictContentSize, customDictContent, dictContentSize,
+ samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
+
+ if (ZDICT_isError(dictContentSize)) {
+ free(largestDictbuffer);
+ free(candidateDictBuffer);
+ return COVER_dictSelectionError(dictContentSize);
+ }
+
+ totalCompressedSize = COVER_checkTotalCompressedSize(params, samplesSizes,
+ samplesBuffer, offsets,
+ nbCheckSamples, nbSamples,
+ largestDictbuffer, dictContentSize);
+
+ if (ZSTD_isError(totalCompressedSize)) {
+ free(largestDictbuffer);
+ free(candidateDictBuffer);
+ return COVER_dictSelectionError(totalCompressedSize);
+ }
+
+ if (params.shrinkDict == 0) {
+ COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize };
+ free(candidateDictBuffer);
+ return selection;
+ }
+
+ largestDict = dictContentSize;
+ largestCompressed = totalCompressedSize;
+ dictContentSize = ZDICT_DICTSIZE_MIN;
+
+ /* Largest dict is initially at least ZDICT_DICTSIZE_MIN */
+ while (dictContentSize < largestDict) {
+ memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
+ dictContentSize = ZDICT_finalizeDictionary(
+ candidateDictBuffer, dictContentSize, customDictContentEnd - dictContentSize, dictContentSize,
+ samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
+
+ if (ZDICT_isError(dictContentSize)) {
+ free(largestDictbuffer);
+ free(candidateDictBuffer);
+ return COVER_dictSelectionError(dictContentSize);
+
+ }
+
+ totalCompressedSize = COVER_checkTotalCompressedSize(params, samplesSizes,
+ samplesBuffer, offsets,
+ nbCheckSamples, nbSamples,
+ candidateDictBuffer, dictContentSize);
+
+ if (ZSTD_isError(totalCompressedSize)) {
+ free(largestDictbuffer);
+ free(candidateDictBuffer);
+ return COVER_dictSelectionError(totalCompressedSize);
+ }
+
+ if (totalCompressedSize <= largestCompressed * regressionTolerance) {
+ COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize };
+ free(largestDictbuffer);
+ return selection;
+ }
+ dictContentSize *= 2;
+ }
+ dictContentSize = largestDict;
+ totalCompressedSize = largestCompressed;
+ {
+ COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize };
+ free(candidateDictBuffer);
+ return selection;
+ }
+}
+
/**
* Parameters for COVER_tryParameters().
*/
@@ -951,6 +1064,7 @@
/* Allocate space for hash table, dict, and freqs */
COVER_map_t activeDmers;
BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+ COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32));
if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) {
DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n");
@@ -966,29 +1080,21 @@
{
const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
dictBufferCapacity, parameters);
- dictBufferCapacity = ZDICT_finalizeDictionary(
- dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
- ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples,
- parameters.zParams);
- if (ZDICT_isError(dictBufferCapacity)) {
- DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
+ selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
+ ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
+ totalCompressedSize);
+
+ if (COVER_dictSelectionIsError(selection)) {
+ DISPLAYLEVEL(1, "Failed to select dictionary\n");
goto _cleanup;
}
}
- /* Check total compressed size */
- totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes,
- ctx->samples, ctx->offsets,
- ctx->nbTrainSamples, ctx->nbSamples,
- dict, dictBufferCapacity);
-
_cleanup:
- COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
- dictBufferCapacity);
+ free(dict);
+ COVER_best_finish(data->best, parameters, selection);
free(data);
COVER_map_destroy(&activeDmers);
- if (dict) {
- free(dict);
- }
+ COVER_dictSelectionFree(selection);
if (freqs) {
free(freqs);
}
@@ -1010,6 +1116,7 @@
const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
const unsigned kIterations =
(1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
+ const unsigned shrinkDict = 0;
/* Local variables */
const int displayLevel = parameters->zParams.notificationLevel;
unsigned iteration = 1;
@@ -1022,15 +1129,15 @@
/* Checks */
if (splitPoint <= 0 || splitPoint > 1) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
- return ERROR(GENERIC);
+ return ERROR(parameter_outOfBound);
}
if (kMinK < kMaxD || kMaxK < kMinK) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n");
- return ERROR(GENERIC);
+ return ERROR(parameter_outOfBound);
}
if (nbSamples == 0) {
DISPLAYLEVEL(1, "Cover must have at least one input file\n");
- return ERROR(GENERIC);
+ return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
@@ -1054,11 +1161,14 @@
/* Initialize the context for this value of d */
COVER_ctx_t ctx;
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
- if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint)) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
- COVER_best_destroy(&best);
- POOL_free(pool);
- return ERROR(GENERIC);
+ {
+ const size_t initVal = COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint);
+ if (ZSTD_isError(initVal)) {
+ LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+ COVER_best_destroy(&best);
+ POOL_free(pool);
+ return initVal;
+ }
}
if (!warned) {
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.suffixSize, displayLevel);
@@ -1075,7 +1185,7 @@
COVER_best_destroy(&best);
COVER_ctx_destroy(&ctx);
POOL_free(pool);
- return ERROR(GENERIC);
+ return ERROR(memory_allocation);
}
data->ctx = &ctx;
data->best = &best;
@@ -1085,6 +1195,7 @@
data->parameters.d = d;
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
+ data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = g_displayLevel;
/* Check the parameters */
if (!COVER_checkParameters(data->parameters, dictBufferCapacity)) {
diff --git a/vendor/github.com/DataDog/zstd/cover.h b/vendor/github.com/DataDog/zstd/cover.h
index efb4680..d9e0636 100644
--- a/vendor/github.com/DataDog/zstd/cover.h
+++ b/vendor/github.com/DataDog/zstd/cover.h
@@ -47,6 +47,15 @@
} COVER_epoch_info_t;
/**
+ * Struct used for the dictionary selection function.
+ */
+typedef struct COVER_dictSelection {
+ BYTE* dictContent;
+ size_t dictSize;
+ size_t totalCompressedSize;
+} COVER_dictSelection_t;
+
+/**
* Computes the number of epochs and the size of each epoch.
* We will make sure that each epoch gets at least 10 * k bytes.
*
@@ -107,6 +116,32 @@
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
* If this dictionary is the best so far save it and its parameters.
*/
-void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
- ZDICT_cover_params_t parameters, void *dict,
- size_t dictSize);
+void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters,
+ COVER_dictSelection_t selection);
+/**
+ * Error function for COVER_selectDict function. Checks if the return
+ * value is an error.
+ */
+unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection);
+
+ /**
+ * Error function for COVER_selectDict function. Returns a struct where
+ * return.totalCompressedSize is a ZSTD error.
+ */
+COVER_dictSelection_t COVER_dictSelectionError(size_t error);
+
+/**
+ * Always call after selectDict is called to free up used memory from
+ * newly created dictionary.
+ */
+void COVER_dictSelectionFree(COVER_dictSelection_t selection);
+
+/**
+ * Called to finalize the dictionary and select one based on whether or not
+ * the shrink-dict flag was enabled. If enabled the dictionary used is the
+ * smallest dictionary within a specified regression of the compressed size
+ * from the largest dictionary.
+ */
+ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
+ size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
+ size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
diff --git a/vendor/github.com/DataDog/zstd/fastcover.c b/vendor/github.com/DataDog/zstd/fastcover.c
index 5b6b941..941bb5a 100644
--- a/vendor/github.com/DataDog/zstd/fastcover.c
+++ b/vendor/github.com/DataDog/zstd/fastcover.c
@@ -287,10 +287,10 @@
* Prepare a context for dictionary building.
* The context is only dependent on the parameter `d` and can used multiple
* times.
- * Returns 1 on success or zero on error.
+ * Returns 0 on success or error code on error.
* The context must be destroyed with `FASTCOVER_ctx_destroy()`.
*/
-static int
+static size_t
FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
const void* samplesBuffer,
const size_t* samplesSizes, unsigned nbSamples,
@@ -310,19 +310,19 @@
totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
(unsigned)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
- return 0;
+ return ERROR(srcSize_wrong);
}
/* Check if there are at least 5 training samples */
if (nbTrainSamples < 5) {
DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples);
- return 0;
+ return ERROR(srcSize_wrong);
}
/* Check if there's testing sample */
if (nbTestSamples < 1) {
DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples);
- return 0;
+ return ERROR(srcSize_wrong);
}
/* Zero the context */
@@ -347,7 +347,7 @@
if (ctx->offsets == NULL) {
DISPLAYLEVEL(1, "Failed to allocate scratch buffers \n");
FASTCOVER_ctx_destroy(ctx);
- return 0;
+ return ERROR(memory_allocation);
}
/* Fill offsets from the samplesSizes */
@@ -364,13 +364,13 @@
if (ctx->freqs == NULL) {
DISPLAYLEVEL(1, "Failed to allocate frequency table \n");
FASTCOVER_ctx_destroy(ctx);
- return 0;
+ return ERROR(memory_allocation);
}
DISPLAYLEVEL(2, "Computing frequencies\n");
FASTCOVER_computeFrequency(ctx->freqs, ctx);
- return 1;
+ return 0;
}
@@ -435,7 +435,6 @@
return tail;
}
-
/**
* Parameters for FASTCOVER_tryParameters().
*/
@@ -464,6 +463,7 @@
U16* segmentFreqs = (U16 *)calloc(((U64)1 << ctx->f), sizeof(U16));
/* Allocate space for hash table, dict, and freqs */
BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
+ COVER_dictSelection_t selection = COVER_dictSelectionError(ERROR(GENERIC));
U32 *freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
if (!segmentFreqs || !dict || !freqs) {
DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
@@ -473,27 +473,24 @@
memcpy(freqs, ctx->freqs, ((U64)1 << ctx->f) * sizeof(U32));
/* Build the dictionary */
{ const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity,
- parameters, segmentFreqs);
+ parameters, segmentFreqs);
+
const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
- dictBufferCapacity = ZDICT_finalizeDictionary(
- dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
- ctx->samples, ctx->samplesSizes, nbFinalizeSamples, parameters.zParams);
- if (ZDICT_isError(dictBufferCapacity)) {
- DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
+ selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
+ ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
+ totalCompressedSize);
+
+ if (COVER_dictSelectionIsError(selection)) {
+ DISPLAYLEVEL(1, "Failed to select dictionary\n");
goto _cleanup;
}
}
- /* Check total compressed size */
- totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes,
- ctx->samples, ctx->offsets,
- ctx->nbTrainSamples, ctx->nbSamples,
- dict, dictBufferCapacity);
_cleanup:
- COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
- dictBufferCapacity);
+ free(dict);
+ COVER_best_finish(data->best, parameters, selection);
free(data);
free(segmentFreqs);
- free(dict);
+ COVER_dictSelectionFree(selection);
free(freqs);
}
@@ -508,6 +505,7 @@
coverParams->nbThreads = fastCoverParams.nbThreads;
coverParams->splitPoint = fastCoverParams.splitPoint;
coverParams->zParams = fastCoverParams.zParams;
+ coverParams->shrinkDict = fastCoverParams.shrinkDict;
}
@@ -524,6 +522,7 @@
fastCoverParams->f = f;
fastCoverParams->accel = accel;
fastCoverParams->zParams = coverParams.zParams;
+ fastCoverParams->shrinkDict = coverParams.shrinkDict;
}
@@ -550,11 +549,11 @@
if (!FASTCOVER_checkParameters(coverParams, dictBufferCapacity, parameters.f,
parameters.accel)) {
DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
- return ERROR(GENERIC);
+ return ERROR(parameter_outOfBound);
}
if (nbSamples == 0) {
DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
- return ERROR(GENERIC);
+ return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
@@ -564,11 +563,14 @@
/* Assign corresponding FASTCOVER_accel_t to accelParams*/
accelParams = FASTCOVER_defaultAccelParameters[parameters.accel];
/* Initialize context */
- if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
+ {
+ size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
coverParams.d, parameters.splitPoint, parameters.f,
- accelParams)) {
- DISPLAYLEVEL(1, "Failed to initialize context\n");
- return ERROR(GENERIC);
+ accelParams);
+ if (ZSTD_isError(initVal)) {
+ DISPLAYLEVEL(1, "Failed to initialize context\n");
+ return initVal;
+ }
}
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, g_displayLevel);
/* Build the dictionary */
@@ -616,6 +618,7 @@
(1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
const unsigned f = parameters->f == 0 ? DEFAULT_F : parameters->f;
const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel;
+ const unsigned shrinkDict = 0;
/* Local variables */
const int displayLevel = parameters->zParams.notificationLevel;
unsigned iteration = 1;
@@ -627,19 +630,19 @@
/* Checks */
if (splitPoint <= 0 || splitPoint > 1) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
- return ERROR(GENERIC);
+ return ERROR(parameter_outOfBound);
}
if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n");
- return ERROR(GENERIC);
+ return ERROR(parameter_outOfBound);
}
if (kMinK < kMaxD || kMaxK < kMinK) {
LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n");
- return ERROR(GENERIC);
+ return ERROR(parameter_outOfBound);
}
if (nbSamples == 0) {
LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n");
- return ERROR(GENERIC);
+ return ERROR(srcSize_wrong);
}
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n",
@@ -666,11 +669,14 @@
/* Initialize the context for this value of d */
FASTCOVER_ctx_t ctx;
LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
- if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams)) {
- LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
- COVER_best_destroy(&best);
- POOL_free(pool);
- return ERROR(GENERIC);
+ {
+ size_t const initVal = FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams);
+ if (ZSTD_isError(initVal)) {
+ LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
+ COVER_best_destroy(&best);
+ POOL_free(pool);
+ return initVal;
+ }
}
if (!warned) {
COVER_warnOnSmallCorpus(dictBufferCapacity, ctx.nbDmers, displayLevel);
@@ -687,7 +693,7 @@
COVER_best_destroy(&best);
FASTCOVER_ctx_destroy(&ctx);
POOL_free(pool);
- return ERROR(GENERIC);
+ return ERROR(memory_allocation);
}
data->ctx = &ctx;
data->best = &best;
@@ -697,6 +703,7 @@
data->parameters.d = d;
data->parameters.splitPoint = splitPoint;
data->parameters.steps = kSteps;
+ data->parameters.shrinkDict = shrinkDict;
data->parameters.zParams.notificationLevel = g_displayLevel;
/* Check the parameters */
if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
diff --git a/vendor/github.com/DataDog/zstd/travis_test_32.sh b/vendor/github.com/DataDog/zstd/travis_test_32.sh
old mode 100755
new mode 100644
diff --git a/vendor/github.com/DataDog/zstd/zdict.c b/vendor/github.com/DataDog/zstd/zdict.c
index c753da0..ee21ee1 100644
--- a/vendor/github.com/DataDog/zstd/zdict.c
+++ b/vendor/github.com/DataDog/zstd/zdict.c
@@ -741,7 +741,7 @@
/* analyze, build stats, starting with literals */
{ size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog);
if (HUF_isError(maxNbBits)) {
- eSize = ERROR(GENERIC);
+ eSize = maxNbBits;
DISPLAYLEVEL(1, " HUF_buildCTable error \n");
goto _cleanup;
}
@@ -764,7 +764,7 @@
total=0; for (u=0; u<=offcodeMax; u++) total+=offcodeCount[u];
errorCode = FSE_normalizeCount(offcodeNCount, Offlog, offcodeCount, total, offcodeMax);
if (FSE_isError(errorCode)) {
- eSize = ERROR(GENERIC);
+ eSize = errorCode;
DISPLAYLEVEL(1, "FSE_normalizeCount error with offcodeCount \n");
goto _cleanup;
}
@@ -773,7 +773,7 @@
total=0; for (u=0; u<=MaxML; u++) total+=matchLengthCount[u];
errorCode = FSE_normalizeCount(matchLengthNCount, mlLog, matchLengthCount, total, MaxML);
if (FSE_isError(errorCode)) {
- eSize = ERROR(GENERIC);
+ eSize = errorCode;
DISPLAYLEVEL(1, "FSE_normalizeCount error with matchLengthCount \n");
goto _cleanup;
}
@@ -782,7 +782,7 @@
total=0; for (u=0; u<=MaxLL; u++) total+=litLengthCount[u];
errorCode = FSE_normalizeCount(litLengthNCount, llLog, litLengthCount, total, MaxLL);
if (FSE_isError(errorCode)) {
- eSize = ERROR(GENERIC);
+ eSize = errorCode;
DISPLAYLEVEL(1, "FSE_normalizeCount error with litLengthCount \n");
goto _cleanup;
}
@@ -791,7 +791,7 @@
/* write result to buffer */
{ size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog);
if (HUF_isError(hhSize)) {
- eSize = ERROR(GENERIC);
+ eSize = hhSize;
DISPLAYLEVEL(1, "HUF_writeCTable error \n");
goto _cleanup;
}
@@ -802,7 +802,7 @@
{ size_t const ohSize = FSE_writeNCount(dstPtr, maxDstSize, offcodeNCount, OFFCODE_MAX, Offlog);
if (FSE_isError(ohSize)) {
- eSize = ERROR(GENERIC);
+ eSize = ohSize;
DISPLAYLEVEL(1, "FSE_writeNCount error with offcodeNCount \n");
goto _cleanup;
}
@@ -813,7 +813,7 @@
{ size_t const mhSize = FSE_writeNCount(dstPtr, maxDstSize, matchLengthNCount, MaxML, mlLog);
if (FSE_isError(mhSize)) {
- eSize = ERROR(GENERIC);
+ eSize = mhSize;
DISPLAYLEVEL(1, "FSE_writeNCount error with matchLengthNCount \n");
goto _cleanup;
}
@@ -824,7 +824,7 @@
{ size_t const lhSize = FSE_writeNCount(dstPtr, maxDstSize, litLengthNCount, MaxLL, llLog);
if (FSE_isError(lhSize)) {
- eSize = ERROR(GENERIC);
+ eSize = lhSize;
DISPLAYLEVEL(1, "FSE_writeNCount error with litlengthNCount \n");
goto _cleanup;
}
@@ -834,7 +834,7 @@
}
if (maxDstSize<12) {
- eSize = ERROR(GENERIC);
+ eSize = ERROR(dstSize_tooSmall);
DISPLAYLEVEL(1, "not enough space to write RepOffsets \n");
goto _cleanup;
}
diff --git a/vendor/github.com/DataDog/zstd/zdict.h b/vendor/github.com/DataDog/zstd/zdict.h
index e229731..37978ec 100644
--- a/vendor/github.com/DataDog/zstd/zdict.h
+++ b/vendor/github.com/DataDog/zstd/zdict.h
@@ -94,6 +94,8 @@
unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
ZDICT_params_t zParams;
} ZDICT_cover_params_t;
@@ -105,6 +107,9 @@
unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */
double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */
unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */
+ unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */
+ unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */
+
ZDICT_params_t zParams;
} ZDICT_fastCover_params_t;
diff --git a/vendor/github.com/DataDog/zstd/zstd.h b/vendor/github.com/DataDog/zstd/zstd.h
index 53470c1..a1910ee 100644
--- a/vendor/github.com/DataDog/zstd/zstd.h
+++ b/vendor/github.com/DataDog/zstd/zstd.h
@@ -71,7 +71,7 @@
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 4
-#define ZSTD_VERSION_RELEASE 0
+#define ZSTD_VERSION_RELEASE 1
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */
@@ -82,16 +82,16 @@
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */
-/***************************************
-* Default constant
-***************************************/
+/* *************************************
+ * Default constant
+ ***************************************/
#ifndef ZSTD_CLEVEL_DEFAULT
# define ZSTD_CLEVEL_DEFAULT 3
#endif
-/***************************************
-* Constants
-***************************************/
+/* *************************************
+ * Constants
+ ***************************************/
/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
@@ -183,9 +183,14 @@
***************************************/
/*= Compression context
* When compressing many times,
- * it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+ * it is recommended to allocate a context just once,
+ * and re-use it for each successive compression operation.
* This will make workload friendlier for system's memory.
- * Use one context per thread for parallel execution in multi-threaded environments. */
+ * Note : re-using context is just a speed / resource optimization.
+ * It doesn't change the compression ratio, which remains identical.
+ * Note 2 : In multi-threaded environments,
+ * use one different context per thread for parallel execution.
+ */
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
@@ -380,6 +385,7 @@
* ZSTD_c_forceMaxWindow
* ZSTD_c_forceAttachDict
* ZSTD_c_literalCompressionMode
+ * ZSTD_c_targetCBlockSize
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
@@ -389,6 +395,7 @@
ZSTD_c_experimentalParam3=1000,
ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002,
+ ZSTD_c_experimentalParam6=1003,
} ZSTD_cParameter;
typedef struct {
@@ -657,17 +664,33 @@
ZSTD_inBuffer* input,
ZSTD_EndDirective endOp);
-ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
-ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
-/*******************************************************************************
- * This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and
- * ZSTD_compressStream2(). It is redundant, but is still fully supported.
+/* These buffer sizes are softly recommended.
+ * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output.
+ * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(),
+ * reducing the amount of memory shuffling and buffering, resulting in minor performance savings.
+ *
+ * However, note that these recommendations are from the perspective of a C caller program.
+ * If the streaming interface is invoked from some other language,
+ * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo,
+ * a major performance rule is to reduce crossing such interface to an absolute minimum.
+ * It's not rare that performance ends being spent more into the interface, rather than compression itself.
+ * In which cases, prefer using large buffers, as large as practical,
+ * for both input and output, to reduce the nb of roundtrips.
+ */
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */
+
+
+/* *****************************************************************************
+ * This following is a legacy streaming API.
+ * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
+ * It is redundant, but remains fully supported.
* Advanced parameters and dictionary compression can only be used through the
* new API.
******************************************************************************/
-/**
+/*!
* Equivalent to:
*
* ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
@@ -675,16 +698,16 @@
* ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
*/
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
-/**
+/*!
* Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).
* NOTE: The return value is different. ZSTD_compressStream() returns a hint for
* the next read size (if non-zero and not an error). ZSTD_compressStream2()
- * returns the number of bytes left to flush (if non-zero and not an error).
+ * returns the minimum nb of bytes left to flush (if non-zero and not an error).
*/
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-/** Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
+/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@@ -969,7 +992,7 @@
#endif /* ZSTD_H_235446 */
-/****************************************************************************************
+/* **************************************************************************************
* ADVANCED AND EXPERIMENTAL FUNCTIONS
****************************************************************************************
* The definitions in the following section are considered experimental.
@@ -1037,6 +1060,10 @@
#define ZSTD_LDM_HASHRATELOG_MIN 0
#define ZSTD_LDM_HASHRATELOG_MAX (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)
+/* Advanced parameter bounds */
+#define ZSTD_TARGETCBLOCKSIZE_MIN 64
+#define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX
+
/* internal */
#define ZSTD_HASHLOG3_MAX 17
@@ -1162,7 +1189,7 @@
* however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
-/** ZSTD_decompressBound() :
+/*! ZSTD_decompressBound() :
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
* `srcSize` must be the _exact_ size of this series
* (i.e. there should be a frame boundary at `src + srcSize`)
@@ -1409,6 +1436,11 @@
*/
#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
+/* Tries to fit compressed block size to be around targetCBlockSize.
+ * No target when targetCBlockSize == 0.
+ * There is no guarantee on compressed block size (default:0) */
+#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6
+
/*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
* and store it into int* value.
@@ -1843,7 +1875,7 @@
unsigned checksumFlag;
} ZSTD_frameHeader;
-/** ZSTD_getFrameHeader() :
+/*! ZSTD_getFrameHeader() :
* decode Frame Header, or requires larger `srcSize`.
* @return : 0, `zfhPtr` is correctly filled,
* >0, `srcSize` is too small, value is wanted `srcSize` amount,
diff --git a/vendor/github.com/DataDog/zstd/zstd_compress.c b/vendor/github.com/DataDog/zstd/zstd_compress.c
index 2e163c8..1476512 100644
--- a/vendor/github.com/DataDog/zstd/zstd_compress.c
+++ b/vendor/github.com/DataDog/zstd/zstd_compress.c
@@ -385,6 +385,11 @@
bounds.upperBound = ZSTD_lcm_uncompressed;
return bounds;
+ case ZSTD_c_targetCBlockSize:
+ bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN;
+ bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX;
+ return bounds;
+
default:
{ ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 };
return boundError;
@@ -452,6 +457,7 @@
case ZSTD_c_ldmHashRateLog:
case ZSTD_c_forceAttachDict:
case ZSTD_c_literalCompressionMode:
+ case ZSTD_c_targetCBlockSize:
default:
return 0;
}
@@ -497,6 +503,7 @@
case ZSTD_c_ldmHashLog:
case ZSTD_c_ldmMinMatch:
case ZSTD_c_ldmBucketSizeLog:
+ case ZSTD_c_targetCBlockSize:
break;
default: RETURN_ERROR(parameter_unsupported);
@@ -671,6 +678,12 @@
CCtxParams->ldmParams.hashRateLog = value;
return CCtxParams->ldmParams.hashRateLog;
+ case ZSTD_c_targetCBlockSize :
+ if (value!=0) /* 0 ==> default */
+ BOUNDCHECK(ZSTD_c_targetCBlockSize, value);
+ CCtxParams->targetCBlockSize = value;
+ return CCtxParams->targetCBlockSize;
+
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
}
@@ -692,13 +705,13 @@
*value = CCtxParams->compressionLevel;
break;
case ZSTD_c_windowLog :
- *value = CCtxParams->cParams.windowLog;
+ *value = (int)CCtxParams->cParams.windowLog;
break;
case ZSTD_c_hashLog :
- *value = CCtxParams->cParams.hashLog;
+ *value = (int)CCtxParams->cParams.hashLog;
break;
case ZSTD_c_chainLog :
- *value = CCtxParams->cParams.chainLog;
+ *value = (int)CCtxParams->cParams.chainLog;
break;
case ZSTD_c_searchLog :
*value = CCtxParams->cParams.searchLog;
@@ -773,6 +786,9 @@
case ZSTD_c_ldmHashRateLog :
*value = CCtxParams->ldmParams.hashRateLog;
break;
+ case ZSTD_c_targetCBlockSize :
+ *value = (int)CCtxParams->targetCBlockSize;
+ break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
return 0;
@@ -930,12 +946,12 @@
@return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{
- BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog);
- BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog);
- BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog);
- BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog);
- BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch);
- BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength);
+ BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog);
+ BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog);
+ BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog);
+ BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog);
+ BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch);
+ BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength);
BOUNDCHECK(ZSTD_c_strategy, cParams.strategy);
return 0;
}
@@ -951,7 +967,7 @@
if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \
else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \
}
-# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int)
+# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned)
CLAMP(ZSTD_c_windowLog, cParams.windowLog);
CLAMP(ZSTD_c_chainLog, cParams.chainLog);
CLAMP(ZSTD_c_hashLog, cParams.hashLog);
@@ -1282,15 +1298,14 @@
}
/*! ZSTD_invalidateMatchState()
- * Invalidate all the matches in the match finder tables.
- * Requires nextSrc and base to be set (can be NULL).
+ * Invalidate all the matches in the match finder tables.
+ * Requires nextSrc and base to be set (can be NULL).
*/
static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms)
{
ZSTD_window_clear(&ms->window);
ms->nextToUpdate = ms->window.dictLimit;
- ms->nextToUpdate3 = ms->window.dictLimit;
ms->loadedDictEnd = 0;
ms->opt.litLengthSum = 0; /* force reset of btopt stats */
ms->dictMatchState = NULL;
@@ -1327,15 +1342,17 @@
typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e;
+typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e;
+
static void*
ZSTD_reset_matchState(ZSTD_matchState_t* ms,
void* ptr,
const ZSTD_compressionParameters* cParams,
- ZSTD_compResetPolicy_e const crp, U32 const forCCtx)
+ ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho)
{
size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog);
size_t const hSize = ((size_t)1) << cParams->hashLog;
- U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
+ U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
size_t const h3Size = ((size_t)1) << hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
@@ -1349,7 +1366,7 @@
ZSTD_invalidateMatchState(ms);
/* opt parser space */
- if (forCCtx && (cParams->strategy >= ZSTD_btopt)) {
+ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
DEBUGLOG(4, "reserving optimal parser space");
ms->opt.litFreq = (unsigned*)ptr;
ms->opt.litLengthFreq = ms->opt.litFreq + (1<<Litbits);
@@ -1377,6 +1394,19 @@
return ptr;
}
+/* ZSTD_indexTooCloseToMax() :
+ * minor optimization : prefer memset() rather than reduceIndex()
+ * which is measurably slow in some circumstances (reported for Visual Studio).
+ * Works when re-using a context for a lot of smallish inputs :
+ * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN,
+ * memset() will be triggered before reduceIndex().
+ */
+#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB)
+static int ZSTD_indexTooCloseToMax(ZSTD_window_t w)
+{
+ return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN);
+}
+
#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */
#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large
* during at least this number of times,
@@ -1388,7 +1418,7 @@
note : `params` are assumed fully validated at this stage */
static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_CCtx_params params,
- U64 pledgedSrcSize,
+ U64 const pledgedSrcSize,
ZSTD_compResetPolicy_e const crp,
ZSTD_buffered_policy_e const zbuff)
{
@@ -1400,13 +1430,21 @@
if (ZSTD_equivalentParams(zc->appliedParams, params,
zc->inBuffSize,
zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit,
- zbuff, pledgedSrcSize)) {
- DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> continue mode (wLog1=%u, blockSize1=%zu)",
- zc->appliedParams.cParams.windowLog, zc->blockSize);
+ zbuff, pledgedSrcSize) ) {
+ DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode");
zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */
- if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION)
+ if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) {
+ DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)",
+ zc->appliedParams.cParams.windowLog, zc->blockSize);
+ if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) {
+ /* prefer a reset, faster than a rescale */
+ ZSTD_reset_matchState(&zc->blockState.matchState,
+ zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32,
+ ¶ms.cParams,
+ crp, ZSTD_resetTarget_CCtx);
+ }
return ZSTD_continueCCtx(zc, params, pledgedSrcSize);
- } }
+ } } }
DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx");
if (params.ldmParams.enableLdm) {
@@ -1449,7 +1487,7 @@
DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize);
if (workSpaceTooSmall || workSpaceWasteful) {
- DEBUGLOG(4, "Need to resize workSpaceSize from %zuKB to %zuKB",
+ DEBUGLOG(4, "Resize workSpaceSize from %zuKB to %zuKB",
zc->workSpaceSize >> 10,
neededSpace >> 10);
@@ -1491,7 +1529,10 @@
ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock);
- ptr = zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32;
+ ptr = ZSTD_reset_matchState(&zc->blockState.matchState,
+ zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32,
+ ¶ms.cParams,
+ crp, ZSTD_resetTarget_CCtx);
/* ldm hash table */
/* initialize bucketOffsets table later for pointer alignment */
@@ -1509,8 +1550,6 @@
}
assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */
- ptr = ZSTD_reset_matchState(&zc->blockState.matchState, ptr, ¶ms.cParams, crp, /* forCCtx */ 1);
-
/* sequences storage */
zc->seqStore.maxNbSeq = maxNbSeq;
zc->seqStore.sequencesStart = (seqDef*)ptr;
@@ -1587,15 +1626,14 @@
* handled in _enforceMaxDist */
}
-static size_t ZSTD_resetCCtx_byAttachingCDict(
- ZSTD_CCtx* cctx,
- const ZSTD_CDict* cdict,
- ZSTD_CCtx_params params,
- U64 pledgedSrcSize,
- ZSTD_buffered_policy_e zbuff)
+static size_t
+ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
+ const ZSTD_CDict* cdict,
+ ZSTD_CCtx_params params,
+ U64 pledgedSrcSize,
+ ZSTD_buffered_policy_e zbuff)
{
- {
- const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
+ { const ZSTD_compressionParameters* const cdict_cParams = &cdict->matchState.cParams;
unsigned const windowLog = params.cParams.windowLog;
assert(windowLog != 0);
/* Resize working context table params for input only, since the dict
@@ -1607,8 +1645,7 @@
assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
}
- {
- const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc
+ { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc
- cdict->matchState.window.base);
const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit;
if (cdictLen == 0) {
@@ -1625,9 +1662,9 @@
cctx->blockState.matchState.window.base + cdictEnd;
ZSTD_window_clear(&cctx->blockState.matchState.window);
}
+ /* loadedDictEnd is expressed within the referential of the active context */
cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit;
- }
- }
+ } }
cctx->dictID = cdict->dictID;
@@ -1681,7 +1718,6 @@
ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState;
dstMatchState->window = srcMatchState->window;
dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
- dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3;
dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
}
@@ -1761,7 +1797,6 @@
ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState;
dstMatchState->window = srcMatchState->window;
dstMatchState->nextToUpdate = srcMatchState->nextToUpdate;
- dstMatchState->nextToUpdate3= srcMatchState->nextToUpdate3;
dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd;
}
dstCCtx->dictID = srcCCtx->dictID;
@@ -1831,16 +1866,15 @@
/*! ZSTD_reduceIndex() :
* rescale all indexes to avoid future overflow (indexes are U32) */
-static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
+static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue)
{
- ZSTD_matchState_t* const ms = &zc->blockState.matchState;
- { U32 const hSize = (U32)1 << zc->appliedParams.cParams.hashLog;
+ { U32 const hSize = (U32)1 << params->cParams.hashLog;
ZSTD_reduceTable(ms->hashTable, hSize, reducerValue);
}
- if (zc->appliedParams.cParams.strategy != ZSTD_fast) {
- U32 const chainSize = (U32)1 << zc->appliedParams.cParams.chainLog;
- if (zc->appliedParams.cParams.strategy == ZSTD_btlazy2)
+ if (params->cParams.strategy != ZSTD_fast) {
+ U32 const chainSize = (U32)1 << params->cParams.chainLog;
+ if (params->cParams.strategy == ZSTD_btlazy2)
ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue);
else
ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue);
@@ -2524,6 +2558,7 @@
op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
else
op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
+ assert(op <= oend);
if (nbSeq==0) {
/* Copy the old tables over as if we repeated them */
memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
@@ -2532,6 +2567,7 @@
/* seqHead : flags for FSE encoding type */
seqHead = op++;
+ assert(op <= oend);
/* convert length/distances into codes */
ZSTD_seqToCodes(seqStorePtr);
@@ -2555,6 +2591,7 @@
if (LLtype == set_compressed)
lastNCount = op;
op += countSize;
+ assert(op <= oend);
} }
/* build CTable for Offsets */
{ unsigned max = MaxOff;
@@ -2577,6 +2614,7 @@
if (Offtype == set_compressed)
lastNCount = op;
op += countSize;
+ assert(op <= oend);
} }
/* build CTable for MatchLengths */
{ unsigned max = MaxML;
@@ -2597,6 +2635,7 @@
if (MLtype == set_compressed)
lastNCount = op;
op += countSize;
+ assert(op <= oend);
} }
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
@@ -2610,6 +2649,7 @@
longOffsets, bmi2);
FORWARD_IF_ERROR(bitstreamSize);
op += bitstreamSize;
+ assert(op <= oend);
/* zstd versions <= 1.3.4 mistakenly report corruption when
* FSE_readNCount() receives a buffer < 4 bytes.
* Fixed by https://github.com/facebook/zstd/pull/1146.
@@ -2721,30 +2761,24 @@
ssPtr->longLengthID = 0;
}
-static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize)
+typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
+
+static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
{
ZSTD_matchState_t* const ms = &zc->blockState.matchState;
- size_t cSize;
- DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
- (unsigned)dstCapacity, (unsigned)ms->window.dictLimit, (unsigned)ms->nextToUpdate);
+ DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize);
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
-
/* Assert that we have correctly flushed the ctx params into the ms's copy */
ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams);
-
if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch);
- cSize = 0;
- goto out; /* don't even attempt compression below a certain srcSize */
+ return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */
}
ZSTD_resetSeqStore(&(zc->seqStore));
/* required for optimal parser to read stats from dictionary */
ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy;
/* tell the optimal parser how we expect to compress literals */
ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode;
-
/* a gap between an attached dict and the current window is not safe,
* they must remain adjacent,
* and when that stops being the case, the dict must be unset */
@@ -2798,6 +2832,21 @@
{ const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize);
} }
+ return ZSTDbss_compress;
+}
+
+static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize)
+{
+ size_t cSize;
+ DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+ (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate);
+
+ { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
+ FORWARD_IF_ERROR(bss);
+ if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; }
+ }
/* encode sequences and literals */
cSize = ZSTD_compressSequences(&zc->seqStore,
@@ -2826,6 +2875,25 @@
}
+static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, void const* ip, void const* iend)
+{
+ if (ZSTD_window_needOverflowCorrection(ms->window, iend)) {
+ U32 const maxDist = (U32)1 << params->cParams.windowLog;
+ U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy);
+ U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip);
+ ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
+ ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+ ZSTD_reduceIndex(ms, params, correction);
+ if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
+ else ms->nextToUpdate -= correction;
+ /* invalidate dictionaries on overflow correction */
+ ms->loadedDictEnd = 0;
+ ms->dictMatchState = NULL;
+ }
+}
+
+
/*! ZSTD_compress_frameChunk() :
* Compress a chunk of data into one or multiple blocks.
* All blocks will be terminated, all input will be consumed.
@@ -2844,7 +2912,7 @@
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
- assert(cctx->appliedParams.cParams.windowLog <= 31);
+ assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX);
DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize);
if (cctx->appliedParams.fParams.checksumFlag && srcSize)
@@ -2859,19 +2927,10 @@
"not enough space to store compressed block");
if (remaining < blockSize) blockSize = remaining;
- if (ZSTD_window_needOverflowCorrection(ms->window, ip + blockSize)) {
- U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
- U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip);
- ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
- ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
- ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
- ZSTD_reduceIndex(cctx, correction);
- if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
- else ms->nextToUpdate -= correction;
- ms->loadedDictEnd = 0;
- ms->dictMatchState = NULL;
- }
- ZSTD_window_enforceMaxDist(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
+ ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, ip, ip + blockSize);
+ ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState);
+
+ /* Ensure hash/chain table insertion resumes no sooner than lowlimit */
if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit;
{ size_t cSize = ZSTD_compressBlock_internal(cctx,
@@ -2899,7 +2958,7 @@
} }
if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
- return op-ostart;
+ return (size_t)(op-ostart);
}
@@ -2991,6 +3050,7 @@
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams,
cctx->pledgedSrcSizePlusOne-1, cctx->dictID);
FORWARD_IF_ERROR(fhSize);
+ assert(fhSize <= dstCapacity);
dstCapacity -= fhSize;
dst = (char*)dst + fhSize;
cctx->stage = ZSTDcs_ongoing;
@@ -3007,18 +3067,7 @@
if (!frame) {
/* overflow check and correction for block mode */
- if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) {
- U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy);
- U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src);
- ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
- ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
- ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
- ZSTD_reduceIndex(cctx, correction);
- if (ms->nextToUpdate < correction) ms->nextToUpdate = 0;
- else ms->nextToUpdate -= correction;
- ms->loadedDictEnd = 0;
- ms->dictMatchState = NULL;
- }
+ ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, src, (BYTE const*)src + srcSize);
}
DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize);
@@ -3074,7 +3123,7 @@
const void* src, size_t srcSize,
ZSTD_dictTableLoadMethod_e dtlm)
{
- const BYTE* const ip = (const BYTE*) src;
+ const BYTE* ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
ZSTD_window_update(&ms->window, src, srcSize);
@@ -3085,32 +3134,42 @@
if (srcSize <= HASH_READ_SIZE) return 0;
- switch(params->cParams.strategy)
- {
- case ZSTD_fast:
- ZSTD_fillHashTable(ms, iend, dtlm);
- break;
- case ZSTD_dfast:
- ZSTD_fillDoubleHashTable(ms, iend, dtlm);
- break;
+ while (iend - ip > HASH_READ_SIZE) {
+ size_t const remaining = iend - ip;
+ size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX);
+ const BYTE* const ichunk = ip + chunk;
- case ZSTD_greedy:
- case ZSTD_lazy:
- case ZSTD_lazy2:
- if (srcSize >= HASH_READ_SIZE)
- ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
- break;
+ ZSTD_overflowCorrectIfNeeded(ms, params, ip, ichunk);
- case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
- case ZSTD_btopt:
- case ZSTD_btultra:
- case ZSTD_btultra2:
- if (srcSize >= HASH_READ_SIZE)
- ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
- break;
+ switch(params->cParams.strategy)
+ {
+ case ZSTD_fast:
+ ZSTD_fillHashTable(ms, ichunk, dtlm);
+ break;
+ case ZSTD_dfast:
+ ZSTD_fillDoubleHashTable(ms, ichunk, dtlm);
+ break;
- default:
- assert(0); /* not possible : not a valid strategy id */
+ case ZSTD_greedy:
+ case ZSTD_lazy:
+ case ZSTD_lazy2:
+ if (chunk >= HASH_READ_SIZE)
+ ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE);
+ break;
+
+ case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
+ case ZSTD_btopt:
+ case ZSTD_btultra:
+ case ZSTD_btultra2:
+ if (chunk >= HASH_READ_SIZE)
+ ZSTD_updateTree(ms, ichunk-HASH_READ_SIZE, ichunk);
+ break;
+
+ default:
+ assert(0); /* not possible : not a valid strategy id */
+ }
+
+ ip = ichunk;
}
ms->nextToUpdate = (U32)(iend - ms->window.base);
@@ -3297,12 +3356,11 @@
FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
ZSTDcrp_continue, zbuff) );
- {
- size_t const dictID = ZSTD_compress_insertDictionary(
+ { size_t const dictID = ZSTD_compress_insertDictionary(
cctx->blockState.prevCBlock, &cctx->blockState.matchState,
¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace);
FORWARD_IF_ERROR(dictID);
- assert(dictID <= (size_t)(U32)-1);
+ assert(dictID <= UINT_MAX);
cctx->dictID = (U32)dictID;
}
return 0;
@@ -3555,10 +3613,10 @@
/* Reset the state to no dictionary */
ZSTD_reset_compressedBlockState(&cdict->cBlockState);
- { void* const end = ZSTD_reset_matchState(
- &cdict->matchState,
- (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32,
- &cParams, ZSTDcrp_continue, /* forCCtx */ 0);
+ { void* const end = ZSTD_reset_matchState(&cdict->matchState,
+ (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32,
+ &cParams,
+ ZSTDcrp_continue, ZSTD_resetTarget_CDict);
assert(end == (char*)cdict->workspace + cdict->workspaceSize);
(void)end;
}
@@ -4068,7 +4126,7 @@
case zcss_flush:
DEBUGLOG(5, "flush stage");
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
- size_t const flushed = ZSTD_limitCopy(op, oend-op,
+ size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op),
zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u",
(unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed);
@@ -4262,7 +4320,7 @@
if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */
/* single thread mode : attempt to calculate remaining to flush more precisely */
{ size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
- size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4;
+ size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4);
size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize;
DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush);
return toFlush;
diff --git a/vendor/github.com/DataDog/zstd/zstd_compress_internal.h b/vendor/github.com/DataDog/zstd/zstd_compress_internal.h
index cc3cbb9..5495899 100644
--- a/vendor/github.com/DataDog/zstd/zstd_compress_internal.h
+++ b/vendor/github.com/DataDog/zstd/zstd_compress_internal.h
@@ -33,13 +33,13 @@
***************************************/
#define kSearchStrength 8
#define HASH_READ_SIZE 8
-#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted".
+#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted".
It could be confused for a real successor at index "1", if sorted as larger than its predecessor.
It's not a big deal though : candidate will just be sorted again.
Additionally, candidate position 1 will be lost.
But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss.
- The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy
- Constant required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */
+ The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy.
+ This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */
/*-*************************************
@@ -128,21 +128,20 @@
BYTE const* base; /* All regular indexes relative to this position */
BYTE const* dictBase; /* extDict indexes relative to this position */
U32 dictLimit; /* below that point, need extDict */
- U32 lowLimit; /* below that point, no more data */
+ U32 lowLimit; /* below that point, no more valid data */
} ZSTD_window_t;
typedef struct ZSTD_matchState_t ZSTD_matchState_t;
struct ZSTD_matchState_t {
ZSTD_window_t window; /* State for window round buffer management */
- U32 loadedDictEnd; /* index of end of dictionary */
+ U32 loadedDictEnd; /* index of end of dictionary, within context's referential. When dict referential is copied into active context (i.e. not attached), effectively same value as dictSize, since referential starts from zero */
U32 nextToUpdate; /* index from which to continue table update */
- U32 nextToUpdate3; /* index from which to continue table update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32* hashTable;
U32* hashTable3;
U32* chainTable;
optState_t opt; /* optimal parser state */
- const ZSTD_matchState_t * dictMatchState;
+ const ZSTD_matchState_t* dictMatchState;
ZSTD_compressionParameters cParams;
};
@@ -195,6 +194,9 @@
int compressionLevel;
int forceWindow; /* force back-references to respect limit of
* 1<<wLog, even for dictionary */
+ size_t targetCBlockSize; /* Tries to fit compressed block size to be around targetCBlockSize.
+ * No target when targetCBlockSize == 0.
+ * There is no guarantee on compressed block size */
ZSTD_dictAttachPref_e attachDictPref;
ZSTD_literalCompressionMode_e literalCompressionMode;
@@ -324,7 +326,7 @@
/* copy Literals */
assert(seqStorePtr->maxNbLit <= 128 KB);
assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit);
- ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
+ ZSTD_wildcopy(seqStorePtr->lit, literals, litLength, ZSTD_no_overlap);
seqStorePtr->lit += litLength;
/* literal Length */
@@ -564,6 +566,9 @@
/*-*************************************
* Round buffer management
***************************************/
+#if (ZSTD_WINDOWLOG_MAX_64 > 31)
+# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX"
+#endif
/* Max current allowed */
#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX))
/* Maximum chunk size before overflow correction needs to be called again */
@@ -675,31 +680,49 @@
* Updates lowLimit so that:
* (srcEnd - base) - lowLimit == maxDist + loadedDictEnd
*
- * This allows a simple check that index >= lowLimit to see if index is valid.
- * This must be called before a block compression call, with srcEnd as the block
- * source end.
+ * It ensures index is valid as long as index >= lowLimit.
+ * This must be called before a block compression call.
*
- * If loadedDictEndPtr is not NULL, we set it to zero once we update lowLimit.
- * This is because dictionaries are allowed to be referenced as long as the last
- * byte of the dictionary is in the window, but once they are out of range,
- * they cannot be referenced. If loadedDictEndPtr is NULL, we use
- * loadedDictEnd == 0.
+ * loadedDictEnd is only defined if a dictionary is in use for current compression.
+ * As the name implies, loadedDictEnd represents the index at end of dictionary.
+ * The value lies within context's referential, it can be directly compared to blockEndIdx.
*
- * In normal dict mode, the dict is between lowLimit and dictLimit. In
- * dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary
- * is below them. forceWindow and dictMatchState are therefore incompatible.
+ * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0.
+ * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit.
+ * This is because dictionaries are allowed to be referenced fully
+ * as long as the last byte of the dictionary is in the window.
+ * Once input has progressed beyond window size, dictionary cannot be referenced anymore.
+ *
+ * In normal dict mode, the dictionary lies between lowLimit and dictLimit.
+ * In dictMatchState mode, lowLimit and dictLimit are the same,
+ * and the dictionary is below them.
+ * forceWindow and dictMatchState are therefore incompatible.
*/
MEM_STATIC void
ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
- void const* srcEnd,
- U32 maxDist,
- U32* loadedDictEndPtr,
+ const void* blockEnd,
+ U32 maxDist,
+ U32* loadedDictEndPtr,
const ZSTD_matchState_t** dictMatchStatePtr)
{
- U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base);
- U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
- DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u",
- (unsigned)blockEndIdx, (unsigned)maxDist);
+ U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
+ U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
+ DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
+ (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
+
+ /* - When there is no dictionary : loadedDictEnd == 0.
+ In which case, the test (blockEndIdx > maxDist) is merely to avoid
+ overflowing next operation `newLowLimit = blockEndIdx - maxDist`.
+ - When there is a standard dictionary :
+ Index referential is copied from the dictionary,
+ which means it starts from 0.
+ In which case, loadedDictEnd == dictSize,
+ and it makes sense to compare `blockEndIdx > maxDist + dictSize`
+ since `blockEndIdx` also starts from zero.
+ - When there is an attached dictionary :
+ loadedDictEnd is expressed within the referential of the context,
+ so it can be directly compared against blockEndIdx.
+ */
if (blockEndIdx > maxDist + loadedDictEnd) {
U32 const newLowLimit = blockEndIdx - maxDist;
if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
@@ -708,10 +731,31 @@
(unsigned)window->dictLimit, (unsigned)window->lowLimit);
window->dictLimit = window->lowLimit;
}
- if (loadedDictEndPtr)
- *loadedDictEndPtr = 0;
- if (dictMatchStatePtr)
- *dictMatchStatePtr = NULL;
+ /* On reaching window size, dictionaries are invalidated */
+ if (loadedDictEndPtr) *loadedDictEndPtr = 0;
+ if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
+ }
+}
+
+/* Similar to ZSTD_window_enforceMaxDist(),
+ * but only invalidates dictionary
+ * when input progresses beyond window size. */
+MEM_STATIC void
+ZSTD_checkDictValidity(ZSTD_window_t* window,
+ const void* blockEnd,
+ U32 maxDist,
+ U32* loadedDictEndPtr,
+ const ZSTD_matchState_t** dictMatchStatePtr)
+{
+ U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base);
+ U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
+ DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u",
+ (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
+
+ if (loadedDictEnd && (blockEndIdx > maxDist + loadedDictEnd)) {
+ /* On reaching window size, dictionaries are invalidated */
+ if (loadedDictEndPtr) *loadedDictEndPtr = 0;
+ if (dictMatchStatePtr) *dictMatchStatePtr = NULL;
}
}
diff --git a/vendor/github.com/DataDog/zstd/zstd_decompress.c b/vendor/github.com/DataDog/zstd/zstd_decompress.c
index 675596f..e42872a 100644
--- a/vendor/github.com/DataDog/zstd/zstd_decompress.c
+++ b/vendor/github.com/DataDog/zstd/zstd_decompress.c
@@ -360,8 +360,11 @@
sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
frameParameter_unsupported);
-
- return skippableHeaderSize + sizeU32;
+ {
+ size_t const skippableSize = skippableHeaderSize + sizeU32;
+ RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong);
+ return skippableSize;
+ }
}
/** ZSTD_findDecompressedSize() :
@@ -378,11 +381,10 @@
if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t const skippableSize = readSkippableFrameSize(src, srcSize);
- if (ZSTD_isError(skippableSize))
- return skippableSize;
- if (srcSize < skippableSize) {
+ if (ZSTD_isError(skippableSize)) {
return ZSTD_CONTENTSIZE_ERROR;
}
+ assert(skippableSize <= srcSize);
src = (const BYTE *)src + skippableSize;
srcSize -= skippableSize;
@@ -467,6 +469,8 @@
if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
&& (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
+ assert(ZSTD_isError(frameSizeInfo.compressedSize) ||
+ frameSizeInfo.compressedSize <= srcSize);
return frameSizeInfo;
} else {
const BYTE* ip = (const BYTE*)src;
@@ -529,7 +533,6 @@
return frameSizeInfo.compressedSize;
}
-
/** ZSTD_decompressBound() :
* compatible with legacy mode
* `src` must point to the start of a ZSTD frame or a skippeable frame
@@ -546,6 +549,7 @@
unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
return ZSTD_CONTENTSIZE_ERROR;
+ assert(srcSize >= compressedSize);
src = (const BYTE*)src + compressedSize;
srcSize -= compressedSize;
bound += decompressedBound;
@@ -738,9 +742,8 @@
(unsigned)magicNumber, ZSTD_MAGICNUMBER);
if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t const skippableSize = readSkippableFrameSize(src, srcSize);
- if (ZSTD_isError(skippableSize))
- return skippableSize;
- RETURN_ERROR_IF(srcSize < skippableSize, srcSize_wrong);
+ FORWARD_IF_ERROR(skippableSize);
+ assert(skippableSize <= srcSize);
src = (const BYTE *)src + skippableSize;
srcSize -= skippableSize;
diff --git a/vendor/github.com/DataDog/zstd/zstd_decompress_block.c b/vendor/github.com/DataDog/zstd/zstd_decompress_block.c
index a2a7eed..24f4859 100644
--- a/vendor/github.com/DataDog/zstd/zstd_decompress_block.c
+++ b/vendor/github.com/DataDog/zstd/zstd_decompress_block.c
@@ -505,7 +505,7 @@
*nbSeqPtr = nbSeq;
/* FSE table descriptors */
- RETURN_ERROR_IF(ip+4 > iend, srcSize_wrong); /* minimum possible size */
+ RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong); /* minimum possible size: 1 byte for symbol encoding types */
{ symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6);
symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3);
symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3);
@@ -637,9 +637,10 @@
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
/* copy Literals */
- ZSTD_copy8(op, *litPtr);
if (sequence.litLength > 8)
- ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+ ZSTD_wildcopy_16min(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+ else
+ ZSTD_copy8(op, *litPtr);
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
@@ -686,13 +687,13 @@
if (oMatchEnd > oend-(16-MINMATCH)) {
if (op < oend_w) {
- ZSTD_wildcopy(op, match, oend_w - op);
+ ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst);
match += oend_w - op;
op = oend_w;
}
while (op < oMatchEnd) *op++ = *match++;
} else {
- ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@@ -717,9 +718,11 @@
if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd);
/* copy Literals */
- ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */
if (sequence.litLength > 8)
- ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+ ZSTD_wildcopy_16min(op, *litPtr, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+ else
+ ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */
+
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
@@ -766,13 +769,13 @@
if (oMatchEnd > oend-(16-MINMATCH)) {
if (op < oend_w) {
- ZSTD_wildcopy(op, match, oend_w - op);
+ ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst);
match += oend_w - op;
op = oend_w;
}
while (op < oMatchEnd) *op++ = *match++;
} else {
- ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
+ ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@@ -889,6 +892,7 @@
}
FORCE_INLINE_TEMPLATE size_t
+DONT_VECTORIZE
ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
@@ -918,6 +922,11 @@
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
+ ZSTD_STATIC_ASSERT(
+ BIT_DStream_unfinished < BIT_DStream_completed &&
+ BIT_DStream_endOfBuffer < BIT_DStream_completed &&
+ BIT_DStream_completed < BIT_DStream_overflow);
+
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
nbSeq--;
{ seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
@@ -930,6 +939,7 @@
/* check if reached exact end */
DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq);
RETURN_ERROR_IF(nbSeq, corruption_detected);
+ RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected);
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
@@ -1131,6 +1141,7 @@
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
static TARGET_ATTRIBUTE("bmi2") size_t
+DONT_VECTORIZE
ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
diff --git a/vendor/github.com/DataDog/zstd/zstd_double_fast.c b/vendor/github.com/DataDog/zstd/zstd_double_fast.c
index 47faf6d..5957255 100644
--- a/vendor/github.com/DataDog/zstd/zstd_double_fast.c
+++ b/vendor/github.com/DataDog/zstd/zstd_double_fast.c
@@ -43,8 +43,7 @@
/* Only load extra positions for ZSTD_dtlm_full */
if (dtlm == ZSTD_dtlm_fast)
break;
- }
- }
+ } }
}
@@ -63,7 +62,10 @@
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
- const U32 prefixLowestIndex = ms->window.dictLimit;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 lowestValid = ms->window.dictLimit;
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid;
const BYTE* const prefixLowest = base + prefixLowestIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE;
@@ -95,8 +97,15 @@
dictCParams->chainLog : hBitsS;
const U32 dictAndPrefixLength = (U32)(ip - prefixLowest + dictEnd - dictStart);
+ DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_generic");
+
assert(dictMode == ZSTD_noDict || dictMode == ZSTD_dictMatchState);
+ /* if a dictionary is attached, it must be within window range */
+ if (dictMode == ZSTD_dictMatchState) {
+ assert(lowestValid + maxDistance >= endIndex);
+ }
+
/* init */
ip += (dictAndPrefixLength == 0);
if (dictMode == ZSTD_noDict) {
@@ -138,7 +147,7 @@
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
ip++;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
goto _match_stored;
}
@@ -147,7 +156,7 @@
&& ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
ip++;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
goto _match_stored;
}
@@ -170,8 +179,7 @@
offset = (U32)(current - dictMatchIndexL - dictIndexDelta);
while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */
goto _match_found;
- }
- }
+ } }
if (matchIndexS > prefixLowestIndex) {
/* check prefix short match */
@@ -186,16 +194,14 @@
if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) {
goto _search_next_long;
- }
- }
+ } }
ip += ((ip-anchor) >> kSearchStrength) + 1;
continue;
_search_next_long:
- {
- size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
+ { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
U32 const matchIndexL3 = hashLong[hl3];
const BYTE* matchL3 = base + matchIndexL3;
@@ -221,9 +227,7 @@
offset = (U32)(current + 1 - dictMatchIndexL3 - dictIndexDelta);
while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */
goto _match_found;
- }
- }
- }
+ } } }
/* if no long +1 match, explore the short match we found */
if (dictMode == ZSTD_dictMatchState && matchIndexS < prefixLowestIndex) {
@@ -242,7 +246,7 @@
offset_2 = offset_1;
offset_1 = offset;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
_match_stored:
/* match found */
@@ -250,11 +254,14 @@
anchor = ip;
if (ip <= ilimit) {
- /* Fill Table */
- hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] =
- hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */
- hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] =
- hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base);
+ /* Complementary insertion */
+ /* done after iLimit test, as candidates could be > iend-8 */
+ { U32 const indexToInsert = current+2;
+ hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
+ hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
+ hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
+ }
/* check immediate repcode */
if (dictMode == ZSTD_dictMatchState) {
@@ -278,8 +285,7 @@
continue;
}
break;
- }
- }
+ } }
if (dictMode == ZSTD_noDict) {
while ( (ip <= ilimit)
@@ -294,14 +300,15 @@
ip += rLength;
anchor = ip;
continue; /* faster when present ... (?) */
- } } } }
+ } } }
+ } /* while (ip < ilimit) */
/* save reps for next block */
rep[0] = offset_1 ? offset_1 : offsetSaved;
rep[1] = offset_2 ? offset_2 : offsetSaved;
/* Return the last literals size */
- return iend - anchor;
+ return (size_t)(iend - anchor);
}
@@ -360,10 +367,15 @@
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
- const U32 prefixStartIndex = ms->window.dictLimit;
const BYTE* const base = ms->window.base;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 lowestValid = ms->window.lowLimit;
+ const U32 lowLimit = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid;
+ const U32 dictStartIndex = lowLimit;
+ const U32 dictLimit = ms->window.dictLimit;
+ const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit;
const BYTE* const prefixStart = base + prefixStartIndex;
- const U32 dictStartIndex = ms->window.lowLimit;
const BYTE* const dictBase = ms->window.dictBase;
const BYTE* const dictStart = dictBase + dictStartIndex;
const BYTE* const dictEnd = dictBase + prefixStartIndex;
@@ -371,6 +383,10 @@
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize);
+ /* if extDict is invalidated due to maxDistance, switch to "regular" variant */
+ if (prefixStartIndex == dictStartIndex)
+ return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict);
+
/* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
@@ -396,7 +412,7 @@
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
ip++;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
} else {
if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
@@ -407,7 +423,7 @@
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
offset_2 = offset_1;
offset_1 = offset;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
@@ -432,23 +448,27 @@
}
offset_2 = offset_1;
offset_1 = offset;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} else {
ip += ((ip-anchor) >> kSearchStrength) + 1;
continue;
} }
- /* found a match : store it */
+ /* move to next sequence start */
ip += mLength;
anchor = ip;
if (ip <= ilimit) {
- /* Fill Table */
- hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2;
- hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2;
- hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base);
- hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ /* Complementary insertion */
+ /* done after iLimit test, as candidates could be > iend-8 */
+ { U32 const indexToInsert = current+2;
+ hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
+ hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
+ hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
+ hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
+ }
+
/* check immediate repcode */
while (ip <= ilimit) {
U32 const current2 = (U32)(ip-base);
@@ -475,7 +495,7 @@
rep[1] = offset_2;
/* Return the last literals size */
- return iend - anchor;
+ return (size_t)(iend - anchor);
}
diff --git a/vendor/github.com/DataDog/zstd/zstd_fast.c b/vendor/github.com/DataDog/zstd/zstd_fast.c
index ed997b4..a05b8a4 100644
--- a/vendor/github.com/DataDog/zstd/zstd_fast.c
+++ b/vendor/github.com/DataDog/zstd/zstd_fast.c
@@ -13,7 +13,8 @@
void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
- void const* end, ZSTD_dictTableLoadMethod_e dtlm)
+ const void* const end,
+ ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
@@ -41,6 +42,7 @@
} } } }
}
+
FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_fast_generic(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -58,7 +60,10 @@
const BYTE* ip0 = istart;
const BYTE* ip1;
const BYTE* anchor = istart;
- const U32 prefixStartIndex = ms->window.dictLimit;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 validStartIndex = ms->window.dictLimit;
+ const U32 prefixStartIndex = (endIndex - validStartIndex > maxDistance) ? endIndex - maxDistance : validStartIndex;
const BYTE* const prefixStart = base + prefixStartIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE;
@@ -165,7 +170,7 @@
rep[1] = offset_2 ? offset_2 : offsetSaved;
/* Return the last literals size */
- return iend - anchor;
+ return (size_t)(iend - anchor);
}
@@ -222,8 +227,15 @@
const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart);
const U32 dictHLog = dictCParams->hashLog;
- /* otherwise, we would get index underflow when translating a dict index
- * into a local index */
+ /* if a dictionary is still attached, it necessarily means that
+ * it is within window size. So we just check it. */
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 endIndex = (U32)((size_t)(ip - base) + srcSize);
+ assert(endIndex - prefixStartIndex <= maxDistance);
+ (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */
+
+ /* ensure there will be no no underflow
+ * when translating a dict index into a local index */
assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
/* init */
@@ -251,7 +263,7 @@
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
ip++;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
} else if ( (matchIndex <= prefixStartIndex) ) {
size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
U32 const dictMatchIndex = dictHashTable[dictHash];
@@ -271,7 +283,7 @@
} /* catch up */
offset_2 = offset_1;
offset_1 = offset;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
}
} else if (MEM_read32(match) != MEM_read32(ip)) {
/* it's not a match, and we're not going to check the dictionary */
@@ -286,7 +298,7 @@
&& (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
offset_2 = offset_1;
offset_1 = offset;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
}
/* match found */
@@ -327,7 +339,7 @@
rep[1] = offset_2 ? offset_2 : offsetSaved;
/* Return the last literals size */
- return iend - anchor;
+ return (size_t)(iend - anchor);
}
size_t ZSTD_compressBlock_fast_dictMatchState(
@@ -366,15 +378,24 @@
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
- const U32 dictStartIndex = ms->window.lowLimit;
+ const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 validLow = ms->window.lowLimit;
+ const U32 lowLimit = (endIndex - validLow > maxDistance) ? endIndex - maxDistance : validLow;
+ const U32 dictStartIndex = lowLimit;
const BYTE* const dictStart = dictBase + dictStartIndex;
- const U32 prefixStartIndex = ms->window.dictLimit;
+ const U32 dictLimit = ms->window.dictLimit;
+ const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit;
const BYTE* const prefixStart = base + prefixStartIndex;
const BYTE* const dictEnd = dictBase + prefixStartIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
U32 offset_1=rep[0], offset_2=rep[1];
+ /* switch to "regular" variant if extDict is invalidated due to maxDistance */
+ if (prefixStartIndex == dictStartIndex)
+ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls);
+
/* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
const size_t h = ZSTD_hashPtr(ip, hlog, mls);
@@ -394,7 +415,7 @@
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
ip++;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH);
} else {
if ( (matchIndex < dictStartIndex) ||
(MEM_read32(match) != MEM_read32(ip)) ) {
@@ -410,7 +431,7 @@
offset = current - matchIndex;
offset_2 = offset_1;
offset_1 = offset;
- ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+ ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
} }
/* found a match : store it */
@@ -445,7 +466,7 @@
rep[1] = offset_2;
/* Return the last literals size */
- return iend - anchor;
+ return (size_t)(iend - anchor);
}
diff --git a/vendor/github.com/DataDog/zstd/zstd_internal.h b/vendor/github.com/DataDog/zstd/zstd_internal.h
index 31f756a..81b16ea 100644
--- a/vendor/github.com/DataDog/zstd/zstd_internal.h
+++ b/vendor/github.com/DataDog/zstd/zstd_internal.h
@@ -34,7 +34,6 @@
#endif
#include "xxhash.h" /* XXH_reset, update, digest */
-
#if defined (__cplusplus)
extern "C" {
#endif
@@ -193,19 +192,72 @@
* Shared functions to include for inlining
*********************************************/
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
+
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
+static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); }
+#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
+
+#define WILDCOPY_OVERLENGTH 8
+#define VECLEN 16
+
+typedef enum {
+ ZSTD_no_overlap,
+ ZSTD_overlap_src_before_dst,
+ /* ZSTD_overlap_dst_before_src, */
+} ZSTD_overlap_e;
/*! ZSTD_wildcopy() :
* custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */
-#define WILDCOPY_OVERLENGTH 8
-MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
+MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
+void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype)
{
+ ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
- do
- COPY8(op, ip)
- while (op < oend);
+
+ assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8));
+ if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) {
+ do
+ COPY8(op, ip)
+ while (op < oend);
+ }
+ else {
+ if ((length & 8) == 0)
+ COPY8(op, ip);
+ do {
+ COPY16(op, ip);
+ }
+ while (op < oend);
+ }
+}
+
+/*! ZSTD_wildcopy_16min() :
+ * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */
+MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE
+void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype)
+{
+ ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src;
+ const BYTE* ip = (const BYTE*)src;
+ BYTE* op = (BYTE*)dst;
+ BYTE* const oend = op + length;
+
+ assert(length >= 8);
+ assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8));
+
+ if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) {
+ do
+ COPY8(op, ip)
+ while (op < oend);
+ }
+ else {
+ if ((length & 8) == 0)
+ COPY8(op, ip);
+ do {
+ COPY16(op, ip);
+ }
+ while (op < oend);
+ }
}
MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
diff --git a/vendor/github.com/DataDog/zstd/zstd_lazy.c b/vendor/github.com/DataDog/zstd/zstd_lazy.c
index 53f998a..94d906c 100644
--- a/vendor/github.com/DataDog/zstd/zstd_lazy.c
+++ b/vendor/github.com/DataDog/zstd/zstd_lazy.c
@@ -83,7 +83,10 @@
U32* largerPtr = smallerPtr + 1;
U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */
U32 dummy32; /* to be nullified at the end */
- U32 const windowLow = ms->window.lowLimit;
+ U32 const windowValid = ms->window.lowLimit;
+ U32 const maxDistance = 1U << cParams->windowLog;
+ U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid;
+
DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)",
current, dictLimit, windowLow);
@@ -239,7 +242,9 @@
const BYTE* const base = ms->window.base;
U32 const current = (U32)(ip-base);
- U32 const windowLow = ms->window.lowLimit;
+ U32 const maxDistance = 1U << cParams->windowLog;
+ U32 const windowValid = ms->window.lowLimit;
+ U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid;
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
@@ -490,8 +495,10 @@
const U32 dictLimit = ms->window.dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
- const U32 lowLimit = ms->window.lowLimit;
const U32 current = (U32)(ip-base);
+ const U32 maxDistance = 1U << cParams->windowLog;
+ const U32 lowValid = ms->window.lowLimit;
+ const U32 lowLimit = (current - lowValid > maxDistance) ? current - maxDistance : lowValid;
const U32 minChain = current > chainSize ? current - chainSize : 0;
U32 nbAttempts = 1U << cParams->searchLog;
size_t ml=4-1;
@@ -653,7 +660,6 @@
/* init */
ip += (dictAndPrefixLength == 0);
- ms->nextToUpdate3 = ms->nextToUpdate;
if (dictMode == ZSTD_noDict) {
U32 const maxRep = (U32)(ip - prefixLowest);
if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0;
@@ -933,7 +939,6 @@
U32 offset_1 = rep[0], offset_2 = rep[1];
/* init */
- ms->nextToUpdate3 = ms->nextToUpdate;
ip += (ip == prefixStart);
/* Match Loop */
diff --git a/vendor/github.com/DataDog/zstd/zstd_ldm.c b/vendor/github.com/DataDog/zstd/zstd_ldm.c
index 784d20f..3dcf86e 100644
--- a/vendor/github.com/DataDog/zstd/zstd_ldm.c
+++ b/vendor/github.com/DataDog/zstd/zstd_ldm.c
@@ -447,7 +447,7 @@
if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) {
U32 const ldmHSize = 1U << params->hashLog;
U32 const correction = ZSTD_window_correctOverflow(
- &ldmState->window, /* cycleLog */ 0, maxDist, src);
+ &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart);
ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction);
}
/* 2. We enforce the maximum offset allowed.
diff --git a/vendor/github.com/DataDog/zstd/zstd_legacy.h b/vendor/github.com/DataDog/zstd/zstd_legacy.h
index e5b383e..0dbd3c7 100644
--- a/vendor/github.com/DataDog/zstd/zstd_legacy.h
+++ b/vendor/github.com/DataDog/zstd/zstd_legacy.h
@@ -238,6 +238,10 @@
frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
break;
}
+ if (!ZSTD_isError(frameSizeInfo.compressedSize) && frameSizeInfo.compressedSize > srcSize) {
+ frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
+ frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR;
+ }
return frameSizeInfo;
}
diff --git a/vendor/github.com/DataDog/zstd/zstd_opt.c b/vendor/github.com/DataDog/zstd/zstd_opt.c
index efb69d3..e32e542 100644
--- a/vendor/github.com/DataDog/zstd/zstd_opt.c
+++ b/vendor/github.com/DataDog/zstd/zstd_opt.c
@@ -255,13 +255,13 @@
* to provide a cost which is directly comparable to a match ending at same position */
static int ZSTD_litLengthContribution(U32 const litLength, const optState_t* const optPtr, int optLevel)
{
- if (optPtr->priceType >= zop_predef) return WEIGHT(litLength, optLevel);
+ if (optPtr->priceType >= zop_predef) return (int)WEIGHT(litLength, optLevel);
/* dynamic statistics */
{ U32 const llCode = ZSTD_LLcode(litLength);
- int const contribution = (LL_bits[llCode] * BITCOST_MULTIPLIER)
- + WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */
- - WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
+ int const contribution = (int)(LL_bits[llCode] * BITCOST_MULTIPLIER)
+ + (int)WEIGHT(optPtr->litLengthFreq[0], optLevel) /* note: log2litLengthSum cancel out */
+ - (int)WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
#if 1
return contribution;
#else
@@ -278,7 +278,7 @@
const optState_t* const optPtr,
int optLevel)
{
- int const contribution = ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel)
+ int const contribution = (int)ZSTD_rawLiteralsCost(literals, litLength, optPtr, optLevel)
+ ZSTD_litLengthContribution(litLength, optPtr, optLevel);
return contribution;
}
@@ -372,13 +372,15 @@
/* Update hashTable3 up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
-static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms, const BYTE* const ip)
+static U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
+ const BYTE* const ip)
{
U32* const hashTable3 = ms->hashTable3;
U32 const hashLog3 = ms->hashLog3;
const BYTE* const base = ms->window.base;
- U32 idx = ms->nextToUpdate3;
- U32 const target = ms->nextToUpdate3 = (U32)(ip - base);
+ U32 idx = *nextToUpdate3;
+ U32 const target = (U32)(ip - base);
size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3);
assert(hashLog3 > 0);
@@ -387,6 +389,7 @@
idx++;
}
+ *nextToUpdate3 = target;
return hashTable3[hash3];
}
@@ -503,9 +506,11 @@
} }
*smallerPtr = *largerPtr = 0;
- if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */
- assert(matchEndIdx > current + 8);
- return matchEndIdx - (current + 8);
+ { U32 positions = 0;
+ if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */
+ assert(matchEndIdx > current + 8);
+ return MAX(positions, matchEndIdx - (current + 8));
+ }
}
FORCE_INLINE_TEMPLATE
@@ -520,8 +525,13 @@
DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
idx, target, dictMode);
- while(idx < target)
- idx += ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict);
+ while(idx < target) {
+ U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, mls, dictMode == ZSTD_extDict);
+ assert(idx < (U32)(idx + forward));
+ idx += forward;
+ }
+ assert((size_t)(ip - base) <= (size_t)(U32)(-1));
+ assert((size_t)(iend - base) <= (size_t)(U32)(-1));
ms->nextToUpdate = target;
}
@@ -531,16 +541,18 @@
FORCE_INLINE_TEMPLATE
U32 ZSTD_insertBtAndGetAllMatches (
+ ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */
ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
- U32 rep[ZSTD_REP_NUM],
+ const U32 rep[ZSTD_REP_NUM],
U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
- ZSTD_match_t* matches,
const U32 lengthToBeat,
U32 const mls /* template */)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
+ U32 const maxDistance = 1U << cParams->windowLog;
const BYTE* const base = ms->window.base;
U32 const current = (U32)(ip-base);
U32 const hashLog = cParams->hashLog;
@@ -556,8 +568,9 @@
U32 const dictLimit = ms->window.dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
- U32 const btLow = btMask >= current ? 0 : current - btMask;
- U32 const windowLow = ms->window.lowLimit;
+ U32 const btLow = (btMask >= current) ? 0 : current - btMask;
+ U32 const windowValid = ms->window.lowLimit;
+ U32 const windowLow = ((current - windowValid) > maxDistance) ? current - maxDistance : windowValid;
U32 const matchLow = windowLow ? windowLow : 1;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = bt + 2*(current&btMask) + 1;
@@ -627,7 +640,7 @@
/* HC3 match finder */
if ((mls == 3) /*static*/ && (bestLength < mls)) {
- U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, ip);
+ U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip);
if ((matchIndex3 >= matchLow)
& (current - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) {
size_t mlen;
@@ -653,9 +666,7 @@
(ip+mlen == iLimit) ) { /* best possible length */
ms->nextToUpdate = current+1; /* skip insertion */
return 1;
- }
- }
- }
+ } } }
/* no dictMatchState lookup: dicts don't have a populated HC3 table */
}
@@ -760,10 +771,13 @@
FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
+ ZSTD_match_t* matches, /* store result (match found, increasing size) in this table */
ZSTD_matchState_t* ms,
+ U32* nextToUpdate3,
const BYTE* ip, const BYTE* const iHighLimit, const ZSTD_dictMode_e dictMode,
- U32 rep[ZSTD_REP_NUM], U32 const ll0,
- ZSTD_match_t* matches, U32 const lengthToBeat)
+ const U32 rep[ZSTD_REP_NUM],
+ U32 const ll0,
+ U32 const lengthToBeat)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const matchLengthSearch = cParams->minMatch;
@@ -772,12 +786,12 @@
ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode);
switch(matchLengthSearch)
{
- case 3 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 3);
+ case 3 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 3);
default :
- case 4 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 4);
- case 5 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 5);
+ case 4 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 4);
+ case 5 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 5);
case 7 :
- case 6 : return ZSTD_insertBtAndGetAllMatches(ms, ip, iHighLimit, dictMode, rep, ll0, matches, lengthToBeat, 6);
+ case 6 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 6);
}
}
@@ -853,6 +867,7 @@
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
+ U32 nextToUpdate3 = ms->nextToUpdate;
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
ZSTD_match_t* const matches = optStatePtr->matchTable;
@@ -862,7 +877,6 @@
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
(U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
assert(optLevel <= 2);
- ms->nextToUpdate3 = ms->nextToUpdate;
ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
ip += (ip==prefixStart);
@@ -873,7 +887,7 @@
/* find first match */
{ U32 const litlen = (U32)(ip - anchor);
U32 const ll0 = !litlen;
- U32 const nbMatches = ZSTD_BtGetAllMatches(ms, ip, iend, dictMode, rep, ll0, matches, minMatch);
+ U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
if (!nbMatches) { ip++; continue; }
/* initialize opt[0] */
@@ -970,7 +984,7 @@
U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
U32 const previousPrice = opt[cur].price;
U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
- U32 const nbMatches = ZSTD_BtGetAllMatches(ms, inr, iend, dictMode, opt[cur].rep, ll0, matches, minMatch);
+ U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
U32 matchNb;
if (!nbMatches) {
DEBUGLOG(7, "rPos:%u : no match found", cur);
@@ -1094,7 +1108,7 @@
} /* while (ip < ilimit) */
/* Return the last literals size */
- return iend - anchor;
+ return (size_t)(iend - anchor);
}
@@ -1158,7 +1172,6 @@
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
- ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
diff --git a/vendor/github.com/DataDog/zstd/zstd_v01.c b/vendor/github.com/DataDog/zstd/zstd_v01.c
index cad2b99..ae8cba2 100644
--- a/vendor/github.com/DataDog/zstd/zstd_v01.c
+++ b/vendor/github.com/DataDog/zstd/zstd_v01.c
@@ -1073,99 +1073,102 @@
const void* cSrc, size_t cSrcSize,
const U16* DTable)
{
- BYTE* const ostart = (BYTE*) dst;
- BYTE* op = ostart;
- BYTE* const omax = op + maxDstSize;
- BYTE* const olimit = omax-15;
-
- const void* ptr = DTable;
- const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
- const U32 dtLog = DTable[0];
- size_t errorCode;
- U32 reloadStatus;
-
- /* Init */
-
- const U16* jumpTable = (const U16*)cSrc;
- const size_t length1 = FSE_readLE16(jumpTable);
- const size_t length2 = FSE_readLE16(jumpTable+1);
- const size_t length3 = FSE_readLE16(jumpTable+2);
- const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
- const char* const start1 = (const char*)(cSrc) + 6;
- const char* const start2 = start1 + length1;
- const char* const start3 = start2 + length2;
- const char* const start4 = start3 + length3;
- FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
-
- if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
-
- errorCode = FSE_initDStream(&bitD1, start1, length1);
- if (FSE_isError(errorCode)) return errorCode;
- errorCode = FSE_initDStream(&bitD2, start2, length2);
- if (FSE_isError(errorCode)) return errorCode;
- errorCode = FSE_initDStream(&bitD3, start3, length3);
- if (FSE_isError(errorCode)) return errorCode;
- errorCode = FSE_initDStream(&bitD4, start4, length4);
- if (FSE_isError(errorCode)) return errorCode;
-
- reloadStatus=FSE_reloadDStream(&bitD2);
-
- /* 16 symbols per loop */
- for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
- op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
+ if (cSrcSize < 6) return (size_t)-FSE_ERROR_srcSize_wrong;
{
-#define HUF_DECODE_SYMBOL_0(n, Dstream) \
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* op = ostart;
+ BYTE* const omax = op + maxDstSize;
+ BYTE* const olimit = omax-15;
-#define HUF_DECODE_SYMBOL_1(n, Dstream) \
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
- if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
+ const void* ptr = DTable;
+ const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1;
+ const U32 dtLog = DTable[0];
+ size_t errorCode;
+ U32 reloadStatus;
-#define HUF_DECODE_SYMBOL_2(n, Dstream) \
- op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
- if (FSE_32bits()) FSE_reloadDStream(&Dstream)
+ /* Init */
- HUF_DECODE_SYMBOL_1( 0, bitD1);
- HUF_DECODE_SYMBOL_1( 1, bitD2);
- HUF_DECODE_SYMBOL_1( 2, bitD3);
- HUF_DECODE_SYMBOL_1( 3, bitD4);
- HUF_DECODE_SYMBOL_2( 4, bitD1);
- HUF_DECODE_SYMBOL_2( 5, bitD2);
- HUF_DECODE_SYMBOL_2( 6, bitD3);
- HUF_DECODE_SYMBOL_2( 7, bitD4);
- HUF_DECODE_SYMBOL_1( 8, bitD1);
- HUF_DECODE_SYMBOL_1( 9, bitD2);
- HUF_DECODE_SYMBOL_1(10, bitD3);
- HUF_DECODE_SYMBOL_1(11, bitD4);
- HUF_DECODE_SYMBOL_0(12, bitD1);
- HUF_DECODE_SYMBOL_0(13, bitD2);
- HUF_DECODE_SYMBOL_0(14, bitD3);
- HUF_DECODE_SYMBOL_0(15, bitD4);
- }
+ const U16* jumpTable = (const U16*)cSrc;
+ const size_t length1 = FSE_readLE16(jumpTable);
+ const size_t length2 = FSE_readLE16(jumpTable+1);
+ const size_t length3 = FSE_readLE16(jumpTable+2);
+ const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !!
+ const char* const start1 = (const char*)(cSrc) + 6;
+ const char* const start2 = start1 + length1;
+ const char* const start3 = start2 + length2;
+ const char* const start4 = start3 + length3;
+ FSE_DStream_t bitD1, bitD2, bitD3, bitD4;
- if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
- return (size_t)-FSE_ERROR_corruptionDetected;
+ if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
- /* tail */
- {
- // bitTail = bitD1; // *much* slower : -20% !??!
- FSE_DStream_t bitTail;
- bitTail.ptr = bitD1.ptr;
- bitTail.bitsConsumed = bitD1.bitsConsumed;
- bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
- bitTail.start = start1;
- for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
+ errorCode = FSE_initDStream(&bitD1, start1, length1);
+ if (FSE_isError(errorCode)) return errorCode;
+ errorCode = FSE_initDStream(&bitD2, start2, length2);
+ if (FSE_isError(errorCode)) return errorCode;
+ errorCode = FSE_initDStream(&bitD3, start3, length3);
+ if (FSE_isError(errorCode)) return errorCode;
+ errorCode = FSE_initDStream(&bitD4, start4, length4);
+ if (FSE_isError(errorCode)) return errorCode;
+
+ reloadStatus=FSE_reloadDStream(&bitD2);
+
+ /* 16 symbols per loop */
+ for ( ; (reloadStatus<FSE_DStream_completed) && (op<olimit); /* D2-3-4 are supposed to be synchronized and finish together */
+ op+=16, reloadStatus = FSE_reloadDStream(&bitD2) | FSE_reloadDStream(&bitD3) | FSE_reloadDStream(&bitD4), FSE_reloadDStream(&bitD1))
{
- HUF_DECODE_SYMBOL_0(0, bitTail);
+ #define HUF_DECODE_SYMBOL_0(n, Dstream) \
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog);
+
+ #define HUF_DECODE_SYMBOL_1(n, Dstream) \
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
+ if (FSE_32bits() && (HUF_MAX_TABLELOG>12)) FSE_reloadDStream(&Dstream)
+
+ #define HUF_DECODE_SYMBOL_2(n, Dstream) \
+ op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \
+ if (FSE_32bits()) FSE_reloadDStream(&Dstream)
+
+ HUF_DECODE_SYMBOL_1( 0, bitD1);
+ HUF_DECODE_SYMBOL_1( 1, bitD2);
+ HUF_DECODE_SYMBOL_1( 2, bitD3);
+ HUF_DECODE_SYMBOL_1( 3, bitD4);
+ HUF_DECODE_SYMBOL_2( 4, bitD1);
+ HUF_DECODE_SYMBOL_2( 5, bitD2);
+ HUF_DECODE_SYMBOL_2( 6, bitD3);
+ HUF_DECODE_SYMBOL_2( 7, bitD4);
+ HUF_DECODE_SYMBOL_1( 8, bitD1);
+ HUF_DECODE_SYMBOL_1( 9, bitD2);
+ HUF_DECODE_SYMBOL_1(10, bitD3);
+ HUF_DECODE_SYMBOL_1(11, bitD4);
+ HUF_DECODE_SYMBOL_0(12, bitD1);
+ HUF_DECODE_SYMBOL_0(13, bitD2);
+ HUF_DECODE_SYMBOL_0(14, bitD3);
+ HUF_DECODE_SYMBOL_0(15, bitD4);
}
- if (FSE_endOfDStream(&bitTail))
- return op-ostart;
+ if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */
+ return (size_t)-FSE_ERROR_corruptionDetected;
+
+ /* tail */
+ {
+ // bitTail = bitD1; // *much* slower : -20% !??!
+ FSE_DStream_t bitTail;
+ bitTail.ptr = bitD1.ptr;
+ bitTail.bitsConsumed = bitD1.bitsConsumed;
+ bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer
+ bitTail.start = start1;
+ for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op<omax) ; op++)
+ {
+ HUF_DECODE_SYMBOL_0(0, bitTail);
+ }
+
+ if (FSE_endOfDStream(&bitTail))
+ return op-ostart;
+ }
+
+ if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
+
+ return (size_t)-FSE_ERROR_corruptionDetected;
}
-
- if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */
-
- return (size_t)-FSE_ERROR_corruptionDetected;
}
@@ -1355,8 +1358,6 @@
static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
-static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
-
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
@@ -1381,16 +1382,9 @@
}
}
-
-static U32 ZSTD_readLE32(const void* memPtr)
+static U32 ZSTD_readLE24(const void* memPtr)
{
- if (ZSTD_isLittleEndian())
- return ZSTD_read32(memPtr);
- else
- {
- const BYTE* p = (const BYTE*)memPtr;
- return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
- }
+ return ZSTD_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
}
static U32 ZSTD_readBE32(const void* memPtr)
@@ -1704,13 +1698,13 @@
seqState->prevOffset = seq->offset;
if (litLength == MaxLL)
{
- U32 add = dumps<de ? *dumps++ : 0;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) litLength += add;
else
{
if (dumps<=(de-3))
{
- litLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ litLength = ZSTD_readLE24(dumps);
dumps += 3;
}
}
@@ -1732,13 +1726,13 @@
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
if (matchLength == MaxML)
{
- U32 add = dumps<de ? *dumps++ : 0;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) matchLength += add;
else
{
if (dumps<=(de-3))
{
- matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ matchLength = ZSTD_readLE24(dumps);
dumps += 3;
}
}
diff --git a/vendor/github.com/DataDog/zstd/zstd_v02.c b/vendor/github.com/DataDog/zstd/zstd_v02.c
index 561bc41..793df60 100644
--- a/vendor/github.com/DataDog/zstd/zstd_v02.c
+++ b/vendor/github.com/DataDog/zstd/zstd_v02.c
@@ -217,6 +217,11 @@
}
}
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
+}
+
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
@@ -3043,11 +3048,11 @@
seqState->prevOffset = seq->offset;
if (litLength == MaxLL)
{
- U32 add = *dumps++;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) litLength += add;
- else
+ else if (dumps + 3 <= de)
{
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ litLength = MEM_readLE24(dumps);
dumps += 3;
}
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -3073,11 +3078,11 @@
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
if (matchLength == MaxML)
{
- U32 add = *dumps++;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) matchLength += add;
- else
+ else if (dumps + 3 <= de)
{
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ matchLength = MEM_readLE24(dumps);
dumps += 3;
}
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
diff --git a/vendor/github.com/DataDog/zstd/zstd_v03.c b/vendor/github.com/DataDog/zstd/zstd_v03.c
index a1bf0fa..7a0e7c9 100644
--- a/vendor/github.com/DataDog/zstd/zstd_v03.c
+++ b/vendor/github.com/DataDog/zstd/zstd_v03.c
@@ -219,6 +219,11 @@
}
}
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
+}
+
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
@@ -2684,11 +2689,11 @@
seqState->prevOffset = seq->offset;
if (litLength == MaxLL)
{
- U32 add = *dumps++;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) litLength += add;
- else
+ else if (dumps + 3 <= de)
{
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ litLength = MEM_readLE24(dumps);
dumps += 3;
}
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
@@ -2714,11 +2719,11 @@
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
if (matchLength == MaxML)
{
- U32 add = *dumps++;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) matchLength += add;
- else
+ else if (dumps + 3 <= de)
{
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ matchLength = MEM_readLE24(dumps);
dumps += 3;
}
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
diff --git a/vendor/github.com/DataDog/zstd/zstd_v04.c b/vendor/github.com/DataDog/zstd/zstd_v04.c
index 4342330..645a6e3 100644
--- a/vendor/github.com/DataDog/zstd/zstd_v04.c
+++ b/vendor/github.com/DataDog/zstd/zstd_v04.c
@@ -189,6 +189,11 @@
}
}
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
+}
+
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
@@ -2808,13 +2813,12 @@
litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
prevOffset = litLength ? seq->offset : seqState->prevOffset;
if (litLength == MaxLL) {
- U32 add = *dumps++;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) litLength += add;
- else {
- litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
+ else if (dumps + 3 <= de) {
+ litLength = MEM_readLE24(dumps);
dumps += 3;
}
- if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
}
@@ -2837,13 +2841,12 @@
/* MatchLength */
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
if (matchLength == MaxML) {
- U32 add = *dumps++;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) matchLength += add;
- else {
- matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
+ else if (dumps + 3 <= de){
+ matchLength = MEM_readLE24(dumps);
dumps += 3;
}
- if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
}
matchLength += MINMATCH;
diff --git a/vendor/github.com/DataDog/zstd/zstd_v05.c b/vendor/github.com/DataDog/zstd/zstd_v05.c
index caaf15f..a7ea606 100644
--- a/vendor/github.com/DataDog/zstd/zstd_v05.c
+++ b/vendor/github.com/DataDog/zstd/zstd_v05.c
@@ -218,6 +218,11 @@
}
}
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+ return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
+}
+
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
@@ -1998,91 +2003,92 @@
const void* cSrc, size_t cSrcSize,
const U16* DTable)
{
- const BYTE* const istart = (const BYTE*) cSrc;
- BYTE* const ostart = (BYTE*) dst;
- BYTE* const oend = ostart + dstSize;
- const void* const dtPtr = DTable;
- const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr) +1;
- const U32 dtLog = DTable[0];
- size_t errorCode;
-
- /* Init */
- BITv05_DStream_t bitD1;
- BITv05_DStream_t bitD2;
- BITv05_DStream_t bitD3;
- BITv05_DStream_t bitD4;
- const size_t length1 = MEM_readLE16(istart);
- const size_t length2 = MEM_readLE16(istart+2);
- const size_t length3 = MEM_readLE16(istart+4);
- size_t length4;
- const BYTE* const istart1 = istart + 6; /* jumpTable */
- const BYTE* const istart2 = istart1 + length1;
- const BYTE* const istart3 = istart2 + length2;
- const BYTE* const istart4 = istart3 + length3;
- const size_t segmentSize = (dstSize+3) / 4;
- BYTE* const opStart2 = ostart + segmentSize;
- BYTE* const opStart3 = opStart2 + segmentSize;
- BYTE* const opStart4 = opStart3 + segmentSize;
- BYTE* op1 = ostart;
- BYTE* op2 = opStart2;
- BYTE* op3 = opStart3;
- BYTE* op4 = opStart4;
- U32 endSignal;
-
/* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
+ {
+ const BYTE* const istart = (const BYTE*) cSrc;
+ BYTE* const ostart = (BYTE*) dst;
+ BYTE* const oend = ostart + dstSize;
+ const void* const dtPtr = DTable;
+ const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr) +1;
+ const U32 dtLog = DTable[0];
+ size_t errorCode;
- length4 = cSrcSize - (length1 + length2 + length3 + 6);
- if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
- errorCode = BITv05_initDStream(&bitD1, istart1, length1);
- if (HUFv05_isError(errorCode)) return errorCode;
- errorCode = BITv05_initDStream(&bitD2, istart2, length2);
- if (HUFv05_isError(errorCode)) return errorCode;
- errorCode = BITv05_initDStream(&bitD3, istart3, length3);
- if (HUFv05_isError(errorCode)) return errorCode;
- errorCode = BITv05_initDStream(&bitD4, istart4, length4);
- if (HUFv05_isError(errorCode)) return errorCode;
+ /* Init */
+ BITv05_DStream_t bitD1;
+ BITv05_DStream_t bitD2;
+ BITv05_DStream_t bitD3;
+ BITv05_DStream_t bitD4;
+ const size_t length1 = MEM_readLE16(istart);
+ const size_t length2 = MEM_readLE16(istart+2);
+ const size_t length3 = MEM_readLE16(istart+4);
+ size_t length4;
+ const BYTE* const istart1 = istart + 6; /* jumpTable */
+ const BYTE* const istart2 = istart1 + length1;
+ const BYTE* const istart3 = istart2 + length2;
+ const BYTE* const istart4 = istart3 + length3;
+ const size_t segmentSize = (dstSize+3) / 4;
+ BYTE* const opStart2 = ostart + segmentSize;
+ BYTE* const opStart3 = opStart2 + segmentSize;
+ BYTE* const opStart4 = opStart3 + segmentSize;
+ BYTE* op1 = ostart;
+ BYTE* op2 = opStart2;
+ BYTE* op3 = opStart3;
+ BYTE* op4 = opStart4;
+ U32 endSignal;
- /* 16-32 symbols per loop (4-8 symbols per stream) */
- endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
- for ( ; (endSignal==BITv05_DStream_unfinished) && (op4<(oend-7)) ; ) {
- HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
- HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
- HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
- HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
- HUFv05_DECODE_SYMBOLX2_1(op1, &bitD1);
- HUFv05_DECODE_SYMBOLX2_1(op2, &bitD2);
- HUFv05_DECODE_SYMBOLX2_1(op3, &bitD3);
- HUFv05_DECODE_SYMBOLX2_1(op4, &bitD4);
- HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
- HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
- HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
- HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
- HUFv05_DECODE_SYMBOLX2_0(op1, &bitD1);
- HUFv05_DECODE_SYMBOLX2_0(op2, &bitD2);
- HUFv05_DECODE_SYMBOLX2_0(op3, &bitD3);
- HUFv05_DECODE_SYMBOLX2_0(op4, &bitD4);
+ length4 = cSrcSize - (length1 + length2 + length3 + 6);
+ if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
+ errorCode = BITv05_initDStream(&bitD1, istart1, length1);
+ if (HUFv05_isError(errorCode)) return errorCode;
+ errorCode = BITv05_initDStream(&bitD2, istart2, length2);
+ if (HUFv05_isError(errorCode)) return errorCode;
+ errorCode = BITv05_initDStream(&bitD3, istart3, length3);
+ if (HUFv05_isError(errorCode)) return errorCode;
+ errorCode = BITv05_initDStream(&bitD4, istart4, length4);
+ if (HUFv05_isError(errorCode)) return errorCode;
+
+ /* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
+ for ( ; (endSignal==BITv05_DStream_unfinished) && (op4<(oend-7)) ; ) {
+ HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUFv05_DECODE_SYMBOLX2_1(op1, &bitD1);
+ HUFv05_DECODE_SYMBOLX2_1(op2, &bitD2);
+ HUFv05_DECODE_SYMBOLX2_1(op3, &bitD3);
+ HUFv05_DECODE_SYMBOLX2_1(op4, &bitD4);
+ HUFv05_DECODE_SYMBOLX2_2(op1, &bitD1);
+ HUFv05_DECODE_SYMBOLX2_2(op2, &bitD2);
+ HUFv05_DECODE_SYMBOLX2_2(op3, &bitD3);
+ HUFv05_DECODE_SYMBOLX2_2(op4, &bitD4);
+ HUFv05_DECODE_SYMBOLX2_0(op1, &bitD1);
+ HUFv05_DECODE_SYMBOLX2_0(op2, &bitD2);
+ HUFv05_DECODE_SYMBOLX2_0(op3, &bitD3);
+ HUFv05_DECODE_SYMBOLX2_0(op4, &bitD4);
+ endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
+ }
+
+ /* check corruption */
+ if (op1 > opStart2) return ERROR(corruption_detected);
+ if (op2 > opStart3) return ERROR(corruption_detected);
+ if (op3 > opStart4) return ERROR(corruption_detected);
+ /* note : op4 supposed already verified within main loop */
+
+ /* finish bitStreams one by one */
+ HUFv05_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
+ HUFv05_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
+ HUFv05_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
+ HUFv05_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
+
+ /* check */
+ endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
+ if (!endSignal) return ERROR(corruption_detected);
+
+ /* decoded size */
+ return dstSize;
}
-
- /* check corruption */
- if (op1 > opStart2) return ERROR(corruption_detected);
- if (op2 > opStart3) return ERROR(corruption_detected);
- if (op3 > opStart4) return ERROR(corruption_detected);
- /* note : op4 supposed already verified within main loop */
-
- /* finish bitStreams one by one */
- HUFv05_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
- HUFv05_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
- HUFv05_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
- HUFv05_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
-
- /* check */
- endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
- if (!endSignal) return ERROR(corruption_detected);
-
- /* decoded size */
- return dstSize;
}
@@ -3150,14 +3156,13 @@
litLength = FSEv05_peakSymbol(&(seqState->stateLL));
prevOffset = litLength ? seq->offset : seqState->prevOffset;
if (litLength == MaxLL) {
- U32 add = *dumps++;
+ const U32 add = *dumps++;
if (add < 255) litLength += add;
- else {
- litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no risk : dumps is always followed by seq tables > 1 byte */
+ else if (dumps + 3 <= de) {
+ litLength = MEM_readLE24(dumps);
if (litLength&1) litLength>>=1, dumps += 3;
else litLength = (U16)(litLength)>>1, dumps += 2;
}
- if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
}
@@ -3184,14 +3189,13 @@
/* MatchLength */
matchLength = FSEv05_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
if (matchLength == MaxML) {
- U32 add = *dumps++;
+ const U32 add = dumps<de ? *dumps++ : 0;
if (add < 255) matchLength += add;
- else {
- matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
+ else if (dumps + 3 <= de) {
+ matchLength = MEM_readLE24(dumps);
if (matchLength&1) matchLength>>=1, dumps += 3;
else matchLength = (U16)(matchLength)>>1, dumps += 2;
}
- if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
}
matchLength += MINMATCH;
diff --git a/vendor/github.com/DataDog/zstd/zstd_v06.c b/vendor/github.com/DataDog/zstd/zstd_v06.c
index a695cbb..f907a3a 100644
--- a/vendor/github.com/DataDog/zstd/zstd_v06.c
+++ b/vendor/github.com/DataDog/zstd/zstd_v06.c
@@ -3242,14 +3242,12 @@
}
/* FSE table descriptors */
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
{ U32 const LLtype = *ip >> 6;
U32 const Offtype = (*ip >> 4) & 3;
U32 const MLtype = (*ip >> 2) & 3;
ip++;
- /* check */
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
-
/* Build DTables */
{ size_t const bhSize = ZSTDv06_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
if (ZSTDv06_isError(bhSize)) return ERROR(corruption_detected);
@@ -3672,7 +3670,7 @@
blockProperties_t blockProperties = { bt_compressed, 0 };
/* Frame Header */
- { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
+ { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, srcSize);
if (ZSTDv06_isError(frameHeaderSize)) {
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
return;
diff --git a/vendor/github.com/DataDog/zstd/zstd_v07.c b/vendor/github.com/DataDog/zstd/zstd_v07.c
index 6b94889..a83ddc9 100644
--- a/vendor/github.com/DataDog/zstd/zstd_v07.c
+++ b/vendor/github.com/DataDog/zstd/zstd_v07.c
@@ -3470,14 +3470,12 @@
}
/* FSE table descriptors */
+ if (ip + 4 > iend) return ERROR(srcSize_wrong); /* min : header byte + all 3 are "raw", hence no header, but at least xxLog bits per type */
{ U32 const LLtype = *ip >> 6;
U32 const OFtype = (*ip >> 4) & 3;
U32 const MLtype = (*ip >> 2) & 3;
ip++;
- /* check */
- if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
-
/* Build DTables */
{ size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected);
@@ -3918,7 +3916,7 @@
}
/* Frame Header */
- { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
+ { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, srcSize);
if (ZSTDv07_isError(frameHeaderSize)) {
ZSTD_errorFrameSizeInfoLegacy(cSize, dBound, frameHeaderSize);
return;
diff --git a/vendor/github.com/DataDog/zstd/zstdmt_compress.c b/vendor/github.com/DataDog/zstd/zstdmt_compress.c
index 38fbb90..9e537b8 100644
--- a/vendor/github.com/DataDog/zstd/zstdmt_compress.c
+++ b/vendor/github.com/DataDog/zstd/zstdmt_compress.c
@@ -1129,9 +1129,14 @@
size_t const produced = ZSTD_isError(cResult) ? 0 : cResult;
size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed;
assert(flushed <= produced);
+ assert(jobPtr->consumed <= jobPtr->src.size);
toFlush = produced - flushed;
- if (toFlush==0 && (jobPtr->consumed >= jobPtr->src.size)) {
- /* doneJobID is not-fully-flushed, but toFlush==0 : doneJobID should be compressing some more data */
+ /* if toFlush==0, nothing is available to flush.
+ * However, jobID is expected to still be active:
+ * if jobID was already completed and fully flushed,
+ * ZSTDMT_flushProduced() should have already moved onto next job.
+ * Therefore, some input has not yet been consumed. */
+ if (toFlush==0) {
assert(jobPtr->consumed < jobPtr->src.size);
}
}
@@ -1148,12 +1153,16 @@
static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
{
- if (params.ldmParams.enableLdm)
+ unsigned jobLog;
+ if (params.ldmParams.enableLdm) {
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on chainLog instead. */
- return MAX(21, params.cParams.chainLog + 4);
- return MAX(20, params.cParams.windowLog + 2);
+ jobLog = MAX(21, params.cParams.chainLog + 4);
+ } else {
+ jobLog = MAX(20, params.cParams.windowLog + 2);
+ }
+ return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX);
}
static int ZSTDMT_overlapLog_default(ZSTD_strategy strat)
@@ -1197,7 +1206,7 @@
ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2)
- overlapRLog;
}
- assert(0 <= ovLog && ovLog <= 30);
+ assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX);
DEBUGLOG(4, "overlapLog : %i", params.overlapLog);
DEBUGLOG(4, "overlap size : %i", 1 << ovLog);
return (ovLog==0) ? 0 : (size_t)1 << ovLog;
@@ -1391,7 +1400,7 @@
FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) );
if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
- if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
+ if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX;
mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */
if (mtctx->singleBlockingThread) {
@@ -1432,6 +1441,8 @@
if (mtctx->targetSectionSize == 0) {
mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params);
}
+ assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX);
+
if (params.rsyncable) {
/* Aim for the targetsectionSize as the average job size. */
U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20);
diff --git a/vendor/github.com/DataDog/zstd/zstdmt_compress.h b/vendor/github.com/DataDog/zstd/zstdmt_compress.h
index 12e6bcb..12a5260 100644
--- a/vendor/github.com/DataDog/zstd/zstdmt_compress.h
+++ b/vendor/github.com/DataDog/zstd/zstdmt_compress.h
@@ -50,6 +50,7 @@
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1 MB)
#endif
+#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30)
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))
diff --git a/vendor/github.com/Shopify/sarama/CHANGELOG.md b/vendor/github.com/Shopify/sarama/CHANGELOG.md
index 4f89e21..02bd0ff 100644
--- a/vendor/github.com/Shopify/sarama/CHANGELOG.md
+++ b/vendor/github.com/Shopify/sarama/CHANGELOG.md
@@ -1,5 +1,13 @@
# Changelog
+#### Version 1.23.1 (2019-07-22)
+
+Bug Fixes:
+- Fix fetch delete bug record
+ ([1425](https://github.com/Shopify/sarama/pull/1425)).
+- Handle SASL/OAUTHBEARER token rejection
+ ([1428](https://github.com/Shopify/sarama/pull/1428)).
+
#### Version 1.23.0 (2019-07-02)
New Features:
diff --git a/vendor/github.com/Shopify/sarama/admin.go b/vendor/github.com/Shopify/sarama/admin.go
index a4d1bc5..1db6a0e 100644
--- a/vendor/github.com/Shopify/sarama/admin.go
+++ b/vendor/github.com/Shopify/sarama/admin.go
@@ -374,29 +374,50 @@
if topic == "" {
return ErrInvalidTopic
}
-
- topics := make(map[string]*DeleteRecordsRequestTopic)
- topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: partitionOffsets}
- request := &DeleteRecordsRequest{
- Topics: topics,
- Timeout: ca.conf.Admin.Timeout,
+ partitionPerBroker := make(map[*Broker][]int32)
+ for partition := range partitionOffsets {
+ broker, err := ca.client.Leader(topic, partition)
+ if err != nil {
+ return err
+ }
+ if _, ok := partitionPerBroker[broker]; ok {
+ partitionPerBroker[broker] = append(partitionPerBroker[broker], partition)
+ } else {
+ partitionPerBroker[broker] = []int32{partition}
+ }
}
+ errs := make([]error, 0)
+ for broker, partitions := range partitionPerBroker {
+ topics := make(map[string]*DeleteRecordsRequestTopic)
+ recordsToDelete := make(map[int32]int64)
+ for _, p := range partitions {
+ recordsToDelete[p] = partitionOffsets[p]
+ }
+ topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: recordsToDelete}
+ request := &DeleteRecordsRequest{
+ Topics: topics,
+ Timeout: ca.conf.Admin.Timeout,
+ }
- b, err := ca.Controller()
- if err != nil {
- return err
+ rsp, err := broker.DeleteRecords(request)
+ if err != nil {
+ errs = append(errs, err)
+ } else {
+ deleteRecordsResponseTopic, ok := rsp.Topics[topic]
+ if !ok {
+ errs = append(errs, ErrIncompleteResponse)
+ } else {
+ for _, deleteRecordsResponsePartition := range deleteRecordsResponseTopic.Partitions {
+ if deleteRecordsResponsePartition.Err != ErrNoError {
+ errs = append(errs, errors.New(deleteRecordsResponsePartition.Err.Error()))
+ }
+ }
+ }
+ }
}
-
- rsp, err := b.DeleteRecords(request)
- if err != nil {
- return err
+ if len(errs) > 0 {
+ return ErrDeleteRecords{MultiError{&errs}}
}
-
- _, ok := rsp.Topics[topic]
- if !ok {
- return ErrIncompleteResponse
- }
-
//todo since we are dealing with couple of partitions it would be good if we return slice of errors
//for each partition instead of one error
return nil
diff --git a/vendor/github.com/Shopify/sarama/broker.go b/vendor/github.com/Shopify/sarama/broker.go
index 7b32a03..9c3e5a0 100644
--- a/vendor/github.com/Shopify/sarama/broker.go
+++ b/vendor/github.com/Shopify/sarama/broker.go
@@ -1013,7 +1013,7 @@
b.correlationID++
- bytesRead, err := b.receiveSASLServerResponse(correlationID)
+ bytesRead, err := b.receiveSASLServerResponse(&SaslAuthenticateResponse{}, correlationID)
b.updateIncomingCommunicationMetrics(bytesRead, time.Since(requestTime))
// With v1 sasl we get an error message set in the response we can return
@@ -1037,26 +1037,53 @@
return err
}
+ message, err := buildClientFirstMessage(token)
+ if err != nil {
+ return err
+ }
+
+ challenged, err := b.sendClientMessage(message)
+ if err != nil {
+ return err
+ }
+
+ if challenged {
+ // Abort the token exchange. The broker returns the failure code.
+ _, err = b.sendClientMessage([]byte(`\x01`))
+ }
+
+ return err
+}
+
+// sendClientMessage sends a SASL/OAUTHBEARER client message and returns true
+// if the broker responds with a challenge, in which case the token is
+// rejected.
+func (b *Broker) sendClientMessage(message []byte) (bool, error) {
+
requestTime := time.Now()
correlationID := b.correlationID
- bytesWritten, err := b.sendSASLOAuthBearerClientResponse(token, correlationID)
+ bytesWritten, err := b.sendSASLOAuthBearerClientMessage(message, correlationID)
if err != nil {
- return err
+ return false, err
}
b.updateOutgoingCommunicationMetrics(bytesWritten)
b.correlationID++
- bytesRead, err := b.receiveSASLServerResponse(correlationID)
- if err != nil {
- return err
- }
+ res := &SaslAuthenticateResponse{}
+ bytesRead, err := b.receiveSASLServerResponse(res, correlationID)
requestLatency := time.Since(requestTime)
b.updateIncomingCommunicationMetrics(bytesRead, requestLatency)
- return nil
+ isChallenge := len(res.SaslAuthBytes) > 0
+
+ if isChallenge && err != nil {
+ Logger.Printf("Broker rejected authentication token: %s", res.SaslAuthBytes)
+ }
+
+ return isChallenge, err
}
func (b *Broker) sendAndReceiveSASLSCRAMv1() error {
@@ -1154,7 +1181,7 @@
// Build SASL/OAUTHBEARER initial client response as described by RFC-7628
// https://tools.ietf.org/html/rfc7628
-func buildClientInitialResponse(token *AccessToken) ([]byte, error) {
+func buildClientFirstMessage(token *AccessToken) ([]byte, error) {
var ext string
if token.Extensions != nil && len(token.Extensions) > 0 {
@@ -1200,11 +1227,7 @@
return b.conn.Write(buf)
}
-func (b *Broker) sendSASLOAuthBearerClientResponse(token *AccessToken, correlationID int32) (int, error) {
- initialResp, err := buildClientInitialResponse(token)
- if err != nil {
- return 0, err
- }
+func (b *Broker) sendSASLOAuthBearerClientMessage(initialResp []byte, correlationID int32) (int, error) {
rb := &SaslAuthenticateRequest{initialResp}
@@ -1222,7 +1245,7 @@
return b.conn.Write(buf)
}
-func (b *Broker) receiveSASLServerResponse(correlationID int32) (int, error) {
+func (b *Broker) receiveSASLServerResponse(res *SaslAuthenticateResponse, correlationID int32) (int, error) {
buf := make([]byte, responseLengthSize+correlationIDSize)
@@ -1250,8 +1273,6 @@
return bytesRead, err
}
- res := &SaslAuthenticateResponse{}
-
if err := versionedDecode(buf, res, 0); err != nil {
return bytesRead, err
}
@@ -1260,10 +1281,6 @@
return bytesRead, res.Err
}
- if len(res.SaslAuthBytes) > 0 {
- Logger.Printf("Received SASL auth response: %s", res.SaslAuthBytes)
- }
-
return bytesRead, nil
}
diff --git a/vendor/github.com/Shopify/sarama/errors.go b/vendor/github.com/Shopify/sarama/errors.go
index 8ecb652..c6a8be7 100644
--- a/vendor/github.com/Shopify/sarama/errors.go
+++ b/vendor/github.com/Shopify/sarama/errors.go
@@ -81,6 +81,28 @@
// See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes
type KError int16
+// MultiError is used to contain multi error.
+type MultiError struct {
+ Errors *[]error
+}
+
+func (mErr MultiError) Error() string {
+ var errString = ""
+ for _, err := range *mErr.Errors {
+ errString += err.Error() + ","
+ }
+ return errString
+}
+
+// ErrDeleteRecords is the type of error returned when fail to delete the required records
+type ErrDeleteRecords struct {
+ MultiError
+}
+
+func (err ErrDeleteRecords) Error() string {
+ return "kafka server: failed to delete records " + err.MultiError.Error()
+}
+
// Numeric error codes returned by the Kafka server.
const (
ErrNoError KError = 0
diff --git a/vendor/github.com/gogo/protobuf/GOLANG_CONTRIBUTORS b/vendor/github.com/gogo/protobuf/GOLANG_CONTRIBUTORS
deleted file mode 100644
index b368efb..0000000
--- a/vendor/github.com/gogo/protobuf/GOLANG_CONTRIBUTORS
+++ /dev/null
@@ -1,5 +0,0 @@
-The contributors to the Go protobuf repository:
-
-# This source code was written by the Go contributors.
-# The master list of contributors is in the main Go distribution,
-# visible at http://tip.golang.org/CONTRIBUTORS.
\ No newline at end of file
diff --git a/vendor/github.com/gogo/protobuf/gogoproto/gogo.pb.go b/vendor/github.com/gogo/protobuf/gogoproto/gogo.pb.go
index e352808..1e91766 100644
--- a/vendor/github.com/gogo/protobuf/gogoproto/gogo.pb.go
+++ b/vendor/github.com/gogo/protobuf/gogoproto/gogo.pb.go
@@ -19,7 +19,7 @@
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
var E_GoprotoEnumPrefix = &proto.ExtensionDesc{
ExtendedType: (*descriptor.EnumOptions)(nil),
diff --git a/vendor/github.com/gogo/protobuf/proto/extensions.go b/vendor/github.com/gogo/protobuf/proto/extensions.go
index 686bd2a..341c6f5 100644
--- a/vendor/github.com/gogo/protobuf/proto/extensions.go
+++ b/vendor/github.com/gogo/protobuf/proto/extensions.go
@@ -527,6 +527,7 @@
// SetExtension sets the specified extension of pb to the specified value.
func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error {
if epb, ok := pb.(extensionsBytes); ok {
+ ClearExtension(pb, extension)
newb, err := encodeExtension(extension, value)
if err != nil {
return err
diff --git a/vendor/github.com/gogo/protobuf/proto/extensions_gogo.go b/vendor/github.com/gogo/protobuf/proto/extensions_gogo.go
index 53ebd8c..6f1ae12 100644
--- a/vendor/github.com/gogo/protobuf/proto/extensions_gogo.go
+++ b/vendor/github.com/gogo/protobuf/proto/extensions_gogo.go
@@ -154,6 +154,10 @@
return EncodeExtensionMap(m.extensionsWrite(), data)
}
+func EncodeInternalExtensionBackwards(m extendableProto, data []byte) (n int, err error) {
+ return EncodeExtensionMapBackwards(m.extensionsWrite(), data)
+}
+
func EncodeExtensionMap(m map[int32]Extension, data []byte) (n int, err error) {
o := 0
for _, e := range m {
@@ -169,6 +173,23 @@
return o, nil
}
+func EncodeExtensionMapBackwards(m map[int32]Extension, data []byte) (n int, err error) {
+ o := 0
+ end := len(data)
+ for _, e := range m {
+ if err := e.Encode(); err != nil {
+ return 0, err
+ }
+ n := copy(data[end-len(e.enc):], e.enc)
+ if n != len(e.enc) {
+ return 0, io.ErrShortBuffer
+ }
+ end -= n
+ o += n
+ }
+ return o, nil
+}
+
func GetRawExtension(m map[int32]Extension, id int32) ([]byte, error) {
e := m[id]
if err := e.Encode(); err != nil {
diff --git a/vendor/github.com/gogo/protobuf/proto/lib.go b/vendor/github.com/gogo/protobuf/proto/lib.go
index d17f802..80db1c1 100644
--- a/vendor/github.com/gogo/protobuf/proto/lib.go
+++ b/vendor/github.com/gogo/protobuf/proto/lib.go
@@ -948,13 +948,19 @@
return false
}
-// ProtoPackageIsVersion2 is referenced from generated protocol buffer files
-// to assert that that code is compatible with this version of the proto package.
-const GoGoProtoPackageIsVersion2 = true
+const (
+ // ProtoPackageIsVersion3 is referenced from generated protocol buffer files
+ // to assert that that code is compatible with this version of the proto package.
+ GoGoProtoPackageIsVersion3 = true
-// ProtoPackageIsVersion1 is referenced from generated protocol buffer files
-// to assert that that code is compatible with this version of the proto package.
-const GoGoProtoPackageIsVersion1 = true
+ // ProtoPackageIsVersion2 is referenced from generated protocol buffer files
+ // to assert that that code is compatible with this version of the proto package.
+ GoGoProtoPackageIsVersion2 = true
+
+ // ProtoPackageIsVersion1 is referenced from generated protocol buffer files
+ // to assert that that code is compatible with this version of the proto package.
+ GoGoProtoPackageIsVersion1 = true
+)
// InternalMessageInfo is a type used internally by generated .pb.go files.
// This type is not intended to be used by non-generated code.
diff --git a/vendor/github.com/gogo/protobuf/proto/properties.go b/vendor/github.com/gogo/protobuf/proto/properties.go
index c9e5fa0..62c5562 100644
--- a/vendor/github.com/gogo/protobuf/proto/properties.go
+++ b/vendor/github.com/gogo/protobuf/proto/properties.go
@@ -400,6 +400,15 @@
return sprop
}
+type (
+ oneofFuncsIface interface {
+ XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{})
+ }
+ oneofWrappersIface interface {
+ XXX_OneofWrappers() []interface{}
+ }
+)
+
// getPropertiesLocked requires that propertiesMu is held.
func getPropertiesLocked(t reflect.Type) *StructProperties {
if prop, ok := propertiesMap[t]; ok {
@@ -441,37 +450,40 @@
// Re-order prop.order.
sort.Sort(prop)
- type oneofMessage interface {
- XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{})
- }
- if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); isOneofMessage && ok {
+ if isOneofMessage {
var oots []interface{}
- _, _, _, oots = om.XXX_OneofFuncs()
-
- // Interpret oneof metadata.
- prop.OneofTypes = make(map[string]*OneofProperties)
- for _, oot := range oots {
- oop := &OneofProperties{
- Type: reflect.ValueOf(oot).Type(), // *T
- Prop: new(Properties),
- }
- sft := oop.Type.Elem().Field(0)
- oop.Prop.Name = sft.Name
- oop.Prop.Parse(sft.Tag.Get("protobuf"))
- // There will be exactly one interface field that
- // this new value is assignable to.
- for i := 0; i < t.NumField(); i++ {
- f := t.Field(i)
- if f.Type.Kind() != reflect.Interface {
- continue
+ switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) {
+ case oneofFuncsIface:
+ _, _, _, oots = m.XXX_OneofFuncs()
+ case oneofWrappersIface:
+ oots = m.XXX_OneofWrappers()
+ }
+ if len(oots) > 0 {
+ // Interpret oneof metadata.
+ prop.OneofTypes = make(map[string]*OneofProperties)
+ for _, oot := range oots {
+ oop := &OneofProperties{
+ Type: reflect.ValueOf(oot).Type(), // *T
+ Prop: new(Properties),
}
- if !oop.Type.AssignableTo(f.Type) {
- continue
+ sft := oop.Type.Elem().Field(0)
+ oop.Prop.Name = sft.Name
+ oop.Prop.Parse(sft.Tag.Get("protobuf"))
+ // There will be exactly one interface field that
+ // this new value is assignable to.
+ for i := 0; i < t.NumField(); i++ {
+ f := t.Field(i)
+ if f.Type.Kind() != reflect.Interface {
+ continue
+ }
+ if !oop.Type.AssignableTo(f.Type) {
+ continue
+ }
+ oop.Field = i
+ break
}
- oop.Field = i
- break
+ prop.OneofTypes[oop.Prop.OrigName] = oop
}
- prop.OneofTypes[oop.Prop.OrigName] = oop
}
}
diff --git a/vendor/github.com/gogo/protobuf/proto/table_marshal.go b/vendor/github.com/gogo/protobuf/proto/table_marshal.go
index 9b1538d..db9927a 100644
--- a/vendor/github.com/gogo/protobuf/proto/table_marshal.go
+++ b/vendor/github.com/gogo/protobuf/proto/table_marshal.go
@@ -389,8 +389,13 @@
// get oneof implementers
var oneofImplementers []interface{}
// gogo: isOneofMessage is needed for embedded oneof messages, without a marshaler and unmarshaler
- if m, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok && isOneofMessage {
- _, _, _, oneofImplementers = m.XXX_OneofFuncs()
+ if isOneofMessage {
+ switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) {
+ case oneofFuncsIface:
+ _, _, _, oneofImplementers = m.XXX_OneofFuncs()
+ case oneofWrappersIface:
+ oneofImplementers = m.XXX_OneofWrappers()
+ }
}
// normal fields
@@ -519,10 +524,6 @@
}
}
-type oneofMessage interface {
- XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{})
-}
-
// wiretype returns the wire encoding of the type.
func wiretype(encoding string) uint64 {
switch encoding {
diff --git a/vendor/github.com/gogo/protobuf/proto/table_merge.go b/vendor/github.com/gogo/protobuf/proto/table_merge.go
index f520106..60dcf70 100644
--- a/vendor/github.com/gogo/protobuf/proto/table_merge.go
+++ b/vendor/github.com/gogo/protobuf/proto/table_merge.go
@@ -530,6 +530,25 @@
}
case reflect.Struct:
switch {
+ case isSlice && !isPointer: // E.g. []pb.T
+ mergeInfo := getMergeInfo(tf)
+ zero := reflect.Zero(tf)
+ mfi.merge = func(dst, src pointer) {
+ // TODO: Make this faster?
+ dstsp := dst.asPointerTo(f.Type)
+ dsts := dstsp.Elem()
+ srcs := src.asPointerTo(f.Type).Elem()
+ for i := 0; i < srcs.Len(); i++ {
+ dsts = reflect.Append(dsts, zero)
+ srcElement := srcs.Index(i).Addr()
+ dstElement := dsts.Index(dsts.Len() - 1).Addr()
+ mergeInfo.merge(valToPointer(dstElement), valToPointer(srcElement))
+ }
+ if dsts.IsNil() {
+ dsts = reflect.MakeSlice(f.Type, 0, 0)
+ }
+ dstsp.Elem().Set(dsts)
+ }
case !isPointer:
mergeInfo := getMergeInfo(tf)
mfi.merge = func(dst, src pointer) {
diff --git a/vendor/github.com/gogo/protobuf/proto/table_unmarshal.go b/vendor/github.com/gogo/protobuf/proto/table_unmarshal.go
index bb2622f..9372293 100644
--- a/vendor/github.com/gogo/protobuf/proto/table_unmarshal.go
+++ b/vendor/github.com/gogo/protobuf/proto/table_unmarshal.go
@@ -371,15 +371,18 @@
}
// Find any types associated with oneof fields.
- // TODO: XXX_OneofFuncs returns more info than we need. Get rid of some of it?
- fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("XXX_OneofFuncs")
// gogo: len(oneofFields) > 0 is needed for embedded oneof messages, without a marshaler and unmarshaler
- if fn.IsValid() && len(oneofFields) > 0 {
- res := fn.Call(nil)[3] // last return value from XXX_OneofFuncs: []interface{}
- for i := res.Len() - 1; i >= 0; i-- {
- v := res.Index(i) // interface{}
- tptr := reflect.ValueOf(v.Interface()).Type() // *Msg_X
- typ := tptr.Elem() // Msg_X
+ if len(oneofFields) > 0 {
+ var oneofImplementers []interface{}
+ switch m := reflect.Zero(reflect.PtrTo(t)).Interface().(type) {
+ case oneofFuncsIface:
+ _, _, _, oneofImplementers = m.XXX_OneofFuncs()
+ case oneofWrappersIface:
+ oneofImplementers = m.XXX_OneofWrappers()
+ }
+ for _, v := range oneofImplementers {
+ tptr := reflect.TypeOf(v) // *Msg_X
+ typ := tptr.Elem() // Msg_X
f := typ.Field(0) // oneof implementers have one field
baseUnmarshal := fieldUnmarshaler(&f)
@@ -407,11 +410,12 @@
u.setTag(fieldNum, of.field, unmarshal, 0, name)
}
}
+
}
}
// Get extension ranges, if any.
- fn = reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray")
+ fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray")
if fn.IsValid() {
if !u.extensions.IsValid() && !u.oldExtensions.IsValid() && !u.bytesExtensions.IsValid() {
panic("a message with extensions, but no extensions field in " + t.Name())
diff --git a/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/descriptor.pb.go b/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/descriptor.pb.go
index cacfa39..d1307d9 100644
--- a/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/descriptor.pb.go
+++ b/vendor/github.com/gogo/protobuf/protoc-gen-gogo/descriptor/descriptor.pb.go
@@ -18,7 +18,7 @@
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
type FieldDescriptorProto_Type int32
diff --git a/vendor/github.com/golang/protobuf/proto/properties.go b/vendor/github.com/golang/protobuf/proto/properties.go
index 79668ff..a4b8c0c 100644
--- a/vendor/github.com/golang/protobuf/proto/properties.go
+++ b/vendor/github.com/golang/protobuf/proto/properties.go
@@ -38,7 +38,6 @@
import (
"fmt"
"log"
- "os"
"reflect"
"sort"
"strconv"
@@ -194,7 +193,7 @@
// "bytes,49,opt,name=foo,def=hello!"
fields := strings.Split(s, ",") // breaks def=, but handled below.
if len(fields) < 2 {
- fmt.Fprintf(os.Stderr, "proto: tag has too few fields: %q\n", s)
+ log.Printf("proto: tag has too few fields: %q", s)
return
}
@@ -214,7 +213,7 @@
p.WireType = WireBytes
// no numeric converter for non-numeric types
default:
- fmt.Fprintf(os.Stderr, "proto: tag has unknown wire type: %q\n", s)
+ log.Printf("proto: tag has unknown wire type: %q", s)
return
}
diff --git a/vendor/github.com/hashicorp/consul/NOTICE.md b/vendor/github.com/hashicorp/consul/NOTICE.md
deleted file mode 100644
index fe34b5e..0000000
--- a/vendor/github.com/hashicorp/consul/NOTICE.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Copyright © 2014-2018 HashiCorp, Inc.
-
-This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this project, you can obtain one at http://mozilla.org/MPL/2.0/.
diff --git a/vendor/github.com/hashicorp/consul/LICENSE b/vendor/github.com/hashicorp/consul/api/LICENSE
similarity index 100%
rename from vendor/github.com/hashicorp/consul/LICENSE
rename to vendor/github.com/hashicorp/consul/api/LICENSE
diff --git a/vendor/github.com/hashicorp/consul/api/agent.go b/vendor/github.com/hashicorp/consul/api/agent.go
index 04043ba..1ef3312 100644
--- a/vendor/github.com/hashicorp/consul/api/agent.go
+++ b/vendor/github.com/hashicorp/consul/api/agent.go
@@ -23,23 +23,11 @@
// service proxies another service within Consul and speaks the connect
// protocol.
ServiceKindConnectProxy ServiceKind = "connect-proxy"
-)
-// ProxyExecMode is the execution mode for a managed Connect proxy.
-type ProxyExecMode string
-
-const (
- // ProxyExecModeDaemon indicates that the proxy command should be long-running
- // and should be started and supervised by the agent until it's target service
- // is deregistered.
- ProxyExecModeDaemon ProxyExecMode = "daemon"
-
- // ProxyExecModeScript indicates that the proxy command should be invoke to
- // completion on each change to the configuration of lifecycle event. The
- // script typically fetches the config and certificates from the agent API and
- // then configures an externally managed daemon, perhaps starting and stopping
- // it if necessary.
- ProxyExecModeScript ProxyExecMode = "script"
+ // ServiceKindMeshGateway is a Mesh Gateway for the Connect feature. This
+ // service will proxy connections based off the SNI header set by other
+ // connect proxies
+ ServiceKindMeshGateway ServiceKind = "mesh-gateway"
)
// UpstreamDestType is the type of upstream discovery mechanism.
@@ -82,15 +70,14 @@
Meta map[string]string
Port int
Address string
+ TaggedAddresses map[string]ServiceAddress `json:",omitempty"`
Weights AgentWeights
EnableTagOverride bool
- CreateIndex uint64 `json:",omitempty" bexpr:"-"`
- ModifyIndex uint64 `json:",omitempty" bexpr:"-"`
- ContentHash string `json:",omitempty" bexpr:"-"`
- // DEPRECATED (ProxyDestination) - remove this field
- ProxyDestination string `json:",omitempty" bexpr:"-"`
- Proxy *AgentServiceConnectProxyConfig `json:",omitempty"`
- Connect *AgentServiceConnect `json:",omitempty"`
+ CreateIndex uint64 `json:",omitempty" bexpr:"-"`
+ ModifyIndex uint64 `json:",omitempty" bexpr:"-"`
+ ContentHash string `json:",omitempty" bexpr:"-"`
+ Proxy *AgentServiceConnectProxyConfig `json:",omitempty"`
+ Connect *AgentServiceConnect `json:",omitempty"`
}
// AgentServiceChecksInfo returns information about a Service and its checks
@@ -103,28 +90,19 @@
// AgentServiceConnect represents the Connect configuration of a service.
type AgentServiceConnect struct {
Native bool `json:",omitempty"`
- Proxy *AgentServiceConnectProxy `json:",omitempty" bexpr:"-"`
SidecarService *AgentServiceRegistration `json:",omitempty" bexpr:"-"`
}
-// AgentServiceConnectProxy represents the Connect Proxy configuration of a
-// service.
-type AgentServiceConnectProxy struct {
- ExecMode ProxyExecMode `json:",omitempty"`
- Command []string `json:",omitempty"`
- Config map[string]interface{} `json:",omitempty" bexpr:"-"`
- Upstreams []Upstream `json:",omitempty"`
-}
-
// AgentServiceConnectProxyConfig is the proxy configuration in a connect-proxy
// ServiceDefinition or response.
type AgentServiceConnectProxyConfig struct {
- DestinationServiceName string
+ DestinationServiceName string `json:",omitempty"`
DestinationServiceID string `json:",omitempty"`
LocalServiceAddress string `json:",omitempty"`
LocalServicePort int `json:",omitempty"`
Config map[string]interface{} `json:",omitempty" bexpr:"-"`
- Upstreams []Upstream
+ Upstreams []Upstream `json:",omitempty"`
+ MeshGateway MeshGatewayConfig `json:",omitempty"`
}
// AgentMember represents a cluster member known to the agent
@@ -157,21 +135,20 @@
// AgentServiceRegistration is used to register a new service
type AgentServiceRegistration struct {
- Kind ServiceKind `json:",omitempty"`
- ID string `json:",omitempty"`
- Name string `json:",omitempty"`
- Tags []string `json:",omitempty"`
- Port int `json:",omitempty"`
- Address string `json:",omitempty"`
- EnableTagOverride bool `json:",omitempty"`
- Meta map[string]string `json:",omitempty"`
- Weights *AgentWeights `json:",omitempty"`
+ Kind ServiceKind `json:",omitempty"`
+ ID string `json:",omitempty"`
+ Name string `json:",omitempty"`
+ Tags []string `json:",omitempty"`
+ Port int `json:",omitempty"`
+ Address string `json:",omitempty"`
+ TaggedAddresses map[string]ServiceAddress `json:",omitempty"`
+ EnableTagOverride bool `json:",omitempty"`
+ Meta map[string]string `json:",omitempty"`
+ Weights *AgentWeights `json:",omitempty"`
Check *AgentServiceCheck
Checks AgentServiceChecks
- // DEPRECATED (ProxyDestination) - remove this field
- ProxyDestination string `json:",omitempty"`
- Proxy *AgentServiceConnectProxyConfig `json:",omitempty"`
- Connect *AgentServiceConnect `json:",omitempty"`
+ Proxy *AgentServiceConnectProxyConfig `json:",omitempty"`
+ Connect *AgentServiceConnect `json:",omitempty"`
}
// AgentCheckRegistration is used to register a new check
@@ -276,12 +253,8 @@
TargetServiceID string
TargetServiceName string
ContentHash string
- // DEPRECATED(managed-proxies) - this struct is re-used for sidecar configs
- // but they don't need ExecMode or Command
- ExecMode ProxyExecMode `json:",omitempty"`
- Command []string `json:",omitempty"`
- Config map[string]interface{} `bexpr:"-"`
- Upstreams []Upstream
+ Config map[string]interface{} `bexpr:"-"`
+ Upstreams []Upstream
}
// Upstream is the response structure for a proxy upstream configuration.
@@ -293,6 +266,7 @@
LocalBindAddress string `json:",omitempty"`
LocalBindPort int `json:",omitempty"`
Config map[string]interface{} `json:",omitempty" bexpr:"-"`
+ MeshGateway MeshGatewayConfig `json:",omitempty"`
}
// Agent can be used to query the Agent endpoints
@@ -815,31 +789,6 @@
return &out, qm, nil
}
-// ConnectProxyConfig gets the configuration for a local managed proxy instance.
-//
-// Note that this uses an unconventional blocking mechanism since it's
-// agent-local state. That means there is no persistent raft index so we block
-// based on object hash instead.
-func (a *Agent) ConnectProxyConfig(proxyServiceID string, q *QueryOptions) (*ConnectProxyConfig, *QueryMeta, error) {
- r := a.c.newRequest("GET", "/v1/agent/connect/proxy/"+proxyServiceID)
- r.setQueryOptions(q)
- rtt, resp, err := requireOK(a.c.doRequest(r))
- if err != nil {
- return nil, nil, err
- }
- defer resp.Body.Close()
-
- qm := &QueryMeta{}
- parseQueryMeta(resp, qm)
- qm.RequestTime = rtt
-
- var out ConnectProxyConfig
- if err := decodeBody(resp, &out); err != nil {
- return nil, nil, err
- }
- return &out, qm, nil
-}
-
// EnableServiceMaintenance toggles service maintenance mode on
// for the given service ID.
func (a *Agent) EnableServiceMaintenance(serviceID, reason string) error {
diff --git a/vendor/github.com/hashicorp/consul/api/api.go b/vendor/github.com/hashicorp/consul/api/api.go
index 11c3a2c..b624b3c 100644
--- a/vendor/github.com/hashicorp/consul/api/api.go
+++ b/vendor/github.com/hashicorp/consul/api/api.go
@@ -143,6 +143,10 @@
// a value from 0 to 5 (inclusive).
RelayFactor uint8
+ // LocalOnly is used in keyring list operation to force the keyring
+ // query to only hit local servers (no WAN traffic).
+ LocalOnly bool
+
// Connect filters prepared query execution to only include Connect-capable
// services. This currently affects prepared query execution.
Connect bool
@@ -655,6 +659,9 @@
if q.RelayFactor != 0 {
r.params.Set("relay-factor", strconv.Itoa(int(q.RelayFactor)))
}
+ if q.LocalOnly {
+ r.params.Set("local-only", fmt.Sprintf("%t", q.LocalOnly))
+ }
if q.Connect {
r.params.Set("connect", "true")
}
diff --git a/vendor/github.com/hashicorp/consul/api/catalog.go b/vendor/github.com/hashicorp/consul/api/catalog.go
index c175c3f..3fb0553 100644
--- a/vendor/github.com/hashicorp/consul/api/catalog.go
+++ b/vendor/github.com/hashicorp/consul/api/catalog.go
@@ -1,5 +1,10 @@
package api
+import (
+ "net"
+ "strconv"
+)
+
type Weights struct {
Passing int
Warning int
@@ -16,6 +21,11 @@
ModifyIndex uint64
}
+type ServiceAddress struct {
+ Address string
+ Port int
+}
+
type CatalogService struct {
ID string
Node string
@@ -26,17 +36,16 @@
ServiceID string
ServiceName string
ServiceAddress string
+ ServiceTaggedAddresses map[string]ServiceAddress
ServiceTags []string
ServiceMeta map[string]string
ServicePort int
ServiceWeights Weights
ServiceEnableTagOverride bool
- // DEPRECATED (ProxyDestination) - remove the next comment!
- // We forgot to ever add ServiceProxyDestination here so no need to deprecate!
- ServiceProxy *AgentServiceConnectProxyConfig
- CreateIndex uint64
- Checks HealthChecks
- ModifyIndex uint64
+ ServiceProxy *AgentServiceConnectProxyConfig
+ CreateIndex uint64
+ Checks HealthChecks
+ ModifyIndex uint64
}
type CatalogNode struct {
@@ -242,3 +251,12 @@
}
return out, qm, nil
}
+
+func ParseServiceAddr(addrPort string) (ServiceAddress, error) {
+ port := 0
+ host, portStr, err := net.SplitHostPort(addrPort)
+ if err == nil {
+ port, err = strconv.Atoi(portStr)
+ }
+ return ServiceAddress{Address: host, Port: port}, err
+}
diff --git a/vendor/github.com/hashicorp/consul/api/config_entry.go b/vendor/github.com/hashicorp/consul/api/config_entry.go
index 0c18963..1588f2e 100644
--- a/vendor/github.com/hashicorp/consul/api/config_entry.go
+++ b/vendor/github.com/hashicorp/consul/api/config_entry.go
@@ -12,8 +12,12 @@
)
const (
- ServiceDefaults string = "service-defaults"
- ProxyDefaults string = "proxy-defaults"
+ ServiceDefaults string = "service-defaults"
+ ProxyDefaults string = "proxy-defaults"
+ ServiceRouter string = "service-router"
+ ServiceSplitter string = "service-splitter"
+ ServiceResolver string = "service-resolver"
+
ProxyConfigGlobal string = "global"
)
@@ -24,10 +28,40 @@
GetModifyIndex() uint64
}
+type MeshGatewayMode string
+
+const (
+ // MeshGatewayModeDefault represents no specific mode and should
+ // be used to indicate that a different layer of the configuration
+ // chain should take precedence
+ MeshGatewayModeDefault MeshGatewayMode = ""
+
+ // MeshGatewayModeNone represents that the Upstream Connect connections
+ // should be direct and not flow through a mesh gateway.
+ MeshGatewayModeNone MeshGatewayMode = "none"
+
+ // MeshGatewayModeLocal represents that the Upstrea Connect connections
+ // should be made to a mesh gateway in the local datacenter. This is
+ MeshGatewayModeLocal MeshGatewayMode = "local"
+
+ // MeshGatewayModeRemote represents that the Upstream Connect connections
+ // should be made to a mesh gateway in a remote datacenter.
+ MeshGatewayModeRemote MeshGatewayMode = "remote"
+)
+
+// MeshGatewayConfig controls how Mesh Gateways are used for upstream Connect
+// services
+type MeshGatewayConfig struct {
+ // Mode is the mode that should be used for the upstream connection.
+ Mode MeshGatewayMode `json:",omitempty"`
+}
+
type ServiceConfigEntry struct {
Kind string
Name string
- Protocol string
+ Protocol string `json:",omitempty"`
+ MeshGateway MeshGatewayConfig `json:",omitempty"`
+ ExternalSNI string `json:",omitempty"`
CreateIndex uint64
ModifyIndex uint64
}
@@ -51,7 +85,8 @@
type ProxyConfigEntry struct {
Kind string
Name string
- Config map[string]interface{}
+ Config map[string]interface{} `json:",omitempty"`
+ MeshGateway MeshGatewayConfig `json:",omitempty"`
CreateIndex uint64
ModifyIndex uint64
}
@@ -80,14 +115,35 @@
func makeConfigEntry(kind, name string) (ConfigEntry, error) {
switch kind {
case ServiceDefaults:
- return &ServiceConfigEntry{Name: name}, nil
+ return &ServiceConfigEntry{Kind: kind, Name: name}, nil
case ProxyDefaults:
- return &ProxyConfigEntry{Name: name}, nil
+ return &ProxyConfigEntry{Kind: kind, Name: name}, nil
+ case ServiceRouter:
+ return &ServiceRouterConfigEntry{Kind: kind, Name: name}, nil
+ case ServiceSplitter:
+ return &ServiceSplitterConfigEntry{Kind: kind, Name: name}, nil
+ case ServiceResolver:
+ return &ServiceResolverConfigEntry{Kind: kind, Name: name}, nil
default:
return nil, fmt.Errorf("invalid config entry kind: %s", kind)
}
}
+func MakeConfigEntry(kind, name string) (ConfigEntry, error) {
+ return makeConfigEntry(kind, name)
+}
+
+// DecodeConfigEntry will decode the result of using json.Unmarshal of a config
+// entry into a map[string]interface{}.
+//
+// Important caveats:
+//
+// - This will NOT work if the map[string]interface{} was produced using HCL
+// decoding as that requires more extensive parsing to work around the issues
+// with map[string][]interface{} that arise.
+//
+// - This will only decode fields using their camel case json field
+// representations.
func DecodeConfigEntry(raw map[string]interface{}) (ConfigEntry, error) {
var entry ConfigEntry
@@ -132,7 +188,19 @@
return DecodeConfigEntry(raw)
}
-// Config can be used to query the Config endpoints
+func decodeConfigEntrySlice(raw []map[string]interface{}) ([]ConfigEntry, error) {
+ var entries []ConfigEntry
+ for _, rawEntry := range raw {
+ entry, err := DecodeConfigEntry(rawEntry)
+ if err != nil {
+ return nil, err
+ }
+ entries = append(entries, entry)
+ }
+ return entries, nil
+}
+
+// ConfigEntries can be used to query the Config endpoints
type ConfigEntries struct {
c *Client
}
@@ -195,13 +263,9 @@
return nil, nil, err
}
- var entries []ConfigEntry
- for _, rawEntry := range raw {
- entry, err := DecodeConfigEntry(rawEntry)
- if err != nil {
- return nil, nil, err
- }
- entries = append(entries, entry)
+ entries, err := decodeConfigEntrySlice(raw)
+ if err != nil {
+ return nil, nil, err
}
return entries, qm, nil
diff --git a/vendor/github.com/hashicorp/consul/api/config_entry_discoverychain.go b/vendor/github.com/hashicorp/consul/api/config_entry_discoverychain.go
new file mode 100644
index 0000000..77acfbd
--- /dev/null
+++ b/vendor/github.com/hashicorp/consul/api/config_entry_discoverychain.go
@@ -0,0 +1,200 @@
+package api
+
+import (
+ "encoding/json"
+ "time"
+)
+
+type ServiceRouterConfigEntry struct {
+ Kind string
+ Name string
+
+ Routes []ServiceRoute `json:",omitempty"`
+
+ CreateIndex uint64
+ ModifyIndex uint64
+}
+
+func (e *ServiceRouterConfigEntry) GetKind() string { return e.Kind }
+func (e *ServiceRouterConfigEntry) GetName() string { return e.Name }
+func (e *ServiceRouterConfigEntry) GetCreateIndex() uint64 { return e.CreateIndex }
+func (e *ServiceRouterConfigEntry) GetModifyIndex() uint64 { return e.ModifyIndex }
+
+type ServiceRoute struct {
+ Match *ServiceRouteMatch `json:",omitempty"`
+ Destination *ServiceRouteDestination `json:",omitempty"`
+}
+
+type ServiceRouteMatch struct {
+ HTTP *ServiceRouteHTTPMatch `json:",omitempty"`
+}
+
+type ServiceRouteHTTPMatch struct {
+ PathExact string `json:",omitempty"`
+ PathPrefix string `json:",omitempty"`
+ PathRegex string `json:",omitempty"`
+
+ Header []ServiceRouteHTTPMatchHeader `json:",omitempty"`
+ QueryParam []ServiceRouteHTTPMatchQueryParam `json:",omitempty"`
+ Methods []string `json:",omitempty"`
+}
+
+type ServiceRouteHTTPMatchHeader struct {
+ Name string
+ Present bool `json:",omitempty"`
+ Exact string `json:",omitempty"`
+ Prefix string `json:",omitempty"`
+ Suffix string `json:",omitempty"`
+ Regex string `json:",omitempty"`
+ Invert bool `json:",omitempty"`
+}
+
+type ServiceRouteHTTPMatchQueryParam struct {
+ Name string
+ Present bool `json:",omitempty"`
+ Exact string `json:",omitempty"`
+ Regex string `json:",omitempty"`
+}
+
+type ServiceRouteDestination struct {
+ Service string `json:",omitempty"`
+ ServiceSubset string `json:",omitempty"`
+ Namespace string `json:",omitempty"`
+ PrefixRewrite string `json:",omitempty"`
+ RequestTimeout time.Duration `json:",omitempty"`
+ NumRetries uint32 `json:",omitempty"`
+ RetryOnConnectFailure bool `json:",omitempty"`
+ RetryOnStatusCodes []uint32 `json:",omitempty"`
+}
+
+func (e *ServiceRouteDestination) MarshalJSON() ([]byte, error) {
+ type Alias ServiceRouteDestination
+ exported := &struct {
+ RequestTimeout string `json:",omitempty"`
+ *Alias
+ }{
+ RequestTimeout: e.RequestTimeout.String(),
+ Alias: (*Alias)(e),
+ }
+ if e.RequestTimeout == 0 {
+ exported.RequestTimeout = ""
+ }
+
+ return json.Marshal(exported)
+}
+
+func (e *ServiceRouteDestination) UnmarshalJSON(data []byte) error {
+ type Alias ServiceRouteDestination
+ aux := &struct {
+ RequestTimeout string
+ *Alias
+ }{
+ Alias: (*Alias)(e),
+ }
+ if err := json.Unmarshal(data, &aux); err != nil {
+ return err
+ }
+ var err error
+ if aux.RequestTimeout != "" {
+ if e.RequestTimeout, err = time.ParseDuration(aux.RequestTimeout); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+type ServiceSplitterConfigEntry struct {
+ Kind string
+ Name string
+
+ Splits []ServiceSplit `json:",omitempty"`
+
+ CreateIndex uint64
+ ModifyIndex uint64
+}
+
+func (e *ServiceSplitterConfigEntry) GetKind() string { return e.Kind }
+func (e *ServiceSplitterConfigEntry) GetName() string { return e.Name }
+func (e *ServiceSplitterConfigEntry) GetCreateIndex() uint64 { return e.CreateIndex }
+func (e *ServiceSplitterConfigEntry) GetModifyIndex() uint64 { return e.ModifyIndex }
+
+type ServiceSplit struct {
+ Weight float32
+ Service string `json:",omitempty"`
+ ServiceSubset string `json:",omitempty"`
+ Namespace string `json:",omitempty"`
+}
+
+type ServiceResolverConfigEntry struct {
+ Kind string
+ Name string
+
+ DefaultSubset string `json:",omitempty"`
+ Subsets map[string]ServiceResolverSubset `json:",omitempty"`
+ Redirect *ServiceResolverRedirect `json:",omitempty"`
+ Failover map[string]ServiceResolverFailover `json:",omitempty"`
+ ConnectTimeout time.Duration `json:",omitempty"`
+
+ CreateIndex uint64
+ ModifyIndex uint64
+}
+
+func (e *ServiceResolverConfigEntry) MarshalJSON() ([]byte, error) {
+ type Alias ServiceResolverConfigEntry
+ exported := &struct {
+ ConnectTimeout string `json:",omitempty"`
+ *Alias
+ }{
+ ConnectTimeout: e.ConnectTimeout.String(),
+ Alias: (*Alias)(e),
+ }
+ if e.ConnectTimeout == 0 {
+ exported.ConnectTimeout = ""
+ }
+
+ return json.Marshal(exported)
+}
+
+func (e *ServiceResolverConfigEntry) UnmarshalJSON(data []byte) error {
+ type Alias ServiceResolverConfigEntry
+ aux := &struct {
+ ConnectTimeout string
+ *Alias
+ }{
+ Alias: (*Alias)(e),
+ }
+ if err := json.Unmarshal(data, &aux); err != nil {
+ return err
+ }
+ var err error
+ if aux.ConnectTimeout != "" {
+ if e.ConnectTimeout, err = time.ParseDuration(aux.ConnectTimeout); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (e *ServiceResolverConfigEntry) GetKind() string { return e.Kind }
+func (e *ServiceResolverConfigEntry) GetName() string { return e.Name }
+func (e *ServiceResolverConfigEntry) GetCreateIndex() uint64 { return e.CreateIndex }
+func (e *ServiceResolverConfigEntry) GetModifyIndex() uint64 { return e.ModifyIndex }
+
+type ServiceResolverSubset struct {
+ Filter string `json:",omitempty"`
+ OnlyPassing bool `json:",omitempty"`
+}
+
+type ServiceResolverRedirect struct {
+ Service string `json:",omitempty"`
+ ServiceSubset string `json:",omitempty"`
+ Namespace string `json:",omitempty"`
+ Datacenter string `json:",omitempty"`
+}
+
+type ServiceResolverFailover struct {
+ Service string `json:",omitempty"`
+ ServiceSubset string `json:",omitempty"`
+ Namespace string `json:",omitempty"`
+ Datacenters []string `json:",omitempty"`
+}
diff --git a/vendor/github.com/hashicorp/consul/api/connect_intention.go b/vendor/github.com/hashicorp/consul/api/connect_intention.go
index a996c03..d25cb84 100644
--- a/vendor/github.com/hashicorp/consul/api/connect_intention.go
+++ b/vendor/github.com/hashicorp/consul/api/connect_intention.go
@@ -54,6 +54,13 @@
// or modified.
CreatedAt, UpdatedAt time.Time
+ // Hash of the contents of the intention
+ //
+ // This is needed mainly for replication purposes. When replicating from
+ // one DC to another keeping the content Hash will allow us to detect
+ // content changes more efficiently than checking every single field
+ Hash []byte
+
CreateIndex uint64
ModifyIndex uint64
}
diff --git a/vendor/github.com/hashicorp/consul/api/discovery_chain.go b/vendor/github.com/hashicorp/consul/api/discovery_chain.go
new file mode 100644
index 0000000..407a3b0
--- /dev/null
+++ b/vendor/github.com/hashicorp/consul/api/discovery_chain.go
@@ -0,0 +1,230 @@
+package api
+
+import (
+ "encoding/json"
+ "fmt"
+ "time"
+)
+
+// DiscoveryChain can be used to query the discovery-chain endpoints
+type DiscoveryChain struct {
+ c *Client
+}
+
+// DiscoveryChain returns a handle to the discovery-chain endpoints
+func (c *Client) DiscoveryChain() *DiscoveryChain {
+ return &DiscoveryChain{c}
+}
+
+func (d *DiscoveryChain) Get(name string, opts *DiscoveryChainOptions, q *QueryOptions) (*DiscoveryChainResponse, *QueryMeta, error) {
+ if name == "" {
+ return nil, nil, fmt.Errorf("Name parameter must not be empty")
+ }
+
+ method := "GET"
+ if opts != nil && opts.requiresPOST() {
+ method = "POST"
+ }
+
+ r := d.c.newRequest(method, fmt.Sprintf("/v1/discovery-chain/%s", name))
+ r.setQueryOptions(q)
+
+ if opts != nil {
+ if opts.EvaluateInDatacenter != "" {
+ r.params.Set("compile-dc", opts.EvaluateInDatacenter)
+ }
+ // TODO(namespaces): handle possible EvaluateInNamespace here
+ }
+
+ if method == "POST" {
+ r.obj = opts
+ }
+
+ rtt, resp, err := requireOK(d.c.doRequest(r))
+ if err != nil {
+ return nil, nil, err
+ }
+ defer resp.Body.Close()
+
+ qm := &QueryMeta{}
+ parseQueryMeta(resp, qm)
+ qm.RequestTime = rtt
+
+ var out DiscoveryChainResponse
+
+ if err := decodeBody(resp, &out); err != nil {
+ return nil, nil, err
+ }
+
+ return &out, qm, nil
+}
+
+type DiscoveryChainOptions struct {
+ EvaluateInDatacenter string `json:"-"`
+
+ // OverrideMeshGateway allows for the mesh gateway setting to be overridden
+ // for any resolver in the compiled chain.
+ OverrideMeshGateway MeshGatewayConfig `json:",omitempty"`
+
+ // OverrideProtocol allows for the final protocol for the chain to be
+ // altered.
+ //
+ // - If the chain ordinarily would be TCP and an L7 protocol is passed here
+ // the chain will not include Routers or Splitters.
+ //
+ // - If the chain ordinarily would be L7 and TCP is passed here the chain
+ // will not include Routers or Splitters.
+ OverrideProtocol string `json:",omitempty"`
+
+ // OverrideConnectTimeout allows for the ConnectTimeout setting to be
+ // overridden for any resolver in the compiled chain.
+ OverrideConnectTimeout time.Duration `json:",omitempty"`
+}
+
+func (o *DiscoveryChainOptions) requiresPOST() bool {
+ if o == nil {
+ return false
+ }
+ return o.OverrideMeshGateway.Mode != "" ||
+ o.OverrideProtocol != "" ||
+ o.OverrideConnectTimeout != 0
+}
+
+type DiscoveryChainResponse struct {
+ Chain *CompiledDiscoveryChain
+}
+
+type CompiledDiscoveryChain struct {
+ ServiceName string
+ Namespace string
+ Datacenter string
+
+ // CustomizationHash is a unique hash of any data that affects the
+ // compilation of the discovery chain other than config entries or the
+ // name/namespace/datacenter evaluation criteria.
+ //
+ // If set, this value should be used to prefix/suffix any generated load
+ // balancer data plane objects to avoid sharing customized and
+ // non-customized versions.
+ CustomizationHash string
+
+ // Protocol is the overall protocol shared by everything in the chain.
+ Protocol string
+
+ // StartNode is the first key into the Nodes map that should be followed
+ // when walking the discovery chain.
+ StartNode string
+
+ // Nodes contains all nodes available for traversal in the chain keyed by a
+ // unique name. You can walk this by starting with StartNode.
+ //
+ // NOTE: The names should be treated as opaque values and are only
+ // guaranteed to be consistent within a single compilation.
+ Nodes map[string]*DiscoveryGraphNode
+
+ // Targets is a list of all targets used in this chain.
+ //
+ // NOTE: The names should be treated as opaque values and are only
+ // guaranteed to be consistent within a single compilation.
+ Targets map[string]*DiscoveryTarget
+}
+
+const (
+ DiscoveryGraphNodeTypeRouter = "router"
+ DiscoveryGraphNodeTypeSplitter = "splitter"
+ DiscoveryGraphNodeTypeResolver = "resolver"
+)
+
+// DiscoveryGraphNode is a single node in the compiled discovery chain.
+type DiscoveryGraphNode struct {
+ Type string
+ Name string // this is NOT necessarily a service
+
+ // fields for Type==router
+ Routes []*DiscoveryRoute
+
+ // fields for Type==splitter
+ Splits []*DiscoverySplit
+
+ // fields for Type==resolver
+ Resolver *DiscoveryResolver
+}
+
+// compiled form of ServiceRoute
+type DiscoveryRoute struct {
+ Definition *ServiceRoute
+ NextNode string
+}
+
+// compiled form of ServiceSplit
+type DiscoverySplit struct {
+ Weight float32
+ NextNode string
+}
+
+// compiled form of ServiceResolverConfigEntry
+type DiscoveryResolver struct {
+ Default bool
+ ConnectTimeout time.Duration
+ Target string
+ Failover *DiscoveryFailover
+}
+
+func (r *DiscoveryResolver) MarshalJSON() ([]byte, error) {
+ type Alias DiscoveryResolver
+ exported := &struct {
+ ConnectTimeout string `json:",omitempty"`
+ *Alias
+ }{
+ ConnectTimeout: r.ConnectTimeout.String(),
+ Alias: (*Alias)(r),
+ }
+ if r.ConnectTimeout == 0 {
+ exported.ConnectTimeout = ""
+ }
+
+ return json.Marshal(exported)
+}
+
+func (r *DiscoveryResolver) UnmarshalJSON(data []byte) error {
+ type Alias DiscoveryResolver
+ aux := &struct {
+ ConnectTimeout string
+ *Alias
+ }{
+ Alias: (*Alias)(r),
+ }
+ if err := json.Unmarshal(data, &aux); err != nil {
+ return err
+ }
+ var err error
+ if aux.ConnectTimeout != "" {
+ if r.ConnectTimeout, err = time.ParseDuration(aux.ConnectTimeout); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// compiled form of ServiceResolverFailover
+type DiscoveryFailover struct {
+ Targets []string
+}
+
+// DiscoveryTarget represents all of the inputs necessary to use a resolver
+// config entry to execute a catalog query to generate a list of service
+// instances during discovery.
+type DiscoveryTarget struct {
+ ID string
+
+ Service string
+ ServiceSubset string
+ Namespace string
+ Datacenter string
+
+ MeshGateway MeshGatewayConfig
+ Subset ServiceResolverSubset
+ External bool
+ SNI string
+ Name string
+}
diff --git a/vendor/github.com/hashicorp/consul/api/go.mod b/vendor/github.com/hashicorp/consul/api/go.mod
index e198218..78fe8a3 100644
--- a/vendor/github.com/hashicorp/consul/api/go.mod
+++ b/vendor/github.com/hashicorp/consul/api/go.mod
@@ -5,7 +5,7 @@
replace github.com/hashicorp/consul/sdk => ../sdk
require (
- github.com/hashicorp/consul/sdk v0.1.1
+ github.com/hashicorp/consul/sdk v0.2.0
github.com/hashicorp/go-cleanhttp v0.5.1
github.com/hashicorp/go-rootcerts v1.0.0
github.com/hashicorp/go-uuid v1.0.1
diff --git a/vendor/github.com/hashicorp/consul/api/go.sum b/vendor/github.com/hashicorp/consul/api/go.sum
index 372ebc1..01591f9 100644
--- a/vendor/github.com/hashicorp/consul/api/go.sum
+++ b/vendor/github.com/hashicorp/consul/api/go.sum
@@ -9,6 +9,8 @@
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/hashicorp/consul/sdk v0.2.0 h1:GWFYFmry/k4b1hEoy7kSkmU8e30GAyI4VZHk0fRxeL4=
+github.com/hashicorp/consul/sdk v0.2.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
diff --git a/vendor/github.com/hashicorp/consul/api/operator_license.go b/vendor/github.com/hashicorp/consul/api/operator_license.go
new file mode 100644
index 0000000..25aa702
--- /dev/null
+++ b/vendor/github.com/hashicorp/consul/api/operator_license.go
@@ -0,0 +1,111 @@
+package api
+
+import (
+ "io/ioutil"
+ "strings"
+ "time"
+)
+
+type License struct {
+ // The unique identifier of the license
+ LicenseID string `json:"license_id"`
+
+ // The customer ID associated with the license
+ CustomerID string `json:"customer_id"`
+
+ // If set, an identifier that should be used to lock the license to a
+ // particular site, cluster, etc.
+ InstallationID string `json:"installation_id"`
+
+ // The time at which the license was issued
+ IssueTime time.Time `json:"issue_time"`
+
+ // The time at which the license starts being valid
+ StartTime time.Time `json:"start_time"`
+
+ // The time after which the license expires
+ ExpirationTime time.Time `json:"expiration_time"`
+
+ // The time at which the license ceases to function and can
+ // no longer be used in any capacity
+ TerminationTime time.Time `json:"termination_time"`
+
+ // The product the license is valid for
+ Product string `json:"product"`
+
+ // License Specific Flags
+ Flags map[string]interface{} `json:"flags"`
+
+ // List of features enabled by the license
+ Features []string `json:"features"`
+}
+
+type LicenseReply struct {
+ Valid bool
+ License *License
+ Warnings []string
+}
+
+func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, error) {
+ var reply LicenseReply
+ if _, err := op.c.query("/v1/operator/license", &reply, q); err != nil {
+ return nil, err
+ } else {
+ return &reply, nil
+ }
+}
+
+func (op *Operator) LicenseGetSigned(q *QueryOptions) (string, error) {
+ r := op.c.newRequest("GET", "/v1/operator/license")
+ r.params.Set("signed", "1")
+ r.setQueryOptions(q)
+ _, resp, err := requireOK(op.c.doRequest(r))
+ if err != nil {
+ return "", err
+ }
+ defer resp.Body.Close()
+
+ data, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return "", err
+ }
+
+ return string(data), nil
+}
+
+// LicenseReset will reset the license to the builtin one if it is still valid.
+// If the builtin license is invalid, the current license stays active.
+func (op *Operator) LicenseReset(opts *WriteOptions) (*LicenseReply, error) {
+ var reply LicenseReply
+ r := op.c.newRequest("DELETE", "/v1/operator/license")
+ r.setWriteOptions(opts)
+ _, resp, err := requireOK(op.c.doRequest(r))
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ if err := decodeBody(resp, &reply); err != nil {
+ return nil, err
+ }
+
+ return &reply, nil
+}
+
+func (op *Operator) LicensePut(license string, opts *WriteOptions) (*LicenseReply, error) {
+ var reply LicenseReply
+ r := op.c.newRequest("PUT", "/v1/operator/license")
+ r.setWriteOptions(opts)
+ r.body = strings.NewReader(license)
+ _, resp, err := requireOK(op.c.doRequest(r))
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ if err := decodeBody(resp, &reply); err != nil {
+ return nil, err
+ }
+
+ return &reply, nil
+}
diff --git a/vendor/github.com/hashicorp/consul/ui-v2/app/styles/components/notice.scss b/vendor/github.com/hashicorp/consul/ui-v2/app/styles/components/notice.scss
deleted file mode 100644
index 3d0a22d..0000000
--- a/vendor/github.com/hashicorp/consul/ui-v2/app/styles/components/notice.scss
+++ /dev/null
@@ -1,10 +0,0 @@
-@import './notice/index';
-.notice.warning {
- @extend %notice-warning;
-}
-.notice.info {
- @extend %notice-info;
-}
-.notice.policy-management {
- @extend %notice-highlight;
-}
diff --git a/vendor/github.com/hashicorp/consul/ui-v2/app/utils/dom/event-target/event-target-shim/LICENSE b/vendor/github.com/hashicorp/consul/ui-v2/app/utils/dom/event-target/event-target-shim/LICENSE
deleted file mode 100644
index c39e694..0000000
--- a/vendor/github.com/hashicorp/consul/ui-v2/app/utils/dom/event-target/event-target-shim/LICENSE
+++ /dev/null
@@ -1,22 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2015 Toru Nagashima
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
diff --git a/vendor/github.com/hashicorp/consul/ui-v2/lib/block-slots/LICENSE.md b/vendor/github.com/hashicorp/consul/ui-v2/lib/block-slots/LICENSE.md
deleted file mode 100644
index c75ad2a..0000000
--- a/vendor/github.com/hashicorp/consul/ui-v2/lib/block-slots/LICENSE.md
+++ /dev/null
@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2016 Ciena Corporation.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/vendor/github.com/hashicorp/consul/website/LICENSE.md b/vendor/github.com/hashicorp/consul/website/LICENSE.md
deleted file mode 100644
index 3189f43..0000000
--- a/vendor/github.com/hashicorp/consul/website/LICENSE.md
+++ /dev/null
@@ -1,10 +0,0 @@
-# Proprietary License
-
-This license is temporary while a more official one is drafted. However,
-this should make it clear:
-
-The text contents of this website are MPL 2.0 licensed.
-
-The design contents of this website are proprietary and may not be reproduced
-or reused in any way other than to run the website locally. The license for
-the design is owned solely by HashiCorp, Inc.
diff --git a/vendor/github.com/hashicorp/consul/website/source/api/operator/license.html.md b/vendor/github.com/hashicorp/consul/website/source/api/operator/license.html.md
deleted file mode 100644
index 5a52c22..0000000
--- a/vendor/github.com/hashicorp/consul/website/source/api/operator/license.html.md
+++ /dev/null
@@ -1,207 +0,0 @@
----
-layout: api
-page_title: License - Operator - HTTP API
-sidebar_current: api-operator-license
-description: |-
- The /operator/license endpoints allow for setting and retrieving the Consul
- Enterprise License.
----
-
-# License - Operator HTTP API
-
-~> **Enterprise Only!** This API endpoint and functionality only exists in
-Consul Enterprise. This is not present in the open source version of Consul.
-
-The licensing functionality described here is available only in
-[Consul Enterprise](https://www.hashicorp.com/products/consul/) version 1.1.0 and later.
-
-## Getting the Consul License
-
-This endpoint gets information about the current license.
-
-| Method | Path | Produces |
-| ------ | ---------------------------- | -------------------------- |
-| `GET` | `/operator/license` | `application/json` |
-
-The table below shows this endpoint's support for
-[blocking queries](/api/features/blocking.html),
-[consistency modes](/api/features/consistency.html),
-[agent caching](/api/features/caching.html), and
-[required ACLs](/api/index.html#authentication).
-
-| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
-| ---------------- | ----------------- | ------------- | ---------------- |
-| `NO` | `all` | `none` | `none` |
-
-### Parameters
-
-- `dc` `(string: "")` - Specifies the datacenter whose license should be retrieved.
- This will default to the datacenter of the agent serving the HTTP request.
- This is specified as a URL query parameter.
-
-### Sample Request
-
-```text
-$ curl \
- http://127.0.0.1:8500/v1/operator/license
-```
-
-### Sample Response
-
-```json
-{
- "Valid": true,
- "License": {
- "license_id": "2afbf681-0d1a-0649-cb6c-333ec9f0989c",
- "customer_id": "0259271d-8ffc-e85e-0830-c0822c1f5f2b",
- "installation_id": "*",
- "issue_time": "2018-05-21T20:03:35.911567355Z",
- "start_time": "2018-05-21T04:00:00Z",
- "expiration_time": "2019-05-22T03:59:59.999Z",
- "product": "consul",
- "flags": {
- "package": "premium"
- },
- "features": [
- "Automated Backups",
- "Automated Upgrades",
- "Enhanced Read Scalability",
- "Network Segments",
- "Redundancy Zone",
- "Advanced Network Federation"
- ],
- "temporary": false
- },
- "Warnings": []
-}
-```
-
-## Updating the Consul License
-
-This endpoint updates the Consul license and returns some of the
-license contents as well as any warning messages regarding its validity.
-
-| Method | Path | Produces |
-| ------ | ---------------------------- | -------------------------- |
-| `PUT` | `/operator/license` | `application/json` |
-
-The table below shows this endpoint's support for
-[blocking queries](/api/features/blocking.html),
-[consistency modes](/api/features/consistency.html),
-[agent caching](/api/features/caching.html), and
-[required ACLs](/api/index.html#authentication).
-
-| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
-| ---------------- | ----------------- | ------------- | ---------------- |
-| `NO` | `none` | `none` | `operator:write` |
-
-### Parameters
-
-- `dc` `(string: "")` - Specifies the datacenter whose license should be updated.
- This will default to the datacenter of the agent serving the HTTP request.
- This is specified as a URL query parameter.
-
-### Sample Payload
-
-The payload is the raw license blob.
-
-### Sample Request
-
-```text
-$ curl \
- --request PUT \
- --data @consul.license \
- http://127.0.0.1:8500/v1/operator/license
-```
-
-### Sample Response
-
-```json
-{
- "Valid": true,
- "License": {
- "license_id": "2afbf681-0d1a-0649-cb6c-333ec9f0989c",
- "customer_id": "0259271d-8ffc-e85e-0830-c0822c1f5f2b",
- "installation_id": "*",
- "issue_time": "2018-05-21T20:03:35.911567355Z",
- "start_time": "2018-05-21T04:00:00Z",
- "expiration_time": "2019-05-22T03:59:59.999Z",
- "product": "consul",
- "flags": {
- "package": "premium"
- },
- "features": [
- "Automated Backups",
- "Automated Upgrades",
- "Enhanced Read Scalability",
- "Network Segments",
- "Redundancy Zone",
- "Advanced Network Federation"
- ],
- "temporary": false
- },
- "Warnings": []
-}
-```
-
-## Resetting the Consul License
-
-This endpoint resets the Consul license to the license included in the Enterprise binary. If the included license is not valid, the replace will fail.
-
-| Method | Path | Produces |
-| -------- | ---------------------------- | -------------------------- |
-| `DELETE` | `/operator/license` | `application/json` |
-
-The table below shows this endpoint's support for
-[blocking queries](/api/features/blocking.html),
-[consistency modes](/api/features/consistency.html),
-[agent caching](/api/features/caching.html), and
-[required ACLs](/api/index.html#authentication).
-
-| Blocking Queries | Consistency Modes | Agent Caching | ACL Required |
-| ---------------- | ----------------- | ------------- | ---------------- |
-| `NO` | `none` | `none` | `operator:write` |
-
-### Parameters
-
-- `dc` `(string: "")` - Specifies the datacenter whose license should be updated.
- This will default to the datacenter of the agent serving the HTTP request.
- This is specified as a URL query parameter.
-
-### Sample Request
-
-```text
-$ curl \
- --request DELETE \
- http://127.0.0.1:8500/v1/operator/license
-```
-
-### Sample Response
-
-```json
-{
- "Valid": true,
- "License": {
- "license_id": "2afbf681-0d1a-0649-cb6c-333ec9f0989c",
- "customer_id": "0259271d-8ffc-e85e-0830-c0822c1f5f2b",
- "installation_id": "*",
- "issue_time": "2018-05-21T20:03:35.911567355Z",
- "start_time": "2018-05-21T04:00:00Z",
- "expiration_time": "2019-05-22T03:59:59.999Z",
- "product": "consul",
- "flags": {
- "package": "premium"
- },
- "features": [
- "Automated Backups",
- "Automated Upgrades",
- "Enhanced Read Scalability",
- "Network Segments",
- "Redundancy Zone",
- "Advanced Network Federation"
- ],
- "temporary": false
- },
- "Warnings": []
-}
-```
diff --git a/vendor/github.com/hashicorp/consul/website/source/docs/commands/license.html.markdown.erb b/vendor/github.com/hashicorp/consul/website/source/docs/commands/license.html.markdown.erb
deleted file mode 100644
index e397a13..0000000
--- a/vendor/github.com/hashicorp/consul/website/source/docs/commands/license.html.markdown.erb
+++ /dev/null
@@ -1,109 +0,0 @@
----
-layout: "docs"
-page_title: "Commands: License"
-sidebar_current: "docs-commands-license"
-description: >
- The license command provides datacenter-level management of the Consul Enterprise license.
----
-
-# Consul License
-
-Command: `consul license`
-
-<%= enterprise_alert :consul %>
-
-The `license` command provides datacenter-level management of the Consul Enterprise license. This was added in Consul 1.1.0.
-
-If ACLs are enabled then a token with operator privileges may be required in
-order to use this command. Requests are forwarded internally to the leader
-if required, so this can be run from any Consul node in a cluster. See the
-[ACL Guide](https://learn.hashicorp.com/consul/security-networking/production-acls) for more information.
-
-
-```text
-Usage: consul license <subcommand> [options] [args]
-
- This command has subcommands for managing the Consul Enterprise license
- Here are some simple examples, and more detailed examples are
- available in the subcommands or the documentation.
-
- Install a new license from a file:
-
- $ consul license put @consul.license
-
- Install a new license from stdin:
-
- $ consul license put -
-
- Install a new license from a string:
-
- $ consul license put "<license blob>"
-
- Retrieve the current license:
-
- $ consul license get
-
- For more examples, ask for subcommand help or view the documentation.
-
-Subcommands:
- get Get the current license
- put Puts a new license in the datacenter
-```
-
-## put
-
-This command sets the Consul Enterprise license.
-
-Usage: `consul license put [options] LICENSE`
-
-#### API Options
-
-<%= partial "docs/commands/http_api_options_client" %>
-<%= partial "docs/commands/http_api_options_server" %>
-
-The output looks like this:
-
-```
-License is valid
-License ID: 2afbf681-0d1a-0649-cb6c-333ec9f0989c
-Customer ID: 0259271d-8ffc-e85e-0830-c0822c1f5f2b
-Expires At: 2019-05-22 03:59:59.999 +0000 UTC
-Datacenter: *
-Package: premium
-Licensed Features:
- Automated Backups
- Automated Upgrades
- Enhanced Read Scalability
- Network Segments
- Redundancy Zone
- Advanced Network Federation
-```
-
-## get
-
-This command gets the Consul Enterprise license.
-
-Usage: `consul license get [options]`
-
-#### API Options
-
-<%= partial "docs/commands/http_api_options_client" %>
-<%= partial "docs/commands/http_api_options_server" %>
-
-The output looks like this:
-
-```
-License is valid
-License ID: 2afbf681-0d1a-0649-cb6c-333ec9f0989c
-Customer ID: 0259271d-8ffc-e85e-0830-c0822c1f5f2b
-Expires At: 2019-05-22 03:59:59.999 +0000 UTC
-Datacenter: *
-Package: premium
-Licensed Features:
- Automated Backups
- Automated Upgrades
- Enhanced Read Scalability
- Network Segments
- Redundancy Zone
- Advanced Network Federation
-```
\ No newline at end of file
diff --git a/vendor/github.com/hashicorp/go-rootcerts/test-fixtures/capath-with-symlinks/securetrust.pem b/vendor/github.com/hashicorp/go-rootcerts/test-fixtures/capath-with-symlinks/securetrust.pem
deleted file mode 120000
index dda0574..0000000
--- a/vendor/github.com/hashicorp/go-rootcerts/test-fixtures/capath-with-symlinks/securetrust.pem
+++ /dev/null
@@ -1 +0,0 @@
-../capath/securetrust.pem
\ No newline at end of file
diff --git a/vendor/github.com/hashicorp/go-rootcerts/test-fixtures/capath-with-symlinks/thawte.pem b/vendor/github.com/hashicorp/go-rootcerts/test-fixtures/capath-with-symlinks/thawte.pem
deleted file mode 120000
index 37ed4f0..0000000
--- a/vendor/github.com/hashicorp/go-rootcerts/test-fixtures/capath-with-symlinks/thawte.pem
+++ /dev/null
@@ -1 +0,0 @@
-../capath/thawte.pem
\ No newline at end of file
diff --git a/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go b/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go
index 5673773..a86c853 100644
--- a/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go
+++ b/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go
@@ -73,6 +73,9 @@
func (c *LRU) Get(key interface{}) (value interface{}, ok bool) {
if ent, ok := c.items[key]; ok {
c.evictList.MoveToFront(ent)
+ if ent.Value.(*entry) == nil {
+ return nil, false
+ }
return ent.Value.(*entry).value, true
}
return
@@ -142,6 +145,19 @@
return c.evictList.Len()
}
+// Resize changes the cache size.
+func (c *LRU) Resize(size int) (evicted int) {
+ diff := c.Len() - size
+ if diff < 0 {
+ diff = 0
+ }
+ for i := 0; i < diff; i++ {
+ c.removeOldest()
+ }
+ c.size = size
+ return diff
+}
+
// removeOldest removes the oldest item from the cache.
func (c *LRU) removeOldest() {
ent := c.evictList.Back()
diff --git a/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go b/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go
index 74c7077..92d7093 100644
--- a/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go
+++ b/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go
@@ -10,7 +10,7 @@
// updates the "recently used"-ness of the key. #value, isFound
Get(key interface{}) (value interface{}, ok bool)
- // Check if a key exsists in cache without updating the recent-ness.
+ // Checks if a key exists in cache without updating the recent-ness.
Contains(key interface{}) (ok bool)
// Returns key's value without updating the "recently used"-ness of the key.
@@ -31,6 +31,9 @@
// Returns the number of items in the cache.
Len() int
- // Clear all cache entries
+ // Clears all cache entries.
Purge()
+
+ // Resizes cache, returning number evicted
+ Resize(int) int
}
diff --git a/vendor/github.com/hashicorp/serf/ops-misc/debian/copyright b/vendor/github.com/hashicorp/serf/ops-misc/debian/copyright
deleted file mode 100644
index 21a1a1b..0000000
--- a/vendor/github.com/hashicorp/serf/ops-misc/debian/copyright
+++ /dev/null
@@ -1,2 +0,0 @@
-Name: serf
-Copyright: Hashicorp 2013
diff --git a/vendor/github.com/hashicorp/serf/website/LICENSE.md b/vendor/github.com/hashicorp/serf/website/LICENSE.md
deleted file mode 100644
index 3189f43..0000000
--- a/vendor/github.com/hashicorp/serf/website/LICENSE.md
+++ /dev/null
@@ -1,10 +0,0 @@
-# Proprietary License
-
-This license is temporary while a more official one is drafted. However,
-this should make it clear:
-
-The text contents of this website are MPL 2.0 licensed.
-
-The design contents of this website are proprietary and may not be reproduced
-or reused in any way other than to run the website locally. The license for
-the design is owned solely by HashiCorp, Inc.
diff --git a/vendor/github.com/hashicorp/serf/website/source/LICENSE b/vendor/github.com/hashicorp/serf/website/source/LICENSE
deleted file mode 100644
index 36c29d7..0000000
--- a/vendor/github.com/hashicorp/serf/website/source/LICENSE
+++ /dev/null
@@ -1,10 +0,0 @@
-# Proprietary License
-
-This license is temporary while a more official one is drafted. However,
-this should make it clear:
-
-* The text contents of this website are MPL 2.0 licensed.
-
-* The design contents of this website are proprietary and may not be reproduced
- or reused in any way other than to run the Serf website locally. The license
- for the design is owned solely by HashiCorp, Inc.
diff --git a/vendor/github.com/opencord/voltha-lib-go/pkg/kafka/client.go b/vendor/github.com/opencord/voltha-lib-go/pkg/kafka/client.go
old mode 100755
new mode 100644
diff --git a/vendor/github.com/opencord/voltha-lib-go/pkg/kafka/sarama_client.go b/vendor/github.com/opencord/voltha-lib-go/pkg/kafka/sarama_client.go
old mode 100755
new mode 100644
diff --git a/vendor/github.com/opencord/voltha-lib-go/pkg/ponresourcemanager/ponresourcemanager.go b/vendor/github.com/opencord/voltha-lib-go/pkg/ponresourcemanager/ponresourcemanager.go
old mode 100755
new mode 100644
diff --git a/vendor/github.com/pierrec/lz4/.travis.yml b/vendor/github.com/pierrec/lz4/.travis.yml
index 9028ee8..fd6c6db 100644
--- a/vendor/github.com/pierrec/lz4/.travis.yml
+++ b/vendor/github.com/pierrec/lz4/.travis.yml
@@ -2,7 +2,6 @@
env:
- GO111MODULE=off
- - GO111MODULE=on
go:
- 1.9.x
diff --git a/vendor/github.com/pierrec/lz4/README.md b/vendor/github.com/pierrec/lz4/README.md
index 00899fb..be1f52a 100644
--- a/vendor/github.com/pierrec/lz4/README.md
+++ b/vendor/github.com/pierrec/lz4/README.md
@@ -103,3 +103,4 @@
- [@ikkeps](https://github.com/ikkeps)
Special thanks to [@Zariel](https://github.com/Zariel) for his asm implementation of the decoder
+Special thanks to [@klauspost](https://github.com/klauspost) for his work on optimizing the code
diff --git a/vendor/github.com/pierrec/lz4/block.go b/vendor/github.com/pierrec/lz4/block.go
index 90831a4..5755cda 100644
--- a/vendor/github.com/pierrec/lz4/block.go
+++ b/vendor/github.com/pierrec/lz4/block.go
@@ -2,13 +2,14 @@
import (
"encoding/binary"
+ "fmt"
"math/bits"
)
-// blockHash hashes 4 bytes into a value < winSize.
-func blockHash(x uint32) uint32 {
- const hasher uint32 = 2654435761 // Knuth multiplicative hash.
- return x * hasher >> hashShift
+// blockHash hashes the lower 6 bytes into a value < htSize.
+func blockHash(x uint64) uint32 {
+ const prime6bytes = 227718039650203
+ return uint32(((x << (64 - 48)) * prime6bytes) >> (64 - hashLog))
}
// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible.
@@ -46,43 +47,83 @@
// This significantly speeds up incompressible data and usually has very small impact on compresssion.
// bytes to skip = 1 + (bytes since last match >> adaptSkipLog)
const adaptSkipLog = 7
-
sn, dn := len(src)-mfLimit, len(dst)
if sn <= 0 || dn == 0 {
return 0, nil
}
- var si int
+ if len(hashTable) < htSize {
+ return 0, fmt.Errorf("hash table too small, should be at least %d in size", htSize)
+ }
+ // Prove to the compiler the table has at least htSize elements.
+ // The compiler can see that "uint32() >> hashShift" cannot be out of bounds.
+ hashTable = hashTable[:htSize]
+
+ // si: Current position of the search.
+ // anchor: Position of the current literals.
+ var si, anchor int
// Fast scan strategy: the hash table only stores the last 4 bytes sequences.
-
- anchor := si // Position of the current literals.
-
for si < sn {
- // Hash the next 4 bytes (sequence)...
- match := binary.LittleEndian.Uint32(src[si:])
+ // Hash the next 6 bytes (sequence)...
+ match := binary.LittleEndian.Uint64(src[si:])
h := blockHash(match)
+ h2 := blockHash(match >> 8)
+ // We check a match at s, s+1 and s+2 and pick the first one we get.
+ // Checking 3 only requires us to load the source one.
ref := hashTable[h]
+ ref2 := hashTable[h2]
hashTable[h] = si
- if ref >= sn { // Invalid reference (dirty hashtable).
- si += 1 + (si-anchor)>>adaptSkipLog
- continue
- }
+ hashTable[h2] = si + 1
offset := si - ref
+
+ // If offset <= 0 we got an old entry in the hash table.
if offset <= 0 || offset >= winSize || // Out of window.
- match != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches.
- si += 1 + (si-anchor)>>adaptSkipLog
- continue
+ uint32(match) != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches.
+ // No match. Start calculating another hash.
+ // The processor can usually do this out-of-order.
+ h = blockHash(match >> 16)
+ ref = hashTable[h]
+
+ // Check the second match at si+1
+ si += 1
+ offset = si - ref2
+
+ if offset <= 0 || offset >= winSize ||
+ uint32(match>>8) != binary.LittleEndian.Uint32(src[ref2:]) {
+ // No match. Check the third match at si+2
+ si += 1
+ offset = si - ref
+ hashTable[h] = si
+
+ if offset <= 0 || offset >= winSize ||
+ uint32(match>>16) != binary.LittleEndian.Uint32(src[ref:]) {
+ // Skip one extra byte (at si+3) before we check 3 matches again.
+ si += 2 + (si-anchor)>>adaptSkipLog
+ continue
+ }
+ }
}
// Match found.
- // acc = accInit
lLen := si - anchor // Literal length.
+ // We already matched 4 bytes.
+ mLen := 4
- // Encode match length part 1.
- si += minMatch
- mLen := si // Match length has minMatch already.
- // Find the longest match, first looking by batches of 8 bytes.
+ // Extend backwards if we can, reducing literals.
+ tOff := si - offset - 1
+ for lLen > 0 && tOff >= 0 && src[si-1] == src[tOff] {
+ si--
+ tOff--
+ lLen--
+ mLen++
+ }
+
+ // Add the match length, so we continue search at the end.
+ // Use mLen to store the offset base.
+ si, mLen = si+mLen, si+minMatch
+
+ // Find the longest match by looking by batches of 8 bytes.
for si < sn {
x := binary.LittleEndian.Uint64(src[si:]) ^ binary.LittleEndian.Uint64(src[si-offset:])
if x == 0 {
@@ -134,6 +175,13 @@
dst[di] = byte(mLen)
di++
}
+ // Check if we can load next values.
+ if si >= sn {
+ break
+ }
+ // Hash match end-2
+ h = blockHash(binary.LittleEndian.Uint64(src[si-2:]))
+ hashTable[h] = si - 2
}
if anchor == 0 {
@@ -165,6 +213,12 @@
return di, nil
}
+// blockHash hashes 4 bytes into a value < winSize.
+func blockHashHC(x uint32) uint32 {
+ const hasher uint32 = 2654435761 // Knuth multiplicative hash.
+ return x * hasher >> (32 - winSizeLog)
+}
+
// CompressBlockHC compresses the source buffer src into the destination dst
// with max search depth (use 0 or negative value for no max).
//
@@ -199,7 +253,7 @@
for si < sn {
// Hash the next 4 bytes (sequence).
match := binary.LittleEndian.Uint32(src[si:])
- h := blockHash(match)
+ h := blockHashHC(match)
// Follow the chain until out of window and give the longest match.
mLen := 0
@@ -251,7 +305,7 @@
for si, ml := winStart, si+mLen; si < ml; {
match >>= 8
match |= uint32(src[si+3]) << 24
- h := blockHash(match)
+ h := blockHashHC(match)
chainTable[si&winMask] = hashTable[h]
hashTable[h] = si
si++
diff --git a/vendor/github.com/pierrec/lz4/decode_other.go b/vendor/github.com/pierrec/lz4/decode_other.go
index 002519f..919888e 100644
--- a/vendor/github.com/pierrec/lz4/decode_other.go
+++ b/vendor/github.com/pierrec/lz4/decode_other.go
@@ -19,7 +19,7 @@
// Literals.
if lLen := b >> 4; lLen > 0 {
switch {
- case lLen < 0xF && di+18 < len(dst) && si+16 < len(src):
+ case lLen < 0xF && si+16 < len(src):
// Shortcut 1
// if we have enough room in src and dst, and the literals length
// is small enough (0..14) then copy all 16 bytes, even if not all
@@ -34,7 +34,13 @@
mLen += 4
if offset := int(src[si]) | int(src[si+1])<<8; mLen <= offset {
i := di - offset
- copy(dst[di:], dst[i:i+18])
+ end := i + 18
+ if end > len(dst) {
+ // The remaining buffer may not hold 18 bytes.
+ // See https://github.com/pierrec/lz4/issues/51.
+ end = len(dst)
+ }
+ copy(dst[di:], dst[i:end])
si += 2
di += mLen
continue
diff --git a/vendor/github.com/pierrec/lz4/errors.go b/vendor/github.com/pierrec/lz4/errors.go
index 71a5a38..1c45d18 100644
--- a/vendor/github.com/pierrec/lz4/errors.go
+++ b/vendor/github.com/pierrec/lz4/errors.go
@@ -1,6 +1,11 @@
package lz4
-import "errors"
+import (
+ "errors"
+ "fmt"
+ "os"
+ rdebug "runtime/debug"
+)
var (
// ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed
@@ -10,10 +15,16 @@
ErrInvalid = errors.New("lz4: bad magic number")
// ErrBlockDependency is returned when attempting to decompress an archive created with block dependency.
ErrBlockDependency = errors.New("lz4: block dependency not supported")
+ // ErrUnsupportedSeek is returned when attempting to Seek any way but forward from the current position.
+ ErrUnsupportedSeek = errors.New("lz4: can only seek forward from io.SeekCurrent")
)
func recoverBlock(e *error) {
- if recover() != nil && *e == nil {
+ if r := recover(); r != nil && *e == nil {
+ if debugFlag {
+ fmt.Fprintln(os.Stderr, r)
+ rdebug.PrintStack()
+ }
*e = ErrInvalidSourceShortBuffer
}
}
diff --git a/vendor/github.com/pierrec/lz4/go.mod b/vendor/github.com/pierrec/lz4/go.mod
deleted file mode 100644
index 96090c6..0000000
--- a/vendor/github.com/pierrec/lz4/go.mod
+++ /dev/null
@@ -1,12 +0,0 @@
-module github.com/pierrec/lz4
-
-require (
- code.cloudfoundry.org/bytefmt v0.0.0-20180906201452-2aa6f33b730c
- github.com/frankban/quicktest v1.4.0
- github.com/onsi/ginkgo v1.8.0 // indirect
- github.com/onsi/gomega v1.5.0 // indirect
- github.com/pierrec/cmdflag v0.0.2
- github.com/schollz/progressbar/v2 v2.12.1
- golang.org/x/net v0.0.0-20190311183353-d8887717615a // indirect
- golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
-)
diff --git a/vendor/github.com/pierrec/lz4/go.sum b/vendor/github.com/pierrec/lz4/go.sum
deleted file mode 100644
index 4f1dab3..0000000
--- a/vendor/github.com/pierrec/lz4/go.sum
+++ /dev/null
@@ -1,57 +0,0 @@
-code.cloudfoundry.org/bytefmt v0.0.0-20180906201452-2aa6f33b730c h1:VzwteSWGbW9mxXTEkH+kpnao5jbgLynw3hq742juQh8=
-code.cloudfoundry.org/bytefmt v0.0.0-20180906201452-2aa6f33b730c/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/frankban/quicktest v1.4.0 h1:rCSCih1FnSWJEel/eub9wclBSqpF2F/PuvxUWGWnbO8=
-github.com/frankban/quicktest v1.4.0/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
-github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pierrec/cmdflag v0.0.2 h1:ybjGJnPr/aURn2IKWjO49znx9N0DL6YfGsIxN0PYuVY=
-github.com/pierrec/cmdflag v0.0.2/go.mod h1:a3zKGZ3cdQUfxjd0RGMLZr8xI3nvpJOB+m6o/1X5BmU=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/schollz/progressbar/v2 v2.12.1 h1:0Ce7IBClG+s3lxXN1Noqwh7aToKGL5a3mnMfPJqDlv4=
-github.com/schollz/progressbar/v2 v2.12.1/go.mod h1:fBI3onORwtNtwCWJHsrXtjE3QnJOtqIZrvr3rDaF7L0=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/vendor/github.com/pierrec/lz4/lz4.go b/vendor/github.com/pierrec/lz4/lz4.go
index 8c26412..cdbf961 100644
--- a/vendor/github.com/pierrec/lz4/lz4.go
+++ b/vendor/github.com/pierrec/lz4/lz4.go
@@ -30,9 +30,9 @@
// hashLog determines the size of the hash table used to quickly find a previous match position.
// Its value influences the compression speed and memory usage, the lower the faster,
// but at the expense of the compression ratio.
- // 16 seems to be the best compromise.
- hashLog = 16
- hashShift = uint((minMatch * 8) - hashLog)
+ // 16 seems to be the best compromise for fast compression.
+ hashLog = 16
+ htSize = 1 << hashLog
mfLimit = 8 + minMatch // The last match cannot start within the last 12 bytes.
)
diff --git a/vendor/github.com/pierrec/lz4/reader.go b/vendor/github.com/pierrec/lz4/reader.go
index 90e8efe..126b792 100644
--- a/vendor/github.com/pierrec/lz4/reader.go
+++ b/vendor/github.com/pierrec/lz4/reader.go
@@ -25,6 +25,8 @@
data []byte // Uncompressed data.
idx int // Index of unread bytes into data.
checksum xxh32.XXHZero // Frame hash.
+ skip int64 // Bytes to skip before next read.
+ dpos int64 // Position in dest
}
// NewReader returns a new LZ4 frame decoder.
@@ -275,8 +277,20 @@
z.idx = 0
}
+ if z.skip > int64(len(z.data[z.idx:])) {
+ z.skip -= int64(len(z.data[z.idx:]))
+ z.dpos += int64(len(z.data[z.idx:]))
+ z.idx = len(z.data)
+ return 0, nil
+ }
+
+ z.idx += int(z.skip)
+ z.dpos += z.skip
+ z.skip = 0
+
n := copy(buf, z.data[z.idx:])
z.idx += n
+ z.dpos += int64(n)
if debugFlag {
debug("copied %d bytes to input", n)
}
@@ -284,6 +298,20 @@
return n, nil
}
+// Seek implements io.Seeker, but supports seeking forward from the current
+// position only. Any other seek will return an error. Allows skipping output
+// bytes which aren't needed, which in some scenarios is faster than reading
+// and discarding them.
+// Note this may cause future calls to Read() to read 0 bytes if all of the
+// data they would have returned is skipped.
+func (z *Reader) Seek(offset int64, whence int) (int64, error) {
+ if offset < 0 || whence != io.SeekCurrent {
+ return z.dpos + z.skip, ErrUnsupportedSeek
+ }
+ z.skip += offset
+ return z.dpos + z.skip, nil
+}
+
// Reset discards the Reader's state and makes it equivalent to the
// result of its original state from NewReader, but reading from r instead.
// This permits reusing a Reader rather than allocating a new one.
diff --git a/vendor/github.com/pierrec/lz4/writer.go b/vendor/github.com/pierrec/lz4/writer.go
index 804a68c..2cc8d95 100644
--- a/vendor/github.com/pierrec/lz4/writer.go
+++ b/vendor/github.com/pierrec/lz4/writer.go
@@ -46,11 +46,13 @@
}
// Allocate the compressed/uncompressed buffers.
// The compressed buffer cannot exceed the uncompressed one.
- if n := 2 * bSize; cap(z.zdata) < n {
- z.zdata = make([]byte, n, n)
+ if cap(z.zdata) < bSize {
+ // Only allocate if there is not enough capacity.
+ // Allocate both buffers at once.
+ z.zdata = make([]byte, 2*bSize)
}
- z.data = z.zdata[:bSize]
- z.zdata = z.zdata[:cap(z.zdata)][bSize:]
+ z.data = z.zdata[:bSize] // Uncompressed buffer is the first half.
+ z.zdata = z.zdata[:cap(z.zdata)][bSize:] // Compressed buffer is the second half.
z.idx = 0
// Size is optional.
diff --git a/vendor/github.com/rcrowley/go-metrics/.travis.yml b/vendor/github.com/rcrowley/go-metrics/.travis.yml
index 117763e..aead076 100644
--- a/vendor/github.com/rcrowley/go-metrics/.travis.yml
+++ b/vendor/github.com/rcrowley/go-metrics/.travis.yml
@@ -1,13 +1,16 @@
language: go
go:
- - 1.3
- - 1.4
- - 1.5
- - 1.6
- - 1.7
- - 1.8
- - 1.9
+ - "1.3"
+ - "1.4"
+ - "1.5"
+ - "1.6"
+ - "1.7"
+ - "1.8"
+ - "1.9"
+ - "1.10"
+ - "1.11"
+ - "1.12"
script:
- ./validate.sh
diff --git a/vendor/github.com/rcrowley/go-metrics/registry.go b/vendor/github.com/rcrowley/go-metrics/registry.go
index c6115c0..a8e6722 100644
--- a/vendor/github.com/rcrowley/go-metrics/registry.go
+++ b/vendor/github.com/rcrowley/go-metrics/registry.go
@@ -219,9 +219,9 @@
}
func (r *StandardRegistry) registered() []metricKV {
- metrics := make([]metricKV, 0, len(r.metrics))
r.mutex.RLock()
defer r.mutex.RUnlock()
+ metrics := make([]metricKV, 0, len(r.metrics))
for name, i := range r.metrics {
metrics = append(metrics, metricKV{
name: name,
diff --git a/vendor/github.com/rcrowley/go-metrics/validate.sh b/vendor/github.com/rcrowley/go-metrics/validate.sh
old mode 100755
new mode 100644