1 /*-*************************************
3 ***************************************/
4 #include <stdio.h> /* fprintf */
5 #include <stdlib.h> /* malloc, free, qsort */
6 #include <string.h> /* memset */
7 #include <time.h> /* clock */
9 #include "mem.h" /* read */
11 #include "threading.h"
13 #include "zstd_internal.h" /* includes zstd.h */
14 #ifndef ZDICT_STATIC_LINKING_ONLY
15 #define ZDICT_STATIC_LINKING_ONLY
20 /*-*************************************
22 ***************************************/
23 #define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB))
24 #define FASTCOVER_MAX_F 31
25 #define FASTCOVER_MAX_ACCEL 10
26 #define DEFAULT_SPLITPOINT 0.75
28 #define DEFAULT_ACCEL 1
31 /*-*************************************
33 ***************************************/
34 static int g_displayLevel = 2;
35 #define DISPLAY(...) \
37 fprintf(stderr, __VA_ARGS__); \
40 #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \
41 if (displayLevel >= l) { \
42 DISPLAY(__VA_ARGS__); \
43 } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */
44 #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__)
46 #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \
47 if (displayLevel >= l) { \
48 if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \
50 DISPLAY(__VA_ARGS__); \
53 #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__)
54 static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
55 static clock_t g_time = 0;
58 /*-*************************************
60 ***************************************/
61 static const U64 prime6bytes = 227718039650203ULL;
62 static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; }
63 static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
65 static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
66 static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
67 static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
71 * Hash the d-byte value pointed to by p and mod 2^f
73 static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) {
75 return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1);
77 return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1);
81 /*-*************************************
83 ***************************************/
85 unsigned finalize; /* Percentage of training samples used for ZDICT_finalizeDictionary */
86 unsigned skip; /* Number of dmer skipped between each dmer counted in computeFrequency */
90 static const FASTCOVER_accel_t FASTCOVER_defaultAccelParameters[FASTCOVER_MAX_ACCEL+1] = {
91 { 100, 0 }, /* accel = 0, should not happen because accel = 0 defaults to accel = 1 */
92 { 100, 0 }, /* accel = 1 */
93 { 50, 1 }, /* accel = 2 */
94 { 34, 2 }, /* accel = 3 */
95 { 25, 3 }, /* accel = 4 */
96 { 20, 4 }, /* accel = 5 */
97 { 17, 5 }, /* accel = 6 */
98 { 14, 6 }, /* accel = 7 */
99 { 13, 7 }, /* accel = 8 */
100 { 11, 8 }, /* accel = 9 */
101 { 10, 9 }, /* accel = 10 */
105 /*-*************************************
107 ***************************************/
111 const size_t *samplesSizes;
113 size_t nbTrainSamples;
114 size_t nbTestSamples;
119 FASTCOVER_accel_t accelParams;
123 /*-*************************************
125 ***************************************/
127 * Selects the best segment in an epoch.
128 * Segments of are scored according to the function:
130 * Let F(d) be the frequency of all dmers with hash value d.
131 * Let S_i be hash value of the dmer at position i of segment S which has length k.
133 * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1})
135 * Once the dmer with hash value d is in the dictionay we set F(d) = 0.
137 static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx,
138 U32 *freqs, U32 begin, U32 end,
139 ZDICT_cover_params_t parameters,
142 const U32 k = parameters.k;
143 const U32 d = parameters.d;
144 const U32 f = ctx->f;
145 const U32 dmersInK = k - d + 1;
147 /* Try each segment (activeSegment) and save the best (bestSegment) */
148 COVER_segment_t bestSegment = {0, 0, 0};
149 COVER_segment_t activeSegment;
151 /* Reset the activeDmers in the segment */
152 /* The activeSegment starts at the beginning of the epoch. */
153 activeSegment.begin = begin;
154 activeSegment.end = begin;
155 activeSegment.score = 0;
157 /* Slide the activeSegment through the whole epoch.
158 * Save the best segment in bestSegment.
160 while (activeSegment.end < end) {
161 /* Get hash value of current dmer */
162 const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d);
164 /* Add frequency of this index to score if this is the first occurence of index in active segment */
165 if (segmentFreqs[index] == 0) {
166 activeSegment.score += freqs[index];
168 /* Increment end of segment and segmentFreqs*/
169 activeSegment.end += 1;
170 segmentFreqs[index] += 1;
171 /* If the window is now too large, drop the first position */
172 if (activeSegment.end - activeSegment.begin == dmersInK + 1) {
173 /* Get hash value of the dmer to be eliminated from active segment */
174 const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d);
175 segmentFreqs[delIndex] -= 1;
176 /* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */
177 if (segmentFreqs[delIndex] == 0) {
178 activeSegment.score -= freqs[delIndex];
180 /* Increment start of segment */
181 activeSegment.begin += 1;
184 /* If this segment is the best so far save it */
185 if (activeSegment.score > bestSegment.score) {
186 bestSegment = activeSegment;
190 /* Zero out rest of segmentFreqs array */
191 while (activeSegment.begin < end) {
192 const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d);
193 segmentFreqs[delIndex] -= 1;
194 activeSegment.begin += 1;
198 /* Zero the frequency of hash value of each dmer covered by the chosen segment. */
200 for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) {
201 const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, f, d);
210 static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters,
211 size_t maxDictSize, unsigned f,
213 /* k, d, and f are required parameters */
214 if (parameters.d == 0 || parameters.k == 0) {
217 /* d has to be 6 or 8 */
218 if (parameters.d != 6 && parameters.d != 8) {
221 /* k <= maxDictSize */
222 if (parameters.k > maxDictSize) {
226 if (parameters.d > parameters.k) {
229 /* 0 < f <= FASTCOVER_MAX_F*/
230 if (f > FASTCOVER_MAX_F || f == 0) {
233 /* 0 < splitPoint <= 1 */
234 if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) {
237 /* 0 < accel <= 10 */
238 if (accel > 10 || accel == 0) {
246 * Clean up a context initialized with `FASTCOVER_ctx_init()`.
249 FASTCOVER_ctx_destroy(FASTCOVER_ctx_t* ctx)
262 * Calculate for frequency of hash value of each dmer in ctx->samples
265 FASTCOVER_computeFrequency(U32* freqs, const FASTCOVER_ctx_t* ctx)
267 const unsigned f = ctx->f;
268 const unsigned d = ctx->d;
269 const unsigned skip = ctx->accelParams.skip;
270 const unsigned readLength = MAX(d, 8);
272 assert(ctx->nbTrainSamples >= 5);
273 assert(ctx->nbTrainSamples <= ctx->nbSamples);
274 for (i = 0; i < ctx->nbTrainSamples; i++) {
275 size_t start = ctx->offsets[i]; /* start of current dmer */
276 size_t const currSampleEnd = ctx->offsets[i+1];
277 while (start + readLength <= currSampleEnd) {
278 const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d);
280 start = start + skip + 1;
287 * Prepare a context for dictionary building.
288 * The context is only dependent on the parameter `d` and can used multiple
290 * Returns 1 on success or zero on error.
291 * The context must be destroyed with `FASTCOVER_ctx_destroy()`.
294 FASTCOVER_ctx_init(FASTCOVER_ctx_t* ctx,
295 const void* samplesBuffer,
296 const size_t* samplesSizes, unsigned nbSamples,
297 unsigned d, double splitPoint, unsigned f,
298 FASTCOVER_accel_t accelParams)
300 const BYTE* const samples = (const BYTE*)samplesBuffer;
301 const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples);
302 /* Split samples into testing and training sets */
303 const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples;
304 const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples;
305 const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize;
306 const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize;
309 if (totalSamplesSize < MAX(d, sizeof(U64)) ||
310 totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) {
311 DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n",
312 (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20));
316 /* Check if there are at least 5 training samples */
317 if (nbTrainSamples < 5) {
318 DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples);
322 /* Check if there's testing sample */
323 if (nbTestSamples < 1) {
324 DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples);
328 /* Zero the context */
329 memset(ctx, 0, sizeof(*ctx));
330 DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples,
331 (U32)trainingSamplesSize);
332 DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples,
333 (U32)testSamplesSize);
335 ctx->samples = samples;
336 ctx->samplesSizes = samplesSizes;
337 ctx->nbSamples = nbSamples;
338 ctx->nbTrainSamples = nbTrainSamples;
339 ctx->nbTestSamples = nbTestSamples;
340 ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1;
343 ctx->accelParams = accelParams;
345 /* The offsets of each file */
346 ctx->offsets = (size_t*)calloc((nbSamples + 1), sizeof(size_t));
347 if (ctx->offsets == NULL) {
348 DISPLAYLEVEL(1, "Failed to allocate scratch buffers \n");
349 FASTCOVER_ctx_destroy(ctx);
353 /* Fill offsets from the samplesSizes */
356 assert(nbSamples >= 5);
357 for (i = 1; i <= nbSamples; ++i) {
358 ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1];
362 /* Initialize frequency array of size 2^f */
363 ctx->freqs = (U32*)calloc(((U64)1 << f), sizeof(U32));
364 if (ctx->freqs == NULL) {
365 DISPLAYLEVEL(1, "Failed to allocate frequency table \n");
366 FASTCOVER_ctx_destroy(ctx);
370 DISPLAYLEVEL(2, "Computing frequencies\n");
371 FASTCOVER_computeFrequency(ctx->freqs, ctx);
378 * Given the prepared context build the dictionary.
381 FASTCOVER_buildDictionary(const FASTCOVER_ctx_t* ctx,
383 void* dictBuffer, size_t dictBufferCapacity,
384 ZDICT_cover_params_t parameters,
387 BYTE *const dict = (BYTE *)dictBuffer;
388 size_t tail = dictBufferCapacity;
389 /* Divide the data up into epochs of equal size.
390 * We will select at least one segment from each epoch.
392 const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k));
393 const U32 epochSize = (U32)(ctx->nbDmers / epochs);
395 DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs,
397 /* Loop through the epochs until there are no more segments or the dictionary
400 for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) {
401 const U32 epochBegin = (U32)(epoch * epochSize);
402 const U32 epochEnd = epochBegin + epochSize;
404 /* Select a segment */
405 COVER_segment_t segment = FASTCOVER_selectSegment(
406 ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs);
408 /* If the segment covers no dmers, then we are out of content */
409 if (segment.score == 0) {
413 /* Trim the segment if necessary and if it is too small then we are done */
414 segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail);
415 if (segmentSize < parameters.d) {
419 /* We fill the dictionary from the back to allow the best segments to be
420 * referenced with the smallest offsets.
423 memcpy(dict + tail, ctx->samples + segment.begin, segmentSize);
426 (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity));
428 DISPLAYLEVEL(2, "\r%79s\r", "");
434 * Parameters for FASTCOVER_tryParameters().
436 typedef struct FASTCOVER_tryParameters_data_s {
437 const FASTCOVER_ctx_t* ctx;
439 size_t dictBufferCapacity;
440 ZDICT_cover_params_t parameters;
441 } FASTCOVER_tryParameters_data_t;
445 * Tries a set of parameters and updates the COVER_best_t with the results.
446 * This function is thread safe if zstd is compiled with multithreaded support.
447 * It takes its parameters as an *OWNING* opaque pointer to support threading.
449 static void FASTCOVER_tryParameters(void *opaque)
451 /* Save parameters as local variables */
452 FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque;
453 const FASTCOVER_ctx_t *const ctx = data->ctx;
454 const ZDICT_cover_params_t parameters = data->parameters;
455 size_t dictBufferCapacity = data->dictBufferCapacity;
456 size_t totalCompressedSize = ERROR(GENERIC);
457 /* Initialize array to keep track of frequency of dmer within activeSegment */
458 U16* segmentFreqs = (U16 *)calloc(((U64)1 << ctx->f), sizeof(U16));
459 /* Allocate space for hash table, dict, and freqs */
460 BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity);
461 U32 *freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32));
462 if (!segmentFreqs || !dict || !freqs) {
463 DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n");
466 /* Copy the frequencies because we need to modify them */
467 memcpy(freqs, ctx->freqs, ((U64)1 << ctx->f) * sizeof(U32));
468 /* Build the dictionary */
469 { const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity,
470 parameters, segmentFreqs);
471 const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
472 dictBufferCapacity = ZDICT_finalizeDictionary(
473 dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
474 ctx->samples, ctx->samplesSizes, nbFinalizeSamples, parameters.zParams);
475 if (ZDICT_isError(dictBufferCapacity)) {
476 DISPLAYLEVEL(1, "Failed to finalize dictionary\n");
480 /* Check total compressed size */
481 totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes,
482 ctx->samples, ctx->offsets,
483 ctx->nbTrainSamples, ctx->nbSamples,
484 dict, dictBufferCapacity);
486 COVER_best_finish(data->best, totalCompressedSize, parameters, dict,
496 FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams,
497 ZDICT_cover_params_t* coverParams)
499 coverParams->k = fastCoverParams.k;
500 coverParams->d = fastCoverParams.d;
501 coverParams->steps = fastCoverParams.steps;
502 coverParams->nbThreads = fastCoverParams.nbThreads;
503 coverParams->splitPoint = fastCoverParams.splitPoint;
504 coverParams->zParams = fastCoverParams.zParams;
509 FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams,
510 ZDICT_fastCover_params_t* fastCoverParams,
511 unsigned f, unsigned accel)
513 fastCoverParams->k = coverParams.k;
514 fastCoverParams->d = coverParams.d;
515 fastCoverParams->steps = coverParams.steps;
516 fastCoverParams->nbThreads = coverParams.nbThreads;
517 fastCoverParams->splitPoint = coverParams.splitPoint;
518 fastCoverParams->f = f;
519 fastCoverParams->accel = accel;
520 fastCoverParams->zParams = coverParams.zParams;
525 ZDICT_trainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity,
526 const void* samplesBuffer,
527 const size_t* samplesSizes, unsigned nbSamples,
528 ZDICT_fastCover_params_t parameters)
530 BYTE* const dict = (BYTE*)dictBuffer;
532 ZDICT_cover_params_t coverParams;
533 FASTCOVER_accel_t accelParams;
534 /* Initialize global data */
535 g_displayLevel = parameters.zParams.notificationLevel;
536 /* Assign splitPoint and f if not provided */
537 parameters.splitPoint = 1.0;
538 parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f;
539 parameters.accel = parameters.accel == 0 ? DEFAULT_ACCEL : parameters.accel;
540 /* Convert to cover parameter */
541 memset(&coverParams, 0 , sizeof(coverParams));
542 FASTCOVER_convertToCoverParams(parameters, &coverParams);
544 if (!FASTCOVER_checkParameters(coverParams, dictBufferCapacity, parameters.f,
546 DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
547 return ERROR(GENERIC);
549 if (nbSamples == 0) {
550 DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n");
551 return ERROR(GENERIC);
553 if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
554 DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n",
556 return ERROR(dstSize_tooSmall);
558 /* Assign corresponding FASTCOVER_accel_t to accelParams*/
559 accelParams = FASTCOVER_defaultAccelParameters[parameters.accel];
560 /* Initialize context */
561 if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples,
562 coverParams.d, parameters.splitPoint, parameters.f,
564 DISPLAYLEVEL(1, "Failed to initialize context\n");
565 return ERROR(GENERIC);
567 /* Build the dictionary */
568 DISPLAYLEVEL(2, "Building dictionary\n");
570 /* Initialize array to keep track of frequency of dmer within activeSegment */
571 U16* segmentFreqs = (U16 *)calloc(((U64)1 << parameters.f), sizeof(U16));
572 const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer,
573 dictBufferCapacity, coverParams, segmentFreqs);
574 const unsigned nbFinalizeSamples = (unsigned)(ctx.nbTrainSamples * ctx.accelParams.finalize / 100);
575 const size_t dictionarySize = ZDICT_finalizeDictionary(
576 dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail,
577 samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams);
578 if (!ZSTD_isError(dictionarySize)) {
579 DISPLAYLEVEL(2, "Constructed dictionary of size %u\n",
580 (U32)dictionarySize);
582 FASTCOVER_ctx_destroy(&ctx);
584 return dictionarySize;
590 ZDICT_optimizeTrainFromBuffer_fastCover(
591 void* dictBuffer, size_t dictBufferCapacity,
592 const void* samplesBuffer,
593 const size_t* samplesSizes, unsigned nbSamples,
594 ZDICT_fastCover_params_t* parameters)
596 ZDICT_cover_params_t coverParams;
597 FASTCOVER_accel_t accelParams;
599 const unsigned nbThreads = parameters->nbThreads;
600 const double splitPoint =
601 parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint;
602 const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d;
603 const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d;
604 const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k;
605 const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k;
606 const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps;
607 const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1);
608 const unsigned kIterations =
609 (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize);
610 const unsigned f = parameters->f == 0 ? DEFAULT_F : parameters->f;
611 const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel;
612 /* Local variables */
613 const int displayLevel = parameters->zParams.notificationLevel;
614 unsigned iteration = 1;
618 POOL_ctx *pool = NULL;
620 if (splitPoint <= 0 || splitPoint > 1) {
621 LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n");
622 return ERROR(GENERIC);
624 if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) {
625 LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n");
626 return ERROR(GENERIC);
628 if (kMinK < kMaxD || kMaxK < kMinK) {
629 LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n");
630 return ERROR(GENERIC);
632 if (nbSamples == 0) {
633 LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n");
634 return ERROR(GENERIC);
636 if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) {
637 LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n",
639 return ERROR(dstSize_tooSmall);
642 pool = POOL_create(nbThreads, 1);
644 return ERROR(memory_allocation);
648 COVER_best_init(&best);
649 memset(&coverParams, 0 , sizeof(coverParams));
650 FASTCOVER_convertToCoverParams(*parameters, &coverParams);
651 accelParams = FASTCOVER_defaultAccelParameters[accel];
652 /* Turn down global display level to clean up display at level 2 and below */
653 g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1;
654 /* Loop through d first because each new value needs a new context */
655 LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n",
657 for (d = kMinD; d <= kMaxD; d += 2) {
658 /* Initialize the context for this value of d */
660 LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d);
661 if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams)) {
662 LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
663 COVER_best_destroy(&best);
665 return ERROR(GENERIC);
667 /* Loop through k reusing the same context */
668 for (k = kMinK; k <= kMaxK; k += kStepSize) {
669 /* Prepare the arguments */
670 FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc(
671 sizeof(FASTCOVER_tryParameters_data_t));
672 LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k);
674 LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
675 COVER_best_destroy(&best);
676 FASTCOVER_ctx_destroy(&ctx);
678 return ERROR(GENERIC);
682 data->dictBufferCapacity = dictBufferCapacity;
683 data->parameters = coverParams;
684 data->parameters.k = k;
685 data->parameters.d = d;
686 data->parameters.splitPoint = splitPoint;
687 data->parameters.steps = kSteps;
688 data->parameters.zParams.notificationLevel = g_displayLevel;
689 /* Check the parameters */
690 if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity,
691 data->ctx->f, accel)) {
692 DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n");
696 /* Call the function and pass ownership of data to it */
697 COVER_best_start(&best);
699 POOL_add(pool, &FASTCOVER_tryParameters, data);
701 FASTCOVER_tryParameters(data);
704 LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ",
705 (U32)((iteration * 100) / kIterations));
708 COVER_best_wait(&best);
709 FASTCOVER_ctx_destroy(&ctx);
711 LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", "");
712 /* Fill the output buffer and parameters with output of the best parameters */
714 const size_t dictSize = best.dictSize;
715 if (ZSTD_isError(best.compressedSize)) {
716 const size_t compressedSize = best.compressedSize;
717 COVER_best_destroy(&best);
719 return compressedSize;
721 FASTCOVER_convertToFastCoverParams(best.parameters, parameters, f, accel);
722 memcpy(dictBuffer, best.dict, dictSize);
723 COVER_best_destroy(&best);