fse_compress.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625
  1. /* ******************************************************************
  2. * FSE : Finite State Entropy encoder
  3. * Copyright (c) Yann Collet, Facebook, Inc.
  4. *
  5. * You can contact the author at :
  6. * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
  7. * - Public forum : https://groups.google.com/forum/#!forum/lz4c
  8. *
  9. * This source code is licensed under both the BSD-style license (found in the
  10. * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  11. * in the COPYING file in the root directory of this source tree).
  12. * You may select, at your option, one of the above-listed licenses.
  13. ****************************************************************** */
  14. /* **************************************************************
  15. * Includes
  16. ****************************************************************/
  17. #include "../common/compiler.h"
  18. #include "../common/mem.h" /* U32, U16, etc. */
  19. #include "../common/debug.h" /* assert, DEBUGLOG */
  20. #include "hist.h" /* HIST_count_wksp */
  21. #include "../common/bitstream.h"
  22. #define FSE_STATIC_LINKING_ONLY
  23. #include "../common/fse.h"
  24. #include "../common/error_private.h"
  25. #define ZSTD_DEPS_NEED_MALLOC
  26. #define ZSTD_DEPS_NEED_MATH64
  27. #include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
  28. /* **************************************************************
  29. * Error Management
  30. ****************************************************************/
  31. #define FSE_isError ERR_isError
  32. /* **************************************************************
  33. * Templates
  34. ****************************************************************/
  35. /*
  36. designed to be included
  37. for type-specific functions (template emulation in C)
  38. Objective is to write these functions only once, for improved maintenance
  39. */
  40. /* safety checks */
  41. #ifndef FSE_FUNCTION_EXTENSION
  42. # error "FSE_FUNCTION_EXTENSION must be defined"
  43. #endif
  44. #ifndef FSE_FUNCTION_TYPE
  45. # error "FSE_FUNCTION_TYPE must be defined"
  46. #endif
  47. /* Function names */
  48. #define FSE_CAT(X,Y) X##Y
  49. #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
  50. #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
  51. /* Function templates */
  52. /* FSE_buildCTable_wksp() :
  53. * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
  54. * wkspSize should be sized to handle worst case situation, which is `1<<max_tableLog * sizeof(FSE_FUNCTION_TYPE)`
  55. * workSpace must also be properly aligned with FSE_FUNCTION_TYPE requirements
  56. */
  57. size_t FSE_buildCTable_wksp(FSE_CTable* ct,
  58. const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
  59. void* workSpace, size_t wkspSize)
  60. {
  61. U32 const tableSize = 1 << tableLog;
  62. U32 const tableMask = tableSize - 1;
  63. void* const ptr = ct;
  64. U16* const tableU16 = ( (U16*) ptr) + 2;
  65. void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
  66. FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
  67. U32 const step = FSE_TABLESTEP(tableSize);
  68. U32* cumul = (U32*)workSpace;
  69. FSE_FUNCTION_TYPE* tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSymbolValue + 2));
  70. U32 highThreshold = tableSize-1;
  71. if ((size_t)workSpace & 3) return ERROR(GENERIC); /* Must be 4 byte aligned */
  72. if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge);
  73. /* CTable header */
  74. tableU16[-2] = (U16) tableLog;
  75. tableU16[-1] = (U16) maxSymbolValue;
  76. assert(tableLog < 16); /* required for threshold strategy to work */
  77. /* For explanations on how to distribute symbol values over the table :
  78. * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
  79. #ifdef __clang_analyzer__
  80. ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */
  81. #endif
  82. /* symbol start positions */
  83. { U32 u;
  84. cumul[0] = 0;
  85. for (u=1; u <= maxSymbolValue+1; u++) {
  86. if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
  87. cumul[u] = cumul[u-1] + 1;
  88. tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
  89. } else {
  90. cumul[u] = cumul[u-1] + normalizedCounter[u-1];
  91. } }
  92. cumul[maxSymbolValue+1] = tableSize+1;
  93. }
  94. /* Spread symbols */
  95. { U32 position = 0;
  96. U32 symbol;
  97. for (symbol=0; symbol<=maxSymbolValue; symbol++) {
  98. int nbOccurrences;
  99. int const freq = normalizedCounter[symbol];
  100. for (nbOccurrences=0; nbOccurrences<freq; nbOccurrences++) {
  101. tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
  102. position = (position + step) & tableMask;
  103. while (position > highThreshold)
  104. position = (position + step) & tableMask; /* Low proba area */
  105. } }
  106. assert(position==0); /* Must have initialized all positions */
  107. }
  108. /* Build table */
  109. { U32 u; for (u=0; u<tableSize; u++) {
  110. FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
  111. tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
  112. } }
  113. /* Build Symbol Transformation Table */
  114. { unsigned total = 0;
  115. unsigned s;
  116. for (s=0; s<=maxSymbolValue; s++) {
  117. switch (normalizedCounter[s])
  118. {
  119. case 0:
  120. /* filling nonetheless, for compatibility with FSE_getMaxNbBits() */
  121. symbolTT[s].deltaNbBits = ((tableLog+1) << 16) - (1<<tableLog);
  122. break;
  123. case -1:
  124. case 1:
  125. symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
  126. symbolTT[s].deltaFindState = total - 1;
  127. total ++;
  128. break;
  129. default :
  130. {
  131. U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
  132. U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
  133. symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
  134. symbolTT[s].deltaFindState = total - normalizedCounter[s];
  135. total += normalizedCounter[s];
  136. } } } }
  137. #if 0 /* debug : symbol costs */
  138. DEBUGLOG(5, "\n --- table statistics : ");
  139. { U32 symbol;
  140. for (symbol=0; symbol<=maxSymbolValue; symbol++) {
  141. DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f",
  142. symbol, normalizedCounter[symbol],
  143. FSE_getMaxNbBits(symbolTT, symbol),
  144. (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256);
  145. }
  146. }
  147. #endif
  148. return 0;
  149. }
  150. #ifndef FSE_COMMONDEFS_ONLY
  151. /*-**************************************************************
  152. * FSE NCount encoding
  153. ****************************************************************/
  154. size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
  155. {
  156. size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
  157. return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
  158. }
  159. static size_t
  160. FSE_writeNCount_generic (void* header, size_t headerBufferSize,
  161. const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
  162. unsigned writeIsSafe)
  163. {
  164. BYTE* const ostart = (BYTE*) header;
  165. BYTE* out = ostart;
  166. BYTE* const oend = ostart + headerBufferSize;
  167. int nbBits;
  168. const int tableSize = 1 << tableLog;
  169. int remaining;
  170. int threshold;
  171. U32 bitStream = 0;
  172. int bitCount = 0;
  173. unsigned symbol = 0;
  174. unsigned const alphabetSize = maxSymbolValue + 1;
  175. int previousIs0 = 0;
  176. /* Table Size */
  177. bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
  178. bitCount += 4;
  179. /* Init */
  180. remaining = tableSize+1; /* +1 for extra accuracy */
  181. threshold = tableSize;
  182. nbBits = tableLog+1;
  183. while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */
  184. if (previousIs0) {
  185. unsigned start = symbol;
  186. while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++;
  187. if (symbol == alphabetSize) break; /* incorrect distribution */
  188. while (symbol >= start+24) {
  189. start+=24;
  190. bitStream += 0xFFFFU << bitCount;
  191. if ((!writeIsSafe) && (out > oend-2))
  192. return ERROR(dstSize_tooSmall); /* Buffer overflow */
  193. out[0] = (BYTE) bitStream;
  194. out[1] = (BYTE)(bitStream>>8);
  195. out+=2;
  196. bitStream>>=16;
  197. }
  198. while (symbol >= start+3) {
  199. start+=3;
  200. bitStream += 3 << bitCount;
  201. bitCount += 2;
  202. }
  203. bitStream += (symbol-start) << bitCount;
  204. bitCount += 2;
  205. if (bitCount>16) {
  206. if ((!writeIsSafe) && (out > oend - 2))
  207. return ERROR(dstSize_tooSmall); /* Buffer overflow */
  208. out[0] = (BYTE)bitStream;
  209. out[1] = (BYTE)(bitStream>>8);
  210. out += 2;
  211. bitStream >>= 16;
  212. bitCount -= 16;
  213. } }
  214. { int count = normalizedCounter[symbol++];
  215. int const max = (2*threshold-1) - remaining;
  216. remaining -= count < 0 ? -count : count;
  217. count++; /* +1 for extra accuracy */
  218. if (count>=threshold)
  219. count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
  220. bitStream += count << bitCount;
  221. bitCount += nbBits;
  222. bitCount -= (count<max);
  223. previousIs0 = (count==1);
  224. if (remaining<1) return ERROR(GENERIC);
  225. while (remaining<threshold) { nbBits--; threshold>>=1; }
  226. }
  227. if (bitCount>16) {
  228. if ((!writeIsSafe) && (out > oend - 2))
  229. return ERROR(dstSize_tooSmall); /* Buffer overflow */
  230. out[0] = (BYTE)bitStream;
  231. out[1] = (BYTE)(bitStream>>8);
  232. out += 2;
  233. bitStream >>= 16;
  234. bitCount -= 16;
  235. } }
  236. if (remaining != 1)
  237. return ERROR(GENERIC); /* incorrect normalized distribution */
  238. assert(symbol <= alphabetSize);
  239. /* flush remaining bitStream */
  240. if ((!writeIsSafe) && (out > oend - 2))
  241. return ERROR(dstSize_tooSmall); /* Buffer overflow */
  242. out[0] = (BYTE)bitStream;
  243. out[1] = (BYTE)(bitStream>>8);
  244. out+= (bitCount+7) /8;
  245. return (out-ostart);
  246. }
  247. size_t FSE_writeNCount (void* buffer, size_t bufferSize,
  248. const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
  249. {
  250. if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */
  251. if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
  252. if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
  253. return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
  254. return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */);
  255. }
  256. /*-**************************************************************
  257. * FSE Compression Code
  258. ****************************************************************/
  259. FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
  260. {
  261. size_t size;
  262. if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
  263. size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
  264. return (FSE_CTable*)ZSTD_malloc(size);
  265. }
  266. void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
  267. /* provides the minimum logSize to safely represent a distribution */
  268. static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
  269. {
  270. U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1;
  271. U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
  272. U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
  273. assert(srcSize > 1); /* Not supported, RLE should be used instead */
  274. return minBits;
  275. }
  276. unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
  277. {
  278. U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
  279. U32 tableLog = maxTableLog;
  280. U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
  281. assert(srcSize > 1); /* Not supported, RLE should be used instead */
  282. if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
  283. if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
  284. if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
  285. if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
  286. if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
  287. return tableLog;
  288. }
  289. unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
  290. {
  291. return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
  292. }
  293. /* Secondary normalization method.
  294. To be used when primary method fails. */
  295. static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount)
  296. {
  297. short const NOT_YET_ASSIGNED = -2;
  298. U32 s;
  299. U32 distributed = 0;
  300. U32 ToDistribute;
  301. /* Init */
  302. U32 const lowThreshold = (U32)(total >> tableLog);
  303. U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
  304. for (s=0; s<=maxSymbolValue; s++) {
  305. if (count[s] == 0) {
  306. norm[s]=0;
  307. continue;
  308. }
  309. if (count[s] <= lowThreshold) {
  310. norm[s] = lowProbCount;
  311. distributed++;
  312. total -= count[s];
  313. continue;
  314. }
  315. if (count[s] <= lowOne) {
  316. norm[s] = 1;
  317. distributed++;
  318. total -= count[s];
  319. continue;
  320. }
  321. norm[s]=NOT_YET_ASSIGNED;
  322. }
  323. ToDistribute = (1 << tableLog) - distributed;
  324. if (ToDistribute == 0)
  325. return 0;
  326. if ((total / ToDistribute) > lowOne) {
  327. /* risk of rounding to zero */
  328. lowOne = (U32)((total * 3) / (ToDistribute * 2));
  329. for (s=0; s<=maxSymbolValue; s++) {
  330. if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
  331. norm[s] = 1;
  332. distributed++;
  333. total -= count[s];
  334. continue;
  335. } }
  336. ToDistribute = (1 << tableLog) - distributed;
  337. }
  338. if (distributed == maxSymbolValue+1) {
  339. /* all values are pretty poor;
  340. probably incompressible data (should have already been detected);
  341. find max, then give all remaining points to max */
  342. U32 maxV = 0, maxC = 0;
  343. for (s=0; s<=maxSymbolValue; s++)
  344. if (count[s] > maxC) { maxV=s; maxC=count[s]; }
  345. norm[maxV] += (short)ToDistribute;
  346. return 0;
  347. }
  348. if (total == 0) {
  349. /* all of the symbols were low enough for the lowOne or lowThreshold */
  350. for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
  351. if (norm[s] > 0) { ToDistribute--; norm[s]++; }
  352. return 0;
  353. }
  354. { U64 const vStepLog = 62 - tableLog;
  355. U64 const mid = (1ULL << (vStepLog-1)) - 1;
  356. U64 const rStep = ZSTD_div64((((U64)1<<vStepLog) * ToDistribute) + mid, (U32)total); /* scale on remaining */
  357. U64 tmpTotal = mid;
  358. for (s=0; s<=maxSymbolValue; s++) {
  359. if (norm[s]==NOT_YET_ASSIGNED) {
  360. U64 const end = tmpTotal + (count[s] * rStep);
  361. U32 const sStart = (U32)(tmpTotal >> vStepLog);
  362. U32 const sEnd = (U32)(end >> vStepLog);
  363. U32 const weight = sEnd - sStart;
  364. if (weight < 1)
  365. return ERROR(GENERIC);
  366. norm[s] = (short)weight;
  367. tmpTotal = end;
  368. } } }
  369. return 0;
  370. }
  371. size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
  372. const unsigned* count, size_t total,
  373. unsigned maxSymbolValue, unsigned useLowProbCount)
  374. {
  375. /* Sanity checks */
  376. if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
  377. if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */
  378. if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */
  379. if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
  380. { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
  381. short const lowProbCount = useLowProbCount ? -1 : 1;
  382. U64 const scale = 62 - tableLog;
  383. U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */
  384. U64 const vStep = 1ULL<<(scale-20);
  385. int stillToDistribute = 1<<tableLog;
  386. unsigned s;
  387. unsigned largest=0;
  388. short largestP=0;
  389. U32 lowThreshold = (U32)(total >> tableLog);
  390. for (s=0; s<=maxSymbolValue; s++) {
  391. if (count[s] == total) return 0; /* rle special case */
  392. if (count[s] == 0) { normalizedCounter[s]=0; continue; }
  393. if (count[s] <= lowThreshold) {
  394. normalizedCounter[s] = lowProbCount;
  395. stillToDistribute--;
  396. } else {
  397. short proba = (short)((count[s]*step) >> scale);
  398. if (proba<8) {
  399. U64 restToBeat = vStep * rtbTable[proba];
  400. proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
  401. }
  402. if (proba > largestP) { largestP=proba; largest=s; }
  403. normalizedCounter[s] = proba;
  404. stillToDistribute -= proba;
  405. } }
  406. if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
  407. /* corner case, need another normalization method */
  408. size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount);
  409. if (FSE_isError(errorCode)) return errorCode;
  410. }
  411. else normalizedCounter[largest] += (short)stillToDistribute;
  412. }
  413. #if 0
  414. { /* Print Table (debug) */
  415. U32 s;
  416. U32 nTotal = 0;
  417. for (s=0; s<=maxSymbolValue; s++)
  418. RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]);
  419. for (s=0; s<=maxSymbolValue; s++)
  420. nTotal += abs(normalizedCounter[s]);
  421. if (nTotal != (1U<<tableLog))
  422. RAWLOG(2, "Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
  423. getchar();
  424. }
  425. #endif
  426. return tableLog;
  427. }
  428. /* fake FSE_CTable, for raw (uncompressed) input */
  429. size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
  430. {
  431. const unsigned tableSize = 1 << nbBits;
  432. const unsigned tableMask = tableSize - 1;
  433. const unsigned maxSymbolValue = tableMask;
  434. void* const ptr = ct;
  435. U16* const tableU16 = ( (U16*) ptr) + 2;
  436. void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */
  437. FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
  438. unsigned s;
  439. /* Sanity checks */
  440. if (nbBits < 1) return ERROR(GENERIC); /* min size */
  441. /* header */
  442. tableU16[-2] = (U16) nbBits;
  443. tableU16[-1] = (U16) maxSymbolValue;
  444. /* Build table */
  445. for (s=0; s<tableSize; s++)
  446. tableU16[s] = (U16)(tableSize + s);
  447. /* Build Symbol Transformation Table */
  448. { const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
  449. for (s=0; s<=maxSymbolValue; s++) {
  450. symbolTT[s].deltaNbBits = deltaNbBits;
  451. symbolTT[s].deltaFindState = s-1;
  452. } }
  453. return 0;
  454. }
  455. /* fake FSE_CTable, for rle input (always same symbol) */
  456. size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
  457. {
  458. void* ptr = ct;
  459. U16* tableU16 = ( (U16*) ptr) + 2;
  460. void* FSCTptr = (U32*)ptr + 2;
  461. FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
  462. /* header */
  463. tableU16[-2] = (U16) 0;
  464. tableU16[-1] = (U16) symbolValue;
  465. /* Build table */
  466. tableU16[0] = 0;
  467. tableU16[1] = 0; /* just in case */
  468. /* Build Symbol Transformation Table */
  469. symbolTT[symbolValue].deltaNbBits = 0;
  470. symbolTT[symbolValue].deltaFindState = 0;
  471. return 0;
  472. }
  473. static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
  474. const void* src, size_t srcSize,
  475. const FSE_CTable* ct, const unsigned fast)
  476. {
  477. const BYTE* const istart = (const BYTE*) src;
  478. const BYTE* const iend = istart + srcSize;
  479. const BYTE* ip=iend;
  480. BIT_CStream_t bitC;
  481. FSE_CState_t CState1, CState2;
  482. /* init */
  483. if (srcSize <= 2) return 0;
  484. { size_t const initError = BIT_initCStream(&bitC, dst, dstSize);
  485. if (FSE_isError(initError)) return 0; /* not enough space available to write a bitstream */ }
  486. #define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
  487. if (srcSize & 1) {
  488. FSE_initCState2(&CState1, ct, *--ip);
  489. FSE_initCState2(&CState2, ct, *--ip);
  490. FSE_encodeSymbol(&bitC, &CState1, *--ip);
  491. FSE_FLUSHBITS(&bitC);
  492. } else {
  493. FSE_initCState2(&CState2, ct, *--ip);
  494. FSE_initCState2(&CState1, ct, *--ip);
  495. }
  496. /* join to mod 4 */
  497. srcSize -= 2;
  498. if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */
  499. FSE_encodeSymbol(&bitC, &CState2, *--ip);
  500. FSE_encodeSymbol(&bitC, &CState1, *--ip);
  501. FSE_FLUSHBITS(&bitC);
  502. }
  503. /* 2 or 4 encoding per loop */
  504. while ( ip>istart ) {
  505. FSE_encodeSymbol(&bitC, &CState2, *--ip);
  506. if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
  507. FSE_FLUSHBITS(&bitC);
  508. FSE_encodeSymbol(&bitC, &CState1, *--ip);
  509. if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */
  510. FSE_encodeSymbol(&bitC, &CState2, *--ip);
  511. FSE_encodeSymbol(&bitC, &CState1, *--ip);
  512. }
  513. FSE_FLUSHBITS(&bitC);
  514. }
  515. FSE_flushCState(&bitC, &CState2);
  516. FSE_flushCState(&bitC, &CState1);
  517. return BIT_closeCStream(&bitC);
  518. }
  519. size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
  520. const void* src, size_t srcSize,
  521. const FSE_CTable* ct)
  522. {
  523. unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
  524. if (fast)
  525. return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
  526. else
  527. return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
  528. }
  529. size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
  530. #endif /* FSE_COMMONDEFS_ONLY */