zstd_ddict.c 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241
  1. /*
  2. * Copyright (c) Yann Collet, Facebook, Inc.
  3. * All rights reserved.
  4. *
  5. * This source code is licensed under both the BSD-style license (found in the
  6. * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  7. * in the COPYING file in the root directory of this source tree).
  8. * You may select, at your option, one of the above-listed licenses.
  9. */
  10. /* zstd_ddict.c :
  11. * concentrates all logic that needs to know the internals of ZSTD_DDict object */
  12. /*-*******************************************************
  13. * Dependencies
  14. *********************************************************/
  15. #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
  16. #include "../common/cpu.h" /* bmi2 */
  17. #include "../common/mem.h" /* low level memory routines */
  18. #define FSE_STATIC_LINKING_ONLY
  19. #include "../common/fse.h"
  20. #define HUF_STATIC_LINKING_ONLY
  21. #include "../common/huf.h"
  22. #include "zstd_decompress_internal.h"
  23. #include "zstd_ddict.h"
  24. /*-*******************************************************
  25. * Types
  26. *********************************************************/
  27. struct ZSTD_DDict_s {
  28. void* dictBuffer;
  29. const void* dictContent;
  30. size_t dictSize;
  31. ZSTD_entropyDTables_t entropy;
  32. U32 dictID;
  33. U32 entropyPresent;
  34. ZSTD_customMem cMem;
  35. }; /* typedef'd to ZSTD_DDict within "zstd.h" */
  36. const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
  37. {
  38. assert(ddict != NULL);
  39. return ddict->dictContent;
  40. }
  41. size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
  42. {
  43. assert(ddict != NULL);
  44. return ddict->dictSize;
  45. }
  46. void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
  47. {
  48. DEBUGLOG(4, "ZSTD_copyDDictParameters");
  49. assert(dctx != NULL);
  50. assert(ddict != NULL);
  51. dctx->dictID = ddict->dictID;
  52. dctx->prefixStart = ddict->dictContent;
  53. dctx->virtualStart = ddict->dictContent;
  54. dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
  55. dctx->previousDstEnd = dctx->dictEnd;
  56. #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
  57. dctx->dictContentBeginForFuzzing = dctx->prefixStart;
  58. dctx->dictContentEndForFuzzing = dctx->previousDstEnd;
  59. #endif
  60. if (ddict->entropyPresent) {
  61. dctx->litEntropy = 1;
  62. dctx->fseEntropy = 1;
  63. dctx->LLTptr = ddict->entropy.LLTable;
  64. dctx->MLTptr = ddict->entropy.MLTable;
  65. dctx->OFTptr = ddict->entropy.OFTable;
  66. dctx->HUFptr = ddict->entropy.hufTable;
  67. dctx->entropy.rep[0] = ddict->entropy.rep[0];
  68. dctx->entropy.rep[1] = ddict->entropy.rep[1];
  69. dctx->entropy.rep[2] = ddict->entropy.rep[2];
  70. } else {
  71. dctx->litEntropy = 0;
  72. dctx->fseEntropy = 0;
  73. }
  74. }
  75. static size_t
  76. ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
  77. ZSTD_dictContentType_e dictContentType)
  78. {
  79. ddict->dictID = 0;
  80. ddict->entropyPresent = 0;
  81. if (dictContentType == ZSTD_dct_rawContent) return 0;
  82. if (ddict->dictSize < 8) {
  83. if (dictContentType == ZSTD_dct_fullDict)
  84. return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
  85. return 0; /* pure content mode */
  86. }
  87. { U32 const magic = MEM_readLE32(ddict->dictContent);
  88. if (magic != ZSTD_MAGIC_DICTIONARY) {
  89. if (dictContentType == ZSTD_dct_fullDict)
  90. return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
  91. return 0; /* pure content mode */
  92. }
  93. }
  94. ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
  95. /* load entropy tables */
  96. RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy(
  97. &ddict->entropy, ddict->dictContent, ddict->dictSize)),
  98. dictionary_corrupted, "");
  99. ddict->entropyPresent = 1;
  100. return 0;
  101. }
  102. static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
  103. const void* dict, size_t dictSize,
  104. ZSTD_dictLoadMethod_e dictLoadMethod,
  105. ZSTD_dictContentType_e dictContentType)
  106. {
  107. if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
  108. ddict->dictBuffer = NULL;
  109. ddict->dictContent = dict;
  110. if (!dict) dictSize = 0;
  111. } else {
  112. void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem);
  113. ddict->dictBuffer = internalBuffer;
  114. ddict->dictContent = internalBuffer;
  115. if (!internalBuffer) return ERROR(memory_allocation);
  116. ZSTD_memcpy(internalBuffer, dict, dictSize);
  117. }
  118. ddict->dictSize = dictSize;
  119. ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
  120. /* parse dictionary content */
  121. FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
  122. return 0;
  123. }
  124. ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
  125. ZSTD_dictLoadMethod_e dictLoadMethod,
  126. ZSTD_dictContentType_e dictContentType,
  127. ZSTD_customMem customMem)
  128. {
  129. if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
  130. { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem);
  131. if (ddict == NULL) return NULL;
  132. ddict->cMem = customMem;
  133. { size_t const initResult = ZSTD_initDDict_internal(ddict,
  134. dict, dictSize,
  135. dictLoadMethod, dictContentType);
  136. if (ZSTD_isError(initResult)) {
  137. ZSTD_freeDDict(ddict);
  138. return NULL;
  139. } }
  140. return ddict;
  141. }
  142. }
  143. /*! ZSTD_createDDict() :
  144. * Create a digested dictionary, to start decompression without startup delay.
  145. * `dict` content is copied inside DDict.
  146. * Consequently, `dict` can be released after `ZSTD_DDict` creation */
  147. ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
  148. {
  149. ZSTD_customMem const allocator = { NULL, NULL, NULL };
  150. return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
  151. }
  152. /*! ZSTD_createDDict_byReference() :
  153. * Create a digested dictionary, to start decompression without startup delay.
  154. * Dictionary content is simply referenced, it will be accessed during decompression.
  155. * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
  156. ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
  157. {
  158. ZSTD_customMem const allocator = { NULL, NULL, NULL };
  159. return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
  160. }
  161. const ZSTD_DDict* ZSTD_initStaticDDict(
  162. void* sBuffer, size_t sBufferSize,
  163. const void* dict, size_t dictSize,
  164. ZSTD_dictLoadMethod_e dictLoadMethod,
  165. ZSTD_dictContentType_e dictContentType)
  166. {
  167. size_t const neededSpace = sizeof(ZSTD_DDict)
  168. + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
  169. ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
  170. assert(sBuffer != NULL);
  171. assert(dict != NULL);
  172. if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
  173. if (sBufferSize < neededSpace) return NULL;
  174. if (dictLoadMethod == ZSTD_dlm_byCopy) {
  175. ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */
  176. dict = ddict+1;
  177. }
  178. if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
  179. dict, dictSize,
  180. ZSTD_dlm_byRef, dictContentType) ))
  181. return NULL;
  182. return ddict;
  183. }
  184. size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
  185. {
  186. if (ddict==NULL) return 0; /* support free on NULL */
  187. { ZSTD_customMem const cMem = ddict->cMem;
  188. ZSTD_customFree(ddict->dictBuffer, cMem);
  189. ZSTD_customFree(ddict, cMem);
  190. return 0;
  191. }
  192. }
  193. /*! ZSTD_estimateDDictSize() :
  194. * Estimate amount of memory that will be needed to create a dictionary for decompression.
  195. * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
  196. size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
  197. {
  198. return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
  199. }
  200. size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
  201. {
  202. if (ddict==NULL) return 0; /* support sizeof on NULL */
  203. return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
  204. }
  205. /*! ZSTD_getDictID_fromDDict() :
  206. * Provides the dictID of the dictionary loaded into `ddict`.
  207. * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
  208. * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
  209. unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
  210. {
  211. if (ddict==NULL) return 0;
  212. return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
  213. }