1 /* ******************************************* 1 /* ******************************************************************
2 * huff0 huffman codec, 2 * huff0 huffman codec,
3 * part of Finite State Entropy library 3 * part of Finite State Entropy library
4 * Copyright (c) Yann Collet, Facebook, Inc. 4 * Copyright (c) Yann Collet, Facebook, Inc.
5 * 5 *
6 * You can contact the author at : 6 * You can contact the author at :
7 * - Source repository : https://github.com/Cy 7 * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
8 * 8 *
9 * This source code is licensed under both the 9 * This source code is licensed under both the BSD-style license (found in the
10 * LICENSE file in the root directory of this 10 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
11 * in the COPYING file in the root directory o 11 * in the COPYING file in the root directory of this source tree).
12 * You may select, at your option, one of the 12 * You may select, at your option, one of the above-listed licenses.
13 ********************************************** 13 ****************************************************************** */
14 14
15 15
16 #ifndef HUF_H_298734234 16 #ifndef HUF_H_298734234
17 #define HUF_H_298734234 17 #define HUF_H_298734234
18 18
19 /* *** Dependencies *** */ 19 /* *** Dependencies *** */
20 #include "zstd_deps.h" /* size_t */ 20 #include "zstd_deps.h" /* size_t */
21 21
22 22
23 /* *** library symbols visibility *** */ 23 /* *** library symbols visibility *** */
24 /* Note : when linking with -fvisibility=hidde 24 /* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
25 * HUF symbols remain "private" (intern 25 * HUF symbols remain "private" (internal symbols for library only).
26 * Set macro FSE_DLL_EXPORT to 1 if you 26 * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
27 #if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT 27 #if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
28 # define HUF_PUBLIC_API __attribute__ ((visib 28 # define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
29 #elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPO 29 #elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */
30 # define HUF_PUBLIC_API __declspec(dllexport) 30 # define HUF_PUBLIC_API __declspec(dllexport)
31 #elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPO 31 #elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
32 # define HUF_PUBLIC_API __declspec(dllimport) 32 # define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
33 #else 33 #else
34 # define HUF_PUBLIC_API 34 # define HUF_PUBLIC_API
35 #endif 35 #endif
36 36
37 37
38 /* ========================== */ 38 /* ========================== */
39 /* *** simple functions *** */ 39 /* *** simple functions *** */
40 /* ========================== */ 40 /* ========================== */
41 41
42 /* HUF_compress() : 42 /* HUF_compress() :
43 * Compress content from buffer 'src', of siz 43 * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
44 * 'dst' buffer must be already allocated. 44 * 'dst' buffer must be already allocated.
45 * Compression runs faster if `dstCapacity` > 45 * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
46 * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 46 * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
47 * @return : size of compressed data (<= `dstC 47 * @return : size of compressed data (<= `dstCapacity`).
48 * Special values : if return == 0, srcData i 48 * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
49 * if HUF_isError(return), c 49 * if HUF_isError(return), compression failed (more details using HUF_getErrorName())
50 */ 50 */
51 HUF_PUBLIC_API size_t HUF_compress(void* dst, 51 HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
52 const void* src, 52 const void* src, size_t srcSize);
53 53
54 /* HUF_decompress() : 54 /* HUF_decompress() :
55 * Decompress HUF data from buffer 'cSrc', of 55 * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
56 * into already allocated buffer 'dst', of mi 56 * into already allocated buffer 'dst', of minimum size 'dstSize'.
57 * `originalSize` : **must** be the ***exact** 57 * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
58 * Note : in contrast with FSE, HUF_decompres 58 * Note : in contrast with FSE, HUF_decompress can regenerate
59 * RLE (cSrcSize==1) and uncompressed 59 * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
60 * because it knows size to regenerate 60 * because it knows size to regenerate (originalSize).
61 * @return : size of regenerated data (== orig 61 * @return : size of regenerated data (== originalSize),
62 * or an error code, which can be te 62 * or an error code, which can be tested using HUF_isError()
63 */ 63 */
64 HUF_PUBLIC_API size_t HUF_decompress(void* dst 64 HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize,
65 const void* cSr 65 const void* cSrc, size_t cSrcSize);
66 66
67 67
68 /* *** Tool functions *** */ 68 /* *** Tool functions *** */
69 #define HUF_BLOCKSIZE_MAX (128 * 1024) 69 #define HUF_BLOCKSIZE_MAX (128 * 1024) /*< maximum input size for a single block compressed with HUF_compress */
70 HUF_PUBLIC_API size_t HUF_compressBound(size_t 70 HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /*< maximum compressed size (worst case) */
71 71
72 /* Error Management */ 72 /* Error Management */
73 HUF_PUBLIC_API unsigned HUF_isError(size_t 73 HUF_PUBLIC_API unsigned HUF_isError(size_t code); /*< tells if a return value is an error code */
74 HUF_PUBLIC_API const char* HUF_getErrorName(si 74 HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /*< provides error code string (useful for debugging) */
75 75
76 76
77 /* *** Advanced function *** */ 77 /* *** Advanced function *** */
78 78
79 /* HUF_compress2() : 79 /* HUF_compress2() :
80 * Same as HUF_compress(), but offers control 80 * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
81 * `maxSymbolValue` must be <= HUF_SYMBOLVALUE 81 * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
82 * `tableLog` must be `<= HUF_TABLELOG_MAX` . 82 * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
83 HUF_PUBLIC_API size_t HUF_compress2 (void* dst 83 HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
84 const void* src 84 const void* src, size_t srcSize,
85 unsigned maxSym 85 unsigned maxSymbolValue, unsigned tableLog);
86 86
87 /* HUF_compress4X_wksp() : 87 /* HUF_compress4X_wksp() :
88 * Same as HUF_compress2(), but uses external 88 * Same as HUF_compress2(), but uses externally allocated `workSpace`.
89 * `workspace` must be at least as large as HU 89 * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
90 #define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* 90 #define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
91 #define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_ 91 #define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
92 HUF_PUBLIC_API size_t HUF_compress4X_wksp (voi 92 HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
93 const voi 93 const void* src, size_t srcSize,
94 unsigned 94 unsigned maxSymbolValue, unsigned tableLog,
95 void* wor 95 void* workSpace, size_t wkspSize);
96 96
97 #endif /* HUF_H_298734234 */ 97 #endif /* HUF_H_298734234 */
98 98
99 /* ******************************************* 99 /* ******************************************************************
100 * WARNING !! 100 * WARNING !!
101 * The following section contains advanced an 101 * The following section contains advanced and experimental definitions
102 * which shall never be used in the context o 102 * which shall never be used in the context of a dynamic library,
103 * because they are not guaranteed to remain 103 * because they are not guaranteed to remain stable in the future.
104 * Only consider them in association with sta 104 * Only consider them in association with static linking.
105 * ******************************************* 105 * *****************************************************************/
106 #if !defined(HUF_H_HUF_STATIC_LINKING_ONLY) 106 #if !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
107 #define HUF_H_HUF_STATIC_LINKING_ONLY 107 #define HUF_H_HUF_STATIC_LINKING_ONLY
108 108
109 /* *** Dependencies *** */ 109 /* *** Dependencies *** */
110 #include "mem.h" /* U32 */ 110 #include "mem.h" /* U32 */
111 #define FSE_STATIC_LINKING_ONLY 111 #define FSE_STATIC_LINKING_ONLY
112 #include "fse.h" 112 #include "fse.h"
113 113
114 114
115 /* *** Constants *** */ 115 /* *** Constants *** */
116 #define HUF_TABLELOG_MAX 12 /* max r 116 #define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
117 #define HUF_TABLELOG_DEFAULT 11 /* defau 117 #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */
118 #define HUF_SYMBOLVALUE_MAX 255 118 #define HUF_SYMBOLVALUE_MAX 255
119 119
120 #define HUF_TABLELOG_ABSOLUTEMAX 12 /* absol 120 #define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
121 #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEM 121 #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
122 # error "HUF_TABLELOG_MAX is too large !" 122 # error "HUF_TABLELOG_MAX is too large !"
123 #endif 123 #endif
124 124
125 125
126 /* **************************************** 126 /* ****************************************
127 * Static allocation 127 * Static allocation
128 ******************************************/ 128 ******************************************/
129 /* HUF buffer bounds */ 129 /* HUF buffer bounds */
130 #define HUF_CTABLEBOUND 129 130 #define HUF_CTABLEBOUND 129
131 #define HUF_BLOCKBOUND(size) (size + (size>>8) 131 #define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */
132 #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOU 132 #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
133 133
134 /* static allocation of HUF's Compression Tabl 134 /* static allocation of HUF's Compression Table */
135 /* this is a private definition, just exposed 135 /* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */
136 typedef size_t HUF_CElt; /* consider it an i 136 typedef size_t HUF_CElt; /* consider it an incomplete type */
137 #define HUF_CTABLE_SIZE_ST(maxSymbolValue) ( 137 #define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */
138 #define HUF_CTABLE_SIZE(maxSymbolValue) 138 #define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t))
139 #define HUF_CREATE_STATIC_CTABLE(name, maxSymb 139 #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
140 HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbol 140 HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */
141 141
142 /* static allocation of HUF's DTable */ 142 /* static allocation of HUF's DTable */
143 typedef U32 HUF_DTable; 143 typedef U32 HUF_DTable;
144 #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1 144 #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
145 #define HUF_CREATE_STATIC_DTABLEX1(DTable, max 145 #define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \
146 HUF_DTable DTable[HUF_DTABLE_SIZE((max 146 HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
147 #define HUF_CREATE_STATIC_DTABLEX2(DTable, max 147 #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
148 HUF_DTable DTable[HUF_DTABLE_SIZE(maxT 148 HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
149 149
150 150
151 /* **************************************** 151 /* ****************************************
152 * Advanced decompression functions 152 * Advanced decompression functions
153 ******************************************/ 153 ******************************************/
154 size_t HUF_decompress4X1 (void* dst, size_t ds 154 size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder */
155 #ifndef HUF_FORCE_DECOMPRESS_X1 155 #ifndef HUF_FORCE_DECOMPRESS_X1
156 size_t HUF_decompress4X2 (void* dst, size_t ds 156 size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder */
157 #endif 157 #endif
158 158
159 size_t HUF_decompress4X_DCtx (HUF_DTable* dctx 159 size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< decodes RLE and uncompressed */
160 size_t HUF_decompress4X_hufOnly(HUF_DTable* dc 160 size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< considers RLE and uncompressed as errors */
161 size_t HUF_decompress4X_hufOnly_wksp(HUF_DTabl 161 size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< considers RLE and uncompressed as errors */
162 size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx 162 size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder */
163 size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* 163 size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< single-symbol decoder */
164 #ifndef HUF_FORCE_DECOMPRESS_X1 164 #ifndef HUF_FORCE_DECOMPRESS_X1
165 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx 165 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder */
166 size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* 166 size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< double-symbols decoder */
167 #endif 167 #endif
168 168
169 169
170 /* **************************************** 170 /* ****************************************
171 * HUF detailed API 171 * HUF detailed API
172 * ****************************************/ 172 * ****************************************/
173 173
174 /*! HUF_compress() does the following: 174 /*! HUF_compress() does the following:
175 * 1. count symbol occurrence from source[] i 175 * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
176 * 2. (optional) refine tableLog using HUF_op 176 * 2. (optional) refine tableLog using HUF_optimalTableLog()
177 * 3. build Huffman table from count using HU 177 * 3. build Huffman table from count using HUF_buildCTable()
178 * 4. save Huffman table to memory buffer usi 178 * 4. save Huffman table to memory buffer using HUF_writeCTable()
179 * 5. encode the data stream using HUF_compre 179 * 5. encode the data stream using HUF_compress4X_usingCTable()
180 * 180 *
181 * The following API allows targeting specifi 181 * The following API allows targeting specific sub-functions for advanced tasks.
182 * For example, it's possible to compress sev 182 * For example, it's possible to compress several blocks using the same 'CTable',
183 * or to save and regenerate 'CTable' using e 183 * or to save and regenerate 'CTable' using external methods.
184 */ 184 */
185 unsigned HUF_optimalTableLog(unsigned maxTable 185 unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
186 size_t HUF_buildCTable (HUF_CElt* CTable, cons 186 size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
187 size_t HUF_writeCTable (void* dst, size_t maxD 187 size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
188 size_t HUF_writeCTable_wksp(void* dst, size_t 188 size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
189 size_t HUF_compress4X_usingCTable(void* dst, s 189 size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
190 size_t HUF_compress4X_usingCTable_bmi2(void* d 190 size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
191 size_t HUF_estimateCompressedSize(const HUF_CE 191 size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
192 int HUF_validateCTable(const HUF_CElt* CTable, 192 int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
193 193
194 typedef enum { 194 typedef enum {
195 HUF_repeat_none, /*< Cannot use the previo 195 HUF_repeat_none, /*< Cannot use the previous table */
196 HUF_repeat_check, /*< Can use the previous 196 HUF_repeat_check, /*< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
197 HUF_repeat_valid /*< Can use the previous 197 HUF_repeat_valid /*< Can use the previous table and it is assumed to be valid */
198 } HUF_repeat; 198 } HUF_repeat;
199 /* HUF_compress4X_repeat() : 199 /* HUF_compress4X_repeat() :
200 * Same as HUF_compress4X_wksp(), but conside 200 * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
201 * If it uses hufTable it does not modify huf 201 * If it uses hufTable it does not modify hufTable or repeat.
202 * If it doesn't, it sets *repeat = HUF_repea 202 * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
203 * If preferRepeat then the old table will al 203 * If preferRepeat then the old table will always be used if valid.
204 * If suspectUncompressible then some samplin 204 * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
205 size_t HUF_compress4X_repeat(void* dst, size_t 205 size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
206 const void* src, size_t 206 const void* src, size_t srcSize,
207 unsigned maxSymbolValue 207 unsigned maxSymbolValue, unsigned tableLog,
208 void* workSpace, size_t 208 void* workSpace, size_t wkspSize, /*< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
209 HUF_CElt* hufTable, HUF 209 HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
210 210
211 /* HUF_buildCTable_wksp() : 211 /* HUF_buildCTable_wksp() :
212 * Same as HUF_buildCTable(), but using exter 212 * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
213 * `workSpace` must be aligned on 4-bytes boun 213 * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
214 */ 214 */
215 #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_S 215 #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
216 #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_ 216 #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
217 size_t HUF_buildCTable_wksp (HUF_CElt* tree, 217 size_t HUF_buildCTable_wksp (HUF_CElt* tree,
218 const unsigned* count, 218 const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
219 void* workSpace, 219 void* workSpace, size_t wkspSize);
220 220
221 /*! HUF_readStats() : 221 /*! HUF_readStats() :
222 * Read compact Huffman tree, saved by HUF_wr 222 * Read compact Huffman tree, saved by HUF_writeCTable().
223 * `huffWeight` is destination buffer. 223 * `huffWeight` is destination buffer.
224 * @return : size read from `src` , or an erro 224 * @return : size read from `src` , or an error Code .
225 * Note : Needed by HUF_readCTable() and HUF_ 225 * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
226 size_t HUF_readStats(BYTE* huffWeight, size_t 226 size_t HUF_readStats(BYTE* huffWeight, size_t hwSize,
227 U32* rankStats, U32* nbSy 227 U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
228 const void* src, size_t s 228 const void* src, size_t srcSize);
229 229
230 /*! HUF_readStats_wksp() : 230 /*! HUF_readStats_wksp() :
231 * Same as HUF_readStats() but takes an extern 231 * Same as HUF_readStats() but takes an external workspace which must be
232 * 4-byte aligned and its size must be >= HUF_ 232 * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE.
233 * If the CPU has BMI2 support, pass bmi2=1, o 233 * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
234 */ 234 */
235 #define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_ 235 #define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1)
236 #define HUF_READ_STATS_WORKSPACE_SIZE (HUF_REA 236 #define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned))
237 size_t HUF_readStats_wksp(BYTE* huffWeight, si 237 size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
238 U32* rankStats, U32* 238 U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
239 const void* src, siz 239 const void* src, size_t srcSize,
240 void* workspace, siz 240 void* workspace, size_t wkspSize,
241 int bmi2); 241 int bmi2);
242 242
243 /* HUF_readCTable() : 243 /* HUF_readCTable() :
244 * Loading a CTable saved with HUF_writeCTabl 244 * Loading a CTable saved with HUF_writeCTable() */
245 size_t HUF_readCTable (HUF_CElt* CTable, unsig 245 size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights);
246 246
247 /* HUF_getNbBitsFromCTable() : 247 /* HUF_getNbBitsFromCTable() :
248 * Read nbBits from CTable symbolTable, for s 248 * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX
249 * Note 1 : is not inlined, as HUF_CElt defin 249 * Note 1 : is not inlined, as HUF_CElt definition is private */
250 U32 HUF_getNbBitsFromCTable(const HUF_CElt* sy 250 U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue);
251 251
252 /* 252 /*
253 * HUF_decompress() does the following: 253 * HUF_decompress() does the following:
254 * 1. select the decompression algorithm (X1, 254 * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics
255 * 2. build Huffman table from save, using HUF 255 * 2. build Huffman table from save, using HUF_readDTableX?()
256 * 3. decode 1 or 4 segments in parallel using 256 * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable()
257 */ 257 */
258 258
259 /* HUF_selectDecoder() : 259 /* HUF_selectDecoder() :
260 * Tells which decoder is likely to decode fa 260 * Tells which decoder is likely to decode faster,
261 * based on a set of pre-computed metrics. 261 * based on a set of pre-computed metrics.
262 * @return : 0==HUF_decompress4X1, 1==HUF_deco 262 * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
263 * Assumption : 0 < dstSize <= 128 KB */ 263 * Assumption : 0 < dstSize <= 128 KB */
264 U32 HUF_selectDecoder (size_t dstSize, size_t 264 U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
265 265
266 /* 266 /*
267 * The minimum workspace size for the `workSp 267 * The minimum workspace size for the `workSpace` used in
268 * HUF_readDTableX1_wksp() and HUF_readDTable 268 * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp().
269 * 269 *
270 * The space used depends on HUF_TABLELOG_MAX 270 * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when
271 * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when H 271 * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15.
272 * Buffer overflow errors may potentially occ 272 * Buffer overflow errors may potentially occur if code modifications result in
273 * a required workspace size greater than tha 273 * a required workspace size greater than that specified in the following
274 * macro. 274 * macro.
275 */ 275 */
276 #define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 1 276 #define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
277 #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF 277 #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
278 278
279 #ifndef HUF_FORCE_DECOMPRESS_X2 279 #ifndef HUF_FORCE_DECOMPRESS_X2
280 size_t HUF_readDTableX1 (HUF_DTable* DTable, c 280 size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
281 size_t HUF_readDTableX1_wksp (HUF_DTable* DTab 281 size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
282 #endif 282 #endif
283 #ifndef HUF_FORCE_DECOMPRESS_X1 283 #ifndef HUF_FORCE_DECOMPRESS_X1
284 size_t HUF_readDTableX2 (HUF_DTable* DTable, c 284 size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
285 size_t HUF_readDTableX2_wksp (HUF_DTable* DTab 285 size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
286 #endif 286 #endif
287 287
288 size_t HUF_decompress4X_usingDTable(void* dst, 288 size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
289 #ifndef HUF_FORCE_DECOMPRESS_X2 289 #ifndef HUF_FORCE_DECOMPRESS_X2
290 size_t HUF_decompress4X1_usingDTable(void* dst 290 size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
291 #endif 291 #endif
292 #ifndef HUF_FORCE_DECOMPRESS_X1 292 #ifndef HUF_FORCE_DECOMPRESS_X1
293 size_t HUF_decompress4X2_usingDTable(void* dst 293 size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
294 #endif 294 #endif
295 295
296 296
297 /* ====================== */ 297 /* ====================== */
298 /* single stream variants */ 298 /* single stream variants */
299 /* ====================== */ 299 /* ====================== */
300 300
301 size_t HUF_compress1X (void* dst, size_t dstSi 301 size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
302 size_t HUF_compress1X_wksp (void* dst, size_t 302 size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /*< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */
303 size_t HUF_compress1X_usingCTable(void* dst, s 303 size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
304 size_t HUF_compress1X_usingCTable_bmi2(void* d 304 size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
305 /* HUF_compress1X_repeat() : 305 /* HUF_compress1X_repeat() :
306 * Same as HUF_compress1X_wksp(), but conside 306 * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
307 * If it uses hufTable it does not modify huf 307 * If it uses hufTable it does not modify hufTable or repeat.
308 * If it doesn't, it sets *repeat = HUF_repea 308 * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
309 * If preferRepeat then the old table will al 309 * If preferRepeat then the old table will always be used if valid.
310 * If suspectUncompressible then some samplin 310 * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
311 size_t HUF_compress1X_repeat(void* dst, size_t 311 size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
312 const void* src, size_t 312 const void* src, size_t srcSize,
313 unsigned maxSymbolValue 313 unsigned maxSymbolValue, unsigned tableLog,
314 void* workSpace, size_t 314 void* workSpace, size_t wkspSize, /*< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
315 HUF_CElt* hufTable, HUF 315 HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
316 316
317 size_t HUF_decompress1X1 (void* dst, size_t ds 317 size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
318 #ifndef HUF_FORCE_DECOMPRESS_X1 318 #ifndef HUF_FORCE_DECOMPRESS_X1
319 size_t HUF_decompress1X2 (void* dst, size_t ds 319 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
320 #endif 320 #endif
321 321
322 size_t HUF_decompress1X_DCtx (HUF_DTable* dctx 322 size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
323 size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* 323 size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
324 #ifndef HUF_FORCE_DECOMPRESS_X2 324 #ifndef HUF_FORCE_DECOMPRESS_X2
325 size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx 325 size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< single-symbol decoder */
326 size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* 326 size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< single-symbol decoder */
327 #endif 327 #endif
328 #ifndef HUF_FORCE_DECOMPRESS_X1 328 #ifndef HUF_FORCE_DECOMPRESS_X1
329 size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx 329 size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< double-symbols decoder */
330 size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* 330 size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< double-symbols decoder */
331 #endif 331 #endif
332 332
333 size_t HUF_decompress1X_usingDTable(void* dst, 333 size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /*< automatic selection of sing or double symbol decoder, based on DTable */
334 #ifndef HUF_FORCE_DECOMPRESS_X2 334 #ifndef HUF_FORCE_DECOMPRESS_X2
335 size_t HUF_decompress1X1_usingDTable(void* dst 335 size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
336 #endif 336 #endif
337 #ifndef HUF_FORCE_DECOMPRESS_X1 337 #ifndef HUF_FORCE_DECOMPRESS_X1
338 size_t HUF_decompress1X2_usingDTable(void* dst 338 size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
339 #endif 339 #endif
340 340
341 /* BMI2 variants. 341 /* BMI2 variants.
342 * If the CPU has BMI2 support, pass bmi2=1, o 342 * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
343 */ 343 */
344 size_t HUF_decompress1X_usingDTable_bmi2(void* 344 size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
345 #ifndef HUF_FORCE_DECOMPRESS_X2 345 #ifndef HUF_FORCE_DECOMPRESS_X2
346 size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DT 346 size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
347 #endif 347 #endif
348 size_t HUF_decompress4X_usingDTable_bmi2(void* 348 size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
349 size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_ 349 size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
350 #ifndef HUF_FORCE_DECOMPRESS_X2 350 #ifndef HUF_FORCE_DECOMPRESS_X2
351 size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* 351 size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
352 #endif 352 #endif
353 #ifndef HUF_FORCE_DECOMPRESS_X1 353 #ifndef HUF_FORCE_DECOMPRESS_X1
354 size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* 354 size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
355 #endif 355 #endif
356 356
357 #endif /* HUF_STATIC_LINKING_ONLY */ 357 #endif /* HUF_STATIC_LINKING_ONLY */
358 358
359 359
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.