Line data Source code
1 : /*
2 : Zipios -- a small C++ library that provides easy access to .zip files.
3 :
4 : Copyright (C) 2000-2007 Thomas Sondergaard
5 : Copyright (C) 2015-2019 Made to Order Software Corporation
6 :
7 : This library is free software; you can redistribute it and/or
8 : modify it under the terms of the GNU Lesser General Public
9 : License as published by the Free Software Foundation; either
10 : version 2.1 of the License, or (at your option) any later version.
11 :
12 : This library is distributed in the hope that it will be useful,
13 : but WITHOUT ANY WARRANTY; without even the implied warranty of
14 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 : Lesser General Public License for more details.
16 :
17 : You should have received a copy of the GNU Lesser General Public
18 : License along with this library; if not, write to the Free Software
19 : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 : */
21 :
22 : /** \file
23 : * \brief Implementation of zipios::DeflateOutputStreambuf.
24 : *
25 : * This is the counterpart of the zipios::InflateInputStreambuf.
26 : */
27 :
28 : #include "deflateoutputstreambuf.hpp"
29 :
30 : #include "zipios/zipiosexceptions.hpp"
31 :
32 : #include "zipios_common.hpp"
33 :
34 :
35 : namespace zipios
36 : {
37 :
38 : /** \class DeflateOutputStreambuf
39 : * \brief A class to handle stream deflate on the fly.
40 : *
41 : * DeflateOutputStreambuf is an output stream filter, that deflates
42 : * the data that is written to it before it passes it on to the
43 : * output stream it is attached to. Deflation/Inflation is a
44 : * compression/decompression method used in gzip and zip. The zlib
45 : * library is used to perform the actual deflation, this class only
46 : * wraps the functionality in an output stream filter.
47 : */
48 :
49 :
50 : /** \brief Initialize a DeflateOutputStreambuf object.
51 : *
52 : * This function initializes the DeflateOutputStreambuf object to make it
53 : * ready for compressing data using the zlib library.
54 : *
55 : * \param[in,out] outbuf The streambuf to use for output.
56 : */
57 243 : DeflateOutputStreambuf::DeflateOutputStreambuf(std::streambuf *outbuf)
58 : : FilterOutputStreambuf(outbuf)
59 : //, m_overflown_bytes(0) -- auto-init
60 : , m_invec(getBufferSize())
61 : //, m_zs() -- auto-init
62 : //, m_zs_initialized(false) -- auto-init
63 243 : , m_outvec(getBufferSize())
64 : //, m_crc32(0) -- auto-init
65 : {
66 : // NOTICE: It is important that this constructor and the methods it
67 : // calls does not do anything with the output streambuf m_outbuf.
68 : // The reason is that this class can be subclassed, and the
69 : // subclass should get a chance to write to the buffer first.
70 :
71 : // zlib init: (this is done in the class declaration)
72 : //m_zs.zalloc = Z_NULL;
73 : //m_zs.zfree = Z_NULL;
74 : //m_zs.opaque = Z_NULL;
75 243 : }
76 :
77 :
78 : /** \brief Clean up any resources used by this object.
79 : *
80 : * The destructor makes sure that the zlib library is done with all
81 : * the input and output data by calling various flush functions. It
82 : * then makes sure that the remaining data from zlib is printed in
83 : * the output file.
84 : *
85 : * This is similar to calling closeStream() explicitly.
86 : */
87 486 : DeflateOutputStreambuf::~DeflateOutputStreambuf()
88 : {
89 243 : closeStream();
90 243 : }
91 :
92 :
93 : /** \brief Initialize the zlib library.
94 : *
95 : * This method is called in the constructor, so it must not write
96 : * anything to the output streambuf m_outbuf (see notice in
97 : * constructor.)
98 : *
99 : * It will initialize the output stream as required to accept data
100 : * to be compressed using the zlib library. The compression level
101 : * is expected to come from the FileEntry which is about to be
102 : * saved in the file.
103 : *
104 : * \return true if the initialization succeeded, false otherwise.
105 : */
106 93731 : bool DeflateOutputStreambuf::init(FileEntry::CompressionLevel compression_level)
107 : {
108 93731 : if(m_zs_initialized)
109 : {
110 : // This is excluded from the coverage since if we reach this
111 : // line there is an internal error that needs to be fixed.
112 : throw std::logic_error("DeflateOutputStreambuf::init(): initialization function called when the class is already initialized. This is not supported."); // LCOV_EXCL_LINE
113 : }
114 93731 : m_zs_initialized = true;
115 :
116 93731 : int const default_mem_level(8);
117 :
118 93731 : int zlevel(Z_NO_COMPRESSION);
119 93731 : switch(compression_level)
120 : {
121 : case FileEntry::COMPRESSION_LEVEL_DEFAULT:
122 903 : zlevel = Z_DEFAULT_COMPRESSION;
123 903 : break;
124 :
125 : case FileEntry::COMPRESSION_LEVEL_SMALLEST:
126 903 : zlevel = Z_BEST_COMPRESSION;
127 903 : break;
128 :
129 : case FileEntry::COMPRESSION_LEVEL_FASTEST:
130 903 : zlevel = Z_BEST_SPEED;
131 903 : break;
132 :
133 : case FileEntry::COMPRESSION_LEVEL_NONE:
134 : throw std::logic_error("the compression level NONE is not supported in DeflateOutputStreambuf::init()"); // LCOV_EXCL_LINE
135 :
136 : default:
137 91022 : if(compression_level < FileEntry::COMPRESSION_LEVEL_MINIMUM
138 91022 : || compression_level > FileEntry::COMPRESSION_LEVEL_MAXIMUM)
139 : {
140 : // This is excluded from the coverage since if we reach this
141 : // line there is an internal error that needs to be fixed.
142 : throw std::logic_error("the compression level must be defined between -3 and 100, see the zipios/fileentry.hpp for a list of valid levels."); // LCOV_EXCL_LINE
143 : }
144 : // The zlevel is calculated linearly from the user specified value
145 : // of 1 to 100
146 : //
147 : // The calculation goes as follow:
148 : //
149 : // x = user specified value - 1 (0 to 99)
150 : // x = x * 8 (0 to 792)
151 : // x = x + 11 / 2 (5 to 797, i.e. +5 with integers)
152 : // x = x / 99 (0 to 8)
153 : // x = x + 1 (1 to 9)
154 : //
155 91022 : zlevel = ((compression_level - 1) * 8 + 11 / 2) / 99 + 1;
156 91022 : break;
157 :
158 : }
159 :
160 : // m_zs.next_in and avail_in must be set according to
161 : // zlib.h (inline doc).
162 93731 : m_zs.next_in = reinterpret_cast<unsigned char *>(&m_invec[0]);
163 93731 : m_zs.avail_in = 0;
164 :
165 93731 : m_zs.next_out = reinterpret_cast<unsigned char *>(&m_outvec[0]);
166 93731 : m_zs.avail_out = getBufferSize();
167 :
168 : //
169 : // windowBits is passed -MAX_WBITS to tell that no zlib
170 : // header should be written.
171 : //
172 93731 : int const err = deflateInit2(&m_zs, zlevel, Z_DEFLATED, -MAX_WBITS, default_mem_level, Z_DEFAULT_STRATEGY);
173 93731 : if(err != Z_OK)
174 : {
175 : // Not too sure how we could generate an error here, the deflateInit2()
176 : // would fail if (1) there is not enough memory and (2) if a parameter
177 : // is out of wack which neither can be generated from the outside
178 : // (well... not easily)
179 : std::ostringstream msgs; // LCOV_EXCL_LINE
180 : msgs << "DeflateOutputStreambuf::init(): error while initializing zlib, " << zError(err) << std::endl; // LCOV_EXCL_LINE
181 : throw IOException(msgs.str()); // LCOV_EXCL_LINE
182 : }
183 :
184 : // streambuf init:
185 93731 : setp(&m_invec[0], &m_invec[0] + getBufferSize());
186 :
187 93731 : m_crc32 = crc32(0, Z_NULL, 0);
188 :
189 93731 : return err == Z_OK;
190 : }
191 :
192 :
193 : /** \brief Closing the stream.
194 : *
195 : * This function is expected to be called once the stream is getting
196 : * closed (the buffer is destroyed.)
197 : *
198 : * It ensures that the zlib library last few bytes get flushed and
199 : * then mark the class as closed.
200 : *
201 : * Note that this function can be called to close the current zlib
202 : * library stream and start a new one. It is actually called from
203 : * the putNextEntry() function (via the closeEntry() function.)
204 : */
205 93974 : void DeflateOutputStreambuf::closeStream()
206 : {
207 93974 : if(m_zs_initialized)
208 : {
209 93731 : m_zs_initialized = false;
210 :
211 : // flush any remaining data
212 93731 : endDeflation();
213 :
214 93731 : int const err(deflateEnd(&m_zs));
215 93731 : if(err != Z_OK) // when we close a directory, we get the Z_DATA_ERROR!
216 : {
217 : // There are not too many cases which break the deflateEnd()
218 : // function call...
219 : std::ostringstream msgs; // LCOV_EXCL_LINE
220 : msgs << "DeflateOutputStreambuf::closeStream(): deflateEnd failed: " << zError(err) << std::endl; // LCOV_EXCL_LINE
221 : throw IOException(msgs.str()); // LCOV_EXCL_LINE
222 : }
223 : }
224 93974 : }
225 :
226 :
227 : /** \brief Get the CRC32 of the file.
228 : *
229 : * This function returns the CRC32 for the current file.
230 : *
231 : * The returned value is the CRC for the data that has been compressed
232 : * already (due to calls to overflow()). As DeflateOutputStreambuf may
233 : * buffer an arbitrary amount of bytes until closeStream() has been
234 : * invoked, the returned value is not very useful before closeStream()
235 : * has been called.
236 : *
237 : * \return The CRC32 of the last file that was passed through.
238 : */
239 171466 : uint32_t DeflateOutputStreambuf::getCrc32() const
240 : {
241 171466 : return m_crc32;
242 : }
243 :
244 :
245 : /** \brief Retrieve the size of the file deflated.
246 : *
247 : * This function returns the number of bytes written to the
248 : * streambuf object and that were processed from the input
249 : * buffer by the compressor. After closeStream() has been
250 : * called this number is the total number of bytes written
251 : * to the stream. In other words, the size of the uncompressed
252 : * data.
253 : *
254 : * \return The uncompressed size of the file that got written here.
255 : */
256 171466 : size_t DeflateOutputStreambuf::getSize() const
257 : {
258 171466 : return m_overflown_bytes;
259 : }
260 :
261 :
262 : /** \brief Handle an overflow.
263 : *
264 : * This function is called by the streambuf implementation whenever
265 : * "too many bytes" are in the output buffer, ready to be compressed.
266 : *
267 : * \exception IOException
268 : * This exception is raised whenever the overflow() function calls
269 : * a zlib library function which returns an error.
270 : *
271 : * \param[in] c The character (byte) that overflowed the buffer.
272 : *
273 : * \return Always zero (0).
274 : */
275 660277 : int DeflateOutputStreambuf::overflow(int c)
276 : {
277 660277 : int err(Z_OK);
278 :
279 660277 : m_zs.avail_in = pptr() - pbase();
280 660277 : m_zs.next_in = reinterpret_cast<unsigned char *>(&m_invec[0]);
281 :
282 660277 : if(m_zs.avail_in > 0)
283 : {
284 660277 : m_crc32 = crc32(m_crc32, m_zs.next_in, m_zs.avail_in); // update crc32
285 :
286 660277 : m_zs.next_out = reinterpret_cast<unsigned char *>(&m_outvec[0]);
287 660277 : m_zs.avail_out = getBufferSize();
288 :
289 : // Deflate until m_invec is empty.
290 3016451 : while((m_zs.avail_in > 0 || m_zs.avail_out == 0) && err == Z_OK)
291 : {
292 1178087 : if(m_zs.avail_out == 0)
293 : {
294 517810 : flushOutvec();
295 : }
296 :
297 1178087 : err = deflate(&m_zs, Z_NO_FLUSH);
298 : }
299 : }
300 :
301 : // somehow we need this flush here or it fails
302 660277 : flushOutvec();
303 :
304 : // Update 'put' pointers
305 660277 : setp(&m_invec[0], &m_invec[0] + getBufferSize());
306 :
307 660277 : if(err != Z_OK && err != Z_STREAM_END)
308 : {
309 : // Throw an exception to make istream set badbit
310 : //
311 : // This is marked as not cover-able because the calls that
312 : // access this function only happen in an internal loop and
313 : // even if we were to write a direct test, I do not see how
314 : // we could end up with an error here
315 : OutputStringStream msgs; // LCOV_EXCL_LINE
316 : msgs << "Deflation failed:" << zError(err); // LCOV_EXCL_LINE
317 : throw IOException(msgs.str()); // LCOV_EXCL_LINE
318 : }
319 :
320 660277 : if(c != EOF)
321 : {
322 566546 : *pptr() = c;
323 566546 : pbump(1);
324 : }
325 :
326 660277 : return 0;
327 : }
328 :
329 :
330 : /** \brief Synchronize the buffer.
331 : *
332 : * The sync() function is expected to clear the input buffer so that
333 : * any new data read from the input (i.e. a file) are re-read from
334 : * disk. However, a call to sync() could break the filtering
335 : * functionality so we do not implement it at all.
336 : *
337 : * This means you are stuck with the existing buffer. But to make
338 : * sure the system understands that, we always returns -1.
339 : */
340 : int DeflateOutputStreambuf::sync() // LCOV_EXCL_LINE
341 : {
342 : return -1; // LCOV_EXCL_LINE
343 : }
344 :
345 :
346 : /** \brief Flush the cached output data.
347 : *
348 : * This function flushes m_outvec and updates the output pointer
349 : * and size m_zs.next_out and m_zs.avail_out.
350 : */
351 1320120 : void DeflateOutputStreambuf::flushOutvec()
352 : {
353 : /** \TODO
354 : * We need to redesign the class to allow for STORED files to
355 : * flow through without the need to have this crap of bytes to
356 : * skip...
357 : */
358 1320120 : size_t deflated_bytes(getBufferSize() - m_zs.avail_out);
359 1320120 : if(deflated_bytes > 0)
360 : {
361 918748 : size_t const bc(m_outbuf->sputn(&m_outvec[0], deflated_bytes));
362 918748 : if(deflated_bytes != bc)
363 : {
364 : // Without implementing our own stream in our test, this
365 : // cannot really be reached because it is all happening
366 : // inside the same loop in ZipFile::saveCollectionToArchive()
367 : throw IOException("DeflateOutputStreambuf::flushOutvec(): write to buffer failed."); // LCOV_EXCL_LINE
368 : }
369 : }
370 :
371 1320120 : m_zs.next_out = reinterpret_cast<unsigned char *>(&m_outvec[0]);
372 1320120 : m_zs.avail_out = getBufferSize();
373 1320120 : }
374 :
375 :
376 : /** \brief End deflation of current file.
377 : *
378 : * This function flushes the remaining data in the zlib buffers,
379 : * after which the only possible operations are deflateEnd() or
380 : * deflateReset().
381 : */
382 93731 : void DeflateOutputStreambuf::endDeflation()
383 : {
384 93731 : overflow();
385 :
386 93731 : m_zs.next_out = reinterpret_cast<unsigned char *>(&m_outvec[0]);
387 93731 : m_zs.avail_out = getBufferSize();
388 :
389 : // Deflate until _invec is empty.
390 93731 : int err(Z_OK);
391 :
392 : // make sure to NOT call deflate() if nothing was written to the
393 : // deflate output stream, otherwise we get a "spurious" (as far
394 : // Zip archives are concerned) 0x03 0x00 marker from the zlib
395 : // library
396 : //
397 93731 : if(m_overflown_bytes > 0)
398 : {
399 377797 : while(err == Z_OK)
400 : {
401 142033 : if(m_zs.avail_out == 0)
402 : {
403 48302 : flushOutvec();
404 : }
405 :
406 142033 : err = deflate(&m_zs, Z_FINISH);
407 : }
408 : }
409 : else
410 : {
411 : // this is not expected to happen, but it can
412 : err = Z_STREAM_END; // LCOV_EXCL_LINE
413 : }
414 :
415 93731 : flushOutvec();
416 :
417 93731 : if(err != Z_STREAM_END)
418 : {
419 : // This is marked as not cover-able because the calls that
420 : // access this function only happen in an internal loop and
421 : // even if we were to write a direct test, I do not see how
422 : // we could end up with an error here
423 : std::ostringstream msgs; // LCOV_EXCL_LINE
424 : msgs << "DeflateOutputStreambuf::endDeflation(): deflate() failed: " // LCOV_EXCL_LINE
425 : << zError(err) << std::endl; // LCOV_EXCL_LINE
426 : throw IOException(msgs.str()); // LCOV_EXCL_LINE
427 : }
428 93731 : }
429 :
430 :
431 3 : } // namespace
432 :
433 : // Local Variables:
434 : // mode: cpp
435 : // indent-tabs-mode: nil
436 : // c-basic-offset: 4
437 : // tab-width: 4
438 : // End:
439 :
440 : // vim: ts=4 sw=4 et
|
