PCL
Compression.h
Go to the documentation of this file.
1 // ____ ______ __
2 // / __ \ / ____// /
3 // / /_/ // / / /
4 // / ____// /___ / /___ PixInsight Class Library
5 // /_/ \____//_____/ PCL 2.4.0
6 // ----------------------------------------------------------------------------
7 // pcl/Compression.h - Released 2020-07-31T19:33:04Z
8 // ----------------------------------------------------------------------------
9 // This file is part of the PixInsight Class Library (PCL).
10 // PCL is a multiplatform C++ framework for development of PixInsight modules.
11 //
12 // Copyright (c) 2003-2020 Pleiades Astrophoto S.L. All Rights Reserved.
13 //
14 // Redistribution and use in both source and binary forms, with or without
15 // modification, is permitted provided that the following conditions are met:
16 //
17 // 1. All redistributions of source code must retain the above copyright
18 // notice, this list of conditions and the following disclaimer.
19 //
20 // 2. All redistributions in binary form must reproduce the above copyright
21 // notice, this list of conditions and the following disclaimer in the
22 // documentation and/or other materials provided with the distribution.
23 //
24 // 3. Neither the names "PixInsight" and "Pleiades Astrophoto", nor the names
25 // of their contributors, may be used to endorse or promote products derived
26 // from this software without specific prior written permission. For written
27 // permission, please contact info@pixinsight.com.
28 //
29 // 4. All products derived from this software, in any form whatsoever, must
30 // reproduce the following acknowledgment in the end-user documentation
31 // and/or other materials provided with the product:
32 //
33 // "This product is based on software from the PixInsight project, developed
34 // by Pleiades Astrophoto and its contributors (http://pixinsight.com/)."
35 //
36 // Alternatively, if that is where third-party acknowledgments normally
37 // appear, this acknowledgment must be reproduced in the product itself.
38 //
39 // THIS SOFTWARE IS PROVIDED BY PLEIADES ASTROPHOTO AND ITS CONTRIBUTORS
40 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
41 // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PLEIADES ASTROPHOTO OR ITS
43 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
44 // EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, BUSINESS
45 // INTERRUPTION; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; AND LOSS OF USE,
46 // DATA OR PROFITS) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
47 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
48 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
49 // POSSIBILITY OF SUCH DAMAGE.
50 // ----------------------------------------------------------------------------
51 
52 #ifndef __PCL_Compression_h
53 #define __PCL_Compression_h
54 
56 
57 #include <pcl/Defs.h>
58 #include <pcl/Diagnostics.h>
59 
60 #include <pcl/ByteArray.h>
61 #include <pcl/ParallelProcess.h>
62 #include <pcl/String.h>
63 
64 namespace pcl
65 {
66 
67 // ----------------------------------------------------------------------------
68 
83 class PCL_CLASS Compression : public ParallelProcess
84 {
85 public:
86 
91  struct Subblock
92  {
94  size_type uncompressedSize = 0;
95  uint64 checksum = 0;
96  };
97 
102  struct Performance
103  {
104  float sizeReduction = 0;
105  double throughput = 0;
106  int numberOfThreads = 0;
107  };
108 
113 
133  Compression() = default;
134 
138  Compression( const Compression& ) = default;
139 
143  Compression& operator =( const Compression& ) = default;
144 
148  virtual ~Compression()
149  {
150  }
151 
155  virtual String AlgorithmName() const = 0;
156 
162  virtual int MaxCompressionLevel() const = 0;
163 
169  virtual int DefaultCompressionLevel() const = 0;
170 
176  int CompressionLevel() const
177  {
178  return m_compressionLevel;
179  }
180 
190  void SetCompressionLevel( int level )
191  {
192  m_compressionLevel = Range( level, 0, MaxCompressionLevel() );
193  }
194 
204  {
205  SetCompressionLevel( 0 );
206  }
207 
214  bool ByteShufflingEnabled() const
215  {
216  return m_byteShuffle;
217  }
218 
232  void EnableByteShuffling( bool enable = true )
233  {
234  m_byteShuffle = enable;
235  }
236 
242  void DisableByteShuffling( bool disable = true )
243  {
244  EnableByteShuffling( !disable );
245  }
246 
255  {
256  return m_itemSize;
257  }
258 
265  void SetItemSize( size_type itemSize )
266  {
267  m_itemSize = uint8( Range( itemSize, size_type( 1 ), size_type( 128 ) ) );
268  }
269 
276  {
277  return m_subblockSize;
278  }
279 
291  {
292  m_subblockSize = Range( size, size_type( 0 ), MaxBlockSize() );
293  }
294 
301  bool ChecksumsEnabled() const
302  {
303  return m_checksums;
304  }
305 
320  void EnableChecksums( bool enable = true )
321  {
322  m_checksums = enable;
323  }
324 
330  void DisableChecksums( bool disable = true )
331  {
332  EnableChecksums( !disable );
333  }
334 
356  subblock_list Compress( const void* data, size_type size, Performance* perf = nullptr ) const;
357 
382  template <class C>
383  subblock_list Compress( const C& data, Performance* perf = nullptr ) const
384  {
385  return Compress( data.Begin(), data.Size(), perf );
386  }
387 
416  size_type Uncompress( void* data, size_type maxSize,
417  const subblock_list& subblocks, Performance* perf = nullptr ) const;
418 
438  ByteArray Uncompress( const subblock_list& subblocks, Performance* perf = nullptr ) const
439  {
440  size_type uncompressedSize = 0;
441  for ( const Subblock& subblock : subblocks )
442  uncompressedSize += subblock.uncompressedSize;
443  ByteArray data( uncompressedSize );
444  (void)Uncompress( data.Begin(), uncompressedSize, subblocks, perf );
445  return data;
446  }
447 
455  ByteArray Uncompress( const ByteArray& compressedData,
456  size_type uncompressedSize, Performance* perf = nullptr ) const
457  {
458  Subblock subblock;
459  subblock.compressedData = compressedData;
460  subblock.uncompressedSize = uncompressedSize;
461  return Uncompress( subblock_list() << subblock, perf );
462  }
463 
464 private:
465 
466  int m_compressionLevel = 0; // 0 = use codec's default
467  size_type m_subblockSize = 0; // 0 = use largest possible subblocks
468  uint8 m_itemSize = 1;
469  bool m_byteShuffle = true;
470  bool m_checksums = true;
471 
472 protected:
473 
478  virtual size_type MinBlockSize() const = 0;
479 
484  virtual size_type MaxBlockSize() const = 0;
485 
490  virtual size_type MaxCompressedBlockSize( size_type size ) const = 0;
491 
500  virtual size_type CompressBlock( void* outputData, size_type outputSize,
501  const void* inputData, size_type inputSize, int level ) const = 0;
502 
510  virtual size_type UncompressBlock( void* outputData, size_type outputSize,
511  const void* inputData, size_type inputSize ) const = 0;
512 
518  static ByteArray Shuffle( const uint8* data, size_type size, size_type itemSize )
519  {
520  ByteArray shuffled( size );
521  if ( size > 0 && data != nullptr )
522  {
523  size_type numberOfItems = size / itemSize;
524  ByteArray::iterator s = shuffled.Begin();
525  for ( size_type j = 0; j < itemSize; ++j )
526  {
527  ByteArray::const_iterator u = data + j;
528  for ( size_type i = 0; i < numberOfItems; ++i, ++s, u += itemSize )
529  *s = *u;
530  }
531  ::memcpy( s, data + numberOfItems*itemSize, size % itemSize );
532  }
533  return shuffled;
534  }
535 
541  static ByteArray Unshuffle( const uint8* data, size_type size, size_type itemSize )
542  {
543  ByteArray unshuffled( size );
544  if ( size > 0 && data != nullptr )
545  {
546  size_type numberOfItems = size / itemSize;
547  ByteArray::const_iterator s = data;
548  for ( size_type j = 0; j < itemSize; ++j )
549  {
550  ByteArray::iterator u = unshuffled.At( j );
551  for ( size_type i = 0; i < numberOfItems; ++i, ++s, u += itemSize )
552  *u = *s;
553  }
554  ::memcpy( unshuffled.At( numberOfItems*itemSize ), s, size % itemSize );
555  }
556  return unshuffled;
557  }
558 
564  static void InPlaceUnshuffle( uint8* data, size_type size, size_type itemSize )
565  {
566  if ( size > 0 && data != nullptr )
567  {
568  const ByteArray shuffled( data, data+size );
569  size_type numberOfItems = size / itemSize;
570  ByteArray::const_iterator s = shuffled.Begin();
571  for ( size_type j = 0; j < itemSize; ++j )
572  {
573  ByteArray::iterator u = data + j;
574  for ( size_type i = 0; i < numberOfItems; ++i, ++s, u += itemSize )
575  *u = *s;
576  }
577  }
578  }
579 
584  void Throw( const String& errorMessage ) const;
585 
586  friend class PCL_CompressionEngine;
587  friend class PCL_DecompressionEngine;
588 };
589 
590 // ----------------------------------------------------------------------------
591 
615 class PCL_CLASS ZLibCompression : public Compression
616 {
617 public:
618 
622  String AlgorithmName() const override
623  {
624  return "ZLib";
625  }
626 
629  int MaxCompressionLevel() const override;
630 
633  int DefaultCompressionLevel() const override;
634 
635 
636 private:
637 
640  size_type MinBlockSize() const override;
641 
644  size_type MaxBlockSize() const override;
645 
648  size_type MaxCompressedBlockSize( size_type size ) const override;
649 
652  size_type CompressBlock( void* outputData, size_type outputSize,
653  const void* inputData, size_type inputSize, int level ) const override;
654 
657  size_type UncompressBlock( void* outputData, size_type outputSize,
658  const void* inputData, size_type inputSize ) const override;
659 };
660 
661 // ----------------------------------------------------------------------------
662 
685 class PCL_CLASS LZ4Compression : public Compression
686 {
687 public:
688 
692  String AlgorithmName() const override
693  {
694  return "LZ4";
695  }
696 
699  int MaxCompressionLevel() const override;
700 
703  int DefaultCompressionLevel() const override;
704 
705 
706 private:
707 
710  size_type MinBlockSize() const override;
711 
714  size_type MaxBlockSize() const override;
715 
718  size_type MaxCompressedBlockSize( size_type size ) const override;
719 
722  size_type CompressBlock( void* outputData, size_type outputSize,
723  const void* inputData, size_type inputSize, int level ) const override;
724 
727  size_type UncompressBlock( void* outputData, size_type outputSize,
728  const void* inputData, size_type inputSize ) const override;
729 };
730 
731 // ----------------------------------------------------------------------------
732 
756 class PCL_CLASS LZ4HCCompression : public Compression
757 {
758 public:
759 
763  String AlgorithmName() const override
764  {
765  return "LZ4-HC";
766  }
767 
770  int MaxCompressionLevel() const override;
771 
774  int DefaultCompressionLevel() const override;
775 
776 
777 private:
778 
781  size_type MinBlockSize() const override;
782 
785  size_type MaxBlockSize() const override;
786 
789  size_type MaxCompressedBlockSize( size_type size ) const override;
790 
793  size_type CompressBlock( void* outputData, size_type outputSize,
794  const void* inputData, size_type inputSize, int level ) const override;
795 
798  size_type UncompressBlock( void* outputData, size_type outputSize,
799  const void* inputData, size_type inputSize ) const override;
800 };
801 
802 // ----------------------------------------------------------------------------
803 
804 } // pcl
805 
806 #endif // __PCL_Compression_h
807 
808 // ----------------------------------------------------------------------------
809 // EOF pcl/Compression.h - Released 2020-07-31T19:33:04Z
bool ByteShufflingEnabled() const
Definition: Compression.h:214
String AlgorithmName() const override
Definition: Compression.h:763
unsigned char uint8
Definition: Defs.h:576
Compression/decompression performance measurements.
Definition: Compression.h:102
virtual ~Compression()
Definition: Compression.h:148
Implementation of the LZ4 compression algorithm.
Definition: Compression.h:685
PCL root namespace.
Definition: AbstractImage.h:76
subblock_list Compress(const C &data, Performance *perf=nullptr) const
Definition: Compression.h:383
Dynamic array of 8-bit unsigned integers.
static ByteArray Shuffle(const uint8 *data, size_type size, size_type itemSize)
Definition: Compression.h:518
const uint8 * const_iterator
Definition: Array.h:117
Abstract base class of data compression algorithm implementations.
Definition: Compression.h:83
static void InPlaceUnshuffle(uint8 *data, size_type size, size_type itemSize)
Definition: Compression.h:564
bool ChecksumsEnabled() const
Definition: Compression.h:301
constexpr const T & Range(const T &x, const T &a, const T &b)
Definition: Utility.h:190
size_t size_type
Definition: Defs.h:543
void EnableByteShuffling(bool enable=true)
Definition: Compression.h:232
Implementation of the LZ4-HC compression algorithm.
Definition: Compression.h:756
ByteArray Uncompress(const ByteArray &compressedData, size_type uncompressedSize, Performance *perf=nullptr) const
Definition: Compression.h:455
Unicode (UTF-16) string.
Definition: String.h:7916
unsigned long long uint64
Definition: Defs.h:616
size_type uncompressedSize
size in bytes of the uncompressed subblock.
Definition: Compression.h:94
void DisableByteShuffling(bool disable=true)
Definition: Compression.h:242
ByteArray compressedData
Sub-block compressed data.
Definition: Compression.h:93
String AlgorithmName() const override
Definition: Compression.h:622
Generic dynamic array.
Definition: Array.h:99
Implementation of the ZLib/Deflate compression algorithm.
Definition: Compression.h:615
void EnableChecksums(bool enable=true)
Definition: Compression.h:320
size_type SubblockSize() const
Definition: Compression.h:275
void SetDefaultCompressionLevel()
Definition: Compression.h:203
void DisableChecksums(bool disable=true)
Definition: Compression.h:330
void SetSubblockSize(size_type size)
Definition: Compression.h:290
void SetItemSize(size_type itemSize)
Definition: Compression.h:265
void SetCompressionLevel(int level)
Definition: Compression.h:190
String AlgorithmName() const override
Definition: Compression.h:692
int CompressionLevel() const
Definition: Compression.h:176
size_type ItemSize() const
Definition: Compression.h:254
Compression subblock data.
Definition: Compression.h:91
Array< Subblock > subblock_list
Definition: Compression.h:112
static ByteArray Unshuffle(const uint8 *data, size_type size, size_type itemSize)
Definition: Compression.h:541
ByteArray Uncompress(const subblock_list &subblocks, Performance *perf=nullptr) const
Definition: Compression.h:438
A process using multiple concurrent execution threads.
uint8 * iterator
Definition: Array.h:113