hpkg format: compress the whole heap

Instead of handling compression for individual file/attribute data we
do now compress the whole heap where they are stored. This
significantly improves compression ratios. We still divide the
uncompressed data into 64 KiB chunks and use a chunk offset array for
the compressed chunks to allow for quick random access without too much
overhead. The tradeoff is a limited possible compression ratio -- i.e.
we won't be as good as tar.gz (though surprisingly with my test
archives we did better than zip).

The other package file sections (package attributes and TOC) are no
longer compressed individually. Their uncompressed data are simply
pushed onto the heap where the usual compression strategy applies. To
simplify things the repository format has been changed in the same
manner although it doesn't otherwise use the heap, since it only stores
meta data.

Due to the data compression having been exposed in public and private
API, this change touches a lot of package kit using code, including
packagefs and the boot loader packagefs support. The latter two haven't
been tested yet. Moreover packagefs needs a new kind of cache so we
avoid re-reading the same heap chunk for two different data items it
contains.

headers/build/private/package/hpkg/DataWriters.h

@@ -0,0 +1 @@
+#include <../private/package/hpkg/DataWriters.h>

headers/build/private/package/hpkg/PackageFileHeapAccessorBase.h

@@ -0,0 +1 @@
+#include <../private/package/hpkg/PackageFileHeapAccessorBase.h>

headers/build/private/package/hpkg/PackageFileHeapReader.h

@@ -0,0 +1 @@
+#include <../private/package/hpkg/PackageFileHeapReader.h>

headers/build/private/package/hpkg/PackageFileHeapWriter.h

@@ -0,0 +1 @@
+#include <../private/package/hpkg/PackageFileHeapWriter.h>

headers/os/package/hpkg/HPKGDefs.h

@@ -20,7 +20,7 @@ enum {
 	B_HPKG_VERSION	= 2,
 	//
 	B_HPKG_REPO_MAGIC	= 'hpkr',
-	B_HPKG_REPO_VERSION	= 1
+	B_HPKG_REPO_VERSION	= 2
 };
 
 
@@ -98,37 +98,34 @@ enum BHPKGAttributeID {
 	B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE				= 11,
 	B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE_TYPE			= 12,
 	B_HPKG_ATTRIBUTE_ID_DATA						= 13,
-	B_HPKG_ATTRIBUTE_ID_DATA_SIZE					= 14,
-	B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION			= 15,
-	B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE				= 16,
-	B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH				= 17,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_NAME				= 18,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_SUMMARY				= 19,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_DESCRIPTION			= 20,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_VENDOR				= 21,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_PACKAGER			= 22,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_FLAGS				= 23,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_ARCHITECTURE		= 24,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR		= 25,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MINOR		= 26,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MICRO		= 27,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_REVISION	= 28,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_COPYRIGHT			= 29,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_LICENSE				= 30,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES			= 31,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_REQUIRES			= 32,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_SUPPLEMENTS			= 33,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_CONFLICTS			= 34,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_FRESHENS			= 35,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_REPLACES			= 36,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_RESOLVABLE_OPERATOR	= 37,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_CHECKSUM			= 38,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_PRE_RELEASE	= 39,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES_COMPATIBLE	= 40,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_URL					= 41,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_SOURCE_URL			= 42,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_INSTALL_PATH		= 43,
-	B_HPKG_ATTRIBUTE_ID_PACKAGE_BASE_PACKAGE		= 44,
+	B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH				= 14,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_NAME				= 15,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_SUMMARY				= 16,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_DESCRIPTION			= 17,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_VENDOR				= 18,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_PACKAGER			= 19,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_FLAGS				= 20,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_ARCHITECTURE		= 21,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR		= 22,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MINOR		= 23,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MICRO		= 24,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_REVISION	= 25,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_COPYRIGHT			= 26,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_LICENSE				= 27,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES			= 28,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_REQUIRES			= 29,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_SUPPLEMENTS			= 30,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_CONFLICTS			= 31,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_FRESHENS			= 32,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_REPLACES			= 33,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_RESOLVABLE_OPERATOR	= 34,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_CHECKSUM			= 35,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_PRE_RELEASE	= 36,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_PROVIDES_COMPATIBLE	= 37,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_URL					= 38,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_SOURCE_URL			= 39,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_INSTALL_PATH		= 40,
+	B_HPKG_ATTRIBUTE_ID_PACKAGE_BASE_PACKAGE		= 41,
 	//
 	B_HPKG_ATTRIBUTE_ID_ENUM_COUNT,
 };
@@ -154,9 +151,7 @@ enum {
 	B_HPKG_DEFAULT_FILE_TYPE				= B_HPKG_FILE_TYPE_FILE,
 	B_HPKG_DEFAULT_FILE_PERMISSIONS			= 0644,
 	B_HPKG_DEFAULT_DIRECTORY_PERMISSIONS	= 0755,
-	B_HPKG_DEFAULT_SYMLINK_PERMISSIONS		= 0777,
-	B_HPKG_DEFAULT_DATA_COMPRESSION			= B_HPKG_COMPRESSION_NONE,
-	B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB		= 64 * 1024
+	B_HPKG_DEFAULT_SYMLINK_PERMISSIONS		= 0777
 };
 
 

headers/os/package/hpkg/PackageData.h

@@ -18,14 +18,10 @@ class BPackageData {
 public:
 								BPackageData();
 
-			uint64				CompressedSize() const
-									{ return fCompressedSize; }
-			uint64				UncompressedSize() const
-									{ return fUncompressedSize; }
+			uint64				Size() const
+									{ return fSize; }
 			uint64				Offset() const
 									{ return fEncodedInline ? 0 : fOffset; }
-			uint32				Compression() const		{ return fCompression; }
-			uint32				ChunkSize() const		{ return fChunkSize; }
 
 			bool				IsEncodedInline() const
 									{ return fEncodedInline; }
@@ -34,23 +30,13 @@ public:
 			void				SetData(uint64 size, uint64 offset);
 			void				SetData(uint8 size, const void* data);
 
-			void				SetCompression(uint32 compression)
-									{ fCompression = compression; }
-			void				SetUncompressedSize(uint64 size)
-									{ fUncompressedSize = size; }
-			void				SetChunkSize(uint32 size)
-									{ fChunkSize = size; }
-
 private:
-			uint64				fCompressedSize;
-			uint64				fUncompressedSize;
+			uint64				fSize : 63;
+			bool				fEncodedInline : 1;
 			union {
 				uint64			fOffset;
 				uint8			fInlineData[B_HPKG_MAX_INLINE_DATA_SIZE];
 			};
-			uint32				fChunkSize;
-			uint32				fCompression;
-			bool				fEncodedInline;
 };
 
 

headers/os/package/hpkg/PackageDataReader.h

@@ -20,15 +20,12 @@ class BPackageData;
 
 class BPackageDataReaderFactory {
 public:
-								BPackageDataReaderFactory(
-									BBufferPool* bufferPool);
+								BPackageDataReaderFactory();
 
-			status_t			CreatePackageDataReader(BDataReader* dataReader,
+			status_t			CreatePackageDataReader(
+									BAbstractBufferedDataReader* dataReader,
 									const BPackageData& data,
 									BAbstractBufferedDataReader*& _reader);
-
-private:
-			BBufferPool*		fBufferPool;
 };
 
 

headers/os/package/hpkg/PackageReader.h

@@ -19,6 +19,8 @@ namespace BPrivate {
 }
 using BPrivate::PackageReaderImpl;
 
+
+class BAbstractBufferedDataReader;
 class BErrorOutput;
 class BLowLevelPackageContentHandler;
 class BPackageContentHandler;
@@ -26,8 +28,7 @@ class BPackageContentHandler;
 
 class BPackageReader {
 public:
-								BPackageReader(
-									BErrorOutput* errorOutput);
+								BPackageReader(BErrorOutput* errorOutput);
 								~BPackageReader();
 
 			status_t			Init(const char* fileName);
@@ -38,6 +39,10 @@ public:
 										contentHandler);
 
 			int					PackageFileFD();
+
+			BAbstractBufferedDataReader* HeapReader() const;
+									// Only valid as long as the reader lives.
+
 private:
 			PackageReaderImpl*	fImpl;
 };

headers/os/package/hpkg/RepositoryReader.h

@@ -19,6 +19,7 @@ namespace BPrivate {
 }
 using BPrivate::RepositoryReaderImpl;
 
+
 class BErrorOutput;
 class BRepositoryContentHandler;
 

headers/private/package/hpkg/DataWriters.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2011, Oliver Tappe <zooey@hirschkaefer.de>
+ * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_
+#define _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_
+
+
+#include <package/hpkg/DataOutput.h>
+#include <package/hpkg/ZlibCompressor.h>
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+
+class BErrorOutput;
+
+
+namespace BPrivate {
+
+
+class AbstractDataWriter {
+public:
+								AbstractDataWriter();
+	virtual						~AbstractDataWriter();
+
+			uint64				BytesWritten() const
+									{ return fBytesWritten; }
+
+	virtual	status_t			WriteDataNoThrow(const void* buffer,
+									size_t size) = 0;
+
+			void				WriteDataThrows(const void* buffer,
+									size_t size);
+
+protected:
+			uint64				fBytesWritten;
+};
+
+
+class FDDataWriter : public AbstractDataWriter {
+public:
+								FDDataWriter(int fd, off_t offset,
+									BErrorOutput* errorOutput);
+
+	virtual	status_t			WriteDataNoThrow(const void* buffer,
+									size_t size);
+
+			off_t				Offset() const
+									{ return fOffset; }
+
+private:
+			int					fFD;
+			off_t				fOffset;
+			BErrorOutput*		fErrorOutput;
+};
+
+
+class ZlibDataWriter : public AbstractDataWriter, private BDataOutput {
+public:
+								ZlibDataWriter(AbstractDataWriter* dataWriter);
+
+			void				Init();
+			void				Finish();
+
+	virtual	status_t			WriteDataNoThrow(const void* buffer,
+									size_t size);
+
+private:
+	// BDataOutput
+	virtual	status_t			WriteData(const void* buffer, size_t size);
+
+private:
+			AbstractDataWriter*	fDataWriter;
+			ZlibCompressor		fCompressor;
+};
+
+
+// inline implementations
+
+
+inline void
+AbstractDataWriter::WriteDataThrows(const void* buffer, size_t size)
+{
+	status_t error = WriteDataNoThrow(buffer, size);
+	if (error != B_OK)
+		throw status_t(error);
+}
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit
+
+
+#endif	// _PACKAGE__HPKG__PRIVATE__DATA_WRITERS_H_

headers/private/package/hpkg/HPKGDefsPrivate.h

@@ -18,7 +18,7 @@ namespace BHPKG {
 namespace BPrivate {
 
 
-// header
+// package file header
 struct hpkg_header {
 	uint32	magic;							// "hpkg"
 	uint16	header_size;
@@ -26,40 +26,41 @@ struct hpkg_header {
 	uint64	total_size;
 
 	// heap
-//	uint64	heap_compression;
+	uint32	heap_compression;
+	uint32	heap_chunk_size;
+	uint64	heap_size_compressed;
+	uint64	heap_size_uncompressed;
 
 	// package attributes section
-	uint32	attributes_compression;
-	uint32	attributes_length_compressed;
-	uint32	attributes_length_uncompressed;
+	uint32	attributes_length;
 	uint32	attributes_strings_length;
 	uint32	attributes_strings_count;
 
 	// TOC section
-	uint32	toc_compression;
-	uint64	toc_length_compressed;
-	uint64	toc_length_uncompressed;
+	uint64	toc_length;
 	uint64	toc_strings_length;
 	uint64	toc_strings_count;
 };
 
 
-// header
+// repository file header
 struct hpkg_repo_header {
 	uint32	magic;							// "hpkr"
 	uint16	header_size;
 	uint16	version;
 	uint64	total_size;
 
+	// heap
+	uint32	heap_compression;
+	uint32	heap_chunk_size;
+	uint64	heap_size_compressed;
+	uint64	heap_size_uncompressed;
+
 	// repository info section
-	uint32	info_compression;
-	uint32	info_length_compressed;
-	uint32	info_length_uncompressed;
+	uint32	info_length;
 
 	// package attributes section
-	uint32	packages_compression;
-	uint64	packages_length_compressed;
-	uint64	packages_length_uncompressed;
+	uint64	packages_length;
 	uint64	packages_strings_length;
 	uint64	packages_strings_count;
 };

headers/private/package/hpkg/PackageFileHeapAccessorBase.h

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_
+#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_
+
+
+#include <SupportDefs.h>
+
+#include <package/hpkg/DataReader.h>
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+
+class BErrorOutput;
+
+
+namespace BPrivate {
+
+
+class PackageFileHeapAccessorBase : public BAbstractBufferedDataReader {
+public:
+			class OffsetArray;
+
+public:
+								PackageFileHeapAccessorBase(
+									BErrorOutput* errorOutput, int fd,
+									off_t heapOffset);
+	virtual						~PackageFileHeapAccessorBase();
+
+			off_t				HeapOffset() const
+									{ return fHeapOffset; }
+			off_t				CompressedHeapSize() const
+									{ return fCompressedHeapSize; }
+			uint64				UncompressedHeapSize() const
+									{ return fUncompressedHeapSize; }
+			size_t				ChunkSize() const
+									{ return kChunkSize; }
+
+			// normally used after cloning a PackageFileHeapReader only
+			void				SetErrorOutput(BErrorOutput* errorOutput)
+									{ fErrorOutput = errorOutput; }
+			void				SetFD(int fd)
+									{ fFD = fd; }
+
+	// BAbstractBufferedDataReader
+	virtual	status_t			ReadDataToOutput(off_t offset, size_t size,
+									BDataOutput* output);
+
+public:
+	static	const size_t		kChunkSize = 64 * 1024;
+
+protected:
+	virtual	status_t			ReadAndDecompressChunk(size_t chunkIndex,
+									void* compressedDataBuffer,
+									void* uncompressedDataBuffer) = 0;
+			status_t			ReadAndDecompressChunkData(uint64 offset,
+									size_t compressedSize,
+									size_t uncompressedSize,
+									void* compressedDataBuffer,
+									void* uncompressedDataBuffer);
+			status_t			ReadFileData(uint64 offset, void* buffer,
+									size_t size);
+
+protected:
+			BErrorOutput*		fErrorOutput;
+			int					fFD;
+			off_t				fHeapOffset;
+			uint64				fCompressedHeapSize;
+			uint64				fUncompressedHeapSize;
+};
+
+
+/*!	Stores the chunk offsets in a compact way, while still providing quick
+	access.
+	- The object doesn't store the number of chunks/offsets it contains. During
+	  initialization the chunk count is provided. Later, when getting an offset,
+	  the caller is responsible for ensuring a valid index.
+	- The first (index 0) chunk offset is omitted, since it is always 0.
+	- The chunk offsets that fit in a 32 bit number use only one 32 bit element
+	  in the offsets array.
+	- The chunk offsets that don't fit in a 32 bit number use two elements in
+	  the offsets array.
+	Memory use is one pointer, if the chunk count is <= 1 (uncompressed heap size
+	<= 64 KiB). Afterwards it's one pointer plus 32 bit per chunk as long as the
+	last offset still fits 32 bit (compressed heap size < 4GiB). For any further
+	chunks it is 64 bit per chunk. So, for the common case we use sizeof(void*)
+	plus 1 KiB per 16 MiB of uncompressed heap, or about 64 KiB per 1 GiB. Which
+	seems reasonable for packagefs to keep in memory.
+ */
+class PackageFileHeapAccessorBase::OffsetArray {
+public:
+								OffsetArray();
+								~OffsetArray();
+
+			bool				InitChunksOffsets(size_t totalChunkCount,
+									size_t baseIndex, const uint16* chunkSizes,
+									size_t chunkCount);
+
+			bool				Init(size_t totalChunkCount,
+									const OffsetArray& other);
+									// "copy" init
+
+			uint64				operator[](size_t index) const;
+
+private:
+			uint32*				fOffsets;
+									// - NULL, if chunkCount <= 1
+									// - element 0 contains the number of 32 bit
+									//   offsets that follow, or is 0, when all
+									//   offsets are 32 bit only
+									// - the following offsets use two elements
+									//   each (lower followed by upper 32 bit)
+									//   to represent the 64 bit value
+};
+
+
+inline uint64
+PackageFileHeapAccessorBase::OffsetArray::operator[](size_t index) const
+{
+	if (index == 0)
+		return 0;
+
+	if (fOffsets[0] == 0 || index < fOffsets[0])
+		return fOffsets[index];
+
+	index += index - fOffsets[0];
+	return fOffsets[index] | ((uint64)fOffsets[index + 1] << 32);
+}
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit
+
+
+#endif	// _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_ACCESSOR_BASE_H_

headers/private/package/hpkg/PackageFileHeapReader.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_
+#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_
+
+
+#include <Array.h>
+#include <package/hpkg/PackageFileHeapAccessorBase.h>
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+
+class BDataReader;
+class BErrorOutput;
+
+
+namespace BPrivate {
+
+
+class PackageFileHeapReader : public PackageFileHeapAccessorBase {
+public:
+								PackageFileHeapReader(BErrorOutput* errorOutput,
+									int fd, off_t heapOffset,
+									off_t compressedHeapSize,
+									uint64 uncompressedHeapSize);
+								~PackageFileHeapReader();
+
+			status_t			Init();
+
+			PackageFileHeapReader* Clone() const;
+
+			const OffsetArray&	Offsets() const
+									{ return fOffsets; }
+
+protected:
+	virtual	status_t			ReadAndDecompressChunk(size_t chunkIndex,
+									void* compressedDataBuffer,
+									void* uncompressedDataBuffer);
+
+private:
+			OffsetArray			fOffsets;
+};
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit
+
+
+#endif	// _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_READER_H_

headers/private/package/hpkg/PackageFileHeapWriter.h

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_
+#define _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_
+
+
+#include <Array.h>
+#include <package/hpkg/DataWriters.h>
+#include <package/hpkg/PackageFileHeapAccessorBase.h>
+
+
+namespace BPrivate {
+	template<typename Value> class RangeArray;
+}
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+
+class BDataReader;
+class BErrorOutput;
+
+
+namespace BPrivate {
+
+
+class PackageFileHeapReader;
+
+
+class PackageFileHeapWriter : public PackageFileHeapAccessorBase,
+	private AbstractDataWriter {
+public:
+								PackageFileHeapWriter(BErrorOutput* errorOutput,
+									int fd, off_t heapOffset);
+								~PackageFileHeapWriter();
+
+			void				Init();
+			void				Reinit(PackageFileHeapReader* heapReader);
+
+			AbstractDataWriter* DataWriter()
+									{ return this; }
+
+			status_t			AddData(BDataReader& dataReader, off_t size,
+									uint64& _offset);
+			void				RemoveDataRanges(
+									const ::BPrivate::RangeArray<uint64>&
+										ranges);
+									// doesn't truncate the file
+			status_t			Finish();
+
+protected:
+	virtual	status_t			ReadAndDecompressChunk(size_t chunkIndex,
+									void* compressedDataBuffer,
+									void* uncompressedDataBuffer);
+
+private:
+	// AbstractDataWriter
+	virtual	status_t			WriteDataNoThrow(const void* buffer,
+									size_t size);
+
+private:
+			struct Chunk;
+			struct ChunkSegment;
+			struct ChunkBuffer;
+
+private:
+			void				_Uninit();
+
+			status_t			_FlushPendingData();
+			status_t			_WriteChunk(const void* data, size_t size,
+									bool mayCompress);
+			status_t			_WriteDataCompressed(const void* data,
+									size_t size);
+			status_t			_WriteDataUncompressed(const void* data,
+									size_t size);
+
+			void				_PushChunks(ChunkBuffer& chunkBuffer,
+									uint64 startOffset, uint64 endOffset);
+
+private:
+			void*				fPendingDataBuffer;
+			void*				fCompressedDataBuffer;
+			size_t				fPendingDataSize;
+			Array<uint64>		fOffsets;
+};
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit
+
+
+#endif	// _PACKAGE__HPKG__PRIVATE__PACKAGE_FILE_HEAP_WRITER_H_

headers/private/package/hpkg/PackageReaderImpl.h

@@ -25,8 +25,7 @@ namespace BPrivate {
 class PackageReaderImpl : public ReaderImplBase {
 	typedef	ReaderImplBase		inherited;
 public:
-								PackageReaderImpl(
-									BErrorOutput* errorOutput);
+								PackageReaderImpl(BErrorOutput* errorOutput);
 								~PackageReaderImpl();
 
 			status_t			Init(const char* fileName);
@@ -41,13 +40,15 @@ public:
 			uint64				HeapOffset() const;
 			uint64				HeapSize() const;
 
+			PackageFileHeapReader* HeapReader() const
+									{ return inherited::HeapReader(); }
+
 protected:
 								// from ReaderImplBase
 	virtual	status_t			ReadAttributeValue(uint8 type, uint8 encoding,
 									AttributeValue& _value);
 
 private:
-			struct DataAttributeHandler;
 			struct AttributeAttributeHandler;
 			struct EntryAttributeHandler;
 			struct RootAttributeHandler;
@@ -59,11 +60,10 @@ private:
 			status_t			_GetTOCBuffer(size_t size,
 									const void*& _buffer);
 private:
-			uint64				fTotalSize;
 			uint64				fHeapOffset;
 			uint64				fHeapSize;
 
-			SectionInfo			fTOCSection;
+			PackageFileSection	fTOCSection;
 };
 
 

headers/private/package/hpkg/PackageWriterImpl.h

@@ -35,6 +35,8 @@ namespace BPrivate {
 
 
 struct hpkg_header;
+class PackageFileHeapWriter;
+
 
 class PackageWriterImpl : public WriterImplBase {
 	typedef	WriterImplBase		inherited;
@@ -78,30 +80,16 @@ private:
 									Entry* entry, const char* fileName,
 									char* pathBuffer);
 			void				_CompactHeap();
-			void				_MoveHeapChunk(off_t fromOffset, off_t toOffset,
-									off_t size);
 			void				_AttributeRemoved(Attribute* attribute);
 
-			void				_WriteTOC(hpkg_header& header);
-			int32				_WriteTOCCompressed(
-									uint64& _uncompressedStringsSize,
-									uint64& _uncompressedMainSize,
-									uint64& _tocUncompressedSize);
-			int32				_WriteTOCUncompressed(
-									uint64& _uncompressedStringsSize,
-									uint64& _uncompressedMainSize,
-									uint64& _tocUncompressedSize);
-			int32				_WriteTOCSections(uint64& _stringsSize,
-									uint64& _mainSize);
+			void				_WriteTOC(hpkg_header& header, uint64& _length);
 			void				_WriteAttributeChildren(Attribute* attribute);
 
-			void				_WritePackageAttributes(hpkg_header& header);
+			void				_WritePackageAttributes(hpkg_header& header,
+									uint64& _length);
 			uint32				_WritePackageAttributesCompressed(
 									uint32& _stringsLengthUncompressed,
 									uint32& _attributesLengthUncompressed);
-			uint32				_WritePackageAttributesUncompressed(
-									uint32& _stringsLengthUncompressed,
-									uint32& _attributesLengthUncompressed);
 
 			void				_AddEntry(int dirFD, Entry* entry,
 									const char* fileName, char* pathBuffer);
@@ -125,8 +113,6 @@ private:
 
 			status_t			_AddData(BDataReader& dataReader, off_t size);
 
-			status_t			_WriteUncompressedData(BDataReader& dataReader,
-									off_t size, uint64 writeOffset);
 			status_t			_WriteZlibCompressedData(
 									BDataReader& dataReader,
 									off_t size, uint64 writeOffset,
@@ -136,12 +122,9 @@ private:
 			BPackageWriterListener*	fListener;
 
 			off_t				fHeapOffset;
-			off_t				fHeapEnd;
+			uint16				fHeaderSize;
 
-			::BPrivate::RangeArray<off_t>* fHeapRangesToRemove;
-
-			void*				fDataBuffer;
-			const size_t		fDataBufferSize;
+			::BPrivate::RangeArray<uint64>* fHeapRangesToRemove;
 
 			Entry*				fRootEntry;
 

headers/private/package/hpkg/ReaderImplBase.h

@@ -7,6 +7,10 @@
 #define _PACKAGE__HPKG__PRIVATE__READER_IMPL_BASE_H_
 
 
+#include <errno.h>
+#include <sys/stat.h>
+
+#include <ByteOrder.h>
 #include <SupportDefs.h>
 
 #include <util/SinglyLinkedList.h>
@@ -27,176 +31,73 @@ class BErrorOutput;
 namespace BPrivate {
 
 
+class PackageFileHeapReader;
+
+
+struct PackageFileSection {
+	uint32			uncompressedLength;
+	uint8*			data;
+	uint64			offset;
+	uint64			currentOffset;
+	uint64			stringsLength;
+	uint64			stringsCount;
+	char**			strings;
+	const char*		name;
+
+	PackageFileSection(const char* _name)
+		:
+		data(NULL),
+		strings(NULL),
+		name(_name)
+	{
+	}
+
+	~PackageFileSection()
+	{
+		delete[] strings;
+		delete[] data;
+	}
+};
+
+
 class ReaderImplBase {
 protected:
-								ReaderImplBase(
+								ReaderImplBase(const char* fileType,
 									BErrorOutput* errorOutput);
 	virtual						~ReaderImplBase();
 
-	virtual	status_t			Init(int fd, bool keepFD);
-
 			int					FD() const;
 
 			BErrorOutput*		ErrorOutput() const;
 
+			PackageFileHeapReader* HeapReader() const
+									{ return fHeapReader; }
+
 protected:
-			struct AttributeHandlerContext {
-				BErrorOutput*	errorOutput;
-				union {
-					BPackageContentHandler*			packageContentHandler;
-					BLowLevelPackageContentHandler*	lowLevelHandler;
-				};
-				bool			hasLowLevelHandler;
-
-				uint64			heapOffset;
-				uint64			heapSize;
-
-				BHPKGPackageSectionID	section;
-
-				AttributeHandlerContext(BErrorOutput* errorOutput,
-					BPackageContentHandler* packageContentHandler,
-					BHPKGPackageSectionID section);
-
-				AttributeHandlerContext(BErrorOutput* errorOutput,
-					BLowLevelPackageContentHandler* lowLevelHandler,
-					BHPKGPackageSectionID section);
-
-				void ErrorOccurred();
-			};
-
+			class AttributeHandlerContext;
+			class AttributeHandler;
+			class IgnoreAttributeHandler;
+			class PackageVersionAttributeHandler;
+			class PackageResolvableAttributeHandler;
+			class PackageResolvableExpressionAttributeHandler;
+			class PackageAttributeHandler;
+			class LowLevelAttributeHandler;
 
 			typedef BPackageAttributeValue AttributeValue;
-
-			struct AttributeHandler
-				: SinglyLinkedListLinkImpl<AttributeHandler> {
-				virtual ~AttributeHandler();
-
-				void SetLevel(int level);
-				virtual status_t HandleAttribute(
-					AttributeHandlerContext* context, uint8 id,
-					const AttributeValue& value, AttributeHandler** _handler);
-
-				virtual status_t Delete(AttributeHandlerContext* context);
-
-			protected:
-				int	fLevel;
-			};
-
-
-			struct IgnoreAttributeHandler : AttributeHandler {
-			};
-
-
-			struct PackageVersionAttributeHandler : AttributeHandler {
-				PackageVersionAttributeHandler(
-					BPackageInfoAttributeValue& packageInfoValue,
-					BPackageVersionData& versionData, bool notify);
-
-				virtual status_t HandleAttribute(
-					AttributeHandlerContext* context, uint8 id,
-					const AttributeValue& value, AttributeHandler** _handler);
-
-				virtual status_t Delete(AttributeHandlerContext* context);
-
-			private:
-				BPackageInfoAttributeValue&	fPackageInfoValue;
-				BPackageVersionData&		fPackageVersionData;
-				bool						fNotify;
-			};
-
-
-			struct PackageResolvableAttributeHandler : AttributeHandler {
-				PackageResolvableAttributeHandler(
-					BPackageInfoAttributeValue& packageInfoValue);
-
-				virtual status_t HandleAttribute(
-					AttributeHandlerContext* context, uint8 id,
-					const AttributeValue& value, AttributeHandler** _handler);
-
-				virtual status_t Delete(AttributeHandlerContext* context);
-
-			private:
-				BPackageInfoAttributeValue&	fPackageInfoValue;
-			};
-
-
-			struct PackageResolvableExpressionAttributeHandler
-				: AttributeHandler {
-				PackageResolvableExpressionAttributeHandler(
-					BPackageInfoAttributeValue& packageInfoValue);
-
-				virtual status_t HandleAttribute(
-					AttributeHandlerContext* context, uint8 id,
-					const AttributeValue& value, AttributeHandler** _handler);
-
-				virtual status_t Delete(AttributeHandlerContext* context);
-
-			private:
-				BPackageInfoAttributeValue&	fPackageInfoValue;
-			};
-
-
-			struct PackageAttributeHandler : AttributeHandler {
-				virtual status_t HandleAttribute(
-					AttributeHandlerContext* context, uint8 id,
-					const AttributeValue& value, AttributeHandler** _handler);
-
-			private:
-				BPackageInfoAttributeValue	fPackageInfoValue;
-			};
-
-
-			struct LowLevelAttributeHandler : AttributeHandler {
-				LowLevelAttributeHandler();
-				LowLevelAttributeHandler(uint8 id,
-					const BPackageAttributeValue& value, void* parentToken,
-					void* token);
-
-				virtual status_t HandleAttribute(
-					AttributeHandlerContext* context, uint8 id,
-					const AttributeValue& value, AttributeHandler** _handler);
-				virtual status_t Delete(AttributeHandlerContext* context);
-
-			private:
-				void*			fParentToken;
-				void*			fToken;
-				uint8			fID;
-				AttributeValue	fValue;
-			};
-
-
-			struct SectionInfo {
-				uint32			compression;
-				uint32			compressedLength;
-				uint32			uncompressedLength;
-				uint8*			data;
-				uint64			offset;
-				uint64			currentOffset;
-				uint64			stringsLength;
-				uint64			stringsCount;
-				char**			strings;
-				const char*		name;
-
-				SectionInfo(const char* _name)
-					:
-					data(NULL),
-					strings(NULL),
-					name(_name)
-				{
-				}
-
-				~SectionInfo()
-				{
-					delete[] strings;
-					delete[] data;
-				}
-			};
-
 			typedef SinglyLinkedList<AttributeHandler> AttributeHandlerList;
 
 protected:
-			const char*			CheckCompression(
-									const SectionInfo& section) const;
+			template<typename Header, uint32 kMagic, uint16 kVersion>
+			status_t			Init(int fd, bool keepFD, Header& header);
+			status_t			InitHeapReader(uint32 compression,
+									uint32 chunkSize, off_t offset,
+									uint64 compressedSize,
+									uint64 uncompressedSize);
+			status_t			InitSection(PackageFileSection& section,
+									uint64 endOffset, uint64 length,
+									uint64 maxSaneLength, uint64 stringsLength,
+									uint64 stringsCount);
+			status_t			PrepareSection(PackageFileSection& section);
 
 			status_t			ParseStrings();
 
@@ -214,8 +115,7 @@ protected:
 
 			status_t			ReadBuffer(off_t offset, void* buffer,
 									size_t size);
-			status_t			ReadCompressedBuffer(
-									const SectionInfo& section);
+			status_t			ReadSection(const PackageFileSection& section);
 
 	inline	AttributeHandler*	CurrentAttributeHandler() const;
 	inline	void				PushAttributeHandler(
@@ -223,13 +123,15 @@ protected:
 	inline	AttributeHandler*	PopAttributeHandler();
 	inline	void				ClearAttributeHandlerStack();
 
-	inline	SectionInfo*		CurrentSection();
-	inline	void				SetCurrentSection(SectionInfo* section);
+	inline	PackageFileSection*	CurrentSection();
+	inline	void				SetCurrentSection(PackageFileSection* section);
 
 protected:
-			SectionInfo			fPackageAttributesSection;
+			PackageFileSection	fPackageAttributesSection;
 
 private:
+			status_t			_Init(int fd, bool keepFD);
+
 			status_t			_ParseAttributeTree(
 									AttributeHandlerContext* context);
 
@@ -247,11 +149,14 @@ private:
 									size_t* _stringLength = NULL);
 
 private:
+			const char*			fFileType;
 			BErrorOutput*		fErrorOutput;
 			int					fFD;
 			bool				fOwnsFD;
 
-			SectionInfo*		fCurrentSection;
+			PackageFileHeapReader* fHeapReader;
+
+			PackageFileSection*	fCurrentSection;
 
 			AttributeHandlerList fAttributeHandlerStack;
 
@@ -260,6 +165,232 @@ private:
 };
 
 
+// #pragma mark - attribute handlers
+
+
+class ReaderImplBase::AttributeHandlerContext {
+public:
+			BErrorOutput*			errorOutput;
+			union {
+				BPackageContentHandler*			packageContentHandler;
+				BLowLevelPackageContentHandler*	lowLevelHandler;
+			};
+			bool					hasLowLevelHandler;
+		
+			BHPKGPackageSectionID	section;
+
+public:
+								AttributeHandlerContext(
+									BErrorOutput* errorOutput,
+									BPackageContentHandler*
+										packageContentHandler,
+									BHPKGPackageSectionID section);
+								AttributeHandlerContext(
+									BErrorOutput* errorOutput,
+									BLowLevelPackageContentHandler*
+										lowLevelHandler,
+									BHPKGPackageSectionID section);
+
+			void				ErrorOccurred();
+};
+
+
+class ReaderImplBase::AttributeHandler
+	: public SinglyLinkedListLinkImpl<AttributeHandler> {
+public:
+	virtual						~AttributeHandler();
+
+			void				SetLevel(int level);
+	virtual	status_t			HandleAttribute(
+									AttributeHandlerContext* context, uint8 id,
+									const AttributeValue& value,
+									AttributeHandler** _handler);
+
+	virtual	status_t			Delete(AttributeHandlerContext* context);
+
+protected:
+			int					fLevel;
+};
+
+
+class ReaderImplBase::IgnoreAttributeHandler : public AttributeHandler {
+};
+
+
+class ReaderImplBase::PackageVersionAttributeHandler : public AttributeHandler {
+public:
+								PackageVersionAttributeHandler(
+									BPackageInfoAttributeValue&
+										packageInfoValue,
+									BPackageVersionData& versionData,
+									bool notify);
+
+	virtual	status_t			HandleAttribute(
+									AttributeHandlerContext* context, uint8 id,
+									const AttributeValue& value,
+									AttributeHandler** _handler);
+
+	virtual	status_t			Delete(AttributeHandlerContext* context);
+
+private:
+			BPackageInfoAttributeValue& fPackageInfoValue;
+			BPackageVersionData& fPackageVersionData;
+			bool				fNotify;
+};
+
+
+class ReaderImplBase::PackageResolvableAttributeHandler
+	: public AttributeHandler {
+public:
+								PackageResolvableAttributeHandler(
+									BPackageInfoAttributeValue&
+										packageInfoValue);
+
+	virtual	status_t			HandleAttribute(
+									AttributeHandlerContext* context, uint8 id,
+									const AttributeValue& value,
+									AttributeHandler** _handler);
+
+	virtual	status_t			Delete(AttributeHandlerContext* context);
+
+private:
+			BPackageInfoAttributeValue& fPackageInfoValue;
+};
+
+
+class ReaderImplBase::PackageResolvableExpressionAttributeHandler
+	: public AttributeHandler {
+public:
+								PackageResolvableExpressionAttributeHandler(
+									BPackageInfoAttributeValue&
+										packageInfoValue);
+
+	virtual	status_t			HandleAttribute(
+									AttributeHandlerContext* context, uint8 id,
+									const AttributeValue& value,
+									AttributeHandler** _handler);
+
+	virtual	status_t			Delete(AttributeHandlerContext* context);
+
+private:
+			BPackageInfoAttributeValue& fPackageInfoValue;
+};
+
+
+class ReaderImplBase::PackageAttributeHandler : public AttributeHandler {
+public:
+	virtual	status_t			HandleAttribute(
+									AttributeHandlerContext* context, uint8 id,
+									const AttributeValue& value,
+									AttributeHandler** _handler);
+
+private:
+			BPackageInfoAttributeValue fPackageInfoValue;
+};
+
+
+class ReaderImplBase::LowLevelAttributeHandler : public AttributeHandler {
+public:
+								LowLevelAttributeHandler();
+								LowLevelAttributeHandler(uint8 id,
+									const BPackageAttributeValue& value,
+									void* parentToken, void* token);
+
+	virtual	status_t			HandleAttribute(
+									AttributeHandlerContext* context, uint8 id,
+									const AttributeValue& value,
+									AttributeHandler** _handler);
+	virtual	status_t			Delete(AttributeHandlerContext* context);
+
+private:
+			void*				fParentToken;
+			void*				fToken;
+			uint8				fID;
+			AttributeValue		fValue;
+};
+
+
+// #pragma mark - template and inline methods
+
+
+template<typename Header, uint32 kMagic, uint16 kVersion>
+status_t
+ReaderImplBase::Init(int fd, bool keepFD, Header& header)
+{
+	status_t error = _Init(fd, keepFD);
+	if (error != B_OK)
+		return error;
+
+	// stat the file
+	struct stat st;
+	if (fstat(FD(), &st) < 0) {
+		ErrorOutput()->PrintError("Error: Failed to access %s file: %s\n",
+			fFileType, strerror(errno));
+		return errno;
+	}
+
+	// read the header
+	if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK)
+		return error;
+
+	// check the header
+
+	// magic
+	if (B_BENDIAN_TO_HOST_INT32(header.magic) != kMagic) {
+		ErrorOutput()->PrintError("Error: Invalid %s file: Invalid "
+			"magic\n", fFileType);
+		return B_BAD_DATA;
+	}
+
+	// version
+	if (B_BENDIAN_TO_HOST_INT16(header.version) != kVersion) {
+		ErrorOutput()->PrintError("Error: Invalid/unsupported %s file "
+			"version (%d)\n", fFileType,
+			B_BENDIAN_TO_HOST_INT16(header.version));
+		return B_MISMATCHED_VALUES;
+	}
+
+	// header size
+	uint64 heapOffset = B_BENDIAN_TO_HOST_INT16(header.header_size);
+	if (heapOffset < (off_t)sizeof(header)) {
+		ErrorOutput()->PrintError("Error: Invalid %s file: Invalid header "
+			"size (%" B_PRIu64 ")\n", fFileType, heapOffset);
+		return B_BAD_DATA;
+	}
+
+	// total size
+	uint64 totalSize = B_BENDIAN_TO_HOST_INT64(header.total_size);
+	if (totalSize != (uint64)st.st_size) {
+		ErrorOutput()->PrintError("Error: Invalid %s file: Total size in "
+			"header (%" B_PRIu64 ") doesn't agree with total file size (%"
+			B_PRIdOFF ")\n", fFileType, totalSize, st.st_size);
+		return B_BAD_DATA;
+	}
+
+	// heap size
+	uint64 compressedHeapSize
+		= B_BENDIAN_TO_HOST_INT64(header.heap_size_compressed);
+	if (compressedHeapSize > totalSize
+		|| heapOffset > totalSize - compressedHeapSize) {
+		ErrorOutput()->PrintError("Error: Invalid %s file: Heap size in "
+			"header (%" B_PRIu64 ") doesn't agree with total file size (%"
+			B_PRIu64 ") and heap offset (%" B_PRIu64 ")\n", fFileType,
+			compressedHeapSize, totalSize, heapOffset);
+		return B_BAD_DATA;
+	}
+
+	error = InitHeapReader(
+		B_BENDIAN_TO_HOST_INT32(header.heap_compression),
+		B_BENDIAN_TO_HOST_INT32(header.heap_chunk_size), heapOffset,
+		compressedHeapSize,
+		B_BENDIAN_TO_HOST_INT64(header.heap_size_uncompressed));
+	if (error != B_OK)
+		return error;
+
+	return B_OK;
+}
+
+
 inline int
 ReaderImplBase::FD() const
 {
@@ -274,7 +405,7 @@ ReaderImplBase::ErrorOutput() const
 }
 
 
-ReaderImplBase::SectionInfo*
+PackageFileSection*
 ReaderImplBase::CurrentSection()
 {
 	return fCurrentSection;
@@ -282,7 +413,7 @@ ReaderImplBase::CurrentSection()
 
 
 void
-ReaderImplBase::SetCurrentSection(SectionInfo* section)
+ReaderImplBase::SetCurrentSection(PackageFileSection* section)
 {
 	fCurrentSection = section;
 }

headers/private/package/hpkg/RepositoryReaderImpl.h

@@ -25,8 +25,7 @@ namespace BPrivate {
 class RepositoryReaderImpl : public ReaderImplBase {
 	typedef	ReaderImplBase		inherited;
 public:
-								RepositoryReaderImpl(
-									BErrorOutput* errorOutput);
+								RepositoryReaderImpl(BErrorOutput* errorOutput);
 								~RepositoryReaderImpl();
 
 			status_t			Init(const char* fileName);
@@ -42,7 +41,6 @@ private:
 			struct RootAttributeHandler;
 
 private:
-			SectionInfo			fRepositoryInfoSection;
 			BRepositoryInfo		fRepositoryInfo;
 };
 

headers/private/package/hpkg/RepositoryWriterImpl.h

@@ -50,10 +50,9 @@ private:
 
 			status_t			_RegisterCurrentPackageInfo();
 			status_t			_WriteRepositoryInfo(hpkg_repo_header& header,
-									ssize_t& _infoLengthCompressed);
-			off_t				_WritePackageAttributes(
-									hpkg_repo_header& header, off_t startOffset,
-									ssize_t& _packagesLengthCompressed);
+									uint64& _length);
+			void				_WritePackageAttributes(
+									hpkg_repo_header& header, uint64& _length);
 
 			struct PackageNameSet;
 

headers/private/package/hpkg/WriterImplBase.h

@@ -11,6 +11,7 @@
 #include <package/hpkg/HPKGDefsPrivate.h>
 
 #include <package/hpkg/DataOutput.h>
+#include <package/hpkg/DataWriters.h>
 #include <package/hpkg/Strings.h>
 #include <package/hpkg/ZlibCompressor.h>
 
@@ -29,16 +30,19 @@ class BErrorOutput;
 namespace BPrivate {
 
 
+class AbstractDataWriter;
+class PackageFileHeapWriter;
+
 struct hpkg_header;
 
+
 class WriterImplBase {
 public:
-								WriterImplBase(BErrorOutput* errorOutput);
+								WriterImplBase(const char* fileType,
+									BErrorOutput* errorOutput);
 								~WriterImplBase();
 
 protected:
-
-
 			struct AttributeValue {
 				union {
 					int64			signedInt;
@@ -92,63 +96,10 @@ protected:
 				void _DeleteChildren();
 			};
 
-
-			struct AbstractDataWriter {
-				AbstractDataWriter();
-				virtual ~AbstractDataWriter();
-
-				uint64 BytesWritten() const;
-
-				virtual status_t WriteDataNoThrow(const void* buffer,
-					size_t size) = 0;
-
-				void WriteDataThrows(const void* buffer, size_t size);
-
-			protected:
-				uint64	fBytesWritten;
-			};
-
-
-			struct FDDataWriter : AbstractDataWriter {
-				FDDataWriter(int fd, off_t offset, BErrorOutput* errorOutput);
-
-				virtual status_t WriteDataNoThrow(const void* buffer,
-					size_t size);
-
-				off_t Offset() const;
-
-			private:
-				int				fFD;
-				off_t			fOffset;
-				BErrorOutput*	fErrorOutput;
-			};
-
-
-			struct ZlibDataWriter : AbstractDataWriter, private BDataOutput {
-				ZlibDataWriter(AbstractDataWriter* dataWriter);
-
-				void Init();
-
-				void Finish();
-
-				virtual status_t WriteDataNoThrow(const void* buffer,
-					size_t size);
-
-			private:
-				// BDataOutput
-				virtual status_t WriteData(const void* buffer, size_t size);
-
-			private:
-				AbstractDataWriter*	fDataWriter;
-				ZlibCompressor		fCompressor;
-			};
-
-
 			typedef DoublyLinkedList<PackageAttribute> PackageAttributeList;
 
 protected:
-			status_t			Init(const char* fileName, const char* type,
-									uint32 flags);
+			status_t			Init(const char* fileName, uint32 flags);
 
 			void				RegisterPackageInfo(
 									PackageAttributeList& attributeList,
@@ -180,13 +131,16 @@ protected:
 			void				WriteAttributeValue(const AttributeValue& value,
 									uint8 encoding);
 			void				WriteUnsignedLEB128(uint64 value);
-	inline	void				WriteString(const char* string);
 
 	template<typename Type>
 	inline	void				Write(const Type& value);
+	inline	void				WriteString(const char* string);
+	inline	void				WriteBuffer(const void* data, size_t size);
+									// appends data to the heap
 
-			void				WriteBuffer(const void* buffer, size_t size,
+			void				RawWriteBuffer(const void* buffer, size_t size,
 									off_t offset);
+									// writes to the file directly
 
 	inline	int					FD() const;
 	inline	uint32				Flags() const;
@@ -197,11 +151,11 @@ protected:
 	inline	const StringCache&	PackageStringCache() const;
 	inline	StringCache&		PackageStringCache();
 
-	inline	AbstractDataWriter*	DataWriter() const;
-	inline	void				SetDataWriter(AbstractDataWriter* dataWriter);
-
 	inline	void				SetFinished(bool finished);
 
+protected:
+			PackageFileHeapWriter* fHeapWriter;
+
 private:
 	static const BHPKGAttributeID kDefaultVersionAttributeID
 		= B_HPKG_ATTRIBUTE_ID_PACKAGE_VERSION_MAJOR;
@@ -211,6 +165,7 @@ private:
 									const PackageAttributeList& attributes);
 
 private:
+			const char*			fFileType;
 			BErrorOutput*		fErrorOutput;
 			const char*			fFileName;
 			uint32				fFlags;
@@ -224,41 +179,25 @@ private:
 };
 
 
-inline uint64
-WriterImplBase::AbstractDataWriter::BytesWritten() const
-{
-	return fBytesWritten;
-}
-
-
-inline void
-WriterImplBase::AbstractDataWriter::WriteDataThrows(const void* buffer,
-	size_t size)
-{
-	status_t error = WriteDataNoThrow(buffer, size);
-	if (error != B_OK)
-		throw status_t(error);
-}
-
-inline off_t
-WriterImplBase::FDDataWriter::Offset() const
-{
-	return fOffset;
-}
-
-
 template<typename Type>
 inline void
 WriterImplBase::Write(const Type& value)
 {
-	fDataWriter->WriteDataThrows(&value, sizeof(Type));
+	WriteBuffer(&value, sizeof(Type));
 }
 
 
 inline void
 WriterImplBase::WriteString(const char* string)
 {
-	fDataWriter->WriteDataThrows(string, strlen(string) + 1);
+	WriteBuffer(string, strlen(string) + 1);
+}
+
+
+inline void
+WriterImplBase::WriteBuffer(const void* data, size_t size)
+{
+	fDataWriter->WriteDataThrows(data, size);
 }
 
 
@@ -276,20 +215,6 @@ WriterImplBase::Flags() const
 }
 
 
-inline WriterImplBase::AbstractDataWriter*
-WriterImplBase::DataWriter() const
-{
-	return fDataWriter;
-}
-
-
-inline void
-WriterImplBase::SetDataWriter(AbstractDataWriter* dataWriter)
-{
-	fDataWriter = dataWriter;
-}
-
-
 inline const WriterImplBase::PackageAttributeList&
 WriterImplBase::PackageAttributes() const
 {

src/add-ons/kernel/file_systems/packagefs/Jamfile

@@ -83,6 +83,8 @@ HAIKU_PACKAGE_FS_PACKAGE_READER_SOURCES =
 	PackageDataReader.cpp
 	PackageEntry.cpp
 	PackageEntryAttribute.cpp
+	PackageFileHeapAccessorBase.cpp
+	PackageFileHeapReader.cpp
 	PackageReaderImpl.cpp
 	ReaderImplBase.cpp
 

src/add-ons/kernel/file_systems/packagefs/nodes/UnpackingAttributeCookie.cpp

@@ -28,17 +28,9 @@ using BPackageKit::BHPKG::BFDDataReader;
 
 
 static status_t
-read_package_data(const PackageData& data, BDataReader* dataReader,
-	off_t offset, void* buffer, size_t* bufferSize)
+read_package_data(const PackageData& data, BDataReader* reader, off_t offset,
+	void* buffer, size_t* bufferSize)
 {
-	// create a PackageDataReader
-	BAbstractBufferedDataReader* reader;
-	status_t error = GlobalFactory::Default()->CreatePackageDataReader(
-		dataReader, data, reader);
-	if (error != B_OK)
-		RETURN_ERROR(error);
-	ObjectDeleter<BAbstractBufferedDataReader> readerDeleter(reader);
-
 	// check the offset
 	if (offset < 0 || (uint64)offset > data.UncompressedSize())
 		return B_BAD_VALUE;
@@ -131,19 +123,20 @@ UnpackingAttributeCookie::ReadAttribute(PackageNode* packageNode,
 	const PackageData& data = attribute->Data();
 	if (data.IsEncodedInline()) {
 		// inline data
-		BBufferDataReader dataReader(data.InlineData(), data.CompressedSize());
+		BBufferDataReader dataReader(data.InlineData(),
+			data.UncompressedSize());
 		return read_package_data(data, &dataReader, offset, buffer, bufferSize);
 	}
 
-	// data not inline -- open the package
+	// data not inline -- let the package create a data reader for us
 	Package* package = packageNode->GetPackage();
-	int fd = package->Open();
-	if (fd < 0)
-		RETURN_ERROR(fd);
-	PackageCloser packageCloser(package);
+	BAbstractBufferedDataReader* reader;
+	status_t error = package->CreateDataReader(data, reader);
+	if (error != B_OK)
+		return error;
+	ObjectDeleter<BAbstractBufferedDataReader> readerDeleter(reader);
 
-	BFDDataReader dataReader(fd);
-	return read_package_data(data, &dataReader, offset, buffer, bufferSize);
+	return read_package_data(data, reader, offset, buffer, bufferSize);
 }
 
 

src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.cpp

@@ -20,13 +20,10 @@ static const uint32 kMaxCachedBuffers = 32;
 
 GlobalFactory::GlobalFactory()
 	:
-	fBufferPool(BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB,
+	fBufferPool(BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB,
 		kMaxCachedBuffers),
-	fPackageDataReaderFactoryV1(&fBufferPool),
-	fPackageDataReaderFactoryV2(&fBufferPool)
+	fPackageDataReaderFactory(&fBufferPool)
 {
-	STATIC_ASSERT((int)BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB
-		== (int)BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB);
 }
 
 
@@ -73,20 +70,10 @@ GlobalFactory::Default()
 
 status_t
 GlobalFactory::CreatePackageDataReader(BDataReader* dataReader,
-	const PackageData& data, BAbstractBufferedDataReader*& _reader)
+	const PackageDataV1& data, BAbstractBufferedDataReader*& _reader)
 {
-	switch (data.Version()) {
-		case 1:
-			return fPackageDataReaderFactoryV1.CreatePackageDataReader(
-				dataReader, data.DataV1(), _reader);
-
-		case 2:
-			return fPackageDataReaderFactoryV2.CreatePackageDataReader(
-				dataReader, data.DataV2(), _reader);
-
-		default:
-			return B_NOT_SUPPORTED;
-	}
+	return fPackageDataReaderFactory.CreatePackageDataReader(dataReader, data,
+		_reader);
 }
 
 

src/add-ons/kernel/file_systems/packagefs/package/GlobalFactory.h

@@ -1,27 +1,24 @@
 /*
- * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de.
  * Distributed under the terms of the MIT License.
  */
 #ifndef GLOBAL_FACTORY_H
 #define GLOBAL_FACTORY_H
 
 
-#include <package/hpkg/PackageDataReader.h>
 #include <package/hpkg/v1/PackageDataReader.h>
 
 #include "BlockBufferPoolKernel.h"
+#include "PackageData.h"
 
 
+using BPackageKit::BHPKG::BBufferPool;
 using BPackageKit::BHPKG::BDataReader;
 using BPackageKit::BHPKG::BPackageData;
 using BPackageKit::BHPKG::BAbstractBufferedDataReader;
 
 typedef BPackageKit::BHPKG::V1::BPackageDataReaderFactory
 	BPackageDataReaderFactoryV1;
-typedef BPackageKit::BHPKG::BPackageDataReaderFactory
-	BPackageDataReaderFactoryV2;
-
-class PackageData;
 
 
 class GlobalFactory {
@@ -35,7 +32,7 @@ public:
 	static	GlobalFactory*		Default();
 
 			status_t			CreatePackageDataReader(BDataReader* dataReader,
-									const PackageData& data,
+									const PackageDataV1& data,
 									BAbstractBufferedDataReader*& _reader);
 
 private:
@@ -45,8 +42,7 @@ private:
 	static	GlobalFactory*		sDefaultInstance;
 
 			BlockBufferPoolKernel fBufferPool;
-			BPackageDataReaderFactoryV1 fPackageDataReaderFactoryV1;
-			BPackageDataReaderFactoryV2 fPackageDataReaderFactoryV2;
+			BPackageDataReaderFactoryV1 fPackageDataReaderFactory;
 };
 
 #endif	// GLOBAL_FACTORY_H

src/add-ons/kernel/file_systems/packagefs/package/Package.cpp

@@ -12,17 +12,21 @@
 #include <string.h>
 #include <unistd.h>
 
-#include <AutoDeleter.h>
 #include <package/hpkg/ErrorOutput.h>
+#include <package/hpkg/PackageDataReader.h>
 #include <package/hpkg/PackageEntry.h>
 #include <package/hpkg/PackageEntryAttribute.h>
-#include <package/hpkg/PackageReaderImpl.h>
 #include <package/hpkg/v1/PackageEntry.h>
 #include <package/hpkg/v1/PackageEntryAttribute.h>
+
+#include <AutoDeleter.h>
+#include <package/hpkg/PackageFileHeapReader.h>
+#include <package/hpkg/PackageReaderImpl.h>
 #include <package/hpkg/v1/PackageReaderImpl.h>
 #include <util/AutoLock.h>
 
 #include "DebugSupport.h"
+#include "GlobalFactory.h"
 #include "PackageDirectory.h"
 #include "PackageFile.h"
 #include "PackageSymlink.h"
@@ -32,10 +36,12 @@
 
 using namespace BPackageKit;
 
-typedef BPackageKit::BHPKG::BErrorOutput BErrorOutput;
-typedef BPackageKit::BHPKG::BPackageInfoAttributeValue
-	BPackageInfoAttributeValue;
-typedef BPackageKit::BHPKG::BPackageVersionData BPackageVersionData;
+using BPackageKit::BHPKG::BDataOutput;
+using BPackageKit::BHPKG::BErrorOutput;
+using BPackageKit::BHPKG::BFDDataReader;
+using BPackageKit::BHPKG::BPackageInfoAttributeValue;
+using BPackageKit::BHPKG::BPackageVersionData;
+using BPackageKit::BHPKG::BPrivate::PackageFileHeapReader;
 
 // current format version types
 typedef BPackageKit::BHPKG::BPackageContentHandler BPackageContentHandler;
@@ -581,6 +587,129 @@ private:
 };
 
 
+// #pragma mark - HeapReader
+
+
+struct Package::HeapReader {
+	virtual ~HeapReader()
+	{
+	}
+
+	virtual void UpdateFD(int fd) = 0;
+
+	virtual status_t CreateDataReader(const PackageData& data,
+		BAbstractBufferedDataReader*& _reader) = 0;
+};
+
+
+// #pragma mark - HeapReaderV1
+
+
+struct Package::HeapReaderV1 : public HeapReader, private BDataReader {
+public:
+	HeapReaderV1(int fd)
+		:
+		fFileReader(fd)
+	{
+	}
+
+	~HeapReaderV1()
+	{
+	}
+
+	virtual void UpdateFD(int fd)
+	{
+		fFileReader.SetFD(fd);
+	}
+
+	virtual status_t CreateDataReader(const PackageData& data,
+		BAbstractBufferedDataReader*& _reader)
+	{
+		return GlobalFactory::Default()->CreatePackageDataReader(this,
+			data.DataV1(), _reader);
+	}
+
+private:
+	// BDataReader
+
+	virtual status_t ReadData(off_t offset, void* buffer, size_t size)
+	{
+		return fFileReader.ReadData(offset, buffer, size);
+	}
+
+private:
+	BFDDataReader	fFileReader;
+};
+
+
+// #pragma mark - HeapReaderV2
+
+
+struct Package::HeapReaderV2 : public HeapReader,
+	private BAbstractBufferedDataReader, private BErrorOutput {
+public:
+	HeapReaderV2()
+		:
+		fHeapReader(NULL)
+	{
+	}
+
+	~HeapReaderV2()
+	{
+		delete fHeapReader;
+	}
+
+	status_t Init(const PackageFileHeapReader* heapReader, int fd)
+	{
+		fHeapReader = heapReader->Clone();
+		if (fHeapReader == NULL)
+			return B_NO_MEMORY;
+
+		fHeapReader->SetErrorOutput(this);
+		fHeapReader->SetFD(fd);
+
+		return B_OK;
+	}
+
+	virtual void UpdateFD(int fd)
+	{
+		fHeapReader->SetFD(fd);
+	}
+
+	virtual status_t CreateDataReader(const PackageData& data,
+		BAbstractBufferedDataReader*& _reader)
+	{
+		return BPackageKit::BHPKG::BPackageDataReaderFactory()
+			.CreatePackageDataReader(this, data.DataV2(), _reader);
+	}
+
+private:
+	// BAbstractBufferedDataReader
+
+	virtual status_t ReadData(off_t offset, void* buffer, size_t size)
+	{
+		return fHeapReader->ReadData(offset, buffer, size);
+	}
+
+	virtual status_t ReadDataToOutput(off_t offset, size_t size,
+		BDataOutput* output)
+	{
+		return fHeapReader->ReadDataToOutput(offset, size, output);
+	}
+
+private:
+	// BErrorOutput
+
+	virtual void PrintErrorVarArgs(const char* format, va_list args)
+	{
+// TODO:...
+	}
+
+private:
+	PackageFileHeapReader*	fHeapReader;
+};
+
+
 // #pragma mark - Package
 
 
@@ -595,6 +724,7 @@ Package::Package(::Volume* volume, dev_t deviceID, ino_t nodeID)
 	fLinkDirectory(NULL),
 	fFD(-1),
 	fOpenCount(0),
+	fHeapReader(NULL),
 	fNodeID(nodeID),
 	fDeviceID(deviceID)
 {
@@ -604,6 +734,8 @@ Package::Package(::Volume* volume, dev_t deviceID, ino_t nodeID)
 
 Package::~Package()
 {
+	delete fHeapReader;
+
 	while (PackageNode* node = fNodes.RemoveHead())
 		node->ReleaseReference();
 
@@ -652,7 +784,22 @@ Package::Load()
 			if (error != B_OK)
 				RETURN_ERROR(error);
 		
-			RETURN_ERROR(packageReader.ParseContent(&handler));
+			error = packageReader.ParseContent(&handler);
+			if (error != B_OK)
+				RETURN_ERROR(error);
+
+			// create a heap reader
+			HeapReaderV2* heapReader = new(std::nothrow) HeapReaderV2;
+			if (heapReader == NULL)
+				RETURN_ERROR(B_NO_MEMORY);
+
+			error = heapReader->Init(packageReader.HeapReader(), fd);
+			if (error != B_OK) {
+				RETURN_ERROR(error);
+			}
+
+			fHeapReader = heapReader;
+			return B_OK;
 		}
 
 		if (error != B_MISMATCHED_VALUES)
@@ -671,7 +818,16 @@ Package::Load()
 	if (error != B_OK)
 		RETURN_ERROR(error);
 
-	RETURN_ERROR(packageReader.ParseContent(&handler));
+	error = packageReader.ParseContent(&handler);
+	if (error != B_OK)
+		RETURN_ERROR(error);
+
+	// create a heap reader
+	fHeapReader = new(std::nothrow) HeapReaderV1(fd);
+	if (fHeapReader == NULL)
+		RETURN_ERROR(B_NO_MEMORY);
+
+	return B_OK;
 }
 
 
@@ -765,6 +921,10 @@ Package::Open()
 	}
 
 	fOpenCount = 1;
+
+	if (fHeapReader != NULL)
+		fHeapReader->UpdateFD(fFD);
+
 	return fFD;
 }
 
@@ -781,5 +941,19 @@ Package::Close()
 	if (--fOpenCount == 0) {
 		close(fFD);
 		fFD = -1;
+
+		if (fHeapReader != NULL)
+			fHeapReader->UpdateFD(fFD);
 	}
 }
+
+
+status_t
+Package::CreateDataReader(const PackageData& data,
+	BAbstractBufferedDataReader*& _reader)
+{
+	if (fHeapReader == NULL)
+		return B_BAD_VALUE;
+
+	return fHeapReader->CreateDataReader(data, _reader);
+}

src/add-ons/kernel/file_systems/packagefs/package/Package.h

@@ -6,6 +6,7 @@
 #define PACKAGE_H
 
 
+#include <package/hpkg/DataReader.h>
 #include <package/PackageArchitecture.h>
 
 #include <Referenceable.h>
@@ -23,6 +24,7 @@
 
 
 using BPackageKit::BPackageArchitecture;
+using BPackageKit::BHPKG::BAbstractBufferedDataReader;
 
 
 class PackageLinkDirectory;
@@ -82,6 +84,9 @@ public:
 			int					Open();
 			void				Close();
 
+			status_t			CreateDataReader(const PackageData& data,
+									BAbstractBufferedDataReader*& _reader);
+
 			const PackageNodeList& Nodes() const	{ return fNodes; }
 			const ResolvableList& Resolvables() const
 									{ return fResolvables; }
@@ -92,6 +97,9 @@ private:
 			struct LoaderErrorOutput;
 			struct LoaderContentHandler;
 			struct LoaderContentHandlerV1;
+			struct HeapReader;
+			struct HeapReaderV1;
+			struct HeapReaderV2;
 
 private:
 			mutex				fLock;
@@ -104,6 +112,7 @@ private:
 			PackageLinkDirectory* fLinkDirectory;
 			int					fFD;
 			uint32				fOpenCount;
+			HeapReader*			fHeapReader;
 			Package*			fFileNameHashTableNext;
 			ino_t				fNodeID;
 			dev_t				fDeviceID;

src/add-ons/kernel/file_systems/packagefs/package/PackageData.h

@@ -83,7 +83,7 @@ PackageData::CompressedSize() const
 {
 	if (fVersion == 1)
 		return DataV1().CompressedSize();
-	return DataV2().CompressedSize();
+	return DataV2().Size();
 }
 
 
@@ -92,7 +92,7 @@ PackageData::UncompressedSize() const
 {
 	if (fVersion == 1)
 		return DataV1().UncompressedSize();
-	return DataV2().UncompressedSize();
+	return DataV2().Size();
 }
 
 

src/add-ons/kernel/file_systems/packagefs/package/PackageFile.cpp

@@ -48,10 +48,10 @@ private:
 
 
 struct PackageFile::DataAccessor {
-	DataAccessor(PackageData* data)
+	DataAccessor(Package* package, PackageData* data)
 		:
+		fPackage(package),
 		fData(data),
-		fDataReader(NULL),
 		fReader(NULL),
 		fFileCache(NULL)
 	{
@@ -62,27 +62,15 @@ struct PackageFile::DataAccessor {
 	{
 		file_cache_delete(fFileCache);
 		delete fReader;
-		delete fDataReader;
 		mutex_destroy(&fLock);
 	}
 
 	status_t Init(dev_t deviceID, ino_t nodeID, int fd)
 	{
-		// create a BDataReader for the compressed data
-		if (fData->IsEncodedInline()) {
-			fDataReader = new(std::nothrow) BBufferDataReader(
-				fData->InlineData(), fData->CompressedSize());
-		} else
-			fDataReader = new(std::nothrow) BFDDataReader(fd);
-
-		if (fDataReader == NULL)
-			RETURN_ERROR(B_NO_MEMORY);
-
-		// create a PackageDataReader
-		status_t error = GlobalFactory::Default()->CreatePackageDataReader(
-			fDataReader, *fData, fReader);
+		// create a reader for the data
+		status_t error = fPackage->CreateDataReader(*fData, fReader);
 		if (error != B_OK)
-			RETURN_ERROR(error);
+			return error;
 
 		// create a file cache
 		fFileCache = file_cache_create(deviceID, nodeID,
@@ -128,8 +116,8 @@ struct PackageFile::DataAccessor {
 
 private:
 	mutex							fLock;
+	Package*						fPackage;
 	PackageData*					fData;
-	BDataReader*					fDataReader;
 	BAbstractBufferedDataReader*	fReader;
 	void*							fFileCache;
 };
@@ -168,7 +156,7 @@ PackageFile::VFSInit(dev_t deviceID, ino_t nodeID)
 	PackageCloser packageCloser(fPackage);
 
 	// create the data accessor
-	fDataAccessor = new(std::nothrow) DataAccessor(&fData);
+	fDataAccessor = new(std::nothrow) DataAccessor(GetPackage(), &fData);
 	if (fDataAccessor == NULL)
 		RETURN_ERROR(B_NO_MEMORY);
 

src/bin/package/command_dump.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de.
  * Distributed under the terms of the MIT License.
  */
 

src/bin/package/command_extract.cpp

@@ -44,7 +44,9 @@
 using BPackageKit::BHPKG::BAbstractBufferedDataReader;
 using BPackageKit::BHPKG::BBlockBufferPoolNoLock;
 using BPackageKit::BHPKG::BBufferDataReader;
+using BPackageKit::BHPKG::BBufferPool;
 using BPackageKit::BHPKG::BDataReader;
+using BPackageKit::BHPKG::BErrorOutput;
 using BPackageKit::BHPKG::BFDDataReader;
 using BPackageKit::BHPKG::BPackageInfoAttributeValue;
 using BPackageKit::BHPKG::BStandardErrorOutput;
@@ -54,38 +56,93 @@ struct VersionPolicyV1 {
 	typedef BPackageKit::BHPKG::V1::BPackageContentHandler
 		PackageContentHandler;
 	typedef BPackageKit::BHPKG::V1::BPackageData PackageData;
-	typedef BPackageKit::BHPKG::V1::BPackageDataReaderFactory
-		PackageDataReaderFactory;
 	typedef BPackageKit::BHPKG::V1::BPackageEntry PackageEntry;
 	typedef BPackageKit::BHPKG::V1::BPackageEntryAttribute
 		PackageEntryAttribute;
 	typedef BPackageKit::BHPKG::V1::BPackageReader PackageReader;
+	typedef BDataReader HeapReaderBase;
 
-	static const int kDefaultDataChunkSize
-		= BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB;
+	static inline size_t BufferSize()
+	{
+		return BPackageKit::BHPKG::V1::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB;
+	}
 
 	static inline const char* PackageInfoFileName()
 	{
 		return BPackageKit::BHPKG::V1::B_HPKG_PACKAGE_INFO_FILE_NAME;
 	}
+
+	static inline uint64 PackageDataCompressedSize(const PackageData& data)
+	{
+		return data.CompressedSize();
+	}
+
+	static inline uint64 PackageDataUncompressedSize(const PackageData& data)
+	{
+		return data.UncompressedSize();
+	}
+
+	static status_t GetHeapReader(PackageReader& packageReader,
+		HeapReaderBase*& _heapReader, bool& _mustDelete)
+	{
+		_heapReader = new(std::nothrow) BFDDataReader(
+			packageReader.PackageFileFD());
+		_mustDelete = false;
+		return _heapReader != NULL ? B_OK : B_NO_MEMORY;
+	}
+
+	static status_t CreatePackageDataReader(BBufferPool* bufferPool,
+		HeapReaderBase* heapReader, const PackageData& data,
+		BAbstractBufferedDataReader*& _reader)
+	{
+		return BPackageKit::BHPKG::V1::BPackageDataReaderFactory(bufferPool)
+			.CreatePackageDataReader(heapReader, data, _reader);
+	}
 };
 
 struct VersionPolicyV2 {
 	typedef BPackageKit::BHPKG::BPackageContentHandler PackageContentHandler;
 	typedef BPackageKit::BHPKG::BPackageData PackageData;
-	typedef BPackageKit::BHPKG::BPackageDataReaderFactory
-		PackageDataReaderFactory;
 	typedef BPackageKit::BHPKG::BPackageEntry PackageEntry;
 	typedef BPackageKit::BHPKG::BPackageEntryAttribute PackageEntryAttribute;
 	typedef BPackageKit::BHPKG::BPackageReader PackageReader;
+	typedef BAbstractBufferedDataReader HeapReaderBase;
 
-	static const int kDefaultDataChunkSize
-		= BPackageKit::BHPKG::B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB;
+	static inline size_t BufferSize()
+	{
+		return 64 * 1024;
+	}
 
 	static inline const char* PackageInfoFileName()
 	{
 		return BPackageKit::BHPKG::B_HPKG_PACKAGE_INFO_FILE_NAME;
 	}
+
+	static inline uint64 PackageDataCompressedSize(const PackageData& data)
+	{
+		return data.Size();
+	}
+
+	static inline uint64 PackageDataUncompressedSize(const PackageData& data)
+	{
+		return data.Size();
+	}
+
+	static status_t GetHeapReader(PackageReader& packageReader,
+		HeapReaderBase*& _heapReader, bool& _mustDelete)
+	{
+		_heapReader = packageReader.HeapReader();
+		_mustDelete = false;
+		return B_OK;
+	}
+
+	static status_t CreatePackageDataReader(BBufferPool* bufferPool,
+		HeapReaderBase* heapReader, const PackageData& data,
+		BAbstractBufferedDataReader*& _reader)
+	{
+		return BPackageKit::BHPKG::BPackageDataReaderFactory()
+			.CreatePackageDataReader(heapReader, data, _reader);
+	}
 };
 
 
@@ -227,10 +284,11 @@ private:
 
 template<typename VersionPolicy>
 struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler {
-	PackageContentExtractHandler(int packageFileFD)
+	PackageContentExtractHandler(BBufferPool* bufferPool,
+	typename VersionPolicy::HeapReaderBase* heapReader)
 		:
-		fBufferPool(VersionPolicy::kDefaultDataChunkSize, 2),
-		fPackageFileReader(packageFileFD),
+		fBufferPool(bufferPool),
+		fPackageFileReader(heapReader),
 		fDataBuffer(NULL),
 		fDataBufferSize(0),
 		fRootFilterEntry(NULL, NULL, true),
@@ -247,11 +305,7 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler {
 
 	status_t Init()
 	{
-		status_t error = fBufferPool.Init();
-		if (error != B_OK)
-			return error;
-
-		error = fRootFilterEntry.Init();
+		status_t error = fRootFilterEntry.Init();
 		if (error != B_OK)
 			return error;
 
@@ -434,15 +488,8 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler {
 			}
 
 			// write data
-			status_t error;
-			const typename VersionPolicy::PackageData& data = entry->Data();
-			if (data.IsEncodedInline()) {
-				BBufferDataReader dataReader(data.InlineData(),
-					data.CompressedSize());
-				error = _ExtractFileData(&dataReader, data, fd);
-			} else
-				error = _ExtractFileData(&fPackageFileReader, data, fd);
-
+			status_t error = _ExtractFileData(fPackageFileReader, entry->Data(),
+				fd);
 			if (error != B_OK)
 				return error;
 		} else if (S_ISLNK(entry->Mode())) {
@@ -528,14 +575,8 @@ struct PackageContentExtractHandler : VersionPolicy::PackageContentHandler {
 		}
 
 		// write data
-		status_t error;
-		const typename VersionPolicy::PackageData& data = attribute->Data();
-		if (data.IsEncodedInline()) {
-			BBufferDataReader dataReader(data.InlineData(),
-				data.CompressedSize());
-			error = _ExtractFileData(&dataReader, data, fd);
-		} else
-			error = _ExtractFileData(&fPackageFileReader, data, fd);
+		status_t error = _ExtractFileData(fPackageFileReader, attribute->Data(),
+			fd);
 
 		fs_close_attr(fd);
 
@@ -642,20 +683,20 @@ private:
 		return path;
 	}
 
-	status_t _ExtractFileData(BDataReader* dataReader,
+	status_t _ExtractFileData(
+		typename VersionPolicy::HeapReaderBase* dataReader,
 		const typename VersionPolicy::PackageData& data, int fd)
 	{
 		// create a PackageDataReader
 		BAbstractBufferedDataReader* reader;
-		status_t error = typename VersionPolicy::PackageDataReaderFactory(
-				&fBufferPool)
-			.CreatePackageDataReader(dataReader, data, reader);
+		status_t error = VersionPolicy::CreatePackageDataReader(fBufferPool,
+			dataReader, data, reader);
 		if (error != B_OK)
 			return error;
 		ObjectDeleter<BAbstractBufferedDataReader> readerDeleter(reader);
 
 		// write the data
-		off_t bytesRemaining = data.UncompressedSize();
+		off_t bytesRemaining = VersionPolicy::PackageDataUncompressedSize(data);
 		off_t offset = 0;
 		while (bytesRemaining > 0) {
 			// read
@@ -688,14 +729,14 @@ private:
 	}
 
 private:
-	BBlockBufferPoolNoLock	fBufferPool;
-	BFDDataReader			fPackageFileReader;
-	void*					fDataBuffer;
-	size_t					fDataBufferSize;
-	Entry					fRootFilterEntry;
-	int						fBaseDirectory;
-	const char*				fInfoFileName;
-	bool					fErrorOccurred;
+	BBufferPool*							fBufferPool;
+	typename VersionPolicy::HeapReaderBase*	fPackageFileReader;
+	void*									fDataBuffer;
+	size_t									fDataBufferSize;
+	Entry									fRootFilterEntry;
+	int										fBaseDirectory;
+	const char*								fInfoFileName;
+	bool									fErrorOccurred;
 };
 
 
@@ -707,6 +748,12 @@ do_extract(const char* packageFileName, const char* changeToDirectory,
 {
 	// open package
 	BStandardErrorOutput errorOutput;
+	BBlockBufferPoolNoLock bufferPool(VersionPolicy::BufferSize(), 2);
+	if (bufferPool.Init() != B_OK) {
+		errorOutput.PrintError("Error: Out of memory!\n");
+		exit(1);
+	}
+
 	typename VersionPolicy::PackageReader packageReader(&errorOutput);
 	status_t error = packageReader.Init(packageFileName);
 	if (error != B_OK) {
@@ -715,8 +762,20 @@ do_extract(const char* packageFileName, const char* changeToDirectory,
 		exit(1);
 	}
 
-	PackageContentExtractHandler<VersionPolicy> handler(
-		packageReader.PackageFileFD());
+	typename VersionPolicy::HeapReaderBase* heapReader;
+	bool mustDeleteHeapReader;
+	error = VersionPolicy::GetHeapReader(packageReader, heapReader,
+		mustDeleteHeapReader);
+	if (error != B_OK) {
+		fprintf(stderr, "Error: Failed to create heap reader: \"%s\"\n",
+			strerror(error));
+		exit(1);
+	}
+	ObjectDeleter<BDataReader> heapReaderDeleter(
+		mustDeleteHeapReader ? heapReader : NULL);
+
+	PackageContentExtractHandler<VersionPolicy> handler(&bufferPool,
+		heapReader);
 	error = handler.Init();
 	if (error != B_OK)
 		exit(1);
@@ -727,7 +786,7 @@ do_extract(const char* packageFileName, const char* changeToDirectory,
 		for (int i = 0; i < explicitEntryCount; i++) {
 			const char* entryName = explicitEntries[i];
 			if (entryName[0] == '\0' || entryName[0] == '/') {
-				fprintf(stderr, "Error: Invalid entry name: \"%s\".",
+				fprintf(stderr, "Error: Invalid entry name: \"%s\"\n",
 					entryName);
 				exit(1);
 			}

src/bin/package/command_list.cpp

@@ -29,7 +29,9 @@
 
 
 using namespace BPackageKit;
+using BPackageKit::BHPKG::BErrorOutput;
 using BPackageKit::BHPKG::BPackageInfoAttributeValue;
+using BPackageKit::BHPKG::BStandardErrorOutput;
 
 
 struct VersionPolicyV1 {
@@ -39,6 +41,12 @@ struct VersionPolicyV1 {
 	typedef BPackageKit::BHPKG::V1::BPackageEntryAttribute
 		PackageEntryAttribute;
 	typedef BPackageKit::BHPKG::V1::BPackageReader PackageReader;
+
+	static inline uint64 PackageDataSize(
+		const BPackageKit::BHPKG::V1::BPackageData& data)
+	{
+		return data.UncompressedSize();
+	}
 };
 
 struct VersionPolicyV2 {
@@ -46,6 +54,12 @@ struct VersionPolicyV2 {
 	typedef BPackageKit::BHPKG::BPackageEntry PackageEntry;
 	typedef BPackageKit::BHPKG::BPackageEntryAttribute PackageEntryAttribute;
 	typedef BPackageKit::BHPKG::BPackageReader PackageReader;
+
+	static inline uint64 PackageDataSize(
+		const BPackageKit::BHPKG::BPackageData& data)
+	{
+		return data.Size();
+	}
 };
 
 
@@ -67,7 +81,8 @@ struct PackageContentListHandler : VersionPolicy::PackageContentHandler {
 
 		// name and size
 		printf("%-*s", indentation < 32 ? 32 - indentation : 0, entry->Name());
-		printf("  %8llu", (unsigned long long)entry->Data().UncompressedSize());
+		printf("  %8llu",
+			(unsigned long long)VersionPolicy::PackageDataSize(entry->Data()));
 
 		// time
 		struct tm* time = localtime(&entry->ModifiedTime().tv_sec);
@@ -113,7 +128,8 @@ struct PackageContentListHandler : VersionPolicy::PackageContentHandler {
 		printf("%*s<", indentation, "");
 		printf("%-*s  %8llu", indentation < 31 ? 31 - indentation : 0,
 			attribute->Name(),
-			(unsigned long long)attribute->Data().UncompressedSize());
+			(unsigned long long)VersionPolicy::PackageDataSize(
+				attribute->Data()));
 
 		uint32 type = attribute->Type();
 		if (isprint(type & 0xff) && isprint((type >> 8) & 0xff)
@@ -352,7 +368,7 @@ do_list(const char* packageFileName, bool listAttributes, bool filePathsOnly,
 	bool ignoreVersionError)
 {
 	// open package
-	BHPKG::BStandardErrorOutput errorOutput;
+	BStandardErrorOutput errorOutput;
 	typename VersionPolicy::PackageReader packageReader(&errorOutput);
 	status_t error = packageReader.Init(packageFileName);
 	if (error != B_OK) {

src/build/libpackage/Jamfile

@@ -28,6 +28,7 @@ HPKG_SOURCES =
 	PoolBuffer.cpp
 	DataOutput.cpp
 	DataReader.cpp
+	DataWriters.cpp
 	ErrorOutput.cpp
 	FDDataReader.cpp
 	HPKGDefs.cpp
@@ -36,6 +37,9 @@ HPKG_SOURCES =
 	PackageDataReader.cpp
 	PackageEntry.cpp
 	PackageEntryAttribute.cpp
+	PackageFileHeapAccessorBase.cpp
+	PackageFileHeapReader.cpp
+	PackageFileHeapWriter.cpp
 	PackageReader.cpp
 	PackageReaderImpl.cpp
 	PackageWriter.cpp

src/kits/package/Jamfile

@@ -18,6 +18,7 @@ HPKG_SOURCES =
 	BufferPool.cpp
 	DataOutput.cpp
 	DataReader.cpp
+	DataWriters.cpp
 	ErrorOutput.cpp
 	FDDataReader.cpp
 	HPKGDefs.cpp
@@ -26,6 +27,8 @@ HPKG_SOURCES =
 	PackageDataReader.cpp
 	PackageEntry.cpp
 	PackageEntryAttribute.cpp
+	PackageFileHeapAccessorBase.cpp
+	PackageFileHeapReader.cpp
 	PackageFileHeapWriter.cpp
 	PackageReader.cpp
 	PackageReaderImpl.cpp

src/kits/package/hpkg/DataWriters.cpp

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2011, Oliver Tappe <zooey@hirschkaefer.de>
+ * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <package/hpkg/DataWriters.h>
+
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <package/hpkg/ErrorOutput.h>
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+namespace BPrivate {
+
+
+// #pragma mark - AbstractDataWriter
+
+
+AbstractDataWriter::AbstractDataWriter()
+	:
+	fBytesWritten(0)
+{
+}
+
+
+AbstractDataWriter::~AbstractDataWriter()
+{
+}
+
+
+// #pragma mark - FDDataWriter
+
+
+FDDataWriter::FDDataWriter(int fd, off_t offset, BErrorOutput* errorOutput)
+	:
+	fFD(fd),
+	fOffset(offset),
+	fErrorOutput(errorOutput)
+{
+}
+
+
+status_t
+FDDataWriter::WriteDataNoThrow(const void* buffer, size_t size)
+{
+	ssize_t bytesWritten = pwrite(fFD, buffer, size, fOffset);
+	if (bytesWritten < 0) {
+		fErrorOutput->PrintError(
+			"WriteDataNoThrow(%p, %lu) failed to write data: %s\n", buffer,
+			size, strerror(errno));
+		return errno;
+	}
+	if ((size_t)bytesWritten != size) {
+		fErrorOutput->PrintError(
+			"WriteDataNoThrow(%p, %lu) failed to write all data\n", buffer,
+			size);
+		return B_ERROR;
+	}
+
+	fOffset += size;
+	fBytesWritten += size;
+	return B_OK;
+}
+
+
+// #pragma mark - ZlibDataWriter
+
+
+ZlibDataWriter::ZlibDataWriter(AbstractDataWriter* dataWriter)
+	:
+	fDataWriter(dataWriter),
+	fCompressor(this)
+{
+}
+
+
+void
+ZlibDataWriter::Init()
+{
+	status_t error = fCompressor.Init();
+	if (error != B_OK)
+		throw status_t(error);
+}
+
+
+void
+ZlibDataWriter::Finish()
+{
+	status_t error = fCompressor.Finish();
+	if (error != B_OK)
+		throw status_t(error);
+}
+
+
+status_t
+ZlibDataWriter::WriteDataNoThrow(const void* buffer,
+	size_t size)
+{
+	status_t error = fCompressor.CompressNext(buffer, size);
+	if (error == B_OK)
+		fBytesWritten += size;
+	return error;
+}
+
+
+status_t
+ZlibDataWriter::WriteData(const void* buffer, size_t size)
+{
+	return fDataWriter->WriteDataNoThrow(buffer, size);
+}
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit

src/kits/package/hpkg/PackageContentHandler.cpp

@@ -30,9 +30,6 @@ static const char* kAttributeNames[B_HPKG_ATTRIBUTE_ID_ENUM_COUNT + 1] = {
 	"file:attribute",
 	"file:attribute:type",
 	"data",
-	"data:compression",
-	"data:size",
-	"data:chunk_size",
 	"symlink:path",
 	"package:name",
 	"package:summary",

src/kits/package/hpkg/PackageData.cpp

@@ -21,10 +21,7 @@ using namespace BPrivate;
 
 BPackageData::BPackageData()
 	:
-	fCompressedSize(0),
-	fUncompressedSize(0),
-	fChunkSize(0),
-	fCompression(B_HPKG_COMPRESSION_NONE),
+	fSize(0),
 	fEncodedInline(true)
 {
 }
@@ -33,7 +30,7 @@ BPackageData::BPackageData()
 void
 BPackageData::SetData(uint64 size, uint64 offset)
 {
-	fUncompressedSize = fCompressedSize = size;
+	fSize = size;
 	fOffset = offset;
 	fEncodedInline = false;
 }
@@ -42,7 +39,7 @@ BPackageData::SetData(uint64 size, uint64 offset)
 void
 BPackageData::SetData(uint8 size, const void* data)
 {
-	fUncompressedSize = fCompressedSize = size;
+	fSize = size;
 	if (size > 0)
 		memcpy(fInlineData, data, size);
 	fEncodedInline = true;

src/kits/package/hpkg/PackageDataReader.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de.
  * Distributed under the terms of the MIT License.
  */
 
@@ -12,9 +12,6 @@
 #include <new>
 
 #include <package/hpkg/HPKGDefsPrivate.h>
-#include <package/hpkg/BufferPool.h>
-#include <package/hpkg/PoolBuffer.h>
-#include <package/hpkg/DataOutput.h>
 #include <package/hpkg/PackageData.h>
 #include <package/hpkg/ZlibDecompressor.h>
 
@@ -27,59 +24,20 @@ namespace BHPKG {
 using namespace BPrivate;
 
 
-// minimum/maximum zlib chunk size we consider sane
-static const size_t kMinSaneZlibChunkSize = 1024;
-static const size_t kMaxSaneZlibChunkSize = 10 * 1024 * 1024;
-
-// maximum number of entries in the zlib offset table buffer
-static const uint32 kMaxZlibOffsetTableBufferSize = 512;
-
-static const size_t kUncompressedReaderBufferSize
-	= B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB;
+// #pragma mark - PackageDataHeapReader
 
 
-// #pragma mark - BPackageDataReader
-
-
-class PackageDataReader : public BAbstractBufferedDataReader {
+class PackageDataHeapReader : public BAbstractBufferedDataReader {
 public:
-	PackageDataReader(BDataReader* dataReader)
+	PackageDataHeapReader(BAbstractBufferedDataReader* dataReader,
+		const BPackageData& data)
 		:
-		fDataReader(dataReader)
+		fDataReader(dataReader),
+		fOffset(data.Offset()),
+		fSize(data.Size())
 	{
 	}
 
-	virtual ~PackageDataReader()
-	{
-	}
-
-	virtual status_t Init(const BPackageData& data) = 0;
-
-protected:
-			BDataReader*			fDataReader;
-};
-
-
-// #pragma mark - UncompressedPackageDataReader
-
-
-class UncompressedPackageDataReader : public PackageDataReader {
-public:
-	UncompressedPackageDataReader(BDataReader* dataReader,
-		BBufferPool* bufferPool)
-		:
-		PackageDataReader(dataReader),
-		fBufferPool(bufferPool)
-	{
-	}
-
-	status_t Init(const BPackageData& data)
-	{
-		fOffset = data.Offset();
-		fSize = data.UncompressedSize();
-		return B_OK;
-	}
-
 	virtual status_t ReadData(off_t offset, void* buffer, size_t size)
 	{
 		if (size == 0)
@@ -106,332 +64,55 @@ public:
 		if ((uint64)offset > fSize || size > fSize - offset)
 			return B_BAD_VALUE;
 
-		// get a temporary buffer
-		PoolBuffer* buffer = fBufferPool->GetBuffer(
-			kUncompressedReaderBufferSize);
-		if (buffer == NULL)
-			return B_NO_MEMORY;
-		PoolBufferPutter bufferPutter(fBufferPool, &buffer);
-
-		while (size > 0) {
-			// read into the buffer
-			size_t toRead = std::min(size, buffer->Size());
-			status_t error = fDataReader->ReadData(fOffset + offset,
-				buffer->Buffer(), toRead);
-			if (error != B_OK)
-				return error;
-
-			// write to the output
-			error = output->WriteData(buffer->Buffer(), toRead);
-			if (error != B_OK)
-				return error;
-
-			offset += toRead;
-			size -= toRead;
-		}
-
-		return B_OK;
+		return fDataReader->ReadDataToOutput(fOffset + offset, size, output);
 	}
 
 private:
-	BBufferPool*	fBufferPool;
-	uint64			fOffset;
-	uint64			fSize;
+	BAbstractBufferedDataReader*	fDataReader;
+	uint64							fOffset;
+	uint64							fSize;
 };
 
 
-// #pragma mark - ZlibPackageDataReader
+// #pragma mark - PackageDataHeapReader
 
 
-class ZlibPackageDataReader : public PackageDataReader {
+class PackageDataInlineReader : public BBufferDataReader {
 public:
-	ZlibPackageDataReader(BDataReader* dataReader, BBufferPool* bufferPool)
+	PackageDataInlineReader(const BPackageData& data)
 		:
-		PackageDataReader(dataReader),
-		fBufferPool(bufferPool),
-		fUncompressBuffer(NULL),
-		fOffsetTable(NULL)
+		BBufferDataReader(fData.InlineData(), data.Size()),
+		fData(data)
 	{
 	}
 
-	~ZlibPackageDataReader()
-	{
-		delete[] fOffsetTable;
-
-		fBufferPool->PutBuffer(&fUncompressBuffer);
-	}
-
-	status_t Init(const BPackageData& data)
-	{
-		fOffset = data.Offset();
-		fCompressedSize = data.CompressedSize();
-		fUncompressedSize = data.UncompressedSize();
-		fChunkSize = data.ChunkSize();
-
-		// validate chunk size
-		if (fChunkSize == 0)
-			fChunkSize = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB;
-		if (fChunkSize < kMinSaneZlibChunkSize
-			|| fChunkSize > kMaxSaneZlibChunkSize) {
-			return B_BAD_DATA;
-		}
-
-		fChunkCount = (fUncompressedSize + (fChunkSize - 1)) / fChunkSize;
-		fOffsetTableSize = (fChunkCount - 1) * sizeof(uint64);
-		if (fOffsetTableSize >= fCompressedSize)
-			return B_BAD_DATA;
-
-		// allocate a buffer for the offset table
-		if (fChunkCount > 1) {
-			fOffsetTableBufferEntryCount = std::min(fChunkCount - 1,
-				(uint64)kMaxZlibOffsetTableBufferSize);
-			fOffsetTable = new(std::nothrow) uint64[
-				fOffsetTableBufferEntryCount];
-			if (fOffsetTable == NULL)
-				return B_NO_MEMORY;
-
-			fOffsetTableIndex = -1;
-				// mark the table content invalid
-		} else
-			fChunkSize = fUncompressedSize;
-
-		// mark uncompressed content invalid
-		fUncompressedChunk = -1;
-
-		return B_OK;
-	}
-
-	virtual status_t ReadDataToOutput(off_t offset, size_t size,
-		BDataOutput* output)
-	{
-		// check offset and size
-		if (size == 0)
-			return B_OK;
-
-		if (offset < 0)
-			return B_BAD_VALUE;
-
-		if ((uint64)offset > fUncompressedSize
-			|| size > fUncompressedSize - offset) {
-			return B_BAD_VALUE;
-		}
-
-		// get our uncompressed chunk buffer back, if possible
-		bool newBuffer;
-		if (fBufferPool->GetBuffer(fChunkSize, &fUncompressBuffer, &newBuffer)
-				== NULL) {
-			return B_NO_MEMORY;
-		}
-		PoolBufferPutter uncompressBufferPutter(fBufferPool,
-			&fUncompressBuffer);
-
-		if (newBuffer)
-			fUncompressedChunk = -1;
-
-		// uncompress
-		int64 chunkIndex = offset / fChunkSize;
-		off_t chunkOffset = chunkIndex * fChunkSize;
-		size_t inChunkOffset = offset - chunkOffset;
-
-		while (size > 0) {
-			// read and uncompress the chunk
-			status_t error = _ReadChunk(chunkIndex);
-			if (error != B_OK)
-				return error;
-
-			// write data to output
-			size_t toCopy = std::min(size, (size_t)fChunkSize - inChunkOffset);
-			error = output->WriteData(
-				(uint8*)fUncompressBuffer->Buffer() + inChunkOffset, toCopy);
-			if (error != B_OK)
-				return error;
-
-			size -= toCopy;
-
-			chunkIndex++;
-			chunkOffset += fChunkSize;
-			inChunkOffset = 0;
-		}
-
-		return B_OK;
-	}
-
 private:
-	status_t _ReadChunk(int64 chunkIndex)
-	{
-		if (chunkIndex == fUncompressedChunk)
-			return B_OK;
-
-		// get the chunk offset and size
-		uint64 offset;
-		uint32 compressedSize;
-		status_t error = _GetCompressedChunkOffsetAndSize(chunkIndex, offset,
-			compressedSize);
-		if (error != B_OK)
-			return error;
-
-		uint32 uncompressedSize = (uint64)chunkIndex + 1 < fChunkCount
-			? fChunkSize : fUncompressedSize - chunkIndex * fChunkSize;
-
-		// read the chunk
-		if (compressedSize == uncompressedSize) {
-			// the chunk is not compressed -- read it directly into the
-			// uncompressed buffer
-			error = fDataReader->ReadData(offset, fUncompressBuffer->Buffer(),
-				compressedSize);
-		} else {
-			// read to a read buffer and uncompress
-			PoolBuffer* readBuffer = fBufferPool->GetBuffer(fChunkSize);
-			if (readBuffer == NULL)
-				return B_NO_MEMORY;
-			PoolBufferPutter readBufferPutter(fBufferPool, readBuffer);
-
-			error = fDataReader->ReadData(offset, readBuffer->Buffer(),
-				compressedSize);
-			if (error != B_OK)
-				return error;
-
-			size_t actuallyUncompressedSize;
-			error = ZlibDecompressor::DecompressSingleBuffer(
-				readBuffer->Buffer(), compressedSize,
-				fUncompressBuffer->Buffer(), uncompressedSize,
-				actuallyUncompressedSize);
-			if (error == B_OK && actuallyUncompressedSize != uncompressedSize)
-				error = B_BAD_DATA;
-		}
-
-		if (error != B_OK) {
-			// error reading/decompressing data -- mark the cached data invalid
-			fUncompressedChunk = -1;
-			return error;
-		}
-
-		fUncompressedChunk = chunkIndex;
-		return B_OK;
-	}
-
-	status_t _GetCompressedChunkOffsetAndSize(int64 chunkIndex, uint64& _offset,
-		uint32& _size)
-	{
-		// get the offset
-		uint64 offset;
-		if (chunkIndex == 0) {
-			// first chunk is at 0
-			offset = 0;
-		} else {
-			status_t error = _GetCompressedChunkRelativeOffset(chunkIndex,
-				offset);
-			if (error != B_OK)
-				return error;
-		}
-
-		// get the end offset
-		uint64 endOffset;
-		if ((uint64)chunkIndex + 1 == fChunkCount) {
-			// last chunk end with the end of the data
-			endOffset = fCompressedSize - fOffsetTableSize;
-		} else {
-			status_t error = _GetCompressedChunkRelativeOffset(chunkIndex + 1,
-				endOffset);
-			if (error != B_OK)
-				return error;
-		}
-
-		// sanity check
-		if (endOffset < offset)
-			return B_BAD_DATA;
-
-		_offset = fOffset + fOffsetTableSize + offset;
-		_size = endOffset - offset;
-		return B_OK;
-	}
-
-	status_t _GetCompressedChunkRelativeOffset(int64 chunkIndex,
-		uint64& _offset)
-	{
-		if (fOffsetTableIndex < 0 || fOffsetTableIndex > chunkIndex
-			|| fOffsetTableIndex + fOffsetTableBufferEntryCount <= chunkIndex) {
-			// read the table at the given index, or, if we can, the whole table
-			int64 readAtIndex = fChunkCount - 1 > fOffsetTableBufferEntryCount
-				? chunkIndex : 1;
-			uint32 entriesToRead = std::min(
-				(uint64)fOffsetTableBufferEntryCount,
-				fChunkCount - readAtIndex);
-
-			status_t error = fDataReader->ReadData(
-				fOffset + (readAtIndex - 1) * sizeof(uint64),
-				fOffsetTable, entriesToRead * sizeof(uint64));
-			if (error != B_OK) {
-				fOffsetTableIndex = -1;
-				return error;
-			}
-
-			fOffsetTableIndex = readAtIndex;
-		}
-
-		// get and check the offset
-		_offset = fOffsetTable[chunkIndex - fOffsetTableIndex];
-		if (_offset > fCompressedSize - fOffsetTableSize)
-			return B_BAD_DATA;
-
-		return B_OK;
-	}
-
-private:
-	BBufferPool*	fBufferPool;
-	PoolBuffer*		fUncompressBuffer;
-	int64			fUncompressedChunk;
-
-	uint64			fOffset;
-	uint64			fUncompressedSize;
-	uint64			fCompressedSize;
-	uint64			fOffsetTableSize;
-	uint64			fChunkCount;
-	uint32			fChunkSize;
-	uint32			fOffsetTableBufferEntryCount;
-	uint64*			fOffsetTable;
-	int32			fOffsetTableIndex;
+	BPackageData	fData;
 };
 
 
 // #pragma mark - BPackageDataReaderFactory
 
 
-BPackageDataReaderFactory::BPackageDataReaderFactory(BBufferPool* bufferPool)
-	:
-	fBufferPool(bufferPool)
+BPackageDataReaderFactory::BPackageDataReaderFactory()
 {
 }
 
 
 status_t
-BPackageDataReaderFactory::CreatePackageDataReader(BDataReader* dataReader,
-	const BPackageData& data, BAbstractBufferedDataReader*& _reader)
+BPackageDataReaderFactory::CreatePackageDataReader(
+	BAbstractBufferedDataReader* dataReader, const BPackageData& data,
+	BAbstractBufferedDataReader*& _reader)
 {
-	PackageDataReader* reader;
-
-	switch (data.Compression()) {
-		case B_HPKG_COMPRESSION_NONE:
-			reader = new(std::nothrow) UncompressedPackageDataReader(
-				dataReader, fBufferPool);
-			break;
-		case B_HPKG_COMPRESSION_ZLIB:
-			reader = new(std::nothrow) ZlibPackageDataReader(dataReader,
-				fBufferPool);
-			break;
-		default:
-			return B_BAD_VALUE;
-	}
+	BAbstractBufferedDataReader* reader;
+	if (data.IsEncodedInline())
+		reader = new(std::nothrow) PackageDataInlineReader(data);
+	else
+		reader = new(std::nothrow) PackageDataHeapReader(dataReader, data);
 
 	if (reader == NULL)
 		return B_NO_MEMORY;
 
-	status_t error = reader->Init(data);
-	if (error != B_OK) {
-		delete reader;
-		return error;
-	}
-
 	_reader = reader;
 	return B_OK;
 }

src/kits/package/hpkg/PackageFileHeapAccessorBase.cpp

@@ -0,0 +1,247 @@
+/*
+ * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <package/hpkg/PackageFileHeapAccessorBase.h>
+
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <new>
+
+#include <package/hpkg/DataOutput.h>
+#include <package/hpkg/ErrorOutput.h>
+
+#include <AutoDeleter.h>
+#include <package/hpkg/ZlibDecompressor.h>
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+namespace BPrivate {
+
+
+// #pragma mark - OffsetArray
+
+
+PackageFileHeapAccessorBase::OffsetArray::OffsetArray()
+	:
+	fOffsets(NULL)
+{
+}
+
+
+PackageFileHeapAccessorBase::OffsetArray::~OffsetArray()
+{
+	delete[] fOffsets;
+}
+
+
+bool
+PackageFileHeapAccessorBase::OffsetArray::InitChunksOffsets(
+	size_t totalChunkCount, size_t baseIndex, const uint16* chunkSizes,
+	size_t chunkCount)
+{
+	if (totalChunkCount <= 1)
+		return true;
+
+	if (fOffsets == NULL) {
+		fOffsets = new(std::nothrow) uint32[totalChunkCount];
+		if (fOffsets == NULL)
+			return false;
+		fOffsets[0] = 0;
+			// Value that serves as a marker that all offsets are 32 bit. We'll
+			// replace it, when necessary.
+	}
+
+	uint64 offset = (*this)[baseIndex];
+	for (size_t i = 0; i < chunkCount; i++) {
+		offset += (uint64)chunkSizes[i] + 1;
+			// the stored value is chunkSize - 1
+		size_t index = baseIndex + i + 1;
+			// (baseIndex + i) is the index of the chunk whose size is stored in
+			// chunkSizes[i]. We compute the offset of the following element
+			// which we store at index (baseIndex + i + 1).
+
+		if (offset <= ~(uint32)0) {
+			fOffsets[index] = (uint32)offset;
+		} else {
+			if (fOffsets[0] == 0) {
+				// Not scaled to allow for 64 bit offsets yet. Do that.
+				fOffsets[0] = index;
+				uint32* newOffsets = new(std::nothrow) uint32[
+					2 * totalChunkCount - fOffsets[0]];
+				if (newOffsets == NULL)
+					return false;
+
+				memcpy(newOffsets, fOffsets,
+					sizeof(newOffsets[0]) * fOffsets[0]);
+
+				delete[] fOffsets;
+				fOffsets = newOffsets;
+			}
+
+			index += index - fOffsets[0];
+			fOffsets[index] = (uint32)offset;
+			fOffsets[index + 1] = uint32(offset >> 32);
+		}
+	}
+
+	return true;
+}
+
+
+bool
+PackageFileHeapAccessorBase::OffsetArray::Init(size_t totalChunkCount,
+	const OffsetArray& other)
+{
+	if (other.fOffsets == NULL)
+		return true;
+
+	size_t elementCount = other.fOffsets[0] == 0
+		? totalChunkCount
+		: 2 * totalChunkCount - other.fOffsets[0];
+
+	fOffsets = new(std::nothrow) uint32[elementCount];
+	if (fOffsets == NULL)
+		return false;
+
+	memcpy(fOffsets, other.fOffsets, elementCount * sizeof(fOffsets[0]));
+	return true;
+}
+
+
+// #pragma mark - PackageFileHeapAccessorBase
+
+
+PackageFileHeapAccessorBase::PackageFileHeapAccessorBase(
+	BErrorOutput* errorOutput, int fd, off_t heapOffset)
+	:
+	fErrorOutput(errorOutput),
+	fFD(fd),
+	fHeapOffset(heapOffset),
+	fCompressedHeapSize(0),
+	fUncompressedHeapSize(0)
+{
+}
+
+
+PackageFileHeapAccessorBase::~PackageFileHeapAccessorBase()
+{
+}
+
+
+status_t
+PackageFileHeapAccessorBase::ReadDataToOutput(off_t offset, size_t size,
+	BDataOutput* output)
+{
+	if (size == 0)
+		return B_OK;
+
+	if (offset < 0 || (uint64)offset > fUncompressedHeapSize
+		|| size > fUncompressedHeapSize - offset) {
+		return B_BAD_VALUE;
+	}
+
+	// allocate buffers for compressed and uncompressed data
+	uint16* compressedDataBuffer = (uint16*)malloc(kChunkSize);
+	uint16* uncompressedDataBuffer = (uint16*)malloc(kChunkSize);
+	MemoryDeleter compressedDataBufferDeleter(compressedDataBuffer);
+	MemoryDeleter uncompressedDataBufferDeleter(uncompressedDataBuffer);
+	if (compressedDataBuffer == NULL || uncompressedDataBuffer == NULL)
+		return B_NO_MEMORY;
+
+	// read the data
+	size_t chunkIndex = size_t(offset / kChunkSize);
+	size_t inChunkOffset = (uint64)offset - (uint64)chunkIndex * kChunkSize;
+	size_t remainingBytes = size;
+
+	while (remainingBytes > 0) {
+		status_t error = ReadAndDecompressChunk(chunkIndex,
+			compressedDataBuffer, uncompressedDataBuffer);
+		if (error != B_OK)
+			return error;
+
+		size_t toWrite = std::min((size_t)kChunkSize - inChunkOffset,
+			remainingBytes);
+			// The last chunk may be shorter than kChunkSize, but since
+			// size (and thus remainingSize) had been clamped, that doesn't
+			// harm.
+		error = output->WriteData(
+			(char*)uncompressedDataBuffer + inChunkOffset, toWrite);
+		if (error != B_OK)
+			return error;
+
+		remainingBytes -= toWrite;
+		chunkIndex++;
+		inChunkOffset = 0;
+	}
+
+	return B_OK;
+}
+
+
+status_t
+PackageFileHeapAccessorBase::ReadAndDecompressChunkData(uint64 offset,
+	size_t compressedSize, size_t uncompressedSize, void* compressedDataBuffer,
+	void* uncompressedDataBuffer)
+{
+	// if uncompressed, read directly into the uncompressed data buffer
+	if (compressedSize == uncompressedSize)
+		return ReadFileData(offset, uncompressedDataBuffer, compressedSize);
+
+	// otherwise read into the other buffer and decompress
+	status_t error = ReadFileData(offset, compressedDataBuffer, compressedSize);
+	if (error != B_OK)
+		return error;
+
+	size_t actualSize;
+	error = ZlibDecompressor::DecompressSingleBuffer(compressedDataBuffer,
+		compressedSize, uncompressedDataBuffer, uncompressedSize, actualSize);
+	if (error != B_OK) {
+		fErrorOutput->PrintError("Failed to decompress data chunk: %s\n",
+			strerror(error));
+		return error;
+	}
+
+	if (actualSize != uncompressedSize) {
+		fErrorOutput->PrintError("Failed to decompress data chunk: chunk "
+			"size mismatch\n");
+		return B_ERROR;
+	}
+
+	return B_OK;
+}
+
+
+status_t
+PackageFileHeapAccessorBase::ReadFileData(uint64 offset, void* buffer,
+	size_t size)
+{
+	ssize_t bytesRead = pread(fFD, buffer, size, fHeapOffset + (off_t)offset);
+	if (bytesRead < 0) {
+		fErrorOutput->PrintError("_ReadData(%" B_PRIu64 "%p, %zu) failed to "
+			"read data: %s\n", offset, buffer, size, strerror(errno));
+		return errno;
+	}
+	if ((size_t)bytesRead != size) {
+		fErrorOutput->PrintError("_ReadData(%" B_PRIu64 "%p, %zu) failed to "
+			"read all data\n", offset, buffer, size);
+		return B_ERROR;
+	}
+
+	return B_OK;
+}
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit

src/kits/package/hpkg/PackageFileHeapReader.cpp

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <package/hpkg/PackageFileHeapReader.h>
+
+#include <algorithm>
+#include <new>
+
+#include <package/hpkg/ErrorOutput.h>
+#include <package/hpkg/HPKGDefs.h>
+
+#include <AutoDeleter.h>
+#include <package/hpkg/PoolBuffer.h>
+
+
+// minimum length of data we require before trying to zlib compress them
+static const size_t kZlibCompressionSizeThreshold = 64;
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+namespace BPrivate {
+
+
+PackageFileHeapReader::PackageFileHeapReader(BErrorOutput* errorOutput, int fd,
+	off_t heapOffset, off_t compressedHeapSize, uint64 uncompressedHeapSize)
+	:
+	PackageFileHeapAccessorBase(errorOutput, fd, heapOffset),
+	fOffsets()
+{
+	fCompressedHeapSize = compressedHeapSize;
+	fUncompressedHeapSize = uncompressedHeapSize;
+}
+
+
+PackageFileHeapReader::~PackageFileHeapReader()
+{
+}
+
+
+status_t
+PackageFileHeapReader::Init()
+{
+	if (fUncompressedHeapSize == 0)
+		return B_OK;
+
+	// Determine number of chunks and adjust the compressed heap size (subtract
+	// the size of the chunk size array at the end). Note that the size of the
+	// last chunk has not been saved, since it size is implied.
+	ssize_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize;
+	if (chunkCount <= 0)
+		return B_OK;
+
+	fCompressedHeapSize -= (chunkCount - 1) * 2;
+
+	// allocate a buffer
+	uint16* buffer = (uint16*)malloc(kChunkSize);
+	if (buffer == NULL)
+		return B_NO_MEMORY;
+	MemoryDeleter bufferDeleter(buffer);
+
+	// read the chunk size array
+	size_t remainingChunks = chunkCount - 1;
+	size_t index = 0;
+	uint64 offset = fCompressedHeapSize;
+	while (remainingChunks > 0) {
+		size_t toRead = std::min(remainingChunks, kChunkSize / 2);
+		status_t error = ReadFileData(offset, buffer, toRead * 2);
+		if (error != B_OK)
+			return error;
+
+		if (!fOffsets.InitChunksOffsets(chunkCount, index, buffer, toRead))
+			return B_NO_MEMORY;
+
+		remainingChunks -= toRead;
+		index += toRead;
+		offset += toRead * 2;
+	}
+
+	return B_OK;
+}
+
+
+PackageFileHeapReader*
+PackageFileHeapReader::Clone() const
+{
+	PackageFileHeapReader* clone = new(std::nothrow) PackageFileHeapReader(
+		fErrorOutput, fFD, fHeapOffset, fCompressedHeapSize,
+		fUncompressedHeapSize);
+	if (clone == NULL)
+		return NULL;
+
+	ssize_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize;
+	if (!clone->fOffsets.Init(chunkCount, fOffsets)) {
+		delete clone;
+		return NULL;
+	}
+
+	return clone;
+}
+
+
+status_t
+PackageFileHeapReader::ReadAndDecompressChunk(size_t chunkIndex,
+	void* compressedDataBuffer, void* uncompressedDataBuffer)
+{
+	uint64 offset = fOffsets[chunkIndex];
+	bool isLastChunk
+		= uint64(chunkIndex + 1) * kChunkSize >= fUncompressedHeapSize;
+	size_t compressedSize = isLastChunk
+		? fCompressedHeapSize - offset
+		: fOffsets[chunkIndex + 1] - offset;
+	size_t uncompressedSize = isLastChunk
+		? fUncompressedHeapSize - (uint64)chunkIndex * kChunkSize
+		: kChunkSize;
+
+	return ReadAndDecompressChunkData(offset, compressedSize, uncompressedSize,
+		compressedDataBuffer, uncompressedDataBuffer);
+}
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit

src/kits/package/hpkg/PackageFileHeapWriter.cpp

@@ -0,0 +1,645 @@
+/*
+ * Copyright 2013, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Distributed under the terms of the MIT License.
+ */
+
+
+#include <package/hpkg/PackageFileHeapWriter.h>
+
+#include <errno.h>
+
+#include <algorithm>
+#include <new>
+
+#include <List.h>
+#include <package/hpkg/ErrorOutput.h>
+#include <package/hpkg/HPKGDefs.h>
+
+#include <AutoDeleter.h>
+#include <package/hpkg/DataReader.h>
+#include <package/hpkg/PackageFileHeapReader.h>
+#include <package/hpkg/ZlibCompressor.h>
+#include <package/hpkg/ZlibDecompressor.h>
+#include <RangeArray.h>
+
+
+// minimum length of data we require before trying to zlib compress them
+static const size_t kZlibCompressionSizeThreshold = 64;
+
+
+namespace BPackageKit {
+
+namespace BHPKG {
+
+namespace BPrivate {
+
+
+struct PackageFileHeapWriter::Chunk {
+	uint64	offset;
+	uint32	compressedSize;
+	uint32	uncompressedSize;
+	void*	buffer;
+};
+
+
+struct PackageFileHeapWriter::ChunkSegment {
+	ssize_t	chunkIndex;
+	uint32	toKeepOffset;
+	uint32	toKeepSize;
+};
+
+
+struct PackageFileHeapWriter::ChunkBuffer {
+	ChunkBuffer(BErrorOutput* errorOutput, int fd, off_t heapOffset,
+		size_t bufferSize)
+		:
+		fErrorOutput(errorOutput),
+		fFD(fd),
+		fHeapOffset(heapOffset),
+		fChunks(),
+		fCurrentChunkIndex(0),
+		fNextReadIndex(0),
+		fSegments(),
+		fCurrentSegmentIndex(0),
+		fBuffers(),
+		fUnusedBuffers(),
+		fBufferSize(bufferSize)
+	{
+	}
+
+	~ChunkBuffer()
+	{
+		for (int32 i = 0; void* buffer = fBuffers.ItemAt(i); i++)
+			free(buffer);
+	}
+
+	bool PushChunkSegment(uint64 chunkOffset, uint32 compressedSize,
+		uint32 uncompressedSize, uint32 toKeepOffset, uint32 toKeepSize)
+	{
+		ChunkSegment segment;
+		segment.toKeepOffset = toKeepOffset;
+		segment.toKeepSize = toKeepSize;
+
+		// might refer to the last chunk
+		segment.chunkIndex = fChunks.Count() - 1;
+
+		if (segment.chunkIndex < 0
+			|| fChunks.ElementAt(segment.chunkIndex).offset != chunkOffset) {
+			// no, need to push a new chunk
+			segment.chunkIndex++;
+
+			Chunk chunk;
+			chunk.offset = chunkOffset;
+			chunk.compressedSize = compressedSize;
+			chunk.uncompressedSize = uncompressedSize;
+			chunk.buffer = NULL;
+			if (!fChunks.Add(chunk))
+				return false;
+		}
+
+		return fSegments.Add(segment);
+	}
+
+	bool HasMoreSegments() const
+	{
+		return fCurrentSegmentIndex < fSegments.Count();
+	}
+
+	const ChunkSegment& CurrentSegment() const
+	{
+		return fSegments[fCurrentSegmentIndex];
+	}
+
+	const Chunk& ChunkAt(ssize_t index) const
+	{
+		return fChunks[index];
+	}
+
+	bool HasMoreChunksToRead() const
+	{
+		return fNextReadIndex < fChunks.Count();
+	}
+
+	bool HasBufferedChunk() const
+	{
+		return fCurrentChunkIndex < fNextReadIndex;
+	}
+
+	uint64 NextReadOffset() const
+	{
+		return fChunks[fNextReadIndex].offset;
+	}
+
+	void ReadNextChunk()
+	{
+		if (!HasMoreChunksToRead())
+			throw status_t(B_BAD_VALUE);
+
+		Chunk& chunk = fChunks[fNextReadIndex++];
+		chunk.buffer = _GetBuffer();
+
+		ssize_t bytesRead = pread(fFD, chunk.buffer, chunk.compressedSize,
+			fHeapOffset + chunk.offset);
+		if (bytesRead < 0) {
+			fErrorOutput->PrintError("PackageFileHeapWriter::ChunkBuffer::"
+				"ReadNextChunk(offset: %" B_PRIu64 ", size: %" B_PRIu32
+				") failed to read data: %s\n", chunk.offset,
+				chunk.compressedSize, strerror(errno));
+			throw status_t(errno);
+		}
+
+		if ((size_t)bytesRead != chunk.compressedSize) {
+			fErrorOutput->PrintError("PackageFileHeapWriter::ChunkBuffer::"
+				"ReadNextChunk(offset: %" B_PRIu64 ", size: %" B_PRIu32
+				") failed to read all data\n", chunk.offset,
+				chunk.compressedSize);
+			throw status_t(B_ERROR);
+		}
+	}
+
+	void CurrentSegmentDone()
+	{
+		// Unless the next segment refers to the same chunk, advance to the next
+		// chunk.
+		const ChunkSegment& segment = fSegments[fCurrentSegmentIndex++];
+		if (!HasMoreSegments()
+			|| segment.chunkIndex != CurrentSegment().chunkIndex) {
+			_PutBuffer(fChunks[fCurrentChunkIndex++].buffer);
+		}
+	}
+
+private:
+	void* _GetBuffer()
+	{
+		if (!fUnusedBuffers.IsEmpty())
+			return fUnusedBuffers.RemoveItem(fUnusedBuffers.CountItems() - 1);
+
+		void* buffer = malloc(fBufferSize);
+		if (buffer == NULL && !fBuffers.AddItem(buffer)) {
+			free(buffer);
+			throw std::bad_alloc();
+		}
+
+		return buffer;
+	}
+
+	void _PutBuffer(void* buffer)
+	{
+		if (buffer != NULL && !fUnusedBuffers.AddItem(buffer)) {
+			fBuffers.RemoveItem(buffer);
+			free(buffer);
+		}
+	}
+
+private:
+	BErrorOutput*		fErrorOutput;
+	int					fFD;
+	off_t				fHeapOffset;
+
+	Array<Chunk>		fChunks;
+	ssize_t				fCurrentChunkIndex;
+	ssize_t				fNextReadIndex;
+
+	Array<ChunkSegment>	fSegments;
+	ssize_t				fCurrentSegmentIndex;
+
+	BList				fBuffers;
+	BList				fUnusedBuffers;
+	size_t				fBufferSize;
+};
+
+
+PackageFileHeapWriter::PackageFileHeapWriter(BErrorOutput* errorOutput, int fd,
+	off_t heapOffset)
+	:
+	PackageFileHeapAccessorBase(errorOutput, fd, heapOffset),
+	fPendingDataBuffer(NULL),
+	fCompressedDataBuffer(NULL),
+	fPendingDataSize(0),
+	fOffsets()
+{
+}
+
+
+PackageFileHeapWriter::~PackageFileHeapWriter()
+{
+	_Uninit();
+}
+
+
+void
+PackageFileHeapWriter::Init()
+{
+	// allocate data buffers
+	fPendingDataBuffer = malloc(kChunkSize);
+	fCompressedDataBuffer = malloc(kChunkSize);
+	if (fPendingDataBuffer == NULL || fCompressedDataBuffer == NULL)
+		throw std::bad_alloc();
+}
+
+
+void
+PackageFileHeapWriter::Reinit(PackageFileHeapReader* heapReader)
+{
+	fHeapOffset = heapReader->HeapOffset();
+	fCompressedHeapSize = heapReader->CompressedHeapSize();
+	fUncompressedHeapSize = heapReader->UncompressedHeapSize();
+	fPendingDataSize = 0;
+
+	// copy the offsets array
+	size_t chunkCount = (fUncompressedHeapSize + kChunkSize - 1) / kChunkSize;
+	if (chunkCount > 0) {
+		if (!fOffsets.AddUninitialized(chunkCount))
+			throw std::bad_alloc();
+
+		for (size_t i = 0; i < chunkCount; i++)
+			fOffsets[i] = heapReader->Offsets()[i];
+	}
+
+	// If the last chunk is partial, read it in and remove it from the offsets.
+	size_t lastChunkSize = fUncompressedHeapSize % kChunkSize;
+	if (lastChunkSize != 0) {
+		status_t error = heapReader->ReadData(
+			fUncompressedHeapSize - lastChunkSize, fPendingDataBuffer,
+			lastChunkSize);
+		if (error != B_OK)
+			throw error;
+
+		fCompressedHeapSize = fOffsets[fOffsets.Count() - 1];
+		fOffsets.Remove(fOffsets.Count() - 1);
+	}
+}
+
+
+status_t
+PackageFileHeapWriter::AddData(BDataReader& dataReader, off_t size,
+	uint64& _offset)
+{
+	_offset = fUncompressedHeapSize;
+
+	// copy the data to the heap
+	off_t readOffset = 0;
+	off_t remainingSize = size;
+	while (remainingSize > 0) {
+		// read data into pending data buffer
+		size_t toCopy = std::min(remainingSize,
+			off_t(kChunkSize - fPendingDataSize));
+		status_t error = dataReader.ReadData(readOffset,
+			(uint8*)fPendingDataBuffer + fPendingDataSize, toCopy);
+		if (error != B_OK) {
+			fErrorOutput->PrintError("Failed to read data: %s\n",
+				strerror(error));
+			return error;
+		}
+
+		fPendingDataSize += toCopy;
+		fUncompressedHeapSize += toCopy;
+		remainingSize -= toCopy;
+		readOffset += toCopy;
+
+		if (fPendingDataSize == kChunkSize) {
+			error = _FlushPendingData();
+			if (error != B_OK)
+				return error;
+		}
+	}
+
+	return B_OK;
+}
+
+
+void
+PackageFileHeapWriter::RemoveDataRanges(
+	const ::BPrivate::RangeArray<uint64>& ranges)
+{
+	ssize_t rangeCount = ranges.CountRanges();
+	if (rangeCount == 0)
+		return;
+
+	if (fOffsets.IsEmpty()) {
+		fErrorOutput->PrintError("Can't remove ranges from empty heap\n");
+		throw status_t(B_BAD_VALUE);
+	}
+
+	// We potentially have to recompress all data from the first affected chunk
+	// to the end (minus the removed ranges, of course). As a basic algorithm we
+	// can use our usual data writing strategy, i.e. read a chunk, decompress it
+	// to a temporary buffer, and write the data to keep via AddData(). There
+	// are a few complications/optimizations, though:
+	// * As data moves to other chunks, it may actually compress worse than
+	//   before. While unlikely, we still have to take care of this case by
+	//   making sure our reading end is at least a complete uncompressed chunk
+	//   ahead of the writing end.
+	// * When we run into the situation that we have to move complete aligned
+	//   chunks, we want to avoid uncompressing and recompressing them
+	//   needlessly.
+
+	// Build a list of (possibly partial) chunks we want to keep.
+
+	// the first partial chunk (if any) and all chunks between ranges
+	ChunkBuffer chunkBuffer(fErrorOutput, fFD, fHeapOffset, kChunkSize);
+	uint64 writeOffset = ranges[0].offset - ranges[0].offset % kChunkSize;
+	uint64 readOffset = writeOffset;
+	for (ssize_t i = 0; i < rangeCount; i++) {
+		const Range<uint64>& range = ranges[i];
+		if (range.size > 0) {
+			_PushChunks(chunkBuffer, readOffset, range.offset);
+			readOffset = range.offset + range.size;
+		}
+	}
+
+	if (readOffset == writeOffset) {
+		fErrorOutput->PrintError("Only empty ranges to remove from heap\n");
+		throw status_t(B_BAD_VALUE);
+	}
+
+	// all chunks after the last range
+	_PushChunks(chunkBuffer, readOffset, fUncompressedHeapSize);
+
+	// Reset our state to look like all chunks from the first affected one have
+	// been removed and re-add all data we want to keep.
+
+	// truncate the offsets array and reset the heap end offset
+	ssize_t firstChunkIndex = ssize_t(writeOffset / kChunkSize);
+	fCompressedHeapSize = fOffsets[firstChunkIndex];
+	fOffsets.Remove(firstChunkIndex, fOffsets.Count() - firstChunkIndex);
+
+	// we need a decompression buffer
+	void* decompressionBuffer = malloc(kChunkSize);
+	if (decompressionBuffer == NULL)
+		throw std::bad_alloc();
+	MemoryDeleter decompressionBufferDeleter(decompressionBuffer);
+
+	const Chunk* decompressedChunk = NULL;
+
+	while (chunkBuffer.HasMoreSegments()) {
+		const ChunkSegment& segment = chunkBuffer.CurrentSegment();
+
+		// If we have an aligned, complete chunk, copy its compressed data.
+		bool copyCompressed = fPendingDataSize == 0 && segment.toKeepOffset == 0
+			&& segment.toKeepSize == kChunkSize;
+
+		// Read more chunks. We need at least one buffered one to do anything
+		// and we want to buffer as many as necessary to ensure we don't
+		// overwrite one we haven't buffered yet.
+		while (chunkBuffer.HasMoreChunksToRead()
+			&& (!chunkBuffer.HasBufferedChunk()
+				|| (!copyCompressed
+					&& chunkBuffer.NextReadOffset()
+						< fCompressedHeapSize + kChunkSize))) {
+			// read chunk
+			chunkBuffer.ReadNextChunk();
+		}
+
+		// copy compressed chunk data, if possible
+		const Chunk& chunk = chunkBuffer.ChunkAt(segment.chunkIndex);
+		if (copyCompressed) {
+			status_t error = _WriteChunk(chunk.buffer, chunk.compressedSize,
+				false);
+			if (error != B_OK)
+				throw error;
+			continue;
+		}
+
+		// decompress chunk, if compressed
+		void* uncompressedData;
+		if (chunk.uncompressedSize == chunk.compressedSize) {
+			uncompressedData = chunk.buffer;
+		} else if (decompressedChunk == &chunk) {
+			uncompressedData = decompressionBuffer;
+		} else {
+			size_t uncompressedSize;
+			status_t error = ZlibDecompressor::DecompressSingleBuffer(
+				chunk.buffer, chunk.compressedSize,
+				decompressionBuffer, chunk.uncompressedSize, uncompressedSize);
+			if (error != B_OK) {
+				fErrorOutput->PrintError("Failed to decompress data chunk: %s\n",
+					strerror(error));
+				throw error;
+			}
+	
+			if (uncompressedSize != chunk.uncompressedSize) {
+				fErrorOutput->PrintError("Failed to decompress data chunk: chunk "
+					"size mismatch\n");
+				throw status_t(B_ERROR);
+			}
+
+			decompressedChunk = &chunk;
+			uncompressedData = decompressionBuffer;
+		}
+
+		// add chunk data
+		WriteDataThrows((uint8*)uncompressedData + segment.toKeepOffset,
+			segment.toKeepSize);
+
+		chunkBuffer.CurrentSegmentDone();
+	}
+}
+
+
+status_t
+PackageFileHeapWriter::Finish()
+{
+	// flush pending data, if any
+	status_t error = _FlushPendingData();
+	if (error != B_OK)
+		return error;
+
+	// write chunk sizes table 
+
+	// We don't need to write the last chunk size, since it is implied by the
+	// total size minus the sum of all other chunk sizes.
+	ssize_t offsetCount = fOffsets.Count();
+	if (offsetCount < 2)
+		return B_OK;
+
+	// Convert the offsets to 16 bit sizes and write them. We use the (no longer
+	// used) pending data buffer for the conversion.
+	uint16* buffer = (uint16*)fPendingDataBuffer;
+	for (ssize_t offsetIndex = 1; offsetIndex < offsetCount;) {
+		ssize_t toWrite = std::min(offsetCount - offsetIndex,
+			ssize_t(kChunkSize / 2));
+
+		for (ssize_t i = 0; i < toWrite; i++, offsetIndex++) {
+			// store chunkSize - 1, so it fits 16 bit (chunks cannot be empty)
+			buffer[i] = uint16(fOffsets[offsetIndex] - fOffsets[offsetIndex - 1]
+				- 1);
+		}
+
+		error = _WriteDataUncompressed(buffer, toWrite * 2);
+		if (error != B_OK)
+			return error;
+	}
+
+	return B_OK;
+}
+
+
+status_t
+PackageFileHeapWriter::ReadAndDecompressChunk(size_t chunkIndex,
+	void* compressedDataBuffer, void* uncompressedDataBuffer)
+{
+	if (uint64(chunkIndex + 1) * kChunkSize > fUncompressedHeapSize) {
+		// The chunk has not been written to disk yet. Its data are still in the
+		// pending data buffer.
+		memcpy(uncompressedDataBuffer, fPendingDataBuffer, fPendingDataSize);
+		// TODO: This can be optimized. Since we write to a BDataOutput anyway,
+		// there's no need to copy the data.
+		return B_OK;
+	}
+
+	uint64 offset = fOffsets[chunkIndex];
+	size_t compressedSize = chunkIndex + 1 == (size_t)fOffsets.Count()
+		? fCompressedHeapSize - offset
+		: fOffsets[chunkIndex + 1] - offset;
+
+	return ReadAndDecompressChunkData(offset, compressedSize, kChunkSize,
+		compressedDataBuffer, uncompressedDataBuffer);
+}
+
+
+status_t
+PackageFileHeapWriter::WriteDataNoThrow(const void* buffer, size_t size)
+{
+	BBufferDataReader reader(buffer, size);
+	uint64 dummyOffset;
+	return AddData(reader, size, dummyOffset);
+}
+
+
+void
+PackageFileHeapWriter::_Uninit()
+{
+	free(fPendingDataBuffer);
+	free(fCompressedDataBuffer);
+	fPendingDataBuffer = NULL;
+	fCompressedDataBuffer = NULL;
+}
+
+
+status_t
+PackageFileHeapWriter::_FlushPendingData()
+{
+	if (fPendingDataSize == 0)
+		return B_OK;
+
+	status_t error = _WriteChunk(fPendingDataBuffer, fPendingDataSize, true);
+	if (error == B_OK)
+		fPendingDataSize = 0;
+
+	return error;
+}
+
+
+status_t
+PackageFileHeapWriter::_WriteChunk(const void* data, size_t size,
+	bool mayCompress)
+{
+	// add offset
+	if (!fOffsets.Add(fCompressedHeapSize)) {
+		fErrorOutput->PrintError("Out of memory!\n");
+		return B_NO_MEMORY;
+	}
+
+	// Try to use zlib compression only for data large enough.
+	bool compress = mayCompress && size >= (off_t)kZlibCompressionSizeThreshold;
+	if (compress) {
+		status_t error = _WriteDataCompressed(data, size);
+		if (error != B_OK) {
+			if (error != B_BUFFER_OVERFLOW)
+				return error;
+			compress = false;
+		}
+	}
+
+	// Write uncompressed, if necessary.
+	if (!compress) {
+		status_t error = _WriteDataUncompressed(data, size);
+		if (error != B_OK)
+			return error;
+	}
+
+	return B_OK;
+}
+
+
+status_t
+PackageFileHeapWriter::_WriteDataCompressed(const void* data, size_t size)
+{
+	size_t compressedSize;
+	status_t error = ZlibCompressor::CompressSingleBuffer(data, size,
+		fCompressedDataBuffer, size, compressedSize);
+	if (error != B_OK)
+		return error;
+
+	// only use compressed data when we've actually saved space
+	if (compressedSize == size)
+		return B_BUFFER_OVERFLOW;
+
+	return _WriteDataUncompressed(fCompressedDataBuffer, compressedSize);
+}
+
+
+status_t
+PackageFileHeapWriter::_WriteDataUncompressed(const void* data, size_t size)
+{
+	ssize_t bytesWritten = pwrite(fFD, data, size,
+		fHeapOffset + (off_t)fCompressedHeapSize);
+	if (bytesWritten < 0) {
+		fErrorOutput->PrintError("Failed to write data: %s\n", strerror(errno));
+		return errno;
+	}
+	if ((size_t)bytesWritten != size) {
+		fErrorOutput->PrintError("Failed to write all data\n");
+		return B_ERROR;
+	}
+
+	fCompressedHeapSize += bytesWritten;
+
+	return B_OK;
+}
+
+
+void
+PackageFileHeapWriter::_PushChunks(ChunkBuffer& chunkBuffer, uint64 startOffset,
+	uint64 endOffset)
+{
+	if (endOffset > fUncompressedHeapSize) {
+		fErrorOutput->PrintError("Invalid range to remove from heap\n");
+		throw status_t(B_BAD_VALUE);
+	}
+
+	ssize_t chunkIndex = startOffset / kChunkSize;
+	uint64 uncompressedChunkOffset = (uint64)chunkIndex * kChunkSize;
+
+	while (startOffset < endOffset) {
+		uint32 inChunkOffset = uint32(startOffset - uncompressedChunkOffset);
+		uint32 uncompressedChunkSize = chunkIndex + 1 < fOffsets.Count()
+			? kChunkSize
+			: fUncompressedHeapSize - uncompressedChunkOffset;
+		uint64 compressedChunkOffset = fOffsets[chunkIndex];
+		uint32 compressedChunkSize = chunkIndex + 1 < fOffsets.Count()
+			? fOffsets[chunkIndex + 1] - compressedChunkOffset
+			: fCompressedHeapSize - compressedChunkOffset;
+		uint32 toKeepSize = uint32(std::min(
+			(uint64)uncompressedChunkSize - inChunkOffset,
+			endOffset - startOffset));
+
+		if (!chunkBuffer.PushChunkSegment(compressedChunkOffset,
+				compressedChunkSize, uncompressedChunkSize, inChunkOffset,
+				toKeepSize)) {
+			throw std::bad_alloc();
+		}
+
+		startOffset += toKeepSize;
+		chunkIndex++;
+		uncompressedChunkOffset += uncompressedChunkSize;
+	}
+}
+
+
+}	// namespace BPrivate
+
+}	// namespace BHPKG
+
+}	// namespace BPackageKit

src/kits/package/hpkg/PackageReader.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2009-2013, Ingo Weinhold, ingo_weinhold@gmx.de.
  * Distributed under the terms of the MIT License.
  */
 
@@ -9,6 +9,8 @@
 #include <new>
 
 #include <package/hpkg/ErrorOutput.h>
+
+#include <package/hpkg/PackageFileHeapReader.h>
 #include <package/hpkg/PackageReaderImpl.h>
 
 
@@ -80,6 +82,14 @@ BPackageReader::PackageFileFD()
 }
 
 
+
+BAbstractBufferedDataReader*
+BPackageReader::HeapReader() const
+{
+	return fImpl != NULL ? fImpl->HeapReader() : NULL;
+}
+
+
 }	// namespace BHPKG
 
 }	// namespace BPackageKit

src/kits/package/hpkg/PackageReaderImpl.cpp

@@ -27,6 +27,7 @@
 #include <package/hpkg/PackageData.h>
 #include <package/hpkg/PackageEntry.h>
 #include <package/hpkg/PackageEntryAttribute.h>
+#include <package/hpkg/PackageFileHeapReader.h>
 #include <package/hpkg/ZlibDecompressor.h>
 
 
@@ -48,80 +49,16 @@ static const size_t kMaxTOCSize					= 64 * 1024 * 1024;
 static const size_t kMaxPackageAttributesSize	= 1 * 1024 * 1024;
 
 
-// #pragma mark - DataAttributeHandler
-
-
-struct PackageReaderImpl::DataAttributeHandler : AttributeHandler {
-	DataAttributeHandler(BPackageData* data)
-		:
-		fData(data)
-	{
-	}
-
-	static status_t InitData(AttributeHandlerContext* context,
-		BPackageData* data, const AttributeValue& value)
-	{
-		if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE)
-			data->SetData(value.data.size, value.data.raw);
-		else
-			data->SetData(value.data.size, value.data.offset);
-
-		data->SetUncompressedSize(value.data.size);
-
-		return B_OK;
-	}
-
-	static status_t Create(AttributeHandlerContext* context,
-		BPackageData* data, const AttributeValue& value,
-		AttributeHandler*& _handler)
-	{
-		DataAttributeHandler* handler = new(std::nothrow) DataAttributeHandler(
-			data);
-		if (handler == NULL)
-			return B_NO_MEMORY;
-
-		InitData(context, data, value);
-
-		_handler = handler;
-		return B_OK;
-	}
-
-	virtual status_t HandleAttribute(AttributeHandlerContext* context,
-		uint8 id, const AttributeValue& value, AttributeHandler** _handler)
-	{
-		switch (id) {
-			case B_HPKG_ATTRIBUTE_ID_DATA_SIZE:
-				fData->SetUncompressedSize(value.unsignedInt);
-				return B_OK;
-
-			case B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION:
-			{
-				switch (value.unsignedInt) {
-					case B_HPKG_COMPRESSION_NONE:
-					case B_HPKG_COMPRESSION_ZLIB:
-						break;
-					default:
-						context->errorOutput->PrintError("Error: Invalid "
-							"compression type for data (%llu)\n",
-							value.unsignedInt);
-						return B_BAD_DATA;
-				}
-
-				fData->SetCompression(value.unsignedInt);
-				return B_OK;
-			}
-
-			case B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE:
-				fData->SetChunkSize(value.unsignedInt);
-				return B_OK;
-		}
-
-		return AttributeHandler::HandleAttribute(context, id, value, _handler);
-	}
-
-private:
-	BPackageData*	fData;
-};
+static status_t
+set_package_data_from_attribute_value(const BPackageAttributeValue& value,
+	BPackageData& data)
+{
+	if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE)
+		data.SetData(value.data.size, value.data.raw);
+	else
+		data.SetData(value.data.size, value.data.offset);
+	return B_OK;
+}
 
 
 // #pragma mark - AttributeAttributeHandler
@@ -140,12 +77,8 @@ struct PackageReaderImpl::AttributeAttributeHandler : AttributeHandler {
 	{
 		switch (id) {
 			case B_HPKG_ATTRIBUTE_ID_DATA:
-				if (_handler != NULL) {
-					return DataAttributeHandler::Create(context,
-						&fAttribute.Data(), value, *_handler);
-				}
-				return DataAttributeHandler::InitData(context,
-					&fAttribute.Data(), value);
+				return set_package_data_from_attribute_value(value,
+					fAttribute.Data());
 
 			case B_HPKG_ATTRIBUTE_ID_FILE_ATTRIBUTE_TYPE:
 				fAttribute.SetType(value.unsignedInt);
@@ -279,12 +212,8 @@ struct PackageReaderImpl::EntryAttributeHandler : AttributeHandler {
 			}
 
 			case B_HPKG_ATTRIBUTE_ID_DATA:
-				if (_handler != NULL) {
-					return DataAttributeHandler::Create(context, &fEntry.Data(),
-						value, *_handler);
-				}
-				return DataAttributeHandler::InitData(context, &fEntry.Data(),
-					value);
+				return set_package_data_from_attribute_value(value,
+					fEntry.Data());
 
 			case B_HPKG_ATTRIBUTE_ID_SYMLINK_PATH:
 				fEntry.SetSymlinkPath(value.string);
@@ -378,7 +307,7 @@ struct PackageReaderImpl::RootAttributeHandler : PackageAttributeHandler {
 
 PackageReaderImpl::PackageReaderImpl(BErrorOutput* errorOutput)
 	:
-	inherited(errorOutput),
+	inherited("package", errorOutput),
 	fTOCSection("TOC")
 {
 }
@@ -407,179 +336,39 @@ PackageReaderImpl::Init(const char* fileName)
 status_t
 PackageReaderImpl::Init(int fd, bool keepFD)
 {
-	status_t error = inherited::Init(fd, keepFD);
-	if (error != B_OK)
-		return error;
-
-	// stat it
-	struct stat st;
-	if (fstat(FD(), &st) < 0) {
-		ErrorOutput()->PrintError("Error: Failed to access package file: %s\n",
-			strerror(errno));
-		return errno;
-	}
-
-	// read the header
 	hpkg_header header;
-	if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK)
+	status_t error = inherited::Init<hpkg_header, B_HPKG_MAGIC, B_HPKG_VERSION>(
+		fd, keepFD, header);
+	if (error != B_OK)
 		return error;
+	fHeapSize = HeapReader()->UncompressedHeapSize();
 
-	// check the header
-
-	// magic
-	if (B_BENDIAN_TO_HOST_INT32(header.magic) != B_HPKG_MAGIC) {
-		ErrorOutput()->PrintError("Error: Invalid package file: Invalid "
-			"magic\n");
-		return B_BAD_DATA;
-	}
-
-	// version
-	if (B_BENDIAN_TO_HOST_INT16(header.version) != B_HPKG_VERSION) {
-		ErrorOutput()->PrintError("Error: Invalid/unsupported package file "
-			"version (%d)\n", B_BENDIAN_TO_HOST_INT16(header.version));
-		return B_MISMATCHED_VALUES;
-	}
-
-	// header size
-	fHeapOffset = B_BENDIAN_TO_HOST_INT16(header.header_size);
-	if ((size_t)fHeapOffset < sizeof(hpkg_header)) {
-		ErrorOutput()->PrintError("Error: Invalid package file: Invalid header "
-			"size (%llu)\n", fHeapOffset);
-		return B_BAD_DATA;
-	}
-
-	// total size
-	fTotalSize = B_BENDIAN_TO_HOST_INT64(header.total_size);
-	if (fTotalSize != (uint64)st.st_size) {
-		ErrorOutput()->PrintError("Error: Invalid package file: Total size in "
-			"header (%llu) doesn't agree with total file size (%lld)\n",
-			fTotalSize, st.st_size);
-		return B_BAD_DATA;
-	}
-
-	// package attributes length and compression
-	fPackageAttributesSection.compression
-		= B_BENDIAN_TO_HOST_INT32(header.attributes_compression);
-	fPackageAttributesSection.compressedLength
-		= B_BENDIAN_TO_HOST_INT32(header.attributes_length_compressed);
-	fPackageAttributesSection.uncompressedLength
-		= B_BENDIAN_TO_HOST_INT32(header.attributes_length_uncompressed);
-	fPackageAttributesSection.stringsLength
-		= B_BENDIAN_TO_HOST_INT32(header.attributes_strings_length);
-	fPackageAttributesSection.stringsCount
-		= B_BENDIAN_TO_HOST_INT32(header.attributes_strings_count);
-
-	if (const char* errorString = CheckCompression(
-		fPackageAttributesSection)) {
-		ErrorOutput()->PrintError("Error: Invalid package file: package "
-			"attributes section: %s\n", errorString);
-		return B_BAD_DATA;
-	}
-
-	// TOC length and compression
-	fTOCSection.compression = B_BENDIAN_TO_HOST_INT32(header.toc_compression);
-	fTOCSection.compressedLength
-		= B_BENDIAN_TO_HOST_INT64(header.toc_length_compressed);
-	fTOCSection.uncompressedLength
-		= B_BENDIAN_TO_HOST_INT64(header.toc_length_uncompressed);
-
-	if (const char* errorString = CheckCompression(fTOCSection)) {
-		ErrorOutput()->PrintError("Error: Invalid package file: TOC section: "
-			"%s\n", errorString);
-		return B_BAD_DATA;
-	}
-
-	// TOC subsections
-	fTOCSection.stringsLength
-		= B_BENDIAN_TO_HOST_INT64(header.toc_strings_length);
-	fTOCSection.stringsCount
-		= B_BENDIAN_TO_HOST_INT64(header.toc_strings_count);
-
-	if (fTOCSection.stringsLength > fTOCSection.uncompressedLength
-		|| fTOCSection.stringsCount > fTOCSection.stringsLength) {
-		ErrorOutput()->PrintError("Error: Invalid package file: Invalid TOC "
-			"subsections description\n");
-		return B_BAD_DATA;
-	}
-
-	// check whether the sections fit together
-	if (fPackageAttributesSection.compressedLength > fTotalSize
-		|| fTOCSection.compressedLength
-			> fTotalSize - fPackageAttributesSection.compressedLength
-		|| fHeapOffset
-			> fTotalSize - fPackageAttributesSection.compressedLength
-				- fTOCSection.compressedLength) {
-		ErrorOutput()->PrintError("Error: Invalid package file: The sum of the "
-			"sections sizes is greater than the package size\n");
-		return B_BAD_DATA;
-	}
-
-	fPackageAttributesSection.offset
-		= fTotalSize - fPackageAttributesSection.compressedLength;
-	fTOCSection.offset = fPackageAttributesSection.offset
-		- fTOCSection.compressedLength;
-	fHeapSize = fTOCSection.offset - fHeapOffset;
-
-	// TOC size sanity check
-	if (fTOCSection.uncompressedLength > kMaxTOCSize) {
-		ErrorOutput()->PrintError("Error: Package file TOC section size "
-			"is %llu bytes. This is beyond the reader's sanity limit\n",
-			fTOCSection.uncompressedLength);
-		return B_UNSUPPORTED;
-	}
-
-	// package attributes size sanity check
-	if (fPackageAttributesSection.uncompressedLength
-			> kMaxPackageAttributesSize) {
-		ErrorOutput()->PrintError(
-			"Error: Package file package attributes section size "
-			"is %llu bytes. This is beyond the reader's sanity limit\n",
-			fPackageAttributesSection.uncompressedLength);
-		return B_UNSUPPORTED;
-	}
-
-	// read in the complete TOC
-	fTOCSection.data
-		= new(std::nothrow) uint8[fTOCSection.uncompressedLength];
-	if (fTOCSection.data == NULL) {
-		ErrorOutput()->PrintError("Error: Out of memory!\n");
-		return B_NO_MEMORY;
-	}
-	error = ReadCompressedBuffer(fTOCSection);
+	// init package attributes section
+	error = InitSection(fPackageAttributesSection, fHeapSize,
+		B_BENDIAN_TO_HOST_INT32(header.attributes_length),
+		kMaxPackageAttributesSize,
+		B_BENDIAN_TO_HOST_INT32(header.attributes_strings_length),
+		B_BENDIAN_TO_HOST_INT32(header.attributes_strings_count));
 	if (error != B_OK)
 		return error;
 
-	// read in the complete package attributes section
-	fPackageAttributesSection.data
-		= new(std::nothrow) uint8[fPackageAttributesSection.uncompressedLength];
-	if (fPackageAttributesSection.data == NULL) {
-		ErrorOutput()->PrintError("Error: Out of memory!\n");
-		return B_NO_MEMORY;
-	}
-	error = ReadCompressedBuffer(fPackageAttributesSection);
+	// init TOC section
+	error = InitSection(fTOCSection, fPackageAttributesSection.offset,
+		B_BENDIAN_TO_HOST_INT64(header.toc_length), kMaxTOCSize,
+		B_BENDIAN_TO_HOST_INT64(header.toc_strings_length),
+		B_BENDIAN_TO_HOST_INT64(header.toc_strings_count));
 	if (error != B_OK)
 		return error;
 
-	// start parsing the TOC
-	fTOCSection.currentOffset = 0;
-	SetCurrentSection(&fTOCSection);
-
-	// strings
-	error = ParseStrings();
+	// prepare the sections for use
+	error = PrepareSection(fTOCSection);
 	if (error != B_OK)
 		return error;
 
-	// parse strings from package attributes section
-	fPackageAttributesSection.currentOffset = 0;
-	SetCurrentSection(&fPackageAttributesSection);
-
-	// strings
-	error = ParseStrings();
+	error = PrepareSection(fPackageAttributesSection);
 	if (error != B_OK)
 		return error;
 
-	SetCurrentSection(NULL);
-
 	return B_OK;
 }
 
@@ -630,10 +419,6 @@ PackageReaderImpl::_ParseTOC(AttributeHandlerContext* context,
 	fTOCSection.currentOffset = fTOCSection.stringsLength;
 	SetCurrentSection(&fTOCSection);
 
-	// prepare attribute handler context
-	context->heapOffset = fHeapOffset;
-	context->heapSize = fHeapSize;
-
 	// init the attribute handler stack
 	rootAttributeHandler->SetLevel(0);
 	ClearAttributeHandlerStack();
@@ -688,7 +473,7 @@ PackageReaderImpl::ReadAttributeValue(uint8 type, uint8 encoding,
 					return B_BAD_DATA;
 				}
 
-				_value.SetToData(size, fHeapOffset + offset);
+				_value.SetToData(size, offset);
 			} else if (encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) {
 				if (size > B_HPKG_MAX_INLINE_DATA_SIZE) {
 					ErrorOutput()->PrintError("Error: Invalid %s section: "

src/kits/package/hpkg/PackageWriterImpl.cpp

@@ -39,6 +39,7 @@
 
 #include <package/hpkg/DataOutput.h>
 #include <package/hpkg/DataReader.h>
+#include <package/hpkg/PackageFileHeapWriter.h>
 #include <package/hpkg/PackageReaderImpl.h>
 #include <package/hpkg/Stacker.h>
 
@@ -156,12 +157,11 @@ struct PackageWriterImpl::Attribute
 struct PackageWriterImpl::PackageContentHandler
 	: BLowLevelPackageContentHandler {
 	PackageContentHandler(Attribute* rootAttribute, BErrorOutput* errorOutput,
-		StringCache& stringCache, uint64 heapOffset)
+		StringCache& stringCache)
 		:
 		fErrorOutput(errorOutput),
 		fStringCache(stringCache),
 		fRootAttribute(rootAttribute),
-		fHeapOffset(heapOffset),
 		fErrorOccurred(false)
 	{
 	}
@@ -212,7 +212,7 @@ struct PackageWriterImpl::PackageContentHandler
 			case B_HPKG_ATTRIBUTE_TYPE_RAW:
 				if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_HEAP) {
 					attribute->value.SetToData(value.data.size,
-						value.data.offset - fHeapOffset);
+						value.data.offset);
 				} else if (value.encoding
 						== B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) {
 					attribute->value.SetToData(value.data.size, value.data.raw);
@@ -249,7 +249,6 @@ private:
 	BErrorOutput*	fErrorOutput;
 	StringCache&	fStringCache;
 	Attribute*		fRootAttribute;
-	uint64			fHeapOffset;
 	bool			fErrorOccurred;
 };
 
@@ -394,8 +393,8 @@ private:
 
 
 struct PackageWriterImpl::HeapAttributeOffsetter {
-	HeapAttributeOffsetter(const RangeArray<off_t>& ranges,
-		const Array<off_t>& deltas)
+	HeapAttributeOffsetter(const RangeArray<uint64>& ranges,
+		const Array<uint64>& deltas)
 		:
 		fRanges(ranges),
 		fDeltas(deltas)
@@ -409,7 +408,7 @@ struct PackageWriterImpl::HeapAttributeOffsetter {
 
 		if (value.type == B_HPKG_ATTRIBUTE_TYPE_RAW
 			&& value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_HEAP) {
-			off_t delta = fDeltas[fRanges.InsertionIndex(value.data.offset)];
+			uint64 delta = fDeltas[fRanges.InsertionIndex(value.data.offset)];
 			value.data.offset -= delta;
 		}
 
@@ -422,8 +421,8 @@ struct PackageWriterImpl::HeapAttributeOffsetter {
 	}
 
 private:
-	const RangeArray<off_t>&	fRanges;
-	const Array<off_t>&			fDeltas;
+	const RangeArray<uint64>&	fRanges;
+	const Array<uint64>&		fDeltas;
 };
 
 
@@ -445,11 +444,9 @@ PackageWriterImpl::_AddAttribute(BHPKGAttributeID attributeID, Type value)
 
 PackageWriterImpl::PackageWriterImpl(BPackageWriterListener* listener)
 	:
-	inherited(listener),
+	inherited("package", listener),
 	fListener(listener),
 	fHeapRangesToRemove(NULL),
-	fDataBuffer(NULL),
-	fDataBufferSize(2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB),
 	fRootEntry(NULL),
 	fRootAttribute(NULL),
 	fTopAttribute(NULL),
@@ -461,10 +458,7 @@ PackageWriterImpl::PackageWriterImpl(BPackageWriterListener* listener)
 PackageWriterImpl::~PackageWriterImpl()
 {
 	delete fRootAttribute;
-
 	delete fRootEntry;
-
-	free(fDataBuffer);
 }
 
 
@@ -566,7 +560,7 @@ status_t
 PackageWriterImpl::Finish()
 {
 	try {
-		RangeArray<off_t> heapRangesToRemove;
+		RangeArray<uint64> heapRangesToRemove;
 		fHeapRangesToRemove = &heapRangesToRemove;
 
 		if ((Flags() & B_HPKG_WRITER_UPDATE_PACKAGE) != 0) {
@@ -610,15 +604,10 @@ PackageWriterImpl::Finish()
 status_t
 PackageWriterImpl::_Init(const char* fileName, uint32 flags)
 {
-	status_t result = inherited::Init(fileName, "package", flags);
+	status_t result = inherited::Init(fileName, flags);
 	if (result != B_OK)
 		return result;
 
-	// allocate data buffer
-	fDataBuffer = malloc(fDataBufferSize);
-	if (fDataBuffer == NULL)
-		throw std::bad_alloc();
-
 	if (fStringCache.Init() != B_OK)
 		throw std::bad_alloc();
 
@@ -627,7 +616,7 @@ PackageWriterImpl::_Init(const char* fileName, uint32 flags)
 
 	fRootAttribute = new Attribute();
 
-	fHeapOffset = fHeapEnd = sizeof(hpkg_header);
+	fHeapOffset = fHeaderSize = sizeof(hpkg_header);
 	fTopAttribute = fRootAttribute;
 
 	// in update mode, parse the TOC
@@ -638,19 +627,14 @@ PackageWriterImpl::_Init(const char* fileName, uint32 flags)
 			return result;
 
 		fHeapOffset = packageReader.HeapOffset();
-		fHeapEnd = fHeapOffset + packageReader.HeapSize();
 
-		PackageContentHandler handler(fRootAttribute, fListener, fStringCache,
-			fHeapOffset);
+		PackageContentHandler handler(fRootAttribute, fListener, fStringCache);
 
 		result = packageReader.ParseContent(&handler);
 		if (result != B_OK)
 			return result;
 
-		if ((uint64)fHeapOffset > packageReader.HeapOffset()) {
-			fListener->PrintError("Unexpected heap offset in package file.\n");
-			return B_BAD_DATA;
-		}
+		fHeapWriter->Reinit(packageReader.HeapReader());
 	}
 
 	return B_OK;
@@ -815,99 +799,22 @@ PackageWriterImpl::_UpdateReadPackageInfo()
 	if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE)
 		data.SetData(value.data.size, value.data.raw);
 	else
-		data.SetData(value.data.size, value.data.offset + fHeapOffset);
-
-	// get the compression
-	uint8 compression = B_HPKG_DEFAULT_DATA_COMPRESSION;
-	if (Attribute* compressionAttribute = dataAttribute->ChildWithID(
-			B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION)) {
-		if (compressionAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) {
-			fListener->PrintError("%s entry in package file has an invalid "
-				"data compression attribute (not of type uint).\n",
-				B_HPKG_PACKAGE_INFO_FILE_NAME);
-			throw status_t(B_BAD_DATA);
-		}
-		compression = compressionAttribute->value.unsignedInt;
-	}
-
-	data.SetCompression(compression);
-
-	// get the size
-	uint64 size;
-	Attribute* sizeAttribute = dataAttribute->ChildWithID(
-		B_HPKG_ATTRIBUTE_ID_DATA_SIZE);
-	if (sizeAttribute == NULL) {
-		size = value.data.size;
-	} else if (sizeAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) {
-		fListener->PrintError("%s entry in package file has an invalid data "
-			"size attribute (not of type uint).\n",
-			B_HPKG_PACKAGE_INFO_FILE_NAME);
-		throw status_t(B_BAD_DATA);
-	} else
-		size = sizeAttribute->value.unsignedInt;
-
-	data.SetUncompressedSize(size);
-
-	// get the chunk size
-	uint64 chunkSize = compression == B_HPKG_COMPRESSION_ZLIB
-		? B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB : 0;
-	if (Attribute* chunkSizeAttribute = dataAttribute->ChildWithID(
-			B_HPKG_ATTRIBUTE_ID_DATA_CHUNK_SIZE)) {
-		if (chunkSizeAttribute->value.type != B_HPKG_ATTRIBUTE_TYPE_UINT) {
-			fListener->PrintError("%s entry in package file has an invalid "
-				"data chunk size attribute (not of type uint).\n",
-				B_HPKG_PACKAGE_INFO_FILE_NAME);
-			throw status_t(B_BAD_DATA);
-		}
-		chunkSize = chunkSizeAttribute->value.unsignedInt;
-	}
-
-	data.SetChunkSize(chunkSize);
+		data.SetData(value.data.size, value.data.offset);
 
 	// read the value into a string
 	BString valueString;
-	char* valueBuffer = valueString.LockBuffer(size);
+	char* valueBuffer = valueString.LockBuffer(value.data.size);
 	if (valueBuffer == NULL)
 		throw std::bad_alloc();
 
 	if (value.encoding == B_HPKG_ATTRIBUTE_ENCODING_RAW_INLINE) {
 		// data encoded inline -- just copy to buffer
-		if (size != value.data.size) {
-			fListener->PrintError("%s entry in package file has an invalid "
-				"data attribute (mismatching size).\n",
-				B_HPKG_PACKAGE_INFO_FILE_NAME);
-			throw status_t(B_BAD_DATA);
-		}
 		memcpy(valueBuffer, value.data.raw, value.data.size);
 	} else {
 		// data on heap -- read from there
-		BBlockBufferPoolNoLock	bufferCache(16 * 1024, 1);
-		status_t error = bufferCache.Init();
-		if (error != B_OK) {
-			fListener->PrintError("Failed to initialize buffer cache: %s\n",
-				strerror(error));
-			throw status_t(error);
-		}
-
-		// create a PackageDataReader
-		BFDDataReader packageFileReader(FD());
-		BAbstractBufferedDataReader* reader;
-		error = BPackageDataReaderFactory(&bufferCache)
-			.CreatePackageDataReader(&packageFileReader, data, reader);
-		if (error != B_OK) {
-			fListener->PrintError("Failed to create package data reader: %s\n",
-				strerror(error));
-			throw status_t(error);
-		}
-		ObjectDeleter<BAbstractBufferedDataReader> readerDeleter(reader);
-
-		// read the data
-		error = reader->ReadData(0, valueBuffer, size);
-		if (error != B_OK) {
-			fListener->PrintError("Failed to read data of %s entry in package "
-				"file: %s\n", B_HPKG_PACKAGE_INFO_FILE_NAME, strerror(error));
-			throw status_t(error);
-		}
+		status_t error = fHeapWriter->ReadData(data.Offset(), valueBuffer,
+			data.Size());
+		throw error;
 	}
 
 	valueString.UnlockBuffer();
@@ -1101,8 +1008,8 @@ PackageWriterImpl::_CompactHeap()
 		return;
 
 	// compute the move deltas for the ranges
-	Array<off_t> deltas;
-	off_t delta = 0;
+	Array<uint64> deltas;
+	uint64 delta = 0;
 	for (int32 i = 0; i < count; i++) {
 		if (!deltas.Add(delta))
 			throw std::bad_alloc();
@@ -1117,73 +1024,8 @@ PackageWriterImpl::_CompactHeap()
 	HeapAttributeOffsetter(*fHeapRangesToRemove, deltas).ProcessAttribute(
 		fRootAttribute);
 
-	// move the heap chunks in the file around
-	off_t chunkOffset = fHeapOffset;
-	delta = 0;
-
-	for (int32 i = 0; i < count; i++) {
-		const Range<off_t>& range = fHeapRangesToRemove->RangeAt(i);
-
-		if (delta > 0 && chunkOffset < range.offset) {
-			// move chunk
-			_MoveHeapChunk(chunkOffset, chunkOffset - delta,
-				range.offset - chunkOffset);
-		}
-
-		chunkOffset = range.EndOffset();
-		delta += range.size;
-	}
-
-	// move the final chunk
-	off_t heapSize = fHeapEnd - fHeapOffset;
-	if (delta > 0 && chunkOffset < heapSize) {
-		_MoveHeapChunk(chunkOffset, chunkOffset - delta,
-			heapSize - chunkOffset);
-	}
-
-	fHeapEnd -= delta;
-}
-
-
-void
-PackageWriterImpl::_MoveHeapChunk(off_t fromOffset, off_t toOffset, off_t size)
-{
-	// convert heap offsets to file offsets
-	fromOffset += fHeapOffset;
-	toOffset += fHeapOffset;
-
-	while (size > 0) {
-		size_t toCopy = std::min(size, (off_t)fDataBufferSize);
-
-		// read data into buffer
-		ssize_t bytesRead = read_pos(FD(), fromOffset, fDataBuffer, toCopy);
-		if (bytesRead < 0) {
-			fListener->PrintError("Failed to read from package file: %s\n",
-				strerror(errno));
-			throw status_t(errno);
-		}
-		if ((size_t)bytesRead < toCopy) {
-			fListener->PrintError("Failed to read from package file (wanted "
-				"%zu bytes, got %zd).\n", toCopy, bytesRead);
-			throw status_t(B_IO_ERROR);
-		}
-
-		// write data to target offset
-		ssize_t bytesWritten = write_pos(FD(), toOffset, fDataBuffer, toCopy);
-		if (bytesWritten < 0) {
-			fListener->PrintError("Failed to write to package file: %s\n",
-				strerror(errno));
-			throw status_t(errno);
-		}
-		if ((size_t)bytesWritten < toCopy) {
-			fListener->PrintError("Failed to write to package file.\n");
-			throw status_t(B_IO_ERROR);
-		}
-
-		fromOffset += toCopy;
-		toOffset += toCopy;
-		size -= toCopy;
-	}
+	// remove the ranges from the heap
+	fHeapWriter->RemoveDataRanges(*fHeapRangesToRemove);
 }
 
 
@@ -1216,41 +1058,52 @@ PackageWriterImpl::_Finish()
 		_AddEntry(AT_FDCWD, entry, entry->Name(), pathBuffer);
 	}
 
-	off_t heapSize = fHeapEnd - fHeapOffset;
-
 	hpkg_header header;
 
 	// write the TOC and package attributes
-	_WriteTOC(header);
-	_WritePackageAttributes(header);
+	uint64 tocLength;
+	_WriteTOC(header, tocLength);
 
-	off_t totalSize = fHeapEnd;
+	uint64 attributesLength;
+	_WritePackageAttributes(header, attributesLength);
+
+	// flush the heap
+	status_t error = fHeapWriter->Finish();
+	if (error != B_OK)
+		return error;
+
+	uint64 compressedHeapSize = fHeapWriter->CompressedHeapSize();
+
+	header.heap_compression = B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB);
+	header.heap_chunk_size = B_HOST_TO_BENDIAN_INT32(fHeapWriter->ChunkSize());
+	header.heap_size_compressed = B_HOST_TO_BENDIAN_INT64(
+		fHeapWriter->CompressedHeapSize());
+	header.heap_size_uncompressed = B_HOST_TO_BENDIAN_INT64(
+		fHeapWriter->UncompressedHeapSize());
 
 	// Truncate the file to the size it is supposed to have. In update mode, it
-	// can be greater when one or more files are shrunk. In creation mode, when
-	// writing compressed TOC or package attributes yields a larger size than
-	// uncompressed, the file size may also be greater than it should be.
+	// can be greater when one or more files are shrunk. In creation mode it
+	// should already have the correct size.
+	off_t totalSize = fHeapWriter->HeapOffset() + (off_t)compressedHeapSize;
 	if (ftruncate(FD(), totalSize) != 0) {
 		fListener->PrintError("Failed to truncate package file to new "
 			"size: %s\n", strerror(errno));
 		return errno;
 	}
 
-	fListener->OnPackageSizeInfo(fHeapOffset, heapSize,
-		B_BENDIAN_TO_HOST_INT64(header.toc_length_compressed),
-		B_BENDIAN_TO_HOST_INT32(header.attributes_length_compressed),
-		totalSize);
+	fListener->OnPackageSizeInfo(fHeaderSize, compressedHeapSize, tocLength,
+		attributesLength, totalSize);
 
 	// prepare the header
 
 	// general
 	header.magic = B_HOST_TO_BENDIAN_INT32(B_HPKG_MAGIC);
-	header.header_size = B_HOST_TO_BENDIAN_INT16((uint16)fHeapOffset);
+	header.header_size = B_HOST_TO_BENDIAN_INT16(fHeaderSize);
 	header.version = B_HOST_TO_BENDIAN_INT16(B_HPKG_VERSION);
 	header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize);
 
 	// write the header
-	WriteBuffer(&header, sizeof(hpkg_header), 0);
+	RawWriteBuffer(&header, sizeof(hpkg_header), 0);
 
 	SetFinished(true);
 	return B_OK;
@@ -1332,107 +1185,32 @@ PackageWriterImpl::_RegisterEntry(Entry* parent, const char* name,
 
 
 void
-PackageWriterImpl::_WriteTOC(hpkg_header& header)
+PackageWriterImpl::_WriteTOC(hpkg_header& header, uint64& _length)
 {
-	// prepare the writer (zlib writer on top of a file writer)
-	off_t startOffset = fHeapEnd;
+	// write the subsections
+	uint64 startOffset = fHeapWriter->UncompressedHeapSize();
 
-	// write the sections
-	uint32 compression = B_HPKG_COMPRESSION_ZLIB;
-	uint64 uncompressedStringsSize;
-	uint64 uncompressedMainSize;
-	uint64 tocUncompressedSize;
-	int32 cachedStringsWritten = _WriteTOCCompressed(uncompressedStringsSize,
-		uncompressedMainSize, tocUncompressedSize);
-
-	off_t endOffset = fHeapEnd;
-
-	if (endOffset - startOffset >= (off_t)tocUncompressedSize) {
-		// the compressed section isn't shorter -- write uncompressed
-		fHeapEnd = startOffset;
-		compression = B_HPKG_COMPRESSION_NONE;
-		cachedStringsWritten = _WriteTOCUncompressed(uncompressedStringsSize,
-			uncompressedMainSize, tocUncompressedSize);
-
-		endOffset = fHeapEnd;
-	}
-
-	fListener->OnTOCSizeInfo(uncompressedStringsSize, uncompressedMainSize,
-		tocUncompressedSize);
-
-	// update the header
-
-	// TOC
-	header.toc_compression = B_HOST_TO_BENDIAN_INT32(compression);
-	header.toc_length_compressed = B_HOST_TO_BENDIAN_INT64(
-		endOffset - startOffset);
-	header.toc_length_uncompressed = B_HOST_TO_BENDIAN_INT64(
-		tocUncompressedSize);
-
-	// TOC subsections
-	header.toc_strings_length = B_HOST_TO_BENDIAN_INT64(
-		uncompressedStringsSize);
-	header.toc_strings_count = B_HOST_TO_BENDIAN_INT64(cachedStringsWritten);
-}
-
-
-int32
-PackageWriterImpl::_WriteTOCCompressed(uint64& _uncompressedStringsSize,
-	uint64& _uncompressedMainSize, uint64& _tocUncompressedSize)
-{
-	FDDataWriter realWriter(FD(), fHeapEnd, fListener);
-	ZlibDataWriter zlibWriter(&realWriter);
-	SetDataWriter(&zlibWriter);
-	zlibWriter.Init();
-
-	// write the sections
-	int32 cachedStringsWritten
-		= _WriteTOCSections(_uncompressedStringsSize, _uncompressedMainSize);
-
-	// finish the writer
-	zlibWriter.Finish();
-	fHeapEnd = realWriter.Offset();
-	SetDataWriter(NULL);
-
-	_tocUncompressedSize = zlibWriter.BytesWritten();
-	return cachedStringsWritten;
-}
-
-
-int32
-PackageWriterImpl::_WriteTOCUncompressed(uint64& _uncompressedStringsSize,
-	uint64& _uncompressedMainSize, uint64& _tocUncompressedSize)
-{
-	FDDataWriter realWriter(FD(), fHeapEnd, fListener);
-	SetDataWriter(&realWriter);
-
-	// write the sections
-	int32 cachedStringsWritten
-		= _WriteTOCSections(_uncompressedStringsSize, _uncompressedMainSize);
-
-	fHeapEnd = realWriter.Offset();
-	SetDataWriter(NULL);
-
-	_tocUncompressedSize = realWriter.BytesWritten();
-	return cachedStringsWritten;
-}
-
-
-int32
-PackageWriterImpl::_WriteTOCSections(uint64& _stringsSize, uint64& _mainSize)
-{
-	// write the cached strings
-	uint64 cachedStringsOffset = DataWriter()->BytesWritten();
+	// cached strings
+	uint64 cachedStringsOffset = fHeapWriter->UncompressedHeapSize();
 	int32 cachedStringsWritten = WriteCachedStrings(fStringCache, 2);
 
-	// write the main TOC section
-	uint64 mainOffset = DataWriter()->BytesWritten();
+	// main TOC section
+	uint64 mainOffset = fHeapWriter->UncompressedHeapSize();
 	_WriteAttributeChildren(fRootAttribute);
 
-	_stringsSize = mainOffset - cachedStringsOffset;
-	_mainSize = DataWriter()->BytesWritten() - mainOffset;
+	// notify the listener
+	uint64 endOffset = fHeapWriter->UncompressedHeapSize();
+	uint64 stringsSize = mainOffset - cachedStringsOffset;
+	uint64 mainSize = endOffset - mainOffset;
+	uint64 tocSize = endOffset - startOffset;
+	fListener->OnTOCSizeInfo(stringsSize, mainSize, tocSize);
 
-	return cachedStringsWritten;
+	// update the header
+	header.toc_length = B_HOST_TO_BENDIAN_INT64(tocSize);
+	header.toc_strings_length = B_HOST_TO_BENDIAN_INT64(stringsSize);
+	header.toc_strings_count = B_HOST_TO_BENDIAN_INT64(cachedStringsWritten);
+
+	_length = tocSize;
 }
 
 
@@ -1459,85 +1237,25 @@ PackageWriterImpl::_WriteAttributeChildren(Attribute* attribute)
 
 
 void
-PackageWriterImpl::_WritePackageAttributes(hpkg_header& header)
+PackageWriterImpl::_WritePackageAttributes(hpkg_header& header, uint64& _length)
 {
-	// write the package attributes (zlib writer on top of a file writer)
-	off_t startOffset = fHeapEnd;
+	// write cached strings and package attributes tree
+	off_t startOffset = fHeapWriter->UncompressedHeapSize();
 
-	uint32 compression = B_HPKG_COMPRESSION_ZLIB;
-	uint32 stringsLengthUncompressed;
-	uint32 attributesLengthUncompressed;
-	uint32 stringsCount = _WritePackageAttributesCompressed(
-		stringsLengthUncompressed, attributesLengthUncompressed);
+	uint32 stringsLength;
+	uint32 stringsCount = WritePackageAttributes(PackageAttributes(),
+		stringsLength);
 
-	off_t endOffset = fHeapEnd;
-
-	if ((off_t)attributesLengthUncompressed <= endOffset - startOffset) {
-		// the compressed section isn't shorter -- write uncompressed
-		fHeapEnd = startOffset;
-		compression = B_HPKG_COMPRESSION_NONE;
-		stringsCount = _WritePackageAttributesUncompressed(
-			stringsLengthUncompressed, attributesLengthUncompressed);
-
-		endOffset = fHeapEnd;
-	}
-
-	fListener->OnPackageAttributesSizeInfo(stringsCount,
-		attributesLengthUncompressed);
+	// notify listener
+	uint32 attributesLength = fHeapWriter->UncompressedHeapSize() - startOffset;
+	fListener->OnPackageAttributesSizeInfo(stringsCount, attributesLength);
 
 	// update the header
-	header.attributes_compression = B_HOST_TO_BENDIAN_INT32(compression);
-	header.attributes_length_compressed
-		= B_HOST_TO_BENDIAN_INT32(endOffset - startOffset);
-	header.attributes_length_uncompressed
-		= B_HOST_TO_BENDIAN_INT32(attributesLengthUncompressed);
+	header.attributes_length = B_HOST_TO_BENDIAN_INT32(attributesLength);
 	header.attributes_strings_count = B_HOST_TO_BENDIAN_INT32(stringsCount);
-	header.attributes_strings_length
-		= B_HOST_TO_BENDIAN_INT32(stringsLengthUncompressed);
-}
+	header.attributes_strings_length = B_HOST_TO_BENDIAN_INT32(stringsLength);
 
-
-uint32
-PackageWriterImpl::_WritePackageAttributesCompressed(
-	uint32& _stringsLengthUncompressed, uint32& _attributesLengthUncompressed)
-{
-	off_t startOffset = fHeapEnd;
-	FDDataWriter realWriter(FD(), startOffset, fListener);
-	ZlibDataWriter zlibWriter(&realWriter);
-	SetDataWriter(&zlibWriter);
-	zlibWriter.Init();
-
-	// write cached strings and package attributes tree
-	uint32 stringsCount = WritePackageAttributes(PackageAttributes(),
-		_stringsLengthUncompressed);
-
-	zlibWriter.Finish();
-	fHeapEnd = realWriter.Offset();
-	SetDataWriter(NULL);
-
-	_attributesLengthUncompressed = zlibWriter.BytesWritten();
-	return stringsCount;
-}
-
-
-uint32
-PackageWriterImpl::_WritePackageAttributesUncompressed(
-	uint32& _stringsLengthUncompressed, uint32& _attributesLengthUncompressed)
-{
-	off_t startOffset = fHeapEnd;
-	FDDataWriter realWriter(FD(), startOffset, fListener);
-
-	SetDataWriter(&realWriter);
-
-	// write cached strings and package attributes tree
-	uint32 stringsCount = WritePackageAttributes(PackageAttributes(),
-		_stringsLengthUncompressed);
-
-	fHeapEnd = realWriter.Offset();
-	SetDataWriter(NULL);
-
-	_attributesLengthUncompressed = realWriter.BytesWritten();
-	return stringsCount;
+	_length = attributesLength;
 }
 
 
@@ -1804,178 +1522,13 @@ PackageWriterImpl::_AddData(BDataReader& dataReader, off_t size)
 		return B_OK;
 	}
 
-	// longer data -- try to compress
-	uint64 dataOffset = fHeapEnd;
-
-	uint64 compression = B_HPKG_COMPRESSION_NONE;
-	uint64 compressedSize;
-
-	status_t error = _WriteZlibCompressedData(dataReader, size, dataOffset,
-		compressedSize);
-	if (error == B_OK) {
-		compression = B_HPKG_COMPRESSION_ZLIB;
-	} else {
-		error = _WriteUncompressedData(dataReader, size, dataOffset);
-		compressedSize = size;
-	}
+	// add data to heap
+	uint64 dataOffset;
+	status_t error = fHeapWriter->AddData(dataReader, size, dataOffset);
 	if (error != B_OK)
 		return error;
 
-	fHeapEnd = dataOffset + compressedSize;
-
-	// add data attribute
-	Attribute* dataAttribute = _AddDataAttribute(B_HPKG_ATTRIBUTE_ID_DATA,
-		compressedSize, dataOffset - fHeapOffset);
-	Stacker<Attribute> attributeAttributeStacker(fTopAttribute, dataAttribute);
-
-	// if compressed, add compression attributes
-	if (compression != B_HPKG_COMPRESSION_NONE) {
-		_AddAttribute(B_HPKG_ATTRIBUTE_ID_DATA_COMPRESSION, compression);
-		_AddAttribute(B_HPKG_ATTRIBUTE_ID_DATA_SIZE, (uint64)size);
-			// uncompressed size
-	}
-
-	return B_OK;
-}
-
-
-status_t
-PackageWriterImpl::_WriteUncompressedData(BDataReader& dataReader, off_t size,
-	uint64 writeOffset)
-{
-	// copy the data to the heap
-	off_t readOffset = 0;
-	off_t remainingSize = size;
-	while (remainingSize > 0) {
-		// read data
-		size_t toCopy = std::min(remainingSize, (off_t)fDataBufferSize);
-		status_t error = dataReader.ReadData(readOffset, fDataBuffer, toCopy);
-		if (error != B_OK) {
-			fListener->PrintError("Failed to read data: %s\n", strerror(error));
-			return error;
-		}
-
-		// write to heap
-		ssize_t bytesWritten = pwrite(FD(), fDataBuffer, toCopy, writeOffset);
-		if (bytesWritten < 0) {
-			fListener->PrintError("Failed to write data: %s\n",
-				strerror(errno));
-			return errno;
-		}
-		if ((size_t)bytesWritten != toCopy) {
-			fListener->PrintError("Failed to write all data\n");
-			return B_ERROR;
-		}
-
-		remainingSize -= toCopy;
-		readOffset += toCopy;
-		writeOffset += toCopy;
-	}
-
-	return B_OK;
-}
-
-
-status_t
-PackageWriterImpl::_WriteZlibCompressedData(BDataReader& dataReader, off_t size,
-	uint64 writeOffset, uint64& _compressedSize)
-{
-	// Use zlib compression only for data large enough.
-	if (size < (off_t)kZlibCompressionSizeThreshold)
-		return B_BAD_VALUE;
-
-	// fDataBuffer is 2 * B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, so split it into
-	// two halves we can use for reading and compressing
-	const size_t chunkSize = B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB;
-	uint8* inputBuffer = (uint8*)fDataBuffer;
-	uint8* outputBuffer = (uint8*)fDataBuffer + chunkSize;
-
-	// account for the offset table
-	uint64 chunkCount = (size + (chunkSize - 1)) / chunkSize;
-	off_t offsetTableOffset = writeOffset;
-	uint64* offsetTable = NULL;
-	if (chunkCount > 1) {
-		offsetTable = new uint64[chunkCount - 1];
-		writeOffset = offsetTableOffset + (chunkCount - 1) * sizeof(uint64);
-	}
-	ArrayDeleter<uint64> offsetTableDeleter(offsetTable);
-
-	const uint64 dataOffset = writeOffset;
-	const uint64 dataEndLimit = offsetTableOffset + size;
-
-	// read the data, compress them and write them to the heap
-	off_t readOffset = 0;
-	off_t remainingSize = size;
-	uint64 chunkIndex = 0;
-	while (remainingSize > 0) {
-		// read data
-		size_t toCopy = std::min(remainingSize, (off_t)chunkSize);
-		status_t error = dataReader.ReadData(readOffset, inputBuffer, toCopy);
-		if (error != B_OK) {
-			fListener->PrintError("Failed to read data: %s\n", strerror(error));
-			return error;
-		}
-
-		// compress
-		size_t compressedSize;
-		error = ZlibCompressor::CompressSingleBuffer(inputBuffer, toCopy,
-			outputBuffer, toCopy, compressedSize);
-
-		const void* writeBuffer;
-		size_t bytesToWrite;
-		if (error == B_OK) {
-			writeBuffer = outputBuffer;
-			bytesToWrite = compressedSize;
-		} else {
-			if (error != B_BUFFER_OVERFLOW)
-				return error;
-			writeBuffer = inputBuffer;
-			bytesToWrite = toCopy;
-		}
-
-		// check the total compressed data size
-		if (writeOffset + bytesToWrite >= dataEndLimit)
-			return B_BUFFER_OVERFLOW;
-
-		if (chunkIndex > 0)
-			offsetTable[chunkIndex - 1] = writeOffset - dataOffset;
-
-		// write to heap
-		ssize_t bytesWritten = pwrite(FD(), writeBuffer, bytesToWrite,
-			writeOffset);
-		if (bytesWritten < 0) {
-			fListener->PrintError("Failed to write data: %s\n",
-				strerror(errno));
-			return errno;
-		}
-		if ((size_t)bytesWritten != bytesToWrite) {
-			fListener->PrintError("Failed to write all data\n");
-			return B_ERROR;
-		}
-
-		remainingSize -= toCopy;
-		readOffset += toCopy;
-		writeOffset += bytesToWrite;
-		chunkIndex++;
-	}
-
-	// write the offset table
-	if (chunkCount > 1) {
-		size_t bytesToWrite = (chunkCount - 1) * sizeof(uint64);
-		ssize_t bytesWritten = pwrite(FD(), offsetTable, bytesToWrite,
-			offsetTableOffset);
-		if (bytesWritten < 0) {
-			fListener->PrintError("Failed to write data: %s\n",
-				strerror(errno));
-			return errno;
-		}
-		if ((size_t)bytesWritten != bytesToWrite) {
-			fListener->PrintError("Failed to write all data\n");
-			return B_ERROR;
-		}
-	}
-
-	_compressedSize = writeOffset - offsetTableOffset;
+	_AddDataAttribute(B_HPKG_ATTRIBUTE_ID_DATA, size, dataOffset);
 	return B_OK;
 }
 

src/kits/package/hpkg/ReaderImplBase.cpp

@@ -21,6 +21,7 @@
 
 #include <package/hpkg/DataOutput.h>
 #include <package/hpkg/ErrorOutput.h>
+#include <package/hpkg/PackageFileHeapReader.h>
 #include <package/hpkg/ZlibDecompressor.h>
 
 
@@ -536,12 +537,14 @@ ReaderImplBase::LowLevelAttributeHandler::Delete(
 // #pragma mark - ReaderImplBase
 
 
-ReaderImplBase::ReaderImplBase(BErrorOutput* errorOutput)
+ReaderImplBase::ReaderImplBase(const char* fileType, BErrorOutput* errorOutput)
 	:
 	fPackageAttributesSection("package attributes"),
+	fFileType(fileType),
 	fErrorOutput(errorOutput),
 	fFD(-1),
 	fOwnsFD(false),
+	fHeapReader(NULL),
 	fCurrentSection(NULL),
 	fScratchBuffer(NULL),
 	fScratchBufferSize(0)
@@ -551,6 +554,8 @@ ReaderImplBase::ReaderImplBase(BErrorOutput* errorOutput)
 
 ReaderImplBase::~ReaderImplBase()
 {
+	delete fHeapReader;
+
 	if (fOwnsFD && fFD >= 0)
 		close(fFD);
 
@@ -559,7 +564,7 @@ ReaderImplBase::~ReaderImplBase()
 
 
 status_t
-ReaderImplBase::Init(int fd, bool keepFD)
+ReaderImplBase::_Init(int fd, bool keepFD)
 {
 	fFD = fd;
 	fOwnsFD = keepFD;
@@ -576,27 +581,84 @@ ReaderImplBase::Init(int fd, bool keepFD)
 }
 
 
-const char*
-ReaderImplBase::CheckCompression(const SectionInfo& section) const
+status_t
+ReaderImplBase::InitHeapReader(uint32 compression, uint32 chunkSize,
+	off_t offset, uint64 compressedSize, uint64 uncompressedSize)
 {
-	switch (section.compression) {
-		case B_HPKG_COMPRESSION_NONE:
-			if (section.compressedLength != section.uncompressedLength) {
-				return "Uncompressed, but compressed and uncompressed length "
-					"don't match";
-			}
-			return NULL;
-
-		case B_HPKG_COMPRESSION_ZLIB:
-			if (section.compressedLength >= section.uncompressedLength) {
-				return "Compressed, but compressed length is not less than "
-					"uncompressed length";
-			}
-			return NULL;
-
-		default:
-			return "Invalid compression algorithm ID";
+	if (compression != B_HPKG_COMPRESSION_ZLIB) {
+		fErrorOutput->PrintError("Error: Invalid heap compression\n");
+		return B_BAD_DATA;
 	}
+
+	fHeapReader = new(std::nothrow) PackageFileHeapReader(fErrorOutput, fFD,
+		offset, compressedSize, uncompressedSize);
+	return fHeapReader->Init();
+}
+
+
+status_t
+ReaderImplBase::InitSection(PackageFileSection& section, uint64 endOffset,
+	uint64 length, uint64 maxSaneLength, uint64 stringsLength,
+	uint64 stringsCount)
+{
+	// check length vs. endOffset
+	if (length > endOffset) {
+		ErrorOutput()->PrintError("Error: %s file %s section size is %"
+			B_PRIu64 " bytes. This is greater than the available space\n",
+			fFileType, section.name, length);
+		return B_BAD_DATA;
+	}
+
+	// check sanity length
+	if (maxSaneLength > 0 && length > maxSaneLength) {
+		ErrorOutput()->PrintError("Error: %s file %s section size is %"
+			B_PRIu64 " bytes. This is beyond the reader's sanity limit\n",
+			fFileType, section.name, length);
+		return B_NOT_SUPPORTED;
+	}
+
+	// check strings subsection size/count
+	if ((stringsLength == 0) != (stringsCount == 0) || stringsLength > length) {
+		ErrorOutput()->PrintError("Error: strings subsection description of %s "
+			"file %s section is invalid (%" B_PRIu64 " strings, length: %"
+			B_PRIu64 ")\n",
+			fFileType, section.name, length, stringsCount, stringsLength);
+		return B_BAD_DATA;
+	}
+
+	section.uncompressedLength = length;
+	section.offset = endOffset - length;
+	section.currentOffset = 0;
+	section.stringsLength = stringsLength;
+	section.stringsCount = stringsCount;
+
+	return B_OK;
+}
+
+
+status_t
+ReaderImplBase::PrepareSection(PackageFileSection& section)
+{
+	// allocate memory for the section data and read it in
+	section.data = new(std::nothrow) uint8[section.uncompressedLength];
+	if (section.data == NULL) {
+		ErrorOutput()->PrintError("Error: Out of memory!\n");
+		return B_NO_MEMORY;
+	}
+
+	status_t error = ReadSection(section);
+	if (error != B_OK)
+		return error;
+
+	// parse the section strings
+	section.currentOffset = 0;
+	SetCurrentSection(&section);
+
+	error = ParseStrings();
+	if (error != B_OK)
+		return error;
+
+	return B_OK;
 }
 
 
@@ -982,8 +1044,9 @@ ReaderImplBase::_ReadSectionBuffer(void* buffer, size_t size)
 {
 	if (size > fCurrentSection->uncompressedLength
 			- fCurrentSection->currentOffset) {
-		fErrorOutput->PrintError("_ReadBuffer(%lu): read beyond %s end\n",
-			size, fCurrentSection->name);
+		fErrorOutput->PrintError(
+			"_ReadSectionBuffer(%lu): read beyond %s end\n", size,
+			fCurrentSection->name);
 		return B_BAD_DATA;
 	}
 
@@ -1014,63 +1077,11 @@ ReaderImplBase::ReadBuffer(off_t offset, void* buffer, size_t size)
 
 
 status_t
-ReaderImplBase::ReadCompressedBuffer(const SectionInfo& section)
+ReaderImplBase::ReadSection(const PackageFileSection& section)
 {
-	uint32 compressedSize = section.compressedLength;
-	uint64 offset = section.offset;
-
-	switch (section.compression) {
-		case B_HPKG_COMPRESSION_NONE:
-			return ReadBuffer(offset, section.data, compressedSize);
-
-		case B_HPKG_COMPRESSION_ZLIB:
-		{
-			// init the decompressor
-			BBufferDataOutput bufferOutput(section.data,
-				section.uncompressedLength);
-			ZlibDecompressor decompressor(&bufferOutput);
-			status_t error = decompressor.Init();
-			if (error != B_OK)
-				return error;
-
-			while (compressedSize > 0) {
-				// read compressed buffer
-				size_t toRead = std::min((size_t)compressedSize,
-					fScratchBufferSize);
-				error = ReadBuffer(offset, fScratchBuffer, toRead);
-				if (error != B_OK)
-					return error;
-
-				// uncompress
-				error = decompressor.DecompressNext(fScratchBuffer, toRead);
-				if (error != B_OK)
-					return error;
-
-				compressedSize -= toRead;
-				offset += toRead;
-			}
-
-			error = decompressor.Finish();
-			if (error != B_OK)
-				return error;
-
-			// verify that all data have been read
-			if (bufferOutput.BytesWritten() != section.uncompressedLength) {
-				fErrorOutput->PrintError("Error: Missing bytes in uncompressed "
-					"buffer!\n");
-				return B_BAD_DATA;
-			}
-
-			return B_OK;
-		}
-
-		default:
-		{
-			fErrorOutput->PrintError("Error: Invalid compression type: %u\n",
-				section.compression);
-			return B_BAD_DATA;
-		}
-	}
+	BBufferDataOutput output(section.data, section.uncompressedLength);
+	return fHeapReader->ReadDataToOutput(section.offset,
+		section.uncompressedLength, &output);
 }
 
 

src/kits/package/hpkg/RepositoryReaderImpl.cpp

@@ -20,6 +20,7 @@
 
 #include <package/hpkg/ErrorOutput.h>
 #include <package/hpkg/HPKGDefsPrivate.h>
+#include <package/hpkg/PackageFileHeapReader.h>
 #include <package/hpkg/RepositoryContentHandler.h>
 
 
@@ -43,8 +44,7 @@ static const size_t kMaxPackageAttributesSize	= 64 * 1024 * 1024;
 
 RepositoryReaderImpl::RepositoryReaderImpl(BErrorOutput* errorOutput)
 	:
-	inherited(errorOutput),
-	fRepositoryInfoSection("repository info")
+	inherited("repository", errorOutput)
 {
 }
 
@@ -73,146 +73,43 @@ RepositoryReaderImpl::Init(const char* fileName)
 status_t
 RepositoryReaderImpl::Init(int fd, bool keepFD)
 {
-	status_t error = inherited::Init(fd, keepFD);
-	if (error != B_OK)
-		return error;
-
-	// stat it
-	struct stat st;
-	if (fstat(FD(), &st) < 0) {
-		ErrorOutput()->PrintError(
-			"Error: Failed to access repository file: %s\n", strerror(errno));
-		return errno;
-	}
-
-	// read the header
 	hpkg_repo_header header;
-	if ((error = ReadBuffer(0, &header, sizeof(header))) != B_OK)
-		return error;
-
-	// check the header
-
-	// magic
-	if (B_BENDIAN_TO_HOST_INT32(header.magic) != B_HPKG_REPO_MAGIC) {
-		ErrorOutput()->PrintError("Error: Invalid repository file: Invalid "
-			"magic\n");
-		return B_BAD_DATA;
-	}
-
-	// header size
-	size_t headerSize = B_BENDIAN_TO_HOST_INT16(header.header_size);
-	if (headerSize < sizeof(hpkg_repo_header)) {
-		ErrorOutput()->PrintError("Error: Invalid repository file: Invalid "
-			"header size (%llu)\n", headerSize);
-		return B_BAD_DATA;
-	}
-
-	// version
-	if (B_BENDIAN_TO_HOST_INT16(header.version) != B_HPKG_REPO_VERSION) {
-		ErrorOutput()->PrintError("Error: Invalid/unsupported repository file "
-			"version (%d)\n", B_BENDIAN_TO_HOST_INT16(header.version));
-		return B_BAD_DATA;
-	}
-
-	// total size
-	uint64 totalSize = B_BENDIAN_TO_HOST_INT64(header.total_size);
-	if (totalSize != (uint64)st.st_size) {
-		ErrorOutput()->PrintError("Error: Invalid repository file: Total size "
-			"in header (%llu) doesn't agree with total file size (%lld)\n",
-			totalSize, st.st_size);
-		return B_BAD_DATA;
-	}
-
-	// repository info length and compression
-	fRepositoryInfoSection.compression
-		= B_BENDIAN_TO_HOST_INT32(header.info_compression);
-	fRepositoryInfoSection.compressedLength
-		= B_BENDIAN_TO_HOST_INT32(header.info_length_compressed);
-	fRepositoryInfoSection.uncompressedLength
-		= B_BENDIAN_TO_HOST_INT32(header.info_length_uncompressed);
-
-	if (const char* errorString = CheckCompression(fRepositoryInfoSection)) {
-		ErrorOutput()->PrintError(
-			"Error: Invalid repository file: info section: %s\n", errorString);
-		return B_BAD_DATA;
-	}
-
-	// package attributes length and compression
-	fPackageAttributesSection.compression
-		= B_BENDIAN_TO_HOST_INT32(header.packages_compression);
-	fPackageAttributesSection.compressedLength
-		= B_BENDIAN_TO_HOST_INT64(header.packages_length_compressed);
-	fPackageAttributesSection.uncompressedLength
-		= B_BENDIAN_TO_HOST_INT64(header.packages_length_uncompressed);
-	fPackageAttributesSection.stringsLength
-		= B_BENDIAN_TO_HOST_INT64(header.packages_strings_length);
-	fPackageAttributesSection.stringsCount
-		= B_BENDIAN_TO_HOST_INT64(header.packages_strings_count);
-
-	if (const char* errorString = CheckCompression(
-		fPackageAttributesSection)) {
-		ErrorOutput()->PrintError("Error: Invalid repository file: package "
-			"attributes section: %s\n", errorString);
-		return B_BAD_DATA;
-	}
-
-	// check whether the sections fit together
-	if (fPackageAttributesSection.compressedLength > totalSize
-		|| fRepositoryInfoSection.compressedLength
-			> totalSize - fPackageAttributesSection.compressedLength) {
-		ErrorOutput()->PrintError("Error: Invalid repository file: The sum of "
-			"the sections sizes is greater than the repository size\n");
-		return B_BAD_DATA;
-	}
-
-	fPackageAttributesSection.offset
-		= totalSize - fPackageAttributesSection.compressedLength;
-	fRepositoryInfoSection.offset = fPackageAttributesSection.offset
-		- fRepositoryInfoSection.compressedLength;
-
-	// repository info size sanity check
-	if (fRepositoryInfoSection.uncompressedLength > kMaxRepositoryInfoSize) {
-		ErrorOutput()->PrintError("Error: Repository file info section size "
-			"is %llu bytes. This is beyond the reader's sanity limit\n",
-			fRepositoryInfoSection.uncompressedLength);
-		return B_UNSUPPORTED;
-	}
-
-	// package attributes size sanity check
-	if (fPackageAttributesSection.uncompressedLength
-			> kMaxPackageAttributesSize) {
-		ErrorOutput()->PrintError(
-			"Error: Package file package attributes section size "
-			"is %llu bytes. This is beyond the reader's sanity limit\n",
-			fPackageAttributesSection.uncompressedLength);
-		return B_UNSUPPORTED;
-	}
-
-	// read in the complete repository info section
-	fRepositoryInfoSection.data
-		= new(std::nothrow) uint8[fRepositoryInfoSection.uncompressedLength];
-	if (fRepositoryInfoSection.data == NULL) {
-		ErrorOutput()->PrintError("Error: Out of memory!\n");
-		return B_NO_MEMORY;
-	}
-	error = ReadCompressedBuffer(fRepositoryInfoSection);
+	status_t error = inherited::Init<hpkg_repo_header, B_HPKG_REPO_MAGIC,
+		B_HPKG_REPO_VERSION>(fd, keepFD, header);
 	if (error != B_OK)
 		return error;
 
-	// read in the complete package attributes section
-	fPackageAttributesSection.data
-		= new(std::nothrow) uint8[fPackageAttributesSection.uncompressedLength];
-	if (fPackageAttributesSection.data == NULL) {
-		ErrorOutput()->PrintError("Error: Out of memory!\n");
-		return B_NO_MEMORY;
-	}
-	error = ReadCompressedBuffer(fPackageAttributesSection);
+	// init package attributes section
+	error = InitSection(fPackageAttributesSection,
+		HeapReader()->UncompressedHeapSize(),
+		B_BENDIAN_TO_HOST_INT64(header.packages_length),
+		kMaxPackageAttributesSize,
+		B_BENDIAN_TO_HOST_INT64(header.packages_strings_length),
+		B_BENDIAN_TO_HOST_INT64(header.packages_strings_count));
+	if (error != B_OK)
+		return error;
+
+	// init repository info section
+	PackageFileSection repositoryInfoSection("repository info");
+	error = InitSection(repositoryInfoSection,
+		fPackageAttributesSection.offset,
+		B_BENDIAN_TO_HOST_INT32(header.info_length), kMaxRepositoryInfoSize, 0,
+		0);
+	if (error != B_OK)
+		return error;
+
+	// prepare the sections for use
+	error = PrepareSection(repositoryInfoSection);
+	if (error != B_OK)
+		return error;
+
+	error = PrepareSection(fPackageAttributesSection);
 	if (error != B_OK)
 		return error;
 
 	// unarchive repository info
 	BMessage repositoryInfoArchive;
-	error = repositoryInfoArchive.Unflatten((char*)fRepositoryInfoSection.data);
+	error = repositoryInfoArchive.Unflatten((char*)repositoryInfoSection.data);
 	if (error != B_OK) {
 		ErrorOutput()->PrintError(
 			"Error: Unable to unflatten repository info archive!\n");
@@ -225,17 +122,6 @@ RepositoryReaderImpl::Init(int fd, bool keepFD)
 		return error;
 	}
 
-	// parse strings from package attributes section
-	fPackageAttributesSection.currentOffset = 0;
-	SetCurrentSection(&fPackageAttributesSection);
-
-	// strings
-	error = ParseStrings();
-	if (error != B_OK)
-		return error;
-
-	SetCurrentSection(NULL);
-
 	return B_OK;
 }
 

src/kits/package/hpkg/RepositoryWriterImpl.cpp

@@ -20,6 +20,7 @@
 #include <package/hpkg/HPKGDefsPrivate.h>
 #include <package/hpkg/PackageDataReader.h>
 #include <package/hpkg/PackageEntry.h>
+#include <package/hpkg/PackageFileHeapWriter.h>
 #include <package/hpkg/PackageInfoAttributeValue.h>
 #include <package/hpkg/PackageReader.h>
 #include <package/ChecksumAccessors.h>
@@ -42,28 +43,31 @@ using BPackageKit::BPrivate::HashableString;
 namespace {
 
 
+// #pragma mark - PackageEntryDataFetcher
+
+
 struct PackageEntryDataFetcher {
 	PackageEntryDataFetcher(BErrorOutput* errorOutput,
 		BPackageData& packageData)
 		:
 		fErrorOutput(errorOutput),
-		fBufferPool(B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, 2),
 		fPackageData(packageData)
 	{
 	}
 
-	status_t ReadIntoString(BDataReader* dataReader, BString& _contents)
+	status_t ReadIntoString(BAbstractBufferedDataReader* heapReader,
+		BString& _contents)
 	{
 		// create a PackageDataReader
 		BAbstractBufferedDataReader* reader;
-		status_t result = BPackageDataReaderFactory(&fBufferPool)
-			.CreatePackageDataReader(dataReader, fPackageData, reader);
+		status_t result = BPackageDataReaderFactory()
+			.CreatePackageDataReader(heapReader, fPackageData, reader);
 		if (result != B_OK)
 			return result;
 		ObjectDeleter<BAbstractBufferedDataReader> readerDeleter(reader);
 
 		// copy data into the given string
-		int32 bufferSize = fPackageData.UncompressedSize();
+		int32 bufferSize = fPackageData.Size();
 		char* buffer = _contents.LockBuffer(bufferSize);
 		if (buffer == NULL)
 			return B_NO_MEMORY;
@@ -81,17 +85,20 @@ struct PackageEntryDataFetcher {
 
 private:
 	BErrorOutput*			fErrorOutput;
-	BBlockBufferPoolNoLock	fBufferPool;
 	BPackageData&			fPackageData;
 };
 
 
+// #pragma mark - PackageContentHandler
+
+
 struct PackageContentHandler : public BPackageInfoContentHandler {
 	PackageContentHandler(BErrorOutput* errorOutput, BPackageInfo* packageInfo,
-		int packageFileFD, BRepositoryInfo* repositoryInfo)
+		BAbstractBufferedDataReader* heapReader,
+		BRepositoryInfo* repositoryInfo)
 		:
 		BPackageInfoContentHandler(*packageInfo, errorOutput),
-		fPackageFileReader(packageFileFD),
+		fHeapReader(heapReader),
 		fRepositoryInfo(repositoryInfo)
 	{
 	}
@@ -132,16 +139,7 @@ struct PackageContentHandler : public BPackageInfoContentHandler {
 		PackageEntryDataFetcher dataFetcher(fErrorOutput, packageData);
 
 		BString licenseText;
-		status_t result;
-		if (packageData.IsEncodedInline()) {
-			BBufferDataReader dataReader(packageData.InlineData(),
-				packageData.CompressedSize());
-			result = dataFetcher.ReadIntoString(&dataReader, licenseText);
-		} else {
-			result
-				= dataFetcher.ReadIntoString(&fPackageFileReader, licenseText);
-		}
-
+		status_t result = dataFetcher.ReadIntoString(fHeapReader, licenseText);
 		if (result != B_OK)
 			return result;
 
@@ -165,24 +163,30 @@ struct PackageContentHandler : public BPackageInfoContentHandler {
 	}
 
 private:
-	BPackageReader*		fPackageReader;
-	BFDDataReader		fPackageFileReader;
-	BRepositoryInfo*	fRepositoryInfo;
+	BPackageReader*					fPackageReader;
+	BAbstractBufferedDataReader*	fHeapReader;
+	BRepositoryInfo*				fRepositoryInfo;
 };
 
 
 }	// anonymous namespace
 
 
+// #pragma mark - PackageNameSet
+
+
 struct RepositoryWriterImpl::PackageNameSet
 	: public ::BPrivate::HashSet<HashableString> {
 };
 
 
+// #pragma mark - RepositoryWriterImpl
+
+
 RepositoryWriterImpl::RepositoryWriterImpl(BRepositoryWriterListener* listener,
 	BRepositoryInfo* repositoryInfo)
 	:
-	inherited(listener),
+	inherited("repository", listener),
 	fListener(listener),
 	fRepositoryInfo(repositoryInfo),
 	fPackageCount(0),
@@ -246,7 +250,7 @@ RepositoryWriterImpl::Finish()
 status_t
 RepositoryWriterImpl::_Init(const char* fileName)
 {
-	return inherited::Init(fileName, "repository", 0);
+	return inherited::Init(fileName, 0);
 }
 
 
@@ -255,29 +259,34 @@ RepositoryWriterImpl::_Finish()
 {
 	hpkg_repo_header header;
 
-	// write repository header
-	ssize_t infoLengthCompressed;
-	status_t result = _WriteRepositoryInfo(header, infoLengthCompressed);
+	// write repository info
+	uint64 infoLength;
+	status_t result = _WriteRepositoryInfo(header, infoLength);
 	if (result != B_OK)
 		return result;
 
 	// write package attributes
-	ssize_t packagesLengthCompressed;
-	off_t totalSize = _WritePackageAttributes(header,
-		sizeof(header) + infoLengthCompressed, packagesLengthCompressed);
+	uint64 packagesLength;
+	_WritePackageAttributes(header, packagesLength);
 
-	fListener->OnRepositoryDone(sizeof(header), infoLengthCompressed,
+	// flush the heap writer
+	result = fHeapWriter->Finish();
+	if (result != B_OK)
+		return result;
+	uint64 totalSize = fHeapWriter->HeapOffset()
+		+ fHeapWriter->CompressedHeapSize();
+
+	fListener->OnRepositoryDone(sizeof(header), infoLength,
 		fRepositoryInfo->LicenseNames().CountStrings(), fPackageCount,
-		packagesLengthCompressed, totalSize);
+		packagesLength, totalSize);
 
-	// general
+	// update the general header info and write the header
 	header.magic = B_HOST_TO_BENDIAN_INT32(B_HPKG_REPO_MAGIC);
 	header.header_size = B_HOST_TO_BENDIAN_INT16((uint16)sizeof(header));
 	header.version = B_HOST_TO_BENDIAN_INT16(B_HPKG_REPO_VERSION);
 	header.total_size = B_HOST_TO_BENDIAN_INT64(totalSize);
 
-	// write the header
-	WriteBuffer(&header, sizeof(header), 0);
+	RawWriteBuffer(&header, sizeof(header), 0);
 
 	SetFinished(true);
 	return B_OK;
@@ -309,7 +318,7 @@ RepositoryWriterImpl::_AddPackage(const BEntry& packageEntry)
 
 	// parse package
 	PackageContentHandler contentHandler(fListener, &fPackageInfo,
-		packageReader.PackageFileFD(), fRepositoryInfo);
+		packageReader.HeapReader(), fRepositoryInfo);
 	if ((result = packageReader.ParseContent(&contentHandler)) != B_OK)
 		return result;
 
@@ -384,8 +393,9 @@ RepositoryWriterImpl::_RegisterCurrentPackageInfo()
 
 status_t
 RepositoryWriterImpl::_WriteRepositoryInfo(hpkg_repo_header& header,
-	ssize_t& _infoLengthCompressed)
+	uint64& _length)
 {
+	// archive and flatten the repository info and write it
 	BMessage archive;
 	status_t result = fRepositoryInfo->Archive(&archive);
 	if (result != B_OK) {
@@ -400,71 +410,40 @@ RepositoryWriterImpl::_WriteRepositoryInfo(hpkg_repo_header& header,
 		return result;
 	}
 
-	off_t startOffset = sizeof(hpkg_repo_header);
+	WriteBuffer(buffer, flattenedSize);
 
-	// write the package attributes (zlib writer on top of a file writer)
-	FDDataWriter realWriter(FD(), startOffset, fListener);
-	ZlibDataWriter zlibWriter(&realWriter);
-	SetDataWriter(&zlibWriter);
-	zlibWriter.Init();
-
-	DataWriter()->WriteDataThrows(buffer, flattenedSize);
-
-	zlibWriter.Finish();
-	SetDataWriter(NULL);
-
-	fListener->OnRepositoryInfoSectionDone(zlibWriter.BytesWritten());
-
-	_infoLengthCompressed = realWriter.BytesWritten();
+	// notify listener
+	fListener->OnRepositoryInfoSectionDone(flattenedSize);
 
 	// update the header
-	header.info_compression
-		= B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB);
-	header.info_length_compressed
-		= B_HOST_TO_BENDIAN_INT32(_infoLengthCompressed);
-	header.info_length_uncompressed
-		= B_HOST_TO_BENDIAN_INT32(flattenedSize);
+	header.info_length = B_HOST_TO_BENDIAN_INT32(flattenedSize);
 
+	_length = flattenedSize;
 	return B_OK;
 }
 
 
-off_t
+void
 RepositoryWriterImpl::_WritePackageAttributes(hpkg_repo_header& header,
-	off_t startOffset, ssize_t& _packagesLengthCompressed)
+	uint64& _length)
 {
 	// write the package attributes (zlib writer on top of a file writer)
-	FDDataWriter realWriter(FD(), startOffset, fListener);
-	ZlibDataWriter zlibWriter(&realWriter);
-	SetDataWriter(&zlibWriter);
-	zlibWriter.Init();
+	uint64 startOffset = fHeapWriter->UncompressedHeapSize();
 
-	// write cached strings and package attributes tree
-	uint32 stringsLengthUncompressed;
+	uint32 stringsLength;
 	uint32 stringsCount = WritePackageAttributes(PackageAttributes(),
-		stringsLengthUncompressed);
+		stringsLength);
 
-	zlibWriter.Finish();
-	off_t endOffset = realWriter.Offset();
-	SetDataWriter(NULL);
+	uint64 sectionSize = fHeapWriter->UncompressedHeapSize() - startOffset;
 
-	fListener->OnPackageAttributesSectionDone(stringsCount,
-		zlibWriter.BytesWritten());
-
-	_packagesLengthCompressed = endOffset - startOffset;
+	fListener->OnPackageAttributesSectionDone(stringsCount, sectionSize);
 
 	// update the header
-	header.packages_compression
-		= B_HOST_TO_BENDIAN_INT32(B_HPKG_COMPRESSION_ZLIB);
-	header.packages_length_compressed
-		= B_HOST_TO_BENDIAN_INT64(_packagesLengthCompressed);
-	header.packages_length_uncompressed
-		= B_HOST_TO_BENDIAN_INT64(zlibWriter.BytesWritten());
+	header.packages_length = B_HOST_TO_BENDIAN_INT64(sectionSize);
 	header.packages_strings_count = B_HOST_TO_BENDIAN_INT64(stringsCount);
-	header.packages_strings_length
-		= B_HOST_TO_BENDIAN_INT64(stringsLengthUncompressed);
+	header.packages_strings_length = B_HOST_TO_BENDIAN_INT64(stringsLength);
 
-	return endOffset;
+	_length = sectionSize;
 }
 
 

src/kits/package/hpkg/WriterImplBase.cpp

@@ -18,11 +18,12 @@
 
 #include <AutoDeleter.h>
 
-#include <package/hpkg/HPKGDefsPrivate.h>
-
 #include <package/hpkg/DataReader.h>
 #include <package/hpkg/ErrorOutput.h>
 
+#include <package/hpkg/HPKGDefsPrivate.h>
+#include <package/hpkg/PackageFileHeapWriter.h>
+
 
 namespace BPackageKit {
 
@@ -176,106 +177,9 @@ WriterImplBase::AttributeValue::_ApplicableIntEncoding(uint64 value)
 }
 
 
-// #pragma mark - AbstractDataWriter
-
-
-WriterImplBase::AbstractDataWriter::AbstractDataWriter()
-	:
-	fBytesWritten(0)
-{
-}
-
-
-WriterImplBase::AbstractDataWriter::~AbstractDataWriter()
-{
-}
-
-
-// #pragma mark - FDDataWriter
-
-
-WriterImplBase::FDDataWriter::FDDataWriter(int fd, off_t offset,
-	BErrorOutput* errorOutput)
-	:
-	fFD(fd),
-	fOffset(offset),
-	fErrorOutput(errorOutput)
-{
-}
-
-
-status_t
-WriterImplBase::FDDataWriter::WriteDataNoThrow(const void* buffer, size_t size)
-{
-	ssize_t bytesWritten = pwrite(fFD, buffer, size, fOffset);
-	if (bytesWritten < 0) {
-		fErrorOutput->PrintError(
-			"WriteDataNoThrow(%p, %lu) failed to write data: %s\n", buffer,
-			size, strerror(errno));
-		return errno;
-	}
-	if ((size_t)bytesWritten != size) {
-		fErrorOutput->PrintError(
-			"WriteDataNoThrow(%p, %lu) failed to write all data\n", buffer,
-			size);
-		return B_ERROR;
-	}
-
-	fOffset += size;
-	fBytesWritten += size;
-	return B_OK;
-}
-
-
-// #pragma mark - ZlibDataWriter
-
-
-WriterImplBase::ZlibDataWriter::ZlibDataWriter(AbstractDataWriter* dataWriter)
-	:
-	fDataWriter(dataWriter),
-	fCompressor(this)
-{
-}
-
-
-void
-WriterImplBase::ZlibDataWriter::Init()
-{
-	status_t error = fCompressor.Init();
-	if (error != B_OK)
-		throw status_t(error);
-}
-
-
-void
-WriterImplBase::ZlibDataWriter::Finish()
-{
-	status_t error = fCompressor.Finish();
-	if (error != B_OK)
-		throw status_t(error);
-}
-
-
-status_t
-WriterImplBase::ZlibDataWriter::WriteDataNoThrow(const void* buffer,
-	size_t size)
-{
-	status_t error = fCompressor.CompressNext(buffer, size);
-	if (error == B_OK)
-		fBytesWritten += size;
-	return error;
-}
-
-
-status_t
-WriterImplBase::ZlibDataWriter::WriteData(const void* buffer, size_t size)
-{
-	return fDataWriter->WriteDataNoThrow(buffer, size);
-}
-
-
 // #pragma mark - PackageAttribute
 
+
 WriterImplBase::PackageAttribute::PackageAttribute(BHPKGAttributeID id_,
 	uint8 type_, uint8 encoding_)
 	:
@@ -310,8 +214,10 @@ WriterImplBase::PackageAttribute::_DeleteChildren()
 // #pragma mark - WriterImplBase
 
 
-WriterImplBase::WriterImplBase(BErrorOutput* errorOutput)
+WriterImplBase::WriterImplBase(const char* fileType, BErrorOutput* errorOutput)
 	:
+	fHeapWriter(NULL),
+	fFileType(fileType),
 	fErrorOutput(errorOutput),
 	fFileName(NULL),
 	fFlags(0),
@@ -324,6 +230,8 @@ WriterImplBase::WriterImplBase(BErrorOutput* errorOutput)
 
 WriterImplBase::~WriterImplBase()
 {
+	delete fHeapWriter;
+
 	if (fFD >= 0)
 		close(fFD);
 
@@ -335,7 +243,7 @@ WriterImplBase::~WriterImplBase()
 
 
 status_t
-WriterImplBase::Init(const char* fileName, const char* type, uint32 flags)
+WriterImplBase::Init(const char* fileName, uint32 flags)
 {
 	if (fPackageStringCache.Init() != B_OK)
 		throw std::bad_alloc();
@@ -347,14 +255,20 @@ WriterImplBase::Init(const char* fileName, const char* type, uint32 flags)
 
 	fFD = open(fileName, openMode, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
 	if (fFD < 0) {
-		fErrorOutput->PrintError("Failed to open %s file \"%s\": %s\n", type,
-			fileName, strerror(errno));
+		fErrorOutput->PrintError("Failed to open %s file \"%s\": %s\n",
+			fFileType, fileName, strerror(errno));
 		return errno;
 	}
 
 	fFileName = fileName;
 	fFlags = flags;
 
+	// create heap writer
+	fHeapWriter = new PackageFileHeapWriter(fErrorOutput, FD(),
+		sizeof(hpkg_header));
+	fHeapWriter->Init();
+	fDataWriter = fHeapWriter->DataWriter();
+
 	return B_OK;
 }
 
@@ -670,8 +584,10 @@ WriterImplBase::WritePackageAttributes(
 	uint32& _stringsLengthUncompressed)
 {
 	// write the cached strings
+	uint64 startOffset = fHeapWriter->UncompressedHeapSize();
 	uint32 stringsCount = WriteCachedStrings(fPackageStringCache, 2);
-	_stringsLengthUncompressed = DataWriter()->BytesWritten();
+	_stringsLengthUncompressed
+		= fHeapWriter->UncompressedHeapSize() - startOffset;
 
 	_WritePackageAttributes(packageAttributes);
 
@@ -760,18 +676,18 @@ WriterImplBase::WriteUnsignedLEB128(uint64 value)
 
 
 void
-WriterImplBase::WriteBuffer(const void* buffer, size_t size, off_t offset)
+WriterImplBase::RawWriteBuffer(const void* buffer, size_t size, off_t offset)
 {
 	ssize_t bytesWritten = pwrite(fFD, buffer, size, offset);
 	if (bytesWritten < 0) {
 		fErrorOutput->PrintError(
-			"WriteBuffer(%p, %lu) failed to write data: %s\n", buffer, size,
+			"RawWriteBuffer(%p, %lu) failed to write data: %s\n", buffer, size,
 			strerror(errno));
 		throw status_t(errno);
 	}
 	if ((size_t)bytesWritten != size) {
 		fErrorOutput->PrintError(
-			"WriteBuffer(%p, %lu) failed to write all data\n", buffer, size);
+			"RawWriteBuffer(%p, %lu) failed to write all data\n", buffer, size);
 		throw status_t(B_ERROR);
 	}
 }

src/kits/package/hpkg/v1/PackageDataReaderV1.cpp

@@ -397,6 +397,23 @@ private:
 };
 
 
+// #pragma mark - PackageDataHeapReader
+
+
+class PackageDataInlineReader : public BBufferDataReader {
+public:
+	PackageDataInlineReader(const BPackageData& data)
+		:
+		BBufferDataReader(fData.InlineData(), data.UncompressedSize()),
+		fData(data)
+	{
+	}
+
+private:
+	BPackageData	fData;
+};
+
+
 // #pragma mark - BPackageDataReaderFactory
 
 
@@ -411,6 +428,16 @@ status_t
 BPackageDataReaderFactory::CreatePackageDataReader(BDataReader* dataReader,
 	const BPackageData& data, BAbstractBufferedDataReader*& _reader)
 {
+	if (data.IsEncodedInline()) {
+		BAbstractBufferedDataReader* reader
+			= new(std::nothrow) PackageDataInlineReader(data);
+		if (reader == NULL)
+			return B_NO_MEMORY;
+
+		_reader = reader;
+		return B_OK;
+	}
+
 	PackageDataReader* reader;
 
 	switch (data.Compression()) {

src/system/boot/loader/file_systems/packagefs/Jamfile

@@ -35,6 +35,8 @@ BootStaticLibrary boot_packagefs :
 	PackageDataReader.cpp
 	PackageEntry.cpp
 	PackageEntryAttribute.cpp
+	PackageFileHeapAccessorBase.cpp
+	PackageFileHeapReader.cpp
 	PackageReaderImpl.cpp
 	ReaderImplBase.cpp
 

src/system/boot/loader/file_systems/packagefs/packagefs.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2011, Ingo Weinhold, ingo_weinhold@gmx.de.
+ * Copyright 2011-2013, Ingo Weinhold, ingo_weinhold@gmx.de.
  * Distributed under the terms of the MIT License.
  */
 
@@ -9,12 +9,12 @@
 #include <errno.h>
 #include <unistd.h>
 
-#include <package/hpkg/BlockBufferPoolNoLock.h>
 #include <package/hpkg/DataReader.h>
 #include <package/hpkg/ErrorOutput.h>
 #include <package/hpkg/PackageDataReader.h>
 #include <package/hpkg/PackageEntry.h>
 #include <package/hpkg/PackageEntryAttribute.h>
+#include <package/hpkg/PackageFileHeapReader.h>
 #include <package/hpkg/PackageReaderImpl.h>
 
 #include <AutoDeleter.h>
@@ -41,6 +41,7 @@
 
 using namespace BPackageKit;
 using namespace BPackageKit::BHPKG;
+using BPackageKit::BHPKG::BPrivate::PackageFileHeapReader;
 using BPackageKit::BHPKG::BPrivate::PackageReaderImpl;
 
 
@@ -143,7 +144,7 @@ struct PackageFile : PackageNode {
 
 	off_t Size() const
 	{
-		return fData.UncompressedSize();
+		return fData.Size();
 	}
 
 private:
@@ -232,85 +233,60 @@ private:
 };
 
 
-// #pragma mark - FileDataReader
+// #pragma mark - PackageLoaderErrorOutput
 
 
-struct FileDataReader {
-	FileDataReader(int fd, const BPackageData& data)
-		:
-		fDataReader(fd),
-		fData(data),
-		fPackageDataReader(NULL)
+struct PackageLoaderErrorOutput : BErrorOutput {
+	PackageLoaderErrorOutput()
 	{
 	}
 
-	~FileDataReader()
+	virtual void PrintErrorVarArgs(const char* format, va_list args)
 	{
-		delete fPackageDataReader;
 	}
-
-	status_t Init(BPackageDataReaderFactory* dataReaderFactory)
-	{
-		if (fData.IsEncodedInline())
-			return B_OK;
-
-		return dataReaderFactory->CreatePackageDataReader(&fDataReader,
-			fData, fPackageDataReader);
-	}
-
-	status_t ReadData(off_t offset, void* buffer, size_t bufferSize)
-	{
-		if (fData.IsEncodedInline()) {
-			BBufferDataReader dataReader(fData.InlineData(),
-				fData.CompressedSize());
-			return dataReader.ReadData(offset, buffer, bufferSize);
-		}
-
-		return fPackageDataReader->ReadData(offset, buffer, bufferSize);
-	}
-
-private:
-	BFDDataReader		fDataReader;
-	BPackageData		fData;
-	BAbstractBufferedDataReader* fPackageDataReader;
 };
 
 
 // #pragma mark - PackageVolume
 
 
-struct PackageVolume : BReferenceable {
+struct PackageVolume : BReferenceable, private PackageLoaderErrorOutput {
 	PackageVolume()
 		:
 		fNextNodeID(1),
 		fRootDirectory(this, S_IFDIR),
-		fBufferCache(B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, 2),
-		fDataReaderFactory(&fBufferCache),
+		fHeapReader(NULL),
 		fFD(-1)
 	{
 	}
 
 	~PackageVolume()
 	{
+		delete fHeapReader;
+
 		if (fFD >= 0)
 			close(fFD);
 	}
 
-	status_t Init(int fd)
+	status_t Init(int fd, const PackageFileHeapReader* heapReader)
 	{
 		status_t error = fRootDirectory.Init(&fRootDirectory, ".",
 			NextNodeID());
 		if (error != B_OK)
 			return error;
 
-		error = fBufferCache.Init();
-		if (error != B_OK)
-			return error;
-
 		fFD = dup(fd);
 		if (fFD < 0)
 			return errno;
 
+		// clone a heap reader and adjust it for our use
+		fHeapReader = heapReader->Clone();
+		if (fHeapReader == NULL)
+			return B_NO_MEMORY;
+
+		fHeapReader->SetErrorOutput(this);
+		fHeapReader->SetFD(fFD);
+
 		return B_OK;
 	}
 
@@ -325,46 +301,20 @@ struct PackageVolume : BReferenceable {
 	}
 
 	status_t CreateFileDataReader(const BPackageData& data,
-		FileDataReader*& _fileDataReader)
+		BAbstractBufferedDataReader*& _reader)
 	{
-		// create the file reader object
-		FileDataReader* fileDataReader = new(std::nothrow) FileDataReader(fFD,
-			data);
-		if (fileDataReader == NULL)
-			return B_NO_MEMORY;
-		ObjectDeleter<FileDataReader> fileDataReaderDeleter(fileDataReader);
-
-		status_t error = fileDataReader->Init(&fDataReaderFactory);
-		if (error != B_OK)
-			return error;
-
-		_fileDataReader = fileDataReaderDeleter.Detach();
-		return B_OK;
+		return BPackageDataReaderFactory().CreatePackageDataReader(fHeapReader,
+			data, _reader);
 	}
 
 private:
 	ino_t						fNextNodeID;
 	PackageDirectory			fRootDirectory;
-	BBlockBufferPoolNoLock		fBufferCache;
-	BPackageDataReaderFactory	fDataReaderFactory;
+	PackageFileHeapReader*		fHeapReader;
 	int							fFD;
 };
 
 
-// #pragma mark - PackageLoaderErrorOutput
-
-
-struct PackageLoaderErrorOutput : BErrorOutput {
-	PackageLoaderErrorOutput()
-	{
-	}
-
-	virtual void PrintErrorVarArgs(const char* format, va_list args)
-	{
-	}
-};
-
-
 // #pragma mark - PackageLoaderContentHandler
 
 
@@ -503,7 +453,8 @@ struct File : ::Node {
 			bufferSize = size - pos;
 
 		if (bufferSize > 0) {
-			FileDataReader* dataReader = (FileDataReader*)cookie;
+			BAbstractBufferedDataReader* dataReader
+				= (BAbstractBufferedDataReader*)cookie;
 			status_t error = dataReader->ReadData(pos, buffer, bufferSize);
 			if (error != B_OK)
 				return error;
@@ -529,7 +480,7 @@ struct File : ::Node {
 		if ((mode & O_ACCMODE) != O_RDONLY && (mode & O_ACCMODE) != O_RDWR)
 			return B_NOT_ALLOWED;
 
-		FileDataReader* dataReader;
+		BAbstractBufferedDataReader* dataReader;
 		status_t error = fFile->Volume()->CreateFileDataReader(fFile->Data(),
 			dataReader);
 		if (error != B_OK)
@@ -542,7 +493,8 @@ struct File : ::Node {
 
 	virtual status_t Close(void* cookie)
 	{
-		FileDataReader* dataReader = (FileDataReader*)cookie;
+		BAbstractBufferedDataReader* dataReader
+			= (BAbstractBufferedDataReader*)cookie;
 		delete dataReader;
 		Release();
 		return B_OK;
@@ -807,7 +759,7 @@ packagefs_mount_file(int fd, ::Directory*& _mountedDirectory)
 {
 	PackageLoaderErrorOutput errorOutput;
  	PackageReaderImpl packageReader(&errorOutput);
- 	status_t error = packageReader.Init(fd, false);
+	status_t error = packageReader.Init(fd, false);
  	if (error != B_OK)
  		RETURN_ERROR(error);
 
@@ -817,7 +769,7 @@ packagefs_mount_file(int fd, ::Directory*& _mountedDirectory)
 		return B_NO_MEMORY;
 	BReference<PackageVolume> volumeReference(volume, true);
 
-	error = volume->Init(fd);
+	error = volume->Init(fd, packageReader.HeapReader());
 	if (error != B_OK)
 		RETURN_ERROR(error);