* Added new interface BufferCache -- basically a memory allocator with the

option to cache a freed buffer -- and implementations
  BlockBufferCache{NoLock,Kernel}.
* ZlibPackageDataReader does now dynamically get its read and uncompress
  buffers from a provided BufferCache when needed.
* Allocating the buffers once and keeping them over the whole life time was a
  bit too memory heavy, since we create a reader for every file for which a
  vnode is created. A FS module global factory provides a buffer cache.
* Added a mutex to PackageFile::DataAccessor which guards the access to the
  data reader which isn't thread safe.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@34115 a95241bf-73f2-0310-859d-f6bbb57e9c96
This commit is contained in:
Ingo Weinhold 2009-11-18 15:26:11 +00:00
parent 47212dddf5
commit 4e2cd77109
15 changed files with 682 additions and 39 deletions

View File

@ -0,0 +1,37 @@
/*
* Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#include "BlockBufferCacheKernel.h"
BlockBufferCacheKernel::BlockBufferCacheKernel(size_t blockSize,
uint32 maxCachedBlocks)
:
BlockBufferCache(blockSize, maxCachedBlocks)
{
mutex_init(&fLock, "BlockBufferCache");
}
BlockBufferCacheKernel::~BlockBufferCacheKernel()
{
mutex_destroy(&fLock);
}
bool
BlockBufferCacheKernel::Lock()
{
mutex_lock(&fLock);
return true;
}
void
BlockBufferCacheKernel::Unlock()
{
mutex_unlock(&fLock);
}

View File

@ -0,0 +1,28 @@
/*
* Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#ifndef BLOCK_BUFFER_CACHE_KERNEL_H
#define BLOCK_BUFFER_CACHE_KERNEL_H
#include <util/AutoLock.h>
#include "BlockBufferCache.h"
class BlockBufferCacheKernel : public BlockBufferCache {
public:
BlockBufferCacheKernel(size_t blockSize,
uint32 maxCachedBlocks);
virtual ~BlockBufferCacheKernel();
virtual bool Lock();
virtual void Unlock();
private:
mutex fLock;
};
#endif // BLOCK_BUFFER_CACHE_KERNEL_H

View File

@ -0,0 +1,85 @@
/*
* Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#include "GlobalFactory.h"
#include <new>
#include <haiku_package.h>
static const uint32 kMaxCachedBuffers = 32;
/*static*/ GlobalFactory* GlobalFactory::sDefaultInstance = NULL;
GlobalFactory::GlobalFactory()
:
fBufferCache(B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, kMaxCachedBuffers),
fPackageDataReaderFactory(&fBufferCache)
{
}
GlobalFactory::~GlobalFactory()
{
}
/*static*/ status_t
GlobalFactory::CreateDefault()
{
if (sDefaultInstance != NULL)
return B_OK;
GlobalFactory* factory = new(std::nothrow) GlobalFactory;
if (factory == NULL)
return B_NO_MEMORY;
status_t error = factory->_Init();
if (error != B_OK) {
delete factory;
return error;
}
sDefaultInstance = factory;
return B_OK;
}
/*static*/ void
GlobalFactory::DeleteDefault()
{
delete sDefaultInstance;
sDefaultInstance = NULL;
}
/*static*/ GlobalFactory*
GlobalFactory::Default()
{
return sDefaultInstance;
}
status_t
GlobalFactory::CreatePackageDataReader(DataReader* dataReader,
const PackageData& data, PackageDataReader*& _reader)
{
return fPackageDataReaderFactory.CreatePackageDataReader(dataReader, data,
_reader);
}
status_t
GlobalFactory::_Init()
{
status_t error = fBufferCache.Init();
if (error != B_OK)
return error;
return B_OK;
}

View File

@ -0,0 +1,37 @@
/*
* Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#ifndef GLOBAL_FACTORY_H
#define GLOBAL_FACTORY_H
#include "BlockBufferCacheKernel.h"
#include "PackageDataReader.h"
class GlobalFactory {
private:
GlobalFactory();
~GlobalFactory();
public:
static status_t CreateDefault();
static void DeleteDefault();
static GlobalFactory* Default();
status_t CreatePackageDataReader(DataReader* dataReader,
const PackageData& data,
PackageDataReader*& _reader);
private:
status_t _Init();
private:
static GlobalFactory* sDefaultInstance;
BlockBufferCacheKernel fBufferCache;
PackageDataReaderFactory fPackageDataReaderFactory;
};
#endif // GLOBAL_FACTORY_H

View File

@ -10,8 +10,10 @@ DEFINES += B_ENABLE_INCOMPLETE_POSIX_AT_SUPPORT ;
HAIKU_PACKAGE_FS_SOURCES =
BlockBufferCacheKernel.cpp
DebugSupport.cpp
Directory.cpp
GlobalFactory.cpp
kernel_interface.cpp
LeafNode.cpp
Node.cpp
@ -26,6 +28,8 @@ HAIKU_PACKAGE_FS_SOURCES =
;
HAIKU_PACKAGE_FS_PACKAGE_READER_SOURCES =
BlockBufferCache.cpp
BufferCache.cpp
DataOutput.cpp
DataReader.cpp
ErrorOutput.cpp

View File

@ -15,6 +15,7 @@
#include "PackageDataReader.h"
#include "DebugSupport.h"
#include "GlobalFactory.h"
#include "Package.h"
@ -29,6 +30,7 @@ struct PackageFile::DataAccessor {
fReader(NULL),
fFileCache(NULL)
{
mutex_init(&fLock, "file data accessor");
}
~DataAccessor()
@ -36,6 +38,7 @@ struct PackageFile::DataAccessor {
file_cache_delete(fFileCache);
delete fReader;
delete fDataReader;
mutex_destroy(&fLock);
}
status_t Init(dev_t deviceID, ino_t nodeID, int fd)
@ -51,7 +54,7 @@ struct PackageFile::DataAccessor {
RETURN_ERROR(B_NO_MEMORY);
// create a PackageDataReader
status_t error = PackageDataReaderFactory::CreatePackageDataReader(
status_t error = GlobalFactory::Default()->CreatePackageDataReader(
fDataReader, *fData, fReader);
if (error != B_OK)
RETURN_ERROR(error);
@ -74,6 +77,7 @@ struct PackageFile::DataAccessor {
fData->UncompressedSize() - offset);
if (toRead > 0) {
MutexLocker locker(fLock);
status_t error = fReader->ReadData(offset, buffer, toRead);
if (error != B_OK)
RETURN_ERROR(error);
@ -84,6 +88,7 @@ struct PackageFile::DataAccessor {
}
private:
mutex fLock;
PackageData* fData;
DataReader* fDataReader;
PackageDataReader* fReader;

View File

@ -19,6 +19,7 @@
#include "DebugSupport.h"
#include "Directory.h"
#include "GlobalFactory.h"
#include "LeafNode.h"
#include "Volume.h"
@ -651,14 +652,27 @@ packagefs_std_ops(int32 op, ...)
{
switch (op) {
case B_MODULE_INIT:
{
init_debugging();
PRINT("package_std_ops(): B_MODULE_INIT\n");
status_t error = GlobalFactory::CreateDefault();
if (error != B_OK) {
ERROR("Failed to init GlobalFactory\n");
exit_debugging();
return error;
}
return B_OK;
}
case B_MODULE_UNINIT:
{
PRINT("package_std_ops(): B_MODULE_UNINIT\n");
GlobalFactory::DeleteDefault();
exit_debugging();
return B_OK;
}
default:
return B_ERROR;

View File

@ -0,0 +1,208 @@
/*
* Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#include "BlockBufferCache.h"
#include <algorithm>
#include <new>
#include <AutoLocker.h>
// #pragma mark - BlockBufferCache
BlockBufferCache::BlockBufferCache(size_t blockSize, uint32 maxCachedBlocks)
:
fBlockSize(blockSize),
fMaxCachedBlocks(maxCachedBlocks),
fAllocatedBlocks(0)
{
}
BlockBufferCache::~BlockBufferCache()
{
// delete all cached blocks
while (CachedBuffer* block = fCachedBuffers.RemoveHead())
delete block;
while (CachedBuffer* block = fUnusedBuffers.RemoveHead())
delete block;
}
status_t
BlockBufferCache::Init()
{
return B_OK;
}
CachedBuffer*
BlockBufferCache::GetBuffer(size_t size, CachedBuffer** owner, bool* _newBuffer)
{
// for sizes greater than the block size, we always allocate a new buffer
if (size > fBlockSize)
return _AllocateBuffer(size, owner, _newBuffer);
AutoLocker<BlockBufferCache> locker(this);
// if an owner is given and the buffer is still cached, return it
if (owner != NULL && *owner != NULL) {
CachedBuffer* buffer = *owner;
fCachedBuffers.Remove(buffer);
if (_newBuffer != NULL)
*_newBuffer = false;
return buffer;
}
// we need a new buffer -- try unused ones first
CachedBuffer* buffer = fUnusedBuffers.RemoveHead();
if (buffer != NULL) {
buffer->SetOwner(owner);
if (owner != NULL)
*owner = buffer;
if (_newBuffer != NULL)
*_newBuffer = true;
return buffer;
}
// if we have already hit the max block limit, steal a cached block
if (fAllocatedBlocks >= fMaxCachedBlocks) {
buffer = fCachedBuffers.RemoveHead();
if (buffer != NULL) {
buffer->SetCached(false);
*buffer->Owner() = NULL;
buffer->SetOwner(owner);
if (owner != NULL)
*owner = buffer;
if (_newBuffer != NULL)
*_newBuffer = true;
return buffer;
}
}
// allocate a new buffer
locker.Unlock();
return _AllocateBuffer(size, owner, _newBuffer);
}
void
BlockBufferCache::PutBufferAndCache(CachedBuffer** owner)
{
CachedBuffer* buffer = *owner;
// always delete buffers with non-standard size
if (buffer->Size() != fBlockSize) {
*owner = NULL;
delete buffer;
return;
}
AutoLocker<BlockBufferCache> locker(this);
// queue the cached buffer
buffer->SetOwner(owner);
fCachedBuffers.Add(buffer);
buffer->SetCached(true);
if (fAllocatedBlocks > fMaxCachedBlocks) {
// We have exceeded the limit -- we need to free a buffer.
CachedBuffer* otherBuffer = fUnusedBuffers.RemoveHead();
if (otherBuffer == NULL) {
otherBuffer = fCachedBuffers.RemoveHead();
*otherBuffer->Owner() = NULL;
otherBuffer->SetCached(false);
}
delete otherBuffer;
}
}
void
BlockBufferCache::PutBuffer(CachedBuffer** owner)
{
AutoLocker<BlockBufferCache> locker(this);
CachedBuffer* buffer = *owner;
if (buffer == NULL)
return;
if (buffer->IsCached()) {
fCachedBuffers.Remove(buffer);
buffer->SetCached(false);
}
buffer->SetOwner(NULL);
*owner = NULL;
if (buffer->Size() == fBlockSize && fAllocatedBlocks < fMaxCachedBlocks)
fUnusedBuffers.Add(buffer);
else
delete buffer;
}
CachedBuffer*
BlockBufferCache::_AllocateBuffer(size_t size, CachedBuffer** owner,
bool* _newBuffer)
{
CachedBuffer* buffer = new(std::nothrow) CachedBuffer(
std::max(size, fBlockSize));
if (buffer == NULL || buffer->Buffer() == NULL) {
delete buffer;
return NULL;
}
buffer->SetOwner(owner);
if (_newBuffer != NULL)
*_newBuffer = true;
AutoLocker<BlockBufferCache> locker(this);
fAllocatedBlocks++;
if (owner != NULL)
*owner = buffer;
return buffer;
}
// #pragma mark - BlockBufferCacheNoLock
BlockBufferCacheNoLock::BlockBufferCacheNoLock(size_t blockSize,
uint32 maxCachedBlocks)
:
BlockBufferCache(blockSize, maxCachedBlocks)
{
}
BlockBufferCacheNoLock::~BlockBufferCacheNoLock()
{
}
bool
BlockBufferCacheNoLock::Lock()
{
return true;
}
void
BlockBufferCacheNoLock::Unlock()
{
}

View File

@ -0,0 +1,57 @@
/*
* Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#ifndef BLOCK_BUFFER_CACHE_H
#define BLOCK_BUFFER_CACHE_H
#include "BufferCache.h"
class BlockBufferCache : public BufferCache {
public:
BlockBufferCache(size_t blockSize,
uint32 maxCachedBlocks);
virtual ~BlockBufferCache();
virtual status_t Init();
virtual CachedBuffer* GetBuffer(size_t size,
CachedBuffer** owner = NULL,
bool* _newBuffer = NULL);
virtual void PutBufferAndCache(CachedBuffer** owner);
virtual void PutBuffer(CachedBuffer** owner);
virtual bool Lock() = 0;
virtual void Unlock() = 0;
private:
typedef DoublyLinkedList<CachedBuffer> BufferList;
private:
CachedBuffer* _AllocateBuffer(size_t size,
CachedBuffer** owner, bool* _newBuffer);
// object must not be locked
private:
size_t fBlockSize;
uint32 fMaxCachedBlocks;
uint32 fAllocatedBlocks;
BufferList fUnusedBuffers;
BufferList fCachedBuffers;
};
class BlockBufferCacheNoLock : public BlockBufferCache {
public:
BlockBufferCacheNoLock(size_t blockSize,
uint32 maxCachedBlocks);
virtual ~BlockBufferCacheNoLock();
virtual bool Lock();
virtual void Unlock();
};
#endif // BLOCK_BUFFER_CACHE_H

View File

@ -0,0 +1,36 @@
/*
* Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#include "BufferCache.h"
#include <stdlib.h>
// #pragma mark - CachedBuffer
CachedBuffer::CachedBuffer(size_t size)
:
fOwner(NULL),
fBuffer(malloc(size)),
fSize(size),
fCached(false)
{
}
CachedBuffer::~CachedBuffer()
{
free(fBuffer);
}
// #pragma mark - BufferCache
BufferCache::~BufferCache()
{
}

View File

@ -0,0 +1,91 @@
/*
* Copyright 2009, Ingo Weinhold, ingo_weinhold@gmx.de.
* Distributed under the terms of the MIT License.
*/
#ifndef BUFFER_CACHE_H
#define BUFFER_CACHE_H
#include <stddef.h>
#include <util/DoublyLinkedList.h>
class CachedBuffer : public DoublyLinkedListLinkImpl<CachedBuffer> {
public:
CachedBuffer(size_t size);
~CachedBuffer();
void* Buffer() const { return fBuffer; }
size_t Size() const { return fSize; }
// implementation private
CachedBuffer** Owner() const { return fOwner; }
void SetOwner(CachedBuffer** owner)
{ fOwner = owner; }
void SetCached(bool cached) { fCached = cached; }
bool IsCached() const { return fCached; }
private:
CachedBuffer** fOwner;
void* fBuffer;
size_t fSize;
bool fCached;
};
class BufferCache {
public:
virtual ~BufferCache();
virtual CachedBuffer* GetBuffer(size_t size,
CachedBuffer** owner = NULL,
bool* _newBuffer = NULL) = 0;
virtual void PutBufferAndCache(CachedBuffer** owner) = 0;
// caller is buffer owner and wants the
// buffer cached, if possible
virtual void PutBuffer(CachedBuffer** owner) = 0;
// puts the buffer for good, owner might
// have called PutBufferAndCache() before
// and might not own a buffer anymore
};
class CachedBufferPutter {
public:
CachedBufferPutter(BufferCache* cache, CachedBuffer** owner)
:
fCache(cache),
fOwner(owner),
fBuffer(NULL)
{
}
CachedBufferPutter(BufferCache* cache, CachedBuffer* buffer)
:
fCache(cache),
fOwner(NULL),
fBuffer(buffer)
{
}
~CachedBufferPutter()
{
if (fCache != NULL) {
if (fOwner != NULL)
fCache->PutBufferAndCache(fOwner);
else if (fBuffer != NULL)
fCache->PutBuffer(&fBuffer);
}
}
private:
BufferCache* fCache;
CachedBuffer** fOwner;
CachedBuffer* fBuffer;
};
#endif // BUFFER_CACHE_H

View File

@ -10,6 +10,8 @@ DEFINES += B_ENABLE_INCOMPLETE_POSIX_AT_SUPPORT ;
SEARCH_SOURCE += [ FDirName $(SUBDIR) compression ] ;
BinCommand package :
BlockBufferCache.cpp
BufferCache.cpp
command_create.cpp
command_dump.cpp
command_extract.cpp

View File

@ -13,6 +13,7 @@
#include <haiku_package.h>
#include "BufferCache.h"
#include "PackageData.h"
#include "ZlibDecompressor.h"
@ -94,18 +95,20 @@ private:
class ZlibPackageDataReader : public PackageDataReader {
public:
ZlibPackageDataReader(DataReader* dataReader)
ZlibPackageDataReader(DataReader* dataReader, BufferCache* bufferCache)
:
PackageDataReader(dataReader),
fOffsetTable(NULL),
fReadBuffer(NULL)
fBufferCache(bufferCache),
fUncompressBuffer(NULL),
fOffsetTable(NULL)
{
}
~ZlibPackageDataReader()
{
delete[] fOffsetTable;
free(fReadBuffer);
fBufferCache->PutBuffer(&fUncompressBuffer);
}
status_t Init(const PackageData& data)
@ -142,14 +145,8 @@ public:
} else
fChunkSize = fUncompressedSize;
// allocate the read/uncompress buffer
fReadBuffer = (uint8*)malloc(fChunkSize * 2);
if (fReadBuffer == NULL)
return B_NO_MEMORY;
fUncompressBuffer = fReadBuffer + fChunkSize;
// mark uncompressed content invalid
fUncompressedChunk = -1;
// mark content invalid
return B_OK;
}
@ -180,6 +177,17 @@ public:
return B_BAD_VALUE;
}
// get our uncompressed chunk buffer back, if possible
bool newBuffer;
if (fBufferCache->GetBuffer(fChunkSize, &fUncompressBuffer, &newBuffer)
== NULL) {
return B_NO_MEMORY;
}
CachedBufferPutter uncompressBuffer(fBufferCache, &fUncompressBuffer);
if (newBuffer)
fUncompressedChunk = -1;
// uncompress
int64 chunkIndex = offset / fChunkSize;
off_t chunkOffset = chunkIndex * fChunkSize;
@ -193,7 +201,8 @@ public:
// copy data to buffer
size_t toCopy = std::min(size, (size_t)fChunkSize - inChunkOffset);
memcpy(buffer, fUncompressBuffer + inChunkOffset, toCopy);
memcpy(buffer, (uint8*)fUncompressBuffer->Buffer() + inChunkOffset,
toCopy);
buffer += toCopy;
size -= toCopy;
@ -227,17 +236,24 @@ private:
if (compressedSize == uncompressedSize) {
// the chunk is not compressed -- read it directly into the
// uncompressed buffer
error = fDataReader->ReadData(offset, fUncompressBuffer,
error = fDataReader->ReadData(offset, fUncompressBuffer->Buffer(),
compressedSize);
} else {
// read to the read buffer and uncompress
error = fDataReader->ReadData(offset, fReadBuffer, compressedSize);
// read to a read buffer and uncompress
CachedBuffer* readBuffer = fBufferCache->GetBuffer(fChunkSize);
if (readBuffer == NULL)
return B_NO_MEMORY;
CachedBufferPutter readBufferPutter(fBufferCache, readBuffer);
error = fDataReader->ReadData(offset, readBuffer->Buffer(),
compressedSize);
if (error != B_OK)
return error;
size_t actuallyUncompressedSize;
error = ZlibDecompressor::DecompressSingleBuffer(fReadBuffer,
compressedSize, fUncompressBuffer, uncompressedSize,
error = ZlibDecompressor::DecompressSingleBuffer(
readBuffer->Buffer(), compressedSize,
fUncompressBuffer->Buffer(), uncompressedSize,
actuallyUncompressedSize);
if (error == B_OK && actuallyUncompressedSize != uncompressedSize)
error = B_BAD_DATA;
@ -321,25 +337,33 @@ private:
}
private:
uint64 fOffset;
uint64 fUncompressedSize;
uint64 fCompressedSize;
uint64 fOffsetTableSize;
uint64 fChunkCount;
uint32 fChunkSize;
uint32 fOffsetTableBufferEntryCount;
uint64* fOffsetTable;
int32 fOffsetTableIndex;
uint8* fReadBuffer;
uint8* fUncompressBuffer;
int64 fUncompressedChunk;
BufferCache* fBufferCache;
CachedBuffer* fUncompressBuffer;
int64 fUncompressedChunk;
uint64 fOffset;
uint64 fUncompressedSize;
uint64 fCompressedSize;
uint64 fOffsetTableSize;
uint64 fChunkCount;
uint32 fChunkSize;
uint32 fOffsetTableBufferEntryCount;
uint64* fOffsetTable;
int32 fOffsetTableIndex;
};
// #pragma mark - PackageDataReaderFactory
/*static*/ status_t
PackageDataReaderFactory::PackageDataReaderFactory(BufferCache* bufferCache)
:
fBufferCache(bufferCache)
{
}
status_t
PackageDataReaderFactory::CreatePackageDataReader(DataReader* dataReader,
const PackageData& data, PackageDataReader*& _reader)
{
@ -351,7 +375,8 @@ PackageDataReaderFactory::CreatePackageDataReader(DataReader* dataReader,
dataReader);
break;
case B_HPKG_COMPRESSION_ZLIB:
reader = new(std::nothrow) ZlibPackageDataReader(dataReader);
reader = new(std::nothrow) ZlibPackageDataReader(dataReader,
fBufferCache);
break;
default:
return B_BAD_VALUE;

View File

@ -9,6 +9,7 @@
#include "DataReader.h"
class BufferCache;
class PackageData;
@ -29,9 +30,15 @@ protected:
class PackageDataReaderFactory {
public:
static status_t CreatePackageDataReader(DataReader* dataReader,
PackageDataReaderFactory(
BufferCache* bufferCache);
status_t CreatePackageDataReader(DataReader* dataReader,
const PackageData& data,
PackageDataReader*& _reader);
private:
BufferCache* fBufferCache;
};

View File

@ -23,6 +23,7 @@
#include <AutoDeleter.h>
#include "BlockBufferCache.h"
#include "FDCloser.h"
#include "package.h"
#include "PackageDataReader.h"
@ -35,6 +36,7 @@
struct PackageContentExtractHandler : PackageContentHandler {
PackageContentExtractHandler(int packageFileFD)
:
fBufferCache(B_HPKG_DEFAULT_DATA_CHUNK_SIZE_ZLIB, 2),
fPackageFileReader(packageFileFD),
fDataBuffer(NULL),
fDataBufferSize(0),
@ -49,6 +51,10 @@ struct PackageContentExtractHandler : PackageContentHandler {
status_t Init()
{
status_t error = fBufferCache.Init();
if (error != B_OK)
return error;
fDataBufferSize = 64 * 1024;
fDataBuffer = malloc(fDataBufferSize);
if (fDataBuffer == NULL)
@ -245,8 +251,8 @@ private:
{
// create a PackageDataReader
PackageDataReader* reader;
status_t error = PackageDataReaderFactory::CreatePackageDataReader(
dataReader, data, reader);
status_t error = PackageDataReaderFactory(&fBufferCache)
.CreatePackageDataReader(dataReader, data, reader);
if (error != B_OK)
return error;
ObjectDeleter<PackageDataReader> readerDeleter(reader);
@ -284,10 +290,11 @@ private:
}
private:
FDDataReader fPackageFileReader;
void* fDataBuffer;
size_t fDataBufferSize;
bool fErrorOccurred;
BlockBufferCacheNoLock fBufferCache;
FDDataReader fPackageFileReader;
void* fDataBuffer;
size_t fDataBufferSize;
bool fErrorOccurred;
};