mirrors/postgres
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor the pg_dump zlib code from pg_backup_custom.c to a separate file,

Browse Source 
to make it easier to reuse that code. There is no user-visible changes.

This is in preparation for the patch to add a new archive format, a directory,
to perform a custom-like dump but with each table being dumped to a separate
file (that in turn is a prerequisite for parallel pg_dump). This also makes it
easier to add new compression methods in the future, and makes the
pg_backup_custom.c code easier to read, when the compression-related code is
factored out.

Joachim Wieland, with heavy editorialization by me.
This commit is contained in:
Heikki Linnakangas 2010-12-02 21:39:03 +02:00
parent 225f0aa3df
commit bf9aa490db
5 changed files with 552 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/bin/pg_dump/Makefile
View File

@ -20,7 +20,7 @@ override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
OBJS= pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o \ OBJS= pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o \
pg_backup_files.o pg_backup_null.o pg_backup_tar.o \ pg_backup_files.o pg_backup_null.o pg_backup_tar.o \
dumputils.o $(WIN32RES) dumputils.o compress_io.o $(WIN32RES)
KEYWRDOBJS = keywords.o kwlookup.o KEYWRDOBJS = keywords.o kwlookup.o

403
src/bin/pg_dump/compress_io.c Normal file
View File

@ -0,0 +1,403 @@
/*-------------------------------------------------------------------------
*
* compress_io.c
* Routines for archivers to write an uncompressed or compressed data
* stream.
*
* Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* The interface for writing to an archive consists of three functions:
* AllocateCompressor, WriteDataToArchive and EndCompressor. First you call
* AllocateCompressor, then write all the data by calling WriteDataToArchive
* as many times as needed, and finally EndCompressor. WriteDataToArchive
* and EndCompressor will call the WriteFunc that was provided to
* AllocateCompressor for each chunk of compressed data.
*
* The interface for reading an archive consists of just one function:
* ReadDataFromArchive. ReadDataFromArchive reads the whole compressed input
* stream, by repeatedly calling the given ReadFunc. ReadFunc returns the
* compressed data chunk at a time, and ReadDataFromArchive decompresses it
* and passes the decompressed data to ahwrite(), until ReadFunc returns 0
* to signal EOF.
*
* The interface is the same for compressed and uncompressed streams.
*
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.c
*
*-------------------------------------------------------------------------
*/
#include "compress_io.h"
static const char *modulename = gettext_noop("compress_io");
static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
int *level);
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs,
bool flush);
static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF);
static size_t WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
static void EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs);
#endif
/* Routines that support uncompressed data I/O */
static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static size_t WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
/*
* Interprets a numeric 'compression' value. The algorithm implied by the
* value (zlib or none at the moment), is returned in *alg, and the
* zlib compression level in *level.
*/
static void
ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
{
if (compression == Z_DEFAULT_COMPRESSION ||
(compression > 0 && compression <= 9))
*alg = COMPR_ALG_LIBZ;
else if (compression == 0)
*alg = COMPR_ALG_NONE;
else
die_horribly(NULL, modulename, "Invalid compression code: %d\n",
compression);
/* The level is just the passed-in value. */
if (level)
*level = compression;
}
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
AllocateCompressor(int compression, WriteFunc writeF)
{
CompressorState *cs;
CompressionAlgorithm alg;
int level;
ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
if (alg == COMPR_ALG_LIBZ)
die_horribly(NULL, modulename, "not built with zlib support\n");
#endif
cs = (CompressorState *) calloc(1, sizeof(CompressorState));
if (cs == NULL)
die_horribly(NULL, modulename, "out of memory\n");
cs->writeF = writeF;
cs->comprAlg = alg;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
if (alg == COMPR_ALG_LIBZ)
InitCompressorZlib(cs, level);
#endif
return cs;
}
/*
* Read all compressed data from the input stream (via readF) and print it
* out with ahwrite().
*/
void
ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
{
CompressionAlgorithm alg;
ParseCompressionOption(compression, &alg, NULL);
if (alg == COMPR_ALG_NONE)
ReadDataFromArchiveNone(AH, readF);
if (alg == COMPR_ALG_LIBZ)
{
#ifdef HAVE_LIBZ
ReadDataFromArchiveZlib(AH, readF);
#else
die_horribly(NULL, modulename, "not built with zlib support\n");
#endif
}
}
/*
* Compress and write data to the output stream (via writeF).
*/
size_t
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
switch(cs->comprAlg)
{
case COMPR_ALG_LIBZ:
#ifdef HAVE_LIBZ
return WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
die_horribly(NULL, modulename, "not built with zlib support\n");
#endif
case COMPR_ALG_NONE:
return WriteDataToArchiveNone(AH, cs, data, dLen);
}
return 0; /* keep compiler quiet */
}
/*
* Terminate compression library context and flush its buffers.
*/
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
#ifdef HAVE_LIBZ
if (cs->comprAlg == COMPR_ALG_LIBZ)
EndCompressorZlib(AH, cs);
#endif
free(cs);
}
/* Private routines, specific to each compression method. */
#ifdef HAVE_LIBZ
/*
* Functions for zlib compressed output.
*/
static void
InitCompressorZlib(CompressorState *cs, int level)
{
z_streamp zp;
zp = cs->zp = (z_streamp) malloc(sizeof(z_stream));
if (cs->zp == NULL)
die_horribly(NULL, modulename, "out of memory\n");
zp->zalloc = Z_NULL;
zp->zfree = Z_NULL;
zp->opaque = Z_NULL;
/*
* zlibOutSize is the buffer size we tell zlib it can output
* to. We actually allocate one extra byte because some routines
* want to append a trailing zero byte to the zlib output.
*/
cs->zlibOut = (char *) malloc(ZLIB_OUT_SIZE + 1);
cs->zlibOutSize = ZLIB_OUT_SIZE;
if (cs->zlibOut == NULL)
die_horribly(NULL, modulename, "out of memory\n");
if (deflateInit(zp, level) != Z_OK)
die_horribly(NULL, modulename,
"could not initialize compression library: %s\n",
zp->msg);
/* Just be paranoid - maybe End is called after Start, with no Write */
zp->next_out = (void *) cs->zlibOut;
zp->avail_out = cs->zlibOutSize;
}
static void
EndCompressorZlib(ArchiveHandle *AH, CompressorState *cs)
{
z_streamp zp = cs->zp;
zp->next_in = NULL;
zp->avail_in = 0;
/* Flush any remaining data from zlib buffer */
DeflateCompressorZlib(AH, cs, true);
if (deflateEnd(zp) != Z_OK)
die_horribly(AH, modulename,
"could not close compression stream: %s\n", zp->msg);
free(cs->zlibOut);
free(cs->zp);
}
static void
DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush)
{
z_streamp zp = cs->zp;
char *out = cs->zlibOut;
int res = Z_OK;
while (cs->zp->avail_in != 0 || flush)
{
res = deflate(zp, flush ? Z_FINISH : Z_NO_FLUSH);
if (res == Z_STREAM_ERROR)
die_horribly(AH, modulename,
"could not compress data: %s\n", zp->msg);
if ((flush && (zp->avail_out < cs->zlibOutSize))
|| (zp->avail_out == 0)
|| (zp->avail_in != 0)
)
{
/*
* Extra paranoia: avoid zero-length chunks, since a zero length
* chunk is the EOF marker in the custom format. This should never
* happen but...
*/
if (zp->avail_out < cs->zlibOutSize)
{
/*
* Any write function shoud do its own error checking but
* to make sure we do a check here as well...
*/
size_t len = cs->zlibOutSize - zp->avail_out;
if (cs->writeF(AH, out, len) != len)
die_horribly(AH, modulename,
"could not write to output file: %s\n",
strerror(errno));
}
zp->next_out = (void *) out;
zp->avail_out = cs->zlibOutSize;
}
if (res == Z_STREAM_END)
break;
}
}
static size_t
WriteDataToArchiveZlib(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen)
{
cs->zp->next_in = (void *) data;
cs->zp->avail_in = dLen;
DeflateCompressorZlib(AH, cs, false);
/* we have either succeeded in writing dLen bytes or we have called
* die_horribly() */
return dLen;
}
static void
ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF)
{
z_streamp zp;
char *out;
int res = Z_OK;
size_t cnt;
char *buf;
size_t buflen;
zp = (z_streamp) malloc(sizeof(z_stream));
if (zp == NULL)
die_horribly(NULL, modulename, "out of memory\n");
zp->zalloc = Z_NULL;
zp->zfree = Z_NULL;
zp->opaque = Z_NULL;
buf = malloc(ZLIB_IN_SIZE);
if (buf == NULL)
die_horribly(NULL, modulename, "out of memory\n");
buflen = ZLIB_IN_SIZE;
out = malloc(ZLIB_OUT_SIZE + 1);
if (out == NULL)
die_horribly(NULL, modulename, "out of memory\n");
if (inflateInit(zp) != Z_OK)
die_horribly(NULL, modulename,
"could not initialize compression library: %s\n",
zp->msg);
/* no minimal chunk size for zlib */
while ((cnt = readF(AH, &buf, &buflen)))
{
zp->next_in = (void *) buf;
zp->avail_in = cnt;
while (zp->avail_in > 0)
{
zp->next_out = (void *) out;
zp->avail_out = ZLIB_OUT_SIZE;
res = inflate(zp, 0);
if (res != Z_OK && res != Z_STREAM_END)
die_horribly(AH, modulename,
"could not uncompress data: %s\n", zp->msg);
out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
}
}
zp->next_in = NULL;
zp->avail_in = 0;
while (res != Z_STREAM_END)
{
zp->next_out = (void *) out;
zp->avail_out = ZLIB_OUT_SIZE;
res = inflate(zp, 0);
if (res != Z_OK && res != Z_STREAM_END)
die_horribly(AH, modulename,
"could not uncompress data: %s\n", zp->msg);
out[ZLIB_OUT_SIZE - zp->avail_out] = '\0';
ahwrite(out, 1, ZLIB_OUT_SIZE - zp->avail_out, AH);
}
if (inflateEnd(zp) != Z_OK)
die_horribly(AH, modulename,
"could not close compression library: %s\n", zp->msg);
free(buf);
free(out);
free(zp);
}
#endif /* HAVE_LIBZ */
/*
* Functions for uncompressed output.
*/
static void
ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF)
{
size_t cnt;
char *buf;
size_t buflen;
buf = malloc(ZLIB_OUT_SIZE);
if (buf == NULL)
die_horribly(NULL, modulename, "out of memory\n");
buflen = ZLIB_OUT_SIZE;
while ((cnt = readF(AH, &buf, &buflen)))
{
ahwrite(buf, 1, cnt, AH);
}
free(buf);
}
static size_t
WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen)
{
/*
* Any write function should do its own error checking but to make
* sure we do a check here as well...
*/
if (cs->writeF(AH, data, dLen) != dLen)
die_horribly(AH, modulename,
"could not write to output file: %s\n",
strerror(errno));
return dLen;
}

68
src/bin/pg_dump/compress_io.h Normal file
View File

@ -0,0 +1,68 @@
/*-------------------------------------------------------------------------
*
* compress_io.h
* Interface to compress_io.c routines
*
* Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
* src/bin/pg_dump/compress_io.h
*
*-------------------------------------------------------------------------
*/
#ifndef __COMPRESS_IO__
#define __COMPRESS_IO__
#include "postgres_fe.h"
#include "pg_backup_archiver.h"
/* Initial buffer sizes used in zlib compression. */
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
struct _CompressorState;
typedef enum
{
COMPR_ALG_NONE,
COMPR_ALG_LIBZ
} CompressionAlgorithm;
/* Prototype for callback function to WriteDataToArchive() */
typedef size_t (*WriteFunc)(ArchiveHandle *AH, const char *buf, size_t len);
/*
* Prototype for callback function to ReadDataFromArchive()
*
* ReadDataFromArchive will call the read function repeatedly, until it
* returns 0 to signal EOF. ReadDataFromArchive passes a buffer to read the
* data into in *buf, of length *buflen. If that's not big enough for the
* callback function, it can free() it and malloc() a new one, returning the
* new buffer and its size in *buf and *buflen.
*
* Returns the number of bytes read into *buf, or 0 on EOF.
*/
typedef size_t (*ReadFunc)(ArchiveHandle *AH, char **buf, size_t *buflen);
typedef struct _CompressorState
{
CompressionAlgorithm comprAlg;
WriteFunc writeF;
#ifdef HAVE_LIBZ
z_streamp zp;
char *zlibOut;
size_t zlibOutSize;
#endif
} CompressorState;
extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
ReadFunc readF);
extern size_t WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
#endif

7
src/bin/pg_dump/pg_backup_archiver.h
View File

@ -49,6 +49,7 @@
#define GZCLOSE(fh) fclose(fh) #define GZCLOSE(fh) fclose(fh)
#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
#define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZREAD(p, s, n, fh) fread(p, s, n, fh)
/* this is just the redefinition of a libz constant */
#define Z_DEFAULT_COMPRESSION (-1) #define Z_DEFAULT_COMPRESSION (-1)
typedef struct _z_stream typedef struct _z_stream
@ -266,7 +267,11 @@ typedef struct _archiveHandle
DumpId maxDumpId; /* largest DumpId among all TOC entries */ DumpId maxDumpId; /* largest DumpId among all TOC entries */
struct _tocEntry *currToc; /* Used when dumping data */ struct _tocEntry *currToc; /* Used when dumping data */
int compression; /* Compression requested on open */ int compression; /* Compression requested on open
* Possible values for compression:
* -1 Z_DEFAULT_COMPRESSION
* 0 COMPRESSION_NONE
* 1-9 levels for gzip compression */
ArchiveMode mode; /* File mode - r or w */ ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */ void *formatData; /* Header data specific to file format */

365
src/bin/pg_dump/pg_backup_custom.c
View File

@ -25,6 +25,7 @@
*/ */
#include "pg_backup_archiver.h" #include "pg_backup_archiver.h"
#include "compress_io.h"
/*-------- /*--------
* Routines in the format interface * Routines in the format interface
@ -58,20 +59,9 @@ static void _LoadBlobs(ArchiveHandle *AH, bool drop);
static void _Clone(ArchiveHandle *AH); static void _Clone(ArchiveHandle *AH);
static void _DeClone(ArchiveHandle *AH); static void _DeClone(ArchiveHandle *AH);
/*------------
* Buffers used in zlib compression and extra data stored in archive and
* in TOC entries.
*------------
*/
#define zlibOutSize 4096
#define zlibInSize 4096
typedef struct typedef struct
{ {
z_streamp zp; CompressorState *cs;
char *zlibOut;
char *zlibIn;
size_t inSize;
int hasSeek; int hasSeek;
pgoff_t filePos; pgoff_t filePos;
pgoff_t dataStart; pgoff_t dataStart;
@ -89,10 +79,10 @@ typedef struct
*------ *------
*/ */
static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id); static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te);
static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te);
static pgoff_t _getFilePos(ArchiveHandle *AH, lclContext *ctx); static pgoff_t _getFilePos(ArchiveHandle *AH, lclContext *ctx);
static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush);
static size_t _CustomWriteFunc(ArchiveHandle *AH, const char *buf, size_t len);
static size_t _CustomReadFunc(ArchiveHandle *AH, char **buf, size_t *buflen);
static const char *modulename = gettext_noop("custom archiver"); static const char *modulename = gettext_noop("custom archiver");
@ -136,39 +126,20 @@ InitArchiveFmt_Custom(ArchiveHandle *AH)
AH->ClonePtr = _Clone; AH->ClonePtr = _Clone;
AH->DeClonePtr = _DeClone; AH->DeClonePtr = _DeClone;
/* /* Set up a private area. */
* Set up some special context used in compressing data.
*/
ctx = (lclContext *) calloc(1, sizeof(lclContext)); ctx = (lclContext *) calloc(1, sizeof(lclContext));
if (ctx == NULL) if (ctx == NULL)
die_horribly(AH, modulename, "out of memory\n"); die_horribly(AH, modulename, "out of memory\n");
AH->formatData = (void *) ctx; AH->formatData = (void *) ctx;
ctx->zp = (z_streamp) malloc(sizeof(z_stream));
if (ctx->zp == NULL)
die_horribly(AH, modulename, "out of memory\n");
/* Initialize LO buffering */ /* Initialize LO buffering */
AH->lo_buf_size = LOBBUFSIZE; AH->lo_buf_size = LOBBUFSIZE;
AH->lo_buf = (void *) malloc(LOBBUFSIZE); AH->lo_buf = (void *) malloc(LOBBUFSIZE);
if (AH->lo_buf == NULL) if (AH->lo_buf == NULL)
die_horribly(AH, modulename, "out of memory\n"); die_horribly(AH, modulename, "out of memory\n");
/*
* zlibOutSize is the buffer size we tell zlib it can output to. We
* actually allocate one extra byte because some routines want to append a
* trailing zero byte to the zlib output. The input buffer is expansible
* and is always of size ctx->inSize; zlibInSize is just the initial
* default size for it.
*/
ctx->zlibOut = (char *) malloc(zlibOutSize + 1);
ctx->zlibIn = (char *) malloc(zlibInSize);
ctx->inSize = zlibInSize;
ctx->filePos = 0; ctx->filePos = 0;
if (ctx->zlibOut == NULL || ctx->zlibIn == NULL)
die_horribly(AH, modulename, "out of memory\n");
/* /*
* Now open the file * Now open the file
*/ */
@ -324,7 +295,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */ _WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */ WriteInt(AH, te->dumpId); /* For sanity check */
_StartDataCompressor(AH, te); ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
} }
/* /*
@ -340,17 +311,12 @@ static size_t
_WriteData(ArchiveHandle *AH, const void *data, size_t dLen) _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
{ {
lclContext *ctx = (lclContext *) AH->formatData; lclContext *ctx = (lclContext *) AH->formatData;
z_streamp zp = ctx->zp; CompressorState *cs = ctx->cs;
zp->next_in = (void *) data; if (dLen == 0)
zp->avail_in = dLen; return 0;
while (zp->avail_in != 0) return WriteDataToArchive(AH, cs, data, dLen);
{
/* printf("Deflating %lu bytes\n", (unsigned long) dLen); */
_DoDeflate(AH, ctx, 0);
}
return dLen;
} }
/* /*
@ -363,10 +329,11 @@ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
static void static void
_EndData(ArchiveHandle *AH, TocEntry *te) _EndData(ArchiveHandle *AH, TocEntry *te)
{ {
/* lclContext *ctx = (lclContext *) AH->formatData; */ lclContext *ctx = (lclContext *) AH->formatData;
/* lclTocEntry *tctx = (lclTocEntry *) te->formatData; */
_EndDataCompressor(AH, te); EndCompressor(AH, ctx->cs);
/* Send the end marker */
WriteInt(AH, 0);
} }
/* /*
@ -401,11 +368,14 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
static void static void
_StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
{ {
lclContext *ctx = (lclContext *) AH->formatData;
if (oid == 0) if (oid == 0)
die_horribly(AH, modulename, "invalid OID for large object\n"); die_horribly(AH, modulename, "invalid OID for large object\n");
WriteInt(AH, oid); WriteInt(AH, oid);
_StartDataCompressor(AH, te);
ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
} }
/* /*
@ -416,7 +386,11 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
static void static void
_EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
{ {
_EndDataCompressor(AH, te); lclContext *ctx = (lclContext *) AH->formatData;
EndCompressor(AH, ctx->cs);
/* Send the end marker */
WriteInt(AH, 0);
} }
/* /*
@ -532,108 +506,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
static void static void
_PrintData(ArchiveHandle *AH) _PrintData(ArchiveHandle *AH)
{ {
lclContext *ctx = (lclContext *) AH->formatData; ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
z_streamp zp = ctx->zp;
size_t blkLen;
char *in = ctx->zlibIn;
size_t cnt;
#ifdef HAVE_LIBZ
int res;
char *out = ctx->zlibOut;
#endif
#ifdef HAVE_LIBZ
res = Z_OK;
if (AH->compression != 0)
{
zp->zalloc = Z_NULL;
zp->zfree = Z_NULL;
zp->opaque = Z_NULL;
if (inflateInit(zp) != Z_OK)
die_horribly(AH, modulename, "could not initialize compression library: %s\n", zp->msg);
}
#endif
blkLen = ReadInt(AH);
while (blkLen != 0)
{
if (blkLen + 1 > ctx->inSize)
{
free(ctx->zlibIn);
ctx->zlibIn = NULL;
ctx->zlibIn = (char *) malloc(blkLen + 1);
if (!ctx->zlibIn)
die_horribly(AH, modulename, "out of memory\n");
ctx->inSize = blkLen + 1;
in = ctx->zlibIn;
}
cnt = fread(in, 1, blkLen, AH->FH);
if (cnt != blkLen)
{
if (feof(AH->FH))
die_horribly(AH, modulename,
"could not read from input file: end of file\n");
else
die_horribly(AH, modulename,
"could not read from input file: %s\n", strerror(errno));
}
ctx->filePos += blkLen;
zp->next_in = (void *) in;
zp->avail_in = blkLen;
#ifdef HAVE_LIBZ
if (AH->compression != 0)
{
while (zp->avail_in != 0)
{
zp->next_out = (void *) out;
zp->avail_out = zlibOutSize;
res = inflate(zp, 0);
if (res != Z_OK && res != Z_STREAM_END)
die_horribly(AH, modulename, "could not uncompress data: %s\n", zp->msg);
out[zlibOutSize - zp->avail_out] = '\0';
ahwrite(out, 1, zlibOutSize - zp->avail_out, AH);
}
}
else
#endif
{
in[zp->avail_in] = '\0';
ahwrite(in, 1, zp->avail_in, AH);
zp->avail_in = 0;
}
blkLen = ReadInt(AH);
}
#ifdef HAVE_LIBZ
if (AH->compression != 0)
{
zp->next_in = NULL;
zp->avail_in = 0;
while (res != Z_STREAM_END)
{
zp->next_out = (void *) out;
zp->avail_out = zlibOutSize;
res = inflate(zp, 0);
if (res != Z_OK && res != Z_STREAM_END)
die_horribly(AH, modulename, "could not uncompress data: %s\n", zp->msg);
out[zlibOutSize - zp->avail_out] = '\0';
ahwrite(out, 1, zlibOutSize - zp->avail_out, AH);
}
if (inflateEnd(zp) != Z_OK)
die_horribly(AH, modulename, "could not close compression library: %s\n", zp->msg);
}
#endif
} }
static void static void
@ -684,20 +557,21 @@ _skipData(ArchiveHandle *AH)
{ {
lclContext *ctx = (lclContext *) AH->formatData; lclContext *ctx = (lclContext *) AH->formatData;
size_t blkLen; size_t blkLen;
char *in = ctx->zlibIn; char *buf = NULL;
int buflen = 0;
size_t cnt; size_t cnt;
blkLen = ReadInt(AH); blkLen = ReadInt(AH);
while (blkLen != 0) while (blkLen != 0)
{ {
if (blkLen > ctx->inSize) if (blkLen > buflen)
{ {
free(ctx->zlibIn); if (buf)
ctx->zlibIn = (char *) malloc(blkLen); free(buf);
ctx->inSize = blkLen; buf = (char *) malloc(blkLen);
in = ctx->zlibIn; buflen = blkLen;
} }
cnt = fread(in, 1, blkLen, AH->FH); cnt = fread(buf, 1, blkLen, AH->FH);
if (cnt != blkLen) if (cnt != blkLen)
{ {
if (feof(AH->FH)) if (feof(AH->FH))
@ -712,6 +586,9 @@ _skipData(ArchiveHandle *AH)
blkLen = ReadInt(AH); blkLen = ReadInt(AH);
} }
if (buf)
free(buf);
} }
/* /*
@ -961,145 +838,58 @@ _readBlockHeader(ArchiveHandle *AH, int *type, int *id)
} }
/* /*
* If zlib is available, then startit up. This is called from * Callback function for WriteDataToArchive. Writes one block of (compressed)
* StartData & StartBlob. The buffers are setup in the Init routine. * data to the archive.
*/ */
static void static size_t
_StartDataCompressor(ArchiveHandle *AH, TocEntry *te) _CustomWriteFunc(ArchiveHandle *AH, const char *buf, size_t len)
{ {
lclContext *ctx = (lclContext *) AH->formatData; /* never write 0-byte blocks (this should not happen) */
z_streamp zp = ctx->zp; if (len == 0)
return 0;
#ifdef HAVE_LIBZ WriteInt(AH, len);
return _WriteBuf(AH, buf, len);
if (AH->compression < 0 || AH->compression > 9)
AH->compression = Z_DEFAULT_COMPRESSION;
if (AH->compression != 0)
{
zp->zalloc = Z_NULL;
zp->zfree = Z_NULL;
zp->opaque = Z_NULL;
if (deflateInit(zp, AH->compression) != Z_OK)
die_horribly(AH, modulename, "could not initialize compression library: %s\n", zp->msg);
}
#else
AH->compression = 0;
#endif
/* Just be paranoid - maybe End is called after Start, with no Write */
zp->next_out = (void *) ctx->zlibOut;
zp->avail_out = zlibOutSize;
} }
/* /*
* Send compressed data to the output stream (via ahwrite). * Callback function for ReadDataFromArchive. To keep things simple, we
* Each data chunk is preceded by it's length. * always read one compressed block at a time.
* In the case of Z0, or no zlib, just write the raw data.
*
*/ */
static int static size_t
_DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush) _CustomReadFunc(ArchiveHandle *AH, char **buf, size_t *buflen)
{ {
z_streamp zp = ctx->zp; size_t blkLen;
size_t cnt;
#ifdef HAVE_LIBZ /* Read length */
char *out = ctx->zlibOut; blkLen = ReadInt(AH);
int res = Z_OK; if (blkLen == 0)
return 0;
if (AH->compression != 0) /* If the caller's buffer is not large enough, allocate a bigger one */
if (blkLen > *buflen)
{ {
res = deflate(zp, flush); free(*buf);
if (res == Z_STREAM_ERROR) *buf = (char *) malloc(blkLen);
die_horribly(AH, modulename, "could not compress data: %s\n", zp->msg); if (!(*buf))
die_horribly(AH, modulename, "out of memory\n");
if (((flush == Z_FINISH) && (zp->avail_out < zlibOutSize)) *buflen = blkLen;
|| (zp->avail_out == 0)
|| (zp->avail_in != 0)
)
{
/*
* Extra paranoia: avoid zero-length chunks since a zero length
* chunk is the EOF marker. This should never happen but...
*/
if (zp->avail_out < zlibOutSize)
{
/*
* printf("Wrote %lu byte deflated chunk\n", (unsigned long)
* (zlibOutSize - zp->avail_out));
*/
WriteInt(AH, zlibOutSize - zp->avail_out);
if (fwrite(out, 1, zlibOutSize - zp->avail_out, AH->FH) != (zlibOutSize - zp->avail_out))
die_horribly(AH, modulename, "could not write to output file: %s\n", strerror(errno));
ctx->filePos += zlibOutSize - zp->avail_out;
}
zp->next_out = (void *) out;
zp->avail_out = zlibOutSize;
}
} }
else
#endif cnt = _ReadBuf(AH, *buf, blkLen);
if (cnt != blkLen)
{ {
if (zp->avail_in > 0) if (feof(AH->FH))
{ die_horribly(AH, modulename,
WriteInt(AH, zp->avail_in); "could not read from input file: end of file\n");
if (fwrite(zp->next_in, 1, zp->avail_in, AH->FH) != zp->avail_in)
die_horribly(AH, modulename, "could not write to output file: %s\n", strerror(errno));
ctx->filePos += zp->avail_in;
zp->avail_in = 0;
}
else else
{ die_horribly(AH, modulename,
#ifdef HAVE_LIBZ "could not read from input file: %s\n", strerror(errno));
if (flush == Z_FINISH)
res = Z_STREAM_END;
#endif
}
} }
return cnt;
#ifdef HAVE_LIBZ
return res;
#else
return 1;
#endif
} }
/*
* Terminate zlib context and flush it's buffers. If no zlib
* then just return.
*/
static void
_EndDataCompressor(ArchiveHandle *AH, TocEntry *te)
{
#ifdef HAVE_LIBZ
lclContext *ctx = (lclContext *) AH->formatData;
z_streamp zp = ctx->zp;
int res;
if (AH->compression != 0)
{
zp->next_in = NULL;
zp->avail_in = 0;
do
{
/* printf("Ending data output\n"); */
res = _DoDeflate(AH, ctx, Z_FINISH);
} while (res != Z_STREAM_END);
if (deflateEnd(zp) != Z_OK)
die_horribly(AH, modulename, "could not close compression stream: %s\n", zp->msg);
}
#endif
/* Send the end marker */
WriteInt(AH, 0);
}
/* /*
* Clone format-specific fields during parallel restoration. * Clone format-specific fields during parallel restoration.
*/ */
@ -1114,12 +904,9 @@ _Clone(ArchiveHandle *AH)
memcpy(AH->formatData, ctx, sizeof(lclContext)); memcpy(AH->formatData, ctx, sizeof(lclContext));
ctx = (lclContext *) AH->formatData; ctx = (lclContext *) AH->formatData;
ctx->zp = (z_streamp) malloc(sizeof(z_stream)); /* sanity check, shouldn't happen */
ctx->zlibOut = (char *) malloc(zlibOutSize + 1); if (ctx->cs != NULL)
ctx->zlibIn = (char *) malloc(ctx->inSize); die_horribly(AH, modulename, "compressor active\n");
if (ctx->zp == NULL || ctx->zlibOut == NULL || ctx->zlibIn == NULL)
die_horribly(AH, modulename, "out of memory\n");
/* /*
* Note: we do not make a local lo_buf because we expect at most one BLOBS * Note: we do not make a local lo_buf because we expect at most one BLOBS
@ -1133,9 +920,5 @@ static void
_DeClone(ArchiveHandle *AH) _DeClone(ArchiveHandle *AH)
{ {
lclContext *ctx = (lclContext *) AH->formatData; lclContext *ctx = (lclContext *) AH->formatData;
free(ctx->zlibOut);
free(ctx->zlibIn);
free(ctx->zp);
free(ctx); free(ctx);
} }