From 3e23b68dac006e8deb0afa327e855258df8de064 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Fri, 6 Apr 2007 04:21:44 +0000 Subject: [PATCH] Support varlena fields with single-byte headers and unaligned storage. This commit breaks any code that assumes that the mere act of forming a tuple (without writing it to disk) does not "toast" any fields. While all available regression tests pass, I'm not totally sure that we've fixed every nook and cranny, especially in contrib. Greg Stark with some help from Tom Lane --- configure | 232 +++++++++- configure.in | 3 +- contrib/dblink/dblink.c | 6 +- contrib/hstore/hstore_gist.c | 20 +- contrib/intarray/_int_gist.c | 9 + contrib/pg_trgm/trgm_gist.c | 22 +- contrib/tsearch2/ts_cfg.c | 4 +- doc/src/sgml/storage.sgml | 57 +-- src/backend/access/common/heaptuple.c | 447 +++++++++++++++---- src/backend/access/common/indextuple.c | 142 +++--- src/backend/access/heap/tuptoaster.c | 496 +++++++++++++-------- src/backend/catalog/toasting.c | 4 +- src/backend/commands/analyze.c | 6 +- src/backend/executor/execQual.c | 6 +- src/backend/storage/large_object/inv_api.c | 10 +- src/backend/utils/adt/arrayfuncs.c | 84 ++-- src/backend/utils/adt/datum.c | 6 +- src/backend/utils/adt/network.c | 104 +++-- src/backend/utils/adt/pg_lzcompress.c | 4 +- src/backend/utils/adt/varchar.c | 160 +++---- src/backend/utils/adt/varlena.c | 280 ++++++------ src/backend/utils/adt/xml.c | 6 +- src/backend/utils/fmgr/fmgr.c | 17 +- src/backend/utils/init/flatfiles.c | 6 +- src/include/access/heapam.h | 5 +- src/include/access/htup.h | 15 +- src/include/access/tupmacs.h | 91 +++- src/include/access/tuptoaster.h | 8 +- src/include/catalog/catversion.h | 4 +- src/include/catalog/pg_type.h | 12 +- src/include/fmgr.h | 18 +- src/include/pg_config.h.in | 4 + src/include/postgres.h | 243 ++++++++-- src/include/utils/inet.h | 22 +- src/test/regress/expected/rowtypes.out | 17 + src/test/regress/expected/strings.out | 20 + src/test/regress/sql/rowtypes.sql | 4 + src/test/regress/sql/strings.sql | 13 + 38 files changed, 1802 insertions(+), 805 deletions(-) diff --git a/configure b/configure index c5a20b1e12..e53c705d71 100755 --- a/configure +++ b/configure @@ -11143,7 +11143,237 @@ fi ## Types, structures, compiler characteristics ## - echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5 + echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5 +echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6 +if test "${ac_cv_c_bigendian+set}" = set; then + echo $ECHO_N "(cached) $ECHO_C" >&6 +else + # See if sys/param.h defines the BYTE_ORDER macro. +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +#include + +int +main () +{ +#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN + bogus endian macros +#endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + # It does; now see whether it defined to BIG_ENDIAN or not. +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +#include +#include + +int +main () +{ +#if BYTE_ORDER != BIG_ENDIAN + not big endian +#endif + + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_c_bigendian=yes +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +ac_cv_c_bigendian=no +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +# It does not; compile a test program. +if test "$cross_compiling" = yes; then + # try to guess the endianness by grepping values into an object file + ac_cv_c_bigendian=unknown + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 }; +short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 }; +void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; } +short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 }; +short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 }; +void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; } +int +main () +{ + _ascii (); _ebcdic (); + ; + return 0; +} +_ACEOF +rm -f conftest.$ac_objext +if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5 + (eval $ac_compile) 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && + { ac_try='test -z "$ac_c_werror_flag" + || test ! -s conftest.err' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; } && + { ac_try='test -s conftest.$ac_objext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then + ac_cv_c_bigendian=yes +fi +if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then + if test "$ac_cv_c_bigendian" = unknown; then + ac_cv_c_bigendian=no + else + # finding both strings is unlikely to happen, but who knows? + ac_cv_c_bigendian=unknown + fi +fi +else + echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +else + cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ +int +main () +{ + /* Are we little or big endian? From Harbison&Steele. */ + union + { + long l; + char c[sizeof (long)]; + } u; + u.l = 1; + exit (u.c[sizeof (long) - 1] == 1); +} +_ACEOF +rm -f conftest$ac_exeext +if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5 + (eval $ac_link) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { ac_try='./conftest$ac_exeext' + { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5 + (eval $ac_try) 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); }; }; then + ac_cv_c_bigendian=no +else + echo "$as_me: program exited with status $ac_status" >&5 +echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + +( exit $ac_status ) +ac_cv_c_bigendian=yes +fi +rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext +fi +fi +rm -f conftest.err conftest.$ac_objext conftest.$ac_ext +fi +echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5 +echo "${ECHO_T}$ac_cv_c_bigendian" >&6 +case $ac_cv_c_bigendian in + yes) + +cat >>confdefs.h <<\_ACEOF +#define WORDS_BIGENDIAN 1 +_ACEOF + ;; + no) + ;; + *) + { { echo "$as_me:$LINENO: error: unknown endianness +presetting ac_cv_c_bigendian=no (or yes) will help" >&5 +echo "$as_me: error: unknown endianness +presetting ac_cv_c_bigendian=no (or yes) will help" >&2;} + { (exit 1); exit 1; }; } ;; +esac + +echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5 echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6 if test "${ac_cv_c_const+set}" = set; then echo $ECHO_N "(cached) $ECHO_C" >&6 diff --git a/configure.in b/configure.in index 03511fbd46..9f7bf314d5 100644 --- a/configure.in +++ b/configure.in @@ -1,5 +1,5 @@ dnl Process this file with autoconf to produce a configure script. -dnl $PostgreSQL: pgsql/configure.in,v 1.507 2007/03/29 15:30:51 mha Exp $ +dnl $PostgreSQL: pgsql/configure.in,v 1.508 2007/04/06 04:21:41 tgl Exp $ dnl dnl Developers, please strive to achieve this order: dnl @@ -869,6 +869,7 @@ fi ## m4_defun([AC_PROG_CC_STDC], []) dnl We don't want that. +AC_C_BIGENDIAN AC_C_CONST AC_C_INLINE AC_C_STRINGIZE diff --git a/contrib/dblink/dblink.c b/contrib/dblink/dblink.c index 568ac1e5f7..b42dd02672 100644 --- a/contrib/dblink/dblink.c +++ b/contrib/dblink/dblink.c @@ -8,7 +8,7 @@ * Darko Prenosil * Shridhar Daithankar * - * $PostgreSQL: pgsql/contrib/dblink/dblink.c,v 1.62 2007/02/07 00:52:35 petere Exp $ + * $PostgreSQL: pgsql/contrib/dblink/dblink.c,v 1.63 2007/04/06 04:21:41 tgl Exp $ * Copyright (c) 2001-2007, PostgreSQL Global Development Group * ALL RIGHTS RESERVED; * @@ -1752,8 +1752,8 @@ get_text_array_contents(ArrayType *array, int *numitems) { values[i] = DatumGetCString(DirectFunctionCall1(textout, PointerGetDatum(ptr))); - ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr)); - ptr = (char *) att_align(ptr, typalign); + ptr = att_addlength_pointer(ptr, typlen, ptr); + ptr = (char *) att_align_nominal(ptr, typalign); } /* advance bitmap pointer if any */ diff --git a/contrib/hstore/hstore_gist.c b/contrib/hstore/hstore_gist.c index fbee64be7d..19ed74933f 100644 --- a/contrib/hstore/hstore_gist.c +++ b/contrib/hstore/hstore_gist.c @@ -170,7 +170,25 @@ ghstore_compress(PG_FUNCTION_ARGS) Datum ghstore_decompress(PG_FUNCTION_ARGS) { - PG_RETURN_DATUM(PG_GETARG_DATUM(0)); + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + GISTENTRY *retval; + HStore *key; + + key = (HStore *) PG_DETOAST_DATUM(entry->key); + + if (key != (HStore *) DatumGetPointer(entry->key)) + { + /* need to pass back the decompressed item */ + retval = palloc(sizeof(GISTENTRY)); + gistentryinit(*retval, PointerGetDatum(key), + entry->rel, entry->page, entry->offset, entry->leafkey); + PG_RETURN_POINTER(retval); + } + else + { + /* we can return the entry as-is */ + PG_RETURN_POINTER(entry); + } } Datum diff --git a/contrib/intarray/_int_gist.c b/contrib/intarray/_int_gist.c index 56eb0c08c2..3c34cb67a7 100644 --- a/contrib/intarray/_int_gist.c +++ b/contrib/intarray/_int_gist.c @@ -232,7 +232,16 @@ g_int_decompress(PG_FUNCTION_ARGS) CHECKARRVALID(in); if (ARRISVOID(in)) + { + if (in != (ArrayType *) DatumGetPointer(entry->key)) { + retval = palloc(sizeof(GISTENTRY)); + gistentryinit(*retval, PointerGetDatum(in), + entry->rel, entry->page, entry->offset, FALSE); + PG_RETURN_POINTER(retval); + } + PG_RETURN_POINTER(entry); + } lenin = ARRNELEMS(in); diff --git a/contrib/pg_trgm/trgm_gist.c b/contrib/pg_trgm/trgm_gist.c index 476cb1b976..260fe01da4 100644 --- a/contrib/pg_trgm/trgm_gist.c +++ b/contrib/pg_trgm/trgm_gist.c @@ -97,7 +97,7 @@ gtrgm_compress(PG_FUNCTION_ARGS) if (entry->leafkey) { /* trgm */ TRGM *res; - text *val = (text *) DatumGetPointer(PG_DETOAST_DATUM(entry->key)); + text *val = DatumGetTextP(entry->key); res = generate_trgm(VARDATA(val), VARSIZE(val) - VARHDRSZ); retval = (GISTENTRY *) palloc(sizeof(GISTENTRY)); @@ -134,7 +134,25 @@ gtrgm_compress(PG_FUNCTION_ARGS) Datum gtrgm_decompress(PG_FUNCTION_ARGS) { - PG_RETURN_DATUM(PG_GETARG_DATUM(0)); + GISTENTRY *entry = (GISTENTRY *) PG_GETARG_POINTER(0); + GISTENTRY *retval; + text *key; + + key = DatumGetTextP(entry->key); + + if (key != (text *) DatumGetPointer(entry->key)) + { + /* need to pass back the decompressed item */ + retval = palloc(sizeof(GISTENTRY)); + gistentryinit(*retval, PointerGetDatum(key), + entry->rel, entry->page, entry->offset, entry->leafkey); + PG_RETURN_POINTER(retval); + } + else + { + /* we can return the entry as-is */ + PG_RETURN_POINTER(entry); + } } Datum diff --git a/contrib/tsearch2/ts_cfg.c b/contrib/tsearch2/ts_cfg.c index 0dc17703c3..646ffc1481 100644 --- a/contrib/tsearch2/ts_cfg.c +++ b/contrib/tsearch2/ts_cfg.c @@ -62,9 +62,7 @@ init_cfg(Oid id, TSCfgInfo * cfg) ts_error(ERROR, "SPI_execp return %d", stat); if (SPI_processed > 0) { - prsname = (text *) DatumGetPointer( - SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull) - ); + prsname = DatumGetTextP(SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull)); oldcontext = MemoryContextSwitchTo(TopMemoryContext); prsname = ptextdup(prsname); MemoryContextSwitchTo(oldcontext); diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 1973a5b90c..9c3cf7589d 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -1,4 +1,4 @@ - + @@ -210,18 +210,27 @@ value, but in some cases more efficient approaches are possible.) -TOAST usurps the high-order two bits of the varlena length word, +TOAST usurps two bits of the varlena length word (the high-order +bits on big-endian machines, the low-order bits on little-endian machines), thereby limiting the logical size of any value of a TOAST-able data type to 1 GB (230 - 1 bytes). When both bits are zero, -the value is an ordinary un-TOASTed value of the data type. One -of these bits, if set, indicates that the value has been compressed and must -be decompressed before use. The other bit, if set, indicates that the value -has been stored out-of-line. In this case the remainder of the value is -actually just a pointer, and the correct data has to be found elsewhere. When -both bits are set, the out-of-line data has been compressed too. In each case -the length in the low-order bits of the varlena word indicates the actual size -of the datum, not the size of the logical value that would be extracted by -decompression or fetching of the out-of-line data. +the value is an ordinary un-TOASTed value of the data type, and +the remaining bits of the length word give the total datum size (including +length word) in bytes. When the highest-order or lowest-order bit is set, +the value has only a single-byte header instead of the normal four-byte +header, and the remaining bits give the total datum size (including length +byte) in bytes. As a special case, if the remaining bits are all zero +(which would be impossible for a self-inclusive length), the value is a +pointer to out-of-line data stored in a separate TOAST table. (The size of +a TOAST pointer is known a priori, so it doesn't need to be represented in +the header.) Values with single-byte headers aren't aligned on any particular +boundary, either. Lastly, when the highest-order or lowest-order bit is +clear but the adjacent bit is set, the content of the datum has been +compressed and must be decompressed before use. In this case the remaining +bits of the length word give the total size of the compressed datum, not the +original data. Note that compression is also possible for out-of-line data +but the varlena header does not tell whether it has occurred — +the content of the TOAST pointer tells that, instead. @@ -254,8 +263,8 @@ retrieval of the values. A pointer datum representing an out-of-line TOAST table in which to look and the OID of the specific value (its chunk_id). For convenience, pointer datums also store the logical datum size (original uncompressed data length) and actual stored size -(different if compression was applied). Allowing for the varlena header word, -the total size of a TOAST pointer datum is therefore 20 bytes +(different if compression was applied). Allowing for the varlena header byte, +the total size of a TOAST pointer datum is therefore 17 bytes regardless of the actual size of the represented value. @@ -280,7 +289,9 @@ The TOAST code recognizes four different strategies for storing PLAIN prevents either compression or - out-of-line storage. This is the only possible strategy for + out-of-line storage; furthermore it disables use of single-byte headers + for varlena types. + This is the only possible strategy for columns of non-TOAST-able data types. @@ -562,7 +573,7 @@ data. Empty in ordinary tables. All table rows are structured in the same way. There is a fixed-size - header (occupying 27 bytes on most machines), followed by an optional null + header (occupying 23 bytes on most machines), followed by an optional null bitmap, an optional object ID field, and the user data. The header is detailed in . The actual user data @@ -604,12 +615,6 @@ data. Empty in ordinary tables. 4 bytes insert XID stamp - - t_cmin - CommandId - 4 bytes - insert CID stamp - t_xmax TransactionId @@ -617,10 +622,10 @@ data. Empty in ordinary tables. delete XID stamp - t_cmax + t_cid CommandId 4 bytes - delete CID stamp (overlays with t_xvac) + insert and/or delete CID stamp (overlays with t_xvac) t_xvac @@ -635,10 +640,10 @@ data. Empty in ordinary tables. current TID of this or newer row version - t_natts + t_infomask2 int16 2 bytes - number of attributes + number of attributes, plus various flag bits t_infomask @@ -682,7 +687,7 @@ data. Empty in ordinary tables. fixed width field, then all the bytes are simply placed. If it's a variable length field (attlen = -1) then it's a bit more complicated. All variable-length datatypes share the common header structure - varattrib, which includes the total length of the stored + struct varlena, which includes the total length of the stored value and some flag bits. Depending on the flags, the data can be either inline or in a TOAST table; it might be compressed, too (see ). diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index 0c83262c3b..f1a80d6fee 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -11,12 +11,53 @@ * we can get rid of it entirely. * * + * Some notes about varlenas and this code: + * + * Before Postgres 8.3 varlenas always had a 4-byte length header, and + * therefore always needed 4-byte alignment (at least). This wasted space + * for short varlenas, for example CHAR(1) took 5 bytes and could need up to + * 3 additional padding bytes for alignment. + * + * Now, a short varlena (up to 126 data bytes) is reduced to a 1-byte header + * and we don't align it. To hide this from datatype-specific functions that + * don't want to deal with it, such a datum is considered "toasted" and will + * be expanded back to the normal 4-byte-header format by pg_detoast_datum. + * (In performance-critical code paths we can use pg_detoast_datum_packed + * and the appropriate access macros to avoid that overhead.) Note that this + * conversion is performed directly in heap_form_tuple (or heap_formtuple), + * without explicitly invoking the toaster. + * + * This change will break any code that assumes it needn't detoast values + * that have been put into a tuple but never sent to disk. Hopefully there + * are few such places. + * + * Varlenas still have alignment 'i' (or 'd') in pg_type/pg_attribute, since + * that's the normal requirement for the untoasted format. But we ignore that + * for the 1-byte-header format. This means that the actual start position + * of a varlena datum may vary depending on which format it has. To determine + * what is stored, we have to require that alignment padding bytes be zero. + * (Postgres actually has always zeroed them, but now it's required!) Since + * the first byte of a 1-byte-header varlena can never be zero, we can examine + * the first byte after the previous datum to tell if it's a pad byte or the + * start of a 1-byte-header varlena. + * + * Note that while formerly we could rely on the first varlena column of a + * system catalog to be at the offset suggested by the C struct for the + * catalog, this is now risky: it's only safe if the preceding field is + * word-aligned, so that there will never be any padding. + * + * We don't pack varlenas whose attstorage is 'p', since the data type + * isn't expecting to have to detoast values. This is used in particular + * by oidvector and int2vector, which are used in the system catalogs + * and we'd like to still refer to them via C struct offsets. + * + * * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/access/common/heaptuple.c,v 1.116 2007/02/27 23:48:06 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/access/common/heaptuple.c,v 1.117 2007/04/06 04:21:41 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -28,11 +69,20 @@ #include "executor/tuptable.h" +/* Does att's datatype allow packing into the 1-byte-header varlena format? */ +#define ATT_IS_PACKABLE(att) \ + ((att)->attlen == -1 && (att)->attstorage != 'p') +/* Use this if it's already known varlena */ +#define VARLENA_ATT_IS_PACKABLE(att) \ + ((att)->attstorage != 'p') + + /* ---------------------------------------------------------------- * misc support routines * ---------------------------------------------------------------- */ + /* * heap_compute_data_size * Determine size of the data area of a tuple to be constructed @@ -49,11 +99,29 @@ heap_compute_data_size(TupleDesc tupleDesc, for (i = 0; i < numberOfAttributes; i++) { + Datum val; + if (isnull[i]) continue; - data_length = att_align(data_length, att[i]->attalign); - data_length = att_addlength(data_length, att[i]->attlen, values[i]); + val = values[i]; + + if (ATT_IS_PACKABLE(att[i]) && + VARATT_CAN_MAKE_SHORT(DatumGetPointer(val))) + { + /* + * we're anticipating converting to a short varlena header, + * so adjust length and don't count any alignment + */ + data_length += VARATT_CONVERTED_SHORT_SIZE(DatumGetPointer(val)); + } + else + { + data_length = att_align_datum(data_length, att[i]->attalign, + att[i]->attlen, val); + data_length = att_addlength_datum(data_length, att[i]->attlen, + val); + } } return data_length; @@ -79,11 +147,29 @@ ComputeDataSize(TupleDesc tupleDesc, for (i = 0; i < numberOfAttributes; i++) { + Datum val; + if (nulls[i] != ' ') continue; - data_length = att_align(data_length, att[i]->attalign); - data_length = att_addlength(data_length, att[i]->attlen, values[i]); + val = values[i]; + + if (ATT_IS_PACKABLE(att[i]) && + VARATT_CAN_MAKE_SHORT(DatumGetPointer(val))) + { + /* + * we're anticipating converting to a short varlena header, + * so adjust length and don't count any alignment + */ + data_length += VARATT_CONVERTED_SHORT_SIZE(DatumGetPointer(val)); + } + else + { + data_length = att_align_datum(data_length, att[i]->attalign, + att[i]->attlen, val); + data_length = att_addlength_datum(data_length, att[i]->attlen, + val); + } } return data_length; @@ -95,17 +181,23 @@ ComputeDataSize(TupleDesc tupleDesc, * * We also fill the null bitmap (if any) and set the infomask bits * that reflect the tuple's data contents. + * + * NOTE: it is now REQUIRED that the caller have pre-zeroed the data area. */ void heap_fill_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull, - char *data, uint16 *infomask, bits8 *bit) + char *data, Size data_size, + uint16 *infomask, bits8 *bit) { bits8 *bitP; int bitmask; int i; int numberOfAttributes = tupleDesc->natts; Form_pg_attribute *att = tupleDesc->attrs; +#ifdef USE_ASSERT_CHECKING + char *start = data; +#endif if (bit != NULL) { @@ -119,7 +211,7 @@ heap_fill_tuple(TupleDesc tupleDesc, bitmask = 0; } - *infomask &= ~(HEAP_HASNULL | HEAP_HASVARWIDTH | HEAP_HASEXTENDED); + *infomask &= ~(HEAP_HASNULL | HEAP_HASVARWIDTH | HEAP_HASEXTERNAL); for (i = 0; i < numberOfAttributes; i++) { @@ -145,36 +237,66 @@ heap_fill_tuple(TupleDesc tupleDesc, *bitP |= bitmask; } - /* XXX we are aligning the pointer itself, not the offset */ - data = (char *) att_align((long) data, att[i]->attalign); + /* + * XXX we use the att_align macros on the pointer value itself, + * not on an offset. This is a bit of a hack. + */ if (att[i]->attbyval) { /* pass-by-value */ + data = (char *) att_align_nominal((long) data, att[i]->attalign); store_att_byval(data, values[i], att[i]->attlen); data_length = att[i]->attlen; } else if (att[i]->attlen == -1) { /* varlena */ + Pointer val = DatumGetPointer(values[i]); + *infomask |= HEAP_HASVARWIDTH; - if (VARATT_IS_EXTERNAL(values[i])) + if (VARATT_IS_EXTERNAL(val)) + { *infomask |= HEAP_HASEXTERNAL; - if (VARATT_IS_COMPRESSED(values[i])) - *infomask |= HEAP_HASCOMPRESSED; - data_length = VARSIZE(DatumGetPointer(values[i])); - memcpy(data, DatumGetPointer(values[i]), data_length); + /* no alignment, since it's short by definition */ + data_length = VARSIZE_EXTERNAL(val); + memcpy(data, val, data_length); + } + else if (VARATT_IS_SHORT(val)) + { + /* no alignment for short varlenas */ + data_length = VARSIZE_SHORT(val); + memcpy(data, val, data_length); + } + else if (VARLENA_ATT_IS_PACKABLE(att[i]) && + VARATT_CAN_MAKE_SHORT(val)) + { + /* convert to short varlena -- no alignment */ + data_length = VARATT_CONVERTED_SHORT_SIZE(val); + SET_VARSIZE_SHORT(data, data_length); + memcpy(data + 1, VARDATA(val), data_length - 1); + } + else + { + /* full 4-byte header varlena */ + data = (char *) att_align_nominal((long) data, + att[i]->attalign); + data_length = VARSIZE(val); + memcpy(data, val, data_length); + } } else if (att[i]->attlen == -2) { - /* cstring */ + /* cstring ... never needs alignment */ *infomask |= HEAP_HASVARWIDTH; + Assert(att[i]->attalign == 'c'); data_length = strlen(DatumGetCString(values[i])) + 1; memcpy(data, DatumGetPointer(values[i]), data_length); } else { /* fixed-length pass-by-reference */ + data = (char *) att_align_nominal((long) data, att[i]->attalign); Assert(att[i]->attlen > 0); data_length = att[i]->attlen; memcpy(data, DatumGetPointer(values[i]), data_length); @@ -182,6 +304,8 @@ heap_fill_tuple(TupleDesc tupleDesc, data += data_length; } + + Assert((data - start) == data_size); } /* ---------------- @@ -193,18 +317,19 @@ heap_fill_tuple(TupleDesc tupleDesc, * ---------------- */ static void -DataFill(char *data, - TupleDesc tupleDesc, - Datum *values, - char *nulls, - uint16 *infomask, - bits8 *bit) +DataFill(TupleDesc tupleDesc, + Datum *values, char *nulls, + char *data, Size data_size, + uint16 *infomask, bits8 *bit) { bits8 *bitP; int bitmask; int i; int numberOfAttributes = tupleDesc->natts; Form_pg_attribute *att = tupleDesc->attrs; +#ifdef USE_ASSERT_CHECKING + char *start = data; +#endif if (bit != NULL) { @@ -218,7 +343,7 @@ DataFill(char *data, bitmask = 0; } - *infomask &= ~(HEAP_HASNULL | HEAP_HASVARWIDTH | HEAP_HASEXTENDED); + *infomask &= ~(HEAP_HASNULL | HEAP_HASVARWIDTH | HEAP_HASEXTERNAL); for (i = 0; i < numberOfAttributes; i++) { @@ -244,36 +369,66 @@ DataFill(char *data, *bitP |= bitmask; } - /* XXX we are aligning the pointer itself, not the offset */ - data = (char *) att_align((long) data, att[i]->attalign); + /* + * XXX we use the att_align macros on the pointer value itself, + * not on an offset. This is a bit of a hack. + */ if (att[i]->attbyval) { /* pass-by-value */ + data = (char *) att_align_nominal((long) data, att[i]->attalign); store_att_byval(data, values[i], att[i]->attlen); data_length = att[i]->attlen; } else if (att[i]->attlen == -1) { /* varlena */ + Pointer val = DatumGetPointer(values[i]); + *infomask |= HEAP_HASVARWIDTH; - if (VARATT_IS_EXTERNAL(values[i])) + if (VARATT_IS_EXTERNAL(val)) + { *infomask |= HEAP_HASEXTERNAL; - if (VARATT_IS_COMPRESSED(values[i])) - *infomask |= HEAP_HASCOMPRESSED; - data_length = VARSIZE(DatumGetPointer(values[i])); - memcpy(data, DatumGetPointer(values[i]), data_length); + /* no alignment, since it's short by definition */ + data_length = VARSIZE_EXTERNAL(val); + memcpy(data, val, data_length); + } + else if (VARATT_IS_SHORT(val)) + { + /* no alignment for short varlenas */ + data_length = VARSIZE_SHORT(val); + memcpy(data, val, data_length); + } + else if (VARLENA_ATT_IS_PACKABLE(att[i]) && + VARATT_CAN_MAKE_SHORT(val)) + { + /* convert to short varlena -- no alignment */ + data_length = VARATT_CONVERTED_SHORT_SIZE(val); + SET_VARSIZE_SHORT(data, data_length); + memcpy(data + 1, VARDATA(val), data_length - 1); + } + else + { + /* full 4-byte header varlena */ + data = (char *) att_align_nominal((long) data, + att[i]->attalign); + data_length = VARSIZE(val); + memcpy(data, val, data_length); + } } else if (att[i]->attlen == -2) { - /* cstring */ + /* cstring ... never needs alignment */ *infomask |= HEAP_HASVARWIDTH; + Assert(att[i]->attalign == 'c'); data_length = strlen(DatumGetCString(values[i])) + 1; memcpy(data, DatumGetPointer(values[i]), data_length); } else { /* fixed-length pass-by-reference */ + data = (char *) att_align_nominal((long) data, att[i]->attalign); Assert(att[i]->attlen > 0); data_length = att[i]->attlen; memcpy(data, DatumGetPointer(values[i]), data_length); @@ -281,6 +436,8 @@ DataFill(char *data, data += data_length; } + + Assert((data - start) == data_size); } /* ---------------------------------------------------------------- @@ -343,6 +500,8 @@ heap_attisnull(HeapTuple tup, int attnum) * the same attribute descriptor will go much quicker. -cim 5/4/91 * * NOTE: if you need to change this code, see also heap_deform_tuple. + * Also see nocache_index_getattr, which is the same code for index + * tuples. * ---------------- */ Datum @@ -353,20 +512,12 @@ nocachegetattr(HeapTuple tuple, { HeapTupleHeader tup = tuple->t_data; Form_pg_attribute *att = tupleDesc->attrs; - char *tp; /* ptr to att in tuple */ + char *tp; /* ptr to data part of tuple */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ - bool slow = false; /* do we have to walk nulls? */ + bool slow = false; /* do we have to walk attrs? */ + int off; /* current offset within data */ (void) isnull; /* not used */ -#ifdef IN_MACRO -/* This is handled in the macro */ - Assert(attnum > 0); - - if (isnull) - *isnull = false; -#endif - - attnum--; /* ---------------- * Three cases: @@ -377,11 +528,21 @@ nocachegetattr(HeapTuple tuple, * ---------------- */ +#ifdef IN_MACRO +/* This is handled in the macro */ + Assert(attnum > 0); + + if (isnull) + *isnull = false; +#endif + + attnum--; + if (HeapTupleNoNulls(tuple)) { #ifdef IN_MACRO /* This is handled in the macro */ - if (att[attnum]->attcacheoff != -1) + if (att[attnum]->attcacheoff >= 0) { return fetchatt(att[attnum], (char *) tup + tup->t_hoff + @@ -436,24 +597,27 @@ nocachegetattr(HeapTuple tuple, tp = (char *) tup + tup->t_hoff; - /* - * now check for any non-fixed length attrs before our attribute - */ if (!slow) { - if (att[attnum]->attcacheoff != -1) + /* + * If we get here, there are no nulls up to and including the target + * attribute. If we have a cached offset, we can use it. + */ + if (att[attnum]->attcacheoff >= 0) { return fetchatt(att[attnum], tp + att[attnum]->attcacheoff); } - else if (HeapTupleHasVarWidth(tuple)) + + /* + * Otherwise, check for non-fixed-length attrs up to and including + * target. If there aren't any, it's safe to cheaply initialize + * the cached offsets for these attrs. + */ + if (HeapTupleHasVarWidth(tuple)) { int j; - /* - * In for(), we test <= and not < because we want to see if we can - * go past it in initializing offsets. - */ for (j = 0; j <= attnum; j++) { if (att[j]->attlen <= 0) @@ -465,89 +629,109 @@ nocachegetattr(HeapTuple tuple, } } - /* - * If slow is false, and we got here, we know that we have a tuple with no - * nulls or var-widths before the target attribute. If possible, we also - * want to initialize the remainder of the attribute cached offset values. - */ if (!slow) { + int natts = tupleDesc->natts; int j = 1; - long off; - int natts = HeapTupleHeaderGetNatts(tup); /* - * need to set cache for some atts + * If we get here, we have a tuple with no nulls or var-widths up to + * and including the target attribute, so we can use the cached offset + * ... only we don't have it yet, or we'd not have got here. Since + * it's cheap to compute offsets for fixed-width columns, we take the + * opportunity to initialize the cached offsets for *all* the leading + * fixed-width columns, in hope of avoiding future visits to this + * routine. */ - att[0]->attcacheoff = 0; - while (j < attnum && att[j]->attcacheoff > 0) + /* we might have set some offsets in the slow path previously */ + while (j < natts && att[j]->attcacheoff > 0) j++; off = att[j - 1]->attcacheoff + att[j - 1]->attlen; - for (; j <= attnum || - /* Can we compute more? We will probably need them */ - (j < natts && - att[j]->attcacheoff == -1 && - (HeapTupleNoNulls(tuple) || !att_isnull(j, bp)) && - (HeapTupleAllFixed(tuple) || att[j]->attlen > 0)); j++) + for (; j < natts; j++) { - off = att_align(off, att[j]->attalign); + if (att[j]->attlen <= 0) + break; + + off = att_align_nominal(off, att[j]->attalign); att[j]->attcacheoff = off; - off = att_addlength(off, att[j]->attlen, tp + off); + off += att[j]->attlen; } - return fetchatt(att[attnum], tp + att[attnum]->attcacheoff); + Assert(j > attnum); + + off = att[attnum]->attcacheoff; } else { bool usecache = true; - int off = 0; int i; /* - * Now we know that we have to walk the tuple CAREFULLY. + * Now we know that we have to walk the tuple CAREFULLY. But we + * still might be able to cache some offsets for next time. * * Note - This loop is a little tricky. For each non-null attribute, * we have to first account for alignment padding before the attr, * then advance over the attr based on its length. Nulls have no * storage and no alignment padding either. We can use/set - * attcacheoff until we pass either a null or a var-width attribute. + * attcacheoff until we reach either a null or a var-width attribute. */ - - for (i = 0; i < attnum; i++) + off = 0; + for (i = 0; ; i++) /* loop exit is at "break" */ { if (HeapTupleHasNulls(tuple) && att_isnull(i, bp)) { usecache = false; - continue; + continue; /* this cannot be the target att */ } - /* If we know the next offset, we can skip the alignment calc */ - if (usecache && att[i]->attcacheoff != -1) + /* If we know the next offset, we can skip the rest */ + if (usecache && att[i]->attcacheoff >= 0) off = att[i]->attcacheoff; + else if (att[i]->attlen == -1) + { + /* + * We can only cache the offset for a varlena attribute + * if the offset is already suitably aligned, so that there + * would be no pad bytes in any case: then the offset will + * be valid for either an aligned or unaligned value. + */ + if (usecache && + off == att_align_nominal(off, att[i]->attalign)) + att[i]->attcacheoff = off; + else + { + off = att_align_pointer(off, att[i]->attalign, -1, + tp + off); + usecache = false; + } + } else { - off = att_align(off, att[i]->attalign); + /* not varlena, so safe to use att_align_nominal */ + off = att_align_nominal(off, att[i]->attalign); if (usecache) att[i]->attcacheoff = off; } - off = att_addlength(off, att[i]->attlen, tp + off); + if (i == attnum) + break; + + off = att_addlength_pointer(off, att[i]->attlen, tp + off); if (usecache && att[i]->attlen <= 0) usecache = false; } - - off = att_align(off, att[attnum]->attalign); - - return fetchatt(att[attnum], tp + off); } + + return fetchatt(att[attnum], tp + off); } /* ---------------- @@ -671,7 +855,7 @@ heap_form_tuple(TupleDesc tupleDescriptor, { HeapTuple tuple; /* return tuple */ HeapTupleHeader td; /* tuple data */ - unsigned long len; + Size len, data_len; int hoff; bool hasnull = false; Form_pg_attribute *att = tupleDescriptor->attrs; @@ -723,7 +907,9 @@ heap_form_tuple(TupleDesc tupleDescriptor, hoff = len = MAXALIGN(len); /* align user data safely */ - len += heap_compute_data_size(tupleDescriptor, values, isnull); + data_len = heap_compute_data_size(tupleDescriptor, values, isnull); + + len += data_len; /* * Allocate and zero the space needed. Note that the tuple body and @@ -754,6 +940,7 @@ heap_form_tuple(TupleDesc tupleDescriptor, values, isnull, (char *) td + hoff, + data_len, &td->t_infomask, (hasnull ? td->t_bits : NULL)); @@ -778,7 +965,7 @@ heap_formtuple(TupleDesc tupleDescriptor, { HeapTuple tuple; /* return tuple */ HeapTupleHeader td; /* tuple data */ - unsigned long len; + Size len, data_len; int hoff; bool hasnull = false; Form_pg_attribute *att = tupleDescriptor->attrs; @@ -830,7 +1017,9 @@ heap_formtuple(TupleDesc tupleDescriptor, hoff = len = MAXALIGN(len); /* align user data safely */ - len += ComputeDataSize(tupleDescriptor, values, nulls); + data_len = ComputeDataSize(tupleDescriptor, values, nulls); + + len += data_len; /* * Allocate and zero the space needed. Note that the tuple body and @@ -857,16 +1046,18 @@ heap_formtuple(TupleDesc tupleDescriptor, if (tupleDescriptor->tdhasoid) /* else leave infomask = 0 */ td->t_infomask = HEAP_HASOID; - DataFill((char *) td + hoff, - tupleDescriptor, + DataFill(tupleDescriptor, values, nulls, + (char *) td + hoff, + data_len, &td->t_infomask, (hasnull ? td->t_bits : NULL)); return tuple; } + /* * heap_modify_tuple * form a new tuple from an old tuple and a set of replacement values. @@ -1069,9 +1260,28 @@ heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; + else if (thisatt->attlen == -1) + { + /* + * We can only cache the offset for a varlena attribute + * if the offset is already suitably aligned, so that there + * would be no pad bytes in any case: then the offset will + * be valid for either an aligned or unaligned value. + */ + if (!slow && + off == att_align_nominal(off, thisatt->attalign)) + thisatt->attcacheoff = off; + else + { + off = att_align_pointer(off, thisatt->attalign, -1, + tp + off); + slow = true; + } + } else { - off = att_align(off, thisatt->attalign); + /* not varlena, so safe to use att_align_nominal */ + off = att_align_nominal(off, thisatt->attalign); if (!slow) thisatt->attcacheoff = off; @@ -1079,7 +1289,7 @@ heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, values[attnum] = fetchatt(thisatt, tp + off); - off = att_addlength(off, thisatt->attlen, tp + off); + off = att_addlength_pointer(off, thisatt->attlen, tp + off); if (thisatt->attlen <= 0) slow = true; /* can't use attcacheoff anymore */ @@ -1162,9 +1372,28 @@ heap_deformtuple(HeapTuple tuple, if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; + else if (thisatt->attlen == -1) + { + /* + * We can only cache the offset for a varlena attribute + * if the offset is already suitably aligned, so that there + * would be no pad bytes in any case: then the offset will + * be valid for either an aligned or unaligned value. + */ + if (!slow && + off == att_align_nominal(off, thisatt->attalign)) + thisatt->attcacheoff = off; + else + { + off = att_align_pointer(off, thisatt->attalign, -1, + tp + off); + slow = true; + } + } else { - off = att_align(off, thisatt->attalign); + /* not varlena, so safe to use att_align_nominal */ + off = att_align_nominal(off, thisatt->attalign); if (!slow) thisatt->attcacheoff = off; @@ -1172,7 +1401,7 @@ heap_deformtuple(HeapTuple tuple, values[attnum] = fetchatt(thisatt, tp + off); - off = att_addlength(off, thisatt->attlen, tp + off); + off = att_addlength_pointer(off, thisatt->attlen, tp + off); if (thisatt->attlen <= 0) slow = true; /* can't use attcacheoff anymore */ @@ -1252,9 +1481,28 @@ slot_deform_tuple(TupleTableSlot *slot, int natts) if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; + else if (thisatt->attlen == -1) + { + /* + * We can only cache the offset for a varlena attribute + * if the offset is already suitably aligned, so that there + * would be no pad bytes in any case: then the offset will + * be valid for either an aligned or unaligned value. + */ + if (!slow && + off == att_align_nominal(off, thisatt->attalign)) + thisatt->attcacheoff = off; + else + { + off = att_align_pointer(off, thisatt->attalign, -1, + tp + off); + slow = true; + } + } else { - off = att_align(off, thisatt->attalign); + /* not varlena, so safe to use att_align_nominal */ + off = att_align_nominal(off, thisatt->attalign); if (!slow) thisatt->attcacheoff = off; @@ -1262,7 +1510,7 @@ slot_deform_tuple(TupleTableSlot *slot, int natts) values[attnum] = fetchatt(thisatt, tp + off); - off = att_addlength(off, thisatt->attlen, tp + off); + off = att_addlength_pointer(off, thisatt->attlen, tp + off); if (thisatt->attlen <= 0) slow = true; /* can't use attcacheoff anymore */ @@ -1543,7 +1791,7 @@ heap_form_minimal_tuple(TupleDesc tupleDescriptor, bool *isnull) { MinimalTuple tuple; /* return tuple */ - unsigned long len; + Size len, data_len; int hoff; bool hasnull = false; Form_pg_attribute *att = tupleDescriptor->attrs; @@ -1595,7 +1843,9 @@ heap_form_minimal_tuple(TupleDesc tupleDescriptor, hoff = len = MAXALIGN(len); /* align user data safely */ - len += heap_compute_data_size(tupleDescriptor, values, isnull); + data_len = heap_compute_data_size(tupleDescriptor, values, isnull); + + len += data_len; /* * Allocate and zero the space needed. @@ -1616,6 +1866,7 @@ heap_form_minimal_tuple(TupleDesc tupleDescriptor, values, isnull, (char *) tuple + hoff, + data_len, &tuple->t_infomask, (hasnull ? tuple->t_bits : NULL)); diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c index c83e34834c..471d28c28c 100644 --- a/src/backend/access/common/indextuple.c +++ b/src/backend/access/common/indextuple.c @@ -9,7 +9,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/access/common/indextuple.c,v 1.81 2007/02/27 23:48:06 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/access/common/indextuple.c,v 1.82 2007/04/06 04:21:41 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -38,6 +38,7 @@ index_form_tuple(TupleDesc tupleDescriptor, char *tp; /* tuple pointer */ IndexTuple tuple; /* return tuple */ Size size, + data_size, hoff; int i; unsigned short infomask = 0; @@ -74,9 +75,9 @@ index_form_tuple(TupleDesc tupleDescriptor, */ if (VARATT_IS_EXTERNAL(values[i])) { - untoasted_values[i] = PointerGetDatum( - heap_tuple_fetch_attr( - (varattrib *) DatumGetPointer(values[i]))); + untoasted_values[i] = + PointerGetDatum(heap_tuple_fetch_attr((struct varlena *) + DatumGetPointer(values[i]))); untoasted_free[i] = true; } @@ -84,8 +85,8 @@ index_form_tuple(TupleDesc tupleDescriptor, * If value is above size target, and is of a compressible datatype, * try to compress it in-line. */ - if (VARSIZE(untoasted_values[i]) > TOAST_INDEX_TARGET && - !VARATT_IS_EXTENDED(untoasted_values[i]) && + if (!VARATT_IS_EXTENDED(untoasted_values[i]) && + VARSIZE(untoasted_values[i]) > TOAST_INDEX_TARGET && (att->attstorage == 'x' || att->attstorage == 'm')) { Datum cvalue = toast_compress_datum(untoasted_values[i]); @@ -116,12 +117,13 @@ index_form_tuple(TupleDesc tupleDescriptor, hoff = IndexInfoFindDataOffset(infomask); #ifdef TOAST_INDEX_HACK - size = hoff + heap_compute_data_size(tupleDescriptor, - untoasted_values, isnull); + data_size = heap_compute_data_size(tupleDescriptor, + untoasted_values, isnull); #else - size = hoff + heap_compute_data_size(tupleDescriptor, - values, isnull); + data_size = heap_compute_data_size(tupleDescriptor, + values, isnull); #endif + size = hoff + data_size; size = MAXALIGN(size); /* be conservative */ tp = (char *) palloc0(size); @@ -135,6 +137,7 @@ index_form_tuple(TupleDesc tupleDescriptor, #endif isnull, (char *) tp + hoff, + data_size, &tupmask, (hasnull ? (bits8 *) tp + sizeof(IndexTupleData) : NULL)); @@ -201,17 +204,14 @@ nocache_index_getattr(IndexTuple tup, bool *isnull) { Form_pg_attribute *att = tupleDesc->attrs; - char *tp; /* ptr to att in tuple */ - bits8 *bp = NULL; /* ptr to null bitmask in tuple */ - bool slow = false; /* do we have to walk nulls? */ + char *tp; /* ptr to data part of tuple */ + bits8 *bp = NULL; /* ptr to null bitmap in tuple */ + bool slow = false; /* do we have to walk attrs? */ int data_off; /* tuple data offset */ + int off; /* current offset within data */ (void) isnull; /* not used */ - /* - * sanity checks - */ - /* ---------------- * Three cases: * @@ -237,7 +237,7 @@ nocache_index_getattr(IndexTuple tup, { #ifdef IN_MACRO /* This is handled in the macro */ - if (att[attnum]->attcacheoff != -1) + if (att[attnum]->attcacheoff >= 0) { return fetchatt(att[attnum], (char *) tup + data_off + @@ -295,21 +295,28 @@ nocache_index_getattr(IndexTuple tup, tp = (char *) tup + data_off; - /* - * now check for any non-fixed length attrs before our attribute - */ if (!slow) { - if (att[attnum]->attcacheoff != -1) + /* + * If we get here, there are no nulls up to and including the target + * attribute. If we have a cached offset, we can use it. + */ + if (att[attnum]->attcacheoff >= 0) { return fetchatt(att[attnum], tp + att[attnum]->attcacheoff); } - else if (IndexTupleHasVarwidths(tup)) + + /* + * Otherwise, check for non-fixed-length attrs up to and including + * target. If there aren't any, it's safe to cheaply initialize + * the cached offsets for these attrs. + */ + if (IndexTupleHasVarwidths(tup)) { int j; - for (j = 0; j < attnum; j++) + for (j = 0; j <= attnum; j++) { if (att[j]->attlen <= 0) { @@ -320,80 +327,109 @@ nocache_index_getattr(IndexTuple tup, } } - /* - * If slow is false, and we got here, we know that we have a tuple with no - * nulls or var-widths before the target attribute. If possible, we also - * want to initialize the remainder of the attribute cached offset values. - */ if (!slow) { + int natts = tupleDesc->natts; int j = 1; - long off; /* - * need to set cache for some atts + * If we get here, we have a tuple with no nulls or var-widths up to + * and including the target attribute, so we can use the cached offset + * ... only we don't have it yet, or we'd not have got here. Since + * it's cheap to compute offsets for fixed-width columns, we take the + * opportunity to initialize the cached offsets for *all* the leading + * fixed-width columns, in hope of avoiding future visits to this + * routine. */ - att[0]->attcacheoff = 0; - while (j < attnum && att[j]->attcacheoff > 0) + /* we might have set some offsets in the slow path previously */ + while (j < natts && att[j]->attcacheoff > 0) j++; off = att[j - 1]->attcacheoff + att[j - 1]->attlen; - for (; j <= attnum; j++) + for (; j < natts; j++) { - off = att_align(off, att[j]->attalign); + if (att[j]->attlen <= 0) + break; + + off = att_align_nominal(off, att[j]->attalign); att[j]->attcacheoff = off; off += att[j]->attlen; } - return fetchatt(att[attnum], tp + att[attnum]->attcacheoff); + Assert(j > attnum); + + off = att[attnum]->attcacheoff; } else { bool usecache = true; - int off = 0; int i; /* - * Now we know that we have to walk the tuple CAREFULLY. + * Now we know that we have to walk the tuple CAREFULLY. But we + * still might be able to cache some offsets for next time. + * + * Note - This loop is a little tricky. For each non-null attribute, + * we have to first account for alignment padding before the attr, + * then advance over the attr based on its length. Nulls have no + * storage and no alignment padding either. We can use/set + * attcacheoff until we reach either a null or a var-width attribute. */ - - for (i = 0; i < attnum; i++) + off = 0; + for (i = 0; ; i++) /* loop exit is at "break" */ { - if (IndexTupleHasNulls(tup)) + if (IndexTupleHasNulls(tup) && att_isnull(i, bp)) { - if (att_isnull(i, bp)) - { - usecache = false; - continue; - } + usecache = false; + continue; /* this cannot be the target att */ } /* If we know the next offset, we can skip the rest */ - if (usecache && att[i]->attcacheoff != -1) + if (usecache && att[i]->attcacheoff >= 0) off = att[i]->attcacheoff; + else if (att[i]->attlen == -1) + { + /* + * We can only cache the offset for a varlena attribute + * if the offset is already suitably aligned, so that there + * would be no pad bytes in any case: then the offset will + * be valid for either an aligned or unaligned value. + */ + if (usecache && + off == att_align_nominal(off, att[i]->attalign)) + att[i]->attcacheoff = off; + else + { + off = att_align_pointer(off, att[i]->attalign, -1, + tp + off); + usecache = false; + } + } else { - off = att_align(off, att[i]->attalign); + /* not varlena, so safe to use att_align_nominal */ + off = att_align_nominal(off, att[i]->attalign); if (usecache) att[i]->attcacheoff = off; } - off = att_addlength(off, att[i]->attlen, tp + off); + if (i == attnum) + break; + + off = att_addlength_pointer(off, att[i]->attlen, tp + off); if (usecache && att[i]->attlen <= 0) usecache = false; } - - off = att_align(off, att[attnum]->attalign); - - return fetchatt(att[attnum], tp + off); } + + return fetchatt(att[attnum], tp + off); } /* diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index 334d670042..1a3c01bcac 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.73 2007/04/03 04:14:26 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.74 2007/04/06 04:21:41 tgl Exp $ * * * INTERFACE ROUTINES @@ -42,25 +42,39 @@ #undef TOAST_DEBUG +/* + * Testing whether an externally-stored value is compressed now requires + * comparing extsize (the actual length of the external data) to rawsize + * (the original uncompressed datum's size). The latter includes VARHDRSZ + * overhead, the former doesn't. We never use compression unless it actually + * saves space, so we expect either equality or less-than. + */ +#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ + ((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ) + static void toast_delete_datum(Relation rel, Datum value); static Datum toast_save_datum(Relation rel, Datum value, bool use_wal, bool use_fsm); -static varattrib *toast_fetch_datum(varattrib *attr); -static varattrib *toast_fetch_datum_slice(varattrib *attr, +static struct varlena *toast_fetch_datum(struct varlena *attr); +static struct varlena *toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length); /* ---------- * heap_tuple_fetch_attr - * - * Public entry point to get back a toasted value + * Public entry point to get back a toasted value from * external storage (possibly still in compressed format). - * ---------- + * + * This will return a datum that contains all the data internally, ie, not + * relying on external storage, but it can still be compressed or have a short + * header. + ---------- */ -varattrib * -heap_tuple_fetch_attr(varattrib *attr) +struct varlena * +heap_tuple_fetch_attr(struct varlena *attr) { - varattrib *result; + struct varlena *result; if (VARATT_IS_EXTERNAL(attr)) { @@ -88,35 +102,25 @@ heap_tuple_fetch_attr(varattrib *attr) * or external storage. * ---------- */ -varattrib * -heap_tuple_untoast_attr(varattrib *attr) +struct varlena * +heap_tuple_untoast_attr(struct varlena *attr) { - varattrib *result; - if (VARATT_IS_EXTERNAL(attr)) { + /* + * This is an externally stored datum --- fetch it back from there + */ + attr = toast_fetch_datum(attr); + /* If it's compressed, decompress it */ if (VARATT_IS_COMPRESSED(attr)) { - /* ---------- - * This is an external stored compressed value - * Fetch it from the toast heap and decompress. - * ---------- - */ - PGLZ_Header *tmp; + PGLZ_Header *tmp = (PGLZ_Header *) attr; - tmp = (PGLZ_Header *) toast_fetch_datum(attr); - result = (varattrib *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - SET_VARSIZE(result, PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - pglz_decompress(tmp, VARDATA(result)); + attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ); + SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ); + pglz_decompress(tmp, VARDATA(attr)); pfree(tmp); } - else - { - /* - * This is an external stored plain value - */ - result = toast_fetch_datum(attr); - } } else if (VARATT_IS_COMPRESSED(attr)) { @@ -125,18 +129,26 @@ heap_tuple_untoast_attr(varattrib *attr) */ PGLZ_Header *tmp = (PGLZ_Header *) attr; - result = (varattrib *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - SET_VARSIZE(result, PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - pglz_decompress(tmp, VARDATA(result)); + attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ); + SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ); + pglz_decompress(tmp, VARDATA(attr)); } - else - + else if (VARATT_IS_SHORT(attr)) + { /* - * This is a plain value inside of the main tuple - why am I called? + * This is a short-header varlena --- convert to 4-byte header format */ - return attr; + Size data_size = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT; + Size new_size = data_size + VARHDRSZ; + struct varlena *new_attr; - return result; + new_attr = (struct varlena *) palloc(new_size); + SET_VARSIZE(new_attr, new_size); + memcpy(VARDATA(new_attr), VARDATA_SHORT(attr), data_size); + attr = new_attr; + } + + return attr; } @@ -147,44 +159,57 @@ heap_tuple_untoast_attr(varattrib *attr) * from compression or external storage. * ---------- */ -varattrib * -heap_tuple_untoast_attr_slice(varattrib *attr, int32 sliceoffset, int32 slicelength) +struct varlena * +heap_tuple_untoast_attr_slice(struct varlena *attr, + int32 sliceoffset, int32 slicelength) { - varattrib *preslice; - varattrib *result; + struct varlena *preslice; + struct varlena *result; + char *attrdata; int32 attrsize; - if (VARATT_IS_COMPRESSED(attr)) + if (VARATT_IS_EXTERNAL(attr)) { - PGLZ_Header *tmp; + struct varatt_external toast_pointer; - if (VARATT_IS_EXTERNAL(attr)) - tmp = (PGLZ_Header *) toast_fetch_datum(attr); - else - tmp = (PGLZ_Header *) attr; /* compressed in main tuple */ + memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer)); - preslice = (varattrib *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - SET_VARSIZE(preslice, PGLZ_RAW_SIZE(tmp) + VARHDRSZ); + /* fast path for non-compressed external datums */ + if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + return toast_fetch_datum_slice(attr, sliceoffset, slicelength); + + /* fetch it back (compressed marker will get set automatically) */ + preslice = toast_fetch_datum(attr); + } + else + preslice = attr; + + if (VARATT_IS_COMPRESSED(preslice)) + { + PGLZ_Header *tmp = (PGLZ_Header *) preslice; + Size size = PGLZ_RAW_SIZE(tmp) + VARHDRSZ; + + preslice = (struct varlena *) palloc(size); + SET_VARSIZE(preslice, size); pglz_decompress(tmp, VARDATA(preslice)); if (tmp != (PGLZ_Header *) attr) pfree(tmp); } + + if (VARATT_IS_SHORT(preslice)) + { + attrdata = VARDATA_SHORT(preslice); + attrsize = VARSIZE_SHORT(preslice) - VARHDRSZ_SHORT; + } else { - /* Plain value */ - if (VARATT_IS_EXTERNAL(attr)) - { - /* fast path */ - return toast_fetch_datum_slice(attr, sliceoffset, slicelength); - } - else - preslice = attr; + attrdata = VARDATA(preslice); + attrsize = VARSIZE(preslice) - VARHDRSZ; } /* slicing of datum for compressed cases and plain value */ - attrsize = VARSIZE(preslice) - VARHDRSZ; if (sliceoffset >= attrsize) { sliceoffset = 0; @@ -194,10 +219,10 @@ heap_tuple_untoast_attr_slice(varattrib *attr, int32 sliceoffset, int32 slicelen if (((sliceoffset + slicelength) > attrsize) || slicelength < 0) slicelength = attrsize - sliceoffset; - result = (varattrib *) palloc(slicelength + VARHDRSZ); + result = (struct varlena *) palloc(slicelength + VARHDRSZ); SET_VARSIZE(result, slicelength + VARHDRSZ); - memcpy(VARDATA(result), VARDATA(preslice) + sliceoffset, slicelength); + memcpy(VARDATA(result), attrdata + sliceoffset, slicelength); if (preslice != attr) pfree(preslice); @@ -210,29 +235,35 @@ heap_tuple_untoast_attr_slice(varattrib *attr, int32 sliceoffset, int32 slicelen * toast_raw_datum_size - * * Return the raw (detoasted) size of a varlena datum + * (including the VARHDRSZ header) * ---------- */ Size toast_raw_datum_size(Datum value) { - varattrib *attr = (varattrib *) DatumGetPointer(value); + struct varlena *attr = (struct varlena *) DatumGetPointer(value); Size result; - if (VARATT_IS_COMPRESSED(attr)) + if (VARATT_IS_EXTERNAL(attr)) { - /* - * va_rawsize shows the original data size, whether the datum is - * external or not. - */ - result = attr->va_content.va_compressed.va_rawsize + VARHDRSZ; + /* va_rawsize is the size of the original datum -- including header */ + struct varatt_external toast_pointer; + + memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer)); + result = toast_pointer.va_rawsize; } - else if (VARATT_IS_EXTERNAL(attr)) + else if (VARATT_IS_COMPRESSED(attr)) + { + /* here, va_rawsize is just the payload size */ + result = VARRAWSIZE_4B_C(attr) + VARHDRSZ; + } + else if (VARATT_IS_SHORT(attr)) { /* - * an uncompressed external attribute has rawsize including the header - * (not too consistent!) + * we have to normalize the header length to VARHDRSZ or else the + * callers of this function will be confused. */ - result = attr->va_content.va_external.va_rawsize; + result = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT + VARHDRSZ; } else { @@ -251,7 +282,7 @@ toast_raw_datum_size(Datum value) Size toast_datum_size(Datum value) { - varattrib *attr = (varattrib *) DatumGetPointer(value); + struct varlena *attr = (struct varlena *) DatumGetPointer(value); Size result; if (VARATT_IS_EXTERNAL(attr)) @@ -261,7 +292,14 @@ toast_datum_size(Datum value) * compressed or not. We do not count the size of the toast pointer * ... should we? */ - result = attr->va_content.va_external.va_extsize; + struct varatt_external toast_pointer; + + memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer)); + result = toast_pointer.va_extsize; + } + else if (VARATT_IS_SHORT(attr)) + { + result = VARSIZE_SHORT(attr); } else { @@ -413,16 +451,16 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, for (i = 0; i < numAttrs; i++) { - varattrib *old_value; - varattrib *new_value; + struct varlena *old_value; + struct varlena *new_value; if (oldtup != NULL) { /* * For UPDATE get the old and new values of this attribute */ - old_value = (varattrib *) DatumGetPointer(toast_oldvalues[i]); - new_value = (varattrib *) DatumGetPointer(toast_values[i]); + old_value = (struct varlena *) DatumGetPointer(toast_oldvalues[i]); + new_value = (struct varlena *) DatumGetPointer(toast_values[i]); /* * If the old value is an external stored one, check if it has @@ -432,10 +470,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, VARATT_IS_EXTERNAL(old_value)) { if (toast_isnull[i] || !VARATT_IS_EXTERNAL(new_value) || - old_value->va_content.va_external.va_valueid != - new_value->va_content.va_external.va_valueid || - old_value->va_content.va_external.va_toastrelid != - new_value->va_content.va_external.va_toastrelid) + memcmp(VARDATA_SHORT(old_value), + VARDATA_SHORT(new_value), + sizeof(struct varatt_external)) != 0) { /* * The old external stored value isn't needed any more @@ -452,7 +489,6 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, * tuple. */ toast_action[i] = 'p'; - toast_sizes[i] = VARSIZE(toast_values[i]); continue; } } @@ -462,7 +498,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, /* * For INSERT simply get the new value */ - new_value = (varattrib *) DatumGetPointer(toast_values[i]); + new_value = (struct varlena *) DatumGetPointer(toast_values[i]); } /* @@ -503,7 +539,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, /* * Remember the size of this attribute */ - toast_sizes[i] = VARSIZE(new_value); + toast_sizes[i] = VARSIZE_ANY(new_value); } else { @@ -542,7 +578,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, toast_values, toast_isnull) > maxDataLen) { int biggest_attno = -1; - int32 biggest_size = MAXALIGN(sizeof(varattrib)); + int32 biggest_size = MAXALIGN(sizeof(varattrib_pointer)); Datum old_value; Datum new_value; @@ -553,7 +589,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, { if (toast_action[i] != ' ') continue; - if (VARATT_IS_EXTENDED(toast_values[i])) + if (VARATT_IS_EXTERNAL(toast_values[i])) + continue; + if (VARATT_IS_COMPRESSED(toast_values[i])) continue; if (att[i]->attstorage != 'x') continue; @@ -603,7 +641,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, rel->rd_rel->reltoastrelid != InvalidOid) { int biggest_attno = -1; - int32 biggest_size = MAXALIGN(sizeof(varattrib)); + int32 biggest_size = MAXALIGN(sizeof(varattrib_pointer)); Datum old_value; /*------ @@ -639,9 +677,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, use_wal, use_fsm); if (toast_free[i]) pfree(DatumGetPointer(old_value)); - toast_free[i] = true; - toast_sizes[i] = VARSIZE(toast_values[i]); need_change = true; need_free = true; @@ -655,7 +691,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, toast_values, toast_isnull) > maxDataLen) { int biggest_attno = -1; - int32 biggest_size = MAXALIGN(sizeof(varattrib)); + int32 biggest_size = MAXALIGN(sizeof(varattrib_pointer)); Datum old_value; Datum new_value; @@ -666,7 +702,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, { if (toast_action[i] != ' ') continue; - if (VARATT_IS_EXTENDED(toast_values[i])) + if (VARATT_IS_EXTERNAL(toast_values[i])) + continue; + if (VARATT_IS_COMPRESSED(toast_values[i])) continue; if (att[i]->attstorage != 'm') continue; @@ -715,7 +753,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, rel->rd_rel->reltoastrelid != InvalidOid) { int biggest_attno = -1; - int32 biggest_size = MAXALIGN(sizeof(varattrib)); + int32 biggest_size = MAXALIGN(sizeof(varattrib_pointer)); Datum old_value; /*-------- @@ -768,6 +806,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, HeapTupleHeader olddata = newtup->t_data; HeapTupleHeader new_data; int32 new_len; + int32 new_data_len; /* * Calculate the new size of the tuple. Header size should not @@ -780,8 +819,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, new_len += sizeof(Oid); new_len = MAXALIGN(new_len); Assert(new_len == olddata->t_hoff); - new_len += heap_compute_data_size(tupleDesc, - toast_values, toast_isnull); + new_data_len = heap_compute_data_size(tupleDesc, + toast_values, toast_isnull); + new_len += new_data_len; /* * Allocate and zero the space needed, and fill HeapTupleData fields. @@ -802,6 +842,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, toast_values, toast_isnull, (char *) new_data + olddata->t_hoff, + new_data_len, &(new_data->t_infomask), has_nulls ? new_data->t_bits : NULL); } @@ -835,6 +876,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, * This must be invoked on any potentially-composite field that is to be * inserted into a tuple. Doing this preserves the invariant that toasting * goes only one level deep in a tuple. + * + * Note that flattening does not mean expansion of short-header varlenas, + * so in one sense toasting is allowed within composite datums. * ---------- */ Datum @@ -845,6 +889,7 @@ toast_flatten_tuple_attribute(Datum value, HeapTupleHeader olddata; HeapTupleHeader new_data; int32 new_len; + int32 new_data_len; HeapTupleData tmptup; Form_pg_attribute *att; int numAttrs; @@ -891,10 +936,11 @@ toast_flatten_tuple_attribute(Datum value, has_nulls = true; else if (att[i]->attlen == -1) { - varattrib *new_value; + struct varlena *new_value; - new_value = (varattrib *) DatumGetPointer(toast_values[i]); - if (VARATT_IS_EXTENDED(new_value)) + new_value = (struct varlena *) DatumGetPointer(toast_values[i]); + if (VARATT_IS_EXTERNAL(new_value) || + VARATT_IS_COMPRESSED(new_value)) { new_value = heap_tuple_untoast_attr(new_value); toast_values[i] = PointerGetDatum(new_value); @@ -924,7 +970,9 @@ toast_flatten_tuple_attribute(Datum value, new_len += sizeof(Oid); new_len = MAXALIGN(new_len); Assert(new_len == olddata->t_hoff); - new_len += heap_compute_data_size(tupleDesc, toast_values, toast_isnull); + new_data_len = heap_compute_data_size(tupleDesc, + toast_values, toast_isnull); + new_len += new_data_len; new_data = (HeapTupleHeader) palloc0(new_len); @@ -939,6 +987,7 @@ toast_flatten_tuple_attribute(Datum value, toast_values, toast_isnull, (char *) new_data + olddata->t_hoff, + new_data_len, &(new_data->t_infomask), has_nulls ? new_data->t_bits : NULL); @@ -962,21 +1011,26 @@ toast_flatten_tuple_attribute(Datum value, * If we fail (ie, compressed result is actually bigger than original) * then return NULL. We must not use compressed data if it'd expand * the tuple! + * + * We use VAR{SIZE,DATA}_ANY so we can handle short varlenas here without + * copying them. But we can't handle external or compressed datums. * ---------- */ Datum toast_compress_datum(Datum value) { - varattrib *tmp; - int32 valsize = VARSIZE(value) - VARHDRSZ; + struct varlena *tmp; + int32 valsize = VARSIZE_ANY_EXHDR(value); - tmp = (varattrib *) palloc(PGLZ_MAX_OUTPUT(valsize)); - if (pglz_compress(VARDATA(value), valsize, + Assert(!VARATT_IS_EXTERNAL(value)); + Assert(!VARATT_IS_COMPRESSED(value)); + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize)); + if (pglz_compress(VARDATA_ANY(value), valsize, (PGLZ_Header *) tmp, PGLZ_strategy_default) && - VARSIZE(tmp) < VARSIZE(value)) + VARSIZE(tmp) < VARSIZE_ANY(value)) { /* successful compression */ - VARATT_SIZEP_DEPRECATED(tmp) |= VARATT_FLAG_COMPRESSED; return PointerGetDatum(tmp); } else @@ -992,7 +1046,7 @@ toast_compress_datum(Datum value) * toast_save_datum - * * Save one single datum into the secondary relation and return - * a varattrib reference for it. + * a Datum reference for it. * ---------- */ static Datum @@ -1006,7 +1060,8 @@ toast_save_datum(Relation rel, Datum value, Datum t_values[3]; bool t_isnull[3]; CommandId mycid = GetCurrentCommandId(); - varattrib *result; + struct varlena *result; + struct varatt_external toast_pointer; struct { struct varlena hdr; @@ -1027,43 +1082,50 @@ toast_save_datum(Relation rel, Datum value, toastidx = index_open(toastrel->rd_rel->reltoastidxid, RowExclusiveLock); /* - * Create the varattrib reference + * Get the data pointer and length, and compute va_rawsize and va_extsize. + * + * va_rawsize is the size of the equivalent fully uncompressed datum, + * so we have to adjust for short headers. + * + * va_extsize is the actual size of the data payload in the toast records. */ - result = (varattrib *) palloc(sizeof(varattrib)); - - SET_VARSIZE(result, sizeof(varattrib)); - VARATT_SIZEP_DEPRECATED(result) |= VARATT_FLAG_EXTERNAL; - if (VARATT_IS_COMPRESSED(value)) + if (VARATT_IS_SHORT(value)) { - VARATT_SIZEP_DEPRECATED(result) |= VARATT_FLAG_COMPRESSED; - result->va_content.va_external.va_rawsize = - ((varattrib *) value)->va_content.va_compressed.va_rawsize; + data_p = VARDATA_SHORT(value); + data_todo = VARSIZE_SHORT(value) - VARHDRSZ_SHORT; + toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */ + toast_pointer.va_extsize = data_todo; + } + else if (VARATT_IS_COMPRESSED(value)) + { + data_p = VARDATA(value); + data_todo = VARSIZE(value) - VARHDRSZ; + /* rawsize in a compressed datum is just the size of the payload */ + toast_pointer.va_rawsize = VARRAWSIZE_4B_C(value) + VARHDRSZ; + toast_pointer.va_extsize = data_todo; + /* Assert that the numbers look like it's compressed */ + Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)); } else - result->va_content.va_external.va_rawsize = VARSIZE(value); + { + data_p = VARDATA(value); + data_todo = VARSIZE(value) - VARHDRSZ; + toast_pointer.va_rawsize = VARSIZE(value); + toast_pointer.va_extsize = data_todo; + } - result->va_content.va_external.va_extsize = - VARSIZE(value) - VARHDRSZ; - result->va_content.va_external.va_valueid = - GetNewOidWithIndex(toastrel, toastidx); - result->va_content.va_external.va_toastrelid = - rel->rd_rel->reltoastrelid; + toast_pointer.va_valueid = GetNewOidWithIndex(toastrel, toastidx); + toast_pointer.va_toastrelid = rel->rd_rel->reltoastrelid; /* * Initialize constant parts of the tuple data */ - t_values[0] = ObjectIdGetDatum(result->va_content.va_external.va_valueid); + t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid); t_values[2] = PointerGetDatum(&chunk_data); t_isnull[0] = false; t_isnull[1] = false; t_isnull[2] = false; - /* - * Get the data to process - */ - data_p = VARDATA(value); - data_todo = VARSIZE(value) - VARHDRSZ; - /* * Split up the item into chunks */ @@ -1111,11 +1173,18 @@ toast_save_datum(Relation rel, Datum value, } /* - * Done - close toast relation and return the reference + * Done - close toast relation */ index_close(toastidx, RowExclusiveLock); heap_close(toastrel, RowExclusiveLock); + /* + * Create the TOAST pointer value that we'll return + */ + result = (struct varlena *) palloc(sizeof(varattrib_pointer)); + SET_VARSIZE_EXTERNAL(result); + memcpy(VARDATA_SHORT(result), &toast_pointer, sizeof(toast_pointer)); + return PointerGetDatum(result); } @@ -1129,7 +1198,8 @@ toast_save_datum(Relation rel, Datum value, static void toast_delete_datum(Relation rel, Datum value) { - varattrib *attr = (varattrib *) DatumGetPointer(value); + struct varlena *attr = (struct varlena *) DatumGetPointer(value); + struct varatt_external toast_pointer; Relation toastrel; Relation toastidx; ScanKeyData toastkey; @@ -1139,11 +1209,14 @@ toast_delete_datum(Relation rel, Datum value) if (!VARATT_IS_EXTERNAL(attr)) return; + /* Must copy to access aligned fields */ + memcpy(&toast_pointer, VARDATA_SHORT(attr), + sizeof(struct varatt_external)); + /* * Open the toast relation and its index */ - toastrel = heap_open(attr->va_content.va_external.va_toastrelid, - RowExclusiveLock); + toastrel = heap_open(toast_pointer.va_toastrelid, RowExclusiveLock); toastidx = index_open(toastrel->rd_rel->reltoastidxid, RowExclusiveLock); /* @@ -1153,7 +1226,7 @@ toast_delete_datum(Relation rel, Datum value) ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(attr->va_content.va_external.va_valueid)); + ObjectIdGetDatum(toast_pointer.va_valueid)); /* * Find the chunks by index @@ -1180,12 +1253,12 @@ toast_delete_datum(Relation rel, Datum value) /* ---------- * toast_fetch_datum - * - * Reconstruct an in memory varattrib from the chunks saved + * Reconstruct an in memory Datum from the chunks saved * in the toast relation * ---------- */ -static varattrib * -toast_fetch_datum(varattrib *attr) +static struct varlena * +toast_fetch_datum(struct varlena *attr) { Relation toastrel; Relation toastidx; @@ -1193,28 +1266,35 @@ toast_fetch_datum(varattrib *attr) IndexScanDesc toastscan; HeapTuple ttup; TupleDesc toasttupDesc; - varattrib *result; + struct varlena *result; + struct varatt_external toast_pointer; int32 ressize; int32 residx, nextidx; int32 numchunks; Pointer chunk; bool isnull; + char *chunkdata; int32 chunksize; - ressize = attr->va_content.va_external.va_extsize; + /* Must copy to access aligned fields */ + memcpy(&toast_pointer, VARDATA_SHORT(attr), + sizeof(struct varatt_external)); + + ressize = toast_pointer.va_extsize; numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; - result = (varattrib *) palloc(ressize + VARHDRSZ); - SET_VARSIZE(result, ressize + VARHDRSZ); - if (VARATT_IS_COMPRESSED(attr)) - VARATT_SIZEP_DEPRECATED(result) |= VARATT_FLAG_COMPRESSED; + result = (struct varlena *) palloc(ressize + VARHDRSZ); + + if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + SET_VARSIZE_COMPRESSED(result, ressize + VARHDRSZ); + else + SET_VARSIZE(result, ressize + VARHDRSZ); /* * Open the toast relation and its index */ - toastrel = heap_open(attr->va_content.va_external.va_toastrelid, - AccessShareLock); + toastrel = heap_open(toast_pointer.va_toastrelid, AccessShareLock); toasttupDesc = toastrel->rd_att; toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock); @@ -1224,7 +1304,7 @@ toast_fetch_datum(varattrib *attr) ScanKeyInit(&toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(attr->va_content.va_external.va_valueid)); + ObjectIdGetDatum(toast_pointer.va_valueid)); /* * Read the chunks by index @@ -1246,7 +1326,24 @@ toast_fetch_datum(varattrib *attr) Assert(!isnull); chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull)); Assert(!isnull); - chunksize = VARSIZE(chunk) - VARHDRSZ; + if (!VARATT_IS_EXTENDED(chunk)) + { + chunksize = VARSIZE(chunk) - VARHDRSZ; + chunkdata = VARDATA(chunk); + } + else if (VARATT_IS_SHORT(chunk)) + { + /* could happen due to heap_form_tuple doing its thing */ + chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT; + chunkdata = VARDATA_SHORT(chunk); + } + else + { + /* should never happen */ + elog(ERROR, "found toasted toast chunk"); + chunksize = 0; /* keep compiler quiet */ + chunkdata = NULL; + } /* * Some checks on the data we've found @@ -1254,31 +1351,35 @@ toast_fetch_datum(varattrib *attr) if (residx != nextidx) elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u", residx, nextidx, - attr->va_content.va_external.va_valueid); + toast_pointer.va_valueid); if (residx < numchunks - 1) { if (chunksize != TOAST_MAX_CHUNK_SIZE) - elog(ERROR, "unexpected chunk size %d in chunk %d for toast value %u", - chunksize, residx, - attr->va_content.va_external.va_valueid); + elog(ERROR, "unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u", + chunksize, (int) TOAST_MAX_CHUNK_SIZE, + residx, numchunks, + toast_pointer.va_valueid); } - else if (residx < numchunks) + else if (residx == numchunks-1) { if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != ressize) - elog(ERROR, "unexpected chunk size %d in chunk %d for toast value %u", - chunksize, residx, - attr->va_content.va_external.va_valueid); + elog(ERROR, "unexpected chunk size %d (expected %d) in final chunk %d for toast value %u", + chunksize, + (int) (ressize - residx*TOAST_MAX_CHUNK_SIZE), + residx, + toast_pointer.va_valueid); } else - elog(ERROR, "unexpected chunk number %d for toast value %u", + elog(ERROR, "unexpected chunk number %d for toast value %u (out of range %d..%d)", residx, - attr->va_content.va_external.va_valueid); + toast_pointer.va_valueid, + 0, numchunks-1); /* * Copy the data into proper place in our result */ memcpy(VARDATA(result) + residx * TOAST_MAX_CHUNK_SIZE, - VARDATA(chunk), + chunkdata, chunksize); nextidx++; @@ -1290,7 +1391,7 @@ toast_fetch_datum(varattrib *attr) if (nextidx != numchunks) elog(ERROR, "missing chunk number %d for toast value %u", nextidx, - attr->va_content.va_external.va_valueid); + toast_pointer.va_valueid); /* * End scan and close relations @@ -1305,12 +1406,12 @@ toast_fetch_datum(varattrib *attr) /* ---------- * toast_fetch_datum_slice - * - * Reconstruct a segment of a varattrib from the chunks saved + * Reconstruct a segment of a Datum from the chunks saved * in the toast relation * ---------- */ -static varattrib * -toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) +static struct varlena * +toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length) { Relation toastrel; Relation toastidx; @@ -1319,7 +1420,8 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) IndexScanDesc toastscan; HeapTuple ttup; TupleDesc toasttupDesc; - varattrib *result; + struct varlena *result; + struct varatt_external toast_pointer; int32 attrsize; int32 residx; int32 nextidx; @@ -1331,11 +1433,16 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) int totalchunks; Pointer chunk; bool isnull; + char *chunkdata; int32 chunksize; int32 chcpystrt; int32 chcpyend; - attrsize = attr->va_content.va_external.va_extsize; + /* Must copy to access aligned fields */ + memcpy(&toast_pointer, VARDATA_SHORT(attr), + sizeof(struct varatt_external)); + + attrsize = toast_pointer.va_extsize; totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; if (sliceoffset >= attrsize) @@ -1347,11 +1454,12 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) if (((sliceoffset + length) > attrsize) || length < 0) length = attrsize - sliceoffset; - result = (varattrib *) palloc(length + VARHDRSZ); - SET_VARSIZE(result, length + VARHDRSZ); + result = (struct varlena *) palloc(length + VARHDRSZ); - if (VARATT_IS_COMPRESSED(attr)) - VARATT_SIZEP_DEPRECATED(result) |= VARATT_FLAG_COMPRESSED; + if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) + SET_VARSIZE_COMPRESSED(result, length + VARHDRSZ); + else + SET_VARSIZE(result, length + VARHDRSZ); if (length == 0) return result; /* Can save a lot of work at this point! */ @@ -1366,8 +1474,7 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) /* * Open the toast relation and its index */ - toastrel = heap_open(attr->va_content.va_external.va_toastrelid, - AccessShareLock); + toastrel = heap_open(toast_pointer.va_toastrelid, AccessShareLock); toasttupDesc = toastrel->rd_att; toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock); @@ -1378,7 +1485,7 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) ScanKeyInit(&toastkey[0], (AttrNumber) 1, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(attr->va_content.va_external.va_valueid)); + ObjectIdGetDatum(toast_pointer.va_valueid)); /* * Use equality condition for one chunk, a range condition otherwise: @@ -1421,7 +1528,24 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) Assert(!isnull); chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull)); Assert(!isnull); - chunksize = VARSIZE(chunk) - VARHDRSZ; + if (!VARATT_IS_EXTENDED(chunk)) + { + chunksize = VARSIZE(chunk) - VARHDRSZ; + chunkdata = VARDATA(chunk); + } + else if (VARATT_IS_SHORT(chunk)) + { + /* could happen due to heap_form_tuple doing its thing */ + chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT; + chunkdata = VARDATA_SHORT(chunk); + } + else + { + /* should never happen */ + elog(ERROR, "found toasted toast chunk"); + chunksize = 0; /* keep compiler quiet */ + chunkdata = NULL; + } /* * Some checks on the data we've found @@ -1429,21 +1553,29 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) if ((residx != nextidx) || (residx > endchunk) || (residx < startchunk)) elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u", residx, nextidx, - attr->va_content.va_external.va_valueid); + toast_pointer.va_valueid); if (residx < totalchunks - 1) { if (chunksize != TOAST_MAX_CHUNK_SIZE) - elog(ERROR, "unexpected chunk size %d in chunk %d for toast value %u", - chunksize, residx, - attr->va_content.va_external.va_valueid); + elog(ERROR, "unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u when fetching slice", + chunksize, (int) TOAST_MAX_CHUNK_SIZE, + residx, totalchunks, + toast_pointer.va_valueid); } - else + else if (residx == totalchunks-1) { if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != attrsize) - elog(ERROR, "unexpected chunk size %d in chunk %d for toast value %u", - chunksize, residx, - attr->va_content.va_external.va_valueid); + elog(ERROR, "unexpected chunk size %d (expected %d) in final chunk %d for toast value %u when fetching slice", + chunksize, + (int) (attrsize - residx * TOAST_MAX_CHUNK_SIZE), + residx, + toast_pointer.va_valueid); } + else + elog(ERROR, "unexpected chunk number %d for toast value %u (out of range %d..%d)", + residx, + toast_pointer.va_valueid, + 0, totalchunks-1); /* * Copy the data into proper place in our result @@ -1457,7 +1589,7 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) memcpy(VARDATA(result) + (residx * TOAST_MAX_CHUNK_SIZE - sliceoffset) + chcpystrt, - VARDATA(chunk) + chcpystrt, + chunkdata + chcpystrt, (chcpyend - chcpystrt) + 1); nextidx++; @@ -1469,7 +1601,7 @@ toast_fetch_datum_slice(varattrib *attr, int32 sliceoffset, int32 length) if (nextidx != (endchunk + 1)) elog(ERROR, "missing chunk number %d for toast value %u", nextidx, - attr->va_content.va_external.va_valueid); + toast_pointer.va_valueid); /* * End scan and close relations diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 47f5e25fd2..463f038c0f 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -8,7 +8,7 @@ * Portions Copyright (c) 1994, Regents of the University of California * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/catalog/toasting.c,v 1.5 2007/01/09 02:14:11 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/catalog/toasting.c,v 1.6 2007/04/06 04:21:42 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -314,7 +314,7 @@ needs_toast_table(Relation rel) { if (att[i]->attisdropped) continue; - data_length = att_align(data_length, att[i]->attalign); + data_length = att_align_nominal(data_length, att[i]->attalign); if (att[i]->attlen > 0) { /* Fixed-length types are never toastable */ diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index c568f04284..44e743363a 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/analyze.c,v 1.103 2007/01/09 02:14:11 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/commands/analyze.c,v 1.104 2007/04/06 04:21:42 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -1478,7 +1478,7 @@ compute_minimal_stats(VacAttrStatsP stats, */ if (is_varlena) { - total_width += VARSIZE(DatumGetPointer(value)); + total_width += VARSIZE_ANY(DatumGetPointer(value)); /* * If the value is toasted, we want to detoast it just once to @@ -1792,7 +1792,7 @@ compute_scalar_stats(VacAttrStatsP stats, */ if (is_varlena) { - total_width += VARSIZE(DatumGetPointer(value)); + total_width += VARSIZE_ANY(DatumGetPointer(value)); /* * If the value is toasted, we want to detoast it just once to diff --git a/src/backend/executor/execQual.c b/src/backend/executor/execQual.c index 94e6829f54..50ab3ada23 100644 --- a/src/backend/executor/execQual.c +++ b/src/backend/executor/execQual.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/executor/execQual.c,v 1.216 2007/03/27 23:21:08 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/executor/execQual.c,v 1.217 2007/04/06 04:21:42 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -1896,8 +1896,8 @@ ExecEvalScalarArrayOp(ScalarArrayOpExprState *sstate, else { elt = fetch_att(s, typbyval, typlen); - s = att_addlength(s, typlen, PointerGetDatum(s)); - s = (char *) att_align(s, typalign); + s = att_addlength_pointer(s, typlen, s); + s = (char *) att_align_nominal(s, typalign); fcinfo.arg[1] = elt; fcinfo.argnull[1] = false; } diff --git a/src/backend/storage/large_object/inv_api.c b/src/backend/storage/large_object/inv_api.c index 7becf160b4..ea98ab2952 100644 --- a/src/backend/storage/large_object/inv_api.c +++ b/src/backend/storage/large_object/inv_api.c @@ -17,7 +17,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/storage/large_object/inv_api.c,v 1.123 2007/03/03 19:52:46 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/storage/large_object/inv_api.c,v 1.124 2007/04/06 04:21:42 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -336,7 +336,7 @@ inv_getsize(LargeObjectDesc *obj_desc) if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) - heap_tuple_untoast_attr((varattrib *) datafield); + heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } lastbyte = data->pageno * LOBLKSIZE + getbytealen(datafield); @@ -462,7 +462,7 @@ inv_read(LargeObjectDesc *obj_desc, char *buf, int nbytes) if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) - heap_tuple_untoast_attr((varattrib *) datafield); + heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } len = getbytealen(datafield); @@ -580,7 +580,7 @@ inv_write(LargeObjectDesc *obj_desc, const char *buf, int nbytes) if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) - heap_tuple_untoast_attr((varattrib *) datafield); + heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } len = getbytealen(datafield); @@ -756,7 +756,7 @@ inv_truncate(LargeObjectDesc *obj_desc, int len) if (VARATT_IS_EXTENDED(datafield)) { datafield = (bytea *) - heap_tuple_untoast_attr((varattrib *) datafield); + heap_tuple_untoast_attr((struct varlena *) datafield); pfreeit = true; } pagelen = getbytealen(datafield); diff --git a/src/backend/utils/adt/arrayfuncs.c b/src/backend/utils/adt/arrayfuncs.c index 38a86452e3..991c7a46f7 100644 --- a/src/backend/utils/adt/arrayfuncs.c +++ b/src/backend/utils/adt/arrayfuncs.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/arrayfuncs.c,v 1.138 2007/03/27 23:21:10 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/arrayfuncs.c,v 1.139 2007/04/06 04:21:42 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -805,8 +805,8 @@ ReadArrayStr(char *arrayStr, /* let's just make sure data is not toasted */ if (typlen == -1) values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i])); - totbytes = att_addlength(totbytes, typlen, values[i]); - totbytes = att_align(totbytes, typalign); + totbytes = att_addlength_datum(totbytes, typlen, values[i]); + totbytes = att_align_nominal(totbytes, typalign); /* check for overflow of total request */ if (!AllocSizeIsValid(totbytes)) ereport(ERROR, @@ -1011,8 +1011,8 @@ array_out(PG_FUNCTION_ARGS) itemvalue = fetch_att(p, typbyval, typlen); values[i] = OutputFunctionCall(&my_extra->proc, itemvalue); - p = att_addlength(p, typlen, PointerGetDatum(p)); - p = (char *) att_align(p, typalign); + p = att_addlength_pointer(p, typlen, p); + p = (char *) att_align_nominal(p, typalign); /* count data plus backslashes; detect chars needing quotes */ if (values[i][0] == '\0') @@ -1399,8 +1399,8 @@ ReadArrayBinary(StringInfo buf, /* let's just make sure data is not toasted */ if (typlen == -1) values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i])); - totbytes = att_addlength(totbytes, typlen, values[i]); - totbytes = att_align(totbytes, typalign); + totbytes = att_addlength_datum(totbytes, typlen, values[i]); + totbytes = att_align_nominal(totbytes, typalign); /* check for overflow of total request */ if (!AllocSizeIsValid(totbytes)) ereport(ERROR, @@ -1512,8 +1512,8 @@ array_send(PG_FUNCTION_ARGS) VARSIZE(outputbytes) - VARHDRSZ); pfree(outputbytes); - p = att_addlength(p, typlen, PointerGetDatum(p)); - p = (char *) att_align(p, typalign); + p = att_addlength_pointer(p, typlen, p); + p = (char *) att_align_nominal(p, typalign); } /* advance bitmap pointer if any */ @@ -2108,8 +2108,8 @@ array_set(ArrayType *array, olditemlen = 0; else { - olditemlen = att_addlength(0, elmlen, PointerGetDatum(elt_ptr)); - olditemlen = att_align(olditemlen, elmalign); + olditemlen = att_addlength_pointer(0, elmlen, elt_ptr); + olditemlen = att_align_nominal(olditemlen, elmalign); } lenafter = (int) (olddatasize - lenbefore - olditemlen); } @@ -2118,8 +2118,8 @@ array_set(ArrayType *array, newitemlen = 0; else { - newitemlen = att_addlength(0, elmlen, dataValue); - newitemlen = att_align(newitemlen, elmalign); + newitemlen = att_addlength_datum(0, elmlen, dataValue); + newitemlen = att_align_nominal(newitemlen, elmalign); } newsize = overheadlen + lenbefore + newitemlen + lenafter; @@ -2639,8 +2639,8 @@ array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType, else { elt = fetch_att(s, inp_typbyval, inp_typlen); - s = att_addlength(s, inp_typlen, elt); - s = (char *) att_align(s, inp_typalign); + s = att_addlength_datum(s, inp_typlen, elt); + s = (char *) att_align_nominal(s, inp_typalign); fcinfo->arg[0] = elt; fcinfo->argnull[0] = false; } @@ -2679,8 +2679,8 @@ array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType, if (typlen == -1) values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i])); /* Update total result size */ - nbytes = att_addlength(nbytes, typlen, values[i]); - nbytes = att_align(nbytes, typalign); + nbytes = att_addlength_datum(nbytes, typlen, values[i]); + nbytes = att_align_nominal(nbytes, typalign); /* check for overflow of total request */ if (!AllocSizeIsValid(nbytes)) ereport(ERROR, @@ -2827,8 +2827,8 @@ construct_md_array(Datum *elems, /* make sure data is not toasted */ if (elmlen == -1) elems[i] = PointerGetDatum(PG_DETOAST_DATUM(elems[i])); - nbytes = att_addlength(nbytes, elmlen, elems[i]); - nbytes = att_align(nbytes, elmalign); + nbytes = att_addlength_datum(nbytes, elmlen, elems[i]); + nbytes = att_align_nominal(nbytes, elmalign); /* check for overflow of total request */ if (!AllocSizeIsValid(nbytes)) ereport(ERROR, @@ -2947,8 +2947,8 @@ deconstruct_array(ArrayType *array, elems[i] = fetch_att(p, elmbyval, elmlen); if (nulls) nulls[i] = false; - p = att_addlength(p, elmlen, PointerGetDatum(p)); - p = (char *) att_align(p, elmalign); + p = att_addlength_pointer(p, elmlen, p); + p = (char *) att_align_nominal(p, elmalign); } /* advance bitmap pointer if any */ @@ -3064,8 +3064,8 @@ array_eq(PG_FUNCTION_ARGS) { isnull1 = false; elt1 = fetch_att(ptr1, typbyval, typlen); - ptr1 = att_addlength(ptr1, typlen, PointerGetDatum(ptr1)); - ptr1 = (char *) att_align(ptr1, typalign); + ptr1 = att_addlength_pointer(ptr1, typlen, ptr1); + ptr1 = (char *) att_align_nominal(ptr1, typalign); } if (bitmap2 && (*bitmap2 & bitmask) == 0) @@ -3077,8 +3077,8 @@ array_eq(PG_FUNCTION_ARGS) { isnull2 = false; elt2 = fetch_att(ptr2, typbyval, typlen); - ptr2 = att_addlength(ptr2, typlen, PointerGetDatum(ptr2)); - ptr2 = (char *) att_align(ptr2, typalign); + ptr2 = att_addlength_pointer(ptr2, typlen, ptr2); + ptr2 = (char *) att_align_nominal(ptr2, typalign); } /* advance bitmap pointers if any */ @@ -3265,8 +3265,8 @@ array_cmp(FunctionCallInfo fcinfo) { isnull1 = false; elt1 = fetch_att(ptr1, typbyval, typlen); - ptr1 = att_addlength(ptr1, typlen, PointerGetDatum(ptr1)); - ptr1 = (char *) att_align(ptr1, typalign); + ptr1 = att_addlength_pointer(ptr1, typlen, ptr1); + ptr1 = (char *) att_align_nominal(ptr1, typalign); } if (bitmap2 && (*bitmap2 & bitmask) == 0) @@ -3278,8 +3278,8 @@ array_cmp(FunctionCallInfo fcinfo) { isnull2 = false; elt2 = fetch_att(ptr2, typbyval, typlen); - ptr2 = att_addlength(ptr2, typlen, PointerGetDatum(ptr2)); - ptr2 = (char *) att_align(ptr2, typalign); + ptr2 = att_addlength_pointer(ptr2, typlen, ptr2); + ptr2 = (char *) att_align_nominal(ptr2, typalign); } /* advance bitmap pointers if any */ @@ -3468,8 +3468,8 @@ array_contain_compare(ArrayType *array1, ArrayType *array2, bool matchall, { isnull1 = false; elt1 = fetch_att(ptr1, typbyval, typlen); - ptr1 = att_addlength(ptr1, typlen, PointerGetDatum(ptr1)); - ptr1 = (char *) att_align(ptr1, typalign); + ptr1 = att_addlength_pointer(ptr1, typlen, ptr1); + ptr1 = (char *) att_align_nominal(ptr1, typalign); } /* advance bitmap pointer if any */ @@ -3667,14 +3667,14 @@ ArrayCastAndSet(Datum src, store_att_byval(dest, src, typlen); else memmove(dest, DatumGetPointer(src), typlen); - inc = att_align(typlen, typalign); + inc = att_align_nominal(typlen, typalign); } else { Assert(!typbyval); - inc = att_addlength(0, typlen, src); + inc = att_addlength_datum(0, typlen, src); memmove(dest, DatumGetPointer(src), inc); - inc = att_align(inc, typalign); + inc = att_align_nominal(inc, typalign); } return inc; @@ -3700,7 +3700,7 @@ array_seek(char *ptr, int offset, bits8 *nullbitmap, int nitems, /* easy if fixed-size elements and no NULLs */ if (typlen > 0 && !nullbitmap) - return ptr + nitems * ((Size) att_align(typlen, typalign)); + return ptr + nitems * ((Size) att_align_nominal(typlen, typalign)); /* seems worth having separate loops for NULL and no-NULLs cases */ if (nullbitmap) @@ -3712,8 +3712,8 @@ array_seek(char *ptr, int offset, bits8 *nullbitmap, int nitems, { if (*nullbitmap & bitmask) { - ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr)); - ptr = (char *) att_align(ptr, typalign); + ptr = att_addlength_pointer(ptr, typlen, ptr); + ptr = (char *) att_align_nominal(ptr, typalign); } bitmask <<= 1; if (bitmask == 0x100) @@ -3727,8 +3727,8 @@ array_seek(char *ptr, int offset, bits8 *nullbitmap, int nitems, { for (i = 0; i < nitems; i++) { - ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr)); - ptr = (char *) att_align(ptr, typalign); + ptr = att_addlength_pointer(ptr, typlen, ptr); + ptr = (char *) att_align_nominal(ptr, typalign); } } return ptr; @@ -3883,7 +3883,7 @@ array_slice_size(char *arraydataptr, bits8 *arraynullsptr, /* Pretty easy for fixed element length without nulls ... */ if (typlen > 0 && !arraynullsptr) - return ArrayGetNItems(ndim, span) * att_align(typlen, typalign); + return ArrayGetNItems(ndim, span) * att_align_nominal(typlen, typalign); /* Else gotta do it the hard way */ src_offset = ArrayGetOffset(ndim, dim, lb, st); @@ -3904,8 +3904,8 @@ array_slice_size(char *arraydataptr, bits8 *arraynullsptr, } if (!array_get_isnull(arraynullsptr, src_offset)) { - inc = att_addlength(0, typlen, PointerGetDatum(ptr)); - inc = att_align(inc, typalign); + inc = att_addlength_pointer(0, typlen, ptr); + inc = att_align_nominal(inc, typalign); ptr += inc; count += inc; } diff --git a/src/backend/utils/adt/datum.c b/src/backend/utils/adt/datum.c index 21e7cc5541..c025de61f7 100644 --- a/src/backend/utils/adt/datum.c +++ b/src/backend/utils/adt/datum.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/datum.c,v 1.34 2007/02/27 23:48:07 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/datum.c,v 1.35 2007/04/06 04:21:42 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -47,7 +47,7 @@ * Find the "real" size of a datum, given the datum value, * whether it is a "by value", and the declared type length. * - * This is essentially an out-of-line version of the att_addlength() + * This is essentially an out-of-line version of the att_addlength_datum() * macro in access/tupmacs.h. We do a tad more error checking though. *------------------------------------------------------------------------- */ @@ -79,7 +79,7 @@ datumGetSize(Datum value, bool typByVal, int typLen) (errcode(ERRCODE_DATA_EXCEPTION), errmsg("invalid Datum pointer"))); - size = (Size) VARSIZE(s); + size = (Size) VARSIZE_ANY(s); } else if (typLen == -2) { diff --git a/src/backend/utils/adt/network.c b/src/backend/utils/adt/network.c index 88fa427450..c1dadfbcc1 100644 --- a/src/backend/utils/adt/network.c +++ b/src/backend/utils/adt/network.c @@ -1,7 +1,7 @@ /* * PostgreSQL type definitions for the INET and CIDR types. * - * $PostgreSQL: pgsql/src/backend/utils/adt/network.c,v 1.68 2007/02/27 23:48:08 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/network.c,v 1.69 2007/04/06 04:21:43 tgl Exp $ * * Jon Postel RIP 16 Oct 1998 */ @@ -30,23 +30,38 @@ static int ip_addrsize(inet *inetptr); static inet *internal_inetpl(inet *ip, int64 addend); /* - * Access macros. + * Access macros. We use VARDATA_ANY so that we can process short-header + * varlena values without detoasting them. This requires a trick: + * VARDATA_ANY assumes the varlena header is already filled in, which is + * not the case when constructing a new value (until SET_INET_VARSIZE is + * called, which we typically can't do till the end). Therefore, we + * always initialize the newly-allocated value to zeroes (using palloc0). + * A zero length word will look like the not-1-byte case to VARDATA_ANY, + * and so we correctly construct an uncompressed value. + * + * Note that ip_maxbits() and SET_INET_VARSIZE() require + * the family field to be set correctly. */ #define ip_family(inetptr) \ - (((inet_struct *)VARDATA(inetptr))->family) + (((inet_struct *) VARDATA_ANY(inetptr))->family) #define ip_bits(inetptr) \ - (((inet_struct *)VARDATA(inetptr))->bits) + (((inet_struct *) VARDATA_ANY(inetptr))->bits) #define ip_addr(inetptr) \ - (((inet_struct *)VARDATA(inetptr))->ipaddr) + (((inet_struct *) VARDATA_ANY(inetptr))->ipaddr) #define ip_maxbits(inetptr) \ (ip_family(inetptr) == PGSQL_AF_INET ? 32 : 128) +#define SET_INET_VARSIZE(dst) \ + SET_VARSIZE(dst, VARHDRSZ + offsetof(inet_struct, ipaddr) + \ + ip_addrsize(dst)) + + /* - * Return the number of bytes of storage needed for this data type. + * Return the number of bytes of address storage needed for this data type. */ static int ip_addrsize(inet *inetptr) @@ -71,7 +86,7 @@ network_in(char *src, bool is_cidr) int bits; inet *dst; - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); /* * First, check to see if this is an IPv6 or IPv4 address. IPv6 addresses @@ -105,10 +120,8 @@ network_in(char *src, bool is_cidr) errdetail("Value has bits set to right of mask."))); } - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); ip_bits(dst) = bits; + SET_INET_VARSIZE(dst); return dst; } @@ -194,7 +207,7 @@ network_recv(StringInfo buf, bool is_cidr) i; /* make sure any unused bits in a CIDR value are zeroed */ - addr = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + addr = (inet *) palloc0(sizeof(inet)); ip_family(addr) = pq_getmsgbyte(buf); if (ip_family(addr) != PGSQL_AF_INET && @@ -220,9 +233,6 @@ network_recv(StringInfo buf, bool is_cidr) /* translator: %s is inet or cidr */ errmsg("invalid length in external \"%s\" value", is_cidr ? "cidr" : "inet"))); - SET_VARSIZE(addr, VARHDRSZ + - ((char *) ip_addr(addr) - (char *) VARDATA(addr)) + - ip_addrsize(addr)); addrptr = (char *) ip_addr(addr); for (i = 0; i < nb; i++) @@ -240,6 +250,8 @@ network_recv(StringInfo buf, bool is_cidr) errdetail("Value has bits set to right of mask."))); } + SET_INET_VARSIZE(addr); + return addr; } @@ -348,8 +360,8 @@ inet_to_cidr(PG_FUNCTION_ARGS) elog(ERROR, "invalid inet bit length: %d", bits); /* clone the original data */ - dst = (inet *) palloc(VARSIZE(src)); - memcpy(dst, src, VARSIZE(src)); + dst = (inet *) palloc(VARSIZE_ANY(src)); + memcpy(dst, src, VARSIZE_ANY(src)); /* zero out any bits to the right of the netmask */ byte = bits / 8; @@ -387,8 +399,8 @@ inet_set_masklen(PG_FUNCTION_ARGS) errmsg("invalid mask length: %d", bits))); /* clone the original data */ - dst = (inet *) palloc(VARSIZE(src)); - memcpy(dst, src, VARSIZE(src)); + dst = (inet *) palloc(VARSIZE_ANY(src)); + memcpy(dst, src, VARSIZE_ANY(src)); ip_bits(dst) = bits; @@ -414,8 +426,8 @@ cidr_set_masklen(PG_FUNCTION_ARGS) errmsg("invalid mask length: %d", bits))); /* clone the original data */ - dst = (inet *) palloc(VARSIZE(src)); - memcpy(dst, src, VARSIZE(src)); + dst = (inet *) palloc(VARSIZE_ANY(src)); + memcpy(dst, src, VARSIZE_ANY(src)); ip_bits(dst) = bits; @@ -546,7 +558,7 @@ hashinet(PG_FUNCTION_ARGS) int addrsize = ip_addrsize(addr); /* XXX this assumes there are no pad bytes in the data structure */ - return hash_any((unsigned char *) VARDATA(addr), addrsize + 2); + return hash_any((unsigned char *) VARDATA_ANY(addr), addrsize + 2); } /* @@ -762,7 +774,7 @@ network_broadcast(PG_FUNCTION_ARGS) *b; /* make sure any unused bits are zeroed */ - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); if (ip_family(ip) == PGSQL_AF_INET) maxbytes = 4; @@ -793,9 +805,7 @@ network_broadcast(PG_FUNCTION_ARGS) ip_family(dst) = ip_family(ip); ip_bits(dst) = ip_bits(ip); - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); + SET_INET_VARSIZE(dst); PG_RETURN_INET_P(dst); } @@ -812,7 +822,7 @@ network_network(PG_FUNCTION_ARGS) *b; /* make sure any unused bits are zeroed */ - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); bits = ip_bits(ip); a = ip_addr(ip); @@ -838,9 +848,7 @@ network_network(PG_FUNCTION_ARGS) ip_family(dst) = ip_family(ip); ip_bits(dst) = ip_bits(ip); - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); + SET_INET_VARSIZE(dst); PG_RETURN_INET_P(dst); } @@ -856,7 +864,7 @@ network_netmask(PG_FUNCTION_ARGS) unsigned char *b; /* make sure any unused bits are zeroed */ - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); bits = ip_bits(ip); b = ip_addr(dst); @@ -881,9 +889,7 @@ network_netmask(PG_FUNCTION_ARGS) ip_family(dst) = ip_family(ip); ip_bits(dst) = ip_maxbits(ip); - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); + SET_INET_VARSIZE(dst); PG_RETURN_INET_P(dst); } @@ -900,7 +906,7 @@ network_hostmask(PG_FUNCTION_ARGS) unsigned char *b; /* make sure any unused bits are zeroed */ - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); if (ip_family(ip) == PGSQL_AF_INET) maxbytes = 4; @@ -930,9 +936,7 @@ network_hostmask(PG_FUNCTION_ARGS) ip_family(dst) = ip_family(ip); ip_bits(dst) = ip_maxbits(ip); - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); + SET_INET_VARSIZE(dst); PG_RETURN_INET_P(dst); } @@ -1259,7 +1263,7 @@ inetnot(PG_FUNCTION_ARGS) inet *ip = PG_GETARG_INET_P(0); inet *dst; - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); { int nb = ip_addrsize(ip); @@ -1272,9 +1276,7 @@ inetnot(PG_FUNCTION_ARGS) ip_bits(dst) = ip_bits(ip); ip_family(dst) = ip_family(ip); - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); + SET_INET_VARSIZE(dst); PG_RETURN_INET_P(dst); } @@ -1287,7 +1289,7 @@ inetand(PG_FUNCTION_ARGS) inet *ip2 = PG_GETARG_INET_P(1); inet *dst; - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); if (ip_family(ip) != ip_family(ip2)) ereport(ERROR, @@ -1306,9 +1308,7 @@ inetand(PG_FUNCTION_ARGS) ip_bits(dst) = Max(ip_bits(ip), ip_bits(ip2)); ip_family(dst) = ip_family(ip); - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); + SET_INET_VARSIZE(dst); PG_RETURN_INET_P(dst); } @@ -1321,7 +1321,7 @@ inetor(PG_FUNCTION_ARGS) inet *ip2 = PG_GETARG_INET_P(1); inet *dst; - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); if (ip_family(ip) != ip_family(ip2)) ereport(ERROR, @@ -1340,9 +1340,7 @@ inetor(PG_FUNCTION_ARGS) ip_bits(dst) = Max(ip_bits(ip), ip_bits(ip2)); ip_family(dst) = ip_family(ip); - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); + SET_INET_VARSIZE(dst); PG_RETURN_INET_P(dst); } @@ -1353,7 +1351,7 @@ internal_inetpl(inet *ip, int64 addend) { inet *dst; - dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct)); + dst = (inet *) palloc0(sizeof(inet)); { int nb = ip_addrsize(ip); @@ -1391,12 +1389,10 @@ internal_inetpl(inet *ip, int64 addend) (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("result is out of range"))); } - ip_bits(dst) = ip_bits(ip); + ip_bits(dst) = ip_bits(ip); ip_family(dst) = ip_family(ip); - SET_VARSIZE(dst, VARHDRSZ + - ((char *) ip_addr(dst) - (char *) VARDATA(dst)) + - ip_addrsize(dst)); + SET_INET_VARSIZE(dst); return dst; } diff --git a/src/backend/utils/adt/pg_lzcompress.c b/src/backend/utils/adt/pg_lzcompress.c index c756e5707a..085bc63e0d 100644 --- a/src/backend/utils/adt/pg_lzcompress.c +++ b/src/backend/utils/adt/pg_lzcompress.c @@ -166,7 +166,7 @@ * * Copyright (c) 1999-2007, PostgreSQL Global Development Group * - * $PostgreSQL: pgsql/src/backend/utils/adt/pg_lzcompress.c,v 1.25 2007/02/27 23:48:08 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/pg_lzcompress.c,v 1.26 2007/04/06 04:21:43 tgl Exp $ * ---------- */ #include "postgres.h" @@ -618,7 +618,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, /* * Success - need only fill in the actual length of the compressed datum. */ - SET_VARSIZE(dest, result_size + sizeof(PGLZ_Header)); + SET_VARSIZE_COMPRESSED(dest, result_size + sizeof(PGLZ_Header)); return true; } diff --git a/src/backend/utils/adt/varchar.c b/src/backend/utils/adt/varchar.c index 371a8a605d..77a6ab9c4f 100644 --- a/src/backend/utils/adt/varchar.c +++ b/src/backend/utils/adt/varchar.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/varchar.c,v 1.122 2007/02/27 23:48:09 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/varchar.c,v 1.123 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -16,6 +16,7 @@ #include "access/hash.h" +#include "access/tuptoaster.h" #include "libpq/pqformat.h" #include "utils/array.h" #include "utils/builtins.h" @@ -206,14 +207,14 @@ bpcharin(PG_FUNCTION_ARGS) Datum bpcharout(PG_FUNCTION_ARGS) { - BpChar *s = PG_GETARG_BPCHAR_P(0); + BpChar *s = PG_GETARG_BPCHAR_PP(0); char *result; int len; /* copy and add null term */ - len = VARSIZE(s) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(s); result = (char *) palloc(len + 1); - memcpy(result, VARDATA(s), len); + memcpy(result, VARDATA_ANY(s), len); result[len] = '\0'; PG_RETURN_CSTRING(result); @@ -267,7 +268,7 @@ bpcharsend(PG_FUNCTION_ARGS) Datum bpchar(PG_FUNCTION_ARGS) { - BpChar *source = PG_GETARG_BPCHAR_P(0); + BpChar *source = PG_GETARG_BPCHAR_PP(0); int32 maxlen = PG_GETARG_INT32(1); bool isExplicit = PG_GETARG_BOOL(2); BpChar *result; @@ -282,9 +283,12 @@ bpchar(PG_FUNCTION_ARGS) if (maxlen < (int32) VARHDRSZ) PG_RETURN_BPCHAR_P(source); - len = VARSIZE(source); + maxlen -= VARHDRSZ; - charlen = pg_mbstrlen_with_len(VARDATA(source), len - VARHDRSZ) + VARHDRSZ; + len = VARSIZE_ANY_EXHDR(source); + s = VARDATA_ANY(source); + + charlen = pg_mbstrlen_with_len(s, len); /* No work if supplied data matches typmod already */ if (charlen == maxlen) @@ -295,23 +299,22 @@ bpchar(PG_FUNCTION_ARGS) /* Verify that extra characters are spaces, and clip them off */ size_t maxmblen; - maxmblen = pg_mbcharcliplen(VARDATA(source), len - VARHDRSZ, - maxlen - VARHDRSZ) + VARHDRSZ; + maxmblen = pg_mbcharcliplen(s, len, maxlen); if (!isExplicit) { - for (i = maxmblen - VARHDRSZ; i < len - VARHDRSZ; i++) - if (*(VARDATA(source) + i) != ' ') + for (i = maxmblen; i < len; i++) + if (s[i] != ' ') ereport(ERROR, (errcode(ERRCODE_STRING_DATA_RIGHT_TRUNCATION), errmsg("value too long for type character(%d)", - maxlen - VARHDRSZ))); + maxlen))); } len = maxmblen; /* - * XXX: at this point, maxlen is the necessary byte length+VARHDRSZ, + * At this point, maxlen is the necessary byte length, * not the number of CHARACTERS! */ maxlen = len; @@ -319,23 +322,23 @@ bpchar(PG_FUNCTION_ARGS) else { /* - * XXX: at this point, maxlen is the necessary byte length+VARHDRSZ, + * At this point, maxlen is the necessary byte length, * not the number of CHARACTERS! */ maxlen = len + (maxlen - charlen); } - s = VARDATA(source); + Assert(maxlen >= len); - result = palloc(maxlen); - SET_VARSIZE(result, maxlen); + result = palloc(maxlen+VARHDRSZ); + SET_VARSIZE(result, maxlen+VARHDRSZ); r = VARDATA(result); - memcpy(r, s, len - VARHDRSZ); + memcpy(r, s, len); /* blank pad the string if necessary */ if (maxlen > len) - memset(r + len - VARHDRSZ, ' ', maxlen - len); + memset(r + len, ' ', maxlen - len); PG_RETURN_BPCHAR_P(result); } @@ -365,11 +368,13 @@ char_bpchar(PG_FUNCTION_ARGS) Datum bpchar_name(PG_FUNCTION_ARGS) { - BpChar *s = PG_GETARG_BPCHAR_P(0); + BpChar *s = PG_GETARG_BPCHAR_PP(0); + char *s_data; Name result; int len; - len = VARSIZE(s) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(s); + s_data = VARDATA_ANY(s); /* Truncate to max length for a Name */ if (len >= NAMEDATALEN) @@ -378,13 +383,13 @@ bpchar_name(PG_FUNCTION_ARGS) /* Remove trailing blanks */ while (len > 0) { - if (*(VARDATA(s) + len - 1) != ' ') + if (s_data[len - 1] != ' ') break; len--; } result = (NameData *) palloc(NAMEDATALEN); - memcpy(NameStr(*result), VARDATA(s), len); + memcpy(NameStr(*result), s_data, len); /* Now null pad to full length... */ while (len < NAMEDATALEN) @@ -509,14 +514,14 @@ varcharin(PG_FUNCTION_ARGS) Datum varcharout(PG_FUNCTION_ARGS) { - VarChar *s = PG_GETARG_VARCHAR_P(0); + VarChar *s = PG_GETARG_VARCHAR_PP(0); char *result; int32 len; /* copy and add null term */ - len = VARSIZE(s) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(s); result = palloc(len + 1); - memcpy(result, VARDATA(s), len); + memcpy(result, VARDATA_ANY(s), len); result[len] = '\0'; PG_RETURN_CSTRING(result); @@ -570,39 +575,41 @@ varcharsend(PG_FUNCTION_ARGS) Datum varchar(PG_FUNCTION_ARGS) { - VarChar *source = PG_GETARG_VARCHAR_P(0); - int32 maxlen = PG_GETARG_INT32(1); + VarChar *source = PG_GETARG_VARCHAR_PP(0); + int32 typmod = PG_GETARG_INT32(1); bool isExplicit = PG_GETARG_BOOL(2); VarChar *result; - int32 len; + int32 len, maxlen; size_t maxmblen; int i; + char *s_data; + + len = VARSIZE_ANY_EXHDR(source); + s_data = VARDATA_ANY(source); + maxlen = typmod - VARHDRSZ; - len = VARSIZE(source); /* No work if typmod is invalid or supplied data fits it already */ - if (maxlen < (int32) VARHDRSZ || len <= maxlen) + if (maxlen < 0 || len <= maxlen) PG_RETURN_VARCHAR_P(source); /* only reach here if string is too long... */ /* truncate multibyte string preserving multibyte boundary */ - maxmblen = pg_mbcharcliplen(VARDATA(source), len - VARHDRSZ, - maxlen - VARHDRSZ); + maxmblen = pg_mbcharcliplen(s_data, len, maxlen); if (!isExplicit) { - for (i = maxmblen; i < len - VARHDRSZ; i++) - if (*(VARDATA(source) + i) != ' ') + for (i = maxmblen; i < len; i++) + if (s_data[i] != ' ') ereport(ERROR, (errcode(ERRCODE_STRING_DATA_RIGHT_TRUNCATION), - errmsg("value too long for type character varying(%d)", - maxlen - VARHDRSZ))); + errmsg("value too long for type character varying(%d)", + maxlen))); } - len = maxmblen + VARHDRSZ; - result = palloc(len); - SET_VARSIZE(result, len); - memcpy(VARDATA(result), VARDATA(source), len - VARHDRSZ); + result = palloc(maxmblen + VARHDRSZ); + SET_VARSIZE(result, maxmblen + VARHDRSZ); + memcpy(VARDATA(result), s_data, maxmblen); PG_RETURN_VARCHAR_P(result); } @@ -632,11 +639,11 @@ varchartypmodout(PG_FUNCTION_ARGS) static int bcTruelen(BpChar *arg) { - char *s = VARDATA(arg); + char *s = VARDATA_ANY(arg); int i; int len; - len = VARSIZE(arg) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(arg); for (i = len - 1; i >= 0; i--) { if (s[i] != ' ') @@ -648,7 +655,7 @@ bcTruelen(BpChar *arg) Datum bpcharlen(PG_FUNCTION_ARGS) { - BpChar *arg = PG_GETARG_BPCHAR_P(0); + BpChar *arg = PG_GETARG_BPCHAR_PP(0); int len; /* get number of bytes, ignoring trailing spaces */ @@ -656,7 +663,7 @@ bpcharlen(PG_FUNCTION_ARGS) /* in multibyte encoding, convert to number of characters */ if (pg_database_encoding_max_length() != 1) - len = pg_mbstrlen_with_len(VARDATA(arg), len); + len = pg_mbstrlen_with_len(VARDATA_ANY(arg), len); PG_RETURN_INT32(len); } @@ -664,9 +671,10 @@ bpcharlen(PG_FUNCTION_ARGS) Datum bpcharoctetlen(PG_FUNCTION_ARGS) { - BpChar *arg = PG_GETARG_BPCHAR_P(0); + Datum arg = PG_GETARG_DATUM(0); - PG_RETURN_INT32(VARSIZE(arg) - VARHDRSZ); + /* We need not detoast the input at all */ + PG_RETURN_INT32(toast_raw_datum_size(arg) - VARHDRSZ); } @@ -681,8 +689,8 @@ bpcharoctetlen(PG_FUNCTION_ARGS) Datum bpchareq(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; bool result; @@ -697,7 +705,7 @@ bpchareq(PG_FUNCTION_ARGS) if (len1 != len2) result = false; else - result = (strncmp(VARDATA(arg1), VARDATA(arg2), len1) == 0); + result = (strncmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), len1) == 0); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -708,8 +716,8 @@ bpchareq(PG_FUNCTION_ARGS) Datum bpcharne(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; bool result; @@ -724,7 +732,7 @@ bpcharne(PG_FUNCTION_ARGS) if (len1 != len2) result = true; else - result = (strncmp(VARDATA(arg1), VARDATA(arg2), len1) != 0); + result = (strncmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), len1) != 0); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -735,8 +743,8 @@ bpcharne(PG_FUNCTION_ARGS) Datum bpcharlt(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; int cmp; @@ -744,7 +752,7 @@ bpcharlt(PG_FUNCTION_ARGS) len1 = bcTruelen(arg1); len2 = bcTruelen(arg2); - cmp = varstr_cmp(VARDATA(arg1), len1, VARDATA(arg2), len2); + cmp = varstr_cmp(VARDATA_ANY(arg1), len1, VARDATA_ANY(arg2), len2); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -755,8 +763,8 @@ bpcharlt(PG_FUNCTION_ARGS) Datum bpcharle(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; int cmp; @@ -764,7 +772,7 @@ bpcharle(PG_FUNCTION_ARGS) len1 = bcTruelen(arg1); len2 = bcTruelen(arg2); - cmp = varstr_cmp(VARDATA(arg1), len1, VARDATA(arg2), len2); + cmp = varstr_cmp(VARDATA_ANY(arg1), len1, VARDATA_ANY(arg2), len2); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -775,8 +783,8 @@ bpcharle(PG_FUNCTION_ARGS) Datum bpchargt(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; int cmp; @@ -784,7 +792,7 @@ bpchargt(PG_FUNCTION_ARGS) len1 = bcTruelen(arg1); len2 = bcTruelen(arg2); - cmp = varstr_cmp(VARDATA(arg1), len1, VARDATA(arg2), len2); + cmp = varstr_cmp(VARDATA_ANY(arg1), len1, VARDATA_ANY(arg2), len2); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -795,8 +803,8 @@ bpchargt(PG_FUNCTION_ARGS) Datum bpcharge(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; int cmp; @@ -804,7 +812,7 @@ bpcharge(PG_FUNCTION_ARGS) len1 = bcTruelen(arg1); len2 = bcTruelen(arg2); - cmp = varstr_cmp(VARDATA(arg1), len1, VARDATA(arg2), len2); + cmp = varstr_cmp(VARDATA_ANY(arg1), len1, VARDATA_ANY(arg2), len2); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -815,8 +823,8 @@ bpcharge(PG_FUNCTION_ARGS) Datum bpcharcmp(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; int cmp; @@ -824,7 +832,7 @@ bpcharcmp(PG_FUNCTION_ARGS) len1 = bcTruelen(arg1); len2 = bcTruelen(arg2); - cmp = varstr_cmp(VARDATA(arg1), len1, VARDATA(arg2), len2); + cmp = varstr_cmp(VARDATA_ANY(arg1), len1, VARDATA_ANY(arg2), len2); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -835,8 +843,8 @@ bpcharcmp(PG_FUNCTION_ARGS) Datum bpchar_larger(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; int cmp; @@ -844,7 +852,7 @@ bpchar_larger(PG_FUNCTION_ARGS) len1 = bcTruelen(arg1); len2 = bcTruelen(arg2); - cmp = varstr_cmp(VARDATA(arg1), len1, VARDATA(arg2), len2); + cmp = varstr_cmp(VARDATA_ANY(arg1), len1, VARDATA_ANY(arg2), len2); PG_RETURN_BPCHAR_P((cmp >= 0) ? arg1 : arg2); } @@ -852,8 +860,8 @@ bpchar_larger(PG_FUNCTION_ARGS) Datum bpchar_smaller(PG_FUNCTION_ARGS) { - BpChar *arg1 = PG_GETARG_BPCHAR_P(0); - BpChar *arg2 = PG_GETARG_BPCHAR_P(1); + BpChar *arg1 = PG_GETARG_BPCHAR_PP(0); + BpChar *arg2 = PG_GETARG_BPCHAR_PP(1); int len1, len2; int cmp; @@ -861,7 +869,7 @@ bpchar_smaller(PG_FUNCTION_ARGS) len1 = bcTruelen(arg1); len2 = bcTruelen(arg2); - cmp = varstr_cmp(VARDATA(arg1), len1, VARDATA(arg2), len2); + cmp = varstr_cmp(VARDATA_ANY(arg1), len1, VARDATA_ANY(arg2), len2); PG_RETURN_BPCHAR_P((cmp <= 0) ? arg1 : arg2); } @@ -878,12 +886,12 @@ bpchar_smaller(PG_FUNCTION_ARGS) Datum hashbpchar(PG_FUNCTION_ARGS) { - BpChar *key = PG_GETARG_BPCHAR_P(0); + BpChar *key = PG_GETARG_BPCHAR_PP(0); char *keydata; int keylen; Datum result; - keydata = VARDATA(key); + keydata = VARDATA_ANY(key); keylen = bcTruelen(key); result = hash_any((unsigned char *) keydata, keylen); diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 51130efd79..0b5b4fd16a 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/adt/varlena.c,v 1.155 2007/02/27 23:48:09 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/varlena.c,v 1.156 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -175,7 +175,7 @@ byteain(PG_FUNCTION_ARGS) Datum byteaout(PG_FUNCTION_ARGS) { - bytea *vlena = PG_GETARG_BYTEA_P(0); + bytea *vlena = PG_GETARG_BYTEA_PP(0); char *result; char *vp; char *rp; @@ -184,8 +184,8 @@ byteaout(PG_FUNCTION_ARGS) int len; len = 1; /* empty string has 1 char */ - vp = VARDATA(vlena); - for (i = VARSIZE(vlena) - VARHDRSZ; i != 0; i--, vp++) + vp = VARDATA_ANY(vlena); + for (i = VARSIZE_ANY_EXHDR(vlena); i != 0; i--, vp++) { if (*vp == '\\') len += 2; @@ -195,8 +195,8 @@ byteaout(PG_FUNCTION_ARGS) len++; } rp = result = (char *) palloc(len); - vp = VARDATA(vlena); - for (i = VARSIZE(vlena) - VARHDRSZ; i != 0; i--, vp++) + vp = VARDATA_ANY(vlena); + for (i = VARSIZE_ANY_EXHDR(vlena); i != 0; i--, vp++) { if (*vp == '\\') { @@ -277,13 +277,13 @@ textin(PG_FUNCTION_ARGS) Datum textout(PG_FUNCTION_ARGS) { - text *t = PG_GETARG_TEXT_P(0); + text *t = PG_GETARG_TEXT_PP(0); int len; char *result; - len = VARSIZE(t) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(t); result = (char *) palloc(len + 1); - memcpy(result, VARDATA(t), len); + memcpy(result, VARDATA_ANY(t), len); result[len] = '\0'; PG_RETURN_CSTRING(result); @@ -315,11 +315,11 @@ textrecv(PG_FUNCTION_ARGS) Datum textsend(PG_FUNCTION_ARGS) { - text *t = PG_GETARG_TEXT_P(0); + text *t = PG_GETARG_TEXT_PP(0); StringInfoData buf; pq_begintypsend(&buf); - pq_sendtext(&buf, VARDATA(t), VARSIZE(t) - VARHDRSZ); + pq_sendtext(&buf, VARDATA_ANY(t), VARSIZE_ANY_EXHDR(t)); PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } @@ -412,10 +412,10 @@ text_length(Datum str) PG_RETURN_INT32(toast_raw_datum_size(str) - VARHDRSZ); else { - text *t = DatumGetTextP(str); + text *t = DatumGetTextPP(str); - PG_RETURN_INT32(pg_mbstrlen_with_len(VARDATA(t), - VARSIZE(t) - VARHDRSZ)); + PG_RETURN_INT32(pg_mbstrlen_with_len(VARDATA_ANY(t), + VARSIZE_ANY_EXHDR(t))); } } @@ -446,19 +446,19 @@ textoctetlen(PG_FUNCTION_ARGS) Datum textcat(PG_FUNCTION_ARGS) { - text *t1 = PG_GETARG_TEXT_P(0); - text *t2 = PG_GETARG_TEXT_P(1); + text *t1 = PG_GETARG_TEXT_PP(0); + text *t2 = PG_GETARG_TEXT_PP(1); int len1, len2, len; text *result; char *ptr; - len1 = VARSIZE(t1) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(t1); if (len1 < 0) len1 = 0; - len2 = VARSIZE(t2) - VARHDRSZ; + len2 = VARSIZE_ANY_EXHDR(t2); if (len2 < 0) len2 = 0; @@ -471,9 +471,9 @@ textcat(PG_FUNCTION_ARGS) /* Fill data field of result string... */ ptr = VARDATA(result); if (len1 > 0) - memcpy(ptr, VARDATA(t1), len1); + memcpy(ptr, VARDATA_ANY(t1), len1); if (len2 > 0) - memcpy(ptr + len1, VARDATA(t2), len2); + memcpy(ptr + len1, VARDATA_ANY(t2), len2); PG_RETURN_TEXT_P(result); } @@ -1058,12 +1058,12 @@ text_cmp(text *arg1, text *arg2) int len1, len2; - a1p = VARDATA(arg1); - a2p = VARDATA(arg2); - - len1 = VARSIZE(arg1) - VARHDRSZ; - len2 = VARSIZE(arg2) - VARHDRSZ; + a1p = VARDATA_ANY(arg1); + a2p = VARDATA_ANY(arg2); + len1 = VARSIZE_ANY_EXHDR(arg1); + len2 = VARSIZE_ANY_EXHDR(arg2); + return varstr_cmp(a1p, len1, a2p, len2); } @@ -1078,19 +1078,19 @@ text_cmp(text *arg1, text *arg2) Datum texteq(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); bool result; /* * Since we only care about equality or not-equality, we can avoid all the * expense of strcoll() here, and just do bitwise comparison. */ - if (VARSIZE(arg1) != VARSIZE(arg2)) + if (VARSIZE_ANY_EXHDR(arg1) != VARSIZE_ANY_EXHDR(arg2)) result = false; else - result = (strncmp(VARDATA(arg1), VARDATA(arg2), - VARSIZE(arg1) - VARHDRSZ) == 0); + result = (strncmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), + VARSIZE_ANY_EXHDR(arg1)) == 0); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -1101,19 +1101,19 @@ texteq(PG_FUNCTION_ARGS) Datum textne(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); bool result; /* * Since we only care about equality or not-equality, we can avoid all the * expense of strcoll() here, and just do bitwise comparison. */ - if (VARSIZE(arg1) != VARSIZE(arg2)) + if (VARSIZE_ANY_EXHDR(arg1) != VARSIZE_ANY_EXHDR(arg2)) result = true; else - result = (strncmp(VARDATA(arg1), VARDATA(arg2), - VARSIZE(arg1) - VARHDRSZ) != 0); + result = (strncmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), + VARSIZE_ANY_EXHDR(arg1)) != 0); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -1124,8 +1124,8 @@ textne(PG_FUNCTION_ARGS) Datum text_lt(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); bool result; result = (text_cmp(arg1, arg2) < 0); @@ -1139,8 +1139,8 @@ text_lt(PG_FUNCTION_ARGS) Datum text_le(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); bool result; result = (text_cmp(arg1, arg2) <= 0); @@ -1154,8 +1154,8 @@ text_le(PG_FUNCTION_ARGS) Datum text_gt(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); bool result; result = (text_cmp(arg1, arg2) > 0); @@ -1169,8 +1169,8 @@ text_gt(PG_FUNCTION_ARGS) Datum text_ge(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); bool result; result = (text_cmp(arg1, arg2) >= 0); @@ -1184,8 +1184,8 @@ text_ge(PG_FUNCTION_ARGS) Datum bttextcmp(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); int32 result; result = text_cmp(arg1, arg2); @@ -1200,8 +1200,8 @@ bttextcmp(PG_FUNCTION_ARGS) Datum text_larger(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); text *result; result = ((text_cmp(arg1, arg2) > 0) ? arg1 : arg2); @@ -1212,8 +1212,8 @@ text_larger(PG_FUNCTION_ARGS) Datum text_smaller(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); text *result; result = ((text_cmp(arg1, arg2) < 0) ? arg1 : arg2); @@ -1233,13 +1233,13 @@ internal_text_pattern_compare(text *arg1, text *arg2) { int result; - result = memcmp(VARDATA(arg1), VARDATA(arg2), - Min(VARSIZE(arg1), VARSIZE(arg2)) - VARHDRSZ); + result = memcmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), + Min(VARSIZE_ANY_EXHDR(arg1), VARSIZE_ANY_EXHDR(arg2))); if (result != 0) return result; - else if (VARSIZE(arg1) < VARSIZE(arg2)) + else if (VARSIZE_ANY_EXHDR(arg1) < VARSIZE_ANY_EXHDR(arg2)) return -1; - else if (VARSIZE(arg1) > VARSIZE(arg2)) + else if (VARSIZE_ANY_EXHDR(arg1) > VARSIZE_ANY_EXHDR(arg2)) return 1; else return 0; @@ -1249,8 +1249,8 @@ internal_text_pattern_compare(text *arg1, text *arg2) Datum text_pattern_lt(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); int result; result = internal_text_pattern_compare(arg1, arg2); @@ -1265,8 +1265,8 @@ text_pattern_lt(PG_FUNCTION_ARGS) Datum text_pattern_le(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); int result; result = internal_text_pattern_compare(arg1, arg2); @@ -1281,11 +1281,11 @@ text_pattern_le(PG_FUNCTION_ARGS) Datum text_pattern_eq(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); int result; - if (VARSIZE(arg1) != VARSIZE(arg2)) + if (VARSIZE_ANY_EXHDR(arg1) != VARSIZE_ANY_EXHDR(arg2)) result = 1; else result = internal_text_pattern_compare(arg1, arg2); @@ -1300,8 +1300,8 @@ text_pattern_eq(PG_FUNCTION_ARGS) Datum text_pattern_ge(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); int result; result = internal_text_pattern_compare(arg1, arg2); @@ -1316,8 +1316,8 @@ text_pattern_ge(PG_FUNCTION_ARGS) Datum text_pattern_gt(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); int result; result = internal_text_pattern_compare(arg1, arg2); @@ -1332,11 +1332,11 @@ text_pattern_gt(PG_FUNCTION_ARGS) Datum text_pattern_ne(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); int result; - if (VARSIZE(arg1) != VARSIZE(arg2)) + if (VARSIZE_ANY_EXHDR(arg1) != VARSIZE_ANY_EXHDR(arg2)) result = 1; else result = internal_text_pattern_compare(arg1, arg2); @@ -1351,8 +1351,8 @@ text_pattern_ne(PG_FUNCTION_ARGS) Datum bttext_pattern_cmp(PG_FUNCTION_ARGS) { - text *arg1 = PG_GETARG_TEXT_P(0); - text *arg2 = PG_GETARG_TEXT_P(1); + text *arg1 = PG_GETARG_TEXT_PP(0); + text *arg2 = PG_GETARG_TEXT_PP(1); int result; result = internal_text_pattern_compare(arg1, arg2); @@ -1389,19 +1389,19 @@ byteaoctetlen(PG_FUNCTION_ARGS) Datum byteacat(PG_FUNCTION_ARGS) { - bytea *t1 = PG_GETARG_BYTEA_P(0); - bytea *t2 = PG_GETARG_BYTEA_P(1); + bytea *t1 = PG_GETARG_BYTEA_PP(0); + bytea *t2 = PG_GETARG_BYTEA_PP(1); int len1, len2, len; bytea *result; char *ptr; - len1 = VARSIZE(t1) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(t1); if (len1 < 0) len1 = 0; - len2 = VARSIZE(t2) - VARHDRSZ; + len2 = VARSIZE_ANY_EXHDR(t2); if (len2 < 0) len2 = 0; @@ -1414,9 +1414,9 @@ byteacat(PG_FUNCTION_ARGS) /* Fill data field of result string... */ ptr = VARDATA(result); if (len1 > 0) - memcpy(ptr, VARDATA(t1), len1); + memcpy(ptr, VARDATA_ANY(t1), len1); if (len2 > 0) - memcpy(ptr + len1, VARDATA(t2), len2); + memcpy(ptr + len1, VARDATA_ANY(t2), len2); PG_RETURN_BYTEA_P(result); } @@ -1509,8 +1509,8 @@ bytea_substr_no_len(PG_FUNCTION_ARGS) Datum byteapos(PG_FUNCTION_ARGS) { - bytea *t1 = PG_GETARG_BYTEA_P(0); - bytea *t2 = PG_GETARG_BYTEA_P(1); + bytea *t1 = PG_GETARG_BYTEA_PP(0); + bytea *t2 = PG_GETARG_BYTEA_PP(1); int pos; int px, p; @@ -1519,14 +1519,14 @@ byteapos(PG_FUNCTION_ARGS) char *p1, *p2; - if (VARSIZE(t2) <= VARHDRSZ) + len1 = VARSIZE_ANY_EXHDR(t1); + len2 = VARSIZE_ANY_EXHDR(t2); + + if (len2 <= 0) PG_RETURN_INT32(1); /* result for empty pattern */ - len1 = VARSIZE(t1) - VARHDRSZ; - len2 = VARSIZE(t2) - VARHDRSZ; - - p1 = VARDATA(t1); - p2 = VARDATA(t2); + p1 = VARDATA_ANY(t1); + p2 = VARDATA_ANY(t2); pos = 0; px = (len1 - len2); @@ -1553,12 +1553,12 @@ byteapos(PG_FUNCTION_ARGS) Datum byteaGetByte(PG_FUNCTION_ARGS) { - bytea *v = PG_GETARG_BYTEA_P(0); + bytea *v = PG_GETARG_BYTEA_PP(0); int32 n = PG_GETARG_INT32(1); int len; int byte; - len = VARSIZE(v) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(v); if (n < 0 || n >= len) ereport(ERROR, @@ -1566,7 +1566,7 @@ byteaGetByte(PG_FUNCTION_ARGS) errmsg("index %d out of valid range, 0..%d", n, len - 1))); - byte = ((unsigned char *) VARDATA(v))[n]; + byte = ((unsigned char *) VARDATA_ANY(v))[n]; PG_RETURN_INT32(byte); } @@ -1582,14 +1582,14 @@ byteaGetByte(PG_FUNCTION_ARGS) Datum byteaGetBit(PG_FUNCTION_ARGS) { - bytea *v = PG_GETARG_BYTEA_P(0); + bytea *v = PG_GETARG_BYTEA_PP(0); int32 n = PG_GETARG_INT32(1); int byteNo, bitNo; int len; int byte; - len = VARSIZE(v) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(v); if (n < 0 || n >= len * 8) ereport(ERROR, @@ -1600,7 +1600,7 @@ byteaGetBit(PG_FUNCTION_ARGS) byteNo = n / 8; bitNo = n % 8; - byte = ((unsigned char *) VARDATA(v))[byteNo]; + byte = ((unsigned char *) VARDATA_ANY(v))[byteNo]; if (byte & (1 << bitNo)) PG_RETURN_INT32(1); @@ -1715,23 +1715,18 @@ byteaSetBit(PG_FUNCTION_ARGS) Datum text_name(PG_FUNCTION_ARGS) { - text *s = PG_GETARG_TEXT_P(0); + text *s = PG_GETARG_TEXT_PP(0); Name result; int len; - len = VARSIZE(s) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(s); /* Truncate oversize input */ if (len >= NAMEDATALEN) len = NAMEDATALEN - 1; -#ifdef STRINGDEBUG - printf("text- convert string length %d (%d) ->%d\n", - VARSIZE(s) - VARHDRSZ, VARSIZE(s), len); -#endif - result = (Name) palloc(NAMEDATALEN); - memcpy(NameStr(*result), VARDATA(s), len); + memcpy(NameStr(*result), VARDATA_ANY(s), len); /* now null pad to full length... */ while (len < NAMEDATALEN) @@ -1755,11 +1750,6 @@ name_text(PG_FUNCTION_ARGS) len = strlen(NameStr(*s)); -#ifdef STRINGDEBUG - printf("text- convert string length %d (%d) ->%d\n", - VARSIZE(s) - VARHDRSZ, VARSIZE(s), len); -#endif - result = palloc(VARHDRSZ + len); SET_VARSIZE(result, VARHDRSZ + len); memcpy(VARDATA(result), NameStr(*s), len); @@ -1948,20 +1938,20 @@ SplitIdentifierString(char *rawstring, char separator, Datum byteaeq(PG_FUNCTION_ARGS) { - bytea *arg1 = PG_GETARG_BYTEA_P(0); - bytea *arg2 = PG_GETARG_BYTEA_P(1); + bytea *arg1 = PG_GETARG_BYTEA_PP(0); + bytea *arg2 = PG_GETARG_BYTEA_PP(1); int len1, len2; bool result; - len1 = VARSIZE(arg1) - VARHDRSZ; - len2 = VARSIZE(arg2) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(arg1); + len2 = VARSIZE_ANY_EXHDR(arg2); /* fast path for different-length inputs */ if (len1 != len2) result = false; else - result = (memcmp(VARDATA(arg1), VARDATA(arg2), len1) == 0); + result = (memcmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), len1) == 0); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -1972,20 +1962,20 @@ byteaeq(PG_FUNCTION_ARGS) Datum byteane(PG_FUNCTION_ARGS) { - bytea *arg1 = PG_GETARG_BYTEA_P(0); - bytea *arg2 = PG_GETARG_BYTEA_P(1); + bytea *arg1 = PG_GETARG_BYTEA_PP(0); + bytea *arg2 = PG_GETARG_BYTEA_PP(1); int len1, len2; bool result; - len1 = VARSIZE(arg1) - VARHDRSZ; - len2 = VARSIZE(arg2) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(arg1); + len2 = VARSIZE_ANY_EXHDR(arg2); /* fast path for different-length inputs */ if (len1 != len2) result = true; else - result = (memcmp(VARDATA(arg1), VARDATA(arg2), len1) != 0); + result = (memcmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), len1) != 0); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -1996,16 +1986,16 @@ byteane(PG_FUNCTION_ARGS) Datum bytealt(PG_FUNCTION_ARGS) { - bytea *arg1 = PG_GETARG_BYTEA_P(0); - bytea *arg2 = PG_GETARG_BYTEA_P(1); + bytea *arg1 = PG_GETARG_BYTEA_PP(0); + bytea *arg2 = PG_GETARG_BYTEA_PP(1); int len1, len2; int cmp; - len1 = VARSIZE(arg1) - VARHDRSZ; - len2 = VARSIZE(arg2) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(arg1); + len2 = VARSIZE_ANY_EXHDR(arg2); - cmp = memcmp(VARDATA(arg1), VARDATA(arg2), Min(len1, len2)); + cmp = memcmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), Min(len1, len2)); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -2016,16 +2006,16 @@ bytealt(PG_FUNCTION_ARGS) Datum byteale(PG_FUNCTION_ARGS) { - bytea *arg1 = PG_GETARG_BYTEA_P(0); - bytea *arg2 = PG_GETARG_BYTEA_P(1); + bytea *arg1 = PG_GETARG_BYTEA_PP(0); + bytea *arg2 = PG_GETARG_BYTEA_PP(1); int len1, len2; int cmp; - len1 = VARSIZE(arg1) - VARHDRSZ; - len2 = VARSIZE(arg2) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(arg1); + len2 = VARSIZE_ANY_EXHDR(arg2); - cmp = memcmp(VARDATA(arg1), VARDATA(arg2), Min(len1, len2)); + cmp = memcmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), Min(len1, len2)); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -2036,16 +2026,16 @@ byteale(PG_FUNCTION_ARGS) Datum byteagt(PG_FUNCTION_ARGS) { - bytea *arg1 = PG_GETARG_BYTEA_P(0); - bytea *arg2 = PG_GETARG_BYTEA_P(1); + bytea *arg1 = PG_GETARG_BYTEA_PP(0); + bytea *arg2 = PG_GETARG_BYTEA_PP(1); int len1, len2; int cmp; - len1 = VARSIZE(arg1) - VARHDRSZ; - len2 = VARSIZE(arg2) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(arg1); + len2 = VARSIZE_ANY_EXHDR(arg2); - cmp = memcmp(VARDATA(arg1), VARDATA(arg2), Min(len1, len2)); + cmp = memcmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), Min(len1, len2)); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -2056,16 +2046,16 @@ byteagt(PG_FUNCTION_ARGS) Datum byteage(PG_FUNCTION_ARGS) { - bytea *arg1 = PG_GETARG_BYTEA_P(0); - bytea *arg2 = PG_GETARG_BYTEA_P(1); + bytea *arg1 = PG_GETARG_BYTEA_PP(0); + bytea *arg2 = PG_GETARG_BYTEA_PP(1); int len1, len2; int cmp; - len1 = VARSIZE(arg1) - VARHDRSZ; - len2 = VARSIZE(arg2) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(arg1); + len2 = VARSIZE_ANY_EXHDR(arg2); - cmp = memcmp(VARDATA(arg1), VARDATA(arg2), Min(len1, len2)); + cmp = memcmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), Min(len1, len2)); PG_FREE_IF_COPY(arg1, 0); PG_FREE_IF_COPY(arg2, 1); @@ -2076,16 +2066,16 @@ byteage(PG_FUNCTION_ARGS) Datum byteacmp(PG_FUNCTION_ARGS) { - bytea *arg1 = PG_GETARG_BYTEA_P(0); - bytea *arg2 = PG_GETARG_BYTEA_P(1); + bytea *arg1 = PG_GETARG_BYTEA_PP(0); + bytea *arg2 = PG_GETARG_BYTEA_PP(1); int len1, len2; int cmp; - len1 = VARSIZE(arg1) - VARHDRSZ; - len2 = VARSIZE(arg2) - VARHDRSZ; + len1 = VARSIZE_ANY_EXHDR(arg1); + len2 = VARSIZE_ANY_EXHDR(arg2); - cmp = memcmp(VARDATA(arg1), VARDATA(arg2), Min(len1, len2)); + cmp = memcmp(VARDATA_ANY(arg1), VARDATA_ANY(arg2), Min(len1, len2)); if ((cmp == 0) && (len1 != len2)) cmp = (len1 < len2) ? -1 : 1; @@ -2713,8 +2703,8 @@ array_to_text(PG_FUNCTION_ARGS) appendStringInfoString(&buf, value); printed = true; - p = att_addlength(p, typlen, PointerGetDatum(p)); - p = (char *) att_align(p, typalign); + p = att_addlength_pointer(p, typlen, p); + p = (char *) att_align_nominal(p, typalign); } /* advance bitmap pointer if any */ @@ -2795,16 +2785,16 @@ to_hex64(PG_FUNCTION_ARGS) Datum md5_text(PG_FUNCTION_ARGS) { - text *in_text = PG_GETARG_TEXT_P(0); + text *in_text = PG_GETARG_TEXT_PP(0); size_t len; char hexsum[MD5_HASH_LEN + 1]; text *result_text; /* Calculate the length of the buffer using varlena metadata */ - len = VARSIZE(in_text) - VARHDRSZ; + len = VARSIZE_ANY_EXHDR(in_text); /* get the hash result */ - if (pg_md5_hash(VARDATA(in_text), len, hexsum) == false) + if (pg_md5_hash(VARDATA_ANY(in_text), len, hexsum) == false) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); @@ -2821,13 +2811,13 @@ md5_text(PG_FUNCTION_ARGS) Datum md5_bytea(PG_FUNCTION_ARGS) { - bytea *in = PG_GETARG_BYTEA_P(0); + bytea *in = PG_GETARG_BYTEA_PP(0); size_t len; char hexsum[MD5_HASH_LEN + 1]; text *result_text; - len = VARSIZE(in) - VARHDRSZ; - if (pg_md5_hash(VARDATA(in), len, hexsum) == false) + len = VARSIZE_ANY_EXHDR(in); + if (pg_md5_hash(VARDATA_ANY(in), len, hexsum) == false) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); diff --git a/src/backend/utils/adt/xml.c b/src/backend/utils/adt/xml.c index f039bba21e..e873264e3a 100644 --- a/src/backend/utils/adt/xml.c +++ b/src/backend/utils/adt/xml.c @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/backend/utils/adt/xml.c,v 1.41 2007/04/05 13:53:23 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/adt/xml.c,v 1.42 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -3071,8 +3071,8 @@ xmlpath(PG_FUNCTION_ARGS) else ns_uris[i - ns_count] = DatumGetCString(DirectFunctionCall1(textout, PointerGetDatum(ptr))); - ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr)); - ptr = (char *) att_align(ptr, typalign); + ptr = att_addlength_pointer(ptr, typlen, ptr); + ptr = (char *) att_align_nominal(ptr, typalign); } /* advance bitmap pointer if any */ diff --git a/src/backend/utils/fmgr/fmgr.c b/src/backend/utils/fmgr/fmgr.c index a28acdc2ad..5a67a84cf8 100644 --- a/src/backend/utils/fmgr/fmgr.c +++ b/src/backend/utils/fmgr/fmgr.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/utils/fmgr/fmgr.c,v 1.105 2007/03/27 23:21:10 tgl Exp $ + * $PostgreSQL: pgsql/src/backend/utils/fmgr/fmgr.c,v 1.106 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -1962,7 +1962,7 @@ struct varlena * pg_detoast_datum(struct varlena * datum) { if (VARATT_IS_EXTENDED(datum)) - return (struct varlena *) heap_tuple_untoast_attr((varattrib *) datum); + return heap_tuple_untoast_attr(datum); else return datum; } @@ -1971,7 +1971,7 @@ struct varlena * pg_detoast_datum_copy(struct varlena * datum) { if (VARATT_IS_EXTENDED(datum)) - return (struct varlena *) heap_tuple_untoast_attr((varattrib *) datum); + return heap_tuple_untoast_attr(datum); else { /* Make a modifiable copy of the varlena object */ @@ -1987,7 +1987,16 @@ struct varlena * pg_detoast_datum_slice(struct varlena * datum, int32 first, int32 count) { /* Only get the specified portion from the toast rel */ - return (struct varlena *) heap_tuple_untoast_attr_slice((varattrib *) datum, first, count); + return heap_tuple_untoast_attr_slice(datum, first, count); +} + +struct varlena * +pg_detoast_datum_packed(struct varlena * datum) +{ + if (VARATT_IS_COMPRESSED(datum) || VARATT_IS_EXTERNAL(datum)) + return heap_tuple_untoast_attr(datum); + else + return datum; } /*------------------------------------------------------------------------- diff --git a/src/backend/utils/init/flatfiles.c b/src/backend/utils/init/flatfiles.c index 1699979c3d..e388d1edfd 100644 --- a/src/backend/utils/init/flatfiles.c +++ b/src/backend/utils/init/flatfiles.c @@ -23,7 +23,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/backend/utils/init/flatfiles.c,v 1.24 2007/01/05 22:19:43 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/utils/init/flatfiles.c,v 1.25 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -467,7 +467,7 @@ write_auth_file(Relation rel_authid, Relation rel_authmem) auth_info[curr_role].rolpassword = DatumGetCString(DirectFunctionCall1(textout, datum)); /* assume passwd has attlen -1 */ - off = att_addlength(off, -1, tp + off); + off = att_addlength_pointer(off, -1, tp + off); } if (HeapTupleHasNulls(tuple) && @@ -482,7 +482,7 @@ write_auth_file(Relation rel_authid, Relation rel_authmem) * rolvaliduntil is timestamptz, which we assume is double * alignment and pass-by-reference. */ - off = att_align(off, 'd'); + off = att_align_nominal(off, 'd'); datum = PointerGetDatum(tp + off); auth_info[curr_role].rolvaliduntil = DatumGetCString(DirectFunctionCall1(timestamptz_out, datum)); } diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index 6c7c98b3f2..1fbb713b55 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.121 2007/03/29 00:15:39 tgl Exp $ + * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.122 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -200,7 +200,8 @@ extern Size heap_compute_data_size(TupleDesc tupleDesc, Datum *values, bool *isnull); extern void heap_fill_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull, - char *data, uint16 *infomask, bits8 *bit); + char *data, Size data_size, + uint16 *infomask, bits8 *bit); extern bool heap_attisnull(HeapTuple tup, int attnum); extern Datum nocachegetattr(HeapTuple tup, int attnum, TupleDesc att, bool *isnull); diff --git a/src/include/access/htup.h b/src/include/access/htup.h index c37501d910..ee816c568a 100644 --- a/src/include/access/htup.h +++ b/src/include/access/htup.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/access/htup.h,v 1.92 2007/02/27 23:48:09 tgl Exp $ + * $PostgreSQL: pgsql/src/include/access/htup.h,v 1.93 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -160,9 +160,8 @@ typedef HeapTupleHeaderData *HeapTupleHeader; #define HEAP_HASNULL 0x0001 /* has null attribute(s) */ #define HEAP_HASVARWIDTH 0x0002 /* has variable-width attribute(s) */ #define HEAP_HASEXTERNAL 0x0004 /* has external stored attribute(s) */ -#define HEAP_HASCOMPRESSED 0x0008 /* has compressed stored attribute(s) */ -#define HEAP_HASEXTENDED 0x000C /* the two above combined */ -#define HEAP_HASOID 0x0010 /* has an object-id field */ +#define HEAP_HASOID 0x0008 /* has an object-id field */ +/* bit 0x0010 is available */ #define HEAP_COMBOCID 0x0020 /* t_cid is a combo cid */ #define HEAP_XMAX_EXCL_LOCK 0x0040 /* xmax is exclusive locker */ #define HEAP_XMAX_SHARED_LOCK 0x0080 /* xmax is shared locker */ @@ -341,7 +340,7 @@ do { \ * MaxAttrSize is a somewhat arbitrary upper limit on the declared size of * data fields of char(n) and similar types. It need not have anything * directly to do with the *actual* upper limit of varlena values, which - * is currently 1Gb (see struct varattrib in postgres.h). I've set it + * is currently 1Gb (see TOAST structures in postgres.h). I've set it * at 10Mb which seems like a reasonable number --- tgl 8/6/00. */ #define MaxAttrSize (10 * 1024 * 1024) @@ -485,12 +484,6 @@ typedef HeapTupleData *HeapTuple; #define HeapTupleHasExternal(tuple) \ (((tuple)->t_data->t_infomask & HEAP_HASEXTERNAL) != 0) -#define HeapTupleHasCompressed(tuple) \ - (((tuple)->t_data->t_infomask & HEAP_HASCOMPRESSED) != 0) - -#define HeapTupleHasExtended(tuple) \ - (((tuple)->t_data->t_infomask & HEAP_HASEXTENDED) != 0) - #define HeapTupleGetOid(tuple) \ HeapTupleHeaderGetOid((tuple)->t_data) diff --git a/src/include/access/tupmacs.h b/src/include/access/tupmacs.h index 80c2d436f0..f4fb8c7b33 100644 --- a/src/include/access/tupmacs.h +++ b/src/include/access/tupmacs.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/access/tupmacs.h,v 1.32 2007/02/27 23:48:09 tgl Exp $ + * $PostgreSQL: pgsql/src/include/access/tupmacs.h,v 1.33 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -91,26 +91,83 @@ #endif /* SIZEOF_DATUM == 8 */ /* - * att_align aligns the given offset as needed for a datum of alignment - * requirement attalign. The cases are tested in what is hopefully something - * like their frequency of occurrence. + * att_align_datum aligns the given offset as needed for a datum of alignment + * requirement attalign and typlen attlen. attdatum is the Datum variable + * we intend to pack into a tuple (it's only accessed if we are dealing with + * a varlena type). Note that this assumes the Datum will be stored as-is; + * callers that are intending to convert non-short varlena datums to short + * format have to account for that themselves. */ -#define att_align(cur_offset, attalign) \ +#define att_align_datum(cur_offset, attalign, attlen, attdatum) \ ( \ - ((attalign) == 'i') ? INTALIGN(cur_offset) : \ - (((attalign) == 'c') ? ((long)(cur_offset)) : \ - (((attalign) == 'd') ? DOUBLEALIGN(cur_offset) : \ - ( \ - AssertMacro((attalign) == 's'), \ - SHORTALIGN(cur_offset) \ - ))) \ + ((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? (long) (cur_offset) : \ + att_align_nominal(cur_offset, attalign) \ ) /* - * att_addlength increments the given offset by the length of the attribute. - * attval is only accessed if we are dealing with a variable-length attribute. + * att_align_pointer performs the same calculation as att_align_datum, + * but is used when walking a tuple. attptr is the current actual data + * pointer; when accessing a varlena field we have to "peek" to see if we + * are looking at a pad byte or the first byte of a 1-byte-header datum. + * (A zero byte must be either a pad byte, or the first byte of a correctly + * aligned 4-byte length word; in either case we can align safely. A non-zero + * byte must be either a 1-byte length word, or the first byte of a correctly + * aligned 4-byte length word; in either case we need not align.) + * + * Note: some callers pass a "char *" pointer for cur_offset. This is + * a bit of a hack but works OK on all known platforms. It ought to be + * cleaned up someday, though. */ -#define att_addlength(cur_offset, attlen, attval) \ +#define att_align_pointer(cur_offset, attalign, attlen, attptr) \ +( \ + ((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? (long) (cur_offset) : \ + att_align_nominal(cur_offset, attalign) \ +) + +/* + * att_align_nominal aligns the given offset as needed for a datum of alignment + * requirement attalign, ignoring any consideration of packed varlena datums. + * There are three main use cases for using this macro directly: + * * we know that the att in question is not varlena (attlen != -1); + * in this case it is cheaper than the above macros and just as good. + * * we need to estimate alignment padding cost abstractly, ie without + * reference to a real tuple. We must assume the worst case that + * all varlenas are aligned. + * * within arrays, we unconditionally align varlenas (XXX this should be + * revisited, probably). + * + * The attalign cases are tested in what is hopefully something like their + * frequency of occurrence. + */ +#define att_align_nominal(cur_offset, attalign) \ +( \ + ((attalign) == 'i') ? INTALIGN(cur_offset) : \ + (((attalign) == 'c') ? (long) (cur_offset) : \ + (((attalign) == 'd') ? DOUBLEALIGN(cur_offset) : \ + ( \ + AssertMacro((attalign) == 's'), \ + SHORTALIGN(cur_offset) \ + ))) \ +) + +/* + * att_addlength_datum increments the given offset by the space needed for + * the given Datum variable. attdatum is only accessed if we are dealing + * with a variable-length attribute. + */ +#define att_addlength_datum(cur_offset, attlen, attdatum) \ + att_addlength_pointer(cur_offset, attlen, DatumGetPointer(attdatum)) + +/* + * att_addlength_pointer performs the same calculation as att_addlength_datum, + * but is used when walking a tuple --- attptr is the pointer to the field + * within the tuple. + * + * Note: some callers pass a "char *" pointer for cur_offset. This is + * actually perfectly OK, but probably should be cleaned up along with + * the same practice for att_align_pointer. + */ +#define att_addlength_pointer(cur_offset, attlen, attptr) \ ( \ ((attlen) > 0) ? \ ( \ @@ -118,12 +175,12 @@ ) \ : (((attlen) == -1) ? \ ( \ - (cur_offset) + VARSIZE(DatumGetPointer(attval)) \ + (cur_offset) + VARSIZE_ANY(attptr) \ ) \ : \ ( \ AssertMacro((attlen) == -2), \ - (cur_offset) + (strlen(DatumGetCString(attval)) + 1) \ + (cur_offset) + (strlen((char *) (attptr)) + 1) \ )) \ ) diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h index 3ab2997946..ea870aff11 100644 --- a/src/include/access/tuptoaster.h +++ b/src/include/access/tuptoaster.h @@ -6,7 +6,7 @@ * * Copyright (c) 2000-2007, PostgreSQL Global Development Group * - * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.34 2007/04/03 04:14:26 tgl Exp $ + * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.35 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -111,7 +111,7 @@ extern void toast_delete(Relation rel, HeapTuple oldtup); * in compressed format. * ---------- */ -extern varattrib *heap_tuple_fetch_attr(varattrib *attr); +extern struct varlena *heap_tuple_fetch_attr(struct varlena *attr); /* ---------- * heap_tuple_untoast_attr() - @@ -120,7 +120,7 @@ extern varattrib *heap_tuple_fetch_attr(varattrib *attr); * it as needed. * ---------- */ -extern varattrib *heap_tuple_untoast_attr(varattrib *attr); +extern struct varlena *heap_tuple_untoast_attr(struct varlena *attr); /* ---------- * heap_tuple_untoast_attr_slice() - @@ -129,7 +129,7 @@ extern varattrib *heap_tuple_untoast_attr(varattrib *attr); * (Handles all cases for attribute storage) * ---------- */ -extern varattrib *heap_tuple_untoast_attr_slice(varattrib *attr, +extern struct varlena *heap_tuple_untoast_attr_slice(struct varlena *attr, int32 sliceoffset, int32 slicelength); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 9525a198fe..68486709cc 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -37,7 +37,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.399 2007/04/02 03:49:40 tgl Exp $ + * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.400 2007/04/06 04:21:43 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 200704012 +#define CATALOG_VERSION_NO 200704051 #endif diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 6971319600..519be08afc 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -8,7 +8,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/pg_type.h,v 1.181 2007/04/02 03:49:41 tgl Exp $ + * $PostgreSQL: pgsql/src/include/catalog/pg_type.h,v 1.182 2007/04/06 04:21:43 tgl Exp $ * * NOTES * the genbki.sh script reads this file and generates .bki @@ -130,8 +130,10 @@ CATALOG(pg_type,1247) BKI_BOOTSTRAP * 'i' = INT alignment (4 bytes on most machines). * 'd' = DOUBLE alignment (8 bytes on many machines, but by no means all). * - * See include/utils/memutils.h for the macros that compute these - * alignment requirements. + * See include/access/tupmacs.h for the macros that compute these + * alignment requirements. Note also that we allow the nominal alignment + * to be violated when storing "packed" varlenas; the TOAST mechanism + * takes care of hiding that from most code. * * NOTE: for types used in system tables, it is critical that the * size and alignment defined in pg_type agree with the way that the @@ -398,10 +400,10 @@ DATA(insert OID = 791 ( _money PGNSP PGUID -1 f b t \054 0 790 array_in arr DATA(insert OID = 829 ( macaddr PGNSP PGUID 6 f b t \054 0 0 macaddr_in macaddr_out macaddr_recv macaddr_send - - - i p f 0 -1 0 _null_ _null_ )); DESCR("XX:XX:XX:XX:XX:XX, MAC address"); #define MACADDROID 829 -DATA(insert OID = 869 ( inet PGNSP PGUID -1 f b t \054 0 0 inet_in inet_out inet_recv inet_send - - - i p f 0 -1 0 _null_ _null_ )); +DATA(insert OID = 869 ( inet PGNSP PGUID -1 f b t \054 0 0 inet_in inet_out inet_recv inet_send - - - i m f 0 -1 0 _null_ _null_ )); DESCR("IP address/netmask, host address, netmask optional"); #define INETOID 869 -DATA(insert OID = 650 ( cidr PGNSP PGUID -1 f b t \054 0 0 cidr_in cidr_out cidr_recv cidr_send - - - i p f 0 -1 0 _null_ _null_ )); +DATA(insert OID = 650 ( cidr PGNSP PGUID -1 f b t \054 0 0 cidr_in cidr_out cidr_recv cidr_send - - - i m f 0 -1 0 _null_ _null_ )); DESCR("network IP address/netmask, network address"); #define CIDROID 650 diff --git a/src/include/fmgr.h b/src/include/fmgr.h index f6462d43a8..298cd12c77 100644 --- a/src/include/fmgr.h +++ b/src/include/fmgr.h @@ -11,7 +11,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/fmgr.h,v 1.49 2007/01/05 22:19:50 momjian Exp $ + * $PostgreSQL: pgsql/src/include/fmgr.h,v 1.50 2007/04/06 04:21:44 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -153,6 +153,11 @@ extern void fmgr_info_copy(FmgrInfo *dstinfo, FmgrInfo *srcinfo, * if you need a modifiable copy of the input. Caller is expected to have * checked for null inputs first, if necessary. * + * pg_detoast_datum_packed() will return packed (1-byte header) datums + * unmodified. It will still expand an externally toasted or compressed datum. + * The resulting datum can be accessed using VARSIZE_ANY() and VARDATA_ANY() + * (beware of multiple evaluations in those macros!) + * * Note: it'd be nice if these could be macros, but I see no way to do that * without evaluating the arguments multiple times, which is NOT acceptable. */ @@ -160,6 +165,7 @@ extern struct varlena *pg_detoast_datum(struct varlena * datum); extern struct varlena *pg_detoast_datum_copy(struct varlena * datum); extern struct varlena *pg_detoast_datum_slice(struct varlena * datum, int32 first, int32 count); +extern struct varlena *pg_detoast_datum_packed(struct varlena * datum); #define PG_DETOAST_DATUM(datum) \ pg_detoast_datum((struct varlena *) DatumGetPointer(datum)) @@ -168,6 +174,8 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum, #define PG_DETOAST_DATUM_SLICE(datum,f,c) \ pg_detoast_datum_slice((struct varlena *) DatumGetPointer(datum), \ (int32) f, (int32) c) +#define PG_DETOAST_DATUM_PACKED(datum) \ + pg_detoast_datum_packed((struct varlena *) DatumGetPointer(datum)) /* * Support for cleaning up detoasted copies of inputs. This must only @@ -207,9 +215,13 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum, #define PG_GETARG_VARLENA_P(n) PG_DETOAST_DATUM(PG_GETARG_DATUM(n)) /* DatumGetFoo macros for varlena types will typically look like this: */ #define DatumGetByteaP(X) ((bytea *) PG_DETOAST_DATUM(X)) +#define DatumGetByteaPP(X) ((bytea *) PG_DETOAST_DATUM_PACKED(X)) #define DatumGetTextP(X) ((text *) PG_DETOAST_DATUM(X)) +#define DatumGetTextPP(X) ((text *) PG_DETOAST_DATUM_PACKED(X)) #define DatumGetBpCharP(X) ((BpChar *) PG_DETOAST_DATUM(X)) +#define DatumGetBpCharPP(X) ((BpChar *) PG_DETOAST_DATUM_PACKED(X)) #define DatumGetVarCharP(X) ((VarChar *) PG_DETOAST_DATUM(X)) +#define DatumGetVarCharPP(X) ((VarChar *) PG_DETOAST_DATUM_PACKED(X)) #define DatumGetHeapTupleHeader(X) ((HeapTupleHeader) PG_DETOAST_DATUM(X)) /* And we also offer variants that return an OK-to-write copy */ #define DatumGetByteaPCopy(X) ((bytea *) PG_DETOAST_DATUM_COPY(X)) @@ -224,9 +236,13 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum, #define DatumGetVarCharPSlice(X,m,n) ((VarChar *) PG_DETOAST_DATUM_SLICE(X,m,n)) /* GETARG macros for varlena types will typically look like this: */ #define PG_GETARG_BYTEA_P(n) DatumGetByteaP(PG_GETARG_DATUM(n)) +#define PG_GETARG_BYTEA_PP(n) DatumGetByteaPP(PG_GETARG_DATUM(n)) #define PG_GETARG_TEXT_P(n) DatumGetTextP(PG_GETARG_DATUM(n)) +#define PG_GETARG_TEXT_PP(n) DatumGetTextPP(PG_GETARG_DATUM(n)) #define PG_GETARG_BPCHAR_P(n) DatumGetBpCharP(PG_GETARG_DATUM(n)) +#define PG_GETARG_BPCHAR_PP(n) DatumGetBpCharPP(PG_GETARG_DATUM(n)) #define PG_GETARG_VARCHAR_P(n) DatumGetVarCharP(PG_GETARG_DATUM(n)) +#define PG_GETARG_VARCHAR_PP(n) DatumGetVarCharPP(PG_GETARG_DATUM(n)) #define PG_GETARG_HEAPTUPLEHEADER(n) DatumGetHeapTupleHeader(PG_GETARG_DATUM(n)) /* And we also offer variants that return an OK-to-write copy */ #define PG_GETARG_BYTEA_P_COPY(n) DatumGetByteaPCopy(PG_GETARG_DATUM(n)) diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 9b107e27f7..35c1be6f9c 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -671,6 +671,10 @@ /* Define to select Win32-style shared memory. */ #undef USE_WIN32_SHARED_MEMORY +/* Define to 1 if your processor stores words with the most significant byte + first (like Motorola and SPARC, unlike Intel and VAX). */ +#undef WORDS_BIGENDIAN + /* Number of bits in a file offset, on hosts where this is settable. */ #undef _FILE_OFFSET_BITS diff --git a/src/include/postgres.h b/src/include/postgres.h index 5234c34e29..1ff0e97e9b 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -10,7 +10,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1995, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/postgres.h,v 1.78 2007/03/23 20:24:41 tgl Exp $ + * $PostgreSQL: pgsql/src/include/postgres.h,v 1.79 2007/04/06 04:21:44 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -54,56 +54,219 @@ * ---------------------------------------------------------------- */ -/* ---------------- - * struct varattrib is the header of a varlena object that may have been - * TOASTed. Generally, only the code closely associated with TOAST logic - * should mess directly with struct varattrib or use the VARATT_FOO macros. - * ---------------- +/* + * struct varatt_external is a "TOAST pointer", that is, the information + * needed to fetch a stored-out-of-line Datum. The data is compressed + * if and only if va_extsize < va_rawsize - VARHDRSZ. This struct must not + * contain any padding, because we sometimes compare pointers using memcmp. + * + * Note that this information is stored unaligned within actual tuples, so + * you need to memcpy from the tuple into a local struct variable before + * you can look at these fields! (The reason we use memcmp is to avoid + * having to do that just to detect equality of two TOAST pointers...) */ -typedef struct varattrib +struct varatt_external { - int32 va_header_; /* External/compressed storage */ - /* flags and item size */ - union + int32 va_rawsize; /* Original data size (includes header) */ + int32 va_extsize; /* External saved size (doesn't) */ + Oid va_valueid; /* Unique ID of value within TOAST table */ + Oid va_toastrelid; /* RelID of TOAST table containing it */ +}; + +/* + * These structs describe the header of a varlena object that may have been + * TOASTed. Generally, don't reference these structs directly, but use the + * macros below. + * + * We use separate structs for the aligned and unaligned cases because the + * compiler might otherwise think it could generate code that assumes + * alignment while touching fields of a 1-byte-header varlena. + */ +typedef union +{ + struct /* Normal varlena (4-byte length) */ { - struct - { - int32 va_rawsize; /* Plain data size */ - char va_data[1]; /* Compressed data */ - } va_compressed; /* Compressed stored attribute */ + uint32 va_header; + char va_data[1]; + } va_4byte; + struct /* Compressed-in-line format */ + { + uint32 va_header; + uint32 va_rawsize; /* Original data size (excludes header) */ + char va_data[1]; /* Compressed data */ + } va_compressed; +} varattrib_4b; - struct - { - int32 va_rawsize; /* Plain data size */ - int32 va_extsize; /* External saved size */ - Oid va_valueid; /* Unique identifier of value */ - Oid va_toastrelid; /* RelID where to find chunks */ - } va_external; /* External stored attribute */ +typedef struct +{ + uint8 va_header; + char va_data[1]; /* Data or TOAST pointer */ +} varattrib_1b; - char va_data[1]; /* Plain stored attribute */ - } va_content; -} varattrib; +typedef struct +{ + uint8 va_header; + char va_data[sizeof(struct varatt_external)]; +} varattrib_pointer; -#define VARATT_FLAG_EXTERNAL 0x80000000 -#define VARATT_FLAG_COMPRESSED 0x40000000 -#define VARATT_MASK_FLAGS 0xc0000000 -#define VARATT_MASK_SIZE 0x3fffffff +/* + * Bit layouts for varlena headers on big-endian machines: + * + * 00xxxxxx 4-byte length word, aligned, uncompressed data (up to 1G) + * 01xxxxxx 4-byte length word, aligned, *compressed* data (up to 1G) + * 10000000 1-byte length word, unaligned, TOAST pointer + * 1xxxxxxx 1-byte length word, unaligned, uncompressed data (up to 126b) + * + * Bit layouts for varlena headers on little-endian machines: + * + * xxxxxx00 4-byte length word, aligned, uncompressed data (up to 1G) + * xxxxxx10 4-byte length word, aligned, *compressed* data (up to 1G) + * 00000001 1-byte length word, unaligned, TOAST pointer + * xxxxxxx1 1-byte length word, unaligned, uncompressed data (up to 126b) + * + * The "xxx" bits are the length field (which includes itself in all cases). + * In the big-endian case we mask to extract the length, in the little-endian + * case we shift. Note that in both cases the flag bits are in the physically + * first byte. Also, it is not possible for a 1-byte length word to be zero; + * this lets us disambiguate alignment padding bytes from the start of an + * unaligned datum. (We now *require* pad bytes to be filled with zero!) + */ -#define VARATT_SIZEP_DEPRECATED(PTR) (((varattrib *) (PTR))->va_header_) +/* + * Endian-dependent macros. These are considered internal --- use the + * external macros below instead of using these directly. + * + * Note: IS_1B is true for external toast records but VARSIZE_1B will return 0 + * for such records. Hence you should usually check for IS_EXTERNAL before + * checking for IS_1B. + */ -#define VARATT_IS_EXTENDED(PTR) \ - ((VARATT_SIZEP_DEPRECATED(PTR) & VARATT_MASK_FLAGS) != 0) -#define VARATT_IS_EXTERNAL(PTR) \ - ((VARATT_SIZEP_DEPRECATED(PTR) & VARATT_FLAG_EXTERNAL) != 0) -#define VARATT_IS_COMPRESSED(PTR) \ - ((VARATT_SIZEP_DEPRECATED(PTR) & VARATT_FLAG_COMPRESSED) != 0) +#ifdef WORDS_BIGENDIAN -/* These macros are the ones for non-TOAST code to use */ +#define VARATT_IS_4B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00) +#define VARATT_IS_4B_U(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x00) +#define VARATT_IS_4B_C(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0xC0) == 0x40) +#define VARATT_IS_1B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x80) +#define VARATT_IS_1B_E(PTR) \ + ((((varattrib_1b *) (PTR))->va_header) == 0x80) +#define VARATT_NOT_PAD_BYTE(PTR) \ + (*((uint8 *) (PTR)) != 0) -#define VARSIZE(PTR) (VARATT_SIZEP_DEPRECATED(PTR) & VARATT_MASK_SIZE) -#define VARDATA(PTR) (((varattrib *) (PTR))->va_content.va_data) +/* VARSIZE_4B() should only be used on known-aligned data */ +#define VARSIZE_4B(PTR) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF) +#define VARSIZE_1B(PTR) \ + (((varattrib_1b *) (PTR))->va_header & 0x7F) +/* Currently there is only one size of toast pointer, but someday maybe not */ +#define VARSIZE_1B_E(PTR) \ + (sizeof(varattrib_pointer)) -#define SET_VARSIZE(PTR,SIZE) (VARATT_SIZEP_DEPRECATED(PTR) = (SIZE)) +#define SET_VARSIZE_4B(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF) +#define SET_VARSIZE_4B_C(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000) +#define SET_VARSIZE_1B(PTR,len) \ + (((varattrib_1b *) (PTR))->va_header = (len) | 0x80) +#define SET_VARSIZE_1B_E(PTR) \ + (((varattrib_1b *) (PTR))->va_header = 0x80) + +#else /* !WORDS_BIGENDIAN */ + +#define VARATT_IS_4B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00) +#define VARATT_IS_4B_U(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x00) +#define VARATT_IS_4B_C(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x03) == 0x02) +#define VARATT_IS_1B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x01) +#define VARATT_IS_1B_E(PTR) \ + ((((varattrib_1b *) (PTR))->va_header) == 0x01) +#define VARATT_NOT_PAD_BYTE(PTR) \ + (*((uint8 *) (PTR)) != 0) + +/* VARSIZE_4B() should only be used on known-aligned data */ +#define VARSIZE_4B(PTR) \ + ((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF) +#define VARSIZE_1B(PTR) \ + ((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F) +/* Currently there is only one size of toast pointer, but someday maybe not */ +#define VARSIZE_1B_E(PTR) \ + (sizeof(varattrib_pointer)) + +#define SET_VARSIZE_4B(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2)) +#define SET_VARSIZE_4B_C(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02) +#define SET_VARSIZE_1B(PTR,len) \ + (((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01) +#define SET_VARSIZE_1B_E(PTR) \ + (((varattrib_1b *) (PTR))->va_header = 0x01) + +#endif /* WORDS_BIGENDIAN */ + +#define VARHDRSZ_SHORT 1 +#define VARATT_SHORT_MAX 0x7F +#define VARATT_CAN_MAKE_SHORT(PTR) \ + (VARATT_IS_4B_U(PTR) && \ + (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) <= VARATT_SHORT_MAX) +#define VARATT_CONVERTED_SHORT_SIZE(PTR) \ + (VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT) + +#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) +#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data) +#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data) + +#define VARRAWSIZE_4B_C(PTR) \ + (((varattrib_4b *) (PTR))->va_compressed.va_rawsize) + +/* Externally visible macros */ + +/* + * VARDATA, VARSIZE, and SET_VARSIZE are the recommended API for most code + * for varlena datatypes. Note that they only work on untoasted, + * 4-byte-header Datums! + * + * Code that wants to use 1-byte-header values without detoasting should + * use VARSIZE_ANY/VARSIZE_ANY_EXHDR/VARDATA_ANY. The other macros here + * should usually be used only by tuple assembly/disassembly code and + * code that specifically wants to work with still-toasted Datums. + */ +#define VARDATA(PTR) VARDATA_4B(PTR) +#define VARSIZE(PTR) VARSIZE_4B(PTR) + +#define VARSIZE_SHORT(PTR) VARSIZE_1B(PTR) +#define VARDATA_SHORT(PTR) VARDATA_1B(PTR) + +#define VARSIZE_EXTERNAL(PTR) VARSIZE_1B_E(PTR) + +#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR) +#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) +#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR) +#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR)) + +#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len) +#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len) +#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len) +#define SET_VARSIZE_EXTERNAL(PTR) SET_VARSIZE_1B_E(PTR) + +#define VARSIZE_ANY(PTR) \ + (VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR) : \ + (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR) : \ + VARSIZE_4B(PTR))) + +#define VARSIZE_ANY_EXHDR(PTR) \ + (VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-1 : \ + (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-1 : \ + VARSIZE_4B(PTR)-4)) + +/* caution: this will not work on an external or compressed-in-line Datum */ +#define VARDATA_ANY(PTR) \ + (VARATT_IS_1B(PTR) ? VARDATA_1B(PTR) : VARDATA_4B(PTR)) /* ---------------------------------------------------------------- diff --git a/src/include/utils/inet.h b/src/include/utils/inet.h index ca92f4ca9e..d94855bb22 100644 --- a/src/include/utils/inet.h +++ b/src/include/utils/inet.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/utils/inet.h,v 1.25 2007/01/05 22:19:59 momjian Exp $ + * $PostgreSQL: pgsql/src/include/utils/inet.h,v 1.26 2007/04/06 04:21:44 tgl Exp $ * *------------------------------------------------------------------------- */ @@ -39,13 +39,19 @@ typedef struct /* * Both INET and CIDR addresses are represented within Postgres as varlena - * objects, ie, there is a varlena header (basically a length word) in front - * of the struct type depicted above. - * - * Although these types are variable-length, the maximum length - * is pretty short, so we make no provision for TOASTing them. + * objects, ie, there is a varlena header in front of the struct type + * depicted above. This struct depicts what we actually have in memory + * in "uncompressed" cases. Note that since the maximum data size is only + * 18 bytes, INET/CIDR will invariably be stored into tuples using the + * 1-byte-header varlena format. However, we have to be prepared to cope + * with the 4-byte-header format too, because various code may helpfully + * try to "decompress" 1-byte-header datums. */ -typedef struct varlena inet; +typedef struct +{ + int32 vl_len_; /* Do not touch this field directly! */ + inet_struct inet_data; +} inet; /* @@ -64,7 +70,7 @@ typedef struct macaddr /* * fmgr interface macros */ -#define DatumGetInetP(X) ((inet *) DatumGetPointer(X)) +#define DatumGetInetP(X) ((inet *) PG_DETOAST_DATUM_PACKED(X)) #define InetPGetDatum(X) PointerGetDatum(X) #define PG_GETARG_INET_P(n) DatumGetInetP(PG_GETARG_DATUM(n)) #define PG_RETURN_INET_P(x) return InetPGetDatum(x) diff --git a/src/test/regress/expected/rowtypes.out b/src/test/regress/expected/rowtypes.out index dbf163cd69..6af1aa705f 100644 --- a/src/test/regress/expected/rowtypes.out +++ b/src/test/regress/expected/rowtypes.out @@ -248,3 +248,20 @@ order by thousand, tenthous; 999 | 9999 (25 rows) +-- Check some corner cases involving empty rowtypes +select ROW(); + row +----- + () +(1 row) + +select ROW() IS NULL; + ?column? +---------- + t +(1 row) + +select ROW() = ROW(); +ERROR: cannot compare rows of zero length +LINE 1: select ROW() = ROW(); + ^ diff --git a/src/test/regress/expected/strings.out b/src/test/regress/expected/strings.out index e11dfc35d7..bd82846b1a 100644 --- a/src/test/regress/expected/strings.out +++ b/src/test/regress/expected/strings.out @@ -940,6 +940,26 @@ SELECT substr(f1, 99995, 10) from toasttest; 567890 (4 rows) +DROP TABLE toasttest; +-- test internally compressing datums +-- this tests compressing a datum to a very small size which exercises a +-- corner case in packed-varlena handling: even though small, the compressed +-- datum must be given a 4-byte header because there are no bits to indicate +-- compression in a 1-byte header +CREATE TABLE toasttest (c char(4096)); +INSERT INTO toasttest VALUES('x'); +SELECT length(c), c::text FROM toasttest; + length | c +--------+--- + 1 | x +(1 row) + +SELECT c FROM toasttest; + c +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + x +(1 row) + DROP TABLE toasttest; -- -- test length diff --git a/src/test/regress/sql/rowtypes.sql b/src/test/regress/sql/rowtypes.sql index 779b20e863..7e5c554d3b 100644 --- a/src/test/regress/sql/rowtypes.sql +++ b/src/test/regress/sql/rowtypes.sql @@ -109,3 +109,7 @@ select thousand, tenthous from tenk1 where (thousand, tenthous) >= (997, 5000) order by thousand, tenthous; +-- Check some corner cases involving empty rowtypes +select ROW(); +select ROW() IS NULL; +select ROW() = ROW(); diff --git a/src/test/regress/sql/strings.sql b/src/test/regress/sql/strings.sql index 389ff63517..800028e4fc 100644 --- a/src/test/regress/sql/strings.sql +++ b/src/test/regress/sql/strings.sql @@ -329,6 +329,19 @@ SELECT substr(f1, 99995, 10) from toasttest; DROP TABLE toasttest; +-- test internally compressing datums + +-- this tests compressing a datum to a very small size which exercises a +-- corner case in packed-varlena handling: even though small, the compressed +-- datum must be given a 4-byte header because there are no bits to indicate +-- compression in a 1-byte header + +CREATE TABLE toasttest (c char(4096)); +INSERT INTO toasttest VALUES('x'); +SELECT length(c), c::text FROM toasttest; +SELECT c FROM toasttest; +DROP TABLE toasttest; + -- -- test length --