Decouple the values of TOAST_TUPLE_THRESHOLD and TOAST_MAX_CHUNK_SIZE.

Add the latter to the values checked in pg_control, since it can't be changed
without invalidating toast table content.  This commit in itself shouldn't
change any behavior, but it lays some necessary groundwork for experimentation
with these toast-control numbers.

Note: while TOAST_TUPLE_THRESHOLD can now be changed without initdb, some
thought still needs to be given to needs_toast_table() in toasting.c before
unleashing random changes.

doc/src/sgml/storage.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.15 2007/03/02 00:48:44 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.16 2007/04/03 04:14:26 tgl Exp $ -->
 
 <chapter id="storage">
 
@@ -240,8 +240,9 @@ of the LZ family of compression techniques.  See
 
 <para>
 Out-of-line values are divided (after compression if used) into chunks of at
-most <literal>TOAST_MAX_CHUNK_SIZE</> bytes (this value is a little less than
+most <symbol>TOAST_MAX_CHUNK_SIZE</> bytes (by default this value is chosen
-<literal>BLCKSZ/4</>, or about 2000 bytes by default).  Each chunk is stored
+so that four chunk rows will fit on a page, making it about 2000 bytes).
+Each chunk is stored
 as a separate row in the <acronym>TOAST</> table for the owning table.  Every
 <acronym>TOAST</> table has the columns <structfield>chunk_id</> (an OID
 identifying the particular <acronym>TOAST</>ed value),
@@ -260,10 +261,12 @@ regardless of the actual size of the represented value.
 
 <para>
 The <acronym>TOAST</> code is triggered only
-when a row value to be stored in a table is wider than <literal>BLCKSZ/4</>
+when a row value to be stored in a table is wider than
-bytes (normally 2 kB).  The <acronym>TOAST</> code will compress and/or move
+<symbol>TOAST_TUPLE_THRESHOLD</> bytes (normally 2 kB).
+The <acronym>TOAST</> code will compress and/or move
 field values out-of-line until the row value is shorter than
-<literal>BLCKSZ/4</> bytes or no more gains can be had.  During an UPDATE
+<symbol>TOAST_TUPLE_TARGET</> bytes (also normally 2 kB)
+or no more gains can be had.  During an UPDATE
 operation, values of unchanged fields are normally preserved as-is; so an
 UPDATE of a row with out-of-line values incurs no <acronym>TOAST</> costs if
 none of the out-of-line values change.

src/backend/access/heap/heapam.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.230 2007/03/29 00:15:37 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.231 2007/04/03 04:14:26 tgl Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -1420,7 +1420,13 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 	 * Note: below this point, heaptup is the data we actually intend to store
 	 * into the relation; tup is the caller's original untoasted data.
 	 */
-	if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD)
+	if (relation->rd_rel->relkind == RELKIND_TOASTVALUE)
+	{
+		/* toast table entries should never be recursively toasted */
+		Assert(!HeapTupleHasExternal(tup));
+		heaptup = tup;
+	}
+	else if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD)
 		heaptup = toast_insert_or_update(relation, tup, NULL,
 										 use_wal, use_fsm);
 	else
@@ -1777,7 +1783,12 @@ l1:
 	 * because we need to look at the contents of the tuple, but it's OK to
 	 * release the content lock on the buffer first.
 	 */
-	if (HeapTupleHasExternal(&tp))
+	if (relation->rd_rel->relkind == RELKIND_TOASTVALUE)
+	{
+		/* toast table entries should never be recursively toasted */
+		Assert(!HeapTupleHasExternal(&tp));
+	}
+	else if (HeapTupleHasExternal(&tp))
 		toast_delete(relation, &tp);
 
 	/*
@@ -2075,9 +2086,17 @@ l2:
 	 * We need to invoke the toaster if there are already any out-of-line
 	 * toasted values present, or if the new tuple is over-threshold.
 	 */
-	need_toast = (HeapTupleHasExternal(&oldtup) ||
+	if (relation->rd_rel->relkind == RELKIND_TOASTVALUE)
-				  HeapTupleHasExternal(newtup) ||
+	{
-				  newtup->t_len > TOAST_TUPLE_THRESHOLD);
+		/* toast table entries should never be recursively toasted */
+		Assert(!HeapTupleHasExternal(&oldtup));
+		Assert(!HeapTupleHasExternal(newtup));
+		need_toast = false;
+	}
+	else
+		need_toast = (HeapTupleHasExternal(&oldtup) ||
+					  HeapTupleHasExternal(newtup) ||
+					  newtup->t_len > TOAST_TUPLE_THRESHOLD);
 
 	pagefree = PageGetFreeSpace((Page) dp);
 

src/backend/access/heap/tuptoaster.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.72 2007/03/29 00:15:37 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.73 2007/04/03 04:14:26 tgl Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -291,6 +291,12 @@ toast_delete(Relation rel, HeapTuple oldtup)
 	Datum		toast_values[MaxHeapAttributeNumber];
 	bool		toast_isnull[MaxHeapAttributeNumber];
 
+	/*
+	 * We should only ever be called for tuples of plain relations ---
+	 * recursing on a toast rel is bad news.
+	 */
+	Assert(rel->rd_rel->relkind == RELKIND_RELATION);
+
 	/*
 	 * Get the tuple descriptor and break down the tuple into fields.
 	 *
@@ -360,6 +366,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	bool		has_nulls = false;
 
 	Size		maxDataLen;
+	Size		hoff;
 
 	char		toast_action[MaxHeapAttributeNumber];
 	bool		toast_isnull[MaxHeapAttributeNumber];
@@ -370,6 +377,12 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	bool		toast_free[MaxHeapAttributeNumber];
 	bool		toast_delold[MaxHeapAttributeNumber];
 
+	/*
+	 * We should only ever be called for tuples of plain relations ---
+	 * recursing on a toast rel is bad news.
+	 */
+	Assert(rel->rd_rel->relkind == RELKIND_RELATION);
+
 	/*
 	 * Get the tuple descriptor and break down the tuple(s) into fields.
 	 */
@@ -512,15 +525,15 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	 */
 
 	/* compute header overhead --- this should match heap_form_tuple() */
-	maxDataLen = offsetof(HeapTupleHeaderData, t_bits);
+	hoff = offsetof(HeapTupleHeaderData, t_bits);
 	if (has_nulls)
-		maxDataLen += BITMAPLEN(numAttrs);
+		hoff += BITMAPLEN(numAttrs);
 	if (newtup->t_data->t_infomask & HEAP_HASOID)
-		maxDataLen += sizeof(Oid);
+		hoff += sizeof(Oid);
-	maxDataLen = MAXALIGN(maxDataLen);
+	hoff = MAXALIGN(hoff);
-	Assert(maxDataLen == newtup->t_data->t_hoff);
+	Assert(hoff == newtup->t_data->t_hoff);
 	/* now convert to a limit on the tuple data size */
-	maxDataLen = TOAST_TUPLE_TARGET - maxDataLen;
+	maxDataLen = TOAST_TUPLE_TARGET - hoff;
 
 	/*
 	 * Look for attributes with attstorage 'x' to compress
@@ -583,7 +596,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 
 	/*
 	 * Second we look for attributes of attstorage 'x' or 'e' that are still
-	 * inline.
+	 * inline.  But skip this if there's no toast table to push them to.
 	 */
 	while (heap_compute_data_size(tupleDesc,
 								  toast_values, toast_isnull) > maxDataLen &&
@@ -695,7 +708,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	}
 
 	/*
-	 * Finally we store attributes of type 'm' external
+	 * Finally we store attributes of type 'm' external, if possible.
 	 */
 	while (heap_compute_data_size(tupleDesc,
 								  toast_values, toast_isnull) > maxDataLen &&

src/backend/access/transam/xlog.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.265 2007/03/03 20:02:26 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.266 2007/04/03 04:14:26 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -28,6 +28,7 @@
 #include "access/multixact.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
+#include "access/tuptoaster.h"
 #include "access/twophase.h"
 #include "access/xact.h"
 #include "access/xlog_internal.h"
@@ -3634,6 +3635,8 @@ WriteControlFile(void)
 	ControlFile->nameDataLen = NAMEDATALEN;
 	ControlFile->indexMaxKeys = INDEX_MAX_KEYS;
 
+	ControlFile->toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;
+
 #ifdef HAVE_INT64_TIMESTAMP
 	ControlFile->enableIntTimes = TRUE;
 #else
@@ -3824,6 +3827,13 @@ ReadControlFile(void)
 					  " but the server was compiled with INDEX_MAX_KEYS %d.",
 						   ControlFile->indexMaxKeys, INDEX_MAX_KEYS),
 				 errhint("It looks like you need to recompile or initdb.")));
+	if (ControlFile->toast_max_chunk_size != TOAST_MAX_CHUNK_SIZE)
+		ereport(FATAL,
+				(errmsg("database files are incompatible with server"),
+				 errdetail("The database cluster was initialized with TOAST_MAX_CHUNK_SIZE %d,"
+						   " but the server was compiled with TOAST_MAX_CHUNK_SIZE %d.",
+						   ControlFile->toast_max_chunk_size, (int) TOAST_MAX_CHUNK_SIZE),
+				 errhint("It looks like you need to recompile or initdb.")));
 
 #ifdef HAVE_INT64_TIMESTAMP
 	if (ControlFile->enableIntTimes != TRUE)

src/bin/pg_controldata/pg_controldata.c

@@ -6,7 +6,7 @@
  * copyright (c) Oliver Elphick <olly@lfix.co.uk>, 2001;
  * licence: BSD
  *
- * $PostgreSQL: pgsql/src/bin/pg_controldata/pg_controldata.c,v 1.34 2007/03/18 16:50:43 neilc Exp $
+ * $PostgreSQL: pgsql/src/bin/pg_controldata/pg_controldata.c,v 1.35 2007/04/03 04:14:26 tgl Exp $
  */
 #include "postgres.h"
 
@@ -199,6 +199,8 @@ main(int argc, char *argv[])
 		   ControlFile.nameDataLen);
 	printf(_("Maximum columns in an index:          %u\n"),
 		   ControlFile.indexMaxKeys);
+	printf(_("Maximum size of a TOAST chunk:        %u\n"),
+		   ControlFile.toast_max_chunk_size);
 	printf(_("Date/time type storage:               %s\n"),
 		   (ControlFile.enableIntTimes ? _("64-bit integers") : _("floating-point numbers")));
 	printf(_("Maximum length of locale name:        %u\n"),

src/bin/pg_resetxlog/pg_resetxlog.c

@@ -23,7 +23,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/bin/pg_resetxlog/pg_resetxlog.c,v 1.58 2007/03/03 20:02:27 momjian Exp $
+ * $PostgreSQL: pgsql/src/bin/pg_resetxlog/pg_resetxlog.c,v 1.59 2007/04/03 04:14:26 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -41,6 +41,7 @@
 #endif
 
 #include "access/transam.h"
+#include "access/tuptoaster.h"
 #include "access/multixact.h"
 #include "access/xlog_internal.h"
 #include "catalog/catversion.h"
@@ -484,6 +485,7 @@ GuessControlValues(void)
 	ControlFile.xlog_seg_size = XLOG_SEG_SIZE;
 	ControlFile.nameDataLen = NAMEDATALEN;
 	ControlFile.indexMaxKeys = INDEX_MAX_KEYS;
+	ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;
 #ifdef HAVE_INT64_TIMESTAMP
 	ControlFile.enableIntTimes = TRUE;
 #else
@@ -572,6 +574,8 @@ PrintControlValues(bool guessed)
 		   ControlFile.nameDataLen);
 	printf(_("Maximum columns in an index:          %u\n"),
 		   ControlFile.indexMaxKeys);
+	printf(_("Maximum size of a TOAST chunk:        %u\n"),
+		   ControlFile.toast_max_chunk_size);
 	printf(_("Date/time type storage:               %s\n"),
 		   (ControlFile.enableIntTimes ? _("64-bit integers") : _("floating-point numbers")));
 	printf(_("Maximum length of locale name:        %u\n"),

src/include/access/tuptoaster.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 2000-2007, PostgreSQL Global Development Group
  *
- * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.33 2007/03/29 00:15:39 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.34 2007/04/03 04:14:26 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -29,19 +29,26 @@
  * TOAST_TUPLE_TARGET bytes.  Both numbers include all tuple header overhead
  * and between-fields alignment padding, but we do *not* consider any
  * end-of-tuple alignment padding; hence the values can be compared directly
- * to a tuple's t_len field.  We choose TOAST_TUPLE_THRESHOLD with the
+ * to a tuple's t_len field.
- * knowledge that toast-table tuples will be exactly that size, and we'd
- * like to fit four of them per page with minimal space wastage.
  *
- * The numbers need not be the same, though they currently are.
+ * The numbers need not be the same, though they currently are.  It doesn't
+ * make sense for TARGET to exceed THRESHOLD, but it could be useful to make
+ * it be smaller.
  *
- * Note: sizeof(PageHeaderData) includes the first ItemId, but we have
+ * Currently we choose both values to match the largest tuple size for which
- * to allow for 3 more, if we want to fit 4 tuples on a page.
+ * TOAST_TUPLES_PER_PAGE tuples can fit on a disk page.
+ *
+ * XXX while these can be modified without initdb, some thought needs to be
+ * given to needs_toast_table() in toasting.c before unleashing random
+ * changes.
  */
+#define TOAST_TUPLES_PER_PAGE	4
+
+/* Note: sizeof(PageHeaderData) includes the first ItemId on the page */
 #define TOAST_TUPLE_THRESHOLD	\
 	MAXALIGN_DOWN((BLCKSZ - \
-				   MAXALIGN(sizeof(PageHeaderData) + 3 * sizeof(ItemIdData))) \
+				   MAXALIGN(sizeof(PageHeaderData) + (TOAST_TUPLES_PER_PAGE-1) * sizeof(ItemIdData))) \
-				  / 4)
+				  / TOAST_TUPLES_PER_PAGE)
 
 #define TOAST_TUPLE_TARGET		TOAST_TUPLE_THRESHOLD
 
@@ -56,20 +63,26 @@
  * When we store an oversize datum externally, we divide it into chunks
  * containing at most TOAST_MAX_CHUNK_SIZE data bytes.	This number *must*
  * be small enough that the completed toast-table tuple (including the
- * ID and sequence fields and all overhead) is no more than MaxHeapTupleSize
+ * ID and sequence fields and all overhead) will fit on a page.
- * bytes.  It *should* be small enough to make toast-table tuples no more
+ * The coding here sets the size on the theory that we want to fit
- * than TOAST_TUPLE_THRESHOLD bytes, else heapam.c will uselessly invoke
+ * EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page.
- * the toaster on toast-table tuples.  The current coding ensures that the
- * maximum tuple length is exactly TOAST_TUPLE_THRESHOLD bytes.
  *
- * NB: you cannot change this value without forcing initdb, at least not
+ * NB: Changing TOAST_MAX_CHUNK_SIZE requires an initdb.
- * if your DB contains any multi-chunk toasted values.
  */
-#define TOAST_MAX_CHUNK_SIZE	(TOAST_TUPLE_THRESHOLD -			\
+#define EXTERN_TUPLES_PER_PAGE	4				/* tweak only this */
-				MAXALIGN(offsetof(HeapTupleHeaderData, t_bits)) -	\
+
-				sizeof(Oid) -										\
+/* Note: sizeof(PageHeaderData) includes the first ItemId on the page */
-				sizeof(int32) -										\
+#define EXTERN_TUPLE_MAX_SIZE	\
-				VARHDRSZ)
+	MAXALIGN_DOWN((BLCKSZ - \
+				   MAXALIGN(sizeof(PageHeaderData) + (EXTERN_TUPLES_PER_PAGE-1) * sizeof(ItemIdData))) \
+				  / EXTERN_TUPLES_PER_PAGE)
+
+#define TOAST_MAX_CHUNK_SIZE	\
+	(EXTERN_TUPLE_MAX_SIZE -							\
+	 MAXALIGN(offsetof(HeapTupleHeaderData, t_bits)) -	\
+	 sizeof(Oid) -										\
+	 sizeof(int32) -									\
+	 VARHDRSZ)
 
 
 /* ----------

src/include/catalog/pg_control.h

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/pg_control.h,v 1.36 2007/03/03 20:02:27 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/pg_control.h,v 1.37 2007/04/03 04:14:26 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -22,7 +22,7 @@
 
 
 /* Version identifier for this pg_control format */
-#define PG_CONTROL_VERSION	832
+#define PG_CONTROL_VERSION	833
 
 /*
  * Body of CheckPoint XLOG records.  This is declared here because we keep
@@ -135,6 +135,8 @@ typedef struct ControlFileData
 	uint32		nameDataLen;	/* catalog name field width */
 	uint32		indexMaxKeys;	/* max number of columns in an index */
 
+	uint32		toast_max_chunk_size;	/* chunk size in TOAST tables */
+
 	/* flag indicating internal format of timestamp, interval, time */
 	uint32		enableIntTimes; /* int64 storage enabled? */