Minor improvements in backup and recovery:

- create a separate archive_mode GUC, on which archive_command is dependent

- %r option in recovery.conf sends last restartpoint to recovery command

- %r used in pg_standby, updated README

- minor other code cleanup in pg_standby

- doc on Warm Standby now mentions pg_standby and %r

- log_restartpoints recovery option emits LOG message at each restartpoint

- end of recovery now displays last transaction end time, as requested
  by Warren Little; also shown at each restartpoint

- restart archiver if needed to carry away WAL files at shutdown

Simon Riggs

contrib/pg_standby/README.pg_standby

@@ -2,16 +2,18 @@ pg_standby README		2006/12/08 Simon Riggs
 
 o What is pg_standby?
 
-  pg_standby is a production-ready program that can be used to
+  pg_standby allows the creation of a Warm Standby server.
-  create a Warm Standby server. Other configuration is required
+  It is designed to be a production-ready program, as well as a 
-  as well, all of which is described in the main server manual.
+  customisable template should you require specific modifications.
+  Other configuration is required as well, all of which is 
+  described in the main server manual.
 
   The program is designed to be a wait-for restore_command, 
   required to turn a normal archive recovery into a Warm Standby.
   Within the restore_command of the recovery.conf you could
   configure pg_standby in the following way:
 
-    restore_command = 'pg_standby archiveDir %f %p'
+    restore_command = 'pg_standby archiveDir %f %p %r'
 
   which would be sufficient to define that files will be restored
   from archiveDir. 
@@ -42,18 +44,27 @@ o How to use pg_standby?
 
   The basic usage should be like this:
 
-    restore_command = 'pg_standby archiveDir %f %p'
+    restore_command = 'pg_standby archiveDir %f %p %r'
 
   with the pg_standby command usage as
 
-    pg_standby [OPTION]... [ARCHIVELOCATION] [NEXTWALFILE] [XLOGFILEPATH]
+    pg_standby [OPTION]... ARCHIVELOCATION NEXTWALFILE XLOGFILEPATH [RESTARTWALFILE]
 
   When used within the restore_command the %f and %p macros
   will provide the actual file and path required for the restore/recovery.
 
+  pg_standby assumes that ARCHIVELOCATION is directory accessible by the 
+  server-owning user.
+
+  If RESTARTWALFILE is specified, typically by using the %r option, then all files
+  prior to this file will be removed from ARCHIVELOCATION. This then minimises
+  the number of files that need to be held, whilst at the same time maintaining
+  restart capability. This capability additionally assumes that ARCHIVELOCATION
+  directory is writable.
+  
 o options
 
-  pg_standby has number of options.
+  pg_standby allows the following command line switches
 
 	-c
 		use copy/cp command to restore WAL files from archive
@@ -63,7 +74,10 @@ o options
 
 	-k numfiles
 		Cleanup files in the archive so that we maintain no more
-		than this many files in the archive. 
+		than this many files in the archive. This parameter will
+		be silently ignored if RESTARTWALFILE is specified, since
+		that specification method is more accurate in determining
+		the correct cut-off point in archive.
 
 		You should be wary against setting this number too low,
 		since this may mean you cannot restart the standby. This
@@ -75,8 +89,15 @@ o options
 		It is wholly unrelated to the setting of checkpoint_segments
 		on either primary or standby.
 
+		Setting numfiles to be zero will disable deletion of files
+		from ARCHIVELOCATION.
+
 		If in doubt, use a large value or do not set a value at all.
 
+		If you specify neither RESTARTWALFILE nor -k, then -k 0 
+		will be assumed, i.e. keep all files in archive.
+		Default=0, Min=0
+
 	-l 
 		use ln command to restore WAL files from archive
 		WAL files will remain in archive
@@ -84,6 +105,8 @@ o options
 		Link is more efficient, but the default is copy to 
 		allow you to maintain the WAL archive for recovery
 		purposes as well as high-availability.
+		The default setting is not necessarily recommended,
+		consult the main database server manual for discussion.
 
 		This option uses the Windows Vista command mklink
 		to provide a file-to-file symbolic link. -l will
@@ -99,14 +122,14 @@ o options
 		the failure back to the database server. This will be
 		interpreted as and end of recovery and the Standby will come
 		up fully as a result.
-		Default=3
+		Default=3, Min=0
 
 	-s sleeptime
 		the number of seconds to sleep between testing to see
 		if the file to be restored is available in the archive yet.
 		The default setting is not necessarily recommended,
 		consult the main database server manual for discussion.
-		Default=5
+		Default=5, Min=1, Max=60
 
 	-t triggerfile
 		the presence of the triggerfile will cause recovery to end
@@ -119,9 +142,10 @@ o options
 	-w maxwaittime
 		the maximum number of seconds to wait for the next file,
 		after which recovery will end and the Standby will come up.
+		A setting of zero means wait forever.
 		The default setting is not necessarily recommended,
 		consult the main database server manual for discussion.
-		Default=0
+		Default=0, Min=0
 
 	Note: --help is not supported since pg_standby is not intended
 		for interactive use, except during dev/test
@@ -148,8 +172,7 @@ o examples
 	Note that backslashes need to be doubled in the archive_command, but
 	*not* in the restore_command, in 8.2, 8.1, 8.0 on Windows.
 
-	restore_command = 'pg_standby -c -d -s 5 -w 0 -t C:\pgsql.trigger.5442
+	restore_command = 'pg_standby -c -d -s 5 -w 0 -t C:\pgsql.trigger.5442 ..\archive %f %p 2>> standby.log'
-..\archive %f %p 2>> standby.log'
 
   	which will
 	- use a copy command to restore WAL files from archive
@@ -158,7 +181,26 @@ o examples
 	- never timeout if file not found
 	- stop waiting when a trigger file called C:\pgsql.trigger.5442 appears
 
+o supported versions
+
+	pg_standby is designed to work with PostgreSQL 8.2 and later. It is
+	currently compatible across minor changes between the way 8.3 and 8.2
+	operate.
+
+	PostgreSQL 8.3 provides the %r command line substitution, designed to
+	let pg_standby know the last file it needs to keep. If the last
+	parameter is omitted, no error is generated, allowing pg_standby to
+	function correctly with PostgreSQL 8.2 also. With PostgreSQL 8.2,
+	the -k option must be used if archive cleanup is required. This option
+	remains available in 8.3.
+
 o reported test success
 
 	SUSE Linux 10.2
 	Windows XP Pro
+
+o additional design notes
+
+	The use of a move command seems like it would be a good idea, but
+	this would prevent recovery from being restartable. Also, the last WAL
+	file is always requested twice from the archive.

contrib/pg_standby/pg_standby.c

@@ -47,17 +47,20 @@ int maxwaittime = 0;		  	/* how long are we prepared to wait for? */
 int keepfiles = 0;				/* number of WAL files to keep, 0 keep all */
 int maxretries = 3;				/* number of retries on restore command */
 bool debug = false;			 	/* are we debugging? */
-bool triggered = false;
+bool triggered = false;			/* have we been triggered? */
-bool signaled = false;
+bool need_cleanup = false;		/* do we need to remove files from archive? */
+
+static volatile sig_atomic_t signaled = false;
 
 char *archiveLocation;		  /* where to find the archive? */
 char *triggerPath;			  /* where to find the trigger file? */
 char *xlogFilePath;			  /* where we are going to restore to */
 char *nextWALFileName;		  /* the file we need to get from archive */
+char *restartWALFileName;	  /* the file from which we can restart restore */
 char *priorWALFileName;		  /* the file we need to get from archive */
 char WALFilePath[MAXPGPATH];/* the file path including archive */
 char restoreCommand[MAXPGPATH]; /* run this to restore */
-char inclusiveCleanupFileName[MAXPGPATH];		  /* the file we need to get from archive */
+char exclusiveCleanupFileName[MAXPGPATH];		  /* the file we need to get from archive */
 
 #define RESTORE_COMMAND_COPY 0
 #define RESTORE_COMMAND_LINK 1
@@ -204,36 +207,15 @@ CustomizableNextWALFileReady()
 static void
 CustomizableCleanupPriorWALFiles(void)
 {
-	uint32			tli,
-					log,
-					seg;
-	int				signed_log = 0;
-
-	if (keepfiles > 0)
-	{
-		sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg);
-		signed_log = log - (keepfiles / MaxSegmentsPerLogFile);
-		if (keepfiles <= seg)
-			seg -= keepfiles;
-		else
-		{
-			seg = MaxSegmentsPerLogFile - (keepfiles % MaxSegmentsPerLogFile);			
-			signed_log--;	
-		}
-		log = (uint32) signed_log;
-	}
-
 	/*
 	 * Work out name of prior file from current filename
 	 */
-	if (keepfiles > 0 && signed_log >= 0 && nextWALFileType == XLOG_DATA)
+	if (nextWALFileType == XLOG_DATA)
 	{
 		int 			rc;
 		DIR				*xldir;
 		struct dirent	*xlde;
 
-		XLogFileName(inclusiveCleanupFileName, tli, log, seg);
-
 		/*
 		 * Assume its OK to keep failing. The failure situation may change over
 		 * time, so we'd rather keep going on the main processing than fail
@@ -252,11 +234,13 @@ CustomizableCleanupPriorWALFiles(void)
 				 * complicated.
 				 *
 				 * We use the alphanumeric sorting property of the filenames to decide
-				 * which ones are earlier than the inclusiveCleanupFileName file.
+				 * which ones are earlier than the exclusiveCleanupFileName file.
+				 * Note that this means files are not removed in the order they were 
+				 * originally written, in case this worries you.
 				 */
 				if (strlen(xlde->d_name) == XLOG_DATA_FNAME_LEN &&
 					strspn(xlde->d_name, "0123456789ABCDEF") == XLOG_DATA_FNAME_LEN &&
-					strcmp(xlde->d_name + 8, inclusiveCleanupFileName + 8) <= 0)
+					strcmp(xlde->d_name + 8, exclusiveCleanupFileName + 8) < 0)
 				{
 #ifdef WIN32
 					snprintf(WALFilePath, MAXPGPATH, "%s\\%s", archiveLocation, xlde->d_name);
@@ -265,22 +249,26 @@ CustomizableCleanupPriorWALFiles(void)
 #endif
 
 					if (debug)
-					   	fprintf(stderr, "\npg_standby: removing \"%s\"\n", WALFilePath);
+					   	fprintf(stderr, "\nremoving \"%s\"", WALFilePath);
 
 					rc = unlink(WALFilePath);
-					if (rc !=0 )
+					if (rc != 0)
-						fprintf(stderr, "\npg_standby: ERROR failed to remove \"%s\": %s\n", WALFilePath,  strerror(errno));
+					{
-
+						fprintf(stderr, "\npg_standby: ERROR failed to remove \"%s\": %s",
-
+								WALFilePath, strerror(errno));
+						break;
 					}
 				}
 			}
+			if (debug)
+				fprintf(stderr, "\n");
+		}
 		else
 		   	fprintf(stderr, "pg_standby: archiveLocation \"%s\" open error\n", archiveLocation);
 
 		closedir(xldir);
-	}
 		fflush(stderr);
+	}
 }
 
 /* =====================================================================
@@ -288,6 +276,61 @@ CustomizableCleanupPriorWALFiles(void)
  * =====================================================================
  */
 
+/*
+ * SetWALFileNameForCleanup()
+ * 
+ *	  Set the earliest WAL filename that we want to keep on the archive
+ *    and decide whether we need_cleanup
+ */
+static bool
+SetWALFileNameForCleanup(void)
+{
+	uint32			tli = 1,
+					log = 0,
+					seg = 0;
+	uint32			log_diff = 0,
+					seg_diff = 0;
+	bool			cleanup = false;
+
+	if (restartWALFileName)
+	{
+		strcpy(exclusiveCleanupFileName, restartWALFileName);
+		return true;
+	}
+
+	if (keepfiles > 0)
+	{
+		sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg);
+		if (tli > 0 && log >= 0 && seg > 0)
+		{
+			log_diff = keepfiles / MaxSegmentsPerLogFile;
+			seg_diff = keepfiles % MaxSegmentsPerLogFile;
+				if (seg_diff > seg)
+			{
+				log_diff++;
+				seg = MaxSegmentsPerLogFile - seg_diff;
+			}
+			else
+				seg -= seg_diff;
+
+			if (log >= log_diff)
+			{
+				log -= log_diff;
+				cleanup = true;
+			}
+			else
+			{
+				log = 0;
+				seg = 0;
+			}
+		}
+	}
+
+	XLogFileName(exclusiveCleanupFileName, tli, log, seg);
+
+	return cleanup;
+}
+
 /*
  * CheckForExternalTrigger()
  * 
@@ -353,7 +396,7 @@ RestoreWALFileForRecovery(void)
 		{
 			if (debug)
 			{
-				fprintf(stderr, " success\n");
+				fprintf(stderr, " OK");
 				fflush(stderr);
 			}
 			return true;
@@ -370,31 +413,30 @@ RestoreWALFileForRecovery(void)
 }
 
 static void
-usage()
+usage(void)
 {
 	fprintf(stderr, "\npg_standby allows Warm Standby servers to be configured\n");
 	fprintf(stderr, "Usage:\n");
-	fprintf(stderr, "  pg_standby [OPTION]... [ARCHIVELOCATION] [NEXTWALFILE] [XLOGFILEPATH]\n");
+	fprintf(stderr, "  pg_standby [OPTION]... ARCHIVELOCATION NEXTWALFILE XLOGFILEPATH [RESTARTWALFILE]\n");
-	fprintf(stderr, "						  note space between [ARCHIVELOCATION] and [NEXTWALFILE]\n");
+	fprintf(stderr, "				note space between ARCHIVELOCATION and NEXTWALFILE\n");
-	fprintf(stderr, "with main intended use via restore_command in the recovery.conf\n");
+	fprintf(stderr, "with main intended use as a restore_command in the recovery.conf\n");
-	fprintf(stderr, "	 restore_command = 'pg_standby [OPTION]... [ARCHIVELOCATION] %%f %%p'\n");
+	fprintf(stderr, "	 restore_command = 'pg_standby [OPTION]... ARCHIVELOCATION %%f %%p %%r'\n");
-	fprintf(stderr, "e.g. restore_command = 'pg_standby -l /mnt/server/archiverdir %%f %%p'\n");
+	fprintf(stderr, "e.g. restore_command = 'pg_standby -l /mnt/server/archiverdir %%f %%p %%r'\n");
 	fprintf(stderr, "\nOptions:\n");
 	fprintf(stderr, "  -c			copies file from archive (default)\n");
 	fprintf(stderr, "  -d			generate lots of debugging output (testing only)\n");
-	fprintf(stderr, "  -k [NUMFILESTOKEEP]	keeps history of # files in archives; unlinks/removes files beyond that\n");
+	fprintf(stderr, "  -k NUMFILESTOKEEP	if RESTARTWALFILE not used, removes files prior to limit (0 keeps all)\n");
 	fprintf(stderr, "  -l			links into archive (leaves file in archive)\n");
-	fprintf(stderr, "  -t [TRIGGERFILE]	defines a trigger file to initiate failover (no default)\n");
+	fprintf(stderr, "  -r MAXRETRIES		max number of times to retry, with progressive wait (default=3)\n");
-	fprintf(stderr, "  -r [MAXRETRIES]	maximum number of times to retry, with progressive wait (default=3)\n");
+	fprintf(stderr, "  -s SLEEPTIME		seconds to wait between file checks (min=1, max=60, default=5)\n");
-	fprintf(stderr, "  -s [SLEEPTIME]	number of seconds to wait between file checks (default=5)\n");
+	fprintf(stderr, "  -t TRIGGERFILE	defines a trigger file to initiate failover (no default)\n");
-	fprintf(stderr, "  -w [MAXWAITTIME]	max number of seconds to wait for a file (0 disables)(default=0)\n");
+	fprintf(stderr, "  -w MAXWAITTIME	max seconds to wait for a file (0=no limit)(default=0)\n");
 	fflush(stderr);
 }
 
 static void
 sighandler(int sig)
 {
-	triggered = true;
 	signaled = true;
 }
 
@@ -419,9 +461,9 @@ main(int argc, char **argv)
 				break;
 			case 'k':			/* keepfiles */
 				keepfiles = atoi(optarg);
-				if (keepfiles <= 0)
+				if (keepfiles < 0)
 				{
-					fprintf(stderr, "usage: pg_standby -k keepfiles must be > 0\n");
+					fprintf(stderr, "usage: pg_standby -k keepfiles must be >= 0\n");
 					usage();
 					exit(2);
 				}
@@ -433,7 +475,7 @@ main(int argc, char **argv)
 				maxretries = atoi(optarg);
 				if (maxretries < 0)
 				{
-					fprintf(stderr, "usage: pg_standby -r maxretries must be > 0\n");
+					fprintf(stderr, "usage: pg_standby -r maxretries must be >= 0\n");
 					usage();
 					exit(2);
 				}
@@ -519,23 +561,28 @@ main(int argc, char **argv)
 		exit(2);
 	}
 
+	if (optind < argc)
+	{
+		restartWALFileName = argv[optind];
+		optind++;
+	}
+
 	CustomizableInitialize();
 
+	need_cleanup = SetWALFileNameForCleanup();
+
 	if (debug)
 	{
         fprintf(stderr, "\nTrigger file 		: %s", triggerPath ? triggerPath : "<not set>");
-	   	fprintf(stderr, "\nWaiting for WAL file	: %s", WALFilePath);
+	   	fprintf(stderr, "\nWaiting for WAL file	: %s", nextWALFileName);
-	   	fprintf(stderr, "\nWAL file path		: %s", nextWALFileName);
+	   	fprintf(stderr, "\nWAL file path		: %s", WALFilePath);
 	   	fprintf(stderr, "\nRestoring to...		: %s", xlogFilePath);
 	   	fprintf(stderr, "\nSleep interval		: %d second%s", 
 					sleeptime, (sleeptime > 1 ? "s" : " "));
 	   	fprintf(stderr, "\nMax wait interval	: %d %s", 
 					maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
 		fprintf(stderr, "\nCommand for restore	: %s", restoreCommand);
-		if (keepfiles > 0)
+		fprintf(stderr, "\nKeep archive history	: %s and later", exclusiveCleanupFileName);
-			fprintf(stderr, "\nNum archived files kept	: last %d files", keepfiles);
-		else
-			fprintf(stderr, "\nNum archived files kept	: all files");
 		fflush(stderr);
 	}
 
@@ -572,6 +619,7 @@ main(int argc, char **argv)
 
 		if (signaled)
 		{
+			triggered = true;
 			if (debug)
 			{
 			   	fprintf(stderr, "\nsignaled to exit\n");
@@ -614,7 +662,7 @@ main(int argc, char **argv)
 	 * immediately after the failed restore, or when
 	 * we restart recovery.
 	 */
-	if (RestoreWALFileForRecovery())
+	if (RestoreWALFileForRecovery() && need_cleanup)
 		CustomizableCleanupPriorWALFiles();
 
 	return 0;

doc/src/sgml/backup.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/backup.sgml,v 2.101 2007/09/14 13:26:22 momjian Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/backup.sgml,v 2.102 2007/09/26 22:36:30 tgl Exp $ -->
 
 <chapter id="backup">
  <title>Backup and Restore</title>
@@ -518,10 +518,13 @@ tar -cf backup.tar /usr/local/pgsql/data
    </para>
 
    <para>
-    The shell command to use is specified by the <xref
+    To enable WAL archiving, set the <xref
-    linkend="guc-archive-command"> configuration parameter, which in practice
+    linkend="guc-archive-mode"> configuration parameter to <literal>on</>,
-    will always be placed in the <filename>postgresql.conf</filename> file.
+    and specify the shell command to use in the <xref
-    In this string,
+    linkend="guc-archive-command"> configuration parameter.  In practice
+    these settings will always be placed in the
+    <filename>postgresql.conf</filename> file.
+    In <varname>archive_command</>,
     any <literal>%p</> is replaced by the path name of the file to
     archive, while any <literal>%f</> is replaced by the file name only.
     (The path name is relative to the working directory of the server,
@@ -1030,6 +1033,13 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
         it to on the server.
         (The path name is relative to the working directory of the server,
         i.e., the cluster's data directory.)
+        Any <literal>%r</> is replaced by the name of the file containing the
+        last valid restartpoint. That is the earliest file that must be kept
+        to allow a restore to be restartable, so this information can be used
+        to truncate the archive to just the minimum required to support
+        restart of the current restore. <literal>%r</> would only be used in a
+        warm-standby configuration (see <xref
+        linkend="warm-standby-planning">).
         Write <literal>%%</> to embed an actual <literal>%</> character
         in the command.
        </para>
@@ -1118,6 +1128,20 @@ restore_command = 'copy /mnt/server/archivedir/%f "%p"'  # Windows
       </listitem>
      </varlistentry>
 
+     <varlistentry id="log-restartpoints" 
+                   xreflabel="log_restartpoints">
+      <term><varname>log_restartpoints</varname> 
+        (<type>boolean</type>)
+      </term>
+      <listitem>
+       <para>
+        Specifies whether to log each restart point as it occurs. This 
+        can be helpful to track the progress of a long recovery.
+        Default is <literal>false</>.
+       </para>
+      </listitem>
+     </varlistentry>
+
    </variablelist>
 
    </sect3>
@@ -1389,7 +1413,8 @@ restore_command = 'copy /mnt/server/archivedir/%f "%p"'  # Windows
     contact between the two database servers is the archive of WAL files
     that both share: primary writing to the archive, standby reading from
     the archive. Care must be taken to ensure that WAL archives for separate
-    primary servers do not become mixed together or confused.
+    primary servers do not become mixed together or confused. The archive
+    need not be large, if it is only required for the standby operation.
    </para>
 
    <para>
@@ -1426,6 +1451,12 @@ if (!triggered)
 </programlisting>
    </para>
 
+   <para>
+    A working example of a waiting <varname>restore_command</> is provided
+    as a contrib module, named <application>pg_standby</>. This can be
+    extended as needed to support specific configurations or environments.
+   </para>
+
    <para>
     <productname>PostgreSQL</productname> does not provide the system
     software required to identify a failure on the primary and notify
@@ -1449,6 +1480,15 @@ if (!triggered)
     as the explicit creation of a trigger file is less error prone, if
     this can be arranged.
    </para>
+
+   <para>
+    The size of the WAL archive can be minimized by using the <literal>%r</>
+    option of the <varname>restore_command</>. This option specifies the
+    last archive filename that needs to be kept to allow the recovery to
+    restart correctly. This can be used to truncate the archive once
+    files are no longer required, if the archive is writable from the
+    standby server.
+   </para>
   </sect2>
 
   <sect2 id="warm-standby-config">
@@ -1468,8 +1508,10 @@ if (!triggered)
      <listitem>
       <para>
        Set up continuous archiving from the primary to a WAL archive located
-       in a directory on the standby server. Ensure that <xref
+       in a directory on the standby server. Ensure that
-       linkend="guc-archive-command"> and <xref linkend="guc-archive-timeout">
+       <xref linkend="guc-archive-mode">,
+       <xref linkend="guc-archive-command"> and
+       <xref linkend="guc-archive-timeout">
        are set appropriately on the primary
        (see <xref linkend="backup-archiving-wal">).
       </para>
@@ -1631,7 +1673,8 @@ if (!triggered)
     far back you need to keep WAL segment files to have a recoverable
     backup.  You can do this by running <application>pg_controldata</>
     on the standby server to inspect the control file and determine the
-    current checkpoint WAL location.
+    current checkpoint WAL location, or by using the 
+    <varname>log_restartpoints</> option to print values to the server log.
    </para>
   </sect2>
  </sect1>

doc/src/sgml/config.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/config.sgml,v 1.147 2007/09/25 20:03:37 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/config.sgml,v 1.148 2007/09/26 22:36:30 tgl Exp $ -->
 
 <chapter Id="runtime-config">
   <title>Server Configuration</title>
@@ -1620,6 +1620,24 @@ SET ENABLE_SEQSCAN TO OFF;
      <title>Archiving</title>
 
     <variablelist>
+     <varlistentry id="guc-archive-mode" xreflabel="archive_mode">
+      <term><varname>archive_mode</varname> (<type>boolean</type>)</term>
+      <indexterm>
+       <primary><varname>archive_mode</> configuration parameter</primary>
+      </indexterm>
+      <listitem>
+       <para>
+        When <varname>archive_mode</> is enabled, completed WAL segments
+        can be sent to archive storage by setting
+        <xref linkend="guc-archive-command">.
+        <varname>archive_mode</> and <varname>archive_command</> are
+        separate variables so that <varname>archive_command</> can be
+        changed without leaving archiving mode.
+        This parameter can only be set at server start.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-archive-command" xreflabel="archive_command">
       <term><varname>archive_command</varname> (<type>string</type>)</term>
       <indexterm>
@@ -1628,8 +1646,7 @@ SET ENABLE_SEQSCAN TO OFF;
       <listitem>
        <para>
         The shell command to execute to archive a completed segment of
-        the WAL file series. If this is an empty string (the default),
+        the WAL file series.  Any <literal>%p</> in the string is
-        WAL archiving is disabled. Any <literal>%p</> in the string is
         replaced by the path name of the file to archive, and any
         <literal>%f</> is replaced by the file name only.
         (The path name is relative to the working directory of the server,
@@ -1638,7 +1655,12 @@ SET ENABLE_SEQSCAN TO OFF;
         command. For more information see <xref
         linkend="backup-archiving-wal">.
         This parameter can only be set in the <filename>postgresql.conf</>
-        file or on the server command line.
+        file or on the server command line.  It is ignored unless
+        <varname>archive_mode</> was enabled at server start.
+        If <varname>archive_command</> is an empty string (the default) while
+        <varname>archive_mode</> is enabled, then WAL archiving is temporarily
+        disabled, but the server continues to accumulate WAL segment files in
+        the expectation that a command will soon be provided.
        </para>
        <para>
         It is important for the command to return a zero exit status if

doc/src/sgml/perform.sgml

@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/perform.sgml,v 1.64 2007/03/29 00:15:36 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/perform.sgml,v 1.65 2007/09/26 22:36:30 tgl Exp $ -->
 
  <chapter id="performance-tips">
   <title>Performance Tips</title>
@@ -819,7 +819,7 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse;
     needs to be written, because in case of an error, the files
     containing the newly loaded data will be removed anyway.
     However, this consideration does not apply when
-    <xref linkend="guc-archive-command"> is set, as all commands
+    <xref linkend="guc-archive-mode"> is set, as all commands
     must write WAL in that case.
    </para>
 
@@ -892,22 +892,24 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse;
   </sect2>
 
   <sect2 id="populate-pitr">
-   <title>Turn off <varname>archive_command</varname></title>
+   <title>Turn off <varname>archive_mode</varname></title>
 
    <para>
     When loading large amounts of data into an installation that uses
-    WAL archiving, you might want to disable archiving (unset the
+    WAL archiving, you might want to disable archiving (turn off the
-    <xref linkend="guc-archive-command"> configuration variable)
+    <xref linkend="guc-archive-mode"> configuration variable)
     while loading.  It might be
     faster to take a new base backup after the load has completed
     than to process a large amount of incremental WAL data.
+    But note that turning <varname>archive_mode</varname> on or off
+    requires a server restart.
    </para>
 
    <para>
     Aside from avoiding the time for the archiver to process the WAL data,
     doing this will actually make certain commands faster, because they
-    are designed not to write WAL at all if <varname>archive_command</varname>
+    are designed not to write WAL at all if <varname>archive_mode</varname>
-    is unset.  (They can guarantee crash safety more cheaply by doing an
+    is off.  (They can guarantee crash safety more cheaply by doing an
     <function>fsync</> at the end than by writing WAL.)
     This applies to the following commands: 
     <itemizedlist>
@@ -989,8 +991,8 @@ SELECT * FROM x, y, a, b, c WHERE something AND somethingelse;
      <listitem>
       <para>
        If using WAL archiving, consider disabling it during the restore.
-       To do that, unset <varname>archive_command</varname> before loading the
+       To do that, turn off <varname>archive_mode</varname> before loading the
-       dump script, and afterwards restore <varname>archive_command</varname>
+       dump script, and afterwards turn it back on
        and take a fresh base backup.
       </para>
      </listitem>

doc/src/sgml/ref/show.sgml

@@ -1,5 +1,5 @@
 <!--
-$PostgreSQL: pgsql/doc/src/sgml/ref/show.sgml,v 1.43 2007/09/11 00:06:41 tgl Exp $
+$PostgreSQL: pgsql/doc/src/sgml/ref/show.sgml,v 1.44 2007/09/26 22:36:30 tgl Exp $
 PostgreSQL documentation
 -->
 
@@ -171,7 +171,7 @@ SHOW ALL;
               name              |            setting             |                                         description                                          
 --------------------------------+--------------------------------+----------------------------------------------------------------------------------------------
  add_missing_from               | off                            | Automatically adds missing table references to FROM clauses.
- archive_command                | unset                          | WAL archiving command.
+ allow_system_table_mods        | off                            | Allows modifications of the structure of system tables.
     .
     .
     .

src/backend/access/transam/xlog.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.281 2007/09/08 20:31:14 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.282 2007/09/26 22:36:30 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -62,6 +62,7 @@
 int			CheckPointSegments = 3;
 int			XLOGbuffers = 8;
 int			XLogArchiveTimeout = 0;
+bool		XLogArchiveMode = false;
 char	   *XLogArchiveCommand = NULL;
 char	   *XLOG_sync_method = NULL;
 const char	XLOG_sync_method_default[] = DEFAULT_SYNC_METHOD_STR;
@@ -120,8 +121,10 @@ static char *recoveryRestoreCommand = NULL;
 static bool recoveryTarget = false;
 static bool recoveryTargetExact = false;
 static bool recoveryTargetInclusive = true;
+static bool recoveryLogRestartpoints = false;
 static TransactionId recoveryTargetXid;
 static TimestampTz recoveryTargetTime;
+static TimestampTz recoveryLastXTime = 0;
 
 /* if recoveryStopsHere returns true, it saves actual stop xid/time here */
 static TransactionId recoveryStopXid;
@@ -2388,12 +2391,15 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 {
 	char		xlogpath[MAXPGPATH];
 	char		xlogRestoreCmd[MAXPGPATH];
+	char		lastRestartPointFname[MAXPGPATH];
 	char	   *dp;
 	char	   *endp;
 	const char *sp;
 	int			rc;
 	bool		signaled;
 	struct stat stat_buf;
+	uint32 		restartLog;
+	uint32 		restartSeg;
 
 	/*
 	 * When doing archive recovery, we always prefer an archived log file even
@@ -2466,6 +2472,17 @@ RestoreArchivedFile(char *path, const char *xlogfname,
 					StrNCpy(dp, xlogfname, endp - dp);
 					dp += strlen(dp);
 					break;
+				case 'r':
+					/* %r: filename of last restartpoint */
+					sp++;
+					XLByteToSeg(ControlFile->checkPointCopy.redo,
+								restartLog, restartSeg);
+					XLogFileName(lastRestartPointFname, 
+								 ControlFile->checkPointCopy.ThisTimeLineID, 
+								 restartLog, restartSeg);
+					StrNCpy(dp, lastRestartPointFname, endp - dp);
+					dp += strlen(dp);
+					break;
 				case '%':
 					/* convert %% to a single % */
 					sp++;
@@ -4401,6 +4418,21 @@ readRecoveryCommandFile(void)
 			ereport(LOG,
 					(errmsg("recovery_target_inclusive = %s", tok2)));
 		}
+		else if (strcmp(tok1, "log_restartpoints") == 0)
+		{
+			/*
+			 * does nothing if a recovery_target is not also set
+			 */
+			if (strcmp(tok2, "true") == 0)
+				recoveryLogRestartpoints = true;
+			else
+			{
+				recoveryLogRestartpoints = false;
+				tok2 = "false";
+			}
+			ereport(LOG,
+					(errmsg("log_restartpoints = %s", tok2)));
+		}
 		else
 			ereport(FATAL,
 					(errmsg("unrecognized recovery parameter \"%s\"",
@@ -4564,10 +4596,6 @@ recoveryStopsHere(XLogRecord *record, bool *includeThis)
 	uint8		record_info;
 	TimestampTz	recordXtime;
 
-	/* Do we have a PITR target at all? */
-	if (!recoveryTarget)
-		return false;
-
 	/* We only consider stopping at COMMIT or ABORT records */
 	if (record->xl_rmid != RM_XACT_ID)
 		return false;
@@ -4589,6 +4617,13 @@ recoveryStopsHere(XLogRecord *record, bool *includeThis)
 	else
 		return false;
 
+	/* Remember the most recent COMMIT/ABORT time for logging purposes */
+	recoveryLastXTime = recordXtime;
+
+	/* Do we have a PITR target at all? */
+	if (!recoveryTarget)
+		return false;
+
 	if (recoveryTargetExact)
 	{
 		/*
@@ -5015,6 +5050,10 @@ StartupXLOG(void)
 			ereport(LOG,
 					(errmsg("redo done at %X/%X",
 							ReadRecPtr.xlogid, ReadRecPtr.xrecoff)));
+			if (recoveryLastXTime)
+				ereport(LOG,
+						(errmsg("last completed transaction was at log time %s",
+								timestamptz_to_str(recoveryLastXTime))));
 			InRedo = false;
 		}
 		else
@@ -5922,9 +5961,13 @@ RecoveryRestartPoint(const CheckPoint *checkPoint)
 	ControlFile->time = time(NULL);
 	UpdateControlFile();
 
-	ereport(DEBUG2,
+	ereport((recoveryLogRestartpoints ? LOG : DEBUG2),
 			(errmsg("recovery restart point at %X/%X",
 					checkPoint->redo.xlogid, checkPoint->redo.xrecoff)));
+	if (recoveryLastXTime)
+		ereport((recoveryLogRestartpoints ? LOG : DEBUG2),
+				(errmsg("last completed transaction was at log time %s",
+						timestamptz_to_str(recoveryLastXTime))));
 }
 
 /*
@@ -6285,14 +6328,20 @@ pg_start_backup(PG_FUNCTION_ARGS)
 	if (!superuser())
 		ereport(ERROR,
 				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
-				 (errmsg("must be superuser to run a backup"))));
+				 errmsg("must be superuser to run a backup")));
 
 	if (!XLogArchivingActive())
 		ereport(ERROR,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-				 (errmsg("WAL archiving is not active"),
+				 errmsg("WAL archiving is not active"),
-				  (errhint("archive_command must be defined before "
+				 errhint("archive_mode must be enabled at server start.")));
-						   "online backups can be made safely.")))));
+
+	if (!XLogArchiveCommandSet())
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("WAL archiving is not active"),
+				 errhint("archive_command must be defined before "
+						 "online backups can be made safely.")));
 
 	backupidstr = DatumGetCString(DirectFunctionCall1(textout,
 												 PointerGetDatum(backupid)));

src/backend/postmaster/pgarch.c

@@ -19,7 +19,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/postmaster/pgarch.c,v 1.30 2007/08/02 23:39:44 adunstan Exp $
+ *	  $PostgreSQL: pgsql/src/backend/postmaster/pgarch.c,v 1.31 2007/09/26 22:36:30 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -314,8 +314,6 @@ pgarch_MainLoop(void)
 		{
 			got_SIGHUP = false;
 			ProcessConfigFile(PGC_SIGHUP);
-			if (!XLogArchivingActive())
-				break;			/* user wants us to shut down */
 		}
 
 		/* Do what we're here for */
@@ -359,6 +357,14 @@ pgarch_ArchiverCopyLoop(void)
 {
 	char		xlog[MAX_XFN_CHARS + 1];
 
+	if (!XLogArchiveCommandSet())
+	{
+		ereport(WARNING,
+				(errmsg("archive_mode enabled, yet archive_command is not set")));
+		/* can't do anything if no command ... */
+		return;
+	}
+
 	/*
 	 * loop through all xlogs with archive_status of .ready and archive
 	 * them...mostly we expect this to be a single file, though it is possible

src/backend/postmaster/postmaster.c

@@ -37,7 +37,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.541 2007/08/19 01:41:24 adunstan Exp $
+ *	  $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.542 2007/09/26 22:36:30 tgl Exp $
  *
  * NOTES
  *
@@ -1311,8 +1311,12 @@ ServerLoop(void)
 				start_autovac_launcher = false;	/* signal processed */
 		}
 
-		/* If we have lost the archiver, try to start a new one */
+		/* 
-		if (XLogArchivingActive() && PgArchPID == 0 && pmState == PM_RUN)
+		 * If we have lost the archiver, try to start a new one.
+		 * We do this even if we are shutting down, to allow archiver to
+		 * take care of any remaining WAL files.
+		 */
+		if (XLogArchivingActive() && PgArchPID == 0 && pmState >= PM_RUN)
 			PgArchPID = pgarch_start();
 
 		/* If we have lost the stats collector, try to start a new one */
@@ -2232,7 +2236,7 @@ reaper(SIGNAL_ARGS)
 			if (!EXIT_STATUS_0(exitstatus))
 				LogChildExit(LOG, _("archiver process"),
 							 pid, exitstatus);
-			if (XLogArchivingActive() && pmState == PM_RUN)
+			if (XLogArchivingActive() && pmState >= PM_RUN)
 				PgArchPID = pgarch_start();
 			continue;
 		}

src/backend/utils/misc/guc.c

@@ -10,7 +10,7 @@
  * Written by Peter Eisentraut <peter_e@gmx.net>.
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/utils/misc/guc.c,v 1.422 2007/09/25 20:03:38 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/utils/misc/guc.c,v 1.423 2007/09/26 22:36:30 tgl Exp $
  *
  *--------------------------------------------------------------------
  */
@@ -169,7 +169,7 @@ static const char *assign_backslash_quote(const char *newval, bool doit, GucSour
 static const char *assign_timezone_abbreviations(const char *newval, bool doit, GucSource source);
 static const char *assign_xmlbinary(const char *newval, bool doit, GucSource source);
 static const char *assign_xmloption(const char *newval, bool doit, GucSource source);
-
+static const char *show_archive_command(void);
 static bool assign_tcp_keepalives_idle(int newval, bool doit, GucSource source);
 static bool assign_tcp_keepalives_interval(int newval, bool doit, GucSource source);
 static bool assign_tcp_keepalives_count(int newval, bool doit, GucSource source);
@@ -1052,6 +1052,15 @@ static struct config_bool ConfigureNamesBool[] =
 		false, NULL, NULL
 	},
 
+    {
+        {"archive_mode", PGC_POSTMASTER, WAL_SETTINGS,
+            gettext_noop("Allows archiving of WAL files using archive_command."),
+            NULL
+        },
+        &XLogArchiveMode,
+        false, NULL, NULL
+    },
+
 	{
 		{"allow_system_table_mods", PGC_POSTMASTER, DEVELOPER_OPTIONS,
 			gettext_noop("Allows modifications of the structure of system tables."),
@@ -1880,7 +1889,7 @@ static struct config_string ConfigureNamesString[] =
 			NULL
 		},
 		&XLogArchiveCommand,
-		"", NULL, NULL
+		"", NULL, show_archive_command
 	},
 
 	{
@@ -6363,7 +6372,7 @@ GUCArrayDelete(ArrayType *array, const char *name)
 
 
 /*
- * assign_hook subroutines
+ * assign_hook and show_hook subroutines
  */
 
 static const char *
@@ -6972,6 +6981,15 @@ assign_xmloption(const char *newval, bool doit, GucSource source)
 	return newval;
 }
 
+static const char *
+show_archive_command(void)
+{
+	if (XLogArchiveMode)
+		return XLogArchiveCommand;
+	else
+		return "(disabled)";
+}
+
 static bool
 assign_tcp_keepalives_idle(int newval, bool doit, GucSource source)
 {

src/backend/utils/misc/postgresql.conf.sample

@@ -174,6 +174,8 @@
 
 # - Archiving -
 
+#archive_mode = off		# allows archiving to be done
+				# (change requires restart)
 #archive_command = ''		# command to use to archive a logfile segment
 #archive_timeout = 0		# force a logfile segment switch after this
 				# many seconds; 0 is off

src/include/access/xlog.h

@@ -6,7 +6,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.83 2007/09/05 18:10:48 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.84 2007/09/26 22:36:30 tgl Exp $
  */
 #ifndef XLOG_H
 #define XLOG_H
@@ -138,13 +138,15 @@ extern XLogRecPtr XactLastRecEnd;
 /* these variables are GUC parameters related to XLOG */
 extern int	CheckPointSegments;
 extern int	XLOGbuffers;
+extern bool XLogArchiveMode;
 extern char *XLogArchiveCommand;
 extern int	XLogArchiveTimeout;
 extern char *XLOG_sync_method;
 extern const char XLOG_sync_method_default[];
 extern bool log_checkpoints;
 
-#define XLogArchivingActive()	(XLogArchiveCommand[0] != '\0')
+#define XLogArchivingActive()	(XLogArchiveMode)
+#define XLogArchiveCommandSet()	(XLogArchiveCommand[0] != '\0')
 
 #ifdef WAL_DEBUG
 extern bool XLOG_DEBUG;