Replace opendir/closedir calls throughout the backend with AllocateDir
and FreeDir routines modeled on the existing AllocateFile/FreeFile. Like the latter, these routines will avoid failing on EMFILE/ENFILE conditions whenever possible, and will prevent leakage of directory descriptors if an elog() occurs while one is open. Also, reduce PANIC to ERROR in MoveOfflineLogs() --- this is not critical code and there is no reason to force a DB restart on failure. All per recent trouble report from Olivier Hubaut.
This commit is contained in:
Tom Lane
parent
4f571319d3
commit
7a57a67278
@ -1,16 +1,15 @@
|
||||
#include "postgres.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <dirent.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "access/heapam.h"
|
||||
#include "catalog/catalog.h"
|
||||
#include "catalog/namespace.h"
|
||||
#include "commands/dbcommands.h"
|
||||
#include "fmgr.h"
|
||||
#include "storage/fd.h"
|
||||
#include "utils/builtins.h"
|
||||
|
||||
|
||||
@ -58,7 +57,7 @@ database_size(PG_FUNCTION_ARGS)
|
||||
|
||||
dbpath = GetDatabasePath(dbid);
|
||||
|
||||
dirdesc = opendir(dbpath);
|
||||
dirdesc = AllocateDir(dbpath);
|
||||
if (!dirdesc)
|
||||
ereport(ERROR,
|
||||
(errcode_for_file_access(),
|
||||
@ -93,7 +92,7 @@ database_size(PG_FUNCTION_ARGS)
|
||||
pfree(fullname);
|
||||
}
|
||||
|
||||
closedir(dirdesc);
|
||||
FreeDir(dirdesc);
|
||||
|
||||
PG_RETURN_INT64(totalsize);
|
||||
}
|
||||
|
||||
@ -6,15 +6,13 @@
|
||||
* Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
|
||||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* $PostgreSQL: pgsql/src/backend/access/transam/slru.c,v 1.12 2004/02/17 03:45:17 momjian Exp $
|
||||
* $PostgreSQL: pgsql/src/backend/access/transam/slru.c,v 1.13 2004/02/23 23:03:10 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
#include "postgres.h"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
@ -888,7 +886,7 @@ SlruScanDirectory(SlruCtl ctl, int cutoffPage, bool doDeletions)
|
||||
int segpage;
|
||||
char path[MAXPGPATH];
|
||||
|
||||
cldir = opendir(ctl->Dir);
|
||||
cldir = AllocateDir(ctl->Dir);
|
||||
if (cldir == NULL)
|
||||
ereport(ERROR,
|
||||
(errcode_for_file_access(),
|
||||
@ -927,7 +925,7 @@ SlruScanDirectory(SlruCtl ctl, int cutoffPage, bool doDeletions)
|
||||
ereport(ERROR,
|
||||
(errcode_for_file_access(),
|
||||
errmsg("could not read directory \"%s\": %m", ctl->Dir)));
|
||||
closedir(cldir);
|
||||
FreeDir(cldir);
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
* Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
|
||||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.136 2004/02/17 03:45:17 momjian Exp $
|
||||
* $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.137 2004/02/23 23:03:10 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
@ -17,10 +17,8 @@
|
||||
#include <fcntl.h>
|
||||
#include <signal.h>
|
||||
#include <unistd.h>
|
||||
#include <errno.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/time.h>
|
||||
#include <dirent.h>
|
||||
|
||||
#include "access/clog.h"
|
||||
#include "access/transam.h"
|
||||
@ -1753,9 +1751,9 @@ MoveOfflineLogs(uint32 log, uint32 seg, XLogRecPtr endptr)
|
||||
|
||||
XLByteToPrevSeg(endptr, endlogId, endlogSeg);
|
||||
|
||||
xldir = opendir(XLogDir);
|
||||
xldir = AllocateDir(XLogDir);
|
||||
if (xldir == NULL)
|
||||
ereport(PANIC,
|
||||
ereport(ERROR,
|
||||
(errcode_for_file_access(),
|
||||
errmsg("could not open transaction log directory \"%s\": %m",
|
||||
XLogDir)));
|
||||
@ -1812,11 +1810,11 @@ MoveOfflineLogs(uint32 log, uint32 seg, XLogRecPtr endptr)
|
||||
errno = 0;
|
||||
#endif
|
||||
if (errno)
|
||||
ereport(PANIC,
|
||||
ereport(ERROR,
|
||||
(errcode_for_file_access(),
|
||||
errmsg("could not read transaction log directory \"%s\": %m",
|
||||
XLogDir)));
|
||||
closedir(xldir);
|
||||
FreeDir(xldir);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $PostgreSQL: pgsql/src/backend/storage/file/fd.c,v 1.107 2004/02/23 20:45:59 tgl Exp $
|
||||
* $PostgreSQL: pgsql/src/backend/storage/file/fd.c,v 1.108 2004/02/23 23:03:10 tgl Exp $
|
||||
*
|
||||
* NOTES:
|
||||
*
|
||||
@ -43,8 +43,6 @@
|
||||
#include <sys/file.h>
|
||||
#include <sys/param.h>
|
||||
#include <sys/stat.h>
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
@ -87,11 +85,11 @@ int max_files_per_process = 1000;
|
||||
|
||||
/*
|
||||
* Maximum number of file descriptors to open for either VFD entries or
|
||||
* AllocateFile files. This is initialized to a conservative value, and
|
||||
* remains that way indefinitely in bootstrap or standalone-backend cases.
|
||||
* In normal postmaster operation, the postmaster calls set_max_safe_fds()
|
||||
* late in initialization to update the value, and that value is then
|
||||
* inherited by forked subprocesses.
|
||||
* AllocateFile/AllocateDir operations. This is initialized to a conservative
|
||||
* value, and remains that way indefinitely in bootstrap or standalone-backend
|
||||
* cases. In normal postmaster operation, the postmaster calls
|
||||
* set_max_safe_fds() late in initialization to update the value, and that
|
||||
* value is then inherited by forked subprocesses.
|
||||
*
|
||||
* Note: the value of max_files_per_process is taken into account while
|
||||
* setting this variable, and so need not be tested separately.
|
||||
@ -159,6 +157,17 @@ static int nfile = 0;
|
||||
static int numAllocatedFiles = 0;
|
||||
static FILE *allocatedFiles[MAX_ALLOCATED_FILES];
|
||||
|
||||
/*
|
||||
* List of <dirent.h> DIRs opened with AllocateDir.
|
||||
*
|
||||
* Since we don't have heavy use of AllocateDir, it seems OK to put a pretty
|
||||
* small maximum limit on the number of simultaneously allocated dirs.
|
||||
*/
|
||||
#define MAX_ALLOCATED_DIRS 10
|
||||
|
||||
static int numAllocatedDirs = 0;
|
||||
static DIR *allocatedDirs[MAX_ALLOCATED_DIRS];
|
||||
|
||||
/*
|
||||
* Number of temporary files opened during the current session;
|
||||
* this is used in generation of tempfile names.
|
||||
@ -489,7 +498,7 @@ LruInsert(File file)
|
||||
|
||||
if (FileIsNotOpen(file))
|
||||
{
|
||||
while (nfile + numAllocatedFiles >= max_safe_fds)
|
||||
while (nfile + numAllocatedFiles + numAllocatedDirs >= max_safe_fds)
|
||||
{
|
||||
if (!ReleaseLruFile())
|
||||
break;
|
||||
@ -748,7 +757,7 @@ fileNameOpenFile(FileName fileName,
|
||||
file = AllocateVfd();
|
||||
vfdP = &VfdCache[file];
|
||||
|
||||
while (nfile + numAllocatedFiles >= max_safe_fds)
|
||||
while (nfile + numAllocatedFiles + numAllocatedDirs >= max_safe_fds)
|
||||
{
|
||||
if (!ReleaseLruFile())
|
||||
break;
|
||||
@ -1099,8 +1108,8 @@ AllocateFile(char *name, char *mode)
|
||||
* looping.
|
||||
*/
|
||||
if (numAllocatedFiles >= MAX_ALLOCATED_FILES ||
|
||||
numAllocatedFiles >= max_safe_fds - 1)
|
||||
elog(ERROR, "too many private FDs demanded");
|
||||
numAllocatedFiles + numAllocatedDirs >= max_safe_fds - 1)
|
||||
elog(ERROR, "too many private files demanded");
|
||||
|
||||
TryAgain:
|
||||
if ((file = fopen(name, mode)) != NULL)
|
||||
@ -1155,6 +1164,86 @@ FreeFile(FILE *file)
|
||||
return fclose(file);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Routines that want to use <dirent.h> (ie, DIR*) should use AllocateDir
|
||||
* rather than plain opendir(). This lets fd.c deal with freeing FDs if
|
||||
* necessary to open the directory, and with closing it after an elog.
|
||||
* When done, call FreeDir rather than closedir.
|
||||
*
|
||||
* Ideally this should be the *only* direct call of opendir() in the backend.
|
||||
*/
|
||||
DIR *
|
||||
AllocateDir(const char *dirname)
|
||||
{
|
||||
DIR *dir;
|
||||
|
||||
DO_DB(elog(LOG, "AllocateDir: Allocated %d", numAllocatedDirs));
|
||||
|
||||
/*
|
||||
* The test against MAX_ALLOCATED_DIRS prevents us from overflowing
|
||||
* allocatedDirs[]; the test against max_safe_fds prevents AllocateDir
|
||||
* from hogging every one of the available FDs, which'd lead to infinite
|
||||
* looping.
|
||||
*/
|
||||
if (numAllocatedDirs >= MAX_ALLOCATED_DIRS ||
|
||||
numAllocatedDirs + numAllocatedFiles >= max_safe_fds - 1)
|
||||
elog(ERROR, "too many private dirs demanded");
|
||||
|
||||
TryAgain:
|
||||
if ((dir = opendir(dirname)) != NULL)
|
||||
{
|
||||
allocatedDirs[numAllocatedDirs] = dir;
|
||||
numAllocatedDirs++;
|
||||
return dir;
|
||||
}
|
||||
|
||||
if (errno == EMFILE || errno == ENFILE)
|
||||
{
|
||||
int save_errno = errno;
|
||||
|
||||
ereport(LOG,
|
||||
(errcode(ERRCODE_INSUFFICIENT_RESOURCES),
|
||||
errmsg("out of file descriptors: %m; release and retry")));
|
||||
errno = 0;
|
||||
if (ReleaseLruFile())
|
||||
goto TryAgain;
|
||||
errno = save_errno;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Close a directory opened with AllocateDir.
|
||||
*
|
||||
* Note we do not check closedir's return value --- it is up to the caller
|
||||
* to handle close errors.
|
||||
*/
|
||||
int
|
||||
FreeDir(DIR *dir)
|
||||
{
|
||||
int i;
|
||||
|
||||
DO_DB(elog(LOG, "FreeDir: Allocated %d", numAllocatedDirs));
|
||||
|
||||
/* Remove dir from list of allocated dirs, if it's present */
|
||||
for (i = numAllocatedDirs; --i >= 0;)
|
||||
{
|
||||
if (allocatedDirs[i] == dir)
|
||||
{
|
||||
numAllocatedDirs--;
|
||||
allocatedDirs[i] = allocatedDirs[numAllocatedDirs];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (i < 0)
|
||||
elog(WARNING, "dir passed to FreeDir was not obtained from AllocateDir");
|
||||
|
||||
return closedir(dir);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* closeAllVfds
|
||||
*
|
||||
@ -1211,7 +1300,7 @@ AtProcExit_Files(int code, Datum arg)
|
||||
* exiting. If that's the case, we should remove all temporary files; if
|
||||
* that's not the case, we are being called for transaction commit/abort
|
||||
* and should only remove transaction-local temp files. In either case,
|
||||
* also clean up "allocated" stdio files.
|
||||
* also clean up "allocated" stdio files and dirs.
|
||||
*/
|
||||
static void
|
||||
CleanupTempFiles(bool isProcExit)
|
||||
@ -1240,6 +1329,9 @@ CleanupTempFiles(bool isProcExit)
|
||||
|
||||
while (numAllocatedFiles > 0)
|
||||
FreeFile(allocatedFiles[0]);
|
||||
|
||||
while (numAllocatedDirs > 0)
|
||||
FreeDir(allocatedDirs[0]);
|
||||
}
|
||||
|
||||
|
||||
@ -1271,7 +1363,7 @@ RemovePgTempFiles(void)
|
||||
* files.
|
||||
*/
|
||||
snprintf(db_path, sizeof(db_path), "%s/base", DataDir);
|
||||
if ((db_dir = opendir(db_path)) != NULL)
|
||||
if ((db_dir = AllocateDir(db_path)) != NULL)
|
||||
{
|
||||
while ((db_de = readdir(db_dir)) != NULL)
|
||||
{
|
||||
@ -1287,7 +1379,7 @@ RemovePgTempFiles(void)
|
||||
"%s/%s/%s",
|
||||
db_path, db_de->d_name,
|
||||
PG_TEMP_FILES_DIR);
|
||||
if ((temp_dir = opendir(temp_path)) != NULL)
|
||||
if ((temp_dir = AllocateDir(temp_path)) != NULL)
|
||||
{
|
||||
while ((temp_de = readdir(temp_dir)) != NULL)
|
||||
{
|
||||
@ -1310,9 +1402,9 @@ RemovePgTempFiles(void)
|
||||
"unexpected file found in temporary-files directory: \"%s\"",
|
||||
rm_path);
|
||||
}
|
||||
closedir(temp_dir);
|
||||
FreeDir(temp_dir);
|
||||
}
|
||||
}
|
||||
closedir(db_dir);
|
||||
FreeDir(db_dir);
|
||||
}
|
||||
}
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
* Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
|
||||
* Portions Copyright (c) 1994, Regents of the University of California
|
||||
*
|
||||
* $PostgreSQL: pgsql/src/include/storage/fd.h,v 1.43 2004/02/23 20:45:59 tgl Exp $
|
||||
* $PostgreSQL: pgsql/src/include/storage/fd.h,v 1.44 2004/02/23 23:03:10 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
@ -31,10 +31,16 @@
|
||||
* use FreeFile, not fclose, to close it. AVOID using stdio for files
|
||||
* that you intend to hold open for any length of time, since there is
|
||||
* no way for them to share kernel file descriptors with other files.
|
||||
*
|
||||
* Likewise, use AllocateDir/FreeDir, not opendir/closedir, to allocate
|
||||
* open directories (DIR*).
|
||||
*/
|
||||
#ifndef FD_H
|
||||
#define FD_H
|
||||
|
||||
#include <dirent.h>
|
||||
|
||||
|
||||
/*
|
||||
* FileSeek uses the standard UNIX lseek(2) flags.
|
||||
*/
|
||||
@ -65,7 +71,11 @@ extern int FileTruncate(File file, long offset);
|
||||
|
||||
/* Operations that allow use of regular stdio --- USE WITH CAUTION */
|
||||
extern FILE *AllocateFile(char *name, char *mode);
|
||||
extern int FreeFile(FILE *);
|
||||
extern int FreeFile(FILE *file);
|
||||
|
||||
/* Operations to allow use of the <dirent.h> library routines */
|
||||
extern DIR *AllocateDir(const char *dirname);
|
||||
extern int FreeDir(DIR *dir);
|
||||
|
||||
/* If you've really really gotta have a plain kernel FD, use this */
|
||||
extern int BasicOpenFile(FileName fileName, int fileFlags, int fileMode);
|
||||
|
||||
@ -11,18 +11,18 @@
|
||||
* as a service.
|
||||
*
|
||||
* IDENTIFICATION
|
||||
* $PostgreSQL: pgsql/src/port/copydir.c,v 1.7 2003/11/29 19:52:13 pgsql Exp $
|
||||
* $PostgreSQL: pgsql/src/port/copydir.c,v 1.8 2004/02/23 23:03:10 tgl Exp $
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
#include "postgres.h"
|
||||
|
||||
#include "storage/fd.h"
|
||||
|
||||
#undef mkdir /* no reason to use that macro because we
|
||||
* ignore the 2nd arg */
|
||||
|
||||
#include <dirent.h>
|
||||
|
||||
|
||||
/*
|
||||
* copydir: copy a directory (we only need to go one level deep)
|
||||
@ -47,7 +47,7 @@ copydir(char *fromdir, char *todir)
|
||||
errmsg("could not create directory \"%s\": %m", todir)));
|
||||
return -1;
|
||||
}
|
||||
xldir = opendir(fromdir);
|
||||
xldir = AllocateDir(fromdir);
|
||||
if (xldir == NULL)
|
||||
{
|
||||
ereport(WARNING,
|
||||
@ -65,11 +65,11 @@ copydir(char *fromdir, char *todir)
|
||||
ereport(WARNING,
|
||||
(errcode_for_file_access(),
|
||||
errmsg("could not copy file \"%s\": %m", fromfl)));
|
||||
closedir(xldir);
|
||||
FreeDir(xldir);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
closedir(xldir);
|
||||
FreeDir(xldir);
|
||||
return 0;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user