Added a server that caches stat data and directory contents.

To use this feature run StatCacheServer somewhere in the background.

The first run of jam will be slower, but subsequent ones will usually
benefit. Only little, if the BeOS FS cache is able to hold the data,
but as soon as the number of files and directories jam has to inspect
hits the limit at which the cache drops data needed for the next run,
the performance difference will be very noticeable.
E.g. the time for a  "jam -n > /dev/null" in the root of a partially built
tree on my machine dropped from >1 min without the server to 13 secs
with it (not in the first run, of course).

Although tested quite a bit, I would still consider the feature
experimental.


git-svn-id: file:///srv/svn/repos/haiku/trunk/current@9671 a95241bf-73f2-0310-859d-f6bbb57e9c96

src/tools/jam/Jamfile

@@ -41,6 +41,7 @@ if $(PROFILE)
 if $(DEBUG)
 {
 	CCFLAGS += -g ;
+	C++FLAGS += -g ;
 	LINKFLAGS += -g ;
 	LOCATE_TARGET = $(LOCATE_TARGET)/g ;
 }
@@ -58,6 +59,16 @@ DEFINES += OPT_JAMFILE_CACHE_EXT ;
 #
 ### LOCAL CHANGE
 
+### LOCAL CHANGE
+# 
+# Include stat cache server under BeOS.
+#
+if $(OS) = BEOS {
+	DEFINES += OPT_STAT_CACHE_SERVER_EXT ;
+}
+#
+### LOCAL CHANGE
+
 # Do we know yacc?
 
 if $(YACC) 	{ code += jamgram.y ; }
@@ -84,6 +95,16 @@ code += hcache.c ;
 #
 ### LOCAL CHANGE
 
+### LOCAL CHANGE
+# 
+# Include stat cache server under BeOS.
+#
+if $(OS) = BEOS {
+	code += beos_stat_cache.c ;
+}
+#
+### LOCAL CHANGE
+
 #
 # How to build the compiled in jambase.
 #
@@ -119,6 +140,19 @@ Library         libjam.a :
 
 if $(BINDIR) 	{ InstallBin $(BINDIR) : jam ; }
 
+### LOCAL CHANGE
+# 
+# Build stat cache server under BeOS.
+#
+if $(OS) = BEOS {
+	LINK on StatCacheServer = g++ ;
+	LINKLIBS on StatCacheServer
+		= [ on StatCacheServer return $(LINKLIBS) ] -lbe ;
+	Main StatCacheServer : StatCacheServer.cpp ;
+}
+#
+### LOCAL CHANGE
+
 #
 # Distribution making from here on out.
 #

src/tools/jam/README.CHANGES

@@ -130,3 +130,27 @@ adjusted more or less to work with 2.5rc3.
   file is not performed. Setting the SEARCH and LOCATE variables does work
   as expected.
 
+
+* Stat Data and Directory Caching Server (BeOS only)
+
+  Also an optimization for large build systems. Since the BeOS FS cache
+  is terrible, stat()ing targets to get their timestamp or see if they exist
+  at all, and reading directories usually happens on disk, since the data
+  from the previous run are already out of the cache, if the build system
+  is large enough.
+
+  This change externalizes all stat()ing and directory reading into a
+  dedicated server process which caches the data, so that they can be
+  served from memory the next time they are requested. The server uses
+  the BeOS node monitoring to keep the data up to date.
+
+  The feature particularly leverages the header and jamfile caching, since
+  after the first run the timestamps of the jamfiles and headers are
+  cached too, so that reading the jamfiles and performing the header
+  scanning doesn't require any disk accesses at all (besides reading the
+  cache files, of course).
+
+  Drawbacks are that the first run of jam will be slower, mainly due to
+  the communication overhead with the server, and that the server consumes
+  memory to store the cached data. The server's memory footprint is quite
+  reasonable, though.

src/tools/jam/StatCacheServer.h

@@ -0,0 +1,35 @@
+// StatCacheServer.h
+
+#ifndef STAT_CACHE_SERVER_H
+#define STAT_CACHE_SERVER_H
+
+#include <StorageDefs.h>
+
+// common definitions used by server and client 
+
+#define STAT_CACHE_SERVER_PORT_NAME	"stat_cache_server_port"
+
+enum {
+	STAT_CACHE_COMMAND_STAT		= 0,
+	STAT_CACHE_COMMAND_READDIR	= 1,
+};
+
+typedef struct stat_cache_request {
+	port_id		replyPort;
+	int32		command;
+	char		path[B_PATH_NAME_LENGTH];
+} stat_cache_request;
+
+typedef struct stat_cache_stat_reply {
+	status_t	error;
+	struct stat	st;
+} stat_cache_stat_reply;
+
+typedef struct stat_cache_readdir_reply {
+	status_t	error;
+	int32		entryCount;
+	void		*clientData;		// used by the client only
+	uint8		buffer[1];
+} stat_cache_readdir_reply;
+
+#endif	// STAT_CACHE_SERVER_H

src/tools/jam/StatCacheServerImpl.h

@@ -0,0 +1,266 @@
+// StatCacheServerImpl.h
+
+#ifndef STAT_CACHE_SERVER_IMPL_H
+#define STAT_CACHE_SERVER_IMPL_H
+
+#include <hash_map>
+#include <string>
+
+#include <Entry.h>
+#include <Locker.h>
+#include <Looper.h>
+#include <MessageQueue.h>
+#include <Node.h>
+#include <OS.h>
+
+class Directory;
+class Entry;
+class Node;
+class SymLink;
+
+// NodeRefHash
+struct NodeRefHash
+{
+	size_t operator()(const node_ref &nodeRef) const;
+};
+ 
+// EntryRefHash
+struct EntryRefHash
+{
+	size_t operator()(const entry_ref &entryRef) const;
+}; 
+
+// Referencable
+class Referencable {
+public:
+	Referencable()
+		: fReferenceCount(1),
+		  fReferenceBaseCount(0)
+	{
+	}
+
+	virtual ~Referencable()
+	{
+	}
+
+	void AddReference()
+	{
+		fReferenceCount++;
+	}
+
+	bool RemoveReference()
+	{
+		if (--fReferenceCount <= fReferenceBaseCount) {
+			Unreferenced();
+			return true;
+		}
+		return false;
+	}
+
+	int32 CountReferences() const
+	{
+		return fReferenceCount;
+	}
+
+protected:
+	virtual void Unreferenced() {};
+
+protected:
+	int32	fReferenceCount;
+	int32	fReferenceBaseCount;
+	bool	fDeleteWhenUnreferenced;
+};
+
+// Entry
+class Entry : public Referencable {
+public:
+	Entry();
+	~Entry();
+
+	status_t SetTo(Directory *parent, const char *name);
+
+	Directory *GetParent() const;
+
+	const char *GetName() const;
+
+	void SetNode(Node *node);
+	Node *GetNode() const;
+
+	void SetPrevious(Entry *entry);
+	Entry *GetPrevious() const;
+	void SetNext(Entry *entry);
+	Entry *GetNext() const;
+
+	entry_ref GetEntryRef() const;
+	status_t GetPath(string& path);
+
+protected:
+	virtual void Unreferenced();
+
+private:
+	Directory	*fParent;
+	string		fName;
+	Node		*fNode;
+	Entry		*fPrevious;
+	Entry		*fNext;
+};
+
+// Node
+class Node : public Referencable {
+public:
+	Node(const struct stat &st);
+	virtual ~Node();
+
+	virtual status_t SetTo(Entry *entry);
+
+	status_t GetPath(string& path);
+
+	const struct stat &GetStat() const;
+	status_t GetStat(struct stat *st);
+	status_t UpdateStat();
+	void MarkStatInvalid();
+
+	void SetEntry(Entry *entry);
+	Entry *GetEntry() const;
+
+	node_ref GetNodeRef() const;
+
+protected:
+	virtual void Unreferenced();
+
+protected:
+	Entry		*fEntry;
+	struct stat	fStat;
+	bool		fStatValid;
+};
+
+// Directory
+class Directory : public Node {
+public:
+	Directory(const struct stat &st);
+	~Directory();
+
+	virtual status_t SetTo(Entry *entry);
+
+	status_t FindEntry(const char *name, Entry **entry);
+	Entry *GetFirstEntry() const;
+	Entry *GetNextEntry(Entry *entry) const;
+
+	status_t ReadAllEntries();
+
+	bool IsComplete() const;
+
+	void AddEntry(Entry *entry);
+	void RemoveEntry(Entry *entry);
+
+private:
+	Entry	*fFirstEntry;
+	Entry	*fLastEntry;
+	bool	fIsComplete;
+};
+
+// SymLink
+class SymLink : public Node {
+public:
+	SymLink(const struct stat &st);
+	~SymLink();
+
+	virtual status_t SetTo(Entry *entry);
+
+	const char *GetTarget() const;
+
+private:
+	string	fTarget;
+};
+
+// NodeMonitor
+class NodeMonitor : public BLooper {
+public:
+	NodeMonitor();
+	virtual ~NodeMonitor();
+
+	status_t Init();
+
+	virtual void MessageReceived(BMessage *message);
+
+	status_t StartWatching(Node *node);
+	status_t StopWatching(Node *node);
+
+	status_t GetNextMonitoringMessage(BMessage **message);
+
+private:
+	int32			fCurrentNodeMonitorLimit;
+	BMessageQueue	fMessageQueue;
+	sem_id			fMessageCountSem;
+};
+
+// PathResolver
+class PathResolver {
+public:
+	PathResolver();
+
+	status_t FindEntry(const char *path, bool traverse, Entry **_entry);
+	status_t FindEntry(Entry *entry, const char *path, bool traverse,
+		Entry **_entry);
+	status_t FindNode(const char *path, bool traverse, Node **node);
+
+	status_t ResolveSymlink(Node *node, Node **_node);
+	status_t ResolveSymlink(Node *node, Entry **entry);
+	status_t ResolveSymlink(Entry *entry, Entry **_entry);
+
+private:
+	int32	fSymLinkCounter;
+};
+
+// NodeManager
+class NodeManager : public BLocker {
+public:
+	NodeManager();
+	~NodeManager();
+
+	static NodeManager *GetDefault();
+
+	status_t Init();
+
+	Directory *GetRootDirectory() const;
+
+	Node *GetNode(const node_ref &nodeRef);
+	Entry *GetEntry(const entry_ref &entryRef);
+
+	status_t CreateEntry(const entry_ref &entryRef, Entry **_entry);
+	status_t CreateDirectory(const node_ref &nodeRef, Directory **_dir);
+
+	void RemoveEntry(Entry *entry);
+	void MoveEntry(Entry *entry, const entry_ref &newRef);
+
+	void EntryUnreferenced(Entry *entry);
+	void NodeUnreferenced(Node *node);
+
+	status_t StartWatching(Node *node);
+	status_t StopWatching(Node *node);
+
+private:
+	static int32 _NodeMonitoringProcessorEntry(void *data);
+	int32 _NodeMonitoringProcessor();
+
+	status_t _CreateNode(Entry *entry, Node **_node);
+
+	void _EntryCreated(BMessage *message);
+	void _EntryRemoved(BMessage *message);
+	void _EntryMoved(BMessage *message);
+	void _StatChanged(BMessage *message);
+
+private:
+	typedef hash_map<entry_ref, Entry*, EntryRefHash> EntryMap;
+	typedef hash_map<node_ref, Node*, NodeRefHash> NodeMap;
+
+	EntryMap	fEntries;
+	NodeMap		fNodes;
+	Directory	*fRootDirectory;
+	NodeMonitor	*fNodeMonitor;
+	thread_id	fNodeMonitoringProcessor;
+
+	static NodeManager	sManager;
+};
+
+#endif	// STAT_CACHE_SERVER_IMPL_H

src/tools/jam/beos_stat_cache.c

@@ -0,0 +1,221 @@
+// beos_stat_cache.c
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <OS.h>
+
+#include "beos_stat_cache.h"
+#include "pathsys.h"
+#include "StatCacheServer.h"
+
+#define SET_ERRNO_AND_RETURN(error) {	\
+	if ((error) == B_OK)				\
+		return 0;						\
+	errno = (error);					\
+	return -1;							\
+}
+
+// get_server_port
+static
+port_id
+get_server_port()
+{
+	static port_id id = -1;
+	static bool initialized = false;
+	if (!initialized) {
+		id = find_port(STAT_CACHE_SERVER_PORT_NAME);
+		initialized = true;
+	}
+	return id;
+}
+
+// get_reply_port
+static
+port_id
+get_reply_port()
+{
+	static port_id id = -1;
+	if (id < 0)
+		id = create_port(1, "stat cache reply port");
+	return id;
+}
+
+// send_request
+static
+status_t
+send_request(int32 command, const char *path)
+{
+	port_id requestPort = get_server_port();
+	port_id replyPort = get_reply_port();
+	stat_cache_request request;
+	int requestSize;
+
+	// get request port
+	if (requestPort < 0)
+		return requestPort;
+	// get reply port
+	if (replyPort < 0)
+		return replyPort;
+	// normalize the path
+	if (!path || !normalize_path(path, request.path, sizeof(request.path)))
+		return B_BAD_VALUE;
+	requestSize = (request.path + strlen(request.path) + 1) - (char*)&request;
+	// send request
+	request.replyPort = replyPort;
+	request.command = command;
+	return write_port(requestPort, 0, &request, requestSize);
+}
+
+// receive_reply
+static
+status_t
+receive_reply(void **_reply, int32 *_replySize, void *buffer, int32 replySize)
+{
+	port_id replyPort = get_reply_port();
+	ssize_t bytesRead;
+	void *reply;
+	int32 code;
+
+	// get reply port
+	if (replyPort < 0)
+		return replyPort;
+
+	// get the reply size
+	if (!buffer) {
+		replySize = port_buffer_size(replyPort);
+		if (replySize < 0)
+			return replySize;
+	}
+
+	// allocate reply
+	if (buffer) {
+		reply = buffer;
+	} else {
+		reply = malloc(replySize);
+		if (!reply)
+			return B_NO_MEMORY;
+	}
+
+	// read the reply
+	bytesRead = read_port(replyPort, &code, reply, replySize);
+	if (bytesRead < 0) {
+		if (!buffer)
+			free(reply);
+		return bytesRead;
+	}
+	if (bytesRead != replySize) {
+		if (!buffer)
+			free(reply);
+		return B_ERROR;
+	}
+
+	if (_reply)
+		*_reply = reply;
+	if (_replySize)
+		*_replySize = replySize;
+	return B_OK;
+}
+
+// beos_stat_cache_stat
+int
+beos_stat_cache_stat(const char *filename, struct stat *st)
+{
+	stat_cache_stat_reply reply;
+	status_t error;
+
+	// fall back to standard, if there is no server
+	if (get_server_port() < 0)
+		return stat(filename, st);
+
+	// send the request
+	error = send_request(STAT_CACHE_COMMAND_STAT, filename);
+	if (error != B_OK)
+		SET_ERRNO_AND_RETURN(error);
+
+	// get the reply
+	error = receive_reply(NULL, NULL, &reply, sizeof(reply));
+	if (error != B_OK)
+		error = reply.error;
+	if (error != B_OK)
+		SET_ERRNO_AND_RETURN(error);
+
+	*st = reply.st;
+	return 0;
+}
+
+// beos_stat_cache_opendir
+DIR *
+beos_stat_cache_opendir(const char *dirName)
+{
+	stat_cache_readdir_reply *reply;
+	int32 replySize;
+	status_t error;
+
+	// fall back to standard, if there is no server
+	if (get_server_port() < 0)
+		return opendir(dirName);
+
+	// send the request
+	error = send_request(STAT_CACHE_COMMAND_READDIR, dirName);
+	if (error != B_OK) {
+		errno = error;
+		return NULL;
+	}
+
+	// get the reply
+	error = receive_reply((void**)&reply, &replySize, NULL, 0);
+	if (error != B_OK)
+		error = reply->error;
+	if (error != B_OK) {
+		free(reply);
+		errno = error;
+		return NULL;
+	}
+
+	reply->clientData = reply->buffer;
+
+	// a bit ugly, but anyway... 
+	return (DIR*)reply;
+}
+
+// beos_stat_cache_readdir
+struct dirent *
+beos_stat_cache_readdir(DIR *dir)
+{
+	stat_cache_readdir_reply *reply;
+	struct dirent *entry;
+
+	// fall back to standard, if there is no server
+	if (get_server_port() < 0)
+		return readdir(dir);
+
+	reply = (stat_cache_readdir_reply*)dir;
+	if (reply->entryCount == 0)
+		return NULL;
+
+	entry = (struct dirent*)reply->clientData;
+
+	// get the next entry
+	if (--reply->entryCount > 0)
+		reply->clientData = (uint8*)entry + entry->d_reclen;
+
+	return entry;
+}
+
+// beos_stat_cache_closedir
+int
+beos_stat_cache_closedir(DIR *dir)
+{
+	stat_cache_readdir_reply *reply;
+
+	// fall back to standard, if there is no server
+	if (get_server_port() < 0)
+		return closedir(dir);
+
+	reply = (stat_cache_readdir_reply*)dir;
+	free(reply);
+	return 0;
+}
+

src/tools/jam/beos_stat_cache.h

@@ -0,0 +1,15 @@
+// beos_stat_cache.h
+
+#ifndef BEOS_STAT_CACHE_H
+#define BEOS_STAT_CACHE_H
+
+#include <dirent.h>
+#include <sys/stat.h>
+
+int beos_stat_cache_stat(const char *filename, struct stat *st);
+
+DIR* beos_stat_cache_opendir(const char *dirName);
+struct dirent *beos_stat_cache_readdir(DIR *dir);
+int beos_stat_cache_closedir(DIR *dir);
+
+#endif BEOS_STAT_CACHE_H

src/tools/jam/fileunix.c

@@ -99,6 +99,15 @@ struct ar_hdr		/* archive file member header - printable ascii */
 # include <ar.h>
 # endif	
 
+# ifdef OPT_STAT_CACHE_SERVER_EXT
+# include "beos_stat_cache.h"
+inline int stat(const char *filename, struct stat *st)
+	{ return beos_stat_cache_stat(filename, st); }
+# define opendir	beos_stat_cache_opendir
+# define readdir	beos_stat_cache_readdir
+# define closedir	beos_stat_cache_closedir
+# endif
+
 /*
  * file_dirscan() - scan a directory for files
  */