Implement heap classes that each have their own range of allocation sizes they

serve, bin sizes and page size. This minimizes the amount of "large"
allocations made in heaps that don't have a bin for the allocation size
(combining multiple pages). This is desirable as such large allocations
are generally pretty inefficient, and also because it separates larger from
smaller allocations better, making the chance of a heap becoming empty higher.
For now there are three heap classes "small", "large" and "huge", with a
predefined set of bin sizes for each. This might need some finetuning later on.
Reduce the grow size to 4MB though as the allocations should now be spread
across heap classes which each grow on their own.

git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@26210 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/private/kernel/heap.h

@@ -12,10 +12,12 @@
 
 // allocate 16MB initial heap for the kernel
 #define INITIAL_HEAP_SIZE			16 * 1024 * 1024
-// grow by another 8MB each time the heap runs out of memory
+// grow by another 4MB each time the heap runs out of memory
-#define HEAP_GROW_SIZE				8 * 1024 * 1024
+#define HEAP_GROW_SIZE				4 * 1024 * 1024
 // allocate a dedicated 1MB area for dynamic growing
 #define HEAP_DEDICATED_GROW_SIZE	1 * 1024 * 1024
+// use areas for allocations bigger than 1MB
+#define HEAP_AREA_USE_THRESHOLD		1 * 1024 * 1024
 
 
 #ifdef __cplusplus

src/system/kernel/heap.cpp

@@ -97,12 +97,35 @@ typedef struct heap_allocator_s {
 	heap_allocator_s *	next;
 } heap_allocator;
 
+typedef struct heap_class_s {
+	const char *name;
+	uint32		initial_percentage;
+	size_t		max_allocation_size;
+	size_t		page_size;
+	size_t		bin_sizes[20];
+} heap_class;
+
 struct DeferredFreeListEntry : DoublyLinkedListLinkImpl<DeferredFreeListEntry> {
 };
 typedef DoublyLinkedList<DeferredFreeListEntry> DeferredFreeList;
 
-static heap_allocator *sHeapList = NULL;
+// Heap class configuration
-static heap_allocator *sLastGrowRequest = NULL;
+#define HEAP_CLASS_COUNT 3
+static heap_class sHeapClasses[HEAP_CLASS_COUNT] = {
+	{ "small", 50, B_PAGE_SIZE, B_PAGE_SIZE,
+		{ 8, 12, 16, 24, 32, 48, 64, 96, 128, 160, 192, 256, 384, 512, 1024,
+			2048, 4096, 0 }
+	},
+	{ "large", 30, B_PAGE_SIZE * 32, B_PAGE_SIZE * 32,
+		{ 4096, 5120, 6144, 7168, 8192, 12288, 16384, 24576, 32768, 65536,
+			131072, 0 }
+	},
+	{ "huge", 20, HEAP_AREA_USE_THRESHOLD, B_PAGE_SIZE * 64,
+		{ 131072, 262144, 0 }
+	}
+};
+
+static heap_allocator *sHeapList[HEAP_CLASS_COUNT];
 static heap_allocator *sGrowHeapList = NULL;
 static thread_id sHeapGrowThread = -1;
 static sem_id sHeapGrowSem = -1;
@@ -263,25 +286,31 @@ dump_heap_list(int argc, char **argv)
 				heap = heap->next;
 			}
 		} else if (strcmp(argv[1], "stats") == 0) {
+			for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
 				uint32 heapCount = 0;
-			heap_allocator *heap = sHeapList;
+				heap_allocator *heap = sHeapList[i];
 				while (heap) {
 					heapCount++;
 					heap = heap->next;
 				}
 
-			dprintf("current heap count: %ld\n", heapCount);
+				dprintf("current %s heap count: %ld\n", sHeapClasses[i].name,
+					heapCount);
+			}
 		} else
 			print_debugger_command_usage(argv[0]);
 
 		return 0;
 	}
 
-	heap_allocator *heap = sHeapList;
+	for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
+		dprintf("dumping list of %s heaps\n", sHeapClasses[i].name);
+		heap_allocator *heap = sHeapList[i];
 		while (heap) {
 			dump_allocator(heap);
 			heap = heap->next;
 		}
+	}
 
 	return 0;
 }
@@ -313,7 +342,8 @@ dump_allocations(int argc, char **argv)
 
 	size_t totalSize = 0;
 	uint32 totalCount = 0;
-	heap_allocator *heap = sHeapList;
+	uint32 heapClassIndex = 0;
+	heap_allocator *heap = sHeapList[0];
 	while (heap) {
 		// go through all the pages
 		heap_leak_check_info *info = NULL;
@@ -390,6 +420,8 @@ dump_allocations(int argc, char **argv)
 		}
 
 		heap = heap->next;
+		if (heap == NULL && ++heapClassIndex < HEAP_CLASS_COUNT)
+			heap = sHeapList[heapClassIndex];
 	}
 
 	dprintf("total allocations: %lu; total bytes: %lu\n", totalCount, totalSize);
@@ -453,7 +485,8 @@ dump_allocations_per_caller(int argc, char **argv)
 
 	sCallerInfoCount = 0;
 
-	heap_allocator *heap = sHeapList;
+	uint32 heapClassIndex = 0;
+	heap_allocator *heap = sHeapList[0];
 	while (heap) {
 		// go through all the pages
 		heap_leak_check_info *info = NULL;
@@ -520,6 +553,8 @@ dump_allocations_per_caller(int argc, char **argv)
 		}
 
 		heap = heap->next;
+		if (heap == NULL && ++heapClassIndex < HEAP_CLASS_COUNT)
+			heap = sHeapList[heapClassIndex];
 	}
 
 	// sort the array
@@ -674,26 +709,30 @@ heap_validate_heap(heap_allocator *heap)
 
 
 static heap_allocator *
-heap_attach(addr_t base, size_t size)
+heap_attach(addr_t base, size_t size, uint32 heapClass)
 {
 	heap_allocator *heap = (heap_allocator *)base;
 	base += sizeof(heap_allocator);
 	size -= sizeof(heap_allocator);
 
-	size_t binSizes[] = { 8, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512, 1024, 2048, B_PAGE_SIZE };
+	heap->page_size = sHeapClasses[heapClass].page_size;
-	uint32 binCount = sizeof(binSizes) / sizeof(binSizes[0]);
-	heap->page_size = B_PAGE_SIZE;
-	heap->bin_count = binCount;
 	heap->bins = (heap_bin *)base;
-	base += binCount * sizeof(heap_bin);
-	size -= binCount * sizeof(heap_bin);
 
-	for (uint32 i = 0; i < binCount; i++) {
+	heap->bin_count = 0;
-		heap_bin *bin = &heap->bins[i];
+	while (true) {
-		bin->element_size = binSizes[i];
+		size_t binSize = sHeapClasses[heapClass].bin_sizes[heap->bin_count];
-		bin->max_free_count = heap->page_size / binSizes[i];
+		if (binSize == 0)
+			break;
+
+		heap_bin *bin = &heap->bins[heap->bin_count];
+		bin->element_size = binSize;
+		bin->max_free_count = heap->page_size / binSize;
 		bin->page_list = NULL;
-	}
+		heap->bin_count++;
+	};
+
+	base += heap->bin_count * sizeof(heap_bin);
+	size -= heap->bin_count * sizeof(heap_bin);
 
 	uint32 pageCount = size / heap->page_size;
 	size_t pageTableSize = pageCount * sizeof(heap_page);
@@ -726,8 +765,8 @@ heap_attach(addr_t base, size_t size)
 	mutex_init(&heap->lock, "heap_mutex");
 
 	heap->next = NULL;
-	dprintf("heap_attach: attached to %p - usable range 0x%08lx - 0x%08lx\n",
+	dprintf("heap_attach: %s heap attached to %p - usable range 0x%08lx - 0x%08lx\n",
-		heap, heap->base, heap->base + heap->size);
+		sHeapClasses[heapClass].name, heap, heap->base, heap->base + heap->size);
 	return heap;
 }
 
@@ -900,6 +939,15 @@ is_valid_alignment(size_t number)
 #endif
 
 
+inline bool
+heap_should_grow(heap_allocator *heap)
+{
+	// suggest growing if it is the last heap and has less than 10% free pages
+	return (heap->next == NULL)
+		&& heap->free_page_count < heap->page_count / 10;
+}
+
+
 static void *
 heap_memalign(heap_allocator *heap, size_t alignment, size_t size,
 	bool *shouldGrow)
@@ -932,11 +980,8 @@ heap_memalign(heap_allocator *heap, size_t alignment, size_t size,
 
 	TRACE(("memalign(): asked to allocate %lu bytes, returning pointer %p\n", size, address));
 
-	if (heap->next == NULL && shouldGrow) {
+	if (shouldGrow)
-		// suggest growing if we are the last heap and we have
+		*shouldGrow = heap_should_grow(heap);
-		// less than 10% free pages
-		*shouldGrow = heap->free_page_count < heap->page_count / 10;
-	}
 
 #if KERNEL_HEAP_LEAK_CHECK
 	size -= sizeof(heap_leak_check_info);
@@ -1076,6 +1121,9 @@ heap_free(heap_allocator *heap, void *address)
 		}
 	}
 
+	if (heap->free_page_count == heap->page_count)
+		dprintf("heap_free: heap %p is completely empty and could be freed\n", heap);
+
 	T(Free((addr_t)address));
 	mutex_unlock(&heap->lock);
 	return B_OK;
@@ -1168,6 +1216,23 @@ heap_realloc(heap_allocator *heap, void *address, void **newAddress,
 }
 
 
+inline uint32
+heap_class_for(size_t size)
+{
+#if KERNEL_HEAP_LEAK_CHECK
+	// take the extra info size into account
+	size += sizeof(heap_leak_check_info_s);
+#endif
+
+	for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
+		if (size <= sHeapClasses[i].max_allocation_size)
+			return i;
+	}
+
+	return HEAP_CLASS_COUNT - 1;
+}
+
+
 static void
 deferred_deleter(void *arg, int iteration)
 {
@@ -1191,7 +1256,7 @@ deferred_deleter(void *arg, int iteration)
 
 
 static heap_allocator *
-heap_create_new_heap(const char *name, size_t size)
+heap_create_new_heap(const char *name, size_t size, uint32 heapClass)
 {
 	void *heapAddress = NULL;
 	area_id heapArea = create_area(name, &heapAddress,
@@ -1202,7 +1267,7 @@ heap_create_new_heap(const char *name, size_t size)
 		return NULL;
 	}
 
-	heap_allocator *newHeap = heap_attach((addr_t)heapAddress, size);
+	heap_allocator *newHeap = heap_attach((addr_t)heapAddress, size, heapClass);
 	if (newHeap == NULL) {
 		panic("heap: could not attach heap to area!\n");
 		delete_area(heapArea);
@@ -1219,7 +1284,6 @@ heap_create_new_heap(const char *name, size_t size)
 static int32
 heap_grow_thread(void *)
 {
-	heap_allocator *heap = sHeapList;
 	heap_allocator *growHeap = sGrowHeapList;
 	while (true) {
 		// wait for a request to grow the heap list
@@ -1234,7 +1298,7 @@ heap_grow_thread(void *)
 			// a new one to make some room.
 			TRACE(("heap_grower: grow heaps will run out of memory soon\n"));
 			heap_allocator *newHeap = heap_create_new_heap(
-				"additional grow heap", HEAP_DEDICATED_GROW_SIZE);
+				"additional grow heap", HEAP_DEDICATED_GROW_SIZE, 0);
 			if (newHeap != NULL) {
 #if PARANOID_VALIDATION
 				heap_validate_heap(newHeap);
@@ -1245,24 +1309,25 @@ heap_grow_thread(void *)
 			}
 		}
 
+		for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
 			// find the last heap
+			heap_allocator *heap = sHeapList[i];
 			while (heap->next)
 				heap = heap->next;
 
-		if (sLastGrowRequest != heap) {
+			if (heap_should_grow(heap)) {
-			// we have already grown since the latest request, just ignore
+				// grow this heap if it makes sense to
-			continue;
-		}
-
-		TRACE(("heap_grower: kernel heap will run out of memory soon, allocating new one\n"));
 				heap_allocator *newHeap = heap_create_new_heap("additional heap",
-			HEAP_GROW_SIZE);
+					HEAP_GROW_SIZE, i);
 				if (newHeap != NULL) {
 #if PARANOID_VALIDATION
 					heap_validate_heap(newHeap);
 #endif
 					heap->next = newHeap;
-			TRACE(("heap_grower: new heap linked in\n"));
+					TRACE(("heap_grower: new %s heap linked in\n",
+						sHeapClasses[i].name));
+				}
+			}
 		}
 
 		// notify anyone waiting for this request
@@ -1276,7 +1341,11 @@ heap_grow_thread(void *)
 status_t
 heap_init(addr_t base, size_t size)
 {
-	sHeapList = heap_attach(base, size);
+	for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
+		size_t partSize = size * sHeapClasses[i].initial_percentage / 100;
+		sHeapList[i] = heap_attach(base, partSize, i);
+		base += partSize;
+	}
 
 	// set up some debug commands
 	add_debugger_command_etc("heap", &dump_heap_list,
@@ -1329,7 +1398,7 @@ status_t
 heap_init_post_thread()
 {
 	sGrowHeapList = heap_create_new_heap("dedicated grow heap",
-		HEAP_DEDICATED_GROW_SIZE);
+		HEAP_DEDICATED_GROW_SIZE, 0);
 	if (sGrowHeapList == NULL) {
 		panic("heap_init_post_thread(): failed to attach dedicated grow heap\n");
 		return B_ERROR;
@@ -1361,19 +1430,18 @@ memalign(size_t alignment, size_t size)
 		return NULL;
 	}
 
-	if (size > (HEAP_GROW_SIZE * 3) / 4) {
+	if (size > HEAP_AREA_USE_THRESHOLD) {
-		// don't even attempt such a huge allocation
+		// don't even attempt such a huge allocation - use areas instead
 		panic("heap: huge allocation of %lu bytes asked!\n", size);
 		return NULL;
 	}
 
-	heap_allocator *heap = sHeapList;
+	heap_allocator *heap = sHeapList[heap_class_for(size)];
 	while (heap) {
 		bool shouldGrow = false;
 		void *result = heap_memalign(heap, alignment, size, &shouldGrow);
 		if (heap->next == NULL && (shouldGrow || result == NULL)) {
 			// the last heap will or has run out of memory, notify the grower
-			sLastGrowRequest = heap;
 			if (result == NULL) {
 				// urgent request, do the request and wait
 				switch_sem(sHeapGrowSem, sHeapGrownNotify);
@@ -1429,7 +1497,7 @@ malloc_nogrow(size_t size)
 	}
 
 	// try public memory, there might be something available
-	heap_allocator *heap = sHeapList;
+	heap_allocator *heap = sHeapList[heap_class_for(size)];
 	while (heap) {
 		void *result = heap_memalign(heap, 0, size, NULL);
 		if (result != NULL)
@@ -1462,7 +1530,8 @@ free(void *address)
 		return;
 	}
 
-	heap_allocator *heap = sHeapList;
+	for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
+		heap_allocator *heap = sHeapList[i];
 		while (heap) {
 			if (heap_free(heap, address) == B_OK) {
 #if PARANOID_VALIDATION
@@ -1473,9 +1542,10 @@ free(void *address)
 
 			heap = heap->next;
 		}
+	}
 
 	// maybe it was allocated from a dedicated grow heap
-	heap = sGrowHeapList;
+	heap_allocator *heap = sGrowHeapList;
 	while (heap) {
 		if (heap_free(heap, address) == B_OK)
 			return;
@@ -1503,7 +1573,8 @@ realloc(void *address, size_t newSize)
 		return NULL;
 	}
 
-	heap_allocator *heap = sHeapList;
+	for (uint32 i = 0; i < HEAP_CLASS_COUNT; i++) {
+		heap_allocator *heap = sHeapList[i];
 		while (heap) {
 			void *newAddress = NULL;
 			if (heap_realloc(heap, address, &newAddress, newSize) == B_OK) {
@@ -1515,6 +1586,7 @@ realloc(void *address, size_t newSize)
 
 			heap = heap->next;
 		}
+	}
 
 	panic("realloc(): failed to realloc address %p to size %lu\n", address, newSize);
 	return NULL;