Merge pull request #124 from pitust/trunk

bootboot: add initrd support, and add a warning about the init stack

stage23/protos/bootboot.c

@@ -2,6 +2,7 @@
 #include <stddef.h>
 #include <stdbool.h>
 #include <protos/bootboot.h>
+#include <protos/bootboot/initrd.h>
 #include <lib/libc.h>
 #include <lib/elf.h>
 #include <lib/blib.h>
@@ -49,25 +50,105 @@ void bootboot_load(char *config) {
     uint64_t fb_vaddr = BOOTBOOT_FB;
     uint64_t struct_vaddr = BOOTBOOT_INFO;
     uint64_t env_vaddr = BOOTBOOT_ENV;
-    uint64_t init_stack_size = 1024;
+    uint64_t init_stack_size = ~0;
 
     /// Config ///
     char *kernel_path = config_get_value(config, 0, "KERNEL_PATH");
-    if (kernel_path == NULL)
-        panic("bootboot: KERNEL_PATH not specified");
     
-    char *initrd = config_get_value(config, 0, "INITRD");
+    char *initrd = config_get_value(config, 0, "INITRD_PATH");
     if (initrd == NULL) {
-        print("bootboot: warning: no initrd!\n");
+        initrd = kernel_path;
+        kernel_path = NULL;
     }
 
-    /// Kernel loading code ///
+    if (kernel_path == NULL && initrd == NULL)
-    print("bootboot: Loading kernel `%s`...\n", kernel_path);
+        panic("bootboot: no KERNEL_PATH or INITRD_PATH specified!");
-    struct file_handle* kernel_file;
-    if ((kernel_file = uri_open(kernel_path)) == NULL)
-        panic("bootboot: Failed to open kernel with path `%s`. Is the path correct?\n", kernel_path);
 
-    uint8_t* kernel = freadall(kernel_file, MEMMAP_KERNEL_AND_MODULES);
+    /// Initrd loading ///
+    file_t bootboot_initrd_file;
+    uint64_t initrd_start = 0, initrd_size = 0;
+    if (initrd) {
+        struct file_handle* initrd_file;
+        if ((initrd_file = uri_open(initrd)) == NULL)
+            panic("bootboot: Failed to open initrd with path `%s`. Is the path correct?\n", initrd);
+
+        uint8_t* initrd_data = freadall(initrd_file, MEMMAP_KERNEL_AND_MODULES);
+        initrd_size = initrd_file->size;
+        initrd_start = (uint64_t)(size_t)initrd_data;
+        fclose(initrd_file);
+        bootboot_initrd_file.size = initrd_size;
+        bootboot_initrd_file.data = initrd_data;
+    } else {
+        panic("bootboot: logic error: no initrd, even though one MUST be present");
+    }
+
+    /// Load bootboot config ///
+#define _emitenv(e) do { \
+        if(envoff >= 4095) panic("bootboot: too much env data!"); \
+        env[envoff++] = e; \
+    } while (false);
+    uint8_t* env = (uint8_t*)ext_mem_alloc_type_aligned(4096, MEMMAP_BOOTLOADER_RECLAIMABLE, 4096);
+    uint64_t envoff = 0;
+    do {
+        file_t conf = initrd_open_auto(bootboot_initrd_file, "sys/config");
+        if (!conf.data) break;
+
+        uint8_t state = 0;
+        uint8_t ipeq = 0;
+
+        // state 0: kv
+        // state 1: precomment
+        // state 2: comment
+
+        for (uint64_t inoff = 0;inoff < conf.size;inoff++) {
+            if (conf.data[inoff] == ' ' && !(state == 0 && ipeq == 2)) continue;
+            if (conf.data[inoff] != ' ' && state == 0 && ipeq == 1) { ipeq = 2; }
+            if (conf.data[inoff] == '/' && state == 1) { state = 2; continue; }
+            if (conf.data[inoff] == '\n' && state == 2) { state = 0; continue; }
+            if (conf.data[inoff] == '\n') { ipeq = 0; }
+            if (conf.data[inoff] == '/' && state == 0) { state = 1; continue; }
+            if (state == 1) { state = 0; _emitenv('/'); _emitenv(conf.data[inoff]); continue; }
+            if (state == 2) continue;
+            if (state == 0 && conf.data[inoff] == '=') ipeq = 1;
+            _emitenv(conf.data[inoff]);
+        }
+    } while (false);
+    _emitenv(0);
+    print("env: ---------\n%s\n--------\n", env);
+
+    /// Kernel loading code ///
+    uint8_t* kernel;
+    if (kernel_path) {
+        print("bootboot: Loading kernel `%s`...\n", kernel_path);
+        struct file_handle* kernel_file;
+        if ((kernel_file = uri_open(kernel_path)) == NULL)
+            panic("bootboot: Failed to open kernel with path `%s`. Is the path correct?\n", kernel_path);
+
+        kernel = freadall(kernel_file, MEMMAP_KERNEL_AND_MODULES);
+    } else {
+        const char* corefile = config_get_value((char*)env, 0, "kernel");
+        if (!corefile) corefile = "sys/core";
+        file_t file = initrd_open_auto(bootboot_initrd_file, corefile);
+        kernel = file.data;
+        if (!file.size) panic("bootboot: cannot find the kernel!");
+    }
+
+    /// Memory mappings ///
+    pagemap_t pmap = new_pagemap(4);
+
+    /// Load kernel ///
+    uint64_t entry, top, slide, rangecount, physbase, virtbase = 0;
+    struct elf_range* ranges;
+    {
+        if (elf64_load(
+            kernel, &entry, &top, &slide, MEMMAP_KERNEL_AND_MODULES,
+            false, false, &ranges, &rangecount, true, &physbase, &virtbase)) {
+            panic("bootboot: elf64 load failed");
+        }
+        for (uint64_t mapvirt = virtbase, mapphys = physbase; mapphys < top;mapvirt += 0x1000, mapphys += 0x1000) {
+            map_page(pmap, mapvirt, mapphys, VMM_FLAG_PRESENT | VMM_FLAG_WRITE, false);
+        }
+    }
     
     /// Funky macros ///
 #define KOFFSET(type, off) (type)&kernel[(off)]
@@ -86,7 +167,7 @@ void bootboot_load(char *config) {
         if(!strcmp(secname, ".strtab")) string_table = section_header;
     }
     if (!symbol_table || !string_table) {
-        print("bootboot: warning: no symbol/string tables in the ELF!");
+        print("bootboot: warning: no symbol/string tables in the ELF!\n");
     } else {
         struct elf64_sym* symbols = KOFFSET(struct elf64_sym*, symbol_table->sh_offset);
         char* symbol_strings = KOFFSET(char*, string_table->sh_offset);
@@ -101,34 +182,23 @@ void bootboot_load(char *config) {
         }
     }
 
-    printv("bootboot: mapping struct to %X", struct_vaddr);
+    if (init_stack_size == ~0UL) {
-    printv("bootboot: mapping environemnt to %X", env_vaddr);
+        print("bootboot: warning: no init stack size entered, assuming 1024\n");
-    printv("bootboot: mapping framebuffer to %X", fb_vaddr);
+        print("1024 is really small, specify more using initstack=size ini initrd;\n");
-    printv("bootboot: the init stack is %X bytes", init_stack_size);
+        init_stack_size = 1024;
-
+        delay(1000000);
-    uint64_t entry, top, slide, rangecount, physbase, virtbase = 0;
-    struct elf_range* ranges;
-
-    /// Memory mappings ///
-    pagemap_t pmap = new_pagemap(4);
-
-    /// Load kernel ///
-    {
-        if (elf64_load(
-            kernel, &entry, &top, &slide, MEMMAP_KERNEL_AND_MODULES,
-            false, false, &ranges, &rangecount, true, &physbase, &virtbase)) {
-            panic("bootboot: elf64 load failed");
-        }
-        for (uint64_t mapvirt = virtbase, mapphys = physbase; mapphys < top;mapvirt += 0x1000, mapphys += 0x1000) {
-            map_page(pmap, mapvirt, mapphys, VMM_FLAG_PRESENT | VMM_FLAG_WRITE, false);
-        }
     }
+    printv("bootboot: mapping struct to %X\n", struct_vaddr);
+    printv("bootboot: mapping environemnt to %X\n", env_vaddr);
+    printv("bootboot: mapping framebuffer to %X\n", fb_vaddr);
+    printv("bootboot: the init stack is %X bytes\n", init_stack_size);;
+
+    /// Bootboot structure ///
     BOOTBOOT* bootboot = (BOOTBOOT*)ext_mem_alloc_type_aligned(4096, MEMMAP_BOOTLOADER_RECLAIMABLE, 4096);
     map_page(pmap, struct_vaddr, (uint64_t)(size_t)bootboot, VMM_FLAG_PRESENT | VMM_FLAG_WRITE, false);
 
     /// Environment ///
     {
-        char* env = (char*)ext_mem_alloc_type_aligned(4096, MEMMAP_BOOTLOADER_RECLAIMABLE, 4096);
         map_page(pmap, env_vaddr, (uint64_t)(size_t)env, VMM_FLAG_PRESENT | VMM_FLAG_WRITE, false);
         uint32_t index = 0, offset = 0;
         char* cfgent = NULL;
@@ -167,19 +237,6 @@ void bootboot_load(char *config) {
         map_page(pmap, fb_vaddr + current, fbi.framebuffer_addr + current, VMM_FLAG_PRESENT | VMM_FLAG_WRITE, false);
     }
 
-    /// Initrd loading ///
-    uint64_t initrd_start = 0, initrd_size = 0;
-    if (initrd) {
-        struct file_handle* initrd_file;
-        if ((initrd_file = uri_open(initrd)) == NULL)
-            panic("bootboot: Failed to open initrd with path `%s`. Is the path correct?\n", initrd);
-
-        uint8_t* initrd_data = freadall(initrd_file, MEMMAP_KERNEL_AND_MODULES);
-        initrd_size = initrd_file->size;
-        initrd_start = (uint64_t)(size_t)initrd_data;
-        fclose(initrd_file);
-    }
-
     /// Header info ///
     memcpy(bootboot->magic, "BOOT", 4);
 #if bios
@@ -191,7 +248,7 @@ void bootboot_load(char *config) {
     /// SMP info ///
     size_t numcores;
     uint32_t bsplapic;
-    struct smp_information* cores;
+    volatile struct smp_information* cores;
     init_smp(0, (void**)&cores, &numcores, &bsplapic, true, false, pmap, false, false);
     bootboot->numcores = numcores;
     bootboot->bspid = bsplapic;

stage23/protos/bootboot/cpio.c

@@ -0,0 +1,75 @@
+#include "lib/print.h"
+#include <protos/bootboot/initrd.h>
+#include <lib/libc.h>
+#include <stdint.h>
+
+#define HPODC_MAGIC "070707"
+
+/**
+ * cpio archive
+ */
+INITRD_HANDLER(cpio) {
+    /// Some macros///
+#define _offset(cnt) do { offset += (cnt); if (offset > file.size) return (file_t){ 0, NULL }; } while (false);
+#define _atoffset() (&file.data[offset])
+#define _must(n) do { if ((offset + (n)) > file.size) return (file_t){ 0, NULL }; } while (false);
+    uint64_t offset = 0;
+    if (file.size < 6) return (file_t){ 0, NULL };
+
+    /// Check magic ///
+    // this may be a bit unclear, but this checks if the file even **has** a cpio header
+    if (memcmp(file.data,"070701",6) && memcmp(file.data, "070702", 6) && memcmp(file.data, "070707", 6))
+        return (file_t){ 0, NULL };
+    
+    /// Some variables ///
+    uint64_t path_name_size = strlen(path);
+
+    /// hpodc archive ///
+    while (!memcmp(_atoffset(), HPODC_MAGIC, 6)) {
+        _must(22);
+        uint32_t name_size = oct2bin(_atoffset()+ 8 * 6 + 11, 6);
+        uint32_t file_size = oct2bin(_atoffset()+ 8 * 6 + 11 + 6, 11);
+        _must(9 * 6 + 2 * 11 + name_size);
+
+        uint8_t* target_path = _atoffset() + 9 * 6 + 2 * 11;
+
+        if (name_size > 2 && target_path[0] == '.' && target_path[1] == '/') {
+            target_path += 2;
+        }
+
+        if (!memcmp(target_path, path, path_name_size + 1)) {
+            return (file_t){
+                file_size,
+                _atoffset() + 9 * 6 + 2 * 11 + name_size,
+            };
+        }
+        _offset(76 + name_size + file_size);
+    }
+    offset = 0;
+    // newc and crc archive
+    while(!memcmp(_atoffset(), "07070", 5)){
+        uint32_t file_size = hex2bin(_atoffset() + 8 * 6 + 6, 8);
+        uint32_t name_size = hex2bin(_atoffset() + 8 * 11 + 6, 8);
+
+        uint8_t* target_path = _atoffset() + 110;
+
+        if (name_size > 2 && target_path[0] == '.' && target_path[1] == '/') {
+            target_path += 2;
+        }
+        
+        if (!memcmp(target_path, path, path_name_size + 1)) {
+            uint8_t buf[9];
+            memcpy(buf, _atoffset() + 8 * 11 + 6, 8);
+            buf[8] = 0;
+            return (file_t){
+                file_size,
+                (_atoffset() + ((110 + name_size + 3) / 4) * 4),
+            };
+        }
+        _offset(((110 + name_size + 3) / 4) * 4 + ((file_size + 3) / 4) * 4);
+    }
+    char buf[9];
+    memcpy(buf, _atoffset(), 8);
+    buf[8] = 0;
+    return (file_t){ 0, NULL };
+}

stage23/protos/bootboot/initrd.c

@@ -0,0 +1,72 @@
+#include <protos/bootboot/initrd.h>
+#include <lib/print.h>
+#include <lib/libc.h>
+#include <lib/blib.h>
+
+
+INITRD_HANDLER(jamesm);
+INITRD_HANDLER(ustar);
+INITRD_HANDLER(cpio);
+
+INITRD_HANDLER(auto) {
+#define DETECT_FAILED panic("bootboot: cannot read file `%s`: cannot detect initrd type (only ustar, cpio and jamesm is supported).", path)
+    if (file.size < 4) DETECT_FAILED;
+    if (!memcmp(file.data, "\x7f""ELF", 4)) {
+        if (strcmp("sys/core", path) == 0) {
+            printv("bootboot: using ELF as initrd to open sys/core\n");
+            return file;
+        }
+        return (file_t){0, NULL};
+    }
+    if (file.size < 5) DETECT_FAILED;
+    if (file.data[4] == 0xBF) {
+        file_t jamesm_attempt = initrd_open_jamesm(file, path);
+        if (jamesm_attempt.data) {
+            printv("bootboot: jamesm matched when reading file `%s`\n", path);
+            return jamesm_attempt;
+        }
+        panic("bootboot: cannot read file `%s`: no such file or directory", path);
+    }
+    if (!memcmp("07070", file.data, 5)) {
+        file_t cpio_attempt = initrd_open_cpio(file, path);
+        if (cpio_attempt.data) {
+            printv("bootboot: cpio matched when reading file `%s`\n", path);
+            return cpio_attempt;
+        }
+        panic("bootboot: cannot read file `%s`: no such file or directory", path);
+    }
+    if (file.size < 262) DETECT_FAILED;
+    if (!memcmp("ustar", file.data + 257, 5)) {
+        file_t ustar_attempt = initrd_open_ustar(file, path);
+        if (ustar_attempt.data) {
+            printv("bootboot: ustar matched when reading file `%s`\n", path);
+            return ustar_attempt;
+        }
+        panic("bootboot: cannot read file `%s`: no such file or directory", path);
+    }
+    DETECT_FAILED;
+}
+
+/// Utilities ///
+uint32_t oct2bin(uint8_t* str, uint32_t max) {
+    uint32_t value = 0;
+    while(max-- > 0) {
+        value <<= 3;
+        value += *str++ - '0';
+    }
+    return value;
+}
+uint32_t hex2bin(uint8_t* str, uint32_t size) {
+    uint32_t value = 0;
+    while(size-- > 0){
+        value <<= 4;
+        if (*str >= '0' && *str <= '9')
+            value += (uint32_t)((*str) - '0');
+        else if (*str >= 'A' && *str <= 'F')
+            value += (uint32_t)((*str) - 'A' + 10);
+        else if (*str >= 'a' && *str <= 'f')
+            value += (uint32_t)((*str) - 'a' + 10);
+        str++;
+    }
+    return value;
+}

stage23/protos/bootboot/initrd.h

@@ -0,0 +1,19 @@
+#ifndef __PROTOS__BOOTBOOT_FS_H__
+#define __PROTOS__BOOTBOOT_FS_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+
+typedef struct file {
+    uint64_t size;
+    uint8_t* data;
+} file_t;
+
+#define INITRD_HANDLER(name) file_t initrd_open_##name(file_t file, const char* path)
+
+INITRD_HANDLER(auto);
+
+uint32_t oct2bin(uint8_t* str, uint32_t max);
+uint32_t hex2bin(uint8_t* str, uint32_t size);
+
+#endif

stage23/protos/bootboot/jamesm.c

@@ -0,0 +1,32 @@
+#include <protos/bootboot/initrd.h>
+#include <lib/libc.h>
+#include <stdint.h>
+
+// This looks really sketchy, but that's what the official """spec""" says...
+typedef struct initrd_header {
+   unsigned char magic; // The magic number is there to check for consistency.
+   char name[64];
+   unsigned int offset; // Offset in the initrd the file starts.
+   unsigned int length; // Length of the file.
+} initrd_entry_t;
+
+INITRD_HANDLER(jamesm) {
+    if (file.size < 5) return (file_t){};
+
+    uint32_t file_count = *((uint32_t*)(file.data));
+    uint32_t path_len = strlen(path);
+
+    initrd_entry_t* data = (initrd_entry_t*)(file.data + 4);
+    if (data[0].magic != 0xBF) return (file_t){};
+
+    for(uint32_t i = 0;i < file_count;i++) {
+        if (data[i].magic != 0xBF) return (file_t){};
+        if(!memcmp(data[i].name, path, path_len + 1)){
+            return (file_t){
+                data[i].length,
+                file.data + data[i].offset
+            };
+        }
+    }
+    return (file_t){};
+}

stage23/protos/bootboot/ustar.c

@@ -0,0 +1,27 @@
+#include <protos/bootboot/initrd.h>
+#include <lib/libc.h>
+#include <stdint.h>
+
+INITRD_HANDLER(ustar) {
+#define PTR (file.data + offset)
+    if (file.size < 262) return (file_t){};
+
+    if(memcmp(file.data + 257, "ustar", 5))
+        return (file_t){};
+    
+    uint32_t path_size = strlen(path);
+    uint32_t offset = 0;
+
+    while(!memcmp(PTR + 257,"ustar",5)){
+        uint32_t file_size = oct2bin(PTR + 0x7c,11);
+        if(!memcmp(PTR, path, path_size + 1)
+             || (PTR[0] == '.' && PTR[1] == '/' && !memcmp(PTR + 2, path, path_size + 1))) {
+            return (file_t){
+                file_size,
+                PTR+512,
+            };
+        }
+        offset += (((file_size + 511) / 512) + 1) * 512;
+    }
+    return (file_t){};
+}