From b53a56fe7230f75e5e85462f3789a2216025545b Mon Sep 17 00:00:00 2001 From: "K. Lange" Date: Sun, 23 Jan 2022 10:36:46 +0900 Subject: [PATCH] aarch64: begin work on new target --- .gitignore | 1 + Makefile | 123 +-- apps/cpu-name.krk | 5 +- apps/demo.c | 68 +- apps/init.c | 1 + apps/killall.c | 2 +- apps/ps.c | 2 +- apps/ttysize.c | 2 +- base/usr/include/kernel/arch/aarch64/pml.h | 39 + base/usr/include/kernel/arch/aarch64/regs.h | 22 + base/usr/include/kernel/arch/x86_64/mmu.h | 49 -- base/usr/include/kernel/elf.h | 4 + base/usr/include/kernel/mmu.h | 55 +- base/usr/include/kernel/process.h | 23 +- base/usr/include/kernel/signal.h | 7 + base/usr/include/syscall.h | 87 ++ build/aarch64.mk | 47 + build/x86_64.mk | 114 +++ kernel/arch/aarch64/bootstrap.S | 212 +++++ kernel/arch/aarch64/bootstub/bootstrap.S | 8 + kernel/arch/aarch64/bootstub/link.ld | 53 ++ kernel/arch/aarch64/bootstub/main.c | 458 ++++++++++ kernel/arch/aarch64/link.ld | 53 ++ kernel/arch/aarch64/mmu.c | 741 ++++++++++++++++ kernel/arch/aarch64/user.c | 927 ++++++++++++++++++++ kernel/misc/elf64.c | 2 +- kernel/misc/malloc.c | 4 +- kernel/misc/pci.c | 39 +- kernel/misc/string.c | 2 +- kernel/sys/process.c | 34 +- kernel/sys/ptrace.c | 14 +- kernel/sys/syscall.c | 12 +- kernel/vfs/procfs.c | 26 +- kernel/video/lfbvideo.c | 13 +- lib/confreader.c | 2 +- lib/graphics.c | 10 +- lib/jpeg.c | 4 +- libc/arch/aarch64/bad.c | 17 + libc/arch/aarch64/crt0.S | 9 + libc/arch/aarch64/crti.S | 12 + libc/arch/aarch64/crtn.S | 8 + libc/arch/aarch64/memcpy.c | 11 + libc/arch/aarch64/memset.c | 9 + libc/pthread/pthread.c | 8 +- libc/stdlib/malloc.c | 2 +- libc/wchar/wcstol.c | 6 +- linker/link.ld | 1 + linker/linker.c | 110 ++- modules/vmware.c | 3 + util/binutils-gdb | 2 +- util/gcc | 2 +- 51 files changed, 3246 insertions(+), 219 deletions(-) create mode 100644 base/usr/include/kernel/arch/aarch64/pml.h create mode 100644 base/usr/include/kernel/arch/aarch64/regs.h delete mode 100644 base/usr/include/kernel/arch/x86_64/mmu.h create mode 100644 build/aarch64.mk create mode 100644 kernel/arch/aarch64/bootstrap.S create mode 100644 kernel/arch/aarch64/bootstub/bootstrap.S create mode 100644 kernel/arch/aarch64/bootstub/link.ld create mode 100644 kernel/arch/aarch64/bootstub/main.c create mode 100644 kernel/arch/aarch64/link.ld create mode 100644 kernel/arch/aarch64/mmu.c create mode 100644 kernel/arch/aarch64/user.c create mode 100644 libc/arch/aarch64/bad.c create mode 100644 libc/arch/aarch64/crt0.S create mode 100644 libc/arch/aarch64/crti.S create mode 100644 libc/arch/aarch64/crtn.S create mode 100644 libc/arch/aarch64/memcpy.c create mode 100644 libc/arch/aarch64/memset.c diff --git a/.gitignore b/.gitignore index cd495325..b8d2acea 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ /fatbase /image.iso /boot/mbr.sys +/bootstub diff --git a/Makefile b/Makefile index 460e047d..a55fe1fd 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,9 @@ TOOLCHAIN=util BASE=base export PATH := $(shell $(TOOLCHAIN)/activate.sh) -include build/x86_64.mk +ARCH ?= x86_64 + +include build/${ARCH}.mk # Cross compiler binaries CC = ${TARGET}-gcc @@ -20,7 +22,7 @@ KERNEL_CFLAGS += -pedantic -Wwrite-strings ${ARCH_KERNEL_CFLAGS} # Defined constants for the kernel KERNEL_CFLAGS += -D_KERNEL_ -DKERNEL_ARCH=${ARCH} -KERNEL_CFLAGS += -DKERNEL_GIT_TAG=`util/make-version` +KERNEL_CFLAGS += -DKERNEL_GIT_TAG=$(shell util/make-version) # Automatically find kernel sources from relevant paths KERNEL_OBJS = $(patsubst %.c,%.o,$(wildcard kernel/*.c)) @@ -39,38 +41,7 @@ KERNEL_SOURCES += $(wildcard kernel/arch/${ARCH}/*.S) # a single relocatable object file. MODULES = $(patsubst modules/%.c,$(BASE)/mod/%.ko,$(wildcard modules/*.c)) -# Configs you can override. -# SMP: Argument to -smp, use 1 to disable SMP. -# RAM: Argument to -m, QEMU takes suffixes like "M" or "G". -# EXTRA_ARGS: Added raw to the QEMU command line -# EMU_KVM: Unset this (EMU_KVM=) to use TCG, or replace it with something like EMU_KVM=-enable-haxm -# EMU_MACH: Argument to -M, 'pc' should be the older default in QEMU; we use q35 to test AHCI. -SMP ?= 4 -RAM ?= 3G -EXTRA_ARGS ?= -EMU_KVM ?= -enable-kvm -EMU_MACH ?= q35 - -EMU = qemu-system-x86_64 -EMU_ARGS = -M q35 -EMU_ARGS += -m $(RAM) -EMU_ARGS += -smp $(SMP) -EMU_ARGS += ${EMU_KVM} -EMU_ARGS += -no-reboot -EMU_ARGS += -serial mon:stdio -EMU_ARGS += -soundhw pcspk,ac97 - -# UTC is the default setting. -#EMU_ARGS += -rtc base=utc - -# Customize network options here. QEMU's default is an e1000(e) under PIIX (Q35), with user networking -# so we don't need to do anything normally. -#EMU_ARGS += -net user -#EMU_ARGS += -netdev hubport,id=u1,hubid=0, -device e1000e,netdev=u1 -object filter-dump,id=f1,netdev=u1,file=qemu-e1000e.pcap -#EMU_ARGS += -netdev hubport,id=u2,hubid=0, -device e1000e,netdev=u2 - -# Add an XHCI tablet if you want to dev on USB -#EMU_ARGS += -device qemu-xhci -device usb-tablet +EMU = qemu-system-${ARCH} APPS=$(patsubst apps/%.c,%,$(wildcard apps/*.c)) APPS_X=$(foreach app,$(APPS),$(BASE)/bin/$(app)) @@ -103,9 +74,6 @@ LC = $(BASE)/lib/libc.so $(GCC_SHARED) .PHONY: all system clean run shell -all: system -system: image.iso - $(BASE)/mod/%.ko: modules/%.c | dirs ${CC} -c ${KERNEL_CFLAGS} -mcmodel=large -o $@ $< @@ -129,33 +97,6 @@ $(BASE)/lib/libkuroko.so: $(KRK_SRC) | $(LC) $(BASE)/lib/ld.so: linker/linker.c $(BASE)/lib/libc.a | dirs $(LC) $(CC) -g -static -Wl,-static $(CFLAGS) -z max-page-size=0x1000 -o $@ -Os -T linker/link.ld $< -run: system - ${EMU} ${EMU_ARGS} -cdrom image.iso - -fast: system - ${EMU} ${EMU_ARGS} -cdrom image.iso \ - -fw_cfg name=opt/org.toaruos.bootmode,string=normal \ - -run-vga: system - ${EMU} ${EMU_ARGS} -cdrom image.iso \ - -fw_cfg name=opt/org.toaruos.bootmode,string=vga \ - -test: system - ${EMU} -M ${EMU_MACH} -m $(RAM) -smp $(SMP) ${EMU_KVM} -kernel misaka-kernel -initrd ramdisk.igz,util/init.krk -append "root=/dev/ram0 init=/dev/ram1" \ - -nographic -no-reboot -audiodev none,id=id -serial null -serial mon:stdio \ - -device qemu-xhci -device usb-tablet -trace "usb*" - -shell: system - ${EMU} -M ${EMU_MACH} -m $(RAM) -smp $(SMP) ${EMU_KVM} -cdrom image.iso \ - -nographic -no-reboot -audiodev none,id=id -serial null -serial mon:stdio \ - -fw_cfg name=opt/org.toaruos.gettyargs,string="-a local /dev/ttyS1" \ - -fw_cfg name=opt/org.toaruos.bootmode,string=headless \ - -fw_cfg name=opt/org.toaruos.term,string=${TERM} - -misaka-kernel: ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o - ${CC} -g -T kernel/arch/${ARCH}/link.ld ${KERNEL_CFLAGS} -o $@.64 ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o - ${OC} --strip-debug -I elf64-x86-64 -O elf32-i386 $@.64 $@ - kernel/sys/version.o: ${KERNEL_SOURCES} kernel/symbols.o: ${KERNEL_ASMOBJS} ${KERNEL_OBJS} util/gensym.krk @@ -187,7 +128,7 @@ clean: -rm -f boot/efi/*.o boot/bios/*.o libc/%.o: libc/%.c base/usr/include/syscall.h - $(CC) -O2 -std=gnu11 -Wall -Wextra -Wno-unused-parameter -fPIC -c -o $@ $< + $(CC) -O2 -std=gnu11 -ffreestanding -Wall -Wextra -Wno-unused-parameter -fPIC -c -o $@ $< .PHONY: libc libc: $(BASE)/lib/libc.a $(BASE)/lib/libc.so @@ -196,7 +137,7 @@ $(BASE)/lib/libc.a: ${LIBC_OBJS} $(CRTS) $(AR) cr $@ $(LIBC_OBJS) $(BASE)/lib/libc.so: ${LIBC_OBJS} | $(CRTS) - ${CC} -nodefaultlibs -shared -fPIC -o $@ $^ + ${CC} -nodefaultlibs -shared -fPIC -o $@ $^ -lgcc $(BASE)/lib/crt%.o: libc/arch/${ARCH}/crt%.S ${AS} -o $@ $< @@ -278,53 +219,3 @@ SOURCE_FILES += $(wildcard $(BASE)/usr/include/*.h $(BASE)/usr/include/*/*.h $(B tags: $(SOURCE_FILES) ctags -f tags $(SOURCE_FILES) -# Loader stuff, legacy CDs -fatbase/ramdisk.igz: ramdisk.igz - cp $< $@ -fatbase/kernel: misaka-kernel - cp $< $@ - strip $@ - -cdrom/fat.img: fatbase/ramdisk.igz fatbase/kernel fatbase/efi/boot/bootx64.efi util/mkdisk.sh | dirs - util/mkdisk.sh $@ fatbase - -cdrom/boot.sys: boot/bios/boot.o $(patsubst boot/%.c,boot/bios/%.o,$(wildcard boot/*.c)) boot/link.ld | dirs - ${LD} -melf_i386 -T boot/link.ld -o $@ boot/bios/boot.o $(patsubst boot/%.c,boot/bios/%.o,$(wildcard boot/*.c)) - -boot/bios/%.o: boot/%.c boot/*.h | dirs - ${CC} -m32 -c -Os -fno-pic -fno-pie -fno-strict-aliasing -finline-functions -ffreestanding -mgeneral-regs-only -o $@ $< - -boot/bios/boot.o: boot/boot.S | dirs - ${AS} --32 -o $@ $< - -EFI_CFLAGS=-fno-stack-protector -fpic -DEFI_PLATFORM -ffreestanding -fshort-wchar -I /usr/include/efi -mno-red-zone -EFI_SECTIONS=-j .text -j .sdata -j .data -j .dynamic -j .dynsym -j .rel -j .rela -j .reloc -EFI_LINK=/usr/lib/crt0-efi-x86_64.o -nostdlib -znocombreloc -T /usr/lib/elf_x86_64_efi.lds -shared -Bsymbolic -L /usr/lib -lefi -lgnuefi - -boot/efi/%.o: boot/%.c boot/*.h | dirs - $(CC) ${EFI_CFLAGS} -I /usr/include/efi/x86_64 -DEFI_FUNCTION_WRAPPER -c -o $@ $< - -boot/efi64.so: $(patsubst boot/%.c,boot/efi/%.o,$(wildcard boot/*.c)) boot/*.h - $(LD) $(patsubst boot/%.c,boot/efi/%.o,$(wildcard boot/*.c)) ${EFI_LINK} -o $@ - -fatbase/efi/boot/bootx64.efi: boot/efi64.so - mkdir -p fatbase/efi/boot - objcopy ${EFI_SECTIONS} --target=efi-app-x86_64 $< $@ - -BUILD_KRK=$(TOOLCHAIN)/local/bin/kuroko -$(TOOLCHAIN)/local/bin/kuroko: kuroko/src/*.c - mkdir -p $(TOOLCHAIN)/local/bin - cc -Ikuroko/src -DNO_RLINE -DSTATIC_ONLY -DKRK_DISABLE_THREADS -o "${TOOLCHAIN}/local/bin/kuroko" kuroko/src/*.c - -image.iso: cdrom/fat.img cdrom/boot.sys boot/mbr.S util/update-extents.krk | $(BUILD_KRK) - xorriso -as mkisofs -R -J -c bootcat \ - -b boot.sys -no-emul-boot -boot-load-size full \ - -eltorito-alt-boot -e fat.img -no-emul-boot -isohybrid-gpt-basdat \ - -o image.iso cdrom - ${AS} --32 $$(kuroko util/make_mbr.krk) -o boot/mbr.o boot/mbr.S - ${LD} -melf_i386 -T boot/link.ld -o boot/mbr.sys boot/mbr.o - tail -c +513 image.iso > image.dat - cat boot/mbr.sys image.dat > image.iso - rm image.dat - kuroko util/update-extents.krk - diff --git a/apps/cpu-name.krk b/apps/cpu-name.krk index 507eb54e..c6255f05 100644 --- a/apps/cpu-name.krk +++ b/apps/cpu-name.krk @@ -19,4 +19,7 @@ for line in lines: let key, value = line.split(': ') cpus[current][key] = value -print(cpus[0]['Model name']) +if cpus and 'Model name' in cpus[0]: + print(cpus[0]['Model name']) +else if cpus and 'PartNum' in cpus[0]: + print('(ARM64)', cpus[0]['PartNum']) diff --git a/apps/demo.c b/apps/demo.c index fb54881c..6f0b67fb 100644 --- a/apps/demo.c +++ b/apps/demo.c @@ -7,8 +7,72 @@ * Copyright (C) 2021 K. Lange */ #include +#include +#include +#include +#include +#include + +#include +#include + +static sprite_t wallpaper; +static struct TT_Font * _tt_font_thin; +static struct tm * timeinfo; +static gfx_context_t * ctx; + +static void redraw(void) { + draw_sprite(ctx, &wallpaper, 0, 0); + struct utsname u; + uname(&u); + + char string[1024]; + snprintf(string, 1024, + "ToaruOS %s %s %s", + u.release, u.version, u.machine); + tt_draw_string_shadow(ctx, _tt_font_thin, string, 15, 30, 30, rgb(255,255,255), rgb(0,0,0), 4); + + strftime(string,1024,"%a %d %b %Y %T %Z",timeinfo); + + tt_draw_string_shadow(ctx, _tt_font_thin, string, 15, 30, 60, rgb(255,255,255), rgb(0,0,0), 4); + + flip(ctx); +} int main(int argc, char * argv[]) { - printf("Hello, world! I am %s! I was called with '%s'\n", argv[0], argv[1]); - return 0; + fprintf(stderr, "open() = %ld\n", syscall_open("/dev/null", 0, 0)); + fprintf(stderr, "open() = %ld\n", syscall_open("/dev/null", 1, 0)); + fprintf(stderr, "open() = %ld\n", syscall_open("/dev/null", 1, 0)); + + ctx = init_graphics_fullscreen_double_buffer(); + draw_fill(ctx, rgb(120,120,120)); + flip(ctx); + + _tt_font_thin = tt_font_from_file("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"); + load_sprite(&wallpaper, "/usr/share/wallpaper.jpg"); + + draw_fill(ctx, rgb(0,0,0)); + flip(ctx); + + struct timeval now; + gettimeofday(&now, NULL); + + int forked = 0; + while (1) { + time_t last = now.tv_sec; + timeinfo = localtime(&last); + redraw(); + + while (1) { + gettimeofday(&now, NULL); + if (now.tv_sec != last) break; + } + + if (!forked) { + forked = 1; + system("uname -a"); + } + } + + return 0; } diff --git a/apps/init.c b/apps/init.c index a76afd82..8551d9e4 100644 --- a/apps/init.c +++ b/apps/init.c @@ -90,6 +90,7 @@ int start_options(char * args[]) { } int main(int argc, char * argv[]) { + fprintf(stderr, "Hello world, this is /bin/init.\n"); /* Initialize stdin/out/err */ set_console(); diff --git a/apps/killall.c b/apps/killall.c index 211b26c7..968940f5 100644 --- a/apps/killall.c +++ b/apps/killall.c @@ -148,7 +148,7 @@ int main (int argc, char * argv[]) { int signum = SIGTERM; - char c; + int c; while ((c = getopt(argc, argv, "s:?")) != -1) { switch (c) { case 's': diff --git a/apps/ps.c b/apps/ps.c index d9e89f2e..ee2f3b83 100644 --- a/apps/ps.c +++ b/apps/ps.c @@ -253,7 +253,7 @@ void show_usage(int argc, char * argv[]) { int main (int argc, char * argv[]) { /* Parse arguments */ - char c; + int c; while ((c = getopt(argc, argv, "AT?")) != -1) { switch (c) { case 'A': diff --git a/apps/ttysize.c b/apps/ttysize.c index 4183f3b7..0ac6279c 100644 --- a/apps/ttysize.c +++ b/apps/ttysize.c @@ -57,7 +57,7 @@ static void divine_size(int * width, int * height) { char buf[1024] = {0}; size_t i = 0; while (1) { - char c = getc_timeout(stdin, 200); + int c = getc_timeout(stdin, 200); if (c == 'R') break; if (c == -1) goto _done; if (c == '\033') continue; diff --git a/base/usr/include/kernel/arch/aarch64/pml.h b/base/usr/include/kernel/arch/aarch64/pml.h new file mode 100644 index 00000000..5004aae1 --- /dev/null +++ b/base/usr/include/kernel/arch/aarch64/pml.h @@ -0,0 +1,39 @@ +#pragma once +#include + +union PML { + struct { + uint64_t present : 1; + uint64_t table_page : 1; + uint64_t attrindx:3; + uint64_t ns:1; + uint64_t ap:2; + uint64_t sh:2; + uint64_t af:1; + uint64_t ng:1; + uint64_t page:36; + uint64_t reserved:4; + uint64_t contiguous:1; + uint64_t pxn:1; + uint64_t uxn:1; + uint64_t avail:4; + uint64_t ignored:5; + } bits; + + struct { + uint64_t valid : 1; + uint64_t table : 1; + uint64_t next:46; + uint64_t reserved:4; + uint64_t ignored:7; + uint64_t pxntable:1; + uint64_t xntable:1; + uint64_t aptable:2; + uint64_t nstable:1; + } table_bits; + + uint64_t raw; +}; + +#define mmu_page_is_user_readable(p) (0) +#define mmu_page_is_user_writable(p) (0) diff --git a/base/usr/include/kernel/arch/aarch64/regs.h b/base/usr/include/kernel/arch/aarch64/regs.h new file mode 100644 index 00000000..7558fe1b --- /dev/null +++ b/base/usr/include/kernel/arch/aarch64/regs.h @@ -0,0 +1,22 @@ +#pragma once +#include + +struct regs { + uint64_t x30, user_sp; + uint64_t x28, x29; + uint64_t x26, x27; + uint64_t x24, x25; + uint64_t x22, x23; + uint64_t x20, x21; + uint64_t x18, x19; + uint64_t x16, x17; + uint64_t x14, x15; + uint64_t x12, x13; + uint64_t x10, x11; + uint64_t x8, x9; + uint64_t x6, x7; + uint64_t x4, x5; + uint64_t x2, x3; + uint64_t x0, x1; +}; + diff --git a/base/usr/include/kernel/arch/x86_64/mmu.h b/base/usr/include/kernel/arch/x86_64/mmu.h deleted file mode 100644 index 298c3492..00000000 --- a/base/usr/include/kernel/arch/x86_64/mmu.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once -#include -#include -#define MMU_FLAG_KERNEL 0x01 -#define MMU_FLAG_WRITABLE 0x02 -#define MMU_FLAG_NOCACHE 0x04 -#define MMU_FLAG_WRITETHROUGH 0x08 -#define MMU_FLAG_SPEC 0x10 -#define MMU_FLAG_WC (MMU_FLAG_NOCACHE | MMU_FLAG_WRITETHROUGH | MMU_FLAG_SPEC) -#define MMU_FLAG_NOEXECUTE 0x20 - -#define MMU_GET_MAKE 0x01 - - -#define MMU_PTR_NULL 1 -#define MMU_PTR_WRITE 2 - -void mmu_frame_set(uintptr_t frame_addr); -void mmu_frame_clear(uintptr_t frame_addr); -int mmu_frame_test(uintptr_t frame_addr); -uintptr_t mmu_first_n_frames(int n); -uintptr_t mmu_first_frame(void); -void mmu_frame_allocate(union PML * page, unsigned int flags); -void mmu_frame_map_address(union PML * page, unsigned int flags, uintptr_t physAddr); -void mmu_frame_free(union PML * page); -uintptr_t mmu_map_to_physical(union PML * root, uintptr_t virtAddr); -union PML * mmu_get_page(uintptr_t virtAddr, int flags); -void mmu_set_directory(union PML * new_pml); -void mmu_free(union PML * from); -union PML * mmu_clone(union PML * from); -void mmu_init(size_t memsize, uintptr_t firstFreePage); -void mmu_invalidate(uintptr_t addr); -uintptr_t mmu_allocate_a_frame(void); -uintptr_t mmu_allocate_n_frames(int n); -union PML * mmu_get_kernel_directory(void); -void * mmu_map_from_physical(uintptr_t frameaddress); -void * mmu_map_mmio_region(uintptr_t physical_address, size_t size); -void * mmu_map_module(size_t size); -void mmu_unmap_module(uintptr_t base_address, size_t size); - -size_t mmu_count_user(union PML * from); -size_t mmu_count_shm(union PML * from); -size_t mmu_total_memory(void); -size_t mmu_used_memory(void); - -void * sbrk(size_t); - -union PML * mmu_get_page_other(union PML * root, uintptr_t virtAddr); -int mmu_validate_user_pointer(void * addr, size_t size, int flags); diff --git a/base/usr/include/kernel/elf.h b/base/usr/include/kernel/elf.h index 06ec6408..090b5ae4 100644 --- a/base/usr/include/kernel/elf.h +++ b/base/usr/include/kernel/elf.h @@ -263,6 +263,10 @@ typedef struct Elf64_Rela { #define R_X86_64_IRELATIVE 37 /**< @brief @p word64 indirect (B + A) */ +#define R_AARCH64_COPY 1024 +#define R_AARCH64_GLOB_DAT 1025 + + /** * Program header types */ diff --git a/base/usr/include/kernel/mmu.h b/base/usr/include/kernel/mmu.h index f41ef833..48df93a5 100644 --- a/base/usr/include/kernel/mmu.h +++ b/base/usr/include/kernel/mmu.h @@ -1,2 +1,55 @@ #pragma once -#include +#include + +#if defined(__x86_64__) +#include +#elif defined(__aarch64__) +#include +#endif + +#define MMU_FLAG_KERNEL 0x01 +#define MMU_FLAG_WRITABLE 0x02 +#define MMU_FLAG_NOCACHE 0x04 +#define MMU_FLAG_WRITETHROUGH 0x08 +#define MMU_FLAG_SPEC 0x10 +#define MMU_FLAG_WC (MMU_FLAG_NOCACHE | MMU_FLAG_WRITETHROUGH | MMU_FLAG_SPEC) +#define MMU_FLAG_NOEXECUTE 0x20 + +#define MMU_GET_MAKE 0x01 + + +#define MMU_PTR_NULL 1 +#define MMU_PTR_WRITE 2 + +void mmu_frame_set(uintptr_t frame_addr); +void mmu_frame_clear(uintptr_t frame_addr); +int mmu_frame_test(uintptr_t frame_addr); +uintptr_t mmu_first_n_frames(int n); +uintptr_t mmu_first_frame(void); +void mmu_frame_allocate(union PML * page, unsigned int flags); +void mmu_frame_map_address(union PML * page, unsigned int flags, uintptr_t physAddr); +void mmu_frame_free(union PML * page); +uintptr_t mmu_map_to_physical(union PML * root, uintptr_t virtAddr); +union PML * mmu_get_page(uintptr_t virtAddr, int flags); +void mmu_set_directory(union PML * new_pml); +void mmu_free(union PML * from); +union PML * mmu_clone(union PML * from); +void mmu_init(size_t memsize, uintptr_t firstFreePage); +void mmu_invalidate(uintptr_t addr); +uintptr_t mmu_allocate_a_frame(void); +uintptr_t mmu_allocate_n_frames(int n); +union PML * mmu_get_kernel_directory(void); +void * mmu_map_from_physical(uintptr_t frameaddress); +void * mmu_map_mmio_region(uintptr_t physical_address, size_t size); +void * mmu_map_module(size_t size); +void mmu_unmap_module(uintptr_t base_address, size_t size); + +size_t mmu_count_user(union PML * from); +size_t mmu_count_shm(union PML * from); +size_t mmu_total_memory(void); +size_t mmu_used_memory(void); + +void * sbrk(size_t); + +union PML * mmu_get_page_other(union PML * root, uintptr_t virtAddr); +int mmu_validate_user_pointer(void * addr, size_t size, int flags); diff --git a/base/usr/include/kernel/process.h b/base/usr/include/kernel/process.h index b1d06048..46c52726 100644 --- a/base/usr/include/kernel/process.h +++ b/base/usr/include/kernel/process.h @@ -6,11 +6,19 @@ #include #include #include -#include #include #include #include +#ifdef __x86_64__ +#include +#endif + +#ifdef __aarch64__ +#include +#endif + + #define PROC_REUSE_FDS 0x0001 #define KERNEL_STACK_SIZE 0x9000 #define USER_ROOT_UID 0 @@ -26,7 +34,11 @@ typedef struct { uintptr_t bp; /* 8 */ uintptr_t ip; /* 16 */ uintptr_t tls_base; /* 24 */ +#ifdef __x86_64__ uintptr_t saved[5]; /* XXX Arch dependent */ +#elif defined(__aarch64__) + uintptr_t saved[32]; +#endif /** * 32: rbx * 40: r12 @@ -190,6 +202,10 @@ struct ProcessorLocal { char cpu_model_name[48]; const char * cpu_manufacturer; #endif + +#ifdef __aarch64__ + uintptr_t sp_el1; +#endif }; extern struct ProcessorLocal processor_local_data[]; @@ -199,8 +215,13 @@ extern int processor_count; * @brief Core-local kernel data. * * x86-64: Marking this as __seg_gs makes it %gs-base-relative. + * aarch64: We shove this in x18 and ref off of that; -ffixed-x18 and don't forget to reload it from TPIDR_EL1 */ +#ifdef __x86_64__ static struct ProcessorLocal __seg_gs * const this_core = 0; +#else +register struct ProcessorLocal * this_core asm("x18"); +#endif extern unsigned long process_append_fd(process_t * proc, fs_node_t * node); extern long process_move_fd(process_t * proc, long src, long dest); diff --git a/base/usr/include/kernel/signal.h b/base/usr/include/kernel/signal.h index 57e91127..a06a2e81 100644 --- a/base/usr/include/kernel/signal.h +++ b/base/usr/include/kernel/signal.h @@ -2,7 +2,14 @@ #include #include + +#if defined(__x86_64__) #include +#elif defined(__aarch64__) +#include +#else +#error "no regs" +#endif typedef struct { int signum; diff --git a/base/usr/include/syscall.h b/base/usr/include/syscall.h index fe7c7d25..97664f11 100644 --- a/base/usr/include/syscall.h +++ b/base/usr/include/syscall.h @@ -13,6 +13,7 @@ _Begin_C_Header #define DECL_SYSCALL4(fn,p1,p2,p3,p4) long syscall_##fn(p1,p2,p3,p4) #define DECL_SYSCALL5(fn,p1,p2,p3,p4,p5) long syscall_##fn(p1,p2,p3,p4,p5) +#ifdef __x86_64__ #define DEFN_SYSCALL0(fn, num) \ long syscall_##fn() { \ long a = num; __asm__ __volatile__("int $0x7F" : "=a" (a) : "a" ((long)a)); \ @@ -58,6 +59,92 @@ _Begin_C_Header : "a" (__res), "b" ((long)(p1)), "c"((long)(p2)), "d"((long)(p3)), "S"((long)(p4)), "D"((long)(p5))); \ return __res; \ } +#else + +#define DEFN_SYSCALL0(fn, num) \ + long syscall_##fn() { \ + register long __res __asm__ ("x0") = num; \ + __asm__ __volatile__("svc 0" : "=r" (__res) : \ + "r" (__res) \ + ); \ + return __res; \ + } + +#define DEFN_SYSCALL1(fn, num, P1) \ + long syscall_##fn(P1 p1) { \ + register long __res __asm__ ("x0") = num; \ + register long x1 __asm__("x1") = (long)p1; \ + __asm__ __volatile__("svc 0" : "=r" (__res) : \ + "r" (__res), \ + "r" (x1) \ + ); \ + return __res; \ + } + +#define DEFN_SYSCALL2(fn, num, P1, P2) \ + long syscall_##fn(P1 p1, P2 p2) { \ + register long __res __asm__ ("x0") = num; \ + register long x1 __asm__("x1") = (long)p1; \ + register long x2 __asm__("x2") = (long)p2; \ + __asm__ __volatile__("svc 0" : "=r" (__res) : \ + "r" (__res), \ + "r" (x1), \ + "r" (x2) \ + ); \ + return __res; \ + } + +#define DEFN_SYSCALL3(fn, num, P1, P2, P3) \ + long syscall_##fn(P1 p1, P2 p2, P3 p3) { \ + register long __res __asm__ ("x0") = num; \ + register long x1 __asm__("x1") = (long)p1; \ + register long x2 __asm__("x2") = (long)p2; \ + register long x3 __asm__("x3") = (long)p3; \ + __asm__ __volatile__("svc 0" : "=r" (__res) : \ + "r" (__res), \ + "r" (x1), \ + "r" (x2), \ + "r" (x3) \ + ); \ + return __res; \ + } + +#define DEFN_SYSCALL4(fn, num, P1, P2, P3, P4) \ + long syscall_##fn(P1 p1, P2 p2, P3 p3, P4 p4) { \ + register long __res __asm__ ("x0") = num; \ + register long x1 __asm__("x1") = (long)p1; \ + register long x2 __asm__("x2") = (long)p2; \ + register long x3 __asm__("x3") = (long)p3; \ + register long x4 __asm__("x4") = (long)p4; \ + __asm__ __volatile__("svc 0" : "=r" (__res) : \ + "r" (__res), \ + "r" (x1), \ + "r" (x2), \ + "r" (x3), \ + "r" (x4) \ + ); \ + return __res; \ + } + +#define DEFN_SYSCALL5(fn, num, P1, P2, P3, P4, P5) \ + long syscall_##fn(P1 p1, P2 p2, P3 p3, P4 p4, P5 p5) { \ + register long __res __asm__ ("x0") = num; \ + register long x1 __asm__("x1") = (long)p1; \ + register long x2 __asm__("x2") = (long)p2; \ + register long x3 __asm__("x3") = (long)p3; \ + register long x4 __asm__("x4") = (long)p4; \ + register long x5 __asm__("x5") = (long)p5; \ + __asm__ __volatile__("svc 0" : "=r" (__res) : \ + "r" (__res), \ + "r" (x1), \ + "r" (x2), \ + "r" (x3), \ + "r" (x4), \ + "r" (x5) \ + ); \ + return __res; \ + } +#endif DECL_SYSCALL1(exit, int); diff --git a/build/aarch64.mk b/build/aarch64.mk new file mode 100644 index 00000000..3e721bdf --- /dev/null +++ b/build/aarch64.mk @@ -0,0 +1,47 @@ +ARCH=aarch64 + +ARCH_KERNEL_CFLAGS = -z max-page-size=0x1000 -nostdlib -mgeneral-regs-only -mno-outline-atomics -ffixed-x18 + +TARGET=aarch64-unknown-toaru + +all: system +system: misaka-kernel ramdisk.igz bootstub + +misaka-kernel: ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o kernel/arch/aarch64/link.ld + ${CC} -g -T kernel/arch/${ARCH}/link.ld ${KERNEL_CFLAGS} -o $@ ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o + +BOOTSTUB_OBJS = $(patsubst %.c,%.o,$(wildcard kernel/arch/aarch64/bootstub/*.c)) +BOOTSTUB_OBJS += $(patsubst %.S,%.o,$(wildcard kernel/arch/aarch64/bootstub/*.S)) +BOOTSTUB_OBJS += kernel/misc/kprintf.o kernel/misc/string.o + +bootstub: ${BOOTSTUB_OBJS} kernel/arch/aarch64/bootstub/link.ld + ${CC} -g -T kernel/arch/aarch64/bootstub/link.ld ${KERNEL_CFLAGS} -o $@ ${BOOTSTUB_OBJS} -lgcc + +QEMU = ~/Projects/third-party/qemu-git/build/qemu-system-aarch64 + +EMU_MACH = virt-2.12 +EMU_CPU = cortex-a72 +SMP ?= 1 +RAM ?= 4G + +EMU_ARGS = -M $(EMU_MACH) +EMU_ARGS += -m $(RAM) +EMU_ARGS += -smp $(SMP) +EMU_ARGS += -cpu $(EMU_CPU) +EMU_RAGS += -no-reboot +EMU_ARGS += -serial mon:stdio +EMU_ARGS += -device bochs-display +EMU_ARGS += -device nec-usb-xhci # Controller +EMU_ARGS += -device usb-tablet # Mouse with absolute positioning +EMU_ARGS += -device usb-kbd # Keyboard +EMU_ARGS += -d guest_errors + +EMU_RAMDISK = -fw_cfg name=opt/org.toaruos.initrd,file=ramdisk.igz +EMU_KERNEL = -fw_cfg name=opt/org.toaruos.kernel,file=misaka-kernel + +run: system + ${QEMU} ${EMU_ARGS} -kernel bootstub -append "root=/dev/ram0 migrate start=live-session vid=auto" ${EMU_RAMDISK} ${EMU_KERNEL} + +debug: system + ${QEMU} ${EMU_ARGS} -kernel bootstub -append "root=/dev/ram0 migrate start=live-session vid=auto" ${EMU_RAMDISK} ${EMU_KERNEL} -d int 2>&1 + diff --git a/build/x86_64.mk b/build/x86_64.mk index 0135d4ba..de29fee9 100644 --- a/build/x86_64.mk +++ b/build/x86_64.mk @@ -4,3 +4,117 @@ ARCH_KERNEL_CFLAGS = -mno-red-zone -fno-omit-frame-pointer -mfsgsbase ARCH_KERNEL_CFLAGS += -mgeneral-regs-only -z max-page-size=0x1000 -nostdlib TARGET=x86_64-pc-toaru + +# Configs you can override. +# SMP: Argument to -smp, use 1 to disable SMP. +# RAM: Argument to -m, QEMU takes suffixes like "M" or "G". +# EXTRA_ARGS: Added raw to the QEMU command line +# EMU_KVM: Unset this (EMU_KVM=) to use TCG, or replace it with something like EMU_KVM=-enable-haxm +# EMU_MACH: Argument to -M, 'pc' should be the older default in QEMU; we use q35 to test AHCI. +SMP ?= 4 +RAM ?= 3G +EXTRA_ARGS ?= +EMU_MACH ?= q35 + +EMU_KVM ?= -enable-kvm + +EMU_ARGS = -M q35 +EMU_ARGS += -m $(RAM) +EMU_ARGS += -smp $(SMP) +EMU_ARGS += ${EMU_KVM} +EMU_ARGS += -no-reboot +EMU_ARGS += -serial mon:stdio +EMU_ARGS += -soundhw pcspk,ac97 + +# UTC is the default setting. +#EMU_ARGS += -rtc base=utc + +# Customize network options here. QEMU's default is an e1000(e) under PIIX (Q35), with user networking +# so we don't need to do anything normally. +#EMU_ARGS += -net user +#EMU_ARGS += -netdev hubport,id=u1,hubid=0, -device e1000e,netdev=u1 -object filter-dump,id=f1,netdev=u1,file=qemu-e1000e.pcap +#EMU_ARGS += -netdev hubport,id=u2,hubid=0, -device e1000e,netdev=u2 + +# Add an XHCI tablet if you want to dev on USB +#EMU_ARGS += -device qemu-xhci -device usb-tablet + +all: system +system: image.iso + +run: system + ${EMU} ${EMU_ARGS} -cdrom image.iso + +fast: system + ${EMU} ${EMU_ARGS} -cdrom image.iso \ + -fw_cfg name=opt/org.toaruos.bootmode,string=normal \ + +run-vga: system + ${EMU} ${EMU_ARGS} -cdrom image.iso \ + -fw_cfg name=opt/org.toaruos.bootmode,string=vga \ + +test: system + ${EMU} -M ${EMU_MACH} -m $(RAM) -smp $(SMP) ${EMU_KVM} -kernel misaka-kernel -initrd ramdisk.igz,util/init.krk -append "root=/dev/ram0 init=/dev/ram1" \ + -nographic -no-reboot -audiodev none,id=id -serial null -serial mon:stdio \ + -device qemu-xhci -device usb-tablet -trace "usb*" + +shell: system + ${EMU} -M ${EMU_MACH} -m $(RAM) -smp $(SMP) ${EMU_KVM} -cdrom image.iso \ + -nographic -no-reboot -audiodev none,id=id -serial null -serial mon:stdio \ + -fw_cfg name=opt/org.toaruos.gettyargs,string="-a local /dev/ttyS1" \ + -fw_cfg name=opt/org.toaruos.bootmode,string=headless \ + -fw_cfg name=opt/org.toaruos.term,string=${TERM} + +misaka-kernel: ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o + ${CC} -g -T kernel/arch/${ARCH}/link.ld ${KERNEL_CFLAGS} -o $@.64 ${KERNEL_ASMOBJS} ${KERNEL_OBJS} kernel/symbols.o + ${OC} --strip-debug -I elf64-x86-64 -O elf32-i386 $@.64 $@ + +# Loader stuff, legacy CDs +fatbase/ramdisk.igz: ramdisk.igz + cp $< $@ +fatbase/kernel: misaka-kernel + cp $< $@ + strip $@ + +cdrom/fat.img: fatbase/ramdisk.igz fatbase/kernel fatbase/efi/boot/bootx64.efi util/mkdisk.sh | dirs + util/mkdisk.sh $@ fatbase + +cdrom/boot.sys: boot/bios/boot.o $(patsubst boot/%.c,boot/bios/%.o,$(wildcard boot/*.c)) boot/link.ld | dirs + ${LD} -melf_i386 -T boot/link.ld -o $@ boot/bios/boot.o $(patsubst boot/%.c,boot/bios/%.o,$(wildcard boot/*.c)) + +boot/bios/%.o: boot/%.c boot/*.h | dirs + ${CC} -m32 -c -Os -fno-pic -fno-pie -fno-strict-aliasing -finline-functions -ffreestanding -mgeneral-regs-only -o $@ $< + +boot/bios/boot.o: boot/boot.S | dirs + ${AS} --32 -o $@ $< + +EFI_CFLAGS=-fno-stack-protector -fpic -DEFI_PLATFORM -ffreestanding -fshort-wchar -I /usr/include/efi -mno-red-zone +EFI_SECTIONS=-j .text -j .sdata -j .data -j .dynamic -j .dynsym -j .rel -j .rela -j .reloc +EFI_LINK=/usr/lib/crt0-efi-x86_64.o -nostdlib -znocombreloc -T /usr/lib/elf_x86_64_efi.lds -shared -Bsymbolic -L /usr/lib -lefi -lgnuefi + +boot/efi/%.o: boot/%.c boot/*.h | dirs + $(CC) ${EFI_CFLAGS} -I /usr/include/efi/x86_64 -DEFI_FUNCTION_WRAPPER -c -o $@ $< + +boot/efi64.so: $(patsubst boot/%.c,boot/efi/%.o,$(wildcard boot/*.c)) boot/*.h + $(LD) $(patsubst boot/%.c,boot/efi/%.o,$(wildcard boot/*.c)) ${EFI_LINK} -o $@ + +fatbase/efi/boot/bootx64.efi: boot/efi64.so + mkdir -p fatbase/efi/boot + objcopy ${EFI_SECTIONS} --target=efi-app-x86_64 $< $@ + +BUILD_KRK=$(TOOLCHAIN)/local/bin/kuroko +$(TOOLCHAIN)/local/bin/kuroko: kuroko/src/*.c + mkdir -p $(TOOLCHAIN)/local/bin + cc -Ikuroko/src -DNO_RLINE -DSTATIC_ONLY -DKRK_DISABLE_THREADS -o "${TOOLCHAIN}/local/bin/kuroko" kuroko/src/*.c + +image.iso: cdrom/fat.img cdrom/boot.sys boot/mbr.S util/update-extents.krk | $(BUILD_KRK) + xorriso -as mkisofs -R -J -c bootcat \ + -b boot.sys -no-emul-boot -boot-load-size full \ + -eltorito-alt-boot -e fat.img -no-emul-boot -isohybrid-gpt-basdat \ + -o image.iso cdrom + ${AS} --32 $$(kuroko util/make_mbr.krk) -o boot/mbr.o boot/mbr.S + ${LD} -melf_i386 -T boot/link.ld -o boot/mbr.sys boot/mbr.o + tail -c +513 image.iso > image.dat + cat boot/mbr.sys image.dat > image.iso + rm image.dat + kuroko util/update-extents.krk + diff --git a/kernel/arch/aarch64/bootstrap.S b/kernel/arch/aarch64/bootstrap.S new file mode 100644 index 00000000..d745bf70 --- /dev/null +++ b/kernel/arch/aarch64/bootstrap.S @@ -0,0 +1,212 @@ +.extern __bootstrap_stack_top +.globl start +start: + ldr x30, =__bootstrap_stack_top + mov sp, x30 + bl kmain +hang: + b hang + +.globl arch_save_context +arch_save_context: + /* x0 has our struct pointer */ + mrs x1, TPIDR_EL0 + mov x2, sp + stp x2, x29, [x0] + stp x30, x1, [x0, (1 * 16)] + stp x19, x20, [x0, (2 * 16)] + stp x21, x22, [x0, (3 * 16)] + stp x23, x24, [x0, (4 * 16)] + stp x25, x26, [x0, (5 * 16)] + stp x27, x28, [x0, (6 * 16)] + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + stp x1, x2, [x0, (7 * 16)] + mov x0, 0 + ret + +.globl arch_restore_context +arch_restore_context: + ldr x1, [x18, 16] /* get previous */ + ldr x2, [x18, 0] /* get current */ + cmp x2, x1 /* compare current to prev, into x2 */ + beq _restore_context_same + add x1, x1, 20 +_restore_context_loop: + ldxr w2, [x1] + and w2, w2, 0xFFFFfff7 + stlxr w4, w2, [x1] + cbnz w4, _restore_context_loop +_restore_context_same: + /* x0 has our struct pointer */ + ldp x2, x29, [x0] + ldp x30, x1, [x0, (1 * 16)] + ldp x19, x20, [x0, (2 * 16)] + ldp x21, x22, [x0, (3 * 16)] + ldp x23, x24, [x0, (4 * 16)] + ldp x25, x26, [x0, (5 * 16)] + ldp x27, x28, [x0, (6 * 16)] + msr TPIDR_EL0, x1 + mov sp, x2 + ldp x1, x2, [x0, (7 * 16)] + msr ELR_EL1, x1 + msr SPSR_EL1, x2 + mov x0, 1 + ret + + +irq_common: + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x6, x7, [sp, #-16]! + stp x8, x9, [sp, #-16]! + stp x10, x11, [sp, #-16]! + stp x12, x13, [sp, #-16]! + stp x14, x15, [sp, #-16]! + stp x16, x17, [sp, #-16]! + stp x18, x19, [sp, #-16]! + stp x20, x21, [sp, #-16]! + stp x22, x23, [sp, #-16]! + stp x24, x25, [sp, #-16]! + stp x26, x27, [sp, #-16]! + stp x28, x29, [sp, #-16]! + mrs x0, SP_EL0 + stp x30, x0, [sp, #-16]! + mov x0, sp + mrs x18, TPIDR_EL1 + .extern aarch64_sync_enter + bl aarch64_sync_enter + ldp x30, x0, [sp], #16 + msr SP_EL0, x0 + ldp x28, x29, [sp], #16 + ldp x26, x27, [sp], #16 + ldp x24, x25, [sp], #16 + ldp x22, x23, [sp], #16 + ldp x20, x21, [sp], #16 + ldp x18, x19, [sp], #16 + ldp x16, x17, [sp], #16 + ldp x14, x15, [sp], #16 + ldp x12, x13, [sp], #16 + ldp x10, x11, [sp], #16 + ldp x8, x9, [sp], #16 + ldp x6, x7, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + eret + +.globl arch_resume_user +arch_resume_user: + ldp x30, x0, [sp], #16 + msr SP_EL0, x0 + ldp x28, x29, [sp], #16 + ldp x26, x27, [sp], #16 + ldp x24, x25, [sp], #16 + ldp x22, x23, [sp], #16 + ldp x20, x21, [sp], #16 + ldp x18, x19, [sp], #16 + ldp x16, x17, [sp], #16 + ldp x14, x15, [sp], #16 + ldp x12, x13, [sp], #16 + ldp x10, x11, [sp], #16 + ldp x8, x9, [sp], #16 + ldp x6, x7, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + eret + +fault_common: + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x6, x7, [sp, #-16]! + stp x8, x9, [sp, #-16]! + stp x10, x11, [sp, #-16]! + stp x12, x13, [sp, #-16]! + stp x14, x15, [sp, #-16]! + stp x16, x17, [sp, #-16]! + stp x18, x19, [sp, #-16]! + stp x20, x21, [sp, #-16]! + stp x22, x23, [sp, #-16]! + stp x24, x25, [sp, #-16]! + stp x26, x27, [sp, #-16]! + stp x28, x29, [sp, #-16]! + mrs x0, SP_EL0 + stp x30, x0, [sp, #-16]! + mov x0, sp + mrs x18, TPIDR_EL1 + .extern aarch64_fault_enter + bl aarch64_fault_enter + ldp x30, x0, [sp], #16 + msr SP_EL0, x0 + ldp x28, x29, [sp], #16 + ldp x26, x27, [sp], #16 + ldp x24, x25, [sp], #16 + ldp x22, x23, [sp], #16 + ldp x20, x21, [sp], #16 + ldp x18, x19, [sp], #16 + ldp x16, x17, [sp], #16 + ldp x14, x15, [sp], #16 + ldp x12, x13, [sp], #16 + ldp x10, x11, [sp], #16 + ldp x8, x9, [sp], #16 + ldp x6, x7, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + eret + + +.globl _exception_vector +.balign 0x800 +_exception_vector: + +_exc_sp0_sync: + b . +.balign 0x80 +_exc_sp0_irq: + b . +.balign 0x80 +_exc_sp0_fiq: + b . +.balign 0x80 +_exc_sp0_serror: + b . +.balign 0x80 +_exc_spx_sync: + b fault_common +.balign 0x80 +_exc_spx_irq: + b . +.balign 0x80 +_exc_spx_fiq: + b . +.balign 0x80 +_exc_spx_serror: + b . +.balign 0x80 +_exc_lower_sync: + b irq_common +.balign 0x80 +_exc_lower_irq: + b . +.balign 0x80 +_exc_lower_fiq: + b . +.balign 0x80 +_exc_lower_serror: + b . +.balign 0x80 +_exc_lower_32_sync: + b . +.balign 0x80 +_exc_lower_32_irq: + b . +.balign 0x80 +_exc_lower_32_fiq: + b . +.balign 0x80 +_exc_lower_32_serror: + b . diff --git a/kernel/arch/aarch64/bootstub/bootstrap.S b/kernel/arch/aarch64/bootstub/bootstrap.S new file mode 100644 index 00000000..e2a0cd8a --- /dev/null +++ b/kernel/arch/aarch64/bootstub/bootstrap.S @@ -0,0 +1,8 @@ +.extern __bootstrap_stack_top +.globl start +start: + ldr x30, =__bootstrap_stack_top + mov sp, x30 + bl kmain +hang: + b hang diff --git a/kernel/arch/aarch64/bootstub/link.ld b/kernel/arch/aarch64/bootstub/link.ld new file mode 100644 index 00000000..e1987514 --- /dev/null +++ b/kernel/arch/aarch64/bootstub/link.ld @@ -0,0 +1,53 @@ +OUTPUT_FORMAT(elf64-littleaarch64) +ENTRY(start) + +SECTIONS +{ + . = 0x40100000; + phys = .; + + .text BLOCK(4K) : ALIGN(4K) + { + *(.bootstrap) + code = .; + *(.text) + } + + .rodata BLOCK(4K) : ALIGN(4K) + { + *(.rodata) + } + + .data BLOCK(4K) : ALIGN(4K) + { + data = .; + *(.data) + *(.symbols) + PROVIDE(kernel_symbols_start = .); + PROVIDE(kernel_symbols_end = .); + PROVIDE(bss_start = .); + } + + .bss BLOCK(4K) : ALIGN(4K) + { + bss = .; + *(COMMON) + *(.bss) + *(.stack) + } + + /* Some built-in stack space... */ + . = ALIGN(0x1000); + . = . + 0x1000; + __bootstrap_stack_top = .; + + end = .; + + /DISCARD/ : + { + *(.comment) + *(.eh_frame) + *(.note.gnu.build-id) + } + +} diff --git a/kernel/arch/aarch64/bootstub/main.c b/kernel/arch/aarch64/bootstub/main.c new file mode 100644 index 00000000..32842c14 --- /dev/null +++ b/kernel/arch/aarch64/bootstub/main.c @@ -0,0 +1,458 @@ +/** + * @file kernel/arch/aarch64/bootstub/main.c + * @brief Shim loader for QEMU virt machine. + * + * Loads at 0x4010_0000 where RAM is, sets up the MMU to have RAM + * at our kernel virtual load address (0xffff_ffff_8000_0000), as + * well as a direct mapping at -512GB for access to IO devices, + * reads the kernel out of fw-cfg, loads it to the kernel virtual + * load address, and then jumps to it. + * + * @copyright + * This file is part of ToaruOS and is released under the terms + * of the NCSA / University of Illinois License - see LICENSE.md + * Copyright (C) 2022 K. Lange + */ +#include +#include +#include +#include + +static uint32_t swizzle(uint32_t from) { + uint8_t a = from >> 24; + uint8_t b = from >> 16; + uint8_t c = from >> 8; + uint8_t d = from; + return (d << 24) | (c << 16) | (b << 8) | (a); +} + +void * malloc(size_t x) { + printf("panic\n"); + while (1); +} + +static uint64_t swizzle64(uint64_t from) { + uint8_t a = from >> 56; + uint8_t b = from >> 48; + uint8_t c = from >> 40; + uint8_t d = from >> 32; + uint8_t e = from >> 24; + uint8_t f = from >> 16; + uint8_t g = from >> 8; + uint8_t h = from; + return ((uint64_t)h << 56) | ((uint64_t)g << 48) | ((uint64_t)f << 40) | ((uint64_t)e << 32) | (d << 24) | (c << 16) | (b << 8) | (a); +} + +static uint16_t swizzle16(uint16_t from) { + uint8_t a = from >> 8; + uint8_t b = from; + return (b << 8) | (a); +} + +struct fdt_header { + uint32_t magic; + uint32_t totalsize; + uint32_t off_dt_struct; + uint32_t off_dt_strings; + uint32_t off_mem_rsvmap; + uint32_t version; + uint32_t last_comp_version; + uint32_t boot_cpuid_phys; + uint32_t size_dt_strings; + uint32_t size_dt_struct; +}; + +static uint32_t * parse_node(uint32_t * node, char * strings, int x) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 9) return NULL; + if (swizzle(*node) != 1) { + printf("Not a node? Got %x\n", swizzle(*node)); + return NULL; + } + + /* Skip the BEGIN_NODE */ + node++; + + for (int i = 0; i < x; ++i) printf(" "); + + while (1) { + char * x = (char*)node; + if (x[0]) { printf("%c",x[0]); } else { node++; break; } + if (x[1]) { printf("%c",x[1]); } else { node++; break; } + if (x[2]) { printf("%c",x[2]); } else { node++; break; } + if (x[3]) { printf("%c",x[3]); } else { node++; break; } + node++; + } + printf("\n"); + + while (1) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 2) return node+1; + if (swizzle(*node) == 3) { + for (int i = 0; i < x; ++i) printf(" "); + uint32_t len = swizzle(node[1]); + uint32_t nameoff = swizzle(node[2]); + printf(" property %s len=%u\n", strings + nameoff, len); + node += 3; + node += (len + 3) / 4; + } else if (swizzle(*node) == 1) { + node = parse_node(node, strings, x + 1); + } + } + +} + +static void dump_dtb(uintptr_t addr) { + + struct fdt_header * fdt = (struct fdt_header*)addr; + +#define P(o) printf(#o " = %#x\n", swizzle(fdt-> o)) + P(magic); + P(totalsize); + P(off_dt_struct); + P(off_dt_strings); + P(off_mem_rsvmap); + P(version); + P(last_comp_version); + P(boot_cpuid_phys); + P(size_dt_strings); + P(size_dt_struct); + + char * dtb_strings = (char *)(addr + swizzle(fdt->off_dt_strings)); + uint32_t * dtb_struct = (uint32_t *)(addr + swizzle(fdt->off_dt_struct)); + + parse_node(dtb_struct, dtb_strings, 0); +} + +static uint32_t * find_subnode(uint32_t * node, char * strings, const char * name, uint32_t ** node_out, int (*cmp)(const char* a, const char *b)) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 9) return NULL; + if (swizzle(*node) != 1) return NULL; + node++; + + if (cmp((char*)node,name)) { + *node_out = node; + return NULL; + } + + while ((*node & 0xFF000000) && (*node & 0xFF0000) && (*node & 0xFF00) && (*node & 0xFF)) node++; + node++; + + while (1) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 2) return node+1; + if (swizzle(*node) == 3) { + uint32_t len = swizzle(node[1]); + node += 3; + node += (len + 3) / 4; + } else if (swizzle(*node) == 1) { + node = find_subnode(node, strings, name, node_out, cmp); + if (!node) return NULL; + } + } +} + +static uint32_t * find_node_int(const char * name, int (*cmp)(const char*,const char*)) { + uintptr_t addr = 0x40000000; + struct fdt_header * fdt = (struct fdt_header*)addr; + char * dtb_strings = (char *)(addr + swizzle(fdt->off_dt_strings)); + uint32_t * dtb_struct = (uint32_t *)(addr + swizzle(fdt->off_dt_struct)); + + uint32_t * out = NULL; + find_subnode(dtb_struct, dtb_strings, name, &out, cmp); + return out; +} + +static int base_cmp(const char *a, const char *b) { + return !strcmp(a,b); +} +static uint32_t * find_node(const char * name) { + return find_node_int(name,base_cmp); +} + +static int prefix_cmp(const char *a, const char *b) { + return !memcmp(a,b,strlen(b)); +} + +static uint32_t * find_node_prefix(const char * name) { + return find_node_int(name,prefix_cmp); +} + +static uint32_t * node_find_property_int(uint32_t * node, char * strings, const char * property, uint32_t ** out) { + while ((*node & 0xFF000000) && (*node & 0xFF0000) && (*node & 0xFF00) && (*node & 0xFF)) node++; + node++; + + while (1) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 2) return node+1; + if (swizzle(*node) == 3) { + uint32_t len = swizzle(node[1]); + uint32_t nameoff = swizzle(node[2]); + if (!strcmp(strings + nameoff, property)) { + *out = &node[1]; + return NULL; + } + node += 3; + node += (len + 3) / 4; + } else if (swizzle(*node) == 1) { + node = node_find_property_int(node+1, strings, property, out); + if (!node) return NULL; + } + } +} + +static uint32_t * node_find_property(uint32_t * node, const char * property) { + uintptr_t addr = 0x40000000; + struct fdt_header * fdt = (struct fdt_header*)addr; + char * dtb_strings = (char *)(addr + swizzle(fdt->off_dt_strings)); + uint32_t * out = NULL; + node_find_property_int(node, dtb_strings, property, &out); + return out; +} + +static size_t _early_log_write(size_t size, uint8_t * buffer) { + for (unsigned int i = 0; i < size; ++i) { + *(volatile unsigned int *)(0x09000000) = buffer[i]; + } + return size; +} + +static size_t _later_log_write(size_t size, uint8_t * buffer) { + for (unsigned int i = 0; i < size; ++i) { + *(volatile unsigned int *)(0xffffff8009000000) = buffer[i]; + } + return size; +} + +static struct BaseTables { + uintptr_t l0_base[512]; + uintptr_t l1_high_gbs[512]; + uintptr_t l1_low_gbs[512]; +} _baseTables __attribute__((aligned(4096))); + +#define PTE_VALID (1UL << 0) +#define PTE_TABLE (1UL << 1) + +/* Table attributes */ +#define PTE_NSTABLE (1UL << 63) +#define PTE_APTABLE (3UL << 61) /* two bits */ +#define PTE_APTABLE_A (1UL << 62) +#define PTE_APTABLE_B (1UL << 61) +#define PTE_UXNTABLE (1UL << 60) +#define PTE_PXNTABLE (1UL << 59) + +/* Block attributes */ +#define PTE_UXN (1UL << 54) +#define PTE_PXN (1UL << 53) +#define PTE_CONTIGUOUS (1UL << 52) +#define PTE_NG (1UL << 11) +#define PTE_AF (1UL << 10) +#define PTE_SH (3UL << 8) /* two bits */ +#define PTE_SH_A (1UL << 9) +#define PTE_SH_B (1UL << 8) +#define PTE_AP (3UL << 6) /* two bits */ +#define PTE_AP_A (1UL << 7) +#define PTE_AP_B (1UL << 6) +#define PTE_NS (1UL << 5) +#define PTE_ATTRINDX (7UL << 2) /* three bits */ +#define PTE_ATTR_A (1UL << 4) +#define PTE_ATTR_B (1UL << 3) +#define PTE_ATTR_C (1UL << 2) + +static void bootstub_mmu_init(void) { + /* Map memory */ + _baseTables.l0_base[0] = (uintptr_t)&_baseTables.l1_low_gbs | PTE_VALID | PTE_TABLE | PTE_AF; + + /* equivalent to high_base_pml */ + _baseTables.l0_base[511] = (uintptr_t)&_baseTables.l1_high_gbs | PTE_VALID | PTE_TABLE | PTE_AF; + + /* Mapping for us */ + _baseTables.l1_low_gbs[1] = 0x40000000 | PTE_VALID | PTE_AF | PTE_SH_A | (1 << 2); + + /* -512GB is a map of 64GB of memory */ + for (size_t i = 0; i < 64; ++i) { + _baseTables.l1_high_gbs[i] = (i << 30) | PTE_VALID | PTE_AF | PTE_SH_A | (1 << 2); + } + + /* -2GiB, map kernel here */ + _baseTables.l1_high_gbs[510] = 0x80000000 | PTE_VALID | PTE_AF | PTE_SH_A | (1 << 2); + + uint64_t sctlr = 0 + | (1UL << 0) /* mmu enabled */ + | (1UL << 2) /* cachability */ + //| (1UL << 6) + | (1UL << 12) /* instruction cachability */ + | (1UL << 23) /* SPAN */ + | (1UL << 28) /* nTLSMD */ + | (1UL << 29) /* LSMAOE */ + | (1UL << 20) /* TSCXT */ + | (1UL << 7) /* ITD */ + ; + + /* Translate control register */ + uint64_t tcr = 0 + | (3UL << 32) /* IPS 4TB? */ + | (2UL << 30) /* TG1 4KB granules in TTBR1 */ + | (16UL << 16) /* T1SZ 48-bit */ + | (3UL << 28) /* SH1 */ + | (1UL << 26) /* ORGN1 */ + | (1UL << 24) /* IRGN1 */ + | (0UL << 14) /* TG0 4KB granules in TTBR0 */ + | (16UL << 0) /* T0SZ 48-bit */ + | (3UL << 12) /* SH0 */ + | (1UL << 10) /* ORGN0 */ + | (1UL << 8) /* IRGN0 */ + ; + + /* MAIR setup? */ + uint64_t mair = (0x000000000044ff00); + asm volatile ("msr MAIR_EL1,%0" :: "r"(mair)); + + /* Frob bits */ + printf("bootstub: setting base values\n"); + asm volatile ("msr TCR_EL1,%0" : : "r"(tcr)); + asm volatile ("msr TTBR0_EL1,%0" : : "r"(&_baseTables.l0_base)); + asm volatile ("msr TTBR1_EL1,%0" : : "r"(&_baseTables.l0_base)); + printf("bootstub: frobbing bits\n"); + asm volatile ("dsb ishst\ntlbi vmalle1is\ndsb ish\nisb" ::: "memory"); + asm volatile ("msr SCTLR_EL1,%0" : : "r"(sctlr)); + asm volatile ("isb" ::: "memory"); + + /* Point log output at new mmio address */ + printf_output = &_later_log_write; + + printf("bootstub: MMU initialized\n"); +} + +static void bootstub_read_kernel(uintptr_t kernel_load_addr) { + /* See if we can find a qemu fw_cfg interface, we can use that for a ramdisk */ + uint32_t * fw_cfg = find_node_prefix("fw-cfg"); + if (fw_cfg) { + printf("bootstub: found fw-cfg interface\n"); + /* best guess until we bother parsing these */ + uint32_t * regs = node_find_property(fw_cfg, "reg"); + if (regs) { + printf("bootstub: length of regs = %u\n", swizzle(regs[0])); + printf("bootstub: addr of fw-cfg = %#x\n", swizzle(regs[3])); + + volatile uint8_t * fw_cfg_addr = (volatile uint8_t*)(uintptr_t)(swizzle(regs[3]) + 0xffffff8000000000); + volatile uint64_t * fw_cfg_data = (volatile uint64_t *)fw_cfg_addr; + volatile uint32_t * fw_cfg_32 = (volatile uint32_t *)fw_cfg_addr; + volatile uint16_t * fw_cfg_sel = (volatile uint16_t *)(fw_cfg_addr + 8); + + *fw_cfg_sel = 0; + + uint64_t response = fw_cfg_data[0]; + + printf("bootstub: response: %c%c%c%c\n", + (char)(response >> 0), + (char)(response >> 8), + (char)(response >> 16), + (char)(response >> 24)); + + /* Needs to be big-endian */ + *fw_cfg_sel = swizzle16(0x19); + + /* count response is 32-bit BE */ + uint32_t count = swizzle(fw_cfg_32[0]); + printf("bootstub: %u entries\n", count); + + struct fw_cfg_file { + uint32_t size; + uint16_t select; + uint16_t reserved; + char name[56]; + }; + + struct fw_cfg_file file; + uint8_t * tmp = (uint8_t *)&file; + + /* Read count entries */ + for (unsigned int i = 0; i < count; ++i) { + for (unsigned int j = 0; j < sizeof(struct fw_cfg_file); ++j) { + tmp[j] = fw_cfg_addr[0]; + } + + /* endian swap to get file size and selector ID */ + file.size = swizzle(file.size); + file.select = swizzle16(file.select); + + printf("bootstub: 0x%04x %s (%d bytes)\n", + file.select, file.name, file.size); + + if (!strcmp(file.name,"opt/org.toaruos.kernel")) { + printf("bootstub: Found kernel, loading\n"); + uint8_t * x = (uint8_t*)kernel_load_addr; + + struct fwcfg_dma { + volatile uint32_t control; + volatile uint32_t length; + volatile uint64_t address; + } dma; + + dma.control = swizzle((file.select << 16) | (1 << 3) | (1 << 1)); + dma.length = swizzle(file.size); + dma.address = swizzle64((uintptr_t)x); + + fw_cfg_data[2] = swizzle64((uint64_t)&dma); + + if (dma.control) { + printf("bootstub: error on dma read?\n"); + return; + } + + return; + } + } + } + } + +} + + +static void bootstub_load_kernel(Elf64_Header * header) { + /* Find load headers */ + for (int i = 0; i < header->e_phnum; ++i) { + Elf64_Phdr * phdr = (void*)((uintptr_t)header + (header->e_phoff + header->e_phentsize * i)); + if (phdr->p_type == PT_LOAD) { + printf("bootstub: Load %zu bytes @ %zx from off %zx\n", phdr->p_memsz, phdr->p_vaddr, phdr->p_offset); + memset((void*)phdr->p_vaddr, 0, phdr->p_memsz); + memcpy((void*)phdr->p_vaddr, (void*)((uintptr_t)header + phdr->p_offset), phdr->p_filesz); + } else { + printf("bootstub: Skip phdr %d\n", i); + } + } +} + +static void bootstub_start_kernel(Elf64_Header * header) { + printf("bootstub: Jump to kernel entry point at %zx\n", + header->e_entry); + + void (*entry)(void) = (void(*)(void))header->e_entry; + entry(); +} + +int kmain(void) { + extern char end[]; + uintptr_t kernel_load_addr = (uintptr_t)&end; + + /* Initialize log */ + printf_output = &_early_log_write; + printf("bootstub: Starting up\n"); + + /* Set up MMU */ + bootstub_mmu_init(); + + /* Read the kernel from fw-cfg */ + bootstub_read_kernel(kernel_load_addr); + + /* Examine kernel */ + Elf64_Header *header = (void*)kernel_load_addr; + bootstub_load_kernel(header); + + /* Jump to kernel */ + bootstub_start_kernel(header); + + while (1) {} + return 0; +} diff --git a/kernel/arch/aarch64/link.ld b/kernel/arch/aarch64/link.ld new file mode 100644 index 00000000..9706a93d --- /dev/null +++ b/kernel/arch/aarch64/link.ld @@ -0,0 +1,53 @@ +OUTPUT_FORMAT(elf64-littleaarch64) +ENTRY(start) + +SECTIONS +{ + . = 0xffffffff80000000; + phys = .; + + .text BLOCK(4K) : ALIGN(4K) + { + *(.bootstrap) + code = .; + *(.text) + } + + .rodata BLOCK(4K) : ALIGN(4K) + { + *(.rodata) + } + + .data BLOCK(4K) : ALIGN(4K) + { + data = .; + *(.data) + *(.symbols) + PROVIDE(kernel_symbols_start = .); + PROVIDE(kernel_symbols_end = .); + PROVIDE(bss_start = .); + } + + .bss BLOCK(4K) : ALIGN(4K) + { + bss = .; + *(COMMON) + *(.bss) + *(.stack) + } + + /* Some built-in stack space... */ + . = ALIGN(0x1000); + . = . + 0x1000; + __bootstrap_stack_top = .; + + end = .; + + /DISCARD/ : + { + *(.comment) + *(.eh_frame) + *(.note.gnu.build-id) + } + +} diff --git a/kernel/arch/aarch64/mmu.c b/kernel/arch/aarch64/mmu.c new file mode 100644 index 00000000..7cc2d437 --- /dev/null +++ b/kernel/arch/aarch64/mmu.c @@ -0,0 +1,741 @@ + +/** + * @file kernel/arch/aarch64/mmu.c + * @brief Stubs + * + * @copyright + * This file is part of ToaruOS and is released under the terms + * of the NCSA / University of Illinois License - see LICENSE.md + * Copyright (C) 2021-2022 K. Lange + */ +#include +#include +#include +#include +#include +#include +#include +#include + +static volatile uint32_t *frames; +static size_t nframes; +static size_t total_memory = 0; +static size_t unavailable_memory = 0; +static uint8_t * mem_refcounts = NULL; + +#define PAGE_SHIFT 12 +#define PAGE_SIZE 0x1000UL +#define PAGE_SIZE_MASK 0xFFFFffffFFFFf000UL +#define PAGE_LOW_MASK 0x0000000000000FFFUL + +#define LARGE_PAGE_SIZE 0x200000UL + +#define KERNEL_HEAP_START 0xFFFFff0000000000UL +#define MMIO_BASE_START 0xffffff1fc0000000UL +#define HIGH_MAP_REGION 0xffffff8000000000UL +#define MODULE_BASE_START 0xffffffff80000000UL + +#define USER_SHM_LOW 0x0000000200000000UL +#define USER_SHM_HIGH 0x0000000400000000UL +#define USER_DEVICE_MAP 0x0000000100000000UL + +#define PHYS_MASK 0x7fffffffffUL +#define CANONICAL_MASK 0xFFFFffffFFFFUL + +#define INDEX_FROM_BIT(b) ((b) >> 5) +#define OFFSET_FROM_BIT(b) ((b) & 0x1F) + +#define QEMU_VIRT_KERNEL_BASE 0x80000000UL + +#define _pagemap __attribute__((aligned(PAGE_SIZE))) = {0} +union PML init_page_region[512] _pagemap; +union PML high_base_pml[512] _pagemap; +union PML heap_base_pml[512] _pagemap; +union PML heap_base_pd[512] _pagemap; +union PML heap_base_pt[512*3] _pagemap; +union PML kbase_pmls[65][512] _pagemap; + +#define PTE_VALID (1UL << 0) +#define PTE_TABLE (1UL << 1) + +/* Table attributes */ +#define PTE_NSTABLE (1UL << 63) +#define PTE_APTABLE (3UL << 61) /* two bits */ +#define PTE_APTABLE_A (1UL << 62) +#define PTE_APTABLE_B (1UL << 61) +#define PTE_UXNTABLE (1UL << 60) +#define PTE_PXNTABLE (1UL << 59) + +/* Block attributes */ +#define PTE_UXN (1UL << 54) +#define PTE_PXN (1UL << 53) +#define PTE_CONTIGUOUS (1UL << 52) +#define PTE_NG (1UL << 11) +#define PTE_AF (1UL << 10) +#define PTE_SH (3UL << 8) /* two bits */ +#define PTE_SH_A (1UL << 9) +#define PTE_SH_B (1UL << 8) +#define PTE_AP (3UL << 6) /* two bits */ +#define PTE_AP_A (1UL << 7) +#define PTE_AP_B (1UL << 6) +#define PTE_NS (1UL << 5) +#define PTE_ATTRINDX (7UL << 2) /* three bits */ +#define PTE_ATTR_A (1UL << 4) +#define PTE_ATTR_B (1UL << 3) +#define PTE_ATTR_C (1UL << 2) + + +void mmu_frame_set(uintptr_t frame_addr) { + if (frame_addr < nframes * PAGE_SIZE) { + uint64_t frame = frame_addr >> 12; + uint64_t index = INDEX_FROM_BIT(frame); + uint32_t offset = OFFSET_FROM_BIT(frame); + frames[index] |= ((uint32_t)1 << offset); + asm ("" ::: "memory"); + } +} + +static uintptr_t lowest_available = 0; + +void mmu_frame_clear(uintptr_t frame_addr) { + if (frame_addr < nframes * PAGE_SIZE) { + uint64_t frame = frame_addr >> PAGE_SHIFT; + uint64_t index = INDEX_FROM_BIT(frame); + uint32_t offset = OFFSET_FROM_BIT(frame); + frames[index] &= ~((uint32_t)1 << offset); + asm ("" ::: "memory"); + if (frame < lowest_available) lowest_available = frame; + } +} + +int mmu_frame_test(uintptr_t frame_addr) { + if (!(frame_addr < nframes * PAGE_SIZE)) return 1; + uint64_t frame = frame_addr >> PAGE_SHIFT; + uint64_t index = INDEX_FROM_BIT(frame); + uint32_t offset = OFFSET_FROM_BIT(frame); + asm ("" ::: "memory"); + return !!(frames[index] & ((uint32_t)1 << offset)); +} + +static spin_lock_t frame_alloc_lock = { 0 }; +static spin_lock_t kheap_lock = { 0 }; +static spin_lock_t mmio_space_lock = { 0 }; +static spin_lock_t module_space_lock = { 0 }; + +uintptr_t mmu_first_n_frames(int n) { + for (uint64_t i = 0; i < nframes * PAGE_SIZE; i += PAGE_SIZE) { + int bad = 0; + for (int j = 0; j < n; ++j) { + if (mmu_frame_test(i + PAGE_SIZE * j)) { + bad = j + 1; + } + } + if (!bad) { + return i / PAGE_SIZE; + } + } + + arch_fatal_prepare(); + dprintf("Failed to allocate %d contiguous frames.\n", n); + arch_dump_traceback(); + arch_fatal(); + return (uintptr_t)-1; +} + +uintptr_t mmu_first_frame(void) { + uintptr_t i, j; + for (i = INDEX_FROM_BIT(lowest_available); i < INDEX_FROM_BIT(nframes); ++i) { + if (frames[i] != (uint32_t)-1) { + for (j = 0; j < (sizeof(uint32_t)*8); ++j) { + uint32_t testFrame = (uint32_t)1 << j; + if (!(frames[i] & testFrame)) { + uintptr_t out = (i << 5) + j; + lowest_available = out + 1; + return out; + } + } + } + } + + if (lowest_available != 0) { + lowest_available = 0; + return mmu_first_frame(); + } + + arch_fatal_prepare(); + dprintf("Out of memory.\n"); + arch_dump_traceback(); + arch_fatal(); + return (uintptr_t)-1; +} + +void mmu_frame_allocate(union PML * page, unsigned int flags) { + /* If page is not set... */ + if (page->bits.page == 0) { + spin_lock(frame_alloc_lock); + uintptr_t index = mmu_first_frame(); + mmu_frame_set(index << PAGE_SHIFT); + page->bits.page = index; + spin_unlock(frame_alloc_lock); + } + + page->bits.table_page = 1; + page->bits.present = 1; + + page->bits.ap = (!(flags & MMU_FLAG_WRITABLE) ? 2 : 0) | (!(flags & MMU_FLAG_KERNEL) ? 1 : 0); + page->bits.af = 1; + page->bits.sh = 2; + page->bits.attrindx = ((flags & MMU_FLAG_NOCACHE) | (flags & MMU_FLAG_WRITETHROUGH)) ? 0 : 1; + + if (!(flags & MMU_FLAG_KERNEL)) { + page->bits.attrindx = 1; + + if ((flags & MMU_FLAG_WC) == MMU_FLAG_WC) { + page->bits.attrindx = 2; + } + } + + asm volatile ("dsb ishst\ntlbi vmalle1is\ndsb ish\nisb" ::: "memory"); + + #if 0 + page->bits.writable = (flags & MMU_FLAG_WRITABLE) ? 1 : 0; + page->bits.user = (flags & MMU_FLAG_KERNEL) ? 0 : 1; + page->bits.nocache = (flags & MMU_FLAG_NOCACHE) ? 1 : 0; + page->bits.writethrough = (flags & MMU_FLAG_WRITETHROUGH) ? 1 : 0; + page->bits.size = (flags & MMU_FLAG_SPEC) ? 1 : 0; + page->bits.nx = (flags & MMU_FLAG_NOEXECUTE) ? 1 : 0; + #endif + +} + +void mmu_frame_map_address(union PML * page, unsigned int flags, uintptr_t physAddr) { + /* frame set physAddr, set page in entry, call frame_allocate to set attribute bits */ + mmu_frame_set(physAddr); + page->bits.page = physAddr >> PAGE_SHIFT; + mmu_frame_allocate(page, flags); +} + +void * mmu_map_from_physical(uintptr_t frameaddress) { + return (void*)(frameaddress | HIGH_MAP_REGION); +} + +#define PDP_MASK 0x3fffffffUL +#define PD_MASK 0x1fffffUL +#define PT_MASK PAGE_LOW_MASK +#define ENTRY_MASK 0x1FF +union PML * mmu_get_page_other(union PML * root, uintptr_t virtAddr) { + //printf("mmu_get_page_other(%#zx, %#zx);\n", (uintptr_t)root, virtAddr); + /* Walk it */ + uintptr_t realBits = virtAddr & CANONICAL_MASK; + uintptr_t pageAddr = realBits >> PAGE_SHIFT; + unsigned int pml4_entry = (pageAddr >> 27) & ENTRY_MASK; + unsigned int pdp_entry = (pageAddr >> 18) & ENTRY_MASK; + unsigned int pd_entry = (pageAddr >> 9) & ENTRY_MASK; + unsigned int pt_entry = (pageAddr) & ENTRY_MASK; + + /* Get the PML4 entry for this address */ + if (!root[pml4_entry].bits.present) { + return NULL; + } + + union PML * pdp = mmu_map_from_physical((uintptr_t)root[pml4_entry].bits.page << PAGE_SHIFT); + + if (!pdp[pdp_entry].bits.present) { + return NULL; + } + + if (!pdp[pdp_entry].bits.table_page) { + return NULL; + } + + union PML * pd = mmu_map_from_physical((uintptr_t)pdp[pdp_entry].bits.page << PAGE_SHIFT); + + if (!pd[pd_entry].bits.present) { + return NULL; + } + + if (!pd[pd_entry].bits.table_page) { + return NULL; + } + + union PML * pt = mmu_map_from_physical((uintptr_t)pd[pd_entry].bits.page << PAGE_SHIFT); + return (union PML *)&pt[pt_entry]; +} + +uintptr_t mmu_map_to_physical(union PML * root, uintptr_t virtAddr) { + if (!root) { + if (virtAddr >= MODULE_BASE_START) { + return (virtAddr - MODULE_BASE_START) + QEMU_VIRT_KERNEL_BASE; + } else if (virtAddr >= HIGH_MAP_REGION) { + return (virtAddr - HIGH_MAP_REGION); + } + return (uintptr_t)virtAddr; + } + + uintptr_t realBits = virtAddr & CANONICAL_MASK; + uintptr_t pageAddr = realBits >> PAGE_SHIFT; + unsigned int pml4_entry = (pageAddr >> 27) & ENTRY_MASK; + unsigned int pdp_entry = (pageAddr >> 18) & ENTRY_MASK; + unsigned int pd_entry = (pageAddr >> 9) & ENTRY_MASK; + unsigned int pt_entry = (pageAddr) & ENTRY_MASK; + + if (!root[pml4_entry].bits.present) return (uintptr_t)-1; + + union PML * pdp = mmu_map_from_physical((uintptr_t)root[pml4_entry].bits.page << PAGE_SHIFT); + + if (!pdp[pdp_entry].bits.present) return (uintptr_t)-2; + if (!pdp[pdp_entry].bits.table_page) return ((uintptr_t)pdp[pdp_entry].bits.page << PAGE_SHIFT) | (virtAddr & PDP_MASK); + + union PML * pd = mmu_map_from_physical((uintptr_t)pdp[pdp_entry].bits.page << PAGE_SHIFT); + + if (!pd[pd_entry].bits.present) return (uintptr_t)-3; + if (!pd[pd_entry].bits.table_page) return ((uintptr_t)pd[pd_entry].bits.page << PAGE_SHIFT) | (virtAddr & PD_MASK); + + union PML * pt = mmu_map_from_physical((uintptr_t)pd[pd_entry].bits.page << PAGE_SHIFT); + + if (!pt[pt_entry].bits.present) return (uintptr_t)-4; + return ((uintptr_t)pt[pt_entry].bits.page << PAGE_SHIFT) | (virtAddr & PT_MASK); +} + +union PML * mmu_get_page(uintptr_t virtAddr, int flags) { + /* This is all the same as x86, thankfully? */ + uintptr_t realBits = virtAddr & CANONICAL_MASK; + uintptr_t pageAddr = realBits >> PAGE_SHIFT; + unsigned int pml4_entry = (pageAddr >> 27) & ENTRY_MASK; + unsigned int pdp_entry = (pageAddr >> 18) & ENTRY_MASK; + unsigned int pd_entry = (pageAddr >> 9) & ENTRY_MASK; + unsigned int pt_entry = (pageAddr) & ENTRY_MASK; + + union PML * root = this_core->current_pml; + + /* Get the PML4 entry for this address */ + if (!root[pml4_entry].bits.present) { + if (!(flags & MMU_GET_MAKE)) goto _noentry; + spin_lock(frame_alloc_lock); + uintptr_t newPage = mmu_first_frame() << PAGE_SHIFT; + mmu_frame_set(newPage); + spin_unlock(frame_alloc_lock); + /* zero it */ + memset(mmu_map_from_physical(newPage), 0, PAGE_SIZE); + root[pml4_entry].raw = (newPage) | PTE_VALID | PTE_TABLE | PTE_AF; + asm volatile ("dsb ishst\ntlbi vmalle1is\ndsb ish\nisb" ::: "memory"); + } + + union PML * pdp = mmu_map_from_physical((uintptr_t)root[pml4_entry].bits.page << PAGE_SHIFT); + + if (!pdp[pdp_entry].bits.present) { + if (!(flags & MMU_GET_MAKE)) goto _noentry; + spin_lock(frame_alloc_lock); + uintptr_t newPage = mmu_first_frame() << PAGE_SHIFT; + mmu_frame_set(newPage); + spin_unlock(frame_alloc_lock); + /* zero it */ + memset(mmu_map_from_physical(newPage), 0, PAGE_SIZE); + pdp[pdp_entry].raw = (newPage) | PTE_VALID | PTE_TABLE | PTE_AF; + asm volatile ("dsb ishst\ntlbi vmalle1is\ndsb ish\nisb" ::: "memory"); + } + + if (!pdp[pdp_entry].bits.table_page) { + printf("Warning: Tried to get page for a 1GiB block! %d\n", pdp_entry); + return NULL; + } + + union PML * pd = mmu_map_from_physical((uintptr_t)pdp[pdp_entry].bits.page << PAGE_SHIFT); + + if (!pd[pd_entry].bits.present) { + if (!(flags & MMU_GET_MAKE)) goto _noentry; + spin_lock(frame_alloc_lock); + uintptr_t newPage = mmu_first_frame() << PAGE_SHIFT; + mmu_frame_set(newPage); + spin_unlock(frame_alloc_lock); + /* zero it */ + memset(mmu_map_from_physical(newPage), 0, PAGE_SIZE); + pd[pd_entry].raw = (newPage) | PTE_VALID | PTE_TABLE | PTE_AF; + asm volatile ("dsb ishst\ntlbi vmalle1is\ndsb ish\nisb" ::: "memory"); + } + + if (!pd[pd_entry].bits.table_page) { + printf("Warning: Tried to get page for a 2MiB block!\n"); + return NULL; + } + + union PML * pt = mmu_map_from_physical((uintptr_t)pd[pd_entry].bits.page << PAGE_SHIFT); + return (union PML *)&pt[pt_entry]; + +_noentry: + printf("no entry for requested page\n"); + return NULL; +} + +static int copy_page_maybe(union PML * pt_in, union PML * pt_out, size_t l, uintptr_t address) { + spin_lock(frame_alloc_lock); + + /* TODO cow bits */ + + char * page_in = mmu_map_from_physical((uintptr_t)pt_in[l].bits.page << PAGE_SHIFT); + uintptr_t newPage = mmu_first_frame() << PAGE_SHIFT; + mmu_frame_set(newPage); + char * page_out = mmu_map_from_physical(newPage); + memcpy(page_out,page_in,PAGE_SIZE); + asm volatile ("" ::: "memory"); + pt_out[l].raw = 0; + pt_out[l].bits.table_page = 1; + pt_out[l].bits.present = 1; + pt_out[l].bits.ap = pt_in[l].bits.ap; + pt_out[l].bits.af = pt_in[l].bits.af; + pt_out[l].bits.sh = pt_in[l].bits.sh; + pt_out[l].bits.attrindx = pt_in[l].bits.attrindx; + pt_out[l].bits.page = newPage >> PAGE_SHIFT; + asm volatile ("" ::: "memory"); + + spin_unlock(frame_alloc_lock); + return 0; +} + +union PML * mmu_clone(union PML * from) { + /* Clone the current PMLs... */ + if (!from) from = this_core->current_pml; + + /* First get a page for ourselves. */ + spin_lock(frame_alloc_lock); + uintptr_t newPage = mmu_first_frame() << PAGE_SHIFT; + mmu_frame_set(newPage); + spin_unlock(frame_alloc_lock); + union PML * pml4_out = mmu_map_from_physical(newPage); + + /* Zero bottom half */ + memset(&pml4_out[0], 0, 256 * sizeof(union PML)); + + /* Copy top half */ + memcpy(&pml4_out[256], &from[256], 256 * sizeof(union PML)); + + /* Copy PDPs */ + for (size_t i = 0; i < 256; ++i) { + if (from[i].bits.present) { + union PML * pdp_in = mmu_map_from_physical((uintptr_t)from[i].bits.page << PAGE_SHIFT); + spin_lock(frame_alloc_lock); + uintptr_t newPage = mmu_first_frame() << PAGE_SHIFT; + mmu_frame_set(newPage); + spin_unlock(frame_alloc_lock); + union PML * pdp_out = mmu_map_from_physical(newPage); + memset(pdp_out, 0, 512 * sizeof(union PML)); + pml4_out[i].raw = (newPage) | PTE_VALID | PTE_TABLE | PTE_AF; + + /* Copy the PDs */ + for (size_t j = 0; j < 512; ++j) { + if (pdp_in[j].bits.present) { + union PML * pd_in = mmu_map_from_physical((uintptr_t)pdp_in[j].bits.page << PAGE_SHIFT); + spin_lock(frame_alloc_lock); + uintptr_t newPage = mmu_first_frame() << PAGE_SHIFT; + mmu_frame_set(newPage); + spin_unlock(frame_alloc_lock); + union PML * pd_out = mmu_map_from_physical(newPage); + memset(pd_out, 0, 512 * sizeof(union PML)); + pdp_out[j].raw = (newPage) | PTE_VALID | PTE_TABLE | PTE_AF; + + /* Now copy the PTs */ + for (size_t k = 0; k < 512; ++k) { + if (pd_in[k].bits.present) { + union PML * pt_in = mmu_map_from_physical((uintptr_t)pd_in[k].bits.page << PAGE_SHIFT); + spin_lock(frame_alloc_lock); + uintptr_t newPage = mmu_first_frame() << PAGE_SHIFT; + mmu_frame_set(newPage); + spin_unlock(frame_alloc_lock); + union PML * pt_out = mmu_map_from_physical(newPage); + memset(pt_out, 0, 512 * sizeof(union PML)); + pd_out[k].raw = (newPage) | PTE_VALID | PTE_TABLE | PTE_AF; + + /* Now, finally, copy pages */ + for (size_t l = 0; l < 512; ++l) { + uintptr_t address = ((i << (9 * 3 + 12)) | (j << (9*2 + 12)) | (k << (9 + 12)) | (l << PAGE_SHIFT)); + if (address >= USER_DEVICE_MAP && address <= USER_SHM_HIGH) continue; + if (pt_in[l].bits.present) { + if (1) { //pt_in[l].bits.user) { + copy_page_maybe(pt_in, pt_out, l, address); + } else { + /* If it's not a user page, just copy directly */ + pt_out[l].raw = pt_in[l].raw; + } + } /* Else, mmap'd files? */ + } + } + } + } + } + } + } + + return pml4_out; +} + +uintptr_t mmu_allocate_a_frame(void) { + spin_lock(frame_alloc_lock); + uintptr_t index = mmu_first_frame(); + mmu_frame_set(index << PAGE_SHIFT); + spin_unlock(frame_alloc_lock); + return index; +} + +uintptr_t mmu_allocate_n_frames(int n) { + spin_lock(frame_alloc_lock); + uintptr_t index = mmu_first_n_frames(n); + for (int i = 0; i < n; ++i) { + mmu_frame_set((index+i) << PAGE_SHIFT); + } + spin_unlock(frame_alloc_lock); + return index; +} + +size_t mmu_count_user(union PML * from) { + /* We walk 'from' and count user pages */ + return 0; +} + +size_t mmu_count_shm(union PML * from) { + /* We walk 'from' and count shm region stuff */ + return 0; +} + +size_t mmu_total_memory(void) { + return total_memory; +} + +size_t mmu_used_memory(void) { + size_t ret = 0; + size_t i, j; + for (i = 0; i < INDEX_FROM_BIT(nframes); ++i) { + for (j = 0; j < 32; ++j) { + uint32_t testFrame = (uint32_t)0x1 << j; + if (frames[i] & testFrame) { + ret++; + } + } + } + return ret * 4 - unavailable_memory; +} + +void mmu_free(union PML * from) { + /* walk and free pages */ +} + +union PML * mmu_get_kernel_directory(void) { + return (union PML*)&init_page_region; +} + +void mmu_set_directory(union PML * new_pml) { + /* Set the EL0 and EL1 directy things? + * There are two of these... */ + if (!new_pml) new_pml = mmu_map_from_physical((uintptr_t)&init_page_region[0]); + this_core->current_pml = new_pml; + + asm volatile ("msr TTBR0_EL1,%0" : : "r"(mmu_map_to_physical(new_pml, (uintptr_t)new_pml))); + asm volatile ("msr TTBR1_EL1,%0" : : "r"(mmu_map_to_physical(new_pml, (uintptr_t)new_pml))); + asm volatile ("dsb ishst\ntlbi vmalle1is\ndsb ish\nisb" ::: "memory"); +} + +void mmu_invalidate(uintptr_t addr) { +} + +static char * heapStart = NULL; +extern char end[]; + +void * sbrk(size_t bytes) { + if (!heapStart) { + arch_fatal_prepare(); + printf("sbrk: Called before heap was ready.\n"); + arch_dump_traceback(); + arch_fatal(); + } + + if (!bytes) { + /* Skip lock acquisition if we just wanted to know where the break was. */ + return heapStart; + } + + if (bytes & PAGE_LOW_MASK) { + arch_fatal_prepare(); + printf("sbrk: Size must be multiple of 4096, was %#zx\n", bytes); + arch_dump_traceback(); + arch_fatal(); + } + + spin_lock(kheap_lock); + void * out = heapStart; + + for (uintptr_t p = (uintptr_t)out; p < (uintptr_t)out + bytes; p += PAGE_SIZE) { + union PML * page = mmu_get_page(p, MMU_GET_MAKE); + mmu_frame_allocate(page, MMU_FLAG_WRITABLE | MMU_FLAG_KERNEL); + } + + heapStart += bytes; + spin_unlock(kheap_lock); + return out; +} + +void * mmu_map_mmio_region(uintptr_t physical_address, size_t size) { + printf("attempt to map mmio\n"); + return NULL; +} + +void * mmu_map_module(size_t size) { + printf("attempt to map module\n"); + return NULL; +} + +void mmu_unmap_module(uintptr_t start_address, size_t size) { +} + +int mmu_copy_on_write(uintptr_t address) { + return 1; +} + +int mmu_validate_user_pointer(void * addr, size_t size, int flags) { + //printf("mmu_validate_user_pointer(%#zx, %lu, %u);\n", (uintptr_t)addr, size, flags); + if (addr == NULL && !(flags & MMU_PTR_NULL)) return 0; + if (size > 0x800000000000) return 0; + + uintptr_t base = (uintptr_t)addr; + uintptr_t end = size ? (base + (size - 1)) : base; + + /* Get start page, end page */ + uintptr_t page_base = base >> 12; + uintptr_t page_end = end >> 12; + + for (uintptr_t page = page_base; page <= page_end; ++page) { + if ((page & 0xffff800000000) != 0 && (page & 0xffff800000000) != 0xffff800000000) return 0; + union PML * page_entry = mmu_get_page_other(this_core->current_process->thread.page_directory->directory, page << 12); + if (!page_entry) { + return 0; + } + if (!page_entry->bits.present) { + return 0; + } + if (!(page_entry->bits.ap & 1)) { + return 0; + } + if ((page_entry->bits.ap & 2) && (flags & MMU_PTR_WRITE)) { + return 0; + //if (mmu_copy_on_write((uintptr_t)(page << 12))) return 0; + } + } + + return 1; +} + +static uintptr_t k2p(void * x) { + return ((uintptr_t)x - MODULE_BASE_START) + QEMU_VIRT_KERNEL_BASE; +} + +void mmu_init(uintptr_t memsize, uintptr_t firstFreePage) { + this_core->current_pml = (union PML *)&init_page_region; + + /* On this machine, there's 1GiB of unavailable memory. */ + unavailable_memory = 1048576; + total_memory = 4194304; + + /* MAIR setup? */ + uint64_t mair; + asm volatile ("mrs %0,MAIR_EL1" : "=r"(mair)); + dprintf("Current MAIR:\n" + " Attr0: 0x%02zx Attr1: 0x%02zx\n" + " Attr2: 0x%02zx Attr3: 0x%02zx\n" + " Attr4: 0x%02zx Attr5: 0x%02zx\n" + " Attr6: 0x%02zx Attr7: 0x%02zx\n", + ((mair >> 0) & 0xFF), + ((mair >> 8) & 0xFF), + ((mair >> 16) & 0xFF), + ((mair >> 24) & 0xFF), + ((mair >> 32) & 0xFF), + ((mair >> 40) & 0xFF), + ((mair >> 48) & 0xFF), + ((mair >> 52) & 0xFF)); + + //mair &= (0xFFffFFffFF000000); + mair = (0x000000000044ff00); + asm volatile ("msr MAIR_EL1,%0" :: "r"(mair)); + asm volatile ("mrs %0,MAIR_EL1" : "=r"(mair)); + dprintf("Loaded MAIR:\n" + " Attr0: 0x%02zx Attr1: 0x%02zx\n" + " Attr2: 0x%02zx Attr3: 0x%02zx\n" + " Attr4: 0x%02zx Attr5: 0x%02zx\n" + " Attr6: 0x%02zx Attr7: 0x%02zx\n", + ((mair >> 0) & 0xFF), + ((mair >> 8) & 0xFF), + ((mair >> 16) & 0xFF), + ((mair >> 24) & 0xFF), + ((mair >> 32) & 0xFF), + ((mair >> 40) & 0xFF), + ((mair >> 48) & 0xFF), + ((mair >> 52) & 0xFF)); + + asm volatile ("" ::: "memory"); + + /* Replicate the mapping we already have */ + init_page_region[511].raw = k2p(&high_base_pml) | PTE_VALID | PTE_TABLE | PTE_AF; + init_page_region[510].raw = k2p(&heap_base_pml) | PTE_VALID | PTE_TABLE | PTE_AF; + + /* "Identity" map at -512GiB */ + for (size_t i = 0; i < 64; ++i) { + high_base_pml[i].raw = (i << 30) | PTE_VALID | PTE_AF | (1 << 2); + } + + + /* Set up some space to map us */ + + /* init_page_region[511] -> high_base_pml[510] -> kbase_pmls[0] -> kbase_pmls[1+n] */ + high_base_pml[510].raw = k2p(&kbase_pmls[0]) | PTE_VALID | PTE_TABLE | PTE_AF; + for (size_t j = 0; j < 64; ++j) { + kbase_pmls[0][j].raw = k2p(&kbase_pmls[1+j]) | PTE_VALID | PTE_TABLE | PTE_AF; + for (int i = 0; i < 512; ++i) { + kbase_pmls[1+j][i].raw = (uintptr_t)(QEMU_VIRT_KERNEL_BASE + LARGE_PAGE_SIZE * j + PAGE_SIZE * i) | + PTE_VALID | PTE_AF | PTE_SH_A | PTE_TABLE | (1 << 2); + } + } + + /* We should be ready to switch to our page directory? */ + asm volatile ("msr TTBR0_EL1,%0" : : "r"(k2p(&init_page_region))); + asm volatile ("msr TTBR1_EL1,%0" : : "r"(k2p(&init_page_region))); + asm volatile ("dsb ishst\ntlbi vmalle1is\ndsb ish\nisb" ::: "memory"); + + /* Let's map some heap. */ + heap_base_pml[0].raw = k2p(&heap_base_pd) | PTE_VALID | PTE_TABLE | PTE_AF; + heap_base_pd[0].raw = k2p(&heap_base_pt[0]) | PTE_VALID | PTE_TABLE | PTE_AF; + heap_base_pd[1].raw = k2p(&heap_base_pt[512]) | PTE_VALID | PTE_TABLE | PTE_AF; + heap_base_pd[2].raw = k2p(&heap_base_pt[1024]) | PTE_VALID | PTE_TABLE | PTE_AF; + + /* Physical frame allocator. We're gonna do this the same as the one we have x86-64, because + * I can't be bothered to think of anything better right now... */ + nframes = (memsize) >> 12; + size_t bytesOfFrames = INDEX_FROM_BIT(nframes * 8); + bytesOfFrames = (bytesOfFrames + PAGE_LOW_MASK) & PAGE_SIZE_MASK; + + /* TODO we should figure out where the DTB ends on virt, as that's where we can + * start doing this... */ + size_t pagesOfFrames = bytesOfFrames >> 12; + + /* Map pages for it... */ + for (size_t i = 0; i < pagesOfFrames; ++i) { + heap_base_pt[i].raw = (firstFreePage + (i << 12)) | PTE_VALID | PTE_AF | PTE_SH_A | PTE_TABLE | (1 << 2); + } + + asm volatile ("dsb ishst\ntlbi vmalle1is\ndsb ish\nisb" ::: "memory"); + + /* Just assume everything is in use. */ + frames = (void*)((uintptr_t)KERNEL_HEAP_START); + memset((void*)frames, 0x00, bytesOfFrames); + + /* Set frames as in use... this also marks all of the lower gigabyte, conveniently... */ + for (uintptr_t i = 0; i < firstFreePage + bytesOfFrames; i+= PAGE_SIZE) { + mmu_frame_set(i); + } + + /* Set kernel space as in use */ + for (uintptr_t i = 0; i < 64 * LARGE_PAGE_SIZE; i += PAGE_SIZE) { + mmu_frame_set(QEMU_VIRT_KERNEL_BASE + i); + } + + heapStart = (char*)KERNEL_HEAP_START + bytesOfFrames; + + lowest_available = (firstFreePage + bytesOfFrames); + +} diff --git a/kernel/arch/aarch64/user.c b/kernel/arch/aarch64/user.c new file mode 100644 index 00000000..f50f0269 --- /dev/null +++ b/kernel/arch/aarch64/user.c @@ -0,0 +1,927 @@ +/** + * @file kernel/arch/x86_64/user.c + * @brief Various assembly snippets for jumping to usermode and back. + * + * @copyright + * This file is part of ToaruOS and is released under the terms + * of the NCSA / University of Illinois License - see LICENSE.md + * Copyright (C) 2021 K. Lange + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +extern void framebuffer_initialize(void); +extern void fbterm_initialize(void); + +static uint32_t swizzle(uint32_t from) { + uint8_t a = from >> 24; + uint8_t b = from >> 16; + uint8_t c = from >> 8; + uint8_t d = from; + return (d << 24) | (c << 16) | (b << 8) | (a); +} + +static uint64_t swizzle64(uint64_t from) { + uint8_t a = from >> 56; + uint8_t b = from >> 48; + uint8_t c = from >> 40; + uint8_t d = from >> 32; + uint8_t e = from >> 24; + uint8_t f = from >> 16; + uint8_t g = from >> 8; + uint8_t h = from; + return ((uint64_t)h << 56) | ((uint64_t)g << 48) | ((uint64_t)f << 40) | ((uint64_t)e << 32) | (d << 24) | (c << 16) | (b << 8) | (a); +} + +static uint16_t swizzle16(uint16_t from) { + uint8_t a = from >> 8; + uint8_t b = from; + return (b << 8) | (a); +} + +struct fdt_header { + uint32_t magic; + uint32_t totalsize; + uint32_t off_dt_struct; + uint32_t off_dt_strings; + uint32_t off_mem_rsvmap; + uint32_t version; + uint32_t last_comp_version; + uint32_t boot_cpuid_phys; + uint32_t size_dt_strings; + uint32_t size_dt_struct; +}; + +static uint32_t * parse_node(uint32_t * node, char * strings, int x) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 9) return NULL; + if (swizzle(*node) != 1) { + printf("Not a node? Got %x\n", swizzle(*node)); + return NULL; + } + + /* Skip the BEGIN_NODE */ + node++; + + for (int i = 0; i < x; ++i) printf(" "); + + while (1) { + char * x = (char*)node; + if (x[0]) { printf("%c",x[0]); } else { node++; break; } + if (x[1]) { printf("%c",x[1]); } else { node++; break; } + if (x[2]) { printf("%c",x[2]); } else { node++; break; } + if (x[3]) { printf("%c",x[3]); } else { node++; break; } + node++; + } + printf("\n"); + + while (1) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 2) return node+1; + if (swizzle(*node) == 3) { + for (int i = 0; i < x; ++i) printf(" "); + uint32_t len = swizzle(node[1]); + uint32_t nameoff = swizzle(node[2]); + printf(" property %s len=%u\n", strings + nameoff, len); + node += 3; + node += (len + 3) / 4; + } else if (swizzle(*node) == 1) { + node = parse_node(node, strings, x + 1); + } + } + +} + +static void dump_dtb(uintptr_t addr) { + + struct fdt_header * fdt = (struct fdt_header*)addr; + +#define P(o) dprintf(#o " = %#x\n", swizzle(fdt-> o)) + P(magic); + P(totalsize); + P(off_dt_struct); + P(off_dt_strings); + P(off_mem_rsvmap); + P(version); + P(last_comp_version); + P(boot_cpuid_phys); + P(size_dt_strings); + P(size_dt_struct); + + char * dtb_strings = (char *)(addr + swizzle(fdt->off_dt_strings)); + uint32_t * dtb_struct = (uint32_t *)(addr + swizzle(fdt->off_dt_struct)); + + parse_node(dtb_struct, dtb_strings, 0); +} + +static uint32_t * find_subnode(uint32_t * node, char * strings, const char * name, uint32_t ** node_out, int (*cmp)(const char* a, const char *b)) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 9) return NULL; + if (swizzle(*node) != 1) return NULL; + node++; + + if (cmp((char*)node,name)) { + *node_out = node; + return NULL; + } + + while ((*node & 0xFF000000) && (*node & 0xFF0000) && (*node & 0xFF00) && (*node & 0xFF)) node++; + node++; + + while (1) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 2) return node+1; + if (swizzle(*node) == 3) { + uint32_t len = swizzle(node[1]); + node += 3; + node += (len + 3) / 4; + } else if (swizzle(*node) == 1) { + node = find_subnode(node, strings, name, node_out, cmp); + if (!node) return NULL; + } + } +} + +static uint32_t * find_node_int(const char * name, int (*cmp)(const char*,const char*)) { + uintptr_t addr = (uintptr_t)mmu_map_from_physical(0x40000000); + struct fdt_header * fdt = (struct fdt_header*)addr; + char * dtb_strings = (char *)(addr + swizzle(fdt->off_dt_strings)); + uint32_t * dtb_struct = (uint32_t *)(addr + swizzle(fdt->off_dt_struct)); + + uint32_t * out = NULL; + find_subnode(dtb_struct, dtb_strings, name, &out, cmp); + return out; +} + +static int base_cmp(const char *a, const char *b) { + return !strcmp(a,b); +} +static uint32_t * find_node(const char * name) { + return find_node_int(name,base_cmp); +} + +static int prefix_cmp(const char *a, const char *b) { + return !memcmp(a,b,strlen(b)); +} + +static uint32_t * find_node_prefix(const char * name) { + return find_node_int(name,prefix_cmp); +} + +static uint32_t * node_find_property_int(uint32_t * node, char * strings, const char * property, uint32_t ** out) { + while ((*node & 0xFF000000) && (*node & 0xFF0000) && (*node & 0xFF00) && (*node & 0xFF)) node++; + node++; + + while (1) { + while (swizzle(*node) == 4) node++; + if (swizzle(*node) == 2) return node+1; + if (swizzle(*node) == 3) { + uint32_t len = swizzle(node[1]); + uint32_t nameoff = swizzle(node[2]); + if (!strcmp(strings + nameoff, property)) { + *out = &node[1]; + return NULL; + } + node += 3; + node += (len + 3) / 4; + } else if (swizzle(*node) == 1) { + node = node_find_property_int(node+1, strings, property, out); + if (!node) return NULL; + } + } +} + +static uint32_t * node_find_property(uint32_t * node, const char * property) { + uintptr_t addr = (uintptr_t)mmu_map_from_physical(0x40000000); + struct fdt_header * fdt = (struct fdt_header*)addr; + char * dtb_strings = (char *)(addr + swizzle(fdt->off_dt_strings)); + uint32_t * out = NULL; + node_find_property_int(node, dtb_strings, property, &out); + return out; +} + +/** + * @brief Enter userspace. + * + * Called by process startup. + * Does not return. + * + * @param entrypoint Address to "return" to in userspace. + * @param argc Number of arguments to provide to the new process. + * @param argv Argument array to pass to the new process; make sure this is user-accessible! + * @param envp Environment strings array + * @param stack Userspace stack address. + */ +void arch_enter_user(uintptr_t entrypoint, int argc, char * argv[], char * envp[], uintptr_t stack) { + asm volatile( + "msr ELR_EL1, %0\n" /* entrypoint */ + "msr SP_EL0, %1\n" /* stack */ + "msr SPSR_EL1, %2\n" /* EL 0 */ + :: + "r"(entrypoint), "r"(stack), "r"(0)); + + register uint64_t x0 __asm__("x0") = argc; + register uint64_t x1 __asm__("x1") = (uintptr_t)argv; + register uint64_t x2 __asm__("x2") = (uintptr_t)envp; + + asm volatile( + "eret" :: "r"(x0), "r"(x1), "r"(x2)); + + __builtin_unreachable(); +} + +/** + * @brief Enter a userspace signal handler. + * + * Similar to @c arch_enter_user but also setups up magic return addresses. + * + * Since signal handlers do to take complicated argument arrays, this only + * supplies a @p signum argument. + * + * Does not return. + * + * @param entrypoint Userspace address of the signal handler, set by the process. + * @param signum Signal number that caused this entry. + */ +void arch_enter_signal_handler(uintptr_t entrypoint, int signum) { + /* TODO */ + printf("%s() called\n", __func__); + while (1); + __builtin_unreachable(); +} + +/** + * @brief Save FPU registers for this thread. + */ +void arch_restore_floating(process_t * proc) { + asm volatile ( + "ldr q0 , [%0, #(0 * 16)]\n" + "ldr q1 , [%0, #(1 * 16)]\n" + "ldr q2 , [%0, #(2 * 16)]\n" + "ldr q3 , [%0, #(3 * 16)]\n" + "ldr q4 , [%0, #(4 * 16)]\n" + "ldr q5 , [%0, #(5 * 16)]\n" + "ldr q6 , [%0, #(6 * 16)]\n" + "ldr q7 , [%0, #(7 * 16)]\n" + "ldr q8 , [%0, #(8 * 16)]\n" + "ldr q9 , [%0, #(9 * 16)]\n" + "ldr q10, [%0, #(10 * 16)]\n" + "ldr q11, [%0, #(11 * 16)]\n" + "ldr q12, [%0, #(12 * 16)]\n" + "ldr q13, [%0, #(13 * 16)]\n" + "ldr q14, [%0, #(14 * 16)]\n" + "ldr q15, [%0, #(15 * 16)]\n" + "ldr q16, [%0, #(16 * 16)]\n" + "ldr q17, [%0, #(17 * 16)]\n" + "ldr q18, [%0, #(18 * 16)]\n" + "ldr q19, [%0, #(19 * 16)]\n" + "ldr q20, [%0, #(20 * 16)]\n" + "ldr q21, [%0, #(21 * 16)]\n" + "ldr q22, [%0, #(22 * 16)]\n" + "ldr q23, [%0, #(23 * 16)]\n" + "ldr q24, [%0, #(24 * 16)]\n" + "ldr q25, [%0, #(25 * 16)]\n" + "ldr q26, [%0, #(26 * 16)]\n" + "ldr q27, [%0, #(27 * 16)]\n" + "ldr q28, [%0, #(28 * 16)]\n" + "ldr q29, [%0, #(29 * 16)]\n" + "ldr q30, [%0, #(30 * 16)]\n" + "ldr q31, [%0, #(31 * 16)]\n" + ::"r"(&proc->thread.fp_regs)); +} + +/** + * @brief Restore FPU registers for this thread. + */ +void arch_save_floating(process_t * proc) { + asm volatile ( + "str q0 , [%0, #(0 * 16)]\n" + "str q1 , [%0, #(1 * 16)]\n" + "str q2 , [%0, #(2 * 16)]\n" + "str q3 , [%0, #(3 * 16)]\n" + "str q4 , [%0, #(4 * 16)]\n" + "str q5 , [%0, #(5 * 16)]\n" + "str q6 , [%0, #(6 * 16)]\n" + "str q7 , [%0, #(7 * 16)]\n" + "str q8 , [%0, #(8 * 16)]\n" + "str q9 , [%0, #(9 * 16)]\n" + "str q10, [%0, #(10 * 16)]\n" + "str q11, [%0, #(11 * 16)]\n" + "str q12, [%0, #(12 * 16)]\n" + "str q13, [%0, #(13 * 16)]\n" + "str q14, [%0, #(14 * 16)]\n" + "str q15, [%0, #(15 * 16)]\n" + "str q16, [%0, #(16 * 16)]\n" + "str q17, [%0, #(17 * 16)]\n" + "str q18, [%0, #(18 * 16)]\n" + "str q19, [%0, #(19 * 16)]\n" + "str q20, [%0, #(20 * 16)]\n" + "str q21, [%0, #(21 * 16)]\n" + "str q22, [%0, #(22 * 16)]\n" + "str q23, [%0, #(23 * 16)]\n" + "str q24, [%0, #(24 * 16)]\n" + "str q25, [%0, #(25 * 16)]\n" + "str q26, [%0, #(26 * 16)]\n" + "str q27, [%0, #(27 * 16)]\n" + "str q28, [%0, #(28 * 16)]\n" + "str q29, [%0, #(29 * 16)]\n" + "str q30, [%0, #(30 * 16)]\n" + "str q31, [%0, #(31 * 16)]\n" + ::"r"(&proc->thread.fp_regs):"memory"); +} + +/** + * @brief Prepare for a fatal event by stopping all other cores. + */ +void arch_fatal_prepare(void) { + /* TODO Stop other cores */ +} + +/** + * @brief Halt all processors, including this one. + * @see arch_fatal_prepare + */ +void arch_fatal(void) { + arch_fatal_prepare(); + printf("-- fatal panic\n"); + /* TODO */ + while (1) {} +} + +/** + * @brief Reboot the computer. + * + * At least on 'virt', there's a system control + * register we can write to to reboot or at least + * do a full shutdown. + */ +long arch_reboot(void) { + return 0; +} + +void aarch64_regs(struct regs *r) { +#define reg(a,b) printf(" X%02d=0x%016zx X%02d=0x%016zx\n",a,r->x ## a, b, r->x ## b) + reg(0,1); + reg(2,3); + reg(4,5); + reg(6,7); + reg(8,9); + reg(10,11); + reg(12,13); + reg(14,15); + reg(16,17); + reg(18,19); + reg(20,21); + reg(22,23); + reg(24,25); + reg(26,27); + reg(28,29); + printf(" X30=0x%016zx SP=0x%016zx\n", r->x30, r->user_sp); +#undef reg +} + +void aarch64_context(process_t * proc) { + printf(" SP=0x%016zx BP(x29)=0x%016zx\n", proc->thread.context.sp, proc->thread.context.bp); + printf(" IP=0x%016zx TLSBASE=0x%016zx\n", proc->thread.context.ip, proc->thread.context.tls_base); + printf(" X19=0x%016zx X20=%016zx\n", proc->thread.context.saved[0], proc->thread.context.saved[1]); + printf(" X21=0x%016zx X22=%016zx\n", proc->thread.context.saved[2], proc->thread.context.saved[3]); + printf(" X23=0x%016zx X24=%016zx\n", proc->thread.context.saved[4], proc->thread.context.saved[5]); + printf(" X25=0x%016zx X26=%016zx\n", proc->thread.context.saved[6], proc->thread.context.saved[7]); + printf(" X27=0x%016zx X28=%016zx\n", proc->thread.context.saved[8], proc->thread.context.saved[9]); + printf(" ELR=0x%016zx SPSR=%016zx\n", proc->thread.context.saved[10], proc->thread.context.saved[11]); +} + +/* Syscall parameter accessors */ +void arch_syscall_return(struct regs * r, long retval) { r->x0 = retval; } +long arch_syscall_number(struct regs * r) { return r->x0; } +long arch_syscall_arg0(struct regs * r) { return r->x1; } +long arch_syscall_arg1(struct regs * r) { return r->x2; } +long arch_syscall_arg2(struct regs * r) { return r->x3; } +long arch_syscall_arg3(struct regs * r) { return r->x4; } +long arch_syscall_arg4(struct regs * r) { return r->x5; } +long arch_stack_pointer(struct regs * r) { printf("%s() called\n", __func__); return 0; /* TODO */ } +long arch_user_ip(struct regs * r) { printf("%s() called\n", __func__); return 0; /* TODO */ } + +/* No port i/o on arm, but these are still littered around some + * drivers we need to remove... */ +unsigned short inports(unsigned short _port) { return 0; } +unsigned int inportl(unsigned short _port) { return 0; } +unsigned char inportb(unsigned short _port) { return 0; } +void inportsm(unsigned short port, unsigned char * data, unsigned long size) { +} + +void outports(unsigned short _port, unsigned short _data) { +} + +void outportl(unsigned short _port, unsigned int _data) { +} + +void outportb(unsigned short _port, unsigned char _data) { +} + +void outportsm(unsigned short port, unsigned char * data, unsigned long size) { +} + +void arch_framebuffer_initialize(void) { + /* I'm not sure we have any options here... + * lfbvideo calls this expecting it to fill in information + * on a preferred video mode; maybe dtb has that? */ +} + +const char * arch_get_cmdline(void) { + /* this should be available from dtb directly as a string */ + return "start=live-session"; +} + +const char * arch_get_loader(void) { + return ""; +} + +/* These should probably assembly. */ +void arch_enter_tasklet(void) { + /* Pop two arguments, jump to the second one. */ + printf("%s() called\n", __func__); +} + +/* ARM says the system clock tick rate is generally in + * the range of 1-50MHz. Since we throw around integer + * MHz ratings that's not great, so let's give it a few + * more digits for long-term accuracy? */ +uint64_t sys_timer_freq = 0; +uint64_t arch_boot_time = 0; /**< No idea where we're going to source this from, need an RTC. */ +uint64_t basis_time = 0; +#define SUBSECONDS_PER_SECOND 1000000 + +uint64_t arch_perf_timer(void) { + uint64_t val; + asm volatile ("mrs %0,CNTPCT_EL0" : "=r"(val)); + return val * 100; +} + +size_t arch_cpu_mhz(void) { + return sys_timer_freq; +} + +static void arch_clock_initialize() { + void * clock_addr = mmu_map_from_physical(0x09010000); + uint64_t val; + asm volatile ("mrs %0,CNTFRQ_EL0" : "=r"(val)); + sys_timer_freq = val / 10000; + arch_boot_time = *(volatile uint32_t*)clock_addr; + basis_time = arch_perf_timer() / sys_timer_freq; + + dprintf("timer: Using %ld MHz as arch_perf_timer frequency.\n", arch_cpu_mhz()); +} + +static void update_ticks(uint64_t ticks, uint64_t *timer_ticks, uint64_t *timer_subticks) { + *timer_subticks = ticks - basis_time; /* should be basis time from when we read RTC */ + *timer_ticks = *timer_subticks / SUBSECONDS_PER_SECOND; + *timer_subticks = *timer_subticks % SUBSECONDS_PER_SECOND; +} + +int gettimeofday(struct timeval * t, void *z) { + uint64_t tsc = arch_perf_timer(); + uint64_t timer_ticks, timer_subticks; + update_ticks(tsc / sys_timer_freq, &timer_ticks, &timer_subticks); + t->tv_sec = arch_boot_time + timer_ticks; + t->tv_usec = timer_subticks; + return 0; +} + +uint64_t now(void) { + struct timeval t; + gettimeofday(&t, NULL); + return t.tv_sec; +} + +void relative_time(unsigned long seconds, unsigned long subseconds, unsigned long * out_seconds, unsigned long * out_subseconds) { + if (!arch_boot_time) { + *out_seconds = 0; + *out_subseconds = 0; + return; + } + + uint64_t tsc = arch_perf_timer(); + uint64_t timer_ticks, timer_subticks; + update_ticks(tsc / sys_timer_freq, &timer_ticks, &timer_subticks); + if (subseconds + timer_subticks >= SUBSECONDS_PER_SECOND) { + *out_seconds = timer_ticks + seconds + (subseconds + timer_subticks) / SUBSECONDS_PER_SECOND; + *out_subseconds = (subseconds + timer_subticks) % SUBSECONDS_PER_SECOND; + } else { + *out_seconds = timer_ticks + seconds; + *out_subseconds = timer_subticks + subseconds; + } +} + +void arch_dump_traceback(void) { + +} + + +static volatile unsigned int * _log_device_addr = 0; +static size_t _early_log_write(size_t size, uint8_t * buffer) { + for (unsigned int i = 0; i < size; ++i) { + *_log_device_addr = buffer[i]; + } + return size; +} + +static void early_log_initialize(void) { + _log_device_addr = mmu_map_from_physical(0x09000000); + printf_output = &_early_log_write; +} + +void arch_set_core_base(uintptr_t base) { + asm volatile ("msr TPIDR_EL1,%0" : : "r"(base)); + asm volatile ("mrs x18, TPIDR_EL1"); +} + +void arch_set_tls_base(uintptr_t tlsbase) { + asm volatile ("msr TPIDR_EL0,%0" : : "r"(tlsbase)); +} + +void arch_set_kernel_stack(uintptr_t stack) { + this_core->sp_el1 = stack; +} + +void arch_wakeup_others(void) { + /* wakeup */ +} + +static void scan_hit_list(uint32_t device, uint16_t vendorid, uint16_t deviceid, void * extra) { + printf("%02x:%02x.%d (%04x, %04x:%04x)\n", + (int)pci_extract_bus(device), + (int)pci_extract_slot(device), + (int)pci_extract_func(device), + (int)pci_find_type(device), + vendorid, + deviceid); + + printf(" BAR0: 0x%08x", pci_read_field(device, PCI_BAR0, 4)); + printf(" BAR1: 0x%08x", pci_read_field(device, PCI_BAR1, 4)); + printf(" BAR2: 0x%08x", pci_read_field(device, PCI_BAR2, 4)); + printf(" BAR3: 0x%08x", pci_read_field(device, PCI_BAR3, 4)); + printf(" BAR4: 0x%08x", pci_read_field(device, PCI_BAR4, 4)); + printf(" BAR5: 0x%08x\n", pci_read_field(device, PCI_BAR5, 4)); + + printf(" IRQ Line: %d", pci_read_field(device, 0x3C, 1)); + printf(" IRQ Pin: %d", pci_read_field(device, 0x3D, 1)); + printf(" Interrupt: %d", pci_get_interrupt(device)); + printf(" Status: 0x%04x\n", pci_read_field(device, PCI_STATUS, 2)); +} + +static void list_dir(const char * dir) { + fs_node_t * root = kopen(dir,0); + if (root) { + uint64_t index = 0; + dprintf("listing %s: ", dir); + while (1) { + struct dirent * d = readdir_fs(root, index); + if (!d) break; + + dprintf("\a %s", d->d_name); + + free(d); + index++; + } + dprintf("\a\n"); + } + close_fs(root); +} + +static struct fwcfg_dma { + volatile uint32_t control; + volatile uint32_t length; + volatile uint64_t address; +} dma __attribute__((aligned(4096))); + +static void fwcfg_load_initrd(uintptr_t * ramdisk_phys_base, size_t * ramdisk_size) { + uintptr_t z = 0; + size_t z_pages= 0; + uintptr_t uz = 0; + size_t uz_pages = 0; + + extern char end[]; + uintptr_t ramdisk_map_start = ((uintptr_t)&end - 0xffffffff80000000UL) + 0x80000000; + + /* See if we can find a qemu fw_cfg interface, we can use that for a ramdisk */ + uint32_t * fw_cfg = find_node_prefix("fw-cfg"); + if (fw_cfg) { + dprintf("fw-cfg: found interface\n"); + /* best guess until we bother parsing these */ + uint32_t * regs = node_find_property(fw_cfg, "reg"); + if (regs) { + volatile uint8_t * fw_cfg_addr = (volatile uint8_t*)(uintptr_t)(mmu_map_from_physical(swizzle(regs[3]))); + volatile uint64_t * fw_cfg_data = (volatile uint64_t *)fw_cfg_addr; + volatile uint32_t * fw_cfg_32 = (volatile uint32_t *)fw_cfg_addr; + volatile uint16_t * fw_cfg_sel = (volatile uint16_t *)(fw_cfg_addr + 8); + + *fw_cfg_sel = 0; + + uint64_t response = fw_cfg_data[0]; + (void)response; + + /* Needs to be big-endian */ + *fw_cfg_sel = swizzle16(0x19); + + /* count response is 32-bit BE */ + uint32_t count = swizzle(fw_cfg_32[0]); + + struct fw_cfg_file { + uint32_t size; + uint16_t select; + uint16_t reserved; + char name[56]; + }; + + struct fw_cfg_file file; + uint8_t * tmp = (uint8_t *)&file; + + /* Read count entries */ + for (unsigned int i = 0; i < count; ++i) { + for (unsigned int j = 0; j < sizeof(struct fw_cfg_file); ++j) { + tmp[j] = fw_cfg_addr[0]; + } + + /* endian swap to get file size and selector ID */ + file.size = swizzle(file.size); + file.select = swizzle16(file.select); + + if (!strcmp(file.name,"opt/org.toaruos.initrd")) { + dprintf("fw-cfg: initrd found\n"); + z_pages = (file.size + 0xFFF) / 0x1000; + z = ramdisk_map_start; + ramdisk_map_start += z_pages * 0x1000; + uint8_t * x = mmu_map_from_physical(z); + + dma.control = swizzle((file.select << 16) | (1 << 3) | (1 << 1)); + dma.length = swizzle(file.size); + dma.address = swizzle64(z); + + asm volatile ("isb" ::: "memory"); + fw_cfg_data[2] = swizzle64(mmu_map_to_physical(NULL,(uint64_t)&dma)); + asm volatile ("isb" ::: "memory"); + + if (dma.control) { + dprintf("fw-cfg: Error on DMA read (control: %#x)\n", dma.control); + return; + } + + dprintf("fw-cfg: initrd loaded x=%#zx\n", (uintptr_t)x); + + if (x[0] == 0x1F && x[1] == 0x8B) { + dprintf("fw-cfg: will attempt to read size from %#zx\n", (uintptr_t)(x + file.size - sizeof(uint32_t))); + uint32_t size; + memcpy(&size, (x + file.size - sizeof(uint32_t)), sizeof(uint32_t)); + dprintf("fw-cfg: compressed ramdisk unpacks to %u bytes\n", size); + + uz_pages = (size + 0xFFF) / 0x1000; + uz = ramdisk_map_start; + ramdisk_map_start += uz_pages * 0x1000; + uint8_t * ramdisk = mmu_map_from_physical(uz); + + gzip_inputPtr = x; + gzip_outputPtr = ramdisk; + if (gzip_decompress()) { + dprintf("fw-cfg: gzip failure, not mounting ramdisk\n"); + return; + } + + memmove(x, ramdisk, size); + + dprintf("fw-cfg: Unpacked ramdisk at %#zx\n", (uintptr_t)ramdisk); + *ramdisk_phys_base = z; + *ramdisk_size = size; + } else { + dprintf("fw-cfg: Ramdisk at %#zx\n", (uintptr_t)x); + *ramdisk_phys_base = z; + *ramdisk_size = file.size; + } + + return; + } + } + } + } +} + +static void dtb_locate_cmdline(void) { + uint32_t * chosen = find_node("chosen"); + if (chosen) { + uint32_t * prop = node_find_property(chosen, "bootargs"); + if (prop) { + args_parse((char*)&prop[2]); + } + } +} + + +static void exception_handlers(void) { + extern char _exception_vector[]; + + asm volatile("msr VBAR_EL1, %0" :: "r"(&_exception_vector)); +} + +void aarch64_sync_enter(struct regs * r) { + uint64_t esr, far, elr, spsr; + asm volatile ("mrs %0, ESR_EL1" : "=r"(esr)); + asm volatile ("mrs %0, FAR_EL1" : "=r"(far)); + asm volatile ("mrs %0, ELR_EL1" : "=r"(elr)); + asm volatile ("mrs %0, SPSR_EL1" : "=r"(spsr)); + + if ((esr >> 26) == 0x15) { + #if 0 + printf("%s: syscall(%ld) puts sp at %#zx with base of %#zx\n", this_core->current_process->name, r->x0, (uintptr_t)r, + this_core->current_process->image.stack); + printf("SVC call (%d)\n", r->x0); + printf(" x0: %#zx", r->x0); + printf(" x1: %#zx", r->x1); + printf(" x2: %#zx\n", r->x2); + printf(" x3: %#zx", r->x3); + printf(" x4: %#zx", r->x4); + printf(" x5: %#zx\n", r->x5); + #endif + + extern void syscall_handler(struct regs *); + syscall_handler(r); + + #if 0 + printf("Regs at return should be:\n"); + printf(" x0: %#zx", r->x0); + printf(" x1: %#zx", r->x1); + printf(" x2: %#zx\n", r->x2); + printf(" x3: %#zx", r->x3); + printf(" x4: %#zx", r->x4); + printf(" x5: %#zx\n", r->x5); + #endif + return; + } + + if (far == 0x1de7ec7edbadc0de) { + /* KVM is mad at us */ + return; + } + + printf("In process %d (%s)\n", this_core->current_process->id, this_core->current_process->name); + printf("ESR: %#zx FAR: %#zx ELR: %#zx SPSR: %#zx\n", esr, far, elr, spsr); + aarch64_regs(r); + + uint64_t tpidr_el0; + asm volatile ("mrs %0, TPIDR_EL0" : "=r"(tpidr_el0)); + printf(" TPIDR_EL0=%#zx\n", tpidr_el0); + + + while (1); + + task_exit(1); +} + +void aarch64_fault_enter(struct regs * r) { + uint64_t esr, far, elr, spsr; + asm volatile ("mrs %0, ESR_EL1" : "=r"(esr)); + asm volatile ("mrs %0, FAR_EL1" : "=r"(far)); + asm volatile ("mrs %0, ELR_EL1" : "=r"(elr)); + asm volatile ("mrs %0, SPSR_EL1" : "=r"(spsr)); + + //printf("In process %d (%s)\n", this_core->current_process->id, this_core->current_process->name); + printf("ESR: %#zx FAR: %#zx ELR: %#zx SPSR: %#zx\n", esr, far, elr, spsr); + aarch64_regs(r); + + uint64_t tpidr_el0; + asm volatile ("mrs %0, TPIDR_EL0" : "=r"(tpidr_el0)); + printf(" TPIDR_EL0=%#zx\n", tpidr_el0); + + while (1); +} + +static void fpu_enable(void) { + uint64_t cpacr_el1; + asm volatile ("mrs %0, CPACR_EL1" : "=r"(cpacr_el1)); + cpacr_el1 |= (3 << 20) | (3 << 16); + asm volatile ("msr CPACR_EL1, %0" :: "r"(cpacr_el1)); +} + +static void timer_start(void) { + asm volatile ("msr CNTKCTL_EL1, %0" :: + "r"( + (1 << 17) /* lower rate */ + | (23 << 4) /* uh lowest bit? */ + | (1 << 1) /* enable event stream */ + )); +} + +static uint64_t time_slice_basis = 0; /**< When the last clock update happened */ +static void update_clock(void) { + uint64_t clock_ticks = arch_perf_timer() / sys_timer_freq; + uint64_t timer_ticks, timer_subticks; + update_ticks(clock_ticks, &timer_ticks, &timer_subticks); + + if (time_slice_basis + SUBSECONDS_PER_SECOND/4 <= clock_ticks) { + update_process_usage(clock_ticks - time_slice_basis, sys_timer_freq); + time_slice_basis = clock_ticks; + } + + wakeup_sleepers(timer_ticks, timer_subticks); +} + +/** + * @brief Called in a loop by kernel idle tasks. + */ +void arch_pause(void) { + /* TODO: This needs to make sure interrupt delivery is enabled. + * Though that would also require us to be turn it off + * in the first place... get around to this when we have + * literally anything set up in the GIC */ + asm volatile ("wfe"); + + update_clock(); + + asm volatile ( + ".globl _ret_from_preempt_source\n" + "_ret_from_preempt_source:" + ); + + switch_next(); +} + + + +/** + * Main kernel C entrypoint for qemu's -machine virt + * + * By this point, a 'bootstub' has already set up some + * initial page tables so the linear physical mapping + * is where we would normally expect it to be, we're + * at -2GiB, and there's some other mappings so that + * a bit of RAM is 1:1. + */ +int kmain(void) { + early_log_initialize(); + + dprintf("%s %d.%d.%d-%s %s %s\n", + __kernel_name, + __kernel_version_major, + __kernel_version_minor, + __kernel_version_lower, + __kernel_version_suffix, + __kernel_version_codename, + __kernel_arch); + + /* Initialize TPIDR_EL1 */ + arch_set_core_base((uintptr_t)&processor_local_data[0]); + + /* Set up the system timer and get an RTC time. */ + arch_clock_initialize(); + + /* Set up exception handlers early... */ + exception_handlers(); + + /* TODO load boot data from DTB? + * We want to know memory information... */ + uintptr_t ramdisk_phys_base = 0; + size_t ramdisk_size = 0; + fwcfg_load_initrd(&ramdisk_phys_base, &ramdisk_size); + + /* TODO get initial memory map data? + * Eh, we can probably just probe the existing tables... maybe... */ + mmu_init(8UL * 1024 * 1024 * 1024 /* Should be maximum valid physical address */, 0x40100000 /* Should be end of DTB */); + + /* Find the cmdline */ + dtb_locate_cmdline(); + + /* TODO Set up all the other arch-specific stuff here */ + fpu_enable(); + + generic_startup(); + + /* Initialize the framebuffer and fbterm here */ + framebuffer_initialize(); + fbterm_initialize(); + + /* TODO Start other cores here */ + + /* Ramdisk */ + ramdisk_mount(ramdisk_phys_base, ramdisk_size); + + /* TODO Start preemption source here */ + timer_start(); + + /* TODO Install drivers that may need to sleep here */ + + generic_main(); + + return 0; +} diff --git a/kernel/misc/elf64.c b/kernel/misc/elf64.c index c14832ed..9bcb2b97 100644 --- a/kernel/misc/elf64.c +++ b/kernel/misc/elf64.c @@ -302,7 +302,7 @@ int elf_exec(const char * path, fs_node_t * file, int argc, const char *const ar /* Map stack space */ uintptr_t userstack = 0x800000000000; - for (uintptr_t i = userstack - 16 * 0x400; i < userstack; i += 0x1000) { + for (uintptr_t i = userstack - 512 * 0x400; i < userstack; i += 0x1000) { union PML * page = mmu_get_page(i, MMU_GET_MAKE); mmu_frame_allocate(page, MMU_FLAG_WRITABLE); } diff --git a/kernel/misc/malloc.c b/kernel/misc/malloc.c index e1f25d58..e5c80ba9 100644 --- a/kernel/misc/malloc.c +++ b/kernel/misc/malloc.c @@ -43,7 +43,7 @@ * Defines for often-used integral values * related to our binning and paging strategy. */ -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(__aarch64__) #define NUM_BINS 10U /* Number of bins, total, under 64-bit. */ #define SMALLEST_BIN_LOG 3U /* Logarithm base two of the smallest bin: log_2(sizeof(int64)). */ #else @@ -117,10 +117,12 @@ void * __attribute__ ((malloc)) valloc(uintptr_t size) { void free(void * ptr) { spin_lock(mem_lock); +#ifndef __aarch64__ if (ptr < (void*)0xffffff0000000000) { printf("Invalid free detected (%p)\n", ptr); while (1) {}; } +#endif klfree(ptr); spin_unlock(mem_lock); } diff --git a/kernel/misc/pci.c b/kernel/misc/pci.c index bfaf4d03..a2728385 100644 --- a/kernel/misc/pci.c +++ b/kernel/misc/pci.c @@ -21,22 +21,47 @@ #include #include -/* TODO: PCI is sufficiently generic this shouldn't depend - * directly on x86-64 hardware... */ +#include + +#ifdef __x86_64__ #include +#endif + +static uintptr_t pcie_addr(uint32_t device, int field) { + return (pci_extract_bus(device) << 20) | (pci_extract_slot(device) << 15) | (pci_extract_func(device) << 12) | (field); +} /** * @brief Write to a PCI device configuration space field. */ void pci_write_field(uint32_t device, int field, int size, uint32_t value) { +#ifdef __x86_64__ outportl(PCI_ADDRESS_PORT, pci_get_addr(device, field)); outportl(PCI_VALUE_PORT, value); +#else + + /* ECAM space */ + if (size == 4) { + *(volatile uint32_t*)mmu_map_from_physical(0x3f000000 + pcie_addr(device,field)) = value; + return; + } else if (size == 2) { + *(volatile uint16_t*)mmu_map_from_physical(0x3f000000 + pcie_addr(device,field)) = value; + return; + } else if (size == 1) { + *(volatile uint8_t*)mmu_map_from_physical(0x3f000000 + pcie_addr(device,field)) = value; + return; + } + + dprintf("rejected invalid field write\n"); + +#endif } /** * @brief Read from a PCI device configuration space field. */ uint32_t pci_read_field(uint32_t device, int field, int size) { +#ifdef __x86_64__ outportl(PCI_ADDRESS_PORT, pci_get_addr(device, field)); if (size == 4) { @@ -49,6 +74,16 @@ uint32_t pci_read_field(uint32_t device, int field, int size) { uint8_t t = inportb(PCI_VALUE_PORT + (field & 3)); return t; } +#else + uintptr_t field_addr = pcie_addr(device,field); + if (size == 4) { + return *(volatile uint32_t*)mmu_map_from_physical(0x3f000000 + field_addr); + } else if (size == 2) { + return *(volatile uint16_t*)mmu_map_from_physical(0x3f000000 + field_addr); + } else if (size == 1) { + return *(volatile uint8_t*)mmu_map_from_physical(0x3f000000 + field_addr); + } +#endif return 0xFFFF; } diff --git a/kernel/misc/string.c b/kernel/misc/string.c index 416c9984..2f8e8a21 100644 --- a/kernel/misc/string.c +++ b/kernel/misc/string.c @@ -24,7 +24,7 @@ unsigned short * memsetw(unsigned short * dest, unsigned short val, int count) { return dest; } -#if 0 +#if 1 void * memcpy(void * restrict dest, const void * restrict src, size_t n) { char * d = dest; const char * s = src; diff --git a/kernel/sys/process.c b/kernel/sys/process.c index 133d2e50..dc775872 100644 --- a/kernel/sys/process.c +++ b/kernel/sys/process.c @@ -40,7 +40,13 @@ #include /* FIXME: This only needs the size of the regs struct... */ +#if defined(__x86_64__) #include +#elif defined(__aarch64__) +#include +#else +#error "no regs" +#endif tree_t * process_tree; /* Stores the parent-child process relationships; the root of this graph is 'init'. */ list_t * process_list; /* Stores all existing processes. Mostly used for sanity checking or for places where iterating over all processes is useful. */ @@ -142,6 +148,8 @@ void switch_next(void) { /* Mark the process as running and started. */ __sync_or_and_fetch(&this_core->current_process->flags, PROC_FLAG_STARTED); + asm volatile ("" ::: "memory"); + /* Jump to next */ arch_restore_context(&this_core->current_process->thread); __builtin_unreachable(); @@ -1303,7 +1311,7 @@ void task_exit(int retval) { } #define PUSH(stack, type, item) stack -= sizeof(type); \ - *((type *) stack) = item + *((volatile type *) stack) = item pid_t fork(void) { uintptr_t sp, bp; @@ -1322,11 +1330,24 @@ pid_t fork(void) { arch_syscall_return(&r, 0); PUSH(sp, struct regs, r); + new_proc->syscall_registers = (void*)sp; new_proc->thread.context.sp = sp; new_proc->thread.context.bp = bp; new_proc->thread.context.tls_base = parent->thread.context.tls_base; new_proc->thread.context.ip = (uintptr_t)&arch_resume_user; + arch_save_context(&parent->thread); + memcpy(new_proc->thread.context.saved, parent->thread.context.saved, sizeof(parent->thread.context.saved)); + + #if 0 + printf("fork(): resuming with register context\n"); + extern void aarch64_regs(struct regs *); + aarch64_regs(&r); + printf("fork(): and arch context:\n"); + extern void aarch64_context(process_t * proc); + aarch64_context(new_proc); + #endif + if (parent->flags & PROC_FLAG_IS_TASKLET) new_proc->flags |= PROC_FLAG_IS_TASKLET; make_process_ready(new_proc); return new_proc->id; @@ -1355,13 +1376,24 @@ pid_t clone(uintptr_t new_stack, uintptr_t thread_func, uintptr_t arg) { } /* different calling convention */ + #if defined(__x86_64__) r.rdi = arg; + #elif defined(__aarch64__) + r.x0 = arg; + #endif PUSH(new_stack, uintptr_t, (uintptr_t)0xFFFFB00F); PUSH(sp, struct regs, r); new_proc->syscall_registers = (void*)sp; + #if defined(__x86_64__) new_proc->syscall_registers->rsp = new_stack; new_proc->syscall_registers->rbp = new_stack; new_proc->syscall_registers->rip = thread_func; + #elif defined(__aarch64__) + new_proc->syscall_registers->user_sp = new_stack; + new_proc->syscall_registers->x29 = new_stack; + //new_proc->syscall_registers->x30 = thread_func; + new_proc->thread.context.saved[14] = thread_func; + #endif new_proc->thread.context.sp = sp; new_proc->thread.context.bp = bp; new_proc->thread.context.tls_base = this_core->current_process->thread.context.tls_base; diff --git a/kernel/sys/ptrace.c b/kernel/sys/ptrace.c index 5404270d..3bc5438b 100644 --- a/kernel/sys/ptrace.c +++ b/kernel/sys/ptrace.c @@ -20,9 +20,15 @@ #include #include #include +#include +#if defined(__x86_64__) #include -#include +#elif defined(__aarch64__) +#include +#else +#error "no regs" +#endif static void _ptrace_trace(process_t * tracer, process_t * tracee) { spin_lock(tracer->wait_lock); @@ -111,7 +117,7 @@ long ptrace_peek(pid_t pid, void * addr, void * data) { union PML * page_entry = mmu_get_page_other(tracee->thread.page_directory->directory, (uintptr_t)addr); if (!page_entry) return -EFAULT; - if (!page_entry->bits.present || !page_entry->bits.user) return -EFAULT; + if (!mmu_page_is_user_readable(page_entry)) return -EFAULT; uintptr_t mapped_address = mmu_map_to_physical(tracee->thread.page_directory->directory, (uintptr_t)addr); @@ -133,7 +139,7 @@ long ptrace_poke(pid_t pid, void * addr, void * data) { union PML * page_entry = mmu_get_page_other(tracee->thread.page_directory->directory, (uintptr_t)addr); if (!page_entry) return -EFAULT; - if (!page_entry->bits.present || !page_entry->bits.user || !page_entry->bits.writable) return -EFAULT; + if (!mmu_page_is_user_writable(page_entry)) return -EFAULT; uintptr_t mapped_address = mmu_map_to_physical(tracee->thread.page_directory->directory, (uintptr_t)addr); @@ -161,7 +167,9 @@ long ptrace_singlestep(pid_t pid, int sig) { struct regs * target = tracee->interrupt_registers ? tracee->interrupt_registers : tracee->syscall_registers; /* arch_set_singlestep? */ + #if defined(__x86_64__) target->rflags |= (1 << 8); + #endif __sync_and_and_fetch(&tracee->flags, ~(PROC_FLAG_SUSPENDED)); tracee->status = (sig << 8); diff --git a/kernel/sys/syscall.c b/kernel/sys/syscall.c index 04cdde4b..20a87197 100644 --- a/kernel/sys/syscall.c +++ b/kernel/sys/syscall.c @@ -84,8 +84,8 @@ long sys_sysfunc(long fn, char ** args) { * Misaka switched everything to raw printfs, and then also * removed most of them for cleanliness... first task would * be to reintroduce kernel fprintf() to printf to fs_nodes. */ - if (this_core->current_process->user != 0) return -EACCES; - printf("loghere: not implemented\n"); + //if (this_core->current_process->user != 0) return -EACCES; + printf("\033[32m%s\033[0m", (char*)args); return -EINVAL; case TOARU_SYS_FUNC_KDEBUG: @@ -98,6 +98,7 @@ long sys_sysfunc(long fn, char ** args) { case TOARU_SYS_FUNC_INSMOD: /* Linux has init_module as a system call? */ if (this_core->current_process->user != 0) return -EACCES; + return -EINVAL; PTR_VALIDATE(args); if (!args) return -EFAULT; PTR_VALIDATE(args[0]); @@ -191,6 +192,12 @@ long sys_exit(long exitcode) { } long sys_write(int fd, char * ptr, unsigned long len) { +#if 0 + /* Enable this to force stderr output to always be printed by the kernel. */ + if (fd == 2) { + printf_output(len,ptr); + } +#endif if (FD_CHECK(fd)) { PTRCHECK(ptr,len,MMU_PTR_NULL); fs_node_t * node = FD_ENTRY(fd); @@ -853,6 +860,7 @@ long sys_fork(void) { } long sys_clone(uintptr_t new_stack, uintptr_t thread_func, uintptr_t arg) { + printf("sys_clone() from pid=%d\n", this_core->current_process->id); if (!new_stack || !PTR_INRANGE(new_stack)) return -EINVAL; if (!thread_func || !PTR_INRANGE(thread_func)) return -EINVAL; return (int)clone(new_stack, thread_func, arg); diff --git a/kernel/vfs/procfs.c b/kernel/vfs/procfs.c index 22aab386..dfc06cdc 100644 --- a/kernel/vfs/procfs.c +++ b/kernel/vfs/procfs.c @@ -284,6 +284,24 @@ static ssize_t cpuinfo_func(fs_node_t *node, off_t offset, size_t size, uint8_t processor_local_data[i].lapic_id ); } +#elif defined(__aarch64__) + + uint64_t midr; + asm volatile ("mrs %0, MIDR_EL1" : "=r"(midr)); + + _bsize += snprintf(buf + _bsize, 1000, + "Implementer: %#x\n" + "Variant: %#x\n" + "Architecture: %#x\n" + "PartNum: %#x\n" + "Revision: %#x\n" + "\n", + (unsigned int)(midr >> 24) & 0xFF, + (unsigned int)(midr >> 20) & 0xF, + (unsigned int)(midr >> 16) & 0xF, + (unsigned int)(midr >> 4) & 0xFFFF, + (unsigned int)(midr >> 0) & 0xF + ); #endif if ((size_t)offset > _bsize) return 0; @@ -588,6 +606,7 @@ static ssize_t irq_func(fs_node_t *node, off_t offset, size_t size, uint8_t *buf free(buf); return size; } +#endif /** * Basically the same as the kdebug `pci` command. @@ -647,7 +666,6 @@ static ssize_t pci_func(fs_node_t *node, off_t offset, size_t size, uint8_t *buf free(b.buffer); return size; } -#endif static ssize_t idle_func(fs_node_t *node, off_t offset, size_t size, uint8_t *buffer) { char * buf = malloc(4096); @@ -721,10 +739,10 @@ static struct procfs_entry std_entries[] = { {-10,"loader", loader_func}, {-11,"idle", idle_func}, {-12,"kallsyms", kallsyms_func}, + {-13,"pci", pci_func}, #ifdef __x86_64__ - {-13,"irq", irq_func}, - {-14,"pat", pat_func}, - {-15,"pci", pci_func}, + {-14,"irq", irq_func}, + {-15,"pat", pat_func}, #endif }; diff --git a/kernel/video/lfbvideo.c b/kernel/video/lfbvideo.c index faa9b0cf..f3625ead 100644 --- a/kernel/video/lfbvideo.c +++ b/kernel/video/lfbvideo.c @@ -206,8 +206,17 @@ static void qemu_scan_pci(uint32_t device, uint16_t v, uint16_t d, void * extra) uintptr_t * output = extra; if ((v == 0x1234 && d == 0x1111) || (v == 0x10de && d == 0x0a20)) { + #ifndef __x86_64__ + /* we have to configure this thing ourselves */ + uintptr_t t = 0x10000008; + uintptr_t m = 0x11000000; + pci_write_field(device, PCI_BAR0, 4, t); /* video memory? */ + pci_write_field(device, PCI_BAR2, 4, m); /* MMIO? */ + pci_write_field(device, PCI_COMMAND, 2, 4|2|1); + #else uintptr_t t = pci_read_field(device, PCI_BAR0, 4); uintptr_t m = pci_read_field(device, PCI_BAR2, 4); + #endif if (m == 0) { /* Shoot. */ @@ -248,7 +257,7 @@ static void qemu_set_resolution(uint16_t x, uint16_t y) { qemu_mmio_out(QEMU_MMIO_FBWIDTH, x); qemu_mmio_out(QEMU_MMIO_FBHEIGHT, y); qemu_mmio_out(QEMU_MMIO_BPP, PREFERRED_B); - qemu_mmio_out(QEMU_MMIO_VIRTX, x * (PREFERRED_B / 8)); + qemu_mmio_out(QEMU_MMIO_VIRTX, x); qemu_mmio_out(QEMU_MMIO_VIRTY, y); qemu_mmio_out(QEMU_MMIO_ENABLED, 0x41); /* 01h: enabled, 40h: lfb */ @@ -258,7 +267,7 @@ static void qemu_set_resolution(uint16_t x, uint16_t y) { lfb_resolution_x = qemu_mmio_in(QEMU_MMIO_FBWIDTH); lfb_resolution_y = qemu_mmio_in(QEMU_MMIO_FBHEIGHT); lfb_resolution_b = qemu_mmio_in(QEMU_MMIO_BPP); - lfb_resolution_s = lfb_resolution_x * 4; + lfb_resolution_s = qemu_mmio_in(QEMU_MMIO_VIRTX) * (lfb_resolution_b / 8); } static void graphics_install_bochs(uint16_t resolution_x, uint16_t resolution_y); diff --git a/lib/confreader.c b/lib/confreader.c index f415bc72..e6572232 100644 --- a/lib/confreader.c +++ b/lib/confreader.c @@ -89,7 +89,7 @@ confreader_t * confreader_load(const char * file) { char tmp2[1024]; while (!feof(f)) { - char c = fgetc(f); + int c = fgetc(f); tmp[0] = '\0'; tmp2[0] = '\0'; if (c == ';') { diff --git a/lib/graphics.c b/lib/graphics.c index 1681c4d7..7537b3d9 100644 --- a/lib/graphics.c +++ b/lib/graphics.c @@ -15,7 +15,7 @@ #include -#ifndef NO_SSE +#if !defined(NO_SSE) && defined(__x86_64__) #include #include #endif @@ -578,7 +578,7 @@ _cleanup_sprite: return 0; } -#ifndef NO_SSE +#if !defined(NO_SSE) && defined(__x86_64__) static __m128i mask00ff; static __m128i mask0080; static __m128i mask0101; @@ -588,9 +588,9 @@ __attribute__((constructor)) static void _masks(void) { mask0080 = _mm_set1_epi16(0x0080); mask0101 = _mm_set1_epi16(0x0101); } -#endif __attribute__((__force_align_arg_pointer__)) +#endif void draw_sprite(gfx_context_t * ctx, const sprite_t * sprite, int32_t x, int32_t y) { int32_t _left = max(x, 0); @@ -603,7 +603,7 @@ void draw_sprite(gfx_context_t * ctx, const sprite_t * sprite, int32_t x, int32_ if (y + _y < _top) continue; if (y + _y > _bottom) break; if (!_is_in_clip(ctx, y + _y)) continue; -#ifdef NO_SSE +#if defined(NO_SSE) || !defined(__x86_64__) for (uint16_t _x = 0; _x < sprite->width; ++_x) { if (x + _x < _left || x + _x > _right || y + _y < _top || y + _y > _bottom) continue; @@ -770,7 +770,7 @@ static uint32_t gfx_bilinear_interpolation(const sprite_t * tex, double u, doubl static inline void apply_alpha_vector(uint32_t * pixels, size_t width, uint8_t alpha) { size_t i = 0; -#ifndef NO_SSE +#if !defined(NO_SSE) && defined(__x86_64__) __m128i alp = _mm_set_epi16(alpha,alpha,alpha,alpha,alpha,alpha,alpha,alpha); while (i + 3 < width) { __m128i p = _mm_load_si128((void*)&pixels[i]); diff --git a/lib/jpeg.c b/lib/jpeg.c index 95c191aa..3edb37f1 100644 --- a/lib/jpeg.c +++ b/lib/jpeg.c @@ -39,7 +39,7 @@ #include -#ifndef NO_SSE +#if !defined(NO_SSE) && defined(__x86_64__) #include #include #endif @@ -233,7 +233,7 @@ static float cosines[8][8] = { static float premul[8][8][8][8]= {{{{0}}}}; static void add_idc(struct idct * self, int n, int m, int coeff) { -#ifdef NO_SSE +#if defined(NO_SSE) || !defined(__x86_64__) for (int y = 0; y < 8; ++y) { for (int x = 0; x < 8; ++x) { self->base[xy_to_lin(x, y)] += premul[n][m][y][x] * coeff; diff --git a/libc/arch/aarch64/bad.c b/libc/arch/aarch64/bad.c new file mode 100644 index 00000000..6332a805 --- /dev/null +++ b/libc/arch/aarch64/bad.c @@ -0,0 +1,17 @@ +/* bad math */ + +double sqrt(double x) { + return 0.0; +} + +double atan2(double y, double x) { + return 0.0; +} + +double pow(double x, double y) { + return 0.0; +} + +double fmod(double x, double y) { + return 0.0; +} diff --git a/libc/arch/aarch64/crt0.S b/libc/arch/aarch64/crt0.S new file mode 100644 index 00000000..46b0c89a --- /dev/null +++ b/libc/arch/aarch64/crt0.S @@ -0,0 +1,9 @@ +.global _start +.type _start, %function + +_start: + mov x4, x0 + adr x0, main + mov x3, x0 + mov x0, x4 + bl pre_main diff --git a/libc/arch/aarch64/crti.S b/libc/arch/aarch64/crti.S new file mode 100644 index 00000000..bbade8c5 --- /dev/null +++ b/libc/arch/aarch64/crti.S @@ -0,0 +1,12 @@ +.global _init +.section .init +_init: + stp x29,x30,[sp,-16]! + mov x29,sp + +.global _fini +.section .fini +_fini: + stp x29,x30,[sp,-16]! + mov x29,sp + diff --git a/libc/arch/aarch64/crtn.S b/libc/arch/aarch64/crtn.S new file mode 100644 index 00000000..10a70336 --- /dev/null +++ b/libc/arch/aarch64/crtn.S @@ -0,0 +1,8 @@ +.section .init + ldp x29,x30,[sp],#16 + ret + +.section .fini + ldp x29,x30,[sp],#16 + ret + diff --git a/libc/arch/aarch64/memcpy.c b/libc/arch/aarch64/memcpy.c new file mode 100644 index 00000000..fdcbc4ec --- /dev/null +++ b/libc/arch/aarch64/memcpy.c @@ -0,0 +1,11 @@ +#include +#include + +void * memcpy(void * restrict dest, const void * restrict src, size_t n) { + char * d = dest; + const char * s = src; + for (; n > 0; n--) { + *d++ = *s++; + } + return dest; +} diff --git a/libc/arch/aarch64/memset.c b/libc/arch/aarch64/memset.c new file mode 100644 index 00000000..c3bf4de8 --- /dev/null +++ b/libc/arch/aarch64/memset.c @@ -0,0 +1,9 @@ +#include + +void * memset(void * dest, int c, size_t n) { + size_t i = 0; + for ( ; i < n; ++i ) { + ((char *)dest)[i] = c; + } + return dest; +} diff --git a/libc/pthread/pthread.c b/libc/pthread/pthread.c index 591eca1d..a3134c1a 100644 --- a/libc/pthread/pthread.c +++ b/libc/pthread/pthread.c @@ -43,8 +43,14 @@ void * __tls_get_addr(void* input) { void __make_tls(void) { char * tlsSpace = valloc(4096); memset(tlsSpace, 0x0, 4096); +#if defined(__x86_64__) + /* self-pointer at end */ char ** tlsSelf = (char **)(tlsSpace + 4096 - sizeof(char *)); +#elif defined(__aarch64__) + /* self-pointer start? */ + char ** tlsSelf = (char **)(tlsSpace); *tlsSelf = (char*)tlsSelf; +#endif sysfunc(TOARU_SYS_FUNC_SETGSBASE, (char*[]){(char*)tlsSelf}); } @@ -56,7 +62,7 @@ void * __thread_start(void * thread) { } int pthread_create(pthread_t * thread, pthread_attr_t * attr, void *(*start_routine)(void *), void * arg) { - char * stack = malloc(PTHREAD_STACK_SIZE); + char * stack = valloc(PTHREAD_STACK_SIZE); uintptr_t stack_top = (uintptr_t)stack + PTHREAD_STACK_SIZE; thread->stack = stack; diff --git a/libc/stdlib/malloc.c b/libc/stdlib/malloc.c index 7437a598..0a45598f 100644 --- a/libc/stdlib/malloc.c +++ b/libc/stdlib/malloc.c @@ -116,7 +116,7 @@ * Defines for often-used integral values * related to our binning and paging strategy. */ -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(__aarch64__) #define NUM_BINS 10U /* Number of bins, total, under 64-bit. */ #define SMALLEST_BIN_LOG 3U /* Logarithm base two of the smallest bin: log_2(sizeof(int32)). */ #else diff --git a/libc/wchar/wcstol.c b/libc/wchar/wcstol.c index b3d805c8..326b5cdf 100644 --- a/libc/wchar/wcstol.c +++ b/libc/wchar/wcstol.c @@ -7,11 +7,11 @@ static int is_valid(int base, wchar_t c) { if (c < '0') return 0; if (base <= 10) { - return c < ('0' + base); + return c < ('0' + (wchar_t)base); } - if (c >= 'a' && c < 'a' + (base - 10)) return 1; - if (c >= 'A' && c < 'A' + (base - 10)) return 1; + if (c >= 'a' && c < 'a' + ((wchar_t)base - 10)) return 1; + if (c >= 'A' && c < 'A' + ((wchar_t)base - 10)) return 1; if (c >= '0' && c <= '9') return 1; return 0; } diff --git a/linker/link.ld b/linker/link.ld index b3bb33e0..965a5771 100644 --- a/linker/link.ld +++ b/linker/link.ld @@ -33,6 +33,7 @@ SECTIONS .init_array : { PROVIDE_HIDDEN (__init_array_start = .); + *(.init_array); PROVIDE_HIDDEN (__init_array_end = .); } diff --git a/linker/linker.c b/linker/linker.c index 6cf556bb..fcb18e9c 100644 --- a/linker/linker.c +++ b/linker/linker.c @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -78,7 +79,7 @@ static hashmap_t * dumb_symbol_table; static hashmap_t * glob_dat; static hashmap_t * objects_map; static hashmap_t * tls_map; -static size_t current_tls_offset = 0; +static size_t current_tls_offset = 16; /* Used for dlerror */ static char * last_error = NULL; @@ -385,19 +386,33 @@ static int object_postload(elf_t * object) { } /* Whether symbol addresses is needed for a relocation type */ -static int need_symbol_for_type(unsigned char type) { +static int need_symbol_for_type(unsigned int type) { +#ifdef __x86_64__ switch(type) { - case 1: - case 2: - case 5: - case 6: - case 7: - case 14: + case R_X86_64_64: + case R_X86_64_PC32: + case R_X86_64_COPY: + case R_X86_64_GLOB_DAT: + case R_X86_64_JUMP_SLOT: + case R_X86_64_8: case R_X86_64_TPOFF64: return 1; default: return 0; } +#else + switch(type) { + case 1024: + case 1025: + case 1026: + case 1030: + case 257: + return 1; + default: + return 0; + } + +#endif } /* Apply ELF relocations */ @@ -411,10 +426,15 @@ static int object_relocate(elf_t * object) { char * symname = (char *)((uintptr_t)object->dyn_string_table + table->st_name); /* If we haven't added this symbol to our symbol table, do so now. */ - if (!hashmap_has(dumb_symbol_table, symname)) { - if (table->st_shndx) { + if (table->st_shndx) { + if (!hashmap_has(dumb_symbol_table, symname)) { hashmap_set(dumb_symbol_table, symname, (void*)(table->st_value + object->base)); + table->st_value = table->st_value + object->base; + } else { + table->st_value = hashmap_get(dumb_symbol_table, symname); } + } else { + table->st_value = table->st_value + object->base; } table++; @@ -441,12 +461,11 @@ static int object_relocate(elf_t * object) { /* If we need symbol for this, get it. */ char * symname = NULL; - uintptr_t x = sym->st_value + object->base; + uintptr_t x = sym->st_value; if (need_symbol_for_type(type) || (type == 5)) { symname = (char *)((uintptr_t)object->dyn_string_table + sym->st_name); if (symname && hashmap_has(dumb_symbol_table, symname)) { x = (uintptr_t)hashmap_get(dumb_symbol_table, symname); - sym->st_value = x; } else { /* This isn't fatal, but do log a message if debugging is enabled. */ TRACE_LD("Symbol not found: %s", symname); @@ -456,6 +475,7 @@ static int object_relocate(elf_t * object) { /* Relocations, symbol lookups, etc. */ switch (type) { +#if defined(__x86_64__) case R_X86_64_GLOB_DAT: /* 6 */ if (symname && hashmap_has(glob_dat, symname)) { x = (uintptr_t)hashmap_get(glob_dat, symname); @@ -490,6 +510,49 @@ static int object_relocate(elf_t * object) { } memcpy((void *)(table->r_offset + object->base), &x, sizeof(uintptr_t)); break; +#elif defined(__aarch64__) + case 1024: /* COPY */ + memcpy((void *)(table->r_offset + object->base), (void *)x, sym->st_size); + break; + case 1025: /* GLOB_DAT */ + if (symname && hashmap_has(glob_dat, symname)) { + x = (uintptr_t)hashmap_get(glob_dat, symname); + } /* fallthrough */ + case 1026: /* JUMP_SLOT */ + memcpy((void*)(table->r_offset + object->base), &x, sizeof(uintptr_t)); + break; + case 1027: /* RELATIVE */ + x = object->base; + x += table->r_addend; + memcpy((void*)(table->r_offset + object->base), &x, sizeof(uintptr_t)); + break; + case 257: /* ABS64 */ + x += table->r_addend; + memcpy((void*)(table->r_offset + object->base), &x, sizeof(uintptr_t)); + break; + case 1030: /* TLS_TPREL64 TPREL(S+A) */ + x += *((ssize_t *)(table->r_offset + object->base)); /* A */ + if (!hashmap_has(tls_map, symname)) { + if (!sym->st_size) { + fprintf(stderr, "Haven't placed %s in static TLS yet but don't know its size?\n", symname); + } + #if 0 + current_tls_offset += sym->st_size; /* TODO alignment restrictions */ + hashmap_set(tls_map, symname, (void*)(current_tls_offset)); + x -= current_tls_offset; + #else + x += current_tls_offset; + hashmap_set(tls_map, symname, (void*)(current_tls_offset)); + current_tls_offset += sym->st_size; /* TODO alignment restrictions */ + #endif + } else { + x += (size_t)hashmap_get(tls_map, symname); + } + memcpy((void *)(table->r_offset + object->base), &x, sizeof(uintptr_t)); + break; +#else +# error "unsupported" +#endif #if 0 case 6: /* GLOB_DAT */ if (symname && hashmap_has(glob_dat, symname)) { @@ -531,7 +594,14 @@ static int object_relocate(elf_t * object) { break; #endif default: + { + char msg[200]; + snprintf(msg, 200, "Unimplemented relocation (%d) requested, bailing.\n", type); + sysfunc(TOARU_SYS_FUNC_LOGHERE, (char**)msg); + exit(1); + } TRACE_LD("Unknown relocation type: %d", type); + break; } table++; @@ -555,7 +625,13 @@ static void object_find_copy_relocations(elf_t * object) { Elf64_Rel * table = (Elf64_Rel *)(shdr.sh_addr + object->base); while ((uintptr_t)table - ((uintptr_t)shdr.sh_addr + object->base) < shdr.sh_size) { unsigned int type = ELF64_R_TYPE(table->r_info); +#if defined(__x86_64__) if (type == R_X86_64_COPY) { +#elif defined(__aarch64__) + if (type == R_AARCH64_COPY) { +#else +# error "Unsupported" +#endif unsigned int symbol = ELF64_R_SYM(table->r_info); Elf64_Sym * sym = &object->dyn_symbol_table[symbol]; char * symname = (char *)((uintptr_t)object->dyn_string_table + sym->st_name); @@ -568,7 +644,13 @@ static void object_find_copy_relocations(elf_t * object) { Elf64_Rela * table = (Elf64_Rela *)(shdr.sh_addr + object->base); while ((uintptr_t)table - ((uintptr_t)shdr.sh_addr + object->base) < shdr.sh_size) { unsigned int type = ELF64_R_TYPE(table->r_info); +#if defined(__x86_64__) if (type == R_X86_64_COPY) { +#elif defined(__aarch64__) + if (type == R_AARCH64_COPY) { +#else +# error "Unsupported" +#endif unsigned int symbol = ELF64_R_SYM(table->r_info); Elf64_Sym * sym = &object->dyn_symbol_table[symbol]; char * symname = (char *)((uintptr_t)object->dyn_string_table + sym->st_name); @@ -592,7 +674,7 @@ static void * object_find_symbol(elf_t * object, const char * symbol_name) { size_t i = 0; while (i < object->dyn_symbol_table_size) { if (!strcmp(symbol_name, (char *)((uintptr_t)object->dyn_string_table + table->st_name))) { - return (void *)(table->st_value + object->base); + return (void *)(table->st_value); } table++; i++; @@ -624,7 +706,7 @@ static void * do_actual_load(const char * filename, elf_t * lib, int flags) { * This is where we should really be loading things into COW * but we don't have the functionality available. */ - uintptr_t load_addr = (uintptr_t)malloc(lib_size); + uintptr_t load_addr = (uintptr_t)valloc(lib_size); object_load(lib, load_addr); /* Perform cleanup steps */ diff --git a/modules/vmware.c b/modules/vmware.c index 572c11f7..67241051 100644 --- a/modules/vmware.c +++ b/modules/vmware.c @@ -33,6 +33,8 @@ #include #include #include + +#ifdef __x86_64__ #include #define VMWARE_MAGIC 0x564D5868 /* hXMV */ @@ -520,3 +522,4 @@ struct Module metadata = { .fini = fini, }; +#endif diff --git a/util/binutils-gdb b/util/binutils-gdb index 38d769c5..facad00e 160000 --- a/util/binutils-gdb +++ b/util/binutils-gdb @@ -1 +1 @@ -Subproject commit 38d769c5da44843ea18ef15d8d62af9380f26aea +Subproject commit facad00e10bb66e7647be3540a33c978721803cb diff --git a/util/gcc b/util/gcc index 29d232ff..66860610 160000 --- a/util/gcc +++ b/util/gcc @@ -1 +1 @@ -Subproject commit 29d232ff766aae6a54ecad24b5eae287eccf5c50 +Subproject commit 66860610d488c9501b3f0013d599df902fb31bf5