Pull in Kuroko (as a submodule, but with build integration)

.github/workflows/build.yml

@@ -8,6 +8,7 @@ jobs:
         uses: actions/checkout@v2
         with:
             fetch-depth: 0
+            submodules: true
       - name: Pull Builder Image
         run: docker pull toaruos/build-tools:1.8.x
       - name: Run Builder

.gitignore

@@ -71,3 +71,6 @@
 
 # SDL
 /base/usr/include/SDL
+
+# Kuroko build files
+/base/usr/share/kuroko

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "kuroko"]
+	path = kuroko
+	url = git@github.com:klange/kuroko

Makefile

@@ -38,7 +38,7 @@ LC=base/lib/libc.so
 ##
 #  APPS      = C sources from apps/
 #  APPS_X    = binaries
-#  APPS_Y    = generated makefiles for binaries (except init)
+#  APPS_Y    = generated makefiles for binaries
 #  APPS_SH   = shell scripts to copy to base/bin/ and mark executable
 #  APPS_SH_X = destinations for shell scripts
 APPS=$(patsubst apps/%.c,%,$(wildcard apps/*.c))
@@ -63,7 +63,7 @@ tags: $(SOURCE_FILES)
 
 ##
 # Files that must be present in the ramdisk (apps, libraries)
-RAMDISK_FILES= ${APPS_X} ${APPS_SH_X} ${LIBS_X} base/lib/ld.so base/lib/libm.so
+RAMDISK_FILES= ${APPS_X} ${APPS_SH_X} ${LIBS_X} base/lib/ld.so base/lib/libm.so ${KUROKO_FILES}
 
 # Kernel / module flags
 
@@ -162,13 +162,15 @@ base/cdrom:
 	mkdir -p $@
 base/var:
 	mkdir -p $@
+base/usr/share/kuroko:
+	mkdir -p $@
 fatbase/efi/boot:
 	mkdir -p $@
 cdrom:
 	mkdir -p $@
 .make:
 	mkdir -p .make
-dirs: base/dev base/tmp base/proc base/bin base/lib base/cdrom cdrom base/var fatbase/efi/boot .make
+dirs: base/dev base/tmp base/proc base/bin base/lib base/cdrom base/usr/share/kuroko cdrom base/var fatbase/efi/boot .make
 
 # C Library
 
@@ -192,6 +194,34 @@ base/lib/libc.so: ${LIBC_OBJS} | dirs crts
 base/lib/libm.so: util/lm.c | dirs crts
 	$(CC) -nodefaultlibs -o $@ $(CFLAGS) -shared -fPIC $^ -lgcc
 
+KUROKO_OBJS=$(patsubst %.c, %.o, $(filter-out kuroko/rline.c kuroko/kuroko.c, $(sort $(wildcard kuroko/*.c)))) kuroko/builtins.o
+kuroko/builtins.c: kuroko/builtins.krk
+	echo "const char _builtins_src[] = {\n" > $@
+	hexdump -v -e '16/1 "0x%02x,"' -e '"\n"' $< | sed s'/0x  ,//g' >> $@
+	echo "0x00 };" >> $@
+kuroko/%.o: kuroko/%.c
+	$(CC) $(CFLAGS) -fPIC -c -o $@ $^
+
+KUROKO_CMODS=$(patsubst kuroko/src/%.c,%,$(wildcard kuroko/src/*.c))
+KUROKO_CMODS_X=$(foreach lib,$(KUROKO_CMODS),base/usr/share/kuroko/$(lib).so)
+KUROKO_CMODS_Y=$(foreach lib,$(KUROKO_CMODS),.make/$(lib).kmak)
+KUROKO_KRK_MODS=$(patsubst kuroko/modules/%.krk,base/usr/share/kuroko/%.krk,$(wildcard kuroko/modules/*.krk))
+
+KUROKO_FILES=$(KUROKO_CMODS_X) $(KUROKO_KRK_MODS) base/lib/libkuroko.so
+
+base/usr/share/kuroko/%.krk: kuroko/modules/%.krk
+	cp $< $@
+
+.make/%.kmak: kuroko/src/%.c util/auto-dep.py | dirs
+	util/auto-dep.py --makekurokomod $< > $@
+
+ifeq (,$(findstring clean,$(MAKECMDGOALS)))
+-include ${KUROKO_CMODS_Y}
+endif
+
+base/lib/libkuroko.so: $(KUROKO_OBJS)  | dirs crts ${LC}
+	$(CC) $(CFLAGS) -shared -fPIC -o $@ $^ -lgcc
+
 # Userspace Linker/Loader
 
 base/lib/ld.so: linker/linker.c base/lib/libc.a | dirs
@@ -305,6 +335,7 @@ clean:
 	rm -f base/lib/crt*.o
 	rm -f ${MODULES}
 	rm -f ${APPS_Y} ${LIBS_Y} ${EXT_LIBS_Y}
+	rm -f ${KUROKO_FILES}
 
 ifneq (,$(findstring Microsoft,$(shell uname -r)))
   QEMU_ARGS=-serial mon:stdio -m 1G -rtc base=localtime -vnc :0

apps/kuroko.c

@@ -0,0 +1,159 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <toaru/rline.h>
+#include <toaru/rline_exp.h>
+#include <kuroko.h>
+#include "chunk.h"
+#include "debug.h"
+#include "vm.h"
+#include "memory.h"
+
+
+int main(int argc, char * argv[]) {
+	int flags = 0;
+	int opt;
+	while ((opt = getopt(argc, argv, "tdgs")) != -1) {
+		switch (opt) {
+			case 't':
+				flags |= KRK_ENABLE_TRACING;
+				break;
+			case 'd':
+				flags |= KRK_ENABLE_DEBUGGING;
+				break;
+			case 's':
+				flags |= KRK_ENABLE_SCAN_TRACING;
+				break;
+			case 'g':
+				flags |= KRK_ENABLE_STRESS_GC;
+				break;
+		}
+	}
+
+	krk_initVM(flags);
+
+	KrkValue result = INTEGER_VAL(0);
+
+	if (optind == argc) {
+		/* Run the repl */
+		int exit = 0;
+
+		rline_exit_string="";
+		rline_exp_set_syntax("krk");
+		//rline_exp_set_shell_commands(shell_commands, shell_commands_len);
+		//rline_exp_set_tab_complete_func(tab_complete_func);
+
+		while (!exit) {
+			size_t lineCapacity = 8;
+			size_t lineCount = 0;
+			char ** lines = ALLOCATE(char *, lineCapacity);
+			size_t totalData = 0;
+			int valid = 1;
+			char * allData = NULL;
+			int inBlock = 0;
+			int blockWidth = 0;
+
+			rline_exp_set_prompts(">>> ", "", 4, 0);
+
+			while (1) {
+				/* This would be a nice place for line editing */
+				char buf[4096] = {0};
+
+				if (inBlock) {
+					rline_exp_set_prompts("  > ", "", 4, 0);
+					rline_preload = malloc(blockWidth + 1);
+					for (int i = 0; i < blockWidth; ++i) {
+						rline_preload[i] = ' ';
+					}
+					rline_preload[blockWidth] = '\0';
+				}
+
+				rline_scroll = 0;
+				if (rline_experimental(buf, 4096) == 0) {
+					valid = 0;
+					exit = 1;
+					break;
+				}
+				if (buf[strlen(buf)-1] != '\n') {
+					fprintf(stderr, "Expected end of line in repl input. Did you ^D early?\n");
+					valid = 0;
+					break;
+				}
+				if (lineCapacity < lineCount + 1) {
+					size_t old = lineCapacity;
+					lineCapacity = GROW_CAPACITY(old);
+					lines = GROW_ARRAY(char *,lines,old,lineCapacity);
+				}
+
+				int i = lineCount++;
+				lines[i] = strdup(buf);
+
+				size_t lineLength = strlen(lines[i]);
+				totalData += lineLength;
+
+				int is_spaces = 1;
+				int count_spaces = 0;
+				for (size_t j = 0; j < lineLength; ++j) {
+					if (lines[i][j] != ' ' && lines[i][j] != '\n') {
+						is_spaces = 0;
+						break;
+					}
+					count_spaces += 1;
+				}
+
+				if (lineLength > 2 && lines[i][lineLength-2] == ':') {
+					inBlock = 1;
+					blockWidth = count_spaces + 4;
+					continue;
+				} else if (inBlock && lineLength != 1) {
+					if (is_spaces) {
+						free(lines[i]);
+						totalData -= lineLength;
+						lineCount--;
+						break;
+					}
+					blockWidth = count_spaces;
+					continue;
+				}
+
+				break;
+			}
+
+			if (valid) {
+				allData = malloc(totalData + 1);
+				allData[0] = '\0';
+			}
+			for (size_t i = 0; i < lineCount; ++i) {
+				if (valid) strcat(allData, lines[i]);
+				rline_history_insert(strdup(lines[i]));
+				free(lines[i]);
+			}
+			FREE_ARRAY(char *, lines, lineCapacity);
+
+			if (valid) {
+				KrkValue result = krk_interpret(allData, 0, "<module>","<stdin>");
+				if (!IS_NONE(result)) {
+					fprintf(stdout, " \033[1;30m=> ");
+					krk_printValue(stdout, result);
+					fprintf(stdout, "\033[0m\n");
+				}
+			}
+
+		}
+	} else {
+
+		for (int i = optind; i < argc; ++i) {
+			KrkValue out = krk_runfile(argv[i],0,"<module>",argv[i]);
+			if (i + 1 == argc) result = out;
+		}
+	}
+
+	krk_freeVM();
+
+	if (IS_INTEGER(result)) return AS_INTEGER(result);
+
+	return 0;
+}

base/usr/include/toaru/rline_exp.h

@@ -10,5 +10,6 @@ extern int rline_exp_set_prompts(char * left, char * right, int left_width, int
 extern int rline_exp_set_shell_commands(char ** cmds, int len);
 extern int rline_exp_set_tab_complete_func(rline_callback_t func);
 extern int rline_exp_set_syntax(char * name);
+extern char * rline_preload;
 
 _End_C_Header

kuroko

@@ -0,0 +1 @@
+Subproject commit a9fffce795bef002469b6c1cb17aef7962a785bd

lib/rline_exp.c

@@ -726,6 +726,95 @@ int syn_py_calculate(struct syntax_state * state) {
 	return -1;
 }
 
+void paint_single_string(struct syntax_state * state) {
+	/* Assumes you came in from a check of charat() == '\'' */
+	paint(1, FLAG_NUMERAL);
+	while (charat() != -1) {
+		if (charat() == '\\' && nextchar() == '\'') {
+			paint(2, FLAG_ESCAPE);
+		} else if (charat() == '\'') {
+			paint(1, FLAG_NUMERAL);
+			return;
+		} else if (charat() == '\\') {
+			paint(2, FLAG_ESCAPE);
+		} else {
+			paint(1, FLAG_NUMERAL);
+		}
+	}
+}
+
+
+char * syn_krk_keywords[] = {
+	"and","class","def","else","export","for","if","in","import",
+	"let","not","or","print","return","while","try","except","raise",
+	"continue","break",
+	NULL
+};
+
+char * syn_krk_types[] = {
+	/* built-in functions */
+	"self", "super", /* implicit in a class method */
+	"len", "str", "int", "float", "dir", "repr", /* global functions from __builtins__ */
+	"list","dict","range", /* builtin classes */
+	"object","exception","isinstance","type",
+	NULL
+};
+
+char * syn_krk_special[] = {
+	"True","False","None",
+	NULL
+};
+
+int paint_krk_numeral(struct syntax_state * state) {
+	if (charat() == '0' && (nextchar() == 'x' || nextchar() == 'X')) {
+		paint(2, FLAG_NUMERAL);
+		while (isxdigit(charat())) paint(1, FLAG_NUMERAL);
+	} else if (charat() == '0' && (nextchar() == 'o' || nextchar() == 'O')) {
+		paint(2, FLAG_NUMERAL);
+		while (charat() >= '0' && charat() <= '7') paint(1, FLAG_NUMERAL);
+	} else if (charat() == '0' && (nextchar() == 'b' || nextchar() == 'B')) {
+		paint(2, FLAG_NUMERAL);
+		while (charat() == '0' || charat() == '1') paint(1, FLAG_NUMERAL);
+	} else {
+		while (isdigit(charat())) paint(1, FLAG_NUMERAL);
+		if (charat() == '.') {
+			paint(1, FLAG_NUMERAL);
+			while (isdigit(charat())) paint(1, FLAG_NUMERAL);
+		}
+	}
+	return 0;
+}
+
+int syn_krk_calculate(struct syntax_state * state) {
+	switch (state->state) {
+		case -1:
+		case 0:
+			if (charat() == '#') {
+				paint_comment(state);
+			} else if (charat() == '"') {
+				paint_simple_string(state);
+				return 0;
+			} else if (charat() == '\'') {
+				paint_single_string(state);
+				return 0;
+			} else if (find_keywords(state, syn_krk_keywords, FLAG_KEYWORD, c_keyword_qualifier)) {
+				return 0;
+			} else if (lastchar() != '.' && find_keywords(state, syn_krk_types, FLAG_TYPE, c_keyword_qualifier)) {
+				return 0;
+			} else if (find_keywords(state, syn_krk_special, FLAG_NUMERAL, c_keyword_qualifier)) {
+				return 0;
+			} else if (!c_keyword_qualifier(lastchar()) && isdigit(charat())) {
+				paint_krk_numeral(state);
+				return 0;
+			} else if (charat() != -1) {
+				skip();
+				return 0;
+			}
+			break;
+	}
+	return -1;
+}
+
 /**
  * Convert syntax hilighting flag to color code
  */
@@ -767,6 +856,7 @@ struct syntax_definition {
 } syntaxes[] = {
 	{"esh",syn_esh_calculate},
 	{"python",syn_py_calculate},
+	{"krk",syn_krk_calculate},
 	{NULL, NULL},
 };
 
@@ -1579,6 +1669,8 @@ static void call_rline_func(rline_callback_t func, rline_context_t * context) {
 	place_cursor_actual();
 }
 
+char * rline_preload = NULL;
+
 /**
  * Perform actual interactive line editing.
  *
@@ -1600,6 +1692,17 @@ static int read_line(void) {
 	for (int i = 0; i < full_width - 1; ++i) {
 		fprintf(stdout, " ");
 	}
+
+	if (rline_preload) {
+		char * c = rline_preload;
+		while (*c) {
+			insert_char(*c);
+			c++;
+		}
+		free(rline_preload);
+		rline_preload = NULL;
+	}
+
 	render_line();
 	place_cursor_actual();
 

libc/stdio/printf.c

@@ -217,6 +217,7 @@ int xvasprintf(char * buf, const char * fmt, va_list args) {
 			case 'p':
 				if (!arg_width) {
 					arg_width = 8;
+					alt = 1;
 				}
 			case 'x': /* Hexadecimal number */
 				if (alt) {

util/auto-dep.py

@@ -36,6 +36,8 @@ class Classifier(object):
         '<toaru/menu.h>':        (None, '-ltoaru_menu',        ['<toaru/sdf.h>', '<toaru/yutani.h>', '<toaru/icon_cache.h>', '<toaru/graphics.h>', '<toaru/hashmap.h>']),
         '<toaru/textregion.h>':  (None, '-ltoaru_textregion',  ['<toaru/sdf.h>', '<toaru/yutani.h>','<toaru/graphics.h>', '<toaru/hashmap.h>']),
         '<toaru/button.h>':      (None, '-ltoaru_button',      ['<toaru/graphics.h>','<toaru/sdf.h>', '<toaru/icon_cache.h>']),
+        # Kuroko
+        '<kuroko.h>':            ('../../../kuroko', '-lkuroko', []),
         # OPTIONAL third-party libraries, for extensions / ports
         '<ft2build.h>':        ('freetype2', '-lfreetype', []),
         '<pixman.h>':          ('pixman-1', '-lpixman-1', []),
@@ -101,6 +103,8 @@ def todep(name):
         name = name.replace("-l","",1)
         if name.startswith('toaru'):
             return (True, "%s/lib%s.so" % ('base/lib', name))
+        elif name.startswith('kuroko'):
+            return (True, "%s/lib%s.so" % ('base/lib', name))
         else:
             return (True, "%s/lib%s.so" % ('base/usr/lib', name))
     else:
@@ -173,4 +177,15 @@ if __name__ == "__main__":
             libraries=" ".join([x for x in _libs]),
             includes=" ".join([x for x in c.includes if x is not None])
             ))
+    elif command == "--makekurokomod":
+        libname = os.path.basename(filename).replace(".c","")
+        _libs = [x for x in c.libs if not x.startswith('-ltoaru_') or x.replace("-ltoaru_","") != libname]
+        print("base/usr/share/kuroko/{lib}.so: {source} {headers} util/auto-dep.py | {libraryfiles} $(LC)\n\t$(CC) $(CFLAGS) {includes} -shared -fPIC -o $@ $< {libraries}".format(
+            lib=libname,
+            source=filename,
+            headers=" ".join([toheader(x) for x in c.libs]),
+            libraryfiles=" ".join([todep(x)[1] for x in _libs]),
+            libraries=" ".join([x for x in _libs]),
+            includes=" ".join([x for x in c.includes if x is not None])
+            ))