From 609f9d86de4d798b593fa1206bbc8f3efa3bbe9b Mon Sep 17 00:00:00 2001 From: Shreyas Lad Date: Tue, 14 Apr 2020 23:48:35 -0700 Subject: [PATCH] EXT2 and VFS support (#5) * initial header file * finished inode structure * finished inode permissions * updated makefile * working ext2fs build * found superblock * bgdt is ded * parsed bgdt * directory entry structure * parsed inodes * parsed root inode * found directory entries * added ext2 support * fs abstraction support * cached size in FILE * hm * removed debug statements * added to is_ext2() args * fixed read errors * comply with standards * fixed things * more fixes * more makefile fixes * even more fixes --- .gitignore | 1 + Makefile | 23 +++- src/fs/echfs.c | 2 +- src/fs/ext2fs.c | 338 ++++++++++++++++++++++++++++++++++++++++++++++++ src/fs/ext2fs.h | 23 ++++ src/fs/file.c | 17 ++- src/lib/blib.c | 6 +- src/main.c | 2 +- test/test.asm | 2 +- 9 files changed, 406 insertions(+), 8 deletions(-) create mode 100644 src/fs/ext2fs.c create mode 100644 src/fs/ext2fs.h diff --git a/.gitignore b/.gitignore index b0018fe9..0065763d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /**/*.img /bochsout.txt /bx_enh_dbg.ini +.vscode !/qloader2.bin diff --git a/Makefile b/Makefile index 37135341..89405e6a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all clean test +.PHONY: all clean echfs-test ext2-test all: $(MAKE) -C src all @@ -6,7 +6,7 @@ all: clean: $(MAKE) -C src clean -test: all +echfs-test: all $(MAKE) -C test rm -f test.img dd if=/dev/zero bs=1M count=0 seek=64 of=test.img @@ -17,3 +17,22 @@ test: all echfs-utils -m -p0 test.img import test/qloader2.cfg qloader2.cfg ./qloader2-install src/qloader2.bin test.img qemu-system-x86_64 -hda test.img -monitor stdio + +ext2-test: + $(MAKE) -C test + rm -rf test.img test_image/ + mkdir test_image + dd if=/dev/zero bs=1M count=0 seek=64 of=test.img + parted -s test.img mklabel msdos + parted -s test.img mkpart primary 1 100% + sudo losetup -Pf --show test.img > loopback_dev + sudo mkfs.ext2 `cat loopback_dev`p1 + sudo mount `cat loopback_dev`p1 test_image + sudo cp test/test.elf test_image + sudo cp test/qloader2.cfg test_image + sync + sudo umount test_image/ + sudo losetup -d `cat loopback_dev` + rm -rf test_image loopback_dev + ./qloader2-install src/qloader2.bin test.img + qemu-system-x86_64 -hda test.img -monitor stdio \ No newline at end of file diff --git a/src/fs/echfs.c b/src/fs/echfs.c index 2998fbf7..6a077a99 100644 --- a/src/fs/echfs.c +++ b/src/fs/echfs.c @@ -101,4 +101,4 @@ int echfs_open(struct echfs_file_handle *ret, int disk, int partition, const cha print("echfs: file %s not found\n", filename); return -1; -} +} \ No newline at end of file diff --git a/src/fs/ext2fs.c b/src/fs/ext2fs.c new file mode 100644 index 00000000..462bdc8b --- /dev/null +++ b/src/fs/ext2fs.c @@ -0,0 +1,338 @@ +#include + +// it willl most likely be 4096 bytes +#define EXT2_BLOCK_SIZE 4096 + +/* EXT2 Filesystem States */ +#define EXT2_FS_CLEAN 1 +#define EXT2_FS_ERRORS 2 + +/* EXT2 Error Handling */ +#define EXT2_ERR_IGNORE 1 +#define EXT2_ERR_REMOUNT_AS_READ 2 +#define EXT2_ERR_PANIC 3 + +/* EXT2 Creator OS IDs */ +#define EXT2_LINUX 0 +#define EXT2_GNU_HURD 1 +#define EXT2_MASIX 2 +#define EXT2_FREEBSD 3 +#define EXT2_BSD_DERIVATIVE 4 + +/* EXT2 Optional Feature Flags */ +#define EXT2_OPT_PREALLOC 0x0001 // Prealloc x number of blocks (superblock byte 205) +#define EXT2_OPT_AFS_INODES 0x0002 // AFS server inodes exist +#define EXT2_OPT_JOURNAL 0x0004 // FS has a journal (ext3) +#define EXT2_OPT_INODE_EXT_ATTR 0x0008 // Inodes have extended attributes +#define EXT2_OPT_FS_RESIZE 0x0010 // FS can resize itself for larger partitions +#define EXT2_OPT_DIR_HASH_IDX 0x0020 // Directories use a hash index + +/* EXT2 Required Feature Flags */ +#define EXT2_REQ_COMPRESSION 0x0001 // the FS uses compression +#define EXT2_REQ_DIR_TYPE_FIELD 0x0002 // Dir entries contain a type field +#define EXT2_REQ_JOURNAL_REPLAY 0x0004 // FS needs to replay its journal +#define EXT2_REQ_USE_JOURNAL 0x0008 // FS uses a journal device + +/* EXT2 Read-Only Feature Flags */ +#define EXT2_SPARSE 0x0001 // Sparse superblocks and group descriptor tables +#define EXT2_FS_LONG 0x0002 // FS uses 64 bit file sizes +#define EXT2_BTREE 0x0004 // Directory contents are stored in a Binary Tree + +// https://wiki.osdev.org/Ext2#Superblock +// the superblock starts at byte 1024 and occupies 1024 bytes +// the size of each block is located at byte 24 of the superblock + +#define EXT2_S_MAGIC 0xEF53 + +/* Superblock Fields */ +struct ext2fs_superblock { + uint32_t s_inodes_count; // total number of inodes in the system + uint32_t s_blocks_count; // total number of blocks in the system + uint32_t s_r_blocks_count; // blocks that only the superuser can access + uint32_t s_free_blocks_count; // number of free blocks + uint32_t s_free_inodes_count; // number of free inodes + uint32_t s_first_data_block; // block number of block that contains superblock + uint32_t s_log_block_size; // [log2(blocksize) - 10] shift left 1024 to get block size + uint32_t s_log_frag_size; // [log2(fragsize) - 10] sift left 1024 to get fragment size + uint32_t s_blocks_per_group; // number of blocks per block group + uint32_t s_frags_per_group; // number of fragments per block group + uint32_t s_inodes_per_group; // number of inodes per block group + uint32_t s_mtime; // Last mount time + uint32_t s_wtime; // Last write time + + uint16_t s_mnt_count; // number of times the volume was mounted before last consistency check + uint16_t s_max_mnt_count; // number of times the drive can be mounted before a check + uint16_t s_magic; // 0xEF53 | used to confirm ext2 presence + uint16_t s_state; // state of the filesystem + uint16_t s_errors; // what to do incase of an error + uint16_t s_minor_rev_level; // combine with major portion to get full version + + uint32_t s_lastcheck; // timestamp of last consistency check + uint32_t s_checkinterval; // amount of time between required consistency checks + uint32_t s_creator_os; // operating system ID + uint32_t s_rev_level; // combine with minor portion to get full version + uint32_t s_def_resuid; // User ID that can use reserved blocks + uint32_t s_def_gid; // Group ID that can use reserved blocks + + // if version number >= 1, we have to use the ext2 extended superblock as well + + /* Extended Superblock */ + uint32_t s_first_ino; // first non reserved inode in the fs (fixed to 11 when version < 1) + + uint16_t s_inode_size; // size of each inode (in bytes) (fixed to 128 when version < 1) + uint16_t s_block_group_nr; // block group this superblock is part of + + uint32_t s_feature_compat; // if optional features are present + uint32_t s_feature_incompat; // if required features are present + uint32_t s_feature_ro_compat; // features that are unsupported (make FS readonly) + + uint64_t s_uuid[2]; // FS ID + uint64_t s_volume_name[2]; // Volume Name + + uint64_t s_last_mounted[8]; // last path the volume was mounted to (C-style string) + + uint32_t s_algo_bitmap; // Compression algorithm used + + uint8_t s_prealloc_blocks; // Number of blocks to preallocate for files + uint8_t s_prealloc_dir_blocks; // Number of blocks to preallocate for directories + + uint16_t unused; // Unused + + uint64_t s_journal_uuid; // Journal ID + + uint32_t s_journal_inum; // Journal Inode number + uint32_t s_journal_dev; // Journal device + uint32_t s_last_orphan; // Head of orphan inode list + uint32_t s_hash_seed[4]; // Seeds used for hashing algo for dir indexing + + uint8_t s_def_hash_version; // Default hash versrion used for dir indexing + + uint32_t s_default_mnt_opts; // Default mount options + uint32_t s_first_meta_bg; // Block group ID for first meta group + + /* UNUSED */ +} __attribute__((packed)); + +/* EXT2 Block Group Descriptor */ +struct ext2fs_bgd { + uint32_t bg_block_bitmap; // Block address of block usage bitmap + uint32_t bg_inode_bitmap; // Block address of inode usage bitmap + uint32_t bg_inode_table; // Starting block address of inode table + + uint16_t bg_free_blocks_count; // Number of unallocated blocks in group + uint16_t bg_free_inodes_count; // Number of unallocated blocks in inode + uint16_t bg_dirs_count; // Number of directories in group + + uint16_t reserved[7]; +} __attribute__((packed)); + +/* EXT2 Inode Types */ +#define EXT2_INO_FIFO 0x1000 +#define EXT2_INO_CHR_DEV 0x2000 // Character device +#define EXT2_INO_DIRECTORY 0x4000 +#define EXT2_INO_BLK_DEV 0x6000 // Block device +#define EXT2_INO_FILE 0x8000 +#define EXT2_INO_SYMLINK 0xA000 +#define EXT2_INO_UNIX_SOCKET 0xC000 + +/* EXT2 Inode Permissions */ +#define EXT2_INO_X_OTHER 0x001 +#define EXT2_INO_W_OTHER 0x002 +#define EXT2_INO_R_OTHER 0x004 +#define EXT2_INO_X_GROUP 0x008 +#define EXT2_INO_W_GROUP 0x010 +#define EXT2_INO_R_GROUP 0x020 +#define EXT2_INO_X_USER 0x040 +#define EXT2_INO_W_USER 0x080 +#define EXT2_INO_R_USER 0x100 +#define EXT2_INO_STICKY 0x200 +#define EXT2_INO_S_GRP_ID 0x400 // Set User ID +#define EXT2_INO_S_USR_ID 0x800 // Set Group ID + +/* EXT2 Inode Flags */ +#define EXT2_INO_SECURE_DELETION 0x00000001 // Secure deletion (unused) +#define EXT2_INO_KEEP_COPY 0x00000002 // Keep copy of data upon deleting (unused) +#define EXT2_INO_FILE_COMPRESSION 0x00000004 // File compression (unused) +#define EXT2_INO_SYNC_UPDATES 0x00000008 // Sync updates to disk +#define EXT2_INO_FILE_IMMUTABLE 0x00000010 // File is readonly +#define EXT2_INO_APPEND_ONLY 0x00000020 // Append only +#define EXT2_INO_NO_INCLUDE_DUMP 0x00000040 // File not included in dump command +#define EXT2_INO_NO_UDPATE_LAT 0x00000080 // Dont update the last access time +#define EXT2_INO_HASH_IDX_DIR 0x00010000 // Directory is hash indexed +#define EXT2_INO_AFS_DIR 0x00020000 // Is AFS directory +#define EXT2_INO_JOURNAL_DATA 0x00040000 // Journal File Data + +/* EXT2 OS Specific Value 2 (only Linux support) */ +struct ext2fs_linux { + uint8_t frag_num; // Number of fragments + uint8_t frag_size; // Fragment Size + + uint16_t reserved_16; // Reserved + uint16_t user_id_high; // High 16 bits of 32 bit user_id + uint16_t group_id_high; // High 16 bits of 32 bit group_id + + uint32_t reserved_32; // Reserved +} __attribute__((packed)); + +/* EXT2 Inode */ +struct ext2fs_inode { + uint16_t i_mode; // Types and permissions + uint16_t i_uid; // User ID + + uint32_t i_size; // Lower 32 bits of the size (in bytes) + uint32_t i_atime; // Time of last access + uint32_t i_ctime; // Time of creation + uint32_t i_mtime; // Time of last modification + uint32_t i_dtime; // Time of last deletion + + uint16_t i_gid; // Block group ID this inode belongs to + uint16_t i_links_count; // Number of directory entries in this inode + + uint32_t i_blocks_count; // Number of blocks in use by this inode + uint32_t i_flags; // Flags for this inode + uint32_t i_osd1; // OS specific value #1 (linux support only) (unused) + uint32_t i_blocks[15]; // Block Pointers + uint32_t i_generation; // Generation number + + /* EXT2 v >= 1.0 */ + uint32_t i_eab; // Extended Attribute Block + uint32_t i_maj; // If feature bit set, upper 32 bit of file size. Directory ACL if inode is directory + + /* EXT2 vAll */ + uint32_t i_frag_block; // Block address of fragment + + struct ext2fs_linux i_osd2; // OS specific value #2 (linux support only) +} __attribute__((packed)); + +/* EXT2 Directory File Types */ +#define EXT2_FT_UNKNOWN 0 // Unknown +#define EXT2_FT_FILE 1 // Regular file +#define EXT2_FT_DIR 2 // Directory +#define EXT2_FT_CHRDEV 3 // Character Device +#define EXT2_FT_BLKDEV 4 // Block Device +#define EXT2_FT_FIFO 5 // FIFO +#define EXT2_FT_SOCKET 6 // Unix Socket +#define EXT2_FT_SYMLINK 7 // Symbolic Link + +/* EXT2 Directory Entry */ +struct ext2fs_dir_entry { + uint32_t inode; // Inode number of file entry + uint16_t rec_len; // Displacement to next directory entry from start of current one + uint8_t name_len; // Length of the name + uint8_t type; // File type +} __attribute__((packed)); + +static struct ext2fs_superblock *superblock; +static struct ext2fs_inode *root_inode; +static int num_entries = 0; + +// parse an inode given the partition base and inode number +static struct ext2fs_inode *ext2fs_get_inode(uint64_t drive, struct mbr_part *part, uint64_t inode) { + uint64_t base = part->first_sect * 512; + uint64_t bgdt_loc = base + EXT2_BLOCK_SIZE; + + uint64_t ino_blk_grp = (inode - 1) / superblock->s_inodes_per_group; + uint64_t ino_tbl_idx = (inode - 1) % superblock->s_inodes_per_group; + + struct ext2fs_bgd *target_descriptor = balloc(sizeof(struct ext2fs_bgd)); + read(drive, target_descriptor, bgdt_loc + (sizeof(struct ext2fs_bgd) * ino_blk_grp), sizeof(struct ext2fs_bgd)); + + struct ext2fs_inode *target = balloc(sizeof(struct ext2fs_inode)); + read(drive, target, base + (target_descriptor->bg_inode_table * EXT2_BLOCK_SIZE) + (sizeof(struct ext2fs_inode) * ino_tbl_idx), sizeof(struct ext2fs_inode)); + + return target; +} + +static struct ext2fs_dir_entry *ext2fs_parse_dirent(int drive, struct mbr_part *part, const char* filename) { + if (root_inode == NULL) + return NULL; + + uint64_t base = part->first_sect * 512; + uint64_t offset = base + (root_inode->i_blocks[0] * EXT2_BLOCK_SIZE); + + for (uint32_t i = 0; i < num_entries; i++) { + struct ext2fs_dir_entry *dir = balloc(sizeof(struct ext2fs_dir_entry)); + + // preliminary read + read(drive, dir, offset, sizeof(struct ext2fs_dir_entry)); + + // name read + char* name = balloc(sizeof(char) * dir->name_len); + read(drive, name, offset + sizeof(struct ext2fs_dir_entry), dir->name_len); + + if (!strncmp(filename, name, dir->name_len)) { + return dir; + } + + offset += dir->rec_len; + } + + return NULL; +} + +int ext2fs_open(struct ext2fs_file_handle *ret, int drive, int partition, const char* filename) { + struct mbr_part part; + mbr_get_part(&part, drive, partition); + + struct ext2fs_dir_entry *entry = ext2fs_parse_dirent(drive, &part, filename); + + ret->drive = drive; + ret->part = part; + + struct ext2fs_inode *target = ext2fs_get_inode(drive, &part, entry->inode); + + ret->inode_num = entry->inode; + ret->size = target->i_size; + + return 1; +} + +int ext2fs_read(struct ext2fs_file_handle *file, void* buf, uint64_t loc, uint64_t count) { + uint64_t base = file->part.first_sect * 512; + + // read the contents of the inode + // it is assumed that bfread has already done the directory check + + // TODO: add support for the indirect block pointers + // TODO: add support for reading multiple blocks + + struct ext2fs_inode *target = ext2fs_get_inode(file->drive, &file->part, file->inode_num); + + read(file->drive, buf, base + (target->i_blocks[0] * EXT2_BLOCK_SIZE) + loc, count); + + return 1; +} + +static int first_run = 0; + +// attempts to initialize the ext2 filesystem +int ext2fs_check_signature(int drive, int partition) { + struct mbr_part part; + mbr_get_part(&part, drive, partition); + + uint64_t base = part.first_sect * 512; + + int magic = 0; + + // read only the checksum of the superblock + read(drive, &magic, base + 1024 + 56, 2); + + if (magic == EXT2_S_MAGIC) { + if (first_run == 0) { + first_run = 1; + + superblock = balloc(1024); + + // read the entire superblock this time + read(drive, superblock, base + 1024, 1024); + // parse the root inode + root_inode = ext2fs_get_inode(drive, &part, 2); + + num_entries = root_inode->i_links_count + 2; + } + + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/src/fs/ext2fs.h b/src/fs/ext2fs.h new file mode 100644 index 00000000..2c94edb8 --- /dev/null +++ b/src/fs/ext2fs.h @@ -0,0 +1,23 @@ +#ifndef __FS_EXT2FS_H__ +#define __FS_EXT2FS_H__ + +#include +#include +#include +#include +#include +#include + +struct ext2fs_file_handle { + int drive; + struct mbr_part part; + int inode_num; + int size; +}; + +int ext2fs_check_signature(int drive, int partition); + +int ext2fs_open(struct ext2fs_file_handle *ret, int drive, int partition, const char* filename); +int ext2fs_read(struct ext2fs_file_handle *file, void* buf, uint64_t loc, uint64_t count); + +#endif \ No newline at end of file diff --git a/src/fs/file.c b/src/fs/file.c index da06aa2a..4cdf9343 100644 --- a/src/fs/file.c +++ b/src/fs/file.c @@ -2,6 +2,7 @@ #include #include #include +#include #include int fopen(struct file_handle *ret, int disk, int partition, const char *filename) { @@ -21,7 +22,21 @@ int fopen(struct file_handle *ret, int disk, int partition, const char *filename return 0; } - // Append other FS checks here + if (ext2fs_check_signature(disk, partition)) { + struct ext2fs_file_handle *fd = balloc(sizeof(struct ext2fs_file_handle)); + + int r = ext2fs_open(fd, disk, partition, filename); + if (!r) + return 1; + + ret->fd = (void *)fd; + ret->read = (void *)ext2fs_read; + ret->disk = disk; + ret->partition = partition; + ret->size = fd->size; + + return 0; + } print("fs: Could not determine the file system of disk %u partition %u", disk, partition); diff --git a/src/lib/blib.c b/src/lib/blib.c index 6d78a87f..684ecda4 100644 --- a/src/lib/blib.c +++ b/src/lib/blib.c @@ -5,7 +5,6 @@ #include #include #include -#include #include void panic(const char *str) { @@ -21,8 +20,11 @@ static size_t bump_allocator_base = 0x20000; void *balloc(size_t count) { void *ret = (void *)bump_allocator_base; size_t new_base = bump_allocator_base + count; - if (new_base >= BUMP_ALLOCATOR_LIMIT) + if (new_base >= BUMP_ALLOCATOR_LIMIT) { + print("PANIC: Old: %x | New: %x\n", (size_t)ret, new_base); + print("PANIC: Allocated: %x\n", count); panic("Memory allocation failed"); + } bump_allocator_base = new_base; return ret; } diff --git a/src/main.c b/src/main.c index a7e76cb6..a74a93c6 100644 --- a/src/main.c +++ b/src/main.c @@ -109,4 +109,4 @@ void main(int boot_drive) { print("Invalid protocol specified: `%s`.\n", proto); for (;;); } -} +} \ No newline at end of file diff --git a/test/test.asm b/test/test.asm index bce612b6..6f8ab040 100644 --- a/test/test.asm +++ b/test/test.asm @@ -29,4 +29,4 @@ _start: mov [rdx], rax mov [rdx+8], rbx mov [rdx+16], rcx - jmp $ + jmp $ \ No newline at end of file