diff --git a/Makefile b/Makefile index d12b51c6..253ad82c 100644 --- a/Makefile +++ b/Makefile @@ -304,6 +304,28 @@ iso9660-test: xorriso -as mkisofs -b boot/limine-cd.bin -no-emul-boot -boot-load-size 4 -boot-info-table test_image/ -o test.iso qemu-system-x86_64 -net none -smp 4 -cdrom test.iso -debugcon stdio +.PHONY: ntfs-test +ntfs-test: + $(MAKE) test-clean + $(MAKE) test.hdd + $(MAKE) limine-bios + $(MAKE) bin/limine-install + $(MAKE) -C test + rm -rf test_image/ + mkdir test_image + sudo losetup -Pf --show test.hdd > loopback_dev + sudo partprobe `cat loopback_dev` + sudo mkfs.ntfs `cat loopback_dev`p1 + sudo mount `cat loopback_dev`p1 test_image + sudo mkdir test_image/boot + sudo cp -rv bin/* test/* test_image/boot/ + sync + sudo umount test_image/ + sudo losetup -d `cat loopback_dev` + rm -rf test_image loopback_dev + bin/limine-install test.hdd + qemu-system-x86_64 -net none -smp 4 -hda test.hdd -debugcon stdio + .PHONY: full-hybrid-test full-hybrid-test: $(MAKE) ovmf-x64 diff --git a/stage23/fs/file.s2.c b/stage23/fs/file.s2.c index a90e9fcb..88d678d7 100644 --- a/stage23/fs/file.s2.c +++ b/stage23/fs/file.s2.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -97,6 +98,20 @@ int fopen(struct file_handle *ret, struct volume *part, const char *filename) { return 0; } + if (ntfs_check_signature(part)) { + struct ntfs_file_handle *fd = ext_mem_alloc(sizeof(struct ntfs_file_handle)); + + int r = ntfs_open(fd, part, filename); + if (r) + return r; + + ret->fd = (void *)fd; + ret->read = (void *)ntfs_read; + ret->size = fd->size_bytes; + + return 0; + } + return -1; } diff --git a/stage23/fs/ntfs.h b/stage23/fs/ntfs.h new file mode 100644 index 00000000..f2e713e5 --- /dev/null +++ b/stage23/fs/ntfs.h @@ -0,0 +1,58 @@ +#ifndef __FS__NTFS_H__ +#define __FS__NTFS_H__ + +#include +#include +#include +#include +#include + +struct ntfs_bpb { + uint8_t jump[3]; + char oem[8]; + uint16_t bytes_per_sector; + uint8_t sectors_per_cluster; + uint16_t reserved_sectors; + uint8_t fats_count; + uint16_t directory_entries_count; + uint16_t sector_totals; + uint8_t media_descriptor_type; + uint16_t sectors_per_fat_16; + uint16_t sectors_per_track; + uint16_t heads_count; + uint32_t hidden_sectors_count; + uint32_t large_sectors_count; + + // ntfs + uint32_t sectors_per_fat_32; + uint64_t sectors_count_64; + uint64_t mft_cluster; +} __attribute__((packed)); + +struct ntfs_file_handle { + struct volume *part; + + struct ntfs_bpb bpb; + + // file record sizes + uint64_t file_record_size; + uint64_t sectors_per_file_record; + + // MFT info, the offset and its runlist + uint64_t mft_offset; + uint8_t mft_run_list[256]; + + // the runlist, resident index and attribute list of the + // current open file/directory + uint8_t run_list[128]; + + // info about the current file + uint32_t size_bytes; +}; + +int ntfs_check_signature(struct volume *part); + +int ntfs_open(struct ntfs_file_handle *ret, struct volume *part, const char *path); +int ntfs_read(struct ntfs_file_handle *file, void *buf, uint64_t loc, uint64_t count); + +#endif \ No newline at end of file diff --git a/stage23/fs/ntfs.s2.c b/stage23/fs/ntfs.s2.c new file mode 100644 index 00000000..76ae7f04 --- /dev/null +++ b/stage23/fs/ntfs.s2.c @@ -0,0 +1,621 @@ +#include +#include +#include + +#include + +// created using documentation from: +// https://dubeyko.com/development/FileSystems/NTFS/ntfsdoc.pdf + +// This is the total size of a file record, including the attributes +// TODO: calculate this +#define MIN_FILE_RECORD_SIZE 1024 + +// The mft number is only 48bit, the other bits are used +// for something else +#define MFT_RECORD_NO_MASK (0xFFFFFFFFFFFF) + +struct mft_file_record { + char name[4]; + uint16_t update_seq_offset; + uint16_t update_seq_size; + uint64_t log_seq_number; + uint16_t sequence_number; + uint16_t hard_link_count; + uint16_t attribute_offset; + uint16_t flags; + uint32_t real_size; + uint32_t allocated_size; + uint64_t base_record_number; +} __attribute__((packed)); + +struct file_record_attr_header { + uint32_t type; + uint32_t length; + uint8_t non_res_flag; + uint8_t name_length; + uint16_t name_offset; + uint16_t flags; + uint16_t attribute_id; +} __attribute__((packed)); + +#define FR_ATTRIBUTE_LIST 0x00000020 +#define FR_ATTRIBUTE_NAME 0x00000030 +#define FR_ATTRIBUTE_VOLUME_NAME 0x00000060 +#define FR_ATTRIBUTE_DATA 0x00000080 +#define FR_ATTRIBUTE_INDEX_ROOT 0x00000090 +#define FR_ATTRIBUTE_INDEX_ALLOC 0x000000A0 + +struct file_record_attr_header_res { + struct file_record_attr_header header; + uint32_t info_length; + uint16_t info_offset; + uint16_t index_flag; +} __attribute__((packed)); + +struct file_record_attr_header_non_res { + struct file_record_attr_header header; + uint64_t first_vcn; + uint64_t last_vcn; + uint16_t run_offset; +} __attribute__((packed)); + +struct file_record_attr_name { + uint64_t mft_parent_record; + uint64_t creation_time; + uint64_t altered_time; + uint64_t mft_changed_time; + uint64_t read_time; + uint64_t allocated_size; + uint64_t real_size; + uint32_t flags; + uint32_t reparse; + uint8_t name_length; + uint8_t name_type; + uint16_t name[]; +} __attribute__((packed)); + +struct index_record { + char name[4]; + uint16_t update_seq_offset; + uint16_t update_seq_size; + uint64_t log_seq_number; + uint64_t vcn; + uint32_t index_entry_offset; + uint32_t index_entry_size; + uint32_t index_entry_alloc; + uint8_t leaf_node; + uint8_t _reserved[3]; + uint16_t update_seq; + uint16_t sequence_array[]; +} __attribute__((packed)); + +struct index_entry { + uint64_t mft_record; + uint16_t entry_size; + uint16_t name_offset; + uint16_t index_Flag; + uint16_t padding; + uint64_t mft_parent_record; + uint64_t creation_time; + uint64_t altered_time; + uint64_t mft_changed_time; + uint64_t read_time; + uint64_t alloc_size; + uint64_t real_size; + uint64_t file_flags; + uint8_t name_length; + uint8_t name_type; + uint16_t name[]; +} __attribute__((packed)); + +int ntfs_check_signature(struct volume *part) { + struct ntfs_bpb bpb; + if (!volume_read(part, &bpb, 0, sizeof(bpb))) { + return 0; + } + + // + // validate the bpb + // + + if (strncmp(bpb.oem, "NTFS ", SIZEOF_ARRAY(bpb.oem))) { + return 0; + } + + if (bpb.sector_totals != 0) { + return 0; + } + + if (bpb.large_sectors_count != 0) { + return 0; + } + + if (bpb.sectors_count_64 == 0) { + return 0; + } + + // this is a valid ntfs sector + return 1; +} + +// the temp buffer is used for storing dirs and alike +// in memory, because limine only has allocate without +// free we are going to allocate it once globally and just +// make sure to only use it in the ntfs_open function... +static uint8_t *dir_buffer = NULL; +static size_t dir_buffer_size = 0; +static size_t dir_buffer_cap = 0; + +/** + * Get an attribute from the given file record + */ +static bool ntfs_get_file_record_attr(uint8_t* file_record, uint32_t attr_type, uint8_t **out_attr) { + struct mft_file_record *fr = (struct mft_file_record *)file_record; + + // get the offset to the first attribute + uint8_t *cur_attr_ptr = file_record + fr->attribute_offset; + + while (true) { + // TODO: don't check for the min size, but for the actual size... + if (cur_attr_ptr + sizeof(struct file_record_attr_header) > file_record + MIN_FILE_RECORD_SIZE) + panic("NTFS: File record attribute is outside of file record"); + + struct file_record_attr_header *cur_attr = (struct file_record_attr_header *)cur_attr_ptr; + + if (cur_attr->type == attr_type) { + *out_attr = cur_attr_ptr; + return true; + } + + // we either found an attr with higher type or the end type + if (cur_attr->type > attr_type || cur_attr->type == 0xFF) + return false; + + if (cur_attr->length == 0) + panic("NTFS: File record attribute has zero length"); + + cur_attr_ptr += cur_attr->length; + } +} + +/** + * Gets a count and cluster from the runlist, if next is true then it updates the list intenrally + * so the next call will return the next element + * + * if returned false we got to the end of the file. + */ +static bool ntfs_get_next_run_list_element(uint8_t **runlist, uint64_t *out_cluster_count, uint64_t *out_cluster, bool next) { + uint8_t *runlist_ptr = *runlist; + + // we have reached the end of the file + if (runlist_ptr[0] == 0) { + return false; + } + + uint8_t low = runlist_ptr[0] & 0xF; + uint8_t high = (runlist_ptr[0] >> 4) & 0xF; + runlist_ptr++; + + // get the run length + uint64_t count = 0; + for (int i = low; i > 0; i--) { + count <<= 8; + count |= runlist_ptr[i - 1]; + } + runlist_ptr += low; + + // get the high byte first + int8_t high_byte = (int8_t)runlist_ptr[high - 1]; + + // get the run offset + uint64_t cluster = 0; + for (int i = high; i > 0; i--) { + cluster <<= 8; + cluster |= runlist_ptr[i - 1]; + } + runlist_ptr += high; + + // if the offset is negative, fill the empty bytes with 0xff + if (high_byte < 0 && high < 8) { + uint64_t fill = 0; + for (int i = 8; i > high; i--) { + fill >>= 8; + fill |= 0xFF00000000000000; + } + cluster |= fill; + } + + // out it + *out_cluster = cluster; + *out_cluster_count = count; + + // update it + if (next) { + *runlist = runlist_ptr; + } + + return true; +} + +static bool ntfs_get_file_record(struct ntfs_file_handle *handle, uint64_t mft_record_no, uint8_t *file_record_buffer) { + uint8_t *runlist = handle->mft_run_list; + + // make sure we only take the number itself + mft_record_no &= MFT_RECORD_NO_MASK; + + // get the + uint64_t count = 0; + uint64_t cluster = 0; + if (!ntfs_get_next_run_list_element(&runlist, &count, &cluster, true)) { + return false; + } + + size_t bytes_per_cluster = handle->bpb.bytes_per_sector * handle->bpb.sectors_per_cluster; + uint64_t byte_count = count * bytes_per_cluster; + uint64_t sector = cluster * handle->bpb.sectors_per_cluster; + uint64_t record_count = 0; + do { + // consume the items from the current runlist + if (byte_count > 0) { + sector += handle->sectors_per_file_record; + byte_count -= handle->file_record_size; + } else { + // get the next run list... + if (!ntfs_get_next_run_list_element(&runlist, &count, &cluster, true)) { + // reached the end of the mft, did not find it... + return false; + } + byte_count = count * bytes_per_cluster; + sector = cluster * handle->bpb.sectors_per_cluster; + continue; + } + record_count++; + + } while (record_count < mft_record_no); + + // we found the sector of the file record! + uint64_t offset = sector * handle->bpb.bytes_per_sector; + + + if(!volume_read(handle->part, file_record_buffer, offset, handle->file_record_size)) + panic("NTFS: Failed to read file record from mft"); + + // make sure this is a valid file record + struct mft_file_record *fr = (struct mft_file_record *)file_record_buffer; + if (strncmp(fr->name, "FILE", SIZEOF_ARRAY(fr->name))) + panic("NTFS: File record has invalid signature (got %c%c%c%c, should be FILE)!", + fr->name[0], fr->name[1], fr->name[2], fr->name[3]); + + // we good! + return true; +} + +/** + * Read the the directory's file record from the mft + */ +static bool ntfs_read_directory(struct ntfs_file_handle *handle, uint64_t mft_record, uint8_t *file_record) { + // get the record of the directory + if (!ntfs_get_file_record(handle, mft_record, file_record)) + return false; + + // + // get the runlist of the directory + // + + // get the index alloc attribute, it should have the runlist offset + // copy the runlist from it to our handle for easier access + uint8_t *index_alloc_ptr; + if (!ntfs_get_file_record_attr(file_record, FR_ATTRIBUTE_INDEX_ALLOC, &index_alloc_ptr)) + panic("NTFS: Directory has no runlist?!"); + + struct file_record_attr_header_non_res *index_alloc = (struct file_record_attr_header_non_res *)index_alloc_ptr; + uint8_t *runlist_ptr = index_alloc_ptr + index_alloc->run_offset; + if (runlist_ptr - file_record + 128u > handle->file_record_size) + panic("NTFS: runlist is outside of file record!"); + memcpy(handle->run_list, runlist_ptr, sizeof(handle->run_list)); + + // calculate the directory size by just going through the runlist + uint8_t *runlist = handle->run_list; + uint64_t dir_size = 0; + uint64_t cluster = 0; + uint64_t cluster_count = 0; + bool status = false; + do { + status = ntfs_get_next_run_list_element(&runlist, &cluster_count, &cluster, true); + if (status) + dir_size += cluster_count; + } while(status); + dir_size *= handle->bpb.sectors_per_cluster * handle->bpb.bytes_per_sector; + + // allocate a buffer for the directory data + if (dir_buffer == NULL) { + // allocate enough just in case, idk how much is good + dir_buffer_cap = dir_size > 64 * 1024 ? dir_size : 64 * 1024; + dir_buffer = ext_mem_alloc(dir_buffer_cap); + } else { + // we must truncate it... + if (dir_size > dir_buffer_cap) { + dir_size = dir_buffer_cap; + } + } + + // set the size of the dir size + dir_buffer_size = dir_size; + + // read the directory + if (ntfs_read(handle, dir_buffer, 0, dir_size)) + panic("NTFS: EOF before reading directory fully..."); + + return true; +} + +/** + * Prepare for reading a file by reading the root directory into the file handle + */ +static void ntfs_read_root(struct ntfs_file_handle *handle) { + // calculate the offset for the mft + handle->mft_offset = (uint64_t)handle->bpb.mft_cluster * (uint64_t)handle->bpb.sectors_per_cluster * (uint64_t)handle->bpb.bytes_per_sector; + + // read the mft file record, this should be the size of a sector + uint8_t file_record_buffer[MIN_FILE_RECORD_SIZE]; + if (!volume_read(handle->part, file_record_buffer, handle->mft_offset, sizeof(file_record_buffer))) + panic("NTFS: Failed to read MFT file record"); + + // get the file attribute + uint8_t *attr_ptr = NULL; + if (!ntfs_get_file_record_attr(file_record_buffer, FR_ATTRIBUTE_DATA, &attr_ptr)) + panic("NTFS: MFT file record missing DATA attribute"); + struct file_record_attr_header_non_res *attr = (struct file_record_attr_header_non_res *)attr_ptr; + + // verify the attr and run list are in the buffer + if ((uint8_t *)attr + sizeof(*attr) > file_record_buffer + sizeof(file_record_buffer)) + panic("NTFS: MFT file record attribute is outside of file record"); + if ((uint8_t *)attr + attr->run_offset + 256 > file_record_buffer + sizeof(file_record_buffer)) + panic("NTFS: MFT Run list is outside of file record"); + + // save the run list + memcpy(handle->mft_run_list, (uint8_t *)attr + attr->run_offset, sizeof(handle->mft_run_list)); + + // read the root directory record, which has the number 5 + if (!ntfs_read_directory(handle, 5, file_record_buffer)) + panic("NTFS: Missing root directory file record!"); +} + +/** + * Search for a file in the ntfs directory, assumes the directory has been read and is stored in + * the temp buffer + */ +static bool ntfs_find_file_in_directory(struct ntfs_file_handle *handle, const char* filename, struct index_entry** out_entry) { + size_t dir_size = dir_buffer_size; + uint8_t *dir_ptr = dir_buffer; + + // TODO: iterate resident record... + + // get the size of the name we need to compare + const char* temp_filename = filename; + size_t filename_size = 0; + while (*temp_filename != '\0' && *temp_filename != '\\' && *temp_filename != '/') { + filename_size++; + temp_filename++; + } + + // iterate the non-resident files in the directory + size_t offset = 0; + while (dir_size) { + // check if the dir pointer is still in the buffer, if not then we could + // not find the file... + if (dir_ptr + sizeof(struct index_record) > dir_buffer + dir_buffer_size) + panic("NTFS: Tried to read index record outside of directory"); + + // get the index and check it, if it is not valid just return + // we did not find the file + struct index_record *index_record = (struct index_record *)dir_ptr; + if (strncmp(index_record->name, "INDX", SIZEOF_ARRAY(index_record->name))) + return false; + + // calculate the offset to the entry + size_t index_size = index_record->index_entry_size; + offset += index_record->index_entry_offset + offsetof(struct index_record, index_entry_offset); + uint8_t *entry_ptr = dir_ptr + offset; + + // loop the record for all of its indexes + while (index_size) { + // make sure we still have an entry + if (entry_ptr + sizeof(struct index_entry) > dir_buffer + dir_buffer_size) + panic("NTFS: Tried to read index entry outside of directory"); + + // get the entry, if size is zero we done + struct index_entry *entry = (struct index_entry *)entry_ptr; + if (entry->entry_size == 0) + break; + + if (filename_size == entry->name_length) { + // this name seem legit, need to get the real name from the mft + // sometimes it works to use the index name but sometimes it has + // invalid names for whatever reason that I can not understand, so + // just always take it from the mft file record + uint8_t file_record_buffer[MIN_FILE_RECORD_SIZE]; + if (!ntfs_get_file_record(handle, entry->mft_record, file_record_buffer)) + panic("NTFS: Failed to get file record"); + + uint8_t *name_attr = NULL; + if (!ntfs_get_file_record_attr(file_record_buffer, FR_ATTRIBUTE_NAME, &name_attr)) + panic("NTFS: File record missing name attribute"); + + // get the offset to the actual info + struct file_record_attr_header_res *header = (struct file_record_attr_header_res *)name_attr; + struct file_record_attr_name *name = (struct file_record_attr_name *)(name_attr + header->info_offset); + + // compare the name + for (int i = 0; i < name->name_length; i++) { + if (name->name[i] != filename[i]) { + goto next_entry; + } + } + + // name is good, return the entry and return true + // that we found the entry + *out_entry = entry; + return true; + } + + // next entry + next_entry: + entry_ptr += entry->entry_size; + index_size -= entry->entry_size; + } + + // next record, need to do some rounding + index_size = index_record->index_entry_size; + if (index_size < 0x1000) { + index_size = 0x1000; + } else { + index_size = (index_size + 0x100) & 0xffffff00; + } + + dir_ptr += index_size; + dir_size -= index_size; + } + + return false; +} + +int ntfs_open(struct ntfs_file_handle *ret, struct volume *part, const char *path) { + // save the part + ret->part = part; + + // start by reading the bpb so we can access it later on + if (!volume_read(part, &ret->bpb, 0, sizeof(ret->bpb))) + panic("NTFS: Failed to read the BPB"); + + // in NTFS sector size can be 512 to 4096 bytes, file records are + // at least 1024 bytes, in here calculate the sectors per file record + // and the file record size + if (ret->bpb.bytes_per_sector <= MIN_FILE_RECORD_SIZE) { + // this has multiple sectors + ret->sectors_per_file_record = MIN_FILE_RECORD_SIZE / ret->bpb.bytes_per_sector; + ret->file_record_size = MIN_FILE_RECORD_SIZE; + } else { + // this has a single sector + ret->sectors_per_file_record = 1; + ret->file_record_size = ret->bpb.bytes_per_sector; + } + if (ret->file_record_size != MIN_FILE_RECORD_SIZE) + panic("NTFS: TODO: support file record size which is not 1024 bytes"); + + // now prepare the root directory so we can search for + // the rest of the stuff + ntfs_read_root(ret); + + // iterate the directories to find the entry + const char* current_path = path; + struct index_entry* entry = NULL; + for (;;) { + // skip slash + while (*current_path == '\\' || *current_path == '/') { + current_path++; + } + + // find the file in the directory + entry = NULL; + if (!ntfs_find_file_in_directory(ret, current_path, &entry)) + return 1; + + size_t filename_len = entry->name_length; + + // check if this is the last entry + uint8_t file_record_buffer[MIN_FILE_RECORD_SIZE]; + if (*(current_path + filename_len) == '\0') { + // we found the file! + ret->size_bytes = entry->real_size; + + // get its runlist... + if (!ntfs_get_file_record(ret, entry->mft_record, file_record_buffer)) + panic("NTFS: Failed to get file record of file"); + + // get the file attribute + uint8_t *attr_ptr = NULL; + if (!ntfs_get_file_record_attr(file_record_buffer, FR_ATTRIBUTE_DATA, &attr_ptr)) + panic("NTFS: File record missing DATA attribute"); + struct file_record_attr_header_non_res *attr = (struct file_record_attr_header_non_res *)attr_ptr; + + // verify the attr and run list are in the buffer + if ((uint8_t *)attr + sizeof(*attr) > file_record_buffer + sizeof(file_record_buffer)) + panic("NTFS: File record attribute is outside of file record"); + if ((uint8_t *)attr + attr->run_offset + 256 > file_record_buffer + sizeof(file_record_buffer)) + panic("NTFS: Run list is outside of file record"); + + // save the run list + memcpy(ret->run_list, (uint8_t *)attr + attr->run_offset, sizeof(ret->run_list)); + + return 0; + + } else { + // read the directory + if (!ntfs_read_directory(ret, entry->mft_record, file_record_buffer)) + panic("NTFS: Failed to read directory"); + + // next path element + current_path += filename_len; + } + } + + // should not be able to reach here... + __builtin_unreachable(); +} + +int ntfs_read(struct ntfs_file_handle *file, void *buf, uint64_t loc, uint64_t count) { + // get the runlist + uint8_t *runlist = file->run_list; + + // TODO: remember the last read location so we can have faster sequential reads... + + // we are going to go over the runlist until we get to the offset + // once we get to the offset we are going to continue going over + // the runlist while copying bytes + uint64_t bytes_per_cluster = file->bpb.sectors_per_cluster * file->bpb.bytes_per_sector; + do { + // get the next element from the runlist + uint64_t cluster_count; + uint64_t cluster; + if (!ntfs_get_next_run_list_element(&runlist, &cluster_count, &cluster, true)) + break; + + // calculate the cont size and offset on disk + uint64_t total_cont_bytes = cluster_count * bytes_per_cluster; + uint64_t abs_byte = cluster * bytes_per_cluster; + + // check if we arrived at the wanted offset + if (loc != 0) { + if (loc >= total_cont_bytes) { + // we need to go more... + loc -= total_cont_bytes; + } else { + // we got to the offset, adjust base and size + // and set the loc to 0 + total_cont_bytes -= loc; + abs_byte += loc; + loc = 0; + } + } + + if (loc == 0) { + // get how much we wanna read now and + // subtract that from the total we need + // to read + size_t read_now = total_cont_bytes; + if (read_now > count) { + read_now = count; + } + count -= read_now; + + // read it! + if (!volume_read(file->part, buf, abs_byte, read_now)) + panic("NTFS: Runlist points to outside the volume (%x)", abs_byte); + } + } while(count); + + // if we didn't read it all then we got a problem + return count != 0; +}