Merge pull request #113 from Itay2805/trunk

NTFS Support
This commit is contained in:
mint 2021-09-29 01:47:17 +02:00 committed by GitHub
commit 6cd265b497
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 716 additions and 0 deletions

View File

@ -304,6 +304,28 @@ iso9660-test:
xorriso -as mkisofs -b boot/limine-cd.bin -no-emul-boot -boot-load-size 4 -boot-info-table test_image/ -o test.iso
qemu-system-x86_64 -net none -smp 4 -cdrom test.iso -debugcon stdio
.PHONY: ntfs-test
ntfs-test:
$(MAKE) test-clean
$(MAKE) test.hdd
$(MAKE) limine-bios
$(MAKE) bin/limine-install
$(MAKE) -C test
rm -rf test_image/
mkdir test_image
sudo losetup -Pf --show test.hdd > loopback_dev
sudo partprobe `cat loopback_dev`
sudo mkfs.ntfs `cat loopback_dev`p1
sudo mount `cat loopback_dev`p1 test_image
sudo mkdir test_image/boot
sudo cp -rv bin/* test/* test_image/boot/
sync
sudo umount test_image/
sudo losetup -d `cat loopback_dev`
rm -rf test_image loopback_dev
bin/limine-install test.hdd
qemu-system-x86_64 -net none -smp 4 -hda test.hdd -debugcon stdio
.PHONY: full-hybrid-test
full-hybrid-test:
$(MAKE) ovmf-x64

View File

@ -5,6 +5,7 @@
#include <fs/ext2.h>
#include <fs/fat32.h>
#include <fs/iso9660.h>
#include <fs/ntfs.h>
#include <lib/print.h>
#include <lib/blib.h>
#include <mm/pmm.h>
@ -97,6 +98,20 @@ int fopen(struct file_handle *ret, struct volume *part, const char *filename) {
return 0;
}
if (ntfs_check_signature(part)) {
struct ntfs_file_handle *fd = ext_mem_alloc(sizeof(struct ntfs_file_handle));
int r = ntfs_open(fd, part, filename);
if (r)
return r;
ret->fd = (void *)fd;
ret->read = (void *)ntfs_read;
ret->size = fd->size_bytes;
return 0;
}
return -1;
}

58
stage23/fs/ntfs.h Normal file
View File

@ -0,0 +1,58 @@
#ifndef __FS__NTFS_H__
#define __FS__NTFS_H__
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <lib/part.h>
#include <lib/blib.h>
struct ntfs_bpb {
uint8_t jump[3];
char oem[8];
uint16_t bytes_per_sector;
uint8_t sectors_per_cluster;
uint16_t reserved_sectors;
uint8_t fats_count;
uint16_t directory_entries_count;
uint16_t sector_totals;
uint8_t media_descriptor_type;
uint16_t sectors_per_fat_16;
uint16_t sectors_per_track;
uint16_t heads_count;
uint32_t hidden_sectors_count;
uint32_t large_sectors_count;
// ntfs
uint32_t sectors_per_fat_32;
uint64_t sectors_count_64;
uint64_t mft_cluster;
} __attribute__((packed));
struct ntfs_file_handle {
struct volume *part;
struct ntfs_bpb bpb;
// file record sizes
uint64_t file_record_size;
uint64_t sectors_per_file_record;
// MFT info, the offset and its runlist
uint64_t mft_offset;
uint8_t mft_run_list[256];
// the runlist, resident index and attribute list of the
// current open file/directory
uint8_t run_list[128];
// info about the current file
uint32_t size_bytes;
};
int ntfs_check_signature(struct volume *part);
int ntfs_open(struct ntfs_file_handle *ret, struct volume *part, const char *path);
int ntfs_read(struct ntfs_file_handle *file, void *buf, uint64_t loc, uint64_t count);
#endif

621
stage23/fs/ntfs.s2.c Normal file
View File

@ -0,0 +1,621 @@
#include <fs/ntfs.h>
#include <mm/pmm.h>
#include <lib/print.h>
#include <stdbool.h>
// created using documentation from:
// https://dubeyko.com/development/FileSystems/NTFS/ntfsdoc.pdf
// This is the total size of a file record, including the attributes
// TODO: calculate this
#define MIN_FILE_RECORD_SIZE 1024
// The mft number is only 48bit, the other bits are used
// for something else
#define MFT_RECORD_NO_MASK (0xFFFFFFFFFFFF)
struct mft_file_record {
char name[4];
uint16_t update_seq_offset;
uint16_t update_seq_size;
uint64_t log_seq_number;
uint16_t sequence_number;
uint16_t hard_link_count;
uint16_t attribute_offset;
uint16_t flags;
uint32_t real_size;
uint32_t allocated_size;
uint64_t base_record_number;
} __attribute__((packed));
struct file_record_attr_header {
uint32_t type;
uint32_t length;
uint8_t non_res_flag;
uint8_t name_length;
uint16_t name_offset;
uint16_t flags;
uint16_t attribute_id;
} __attribute__((packed));
#define FR_ATTRIBUTE_LIST 0x00000020
#define FR_ATTRIBUTE_NAME 0x00000030
#define FR_ATTRIBUTE_VOLUME_NAME 0x00000060
#define FR_ATTRIBUTE_DATA 0x00000080
#define FR_ATTRIBUTE_INDEX_ROOT 0x00000090
#define FR_ATTRIBUTE_INDEX_ALLOC 0x000000A0
struct file_record_attr_header_res {
struct file_record_attr_header header;
uint32_t info_length;
uint16_t info_offset;
uint16_t index_flag;
} __attribute__((packed));
struct file_record_attr_header_non_res {
struct file_record_attr_header header;
uint64_t first_vcn;
uint64_t last_vcn;
uint16_t run_offset;
} __attribute__((packed));
struct file_record_attr_name {
uint64_t mft_parent_record;
uint64_t creation_time;
uint64_t altered_time;
uint64_t mft_changed_time;
uint64_t read_time;
uint64_t allocated_size;
uint64_t real_size;
uint32_t flags;
uint32_t reparse;
uint8_t name_length;
uint8_t name_type;
uint16_t name[];
} __attribute__((packed));
struct index_record {
char name[4];
uint16_t update_seq_offset;
uint16_t update_seq_size;
uint64_t log_seq_number;
uint64_t vcn;
uint32_t index_entry_offset;
uint32_t index_entry_size;
uint32_t index_entry_alloc;
uint8_t leaf_node;
uint8_t _reserved[3];
uint16_t update_seq;
uint16_t sequence_array[];
} __attribute__((packed));
struct index_entry {
uint64_t mft_record;
uint16_t entry_size;
uint16_t name_offset;
uint16_t index_Flag;
uint16_t padding;
uint64_t mft_parent_record;
uint64_t creation_time;
uint64_t altered_time;
uint64_t mft_changed_time;
uint64_t read_time;
uint64_t alloc_size;
uint64_t real_size;
uint64_t file_flags;
uint8_t name_length;
uint8_t name_type;
uint16_t name[];
} __attribute__((packed));
int ntfs_check_signature(struct volume *part) {
struct ntfs_bpb bpb;
if (!volume_read(part, &bpb, 0, sizeof(bpb))) {
return 0;
}
//
// validate the bpb
//
if (strncmp(bpb.oem, "NTFS ", SIZEOF_ARRAY(bpb.oem))) {
return 0;
}
if (bpb.sector_totals != 0) {
return 0;
}
if (bpb.large_sectors_count != 0) {
return 0;
}
if (bpb.sectors_count_64 == 0) {
return 0;
}
// this is a valid ntfs sector
return 1;
}
// the temp buffer is used for storing dirs and alike
// in memory, because limine only has allocate without
// free we are going to allocate it once globally and just
// make sure to only use it in the ntfs_open function...
static uint8_t *dir_buffer = NULL;
static size_t dir_buffer_size = 0;
static size_t dir_buffer_cap = 0;
/**
* Get an attribute from the given file record
*/
static bool ntfs_get_file_record_attr(uint8_t* file_record, uint32_t attr_type, uint8_t **out_attr) {
struct mft_file_record *fr = (struct mft_file_record *)file_record;
// get the offset to the first attribute
uint8_t *cur_attr_ptr = file_record + fr->attribute_offset;
while (true) {
// TODO: don't check for the min size, but for the actual size...
if (cur_attr_ptr + sizeof(struct file_record_attr_header) > file_record + MIN_FILE_RECORD_SIZE)
panic("NTFS: File record attribute is outside of file record");
struct file_record_attr_header *cur_attr = (struct file_record_attr_header *)cur_attr_ptr;
if (cur_attr->type == attr_type) {
*out_attr = cur_attr_ptr;
return true;
}
// we either found an attr with higher type or the end type
if (cur_attr->type > attr_type || cur_attr->type == 0xFF)
return false;
if (cur_attr->length == 0)
panic("NTFS: File record attribute has zero length");
cur_attr_ptr += cur_attr->length;
}
}
/**
* Gets a count and cluster from the runlist, if next is true then it updates the list intenrally
* so the next call will return the next element
*
* if returned false we got to the end of the file.
*/
static bool ntfs_get_next_run_list_element(uint8_t **runlist, uint64_t *out_cluster_count, uint64_t *out_cluster, bool next) {
uint8_t *runlist_ptr = *runlist;
// we have reached the end of the file
if (runlist_ptr[0] == 0) {
return false;
}
uint8_t low = runlist_ptr[0] & 0xF;
uint8_t high = (runlist_ptr[0] >> 4) & 0xF;
runlist_ptr++;
// get the run length
uint64_t count = 0;
for (int i = low; i > 0; i--) {
count <<= 8;
count |= runlist_ptr[i - 1];
}
runlist_ptr += low;
// get the high byte first
int8_t high_byte = (int8_t)runlist_ptr[high - 1];
// get the run offset
uint64_t cluster = 0;
for (int i = high; i > 0; i--) {
cluster <<= 8;
cluster |= runlist_ptr[i - 1];
}
runlist_ptr += high;
// if the offset is negative, fill the empty bytes with 0xff
if (high_byte < 0 && high < 8) {
uint64_t fill = 0;
for (int i = 8; i > high; i--) {
fill >>= 8;
fill |= 0xFF00000000000000;
}
cluster |= fill;
}
// out it
*out_cluster = cluster;
*out_cluster_count = count;
// update it
if (next) {
*runlist = runlist_ptr;
}
return true;
}
static bool ntfs_get_file_record(struct ntfs_file_handle *handle, uint64_t mft_record_no, uint8_t *file_record_buffer) {
uint8_t *runlist = handle->mft_run_list;
// make sure we only take the number itself
mft_record_no &= MFT_RECORD_NO_MASK;
// get the
uint64_t count = 0;
uint64_t cluster = 0;
if (!ntfs_get_next_run_list_element(&runlist, &count, &cluster, true)) {
return false;
}
size_t bytes_per_cluster = handle->bpb.bytes_per_sector * handle->bpb.sectors_per_cluster;
uint64_t byte_count = count * bytes_per_cluster;
uint64_t sector = cluster * handle->bpb.sectors_per_cluster;
uint64_t record_count = 0;
do {
// consume the items from the current runlist
if (byte_count > 0) {
sector += handle->sectors_per_file_record;
byte_count -= handle->file_record_size;
} else {
// get the next run list...
if (!ntfs_get_next_run_list_element(&runlist, &count, &cluster, true)) {
// reached the end of the mft, did not find it...
return false;
}
byte_count = count * bytes_per_cluster;
sector = cluster * handle->bpb.sectors_per_cluster;
continue;
}
record_count++;
} while (record_count < mft_record_no);
// we found the sector of the file record!
uint64_t offset = sector * handle->bpb.bytes_per_sector;
if(!volume_read(handle->part, file_record_buffer, offset, handle->file_record_size))
panic("NTFS: Failed to read file record from mft");
// make sure this is a valid file record
struct mft_file_record *fr = (struct mft_file_record *)file_record_buffer;
if (strncmp(fr->name, "FILE", SIZEOF_ARRAY(fr->name)))
panic("NTFS: File record has invalid signature (got %c%c%c%c, should be FILE)!",
fr->name[0], fr->name[1], fr->name[2], fr->name[3]);
// we good!
return true;
}
/**
* Read the the directory's file record from the mft
*/
static bool ntfs_read_directory(struct ntfs_file_handle *handle, uint64_t mft_record, uint8_t *file_record) {
// get the record of the directory
if (!ntfs_get_file_record(handle, mft_record, file_record))
return false;
//
// get the runlist of the directory
//
// get the index alloc attribute, it should have the runlist offset
// copy the runlist from it to our handle for easier access
uint8_t *index_alloc_ptr;
if (!ntfs_get_file_record_attr(file_record, FR_ATTRIBUTE_INDEX_ALLOC, &index_alloc_ptr))
panic("NTFS: Directory has no runlist?!");
struct file_record_attr_header_non_res *index_alloc = (struct file_record_attr_header_non_res *)index_alloc_ptr;
uint8_t *runlist_ptr = index_alloc_ptr + index_alloc->run_offset;
if (runlist_ptr - file_record + 128u > handle->file_record_size)
panic("NTFS: runlist is outside of file record!");
memcpy(handle->run_list, runlist_ptr, sizeof(handle->run_list));
// calculate the directory size by just going through the runlist
uint8_t *runlist = handle->run_list;
uint64_t dir_size = 0;
uint64_t cluster = 0;
uint64_t cluster_count = 0;
bool status = false;
do {
status = ntfs_get_next_run_list_element(&runlist, &cluster_count, &cluster, true);
if (status)
dir_size += cluster_count;
} while(status);
dir_size *= handle->bpb.sectors_per_cluster * handle->bpb.bytes_per_sector;
// allocate a buffer for the directory data
if (dir_buffer == NULL) {
// allocate enough just in case, idk how much is good
dir_buffer_cap = dir_size > 64 * 1024 ? dir_size : 64 * 1024;
dir_buffer = ext_mem_alloc(dir_buffer_cap);
} else {
// we must truncate it...
if (dir_size > dir_buffer_cap) {
dir_size = dir_buffer_cap;
}
}
// set the size of the dir size
dir_buffer_size = dir_size;
// read the directory
if (ntfs_read(handle, dir_buffer, 0, dir_size))
panic("NTFS: EOF before reading directory fully...");
return true;
}
/**
* Prepare for reading a file by reading the root directory into the file handle
*/
static void ntfs_read_root(struct ntfs_file_handle *handle) {
// calculate the offset for the mft
handle->mft_offset = (uint64_t)handle->bpb.mft_cluster * (uint64_t)handle->bpb.sectors_per_cluster * (uint64_t)handle->bpb.bytes_per_sector;
// read the mft file record, this should be the size of a sector
uint8_t file_record_buffer[MIN_FILE_RECORD_SIZE];
if (!volume_read(handle->part, file_record_buffer, handle->mft_offset, sizeof(file_record_buffer)))
panic("NTFS: Failed to read MFT file record");
// get the file attribute
uint8_t *attr_ptr = NULL;
if (!ntfs_get_file_record_attr(file_record_buffer, FR_ATTRIBUTE_DATA, &attr_ptr))
panic("NTFS: MFT file record missing DATA attribute");
struct file_record_attr_header_non_res *attr = (struct file_record_attr_header_non_res *)attr_ptr;
// verify the attr and run list are in the buffer
if ((uint8_t *)attr + sizeof(*attr) > file_record_buffer + sizeof(file_record_buffer))
panic("NTFS: MFT file record attribute is outside of file record");
if ((uint8_t *)attr + attr->run_offset + 256 > file_record_buffer + sizeof(file_record_buffer))
panic("NTFS: MFT Run list is outside of file record");
// save the run list
memcpy(handle->mft_run_list, (uint8_t *)attr + attr->run_offset, sizeof(handle->mft_run_list));
// read the root directory record, which has the number 5
if (!ntfs_read_directory(handle, 5, file_record_buffer))
panic("NTFS: Missing root directory file record!");
}
/**
* Search for a file in the ntfs directory, assumes the directory has been read and is stored in
* the temp buffer
*/
static bool ntfs_find_file_in_directory(struct ntfs_file_handle *handle, const char* filename, struct index_entry** out_entry) {
size_t dir_size = dir_buffer_size;
uint8_t *dir_ptr = dir_buffer;
// TODO: iterate resident record...
// get the size of the name we need to compare
const char* temp_filename = filename;
size_t filename_size = 0;
while (*temp_filename != '\0' && *temp_filename != '\\' && *temp_filename != '/') {
filename_size++;
temp_filename++;
}
// iterate the non-resident files in the directory
size_t offset = 0;
while (dir_size) {
// check if the dir pointer is still in the buffer, if not then we could
// not find the file...
if (dir_ptr + sizeof(struct index_record) > dir_buffer + dir_buffer_size)
panic("NTFS: Tried to read index record outside of directory");
// get the index and check it, if it is not valid just return
// we did not find the file
struct index_record *index_record = (struct index_record *)dir_ptr;
if (strncmp(index_record->name, "INDX", SIZEOF_ARRAY(index_record->name)))
return false;
// calculate the offset to the entry
size_t index_size = index_record->index_entry_size;
offset += index_record->index_entry_offset + offsetof(struct index_record, index_entry_offset);
uint8_t *entry_ptr = dir_ptr + offset;
// loop the record for all of its indexes
while (index_size) {
// make sure we still have an entry
if (entry_ptr + sizeof(struct index_entry) > dir_buffer + dir_buffer_size)
panic("NTFS: Tried to read index entry outside of directory");
// get the entry, if size is zero we done
struct index_entry *entry = (struct index_entry *)entry_ptr;
if (entry->entry_size == 0)
break;
if (filename_size == entry->name_length) {
// this name seem legit, need to get the real name from the mft
// sometimes it works to use the index name but sometimes it has
// invalid names for whatever reason that I can not understand, so
// just always take it from the mft file record
uint8_t file_record_buffer[MIN_FILE_RECORD_SIZE];
if (!ntfs_get_file_record(handle, entry->mft_record, file_record_buffer))
panic("NTFS: Failed to get file record");
uint8_t *name_attr = NULL;
if (!ntfs_get_file_record_attr(file_record_buffer, FR_ATTRIBUTE_NAME, &name_attr))
panic("NTFS: File record missing name attribute");
// get the offset to the actual info
struct file_record_attr_header_res *header = (struct file_record_attr_header_res *)name_attr;
struct file_record_attr_name *name = (struct file_record_attr_name *)(name_attr + header->info_offset);
// compare the name
for (int i = 0; i < name->name_length; i++) {
if (name->name[i] != filename[i]) {
goto next_entry;
}
}
// name is good, return the entry and return true
// that we found the entry
*out_entry = entry;
return true;
}
// next entry
next_entry:
entry_ptr += entry->entry_size;
index_size -= entry->entry_size;
}
// next record, need to do some rounding
index_size = index_record->index_entry_size;
if (index_size < 0x1000) {
index_size = 0x1000;
} else {
index_size = (index_size + 0x100) & 0xffffff00;
}
dir_ptr += index_size;
dir_size -= index_size;
}
return false;
}
int ntfs_open(struct ntfs_file_handle *ret, struct volume *part, const char *path) {
// save the part
ret->part = part;
// start by reading the bpb so we can access it later on
if (!volume_read(part, &ret->bpb, 0, sizeof(ret->bpb)))
panic("NTFS: Failed to read the BPB");
// in NTFS sector size can be 512 to 4096 bytes, file records are
// at least 1024 bytes, in here calculate the sectors per file record
// and the file record size
if (ret->bpb.bytes_per_sector <= MIN_FILE_RECORD_SIZE) {
// this has multiple sectors
ret->sectors_per_file_record = MIN_FILE_RECORD_SIZE / ret->bpb.bytes_per_sector;
ret->file_record_size = MIN_FILE_RECORD_SIZE;
} else {
// this has a single sector
ret->sectors_per_file_record = 1;
ret->file_record_size = ret->bpb.bytes_per_sector;
}
if (ret->file_record_size != MIN_FILE_RECORD_SIZE)
panic("NTFS: TODO: support file record size which is not 1024 bytes");
// now prepare the root directory so we can search for
// the rest of the stuff
ntfs_read_root(ret);
// iterate the directories to find the entry
const char* current_path = path;
struct index_entry* entry = NULL;
for (;;) {
// skip slash
while (*current_path == '\\' || *current_path == '/') {
current_path++;
}
// find the file in the directory
entry = NULL;
if (!ntfs_find_file_in_directory(ret, current_path, &entry))
return 1;
size_t filename_len = entry->name_length;
// check if this is the last entry
uint8_t file_record_buffer[MIN_FILE_RECORD_SIZE];
if (*(current_path + filename_len) == '\0') {
// we found the file!
ret->size_bytes = entry->real_size;
// get its runlist...
if (!ntfs_get_file_record(ret, entry->mft_record, file_record_buffer))
panic("NTFS: Failed to get file record of file");
// get the file attribute
uint8_t *attr_ptr = NULL;
if (!ntfs_get_file_record_attr(file_record_buffer, FR_ATTRIBUTE_DATA, &attr_ptr))
panic("NTFS: File record missing DATA attribute");
struct file_record_attr_header_non_res *attr = (struct file_record_attr_header_non_res *)attr_ptr;
// verify the attr and run list are in the buffer
if ((uint8_t *)attr + sizeof(*attr) > file_record_buffer + sizeof(file_record_buffer))
panic("NTFS: File record attribute is outside of file record");
if ((uint8_t *)attr + attr->run_offset + 256 > file_record_buffer + sizeof(file_record_buffer))
panic("NTFS: Run list is outside of file record");
// save the run list
memcpy(ret->run_list, (uint8_t *)attr + attr->run_offset, sizeof(ret->run_list));
return 0;
} else {
// read the directory
if (!ntfs_read_directory(ret, entry->mft_record, file_record_buffer))
panic("NTFS: Failed to read directory");
// next path element
current_path += filename_len;
}
}
// should not be able to reach here...
__builtin_unreachable();
}
int ntfs_read(struct ntfs_file_handle *file, void *buf, uint64_t loc, uint64_t count) {
// get the runlist
uint8_t *runlist = file->run_list;
// TODO: remember the last read location so we can have faster sequential reads...
// we are going to go over the runlist until we get to the offset
// once we get to the offset we are going to continue going over
// the runlist while copying bytes
uint64_t bytes_per_cluster = file->bpb.sectors_per_cluster * file->bpb.bytes_per_sector;
do {
// get the next element from the runlist
uint64_t cluster_count;
uint64_t cluster;
if (!ntfs_get_next_run_list_element(&runlist, &cluster_count, &cluster, true))
break;
// calculate the cont size and offset on disk
uint64_t total_cont_bytes = cluster_count * bytes_per_cluster;
uint64_t abs_byte = cluster * bytes_per_cluster;
// check if we arrived at the wanted offset
if (loc != 0) {
if (loc >= total_cont_bytes) {
// we need to go more...
loc -= total_cont_bytes;
} else {
// we got to the offset, adjust base and size
// and set the loc to 0
total_cont_bytes -= loc;
abs_byte += loc;
loc = 0;
}
}
if (loc == 0) {
// get how much we wanna read now and
// subtract that from the total we need
// to read
size_t read_now = total_cont_bytes;
if (read_now > count) {
read_now = count;
}
count -= read_now;
// read it!
if (!volume_read(file->part, buf, abs_byte, read_now))
panic("NTFS: Runlist points to outside the volume (%x)", abs_byte);
}
} while(count);
// if we didn't read it all then we got a problem
return count != 0;
}