Improve filesystem (#24)

* Add doc about fs * Add console on diskless boot * Enumerate all disks * Avoid hardcoded allocation of root dir * Refactor ATA code * Add BlockDevice * Add mkfs command * Update readme * Add warning in the readme about disk modifications * Use all bits inside BlockBitmap data * Add makefile * Overwrite only bootloader and kernel in disk image * Update readme * Add doc about mkfs * Update mkfs command
2020-02-13 09:42:22 +01:00 · 2020-02-13 09:42:22 +01:00 · 0a4b26a8bb
parent d5ff99c26c
commit 0a4b26a8bb
12 changed files with 263 additions and 140 deletions
--- a/33
+++ b/33
@ -0,0 +1,33 @@
+.PHONY: setup image qemu
+.EXPORT_ALL_VARIABLES:
+
+setup:
+	curl https://sh.rustup.rs -sSf | sh
+	rustup install nightly
+	rustup default nightly
+	rustup component add rust-src
+	rustup component add llvm-tools-preview
+	cargo install cargo-xbuild bootimage
+
+output = vga
+keyboard = qwerty
+
+bin=target/x86_64-moros/release/bootimage-moros.bin
+img=disk.img
+
+$(img):
+	qemu-img create $(img) 32M
+
+# Rebuild MOROS if the features list changed
+image: $(img)
+	touch src/lib.rs
+	cargo bootimage --no-default-features --features $(output),$(keyboard) --release
+	dd conv=notrunc if=$(bin) of=$(img)
+
+opts = -cpu max -nic model=rtl8139 -hda $(img)
+ifeq ($(output),serial)
+	opts += -display none -serial stdio
+endif
+
+qemu:
+	qemu-system-x86_64 $(opts)
--- a/README.md
+++ b/README.md
@ -52,36 +52,26 @@ Install tools:
    rustup component add llvm-tools-preview
    cargo install cargo-xbuild bootimage

-Create disk:
-
-    qemu-img create disk.img 128M
-
 ## Usage

-QEMU with VGA Text Mode:
+Build image:

-    cargo xrun --release -- \
-      -cpu phenom \
-      -nic model=rtl8139 \
-      -hdc disk.img
+    make image output=vga keyboard=qwerty

-QEMU with a serial console:
+Run on QEMU:

-    cargo xrun --release --no-default-features --features serial,dvorak -- \
-      -cpu phenom \
-      -nic model=rtl8139 \
-      -serial stdio \
-      -display none \
-      -hdc disk.img
+    make qemu output=vga

-Bochs instead of QEMU:
+Run on a native x86 computer:

-    sh run/bochs.sh
+    sudo dd if=target/x86_64-moros/release/bootimage-moros.bin of=/dev/sdx && sync
+    sudo reboot

-Or with `cool-retro-term` for a retro console look:
-
-    sh run/cool-retro-term.sh
+MOROS will open a console in diskless mode after boot if no filesystem is
+detected. Use `mkfs` to create a filesystem on a disk.

+**Be careful not to overwrite the disk of your OS when using `dd` inside your OS
+or `mkfs` inside MOROS.**

 ## LICENSE

--- a/doc/filesystem.md
+++ b/doc/filesystem.md
@ -0,0 +1,71 @@
+# MOROS Filesystem
+
+## Disk
+
+A disk is separated in block of 512 bytes, grouped into three areas. The first
+is reserved for future uses, the second is used as a bitmap mapping the
+allocated blocks in the third area. The data stored on the disk use the blocks
+of the third area.
+
+During the first boot of the OS, the root dir will be allocated, using the
+first block of the data area.
+
+A location on the tree of dirs and files is named a path:
+
+  - The root dir is represented by a slash: `/`
+  - A dir inside the root will have its name appended to the slash: `/usr`
+  - Subsequent dirs will append a slash and their names: `/usr/admin`
+
+### Creation with QEMU
+
+    $ qemu-img create disk.img 128M
+    Formatting 'disk.img', fmt=raw size=134217728
+
+### Setup in diskless console
+
+During boot MOROS will detect the disks present on the ATA buses, then the
+filesystems on those disks. If no filesystem is found, MOROS will open a
+console in diskless mode to allow the user to create one with the `mkfs`
+command:
+
+    > mkfs /dev/ata/0/0
+
+## Data
+
+### BlockBitmap
+
+Bitmap of allocated blocks in the data area.
+
+### Block
+
+A block is small area of 512 bytes on a disk, and it is also part of linked
+list representing a file or a directory.
+
+The first 4 bytes of a block is the address of the next block on the list and
+the rest of block is the data stored in the block.
+
+### DirEntry
+
+A directory entry represent a file or a directory contained inside a directory.
+Each entry use a variable number of bytes that must fit inside the data of one
+block. Those bytes represent the kind of entry (file or dir), the address of
+the first block, the filesize (max 4GB), and the filename (max 255 chars) of
+the entry.
+
+Structure:
+
+  - 0..1: kind
+  - 1..5: addr
+  - 5..10: size
+  - 10..11: name len
+  - 11..n: name buf
+
+### Dir
+
+A directory contains the address of the first block where its directory entries
+are stored.
+
+### File
+
+A file contains the address of its first block along with its filesize and
+filename, and a reference to its parent directory.
--- a/doc/network.md
+++ b/doc/network.md
@ -1,4 +1,4 @@
-# MOROS Net
+# MOROS Network

 ## NET

--- a/run/bochs.sh
+++ b/run/bochs.sh
@ -4,9 +4,6 @@ set -e

 dir=$(dirname "$0")

-# Build image if needed
-cd "$dir/.." && cargo bootimage --release
-
 # Clean up lock files that Bochs creates
 rm -f "$dir/../target/x86_64-moros/release/bootimage-moros.bin.lock"
 rm -f "$dir/../disk.img.lock"
--- a/run/cool-retro-term.sh
+++ b/run/cool-retro-term.sh
@ -3,14 +3,10 @@
 set -e

 dir=$(dirname "$0")
-image="target/x86_64-moros/release/bootimage-moros.bin"
-qemu="qemu-system-x86_64 -display curses -cpu max -nic model=rtl8139 -hdc disk.img"
+qemu="qemu-system-x86_64 -display curses -cpu max -nic model=rtl8139 disk.img"
 #qemu="qemu-system-x86_64 -display curses -cpu max -hdc disk.img -netdev user,id=u1,hostfwd=tcp::2222-:22 -device rtl8139,netdev=u1 -object filter-dump,id=f1,netdev=u1,file=/tmp/qemu.pcap"

-# Build image if needed
-cd "$dir/.." && cargo bootimage --release
-
 echo "The MOROS theme at '$dir/cool-retro-term.json' have to be manually imported."

 # Launch qemu inside cool-retro-term
-cool-retro-term --fullscreen --profile "MOROS" --workdir "$dir/.." -e sh -c "$qemu $image 2>/dev/null"
+cool-retro-term --fullscreen --profile "MOROS" --workdir "$dir/.." -e sh -c "$qemu 2>/dev/null"
--- a/src/kernel/ata.rs
+++ b/src/kernel/ata.rs
@ -194,6 +194,8 @@ impl Bus {
    }

    pub fn read(&mut self, drive: u8, block: u32, buf: &mut [u8]) {
+        assert!(buf.len() == 512);
+
        self.setup(drive, block);

        self.write_command(Command::Read);
@ -208,6 +210,8 @@ impl Bus {
    }

    pub fn write(&mut self, drive: u8, block: u32, buf: &[u8]) {
+        assert!(buf.len() == 512);
+
        self.setup(drive, block);

        self.write_command(Command::Write);
@ -226,7 +230,7 @@ impl Bus {
 }

 lazy_static! {
-    pub static ref ATA_BUSES: Mutex<Vec<Bus>> = Mutex::new(Vec::new());
+    pub static ref BUSES: Mutex<Vec<Bus>> = Mutex::new(Vec::new());
 }

 fn disk_size(sectors: u32) -> (u32, String) {
@ -239,74 +243,39 @@ fn disk_size(sectors: u32) -> (u32, String) {
 }

 pub fn init() {
-    let mut buses = ATA_BUSES.lock();
+    let mut buses = BUSES.lock();
    buses.push(Bus::new(0, 0x1F0, 0x3F6, 14));
    buses.push(Bus::new(1, 0x170, 0x376, 15));

-    let bus = 1;
-    let drive = 0;
-    if let Some(buf) = buses[bus].identify_drive(drive) {
-        let mut serial = String::new();
-        for i in 10..20 {
-            for &b in &buf[i].to_be_bytes() {
-                serial.push(b as char);
+    for bus in 0..2 {
+        for drive in 0..2 {
+            if let Some(buf) = buses[bus].identify_drive(drive) {
+                let mut serial = String::new();
+                for i in 10..20 {
+                    for &b in &buf[i].to_be_bytes() {
+                        serial.push(b as char);
+                    }
+                }
+                let mut model = String::new();
+                for i in 27..47 {
+                    for &b in &buf[i].to_be_bytes() {
+                        model.push(b as char);
+                    }
+                }
+                let sectors = (buf[61] as u32) << 16 | (buf[60] as u32);
+                let (size, unit) = disk_size(sectors);
+                log!("ATA {}:{} {} {} ({} {})\n", bus, drive, model.trim(), serial.trim(), size, unit);
            }
        }
-        let mut model = String::new();
-        for i in 27..47 {
-            for &b in &buf[i].to_be_bytes() {
-                model.push(b as char);
-            }
-        }
-        let sectors = (buf[61] as u32) << 16 | (buf[60] as u32);
-        let (size, unit) = disk_size(sectors);
-        log!("ATA {}:{} {} {} ({} {})\n", bus, drive, model.trim(), serial.trim(), size, unit);
    }
-
-    /*
-    let block = 1;
-    let mut buf = [0u8; 512];
-    buses[1].read(drive, block, &mut buf);
-    for i in 0..256 {
-        if i % 8 == 0 {
-            print!("\n{:08X} ", i * 2);
-        }
-        print!("{:02X}{:02X} ", buf[i * 2], buf[i * 2 + 1]);
-    }
-    print!("\n");
-
-    buf[0x42] = 'H' as u8;
-    buf[0x43] = 'e' as u8;
-    buf[0x44] = 'l' as u8;
-    buf[0x45] = 'l' as u8;
-    buf[0x46] = 'o' as u8;
-    for i in 0..256 {
-        if i % 8 == 0 {
-            print!("\n{:08X} ", i * 2);
-        }
-        print!("{:02X}{:02X} ", buf[i * 2], buf[i * 2 + 1]);
-    }
-    print!("\n");
-    buses[1].write(drive, block, &mut buf);
-
-    let mut buf = [0u8; 512];
-    buses[1].read(drive, block, &mut buf);
-    for i in 0..256 {
-        if i % 8 == 0 {
-            print!("\n{:08X} ", i * 2);
-        }
-        print!("{:02X}{:02X} ", buf[i * 2], buf[i * 2 + 1]);
-    }
-    print!("\n");
-    */
 }

 pub fn read(bus: u8, drive: u8, block: u32, mut buf: &mut [u8]) {
-    let mut buses = ATA_BUSES.lock();
+    let mut buses = BUSES.lock();
    buses[bus as usize].read(drive, block, &mut buf);
 }

 pub fn write(bus: u8, drive: u8, block: u32, buf: &[u8]) {
-    let mut buses = ATA_BUSES.lock();
+    let mut buses = BUSES.lock();
    buses[bus as usize].write(drive, block, &buf);
 }
--- a/src/kernel/fs.rs
+++ b/src/kernel/fs.rs
@ -2,7 +2,9 @@ use alloc::format;
 use alloc::string::String;
 use alloc::vec::Vec;
 use bit_field::BitField;
-use crate::kernel;
+use crate::{kernel, log};
+use lazy_static::lazy_static;
+use spin::Mutex;

 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum FileType {
@ -186,17 +188,17 @@ impl Block {
    }

    pub fn read(addr: u32) -> Self {
-        let bus = 1; // TODO
-        let dsk = 0; // TODO
        let mut buf = [0; 512];
-        kernel::ata::read(bus, dsk, addr, &mut buf);
+        if let Some(ref block_device) = *BLOCK_DEVICE.lock() {
+            block_device.read(addr, &mut buf);
+        }
        Self { addr, buf }
    }

    pub fn write(&self) {
-        let bus = 1; // TODO
-        let dsk = 0; // TODO
-        kernel::ata::write(bus, dsk, self.addr, &self.buf);
+        if let Some(ref block_device) = *BLOCK_DEVICE.lock() {
+            block_device.write(self.addr, &self.buf);
+        }
    }

    pub fn alloc() -> Option<Self> {
@ -257,8 +259,10 @@ impl Block {
 const BITMAP_SIZE: u32 = 512 - 4; // TODO: Bitmap should use the full block
 const MAX_BLOCKS: u32 = 2 * 2048;

-const BITMAP_ADDR_OFFSET: u32 = 2048 + 2;
-const DATA_ADDR_OFFSET: u32 = BITMAP_ADDR_OFFSET + MAX_BLOCKS;
+const DISK_OFFSET: u32 = (1 << 20) / 512;
+const SUPERBLOCK_ADDR: u32 = DISK_OFFSET;
+const BITMAP_ADDR_OFFSET: u32 = DISK_OFFSET + 2;
+const DATA_ADDR_OFFSET: u32 = BITMAP_ADDR_OFFSET + MAX_BLOCKS / 8;

 /* Disk Areas
 * 1 => Reserved
@ -266,38 +270,54 @@ const DATA_ADDR_OFFSET: u32 = BITMAP_ADDR_OFFSET + MAX_BLOCKS;
 * 3 => Data (directories and files)
 */

+// A BlockBitmap store the allocation status of (512 -4) * 8 data blocks
 pub struct BlockBitmap {}

 impl BlockBitmap {
+    fn block_index(data_addr: u32) -> u32 {
+        let i = data_addr - DATA_ADDR_OFFSET;
+        BITMAP_ADDR_OFFSET + (i / BITMAP_SIZE / 8)
+    }
+
+    fn buffer_index(data_addr: u32) -> usize {
+        let i = data_addr - DATA_ADDR_OFFSET;
+        (i % BITMAP_SIZE) as usize
+    }
+
    pub fn is_free(addr: u32) -> bool {
-        let block = Block::read(BITMAP_ADDR_OFFSET + ((addr - DATA_ADDR_OFFSET) / BITMAP_SIZE));
+        let block = Block::read(BlockBitmap::block_index(addr));
        let bitmap = block.data(); // TODO: Add block.buffer()
-        bitmap[((addr - DATA_ADDR_OFFSET) % BITMAP_SIZE) as usize] == 0
+        let i = BlockBitmap::buffer_index(addr);
+        bitmap[i / 8].get_bit(i % 8)
    }

    pub fn alloc(addr: u32) {
-        let mut block = Block::read(BITMAP_ADDR_OFFSET + ((addr - DATA_ADDR_OFFSET) / BITMAP_SIZE));
+        let mut block = Block::read(BlockBitmap::block_index(addr));
        let bitmap = block.data_mut();
-        bitmap[((addr - DATA_ADDR_OFFSET) % BITMAP_SIZE) as usize] = 1;
+        let i = BlockBitmap::buffer_index(addr);
+        bitmap[i / 8].set_bit(i % 8, true);
        block.write();
    }

    pub fn free(addr: u32) {
-        let mut block = Block::read(BITMAP_ADDR_OFFSET + ((addr - DATA_ADDR_OFFSET) / BITMAP_SIZE));
+        let mut block = Block::read(BlockBitmap::block_index(addr));
        let bitmap = block.data_mut();
-        bitmap[((addr - DATA_ADDR_OFFSET) % BITMAP_SIZE) as usize] = 0;
+        let i = BlockBitmap::buffer_index(addr);
+        bitmap[i / 8].set_bit(i % 8, false);
        block.write();
    }

    pub fn next_free_addr() -> Option<u32> {
-        let n = MAX_BLOCKS / BITMAP_SIZE;
+        let n = MAX_BLOCKS / BITMAP_SIZE / 8;
        for i in 0..n {
            let block = Block::read(BITMAP_ADDR_OFFSET + i);
            let bitmap = block.data();
            for j in 0..BITMAP_SIZE {
-                if bitmap[j as usize] == 0 {
-                    let addr = DATA_ADDR_OFFSET + i * 512 + j;
-                    return Some(addr);
+                for k in 0..8 {
+                    if !bitmap[j as usize].get_bit(k) {
+                        let addr = DATA_ADDR_OFFSET + i * 512 * 8 + j * 8 + k as u32;
+                        return Some(addr);
+                    }
                }
            }
        }
@ -602,42 +622,55 @@ impl Iterator for ReadDir {
    }
 }

-pub fn init() {
-    let root = Dir::root();
+pub struct BlockDevice {
+    bus: u8,
+    dsk: u8,
+}

-    // Allocate root dir on new filesystems
+impl BlockDevice {
+    pub fn new(bus: u8, dsk: u8) -> Self {
+        Self { bus, dsk }
+    }
+
+    pub fn read(&self, block: u32, mut buf: &mut [u8]) {
+        kernel::ata::read(self.bus, self.dsk, block, &mut buf);
+    }
+
+    pub fn write(&self, block: u32, buf: &[u8]) {
+        kernel::ata::write(self.bus, self.dsk, block, &buf);
+    }
+}
+
+lazy_static! {
+    pub static ref BLOCK_DEVICE: Mutex<Option<BlockDevice>> = Mutex::new(None);
+}
+
+const MAGIC: &'static str = "MOROS FS";
+
+pub fn make(bus: u8, dsk: u8) {
+    // Write superblock
+    let mut buf = MAGIC.as_bytes().to_vec();
+    buf.resize(512, 0);
+    let block_device = BlockDevice::new(bus, dsk);
+    block_device.write(SUPERBLOCK_ADDR, &buf);
+    *BLOCK_DEVICE.lock() = Some(block_device);
+
+    // Allocate root dir
+    let root = Dir::root();
    if BlockBitmap::is_free(root.addr()) {
        BlockBitmap::alloc(root.addr());
    }
+}

-    /*
-    if root.find("test").is_none() {
-        match File::create("/test") {
-            Some(test) => {
-                print!("Created '/test' at block 0x{:08X}\n", test.addr());
-            },
-            None => {
-                print!("Could not create '/test'\n");
+pub fn init() {
+    for bus in 0..2 {
+        for dsk in 0..2 {
+            let mut buf = [0u8; 512];
+            kernel::ata::read(bus, dsk, SUPERBLOCK_ADDR, &mut buf);
+            if String::from_utf8(buf[0..8].to_vec()).unwrap() == MAGIC {
+                log!("MFS Superblock found on ATA {}:{}\n", bus, dsk);
+                *BLOCK_DEVICE.lock() = Some(BlockDevice::new(bus, dsk));
            }
        }
    }
-
-    if let Some(mut file) = File::open("/test") {
-        let contents = "Yolo";
-        file.write(&contents.as_bytes()).unwrap();
-        print!("Wrote to '/test'\n");
-    } else {
-        print!("Could not open '/test'\n");
-    }
-
-    if let Some(file) = File::open("/test") {
-        print!("Reading '/test':\n");
-        print!("{}\n", file.read_to_string());
-    } else {
-        print!("Could not open '/test'\n");
-    }
-
-    let uptime = kernel::clock::uptime();
-    print!("[{:.6}] FS Reading root directory ({} entries)\n", uptime, root.read().count());
-    */
 }
--- a/src/main.rs
+++ b/src/main.rs
@ -30,7 +30,15 @@ fn main(boot_info: &'static BootInfo) -> ! {
    include_file("/ini/passwords.csv", include_str!("../dsk/ini/passwords.csv"));
    include_file("/tmp/alice.txt", include_str!("../dsk/tmp/alice.txt"));
    loop {
-        user::shell::main(&["shell", "/ini/boot.sh"]);
+        let bootrc = "/ini/boot.sh";
+        if kernel::fs::File::open(bootrc).is_some() {
+            user::shell::main(&["shell", bootrc]);
+        } else {
+            print!("Could not find '{}'\n", bootrc);
+            print!("Running console in diskless mode\n");
+            //print!("Use `mkfs` and `reboot` to setup MOROS on disk\n");
+            user::shell::main(&["shell"]);
+        }
    }
 }

--- a/src/user/mkfs.rs
+++ b/src/user/mkfs.rs
@ -0,0 +1,24 @@
+use alloc::vec::Vec;
+use crate::{print, kernel, user};
+
+// Example: mkfs /dev/ata/0/0
+pub fn main(args: &[&str]) -> user::shell::ExitCode {
+    if args.len() != 2 {
+        print!("Usage: mkfs /dev/ata/<bus>/<dsk>\n");
+        return user::shell::ExitCode::CommandError;
+    }
+
+    let path: Vec<_> = args[1].split('/').collect();
+
+    if path.len() != 5 {
+        print!("Could not recognize <device>\n");
+        return user::shell::ExitCode::CommandError;
+    }
+
+    let bus = path[3].parse().expect("Could not parse <bus>");
+    let dsk = path[4].parse().expect("Could not parse <dsk>");
+    kernel::fs::make(bus, dsk);
+    print!("MFS setup on ATA {}:{}\n", bus, dsk);
+
+    user::shell::ExitCode::CommandSuccessful
+}
--- a/src/user/mod.rs
+++ b/src/user/mod.rs
@ -1,7 +1,7 @@
 pub mod base64;
 pub mod clear;
-pub mod copy;
 pub mod colors;
+pub mod copy;
 pub mod date;
 pub mod delete;
 pub mod dhcp;
@ -10,11 +10,12 @@ pub mod geotime;
 pub mod halt;
 pub mod help;
 pub mod hex;
-pub mod http;
 pub mod host;
+pub mod http;
 pub mod ip;
 pub mod list;
 pub mod login;
+pub mod mkfs;
 pub mod net;
 pub mod print;
 pub mod r#move;
--- a/src/user/shell.rs
+++ b/src/user/shell.rs
@ -350,6 +350,7 @@ impl Shell {
            "ip"                   => user::ip::main(&args),
            "geotime"              => user::geotime::main(&args),
            "colors"               => user::colors::main(&args),
+            "mkfs"                 => user::mkfs::main(&args),
            _                      => ExitCode::CommandUnknown,
        }
    }