Redesign the fakefs database to support hardlinks

zerocc2014 · Jan 8, 2018 · 37de7b9 · 37de7b9
1 parent a12587e
commit 37de7b9
Show file tree

Hide file tree

Showing 5 changed files with 109 additions and 86 deletions.
diff --git a/fs/fake.c b/fs/fake.c
@@ -9,6 +9,8 @@
 #include "kernel/process.h"
 #include "kernel/fs.h"
 
+// TODO document database
+
 struct ish_stat {
     dword_t mode;
     dword_t uid;
@@ -25,74 +27,65 @@ static noreturn void gdbm_err(GDBM_FILE db) {
     abort();
 }
 
-static GDBM_FILE get_db(struct mount *mount) {
-    GDBM_FILE db = mount->data;
-    if (db == NULL) {
-        char db_path[PATH_MAX];
-        strcpy(db_path, mount->source);
-        char *basename = strrchr(db_path, '/') + 1;
-        assert(strcmp(basename, "data") == 0);
-        strncpy(basename, "meta.db", 7);
-        db = gdbm_open(db_path, 0, GDBM_WRITER, 0, gdbm_fatal);
-        if (db == NULL) {
-            println("gdbm error: %s", gdbm_strerror(gdbm_errno));
-            abort();
-        }
-        mount->data = db;
-    }
-    return db;
+static datum make_datum(char *data, const char *format, ...) {
+    va_list args;
+    va_start(args, format);
+    int n = vsprintf(data, format, args);
+    va_end(args);
+    return (datum) {.dptr = data, .dsize = n};
 }
 
-static datum build_key(char *keydata, const char *path, const char *type) {
-    strcpy(keydata, type);
-    strcpy(keydata + strlen(type) + 1, path);
-    datum key = {.dptr = keydata, .dsize = strlen(type) + 1 + strlen(path)};
-    return key;
+static datum read_meta(struct mount *mount, datum key) {
+    datum value = gdbm_fetch(mount->db, key);
+    if (value.dptr == NULL && gdbm_last_errno(mount->db) != GDBM_ITEM_NOT_FOUND)
+        gdbm_err(mount->db);
+    return value;
 }
 
-static datum read_meta(struct mount *mount, const char *path, const char *type) {
-    char keydata[MAX_PATH+strlen(type)+1];
-    datum key = build_key(keydata, path, type);
-    GDBM_FILE db = get_db(mount);
-    datum value = gdbm_fetch(db, key);
-    if (value.dptr == NULL && gdbm_last_errno(db) != GDBM_ITEM_NOT_FOUND)
-        gdbm_err(db);
-    return value;
+static void write_meta(struct mount *mount, datum key, datum data) {
+    if (gdbm_store(mount->db, key, data, GDBM_REPLACE) == -1)
+        gdbm_err(mount->db);
 }
 
-static void write_meta(struct mount *mount, const char *path, const char *type, datum data) {
-    char keydata[MAX_PATH+strlen(type)+1];
-    datum key = build_key(keydata, path, type);
-    if (gdbm_store(get_db(mount), key, data, GDBM_REPLACE) == -1)
-        gdbm_err(get_db(mount));
+static void delete_meta(struct mount *mount, datum key) {
+    if (gdbm_delete(mount->db, key) == -1 && gdbm_last_errno(mount->db) != GDBM_ITEM_NOT_FOUND)
+        gdbm_err(mount->db);
 }
 
-static void delete_meta(struct mount *mount, const char *path, const char *type) {
-    char keydata[MAX_PATH+strlen(type)+1];
-    datum key = build_key(keydata, path, type);
-    GDBM_FILE db = get_db(mount);
-    if (gdbm_delete(db, key) == -1 && gdbm_last_errno(db) != GDBM_ITEM_NOT_FOUND)
-        gdbm_err(db);
+static datum stat_key(char *data, struct mount *mount, const char *path) {
+    struct stat stat;
+    if (fstatat(mount->root_fd, fix_path(path), &stat, AT_SYMLINK_NOFOLLOW) < 0)
+        return (datum) {.dptr = NULL, .dsize = 0};
+
+    // remember that this path has this inode in case of tarball
+    char keydata[MAX_PATH+strlen("path")+1];
+    char valuedata[30];
+    write_meta(mount, 
+            make_datum(keydata, "inode %s", path), 
+            make_datum(valuedata, "%lu", stat.st_ino));
+
+    return make_datum(data, "stat %lu", stat.st_ino);
 }
 
-static int read_stat(struct mount *mount, const char *path, struct ish_stat *stat) {
-    datum d = read_meta(mount, path, "meta");
+static bool read_stat(struct mount *mount, const char *path, struct ish_stat *stat) {
+    char keydata[30];
+    datum d = read_meta(mount, stat_key(keydata, mount, path));
     if (d.dptr == NULL)
-        return 0;
+        return false;
     assert(d.dsize == sizeof(struct ish_stat));
     *stat = *(struct ish_stat *) d.dptr;
-    return 1;
+    free(d.dptr);
+    return true;
 }
 
 static void write_stat(struct mount *mount, const char *path, struct ish_stat *stat) {
+    char keydata[30];
+    datum key = stat_key(keydata, mount, path);
+    assert(key.dptr != NULL);
     datum data;
     data.dptr = (void *) stat;
     data.dsize = sizeof(struct ish_stat);
-    write_meta(mount, path, "meta", data);
-}
-
-static void delete_stat(struct mount *mount, const char *path) {
-    delete_meta(mount, path, "meta");
+    write_meta(mount, key, data);
 }
 
 static struct fd *fakefs_open(struct mount *mount, const char *path, int flags, int mode) {
@@ -111,22 +104,12 @@ static struct fd *fakefs_open(struct mount *mount, const char *path, int flags,
 }
 
 static int fakefs_unlink(struct mount *mount, const char *path) {
+    char keydata[30];
+    datum key = stat_key(keydata, mount, path);
     int err = realfs.unlink(mount, path);
     if (err < 0)
         return err;
-    delete_stat(mount, path);
-    return 0;
-}
-
-static int fakefs_rename(struct mount *mount, const char *src, const char *dst) {
-    int err = realfs.rename(mount, src, dst);
-    if (err < 0)
-        return err;
-    struct ish_stat stat;
-    if (!read_stat(mount, src, &stat))
-        return _ENOENT;
-    delete_stat(mount, src);
-    write_stat(mount, dst, &stat);
+    delete_meta(mount, key);
     return 0;
 }
 
@@ -243,7 +226,35 @@ static ssize_t fakefs_readlink(struct mount *mount, const char *path, char *buf,
 }
 
 static int fakefs_mount(struct mount *mount) {
-    // TODO maybe open the database here
+    char db_path[PATH_MAX];
+    strcpy(db_path, mount->source);
+    char *basename = strrchr(db_path, '/') + 1;
+    assert(strcmp(basename, "data") == 0);
+    strncpy(basename, "meta.db", 7);
+    mount->db = gdbm_open(db_path, 0, GDBM_WRITER, 0, gdbm_fatal);
+    if (mount->db == NULL) {
+        println("gdbm error: %s", gdbm_strerror(gdbm_errno));
+        return _EINVAL;
+    }
+
+    // after the filesystem is compressed, transmitted, and uncompressed, the
+    // inode numbers will be different. to detect this, the inode of the
+    // database file is stored inside the database and compared with the actual
+    // database file inode, and if they're different we rebuild the database.
+    struct stat stat;
+    if (fstat(gdbm_fdesc(mount->db), &stat) < 0) DIE("fstat database");
+    datum key = {.dptr = "db inode", .dsize = strlen("db inode")};
+    datum value = gdbm_fetch(mount->db, key);
+    if (value.dptr != NULL && atol(value.dptr) != stat.st_ino) {
+        free(value.dptr);
+        TODO("rebuild database");
+    }
+
+    char keydata[30];
+    value = make_datum(keydata, "%lu", (unsigned long) stat.st_ino);
+    value.dsize++; // make sure to null terminate
+    gdbm_store(mount->db, key, value, GDBM_REPLACE);
+
     return realfs.mount(mount);
 }
 
@@ -262,7 +273,7 @@ const struct fs_ops fakefs = {
     .readlink = fakefs_readlink,
     .access = realfs_access,
     .unlink = fakefs_unlink,
-    .rename = fakefs_rename,
+    .rename = realfs_rename,
     .symlink = fakefs_symlink,
 
     .stat = fakefs_stat,

diff --git a/fs/real.c b/fs/real.c
@@ -201,7 +201,7 @@ static int realfs_unlink(struct mount *mount, const char *path) {
     return res;
 }
 
-static int realfs_rename(struct mount *mount, const char *src, const char *dst) {
+int realfs_rename(struct mount *mount, const char *src, const char *dst) {
     int err = renameat(mount->root_fd, fix_path(src), mount->root_fd, fix_path(dst));
     if (err < 0)
         return errno_map();

diff --git a/kernel/fs.h b/kernel/fs.h
@@ -6,6 +6,7 @@
 #include "fs/stat.h"
 #include "emu/memory.h"
 #include <dirent.h>
+#include <gdbm.h>
 
 #define MAX_PATH 4096
 #define MAX_NAME 256
@@ -93,7 +94,10 @@ struct mount {
     struct mount *next;
 
     int root_fd;
-    void *data;
+    union {
+        void *data;
+        GDBM_FILE db;
+    };
 };
 extern struct mount *mounts;
 
@@ -219,6 +223,7 @@ extern const struct fs_ops realfs;
 int realfs_stat(struct mount *mount, const char *path, struct statbuf *fake_stat, bool follow_links);
 int realfs_fstat(struct fd *fd, struct statbuf *fake_stat);
 int realfs_access(struct mount *mount, const char *path, int mode);
+int realfs_rename(struct mount *mount, const char *src, const char *dst);
 int realfs_statfs(struct mount *mount, struct statfsbuf *stat);
 int realfs_flock(struct fd *fd, int operation);
 extern const struct fd_ops realfs_fdops;

diff --git a/tools/fakefsify.py b/tools/fakefsify.py
@@ -15,7 +15,7 @@ def extract_archive(archive, db):
         path = data/(member.name)
         major = member.devmajor
         minor = member.devminor
-        rdev = ((minor & 0xfff00) << 12) | (major << 8) | (minor & 0xff) # copied from fs/dev.h
+        rdev = ((minor & 0xfff00) << 12) | (major << 8) | (minor & 0xff)
         mode = member.mode
         if member.isfile():
             mode |= 0o100000	
@@ -32,17 +32,6 @@ def extract_archive(archive, db):
         else:
             raise ValueError('unrecognized tar entry type')
 
-        metadata = struct.pack(
-            '=iiii',
-            mode,
-            member.uid,
-            member.gid,
-            rdev,
-        )
-        meta_path = path.relative_to(data)
-        meta_path = b'/' + bytes(meta_path) if meta_path.parts else b''
-        db[b'meta\0' + meta_path] = metadata
-
         if member.isdir():
             path.mkdir(parents=True, exist_ok=True)
         elif member.issym():
@@ -52,6 +41,18 @@ def extract_archive(archive, db):
         else:
             path.touch()
 
+        inode = bytes(str(path.stat().st_ino), 'ascii')
+        meta_path = path.relative_to(data)
+        meta_path = b'/' + bytes(meta_path) if meta_path.parts else b''
+        db[b'inode ' + meta_path] = inode
+        db[b'stat ' + inode] = struct.pack(
+            '=iiii',
+            mode,
+            member.uid,
+            member.gid,
+            rdev,
+        )
+
 _, archive_path, fs = sys.argv
 fs = Path(fs)
 fs.mkdir(parents=True, exist_ok=True)

diff --git a/xX_main_Xx.h b/xX_main_Xx.h
@@ -14,12 +14,19 @@ static inline int xX_main_Xx(int argc, char *const argv[]) {
     int opt;
     const char *root = "";
     bool has_root = false;
-    while ((opt = getopt(argc, argv, "+r:")) != -1) {
+    const struct fs_ops *fs;
+    while ((opt = getopt(argc, argv, "+r:f:")) != -1) {
         switch (opt) {
             case 'r':
+            case 'f':
                 root = optarg;
                 has_root = true;
+                if (opt == 'r')
+                    fs = &realfs;
+                else
+                    fs = &fakefs;
                 break;
+
         }
     }
 
@@ -28,12 +35,11 @@ static inline int xX_main_Xx(int argc, char *const argv[]) {
         perror(root);
         exit(1);
     }
-    const struct fs_ops *fs;
-    if (strcmp(strrchr(root_realpath, '/') + 1, "data") == 0)
-        fs = &fakefs;
-    else
-        fs = &realfs;
-    mount_root(fs, root_realpath);
+    if (fs == &fakefs)
+        strcat(root_realpath, "/data");
+    int err = mount_root(fs, root_realpath);
+    if (err < 0)
+        return err;
 
     create_first_process();
     if (!has_root) {
@@ -44,7 +50,7 @@ static inline int xX_main_Xx(int argc, char *const argv[]) {
         current->pwd = pwd;
     }
 
-    int err = sys_execve(argv[optind], argv + optind, (char *[]) {NULL});
+    err = sys_execve(argv[optind], argv + optind, (char *[]) {NULL});
     if (err < 0)
         return err;
     err = create_stdio(real_tty_driver);