From 49698e431f3197b07a5de22f0592a674072d03fe Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 3 Mar 2018 10:26:06 -0600 Subject: [PATCH 001/139] Separated type/struct fields in dir entries The separation of data-structure vs entry type has been implicit for a while now, and even taken advantage of to simplify the traverse logic. Explicitely separating the data-struct and entry types allows us to introduce new data structures (inlined files). --- lfs.c | 58 +++++++++++++++++++++++++++++----------------------------- lfs.h | 12 +++++++++--- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/lfs.c b/lfs.c index 3aef9fcc..533b1ea7 100644 --- a/lfs.c +++ b/lfs.c @@ -793,7 +793,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, // special case for root dir if (pathname[0] == '\0') { *entry = (lfs_entry_t){ - .d.type = LFS_TYPE_DIR, + .d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR, .d.elen = sizeof(entry->d) - 4, .d.alen = 0, .d.nlen = 0, @@ -844,8 +844,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (((0x7f & entry->d.type) != LFS_TYPE_REG && - (0x7f & entry->d.type) != LFS_TYPE_DIR) || + if (((0x7f & entry->d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && + (0x7f & entry->d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) || entry->d.nlen != pathlen) { continue; } @@ -864,13 +864,13 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } // check that entry has not been moved - if (entry->d.type & 0x80) { + if (entry->d.type & LFS_STRUCT_MOVED) { int moved = lfs_moved(lfs, &entry->d.u); if (moved < 0 || moved) { return (moved < 0) ? moved : LFS_ERR_NOENT; } - entry->d.type &= ~0x80; + entry->d.type &= ~LFS_STRUCT_MOVED; } pathname += pathlen; @@ -880,7 +880,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } // continue on if we hit a directory - if (entry->d.type != LFS_TYPE_DIR) { + if ((0xf & entry->d.type) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } @@ -931,7 +931,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } - entry.d.type = LFS_TYPE_DIR; + entry.d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR; entry.d.elen = sizeof(entry.d) - 4; entry.d.alen = 0; entry.d.nlen = strlen(path); @@ -963,7 +963,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { err = lfs_dir_find(lfs, dir, &entry, &path); if (err) { return err; - } else if (entry.d.type != LFS_TYPE_DIR) { + } else if (entry.d.type != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { return LFS_ERR_NOTDIR; } @@ -1021,13 +1021,13 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { return (err == LFS_ERR_NOENT) ? 0 : err; } - if ((0x7f & entry.d.type) != LFS_TYPE_REG && - (0x7f & entry.d.type) != LFS_TYPE_DIR) { + if ((0x7f & entry.d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && + (0x7f & entry.d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { continue; } // check that entry has not been moved - if (entry.d.type & 0x80) { + if (entry.d.type & LFS_STRUCT_MOVED) { int moved = lfs_moved(lfs, &entry.d.u); if (moved < 0) { return moved; @@ -1037,13 +1037,13 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { continue; } - entry.d.type &= ~0x80; + entry.d.type &= ~LFS_STRUCT_MOVED; } break; } - info->type = entry.d.type; + info->type = 0xf & entry.d.type; if (info->type == LFS_TYPE_REG) { info->size = entry.d.u.file.size; } @@ -1319,7 +1319,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // create entry to remember name - entry.d.type = LFS_TYPE_REG; + entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; entry.d.elen = sizeof(entry.d) - 4; entry.d.alen = 0; entry.d.nlen = strlen(path); @@ -1329,7 +1329,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, if (err) { return err; } - } else if (entry.d.type == LFS_TYPE_DIR) { + } else if ((0xf & entry.d.type) == LFS_TYPE_DIR) { return LFS_ERR_ISDIR; } else if (flags & LFS_O_EXCL) { return LFS_ERR_EXIST; @@ -1537,7 +1537,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - LFS_ASSERT(entry.d.type == LFS_TYPE_REG); + LFS_ASSERT(entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)); entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; @@ -1826,7 +1826,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { } memset(info, 0, sizeof(*info)); - info->type = entry.d.type; + info->type = 0xf & entry.d.type; if (info->type == LFS_TYPE_REG) { info->size = entry.d.u.file.size; } @@ -1867,7 +1867,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } lfs_dir_t dir; - if (entry.d.type == LFS_TYPE_DIR) { + if ((0xf & entry.d.type) == LFS_TYPE_DIR) { // must be empty before removal, checking size // without masking top bit checks for any case where // dir is not empty @@ -1886,7 +1886,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // if we were a directory, find pred, replace tail - if (entry.d.type == LFS_TYPE_DIR) { + if ((0xf & entry.d.type) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, dir.pair, &cwd); if (res < 0) { return res; @@ -1949,7 +1949,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } lfs_dir_t dir; - if (prevexists && preventry.d.type == LFS_TYPE_DIR) { + if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { // must be empty before removal, checking size // without masking top bit checks for any case where // dir is not empty @@ -1962,7 +1962,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // mark as moving - oldentry.d.type |= 0x80; + oldentry.d.type |= LFS_STRUCT_MOVED; err = lfs_dir_update(lfs, &oldcwd, &oldentry, NULL); if (err) { return err; @@ -1976,7 +1976,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // move to new location lfs_entry_t newentry = preventry; newentry.d = oldentry.d; - newentry.d.type &= ~0x80; + newentry.d.type &= ~LFS_STRUCT_MOVED; newentry.d.nlen = strlen(newpath); if (prevexists) { @@ -2003,7 +2003,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // if we were a directory, find pred, replace tail - if (prevexists && preventry.d.type == LFS_TYPE_DIR) { + if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, dir.pair, &newcwd); if (res < 0) { return res; @@ -2134,7 +2134,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // write superblocks lfs_superblock_t superblock = { .off = sizeof(superdir.d), - .d.type = LFS_TYPE_SUPERBLOCK, + .d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK, .d.elen = sizeof(superblock.d) - sizeof(superblock.d.magic) - 4, .d.nlen = sizeof(superblock.d.magic), .d.version = LFS_DISK_VERSION, @@ -2263,7 +2263,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } dir.off += lfs_entry_size(&entry); - if ((0x70 & entry.d.type) == (0x70 & LFS_TYPE_REG)) { + if ((0x70 & entry.d.type) == LFS_STRUCT_CTZ) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, entry.d.u.file.head, entry.d.u.file.size, cb, data); if (err) { @@ -2353,7 +2353,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], break; } - if (((0x70 & entry->d.type) == (0x70 & LFS_TYPE_DIR)) && + if (((0x70 & entry->d.type) == LFS_STRUCT_DIR) && lfs_paircmp(entry->d.u.dir, dir) == 0) { return true; } @@ -2393,7 +2393,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) { break; } - if (!(0x80 & entry.d.type) && + if (!(LFS_STRUCT_MOVED & entry.d.type) && memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) { return true; } @@ -2525,7 +2525,7 @@ int lfs_deorphan(lfs_t *lfs) { } // found moved entry - if (entry.d.type & 0x80) { + if (entry.d.type & LFS_STRUCT_MOVED) { int moved = lfs_moved(lfs, &entry.d.u); if (moved < 0) { return moved; @@ -2541,7 +2541,7 @@ int lfs_deorphan(lfs_t *lfs) { } else { LFS_DEBUG("Found partial move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); - entry.d.type &= ~0x80; + entry.d.type &= ~LFS_STRUCT_MOVED; err = lfs_dir_update(lfs, &cwd, &entry, NULL); if (err) { return err; diff --git a/lfs.h b/lfs.h index 376776f6..4a12c4a2 100644 --- a/lfs.h +++ b/lfs.h @@ -74,9 +74,15 @@ enum lfs_error { // File types enum lfs_type { - LFS_TYPE_REG = 0x11, - LFS_TYPE_DIR = 0x22, - LFS_TYPE_SUPERBLOCK = 0x2e, + // file type + LFS_TYPE_REG = 0x01, + LFS_TYPE_DIR = 0x02, + LFS_TYPE_SUPERBLOCK = 0x0e, + + // on disk structure + LFS_STRUCT_CTZ = 0x10, + LFS_STRUCT_DIR = 0x20, + LFS_STRUCT_MOVED = 0x80, }; // File open flags From 4c35c8655af3400237940f26879cd16b6094f0a2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 3 Mar 2018 22:14:19 -0600 Subject: [PATCH 002/139] Added different sources for commits, now with disk->disk moves Previously, commits could only come from memory in RAM. This meant any entries had to be buffered in their entirety before they could be moved to a different directory pair. By adding parameters for specifying commits from existing entries stored on disk, we allow any sized entries to be moved between directory pairs with a fixed RAM cost. --- lfs.c | 85 ++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/lfs.c b/lfs.c index 533b1ea7..3bedd0df 100644 --- a/lfs.c +++ b/lfs.c @@ -486,7 +486,18 @@ static int lfs_dir_fetch(lfs_t *lfs, struct lfs_region { lfs_off_t oldoff; lfs_size_t oldlen; - const void *newdata; + + enum lfs_region_source { + LFS_FROM_MEM, + LFS_FROM_DISK, + } source; + union { + const void *mem; + struct { + lfs_block_t block; + lfs_off_t off; + } disk; + } u; lfs_size_t newlen; }; @@ -527,42 +538,49 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, } int i = 0; + int j = 0; lfs_off_t oldoff = sizeof(dir->d); lfs_off_t newoff = sizeof(dir->d); while (newoff < (0x7fffffff & dir->d.size)-4) { + while (i < count && oldoff == regions[i].oldoff && + j == regions[i].newlen) { + oldoff += regions[i].oldlen; + i += 1; + j = 0; + } + + uint8_t data; if (i < count && regions[i].oldoff == oldoff) { - lfs_crc(&crc, regions[i].newdata, regions[i].newlen); - err = lfs_bd_prog(lfs, dir->pair[0], - newoff, regions[i].newdata, regions[i].newlen); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + if (regions[i].source == LFS_FROM_DISK) { + err = lfs_bd_read(lfs, regions[i].u.disk.block, + regions[i].u.disk.off + j, &data, 1); + if (err) { + return err; } - return err; + } else { + data = ((const uint8_t *)regions[i].u.mem)[j]; } - oldoff += regions[i].oldlen; - newoff += regions[i].newlen; - i += 1; + j += 1; } else { - uint8_t data; err = lfs_bd_read(lfs, oldpair[1], oldoff, &data, 1); if (err) { return err; } - lfs_crc(&crc, &data, 1); - err = lfs_bd_prog(lfs, dir->pair[0], newoff, &data, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - oldoff += 1; - newoff += 1; } + + lfs_crc(&crc, &data, 1); + err = lfs_bd_prog(lfs, dir->pair[0], newoff, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + newoff += 1; } crc = lfs_tole32(crc); @@ -643,8 +661,10 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const void *data) { lfs_entry_tole32(&entry->d); int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, sizeof(entry->d), &entry->d, sizeof(entry->d)}, - {entry->off+sizeof(entry->d), entry->d.nlen, data, entry->d.nlen} + {entry->off, sizeof(entry->d), + LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, + {entry->off+sizeof(entry->d), entry->d.nlen, + LFS_FROM_MEM, {.mem = data}, entry->d.nlen} }, data ? 2 : 1); lfs_entry_fromle32(&entry->d); return err; @@ -659,8 +679,10 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_tole32(&entry->d); int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, 0, &entry->d, sizeof(entry->d)}, - {entry->off, 0, data, entry->d.nlen} + {entry->off, 0, + LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, + {entry->off, 0, + LFS_FROM_MEM, {.mem = data}, entry->d.nlen} }, 2); lfs_entry_fromle32(&entry->d); return err; @@ -679,8 +701,10 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, entry->off = dir->d.size - 4; lfs_entry_tole32(&entry->d); err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, 0, &entry->d, sizeof(entry->d)}, - {entry->off, 0, data, entry->d.nlen} + {entry->off, 0, + LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, + {entry->off, 0, + LFS_FROM_MEM, {.mem = data}, entry->d.nlen} }, 2); lfs_entry_fromle32(&entry->d); if (err) { @@ -720,7 +744,8 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { // shift out the entry int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, lfs_entry_size(entry), NULL, 0}, + {entry->off, lfs_entry_size(entry), + LFS_FROM_MEM, {.mem = NULL}, 0}, }, 1); if (err) { return err; @@ -2153,7 +2178,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){ {sizeof(superdir.d), sizeof(superblock.d), - &superblock.d, sizeof(superblock.d)} + LFS_FROM_MEM, {.mem = &superblock.d}, sizeof(superblock.d)} }, 1); if (err && err != LFS_ERR_CORRUPT) { return err; From 73d29f05b24765dd171f07dd41df09f4f067dc9f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Mar 2018 18:27:25 -0600 Subject: [PATCH 003/139] Adopted a tiny LISP-like DSL for some extra flexibility Really all this means is that the internal commit function was changed from taking an array of "commit structures" to a linked-list of "commit structures". The benefit of a linked-list is that layers of commit functions can pull off some minor modifications to the description of the commit. Most notably, commit functions can add additional entries that will be atomically written out and CRCed along with the initial commit. Also a minor benefit, this is one less parameter when committing a directory with zero entries. --- lfs.c | 114 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/lfs.c b/lfs.c index 3bedd0df..bc9ae58e 100644 --- a/lfs.c +++ b/lfs.c @@ -484,32 +484,36 @@ static int lfs_dir_fetch(lfs_t *lfs, } struct lfs_region { - lfs_off_t oldoff; - lfs_size_t oldlen; - - enum lfs_region_source { + enum { LFS_FROM_MEM, LFS_FROM_DISK, } source; + + lfs_off_t oldoff; + lfs_size_t oldlen; union { - const void *mem; + struct { + const void *data; + } m; struct { lfs_block_t block; lfs_off_t off; - } disk; + } d; } u; lfs_size_t newlen; + + struct lfs_region *next; }; static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, - const struct lfs_region *regions, int count) { + struct lfs_region *region) { // increment revision count dir->d.rev += 1; // keep pairs in order such that pair[0] is most recent lfs_pairswap(dir->pair); - for (int i = 0; i < count; i++) { - dir->d.size += regions[i].newlen - regions[i].oldlen; + for (struct lfs_region *r = region; r; r = r->next) { + dir->d.size += r->newlen - r->oldlen; } const lfs_block_t oldpair[2] = {dir->pair[0], dir->pair[1]}; @@ -537,28 +541,27 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - int i = 0; + struct lfs_region *r = region; int j = 0; lfs_off_t oldoff = sizeof(dir->d); lfs_off_t newoff = sizeof(dir->d); while (newoff < (0x7fffffff & dir->d.size)-4) { - while (i < count && oldoff == regions[i].oldoff && - j == regions[i].newlen) { - oldoff += regions[i].oldlen; - i += 1; + while (r && r->oldoff == oldoff && r->newlen == j) { + oldoff += r->oldlen; + r = r->next; j = 0; } uint8_t data; - if (i < count && regions[i].oldoff == oldoff) { - if (regions[i].source == LFS_FROM_DISK) { - err = lfs_bd_read(lfs, regions[i].u.disk.block, - regions[i].u.disk.off + j, &data, 1); + if (r && r->oldoff == oldoff) { + if (r->source == LFS_FROM_DISK) { + err = lfs_bd_read(lfs, r->u.d.block, + r->u.d.off + j, &data, 1); if (err) { return err; } } else { - data = ((const uint8_t *)regions[i].u.mem)[j]; + data = ((const uint8_t *)r->u.m.data)[j]; } j += 1; @@ -660,12 +663,15 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const void *data) { lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, sizeof(entry->d), - LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, - {entry->off+sizeof(entry->d), entry->d.nlen, - LFS_FROM_MEM, {.mem = data}, entry->d.nlen} - }, data ? 2 : 1); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, sizeof(entry->d), + {.m.data = &entry->d}, sizeof(entry->d), + data ? + &(struct lfs_region){ + LFS_FROM_MEM, entry->off+sizeof(entry->d), entry->d.nlen, + {.m.data = data}, entry->d.nlen} + : NULL}); lfs_entry_fromle32(&entry->d); return err; } @@ -678,12 +684,13 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, entry->off = dir->d.size - 4; lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, 0, - LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, - {entry->off, 0, - LFS_FROM_MEM, {.mem = data}, entry->d.nlen} - }, 2); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, sizeof(entry->d), + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = data}, entry->d.nlen}}); lfs_entry_fromle32(&entry->d); return err; } @@ -700,12 +707,13 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, dir->d.tail[1] = olddir.d.tail[1]; entry->off = dir->d.size - 4; lfs_entry_tole32(&entry->d); - err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, 0, - LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, - {entry->off, 0, - LFS_FROM_MEM, {.mem = data}, entry->d.nlen} - }, 2); + err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, sizeof(entry->d), + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = data}, entry->d.nlen}}); lfs_entry_fromle32(&entry->d); if (err) { return err; @@ -714,7 +722,7 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, olddir.d.size |= 0x80000000; olddir.d.tail[0] = dir->pair[0]; olddir.d.tail[1] = dir->pair[1]; - return lfs_dir_commit(lfs, &olddir, NULL, 0); + return lfs_dir_commit(lfs, &olddir, NULL); } int err = lfs_dir_fetch(lfs, dir, dir->d.tail); @@ -738,15 +746,14 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { pdir.d.size &= dir->d.size | 0x7fffffff; pdir.d.tail[0] = dir->d.tail[0]; pdir.d.tail[1] = dir->d.tail[1]; - return lfs_dir_commit(lfs, &pdir, NULL, 0); + return lfs_dir_commit(lfs, &pdir, NULL); } } // shift out the entry - int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, lfs_entry_size(entry), - LFS_FROM_MEM, {.mem = NULL}, 0}, - }, 1); + int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ + LFS_FROM_MEM, entry->off, lfs_entry_size(entry), + {.m.data = NULL}, 0}); if (err) { return err; } @@ -951,7 +958,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { dir.d.tail[0] = cwd.d.tail[0]; dir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dir_commit(lfs, &dir, NULL, 0); + err = lfs_dir_commit(lfs, &dir, NULL); if (err) { return err; } @@ -1921,7 +1928,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.d.tail[0]; cwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dir_commit(lfs, &cwd, NULL, 0); + err = lfs_dir_commit(lfs, &cwd, NULL); if (err) { return err; } @@ -2038,7 +2045,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newcwd.d.tail[0] = dir.d.tail[0]; newcwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dir_commit(lfs, &newcwd, NULL, 0); + err = lfs_dir_commit(lfs, &newcwd, NULL); if (err) { return err; } @@ -2148,7 +2155,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_commit(lfs, &root, NULL, 0); + err = lfs_dir_commit(lfs, &root, NULL); if (err) { return err; } @@ -2176,10 +2183,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_tole32(&superblock.d); bool valid = false; for (int i = 0; i < 2; i++) { - err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){ - {sizeof(superdir.d), sizeof(superblock.d), - LFS_FROM_MEM, {.mem = &superblock.d}, sizeof(superblock.d)} - }, 1); + err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ + LFS_FROM_MEM, sizeof(superdir.d), sizeof(superblock.d), + {.m.data = &superblock.d}, sizeof(superblock.d)}); if (err && err != LFS_ERR_CORRUPT) { return err; } @@ -2470,7 +2476,7 @@ static int lfs_relocate(lfs_t *lfs, parent.d.tail[0] = newpair[0]; parent.d.tail[1] = newpair[1]; - return lfs_dir_commit(lfs, &parent, NULL, 0); + return lfs_dir_commit(lfs, &parent, NULL); } // couldn't find dir, must be new @@ -2512,7 +2518,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = cwd.d.tail[0]; pdir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dir_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL); if (err) { return err; } @@ -2528,7 +2534,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = entry.d.u.dir[0]; pdir.d.tail[1] = entry.d.u.dir[1]; - err = lfs_dir_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL); if (err) { return err; } From e3daee26218562d55e939e6ce2ffa97149d12eb4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Mar 2018 18:27:43 -0600 Subject: [PATCH 004/139] Changed dir append to mirror commit DSL Expiremental implementation. This opens up the opportunity to use the same commit description for both commits and appends, which effectively do the same thing. This should lead to better code reuse. --- lfs.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/lfs.c b/lfs.c index bc9ae58e..3c7fb493 100644 --- a/lfs.c +++ b/lfs.c @@ -676,6 +676,59 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, return err; } +static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *region) { + // check if we fit, if top bit is set we do not and move on + while (true) { + if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { + entry->off = dir->d.size - 4; + for (struct lfs_region *r = region; r; r = r->next) { + r->oldoff += entry->off; + } + + lfs_entry_tole32(&entry->d); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, 4, region}); + lfs_entry_fromle32(&entry->d); + return err; + } + + // we need to allocate a new dir block + if (!(0x80000000 & dir->d.size)) { + lfs_dir_t olddir = *dir; + int err = lfs_dir_alloc(lfs, dir); + if (err) { + return err; + } + + dir->d.tail[0] = olddir.d.tail[0]; + dir->d.tail[1] = olddir.d.tail[1]; + entry->off = dir->d.size - 4; + lfs_entry_tole32(&entry->d); + err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, 4, region}); + lfs_entry_fromle32(&entry->d); + if (err) { + return err; + } + + olddir.d.size |= 0x80000000; + olddir.d.tail[0] = dir->pair[0]; + olddir.d.tail[1] = dir->pair[1]; + return lfs_dir_commit(lfs, &olddir, NULL); + } + + int err = lfs_dir_fetch(lfs, dir, dir->d.tail); + if (err) { + return err; + } + } +} + static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const void *data) { // check if we fit, if top bit is set we do not and move on @@ -973,7 +1026,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; - err = lfs_dir_append(lfs, &cwd, &entry, path); + err = lfs_dir_append_(lfs, &cwd, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (uint8_t*)&entry.d + 4}, sizeof(entry.d) - 4, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = path}, entry.d.nlen}}); if (err) { return err; } @@ -1357,7 +1416,13 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.nlen = strlen(path); entry.d.u.file.head = 0xffffffff; entry.d.u.file.size = 0; - err = lfs_dir_append(lfs, &cwd, &entry, path); + err = lfs_dir_append_(lfs, &cwd, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (uint8_t*)&entry.d + 4}, sizeof(entry.d) - 4, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = path}, entry.d.nlen}}); if (err) { return err; } @@ -2017,7 +2082,13 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } } else { - err = lfs_dir_append(lfs, &newcwd, &newentry, newpath); + err = lfs_dir_append_(lfs, &newcwd, &newentry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (uint8_t*)&newentry.d + 4}, sizeof(newentry.d) - 4, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = newpath}, newentry.d.nlen}}); if (err) { return err; } From 692f0c542eb21816fb0a548b392e863f951138bb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Mar 2018 19:27:57 -0600 Subject: [PATCH 005/139] Naive implementation of resizable entries Now, with the off, diff, and len parameters in each commit entry, we can build up directory commits that resize entries. This adds complexity but opens up the directory blocks to be much more flexible. The main concern is that resizing entries can push around neighboring entries in surprising ways, such as pushing them into new directory blocks when a directory splits. This can break littlefs's internal logic in how it tracks in-flight entries. The most problematic example being open files. Fortunately, this is helped by a global linked-list of all files and directories opened by the filesystem. As entries change size, the state of open files/dirs may be updated as needed. Note this already needed to exist for the ability to remove files/dirs, which has the same issue. --- lfs.c | 225 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 131 insertions(+), 94 deletions(-) diff --git a/lfs.c b/lfs.c index 3c7fb493..cf8f2e11 100644 --- a/lfs.c +++ b/lfs.c @@ -489,8 +489,8 @@ struct lfs_region { LFS_FROM_DISK, } source; - lfs_off_t oldoff; - lfs_size_t oldlen; + lfs_off_t off; + lfs_ssize_t diff; union { struct { const void *data; @@ -500,20 +500,20 @@ struct lfs_region { lfs_off_t off; } d; } u; - lfs_size_t newlen; + lfs_size_t len; struct lfs_region *next; }; static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, - struct lfs_region *region) { + struct lfs_region *regions) { // increment revision count dir->d.rev += 1; // keep pairs in order such that pair[0] is most recent lfs_pairswap(dir->pair); - for (struct lfs_region *r = region; r; r = r->next) { - dir->d.size += r->newlen - r->oldlen; + for (struct lfs_region *r = regions; r; r = r->next) { + dir->d.size += r->diff; } const lfs_block_t oldpair[2] = {dir->pair[0], dir->pair[1]}; @@ -541,19 +541,19 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - struct lfs_region *r = region; + struct lfs_region *r = regions; int j = 0; lfs_off_t oldoff = sizeof(dir->d); lfs_off_t newoff = sizeof(dir->d); while (newoff < (0x7fffffff & dir->d.size)-4) { - while (r && r->oldoff == oldoff && r->newlen == j) { - oldoff += r->oldlen; + while (r && r->off == oldoff && r->len == j) { + oldoff += r->len - r->diff; r = r->next; j = 0; } uint8_t data; - if (r && r->oldoff == oldoff) { + if (r && r->off == oldoff) { if (r->source == LFS_FROM_DISK) { err = lfs_bd_read(lfs, r->u.d.block, r->u.d.off + j, &data, 1); @@ -660,37 +660,21 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, const void *data) { - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, sizeof(entry->d), - {.m.data = &entry->d}, sizeof(entry->d), - data ? - &(struct lfs_region){ - LFS_FROM_MEM, entry->off+sizeof(entry->d), entry->d.nlen, - {.m.data = data}, entry->d.nlen} - : NULL}); - lfs_entry_fromle32(&entry->d); - return err; -} - -static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *region) { +static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on while (true) { if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { entry->off = dir->d.size - 4; - for (struct lfs_region *r = region; r; r = r->next) { - r->oldoff += entry->off; + for (struct lfs_region *r = regions; r; r = r->next) { + r->off += entry->off; } lfs_entry_tole32(&entry->d); int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, 4, region}); + LFS_FROM_MEM, entry->off, +4, + {.m.data = &entry->d}, 4, regions}); lfs_entry_fromle32(&entry->d); return err; } @@ -706,11 +690,15 @@ static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, dir->d.tail[0] = olddir.d.tail[0]; dir->d.tail[1] = olddir.d.tail[1]; entry->off = dir->d.size - 4; + for (struct lfs_region *r = regions; r; r = r->next) { + r->off += entry->off; + } + lfs_entry_tole32(&entry->d); err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, 4, region}); + LFS_FROM_MEM, entry->off, +4, + {.m.data = &entry->d}, 4, regions}); lfs_entry_fromle32(&entry->d); if (err) { return err; @@ -729,60 +717,95 @@ static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, } } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, const void *data) { - // check if we fit, if top bit is set we do not and move on - while (true) { - if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { - entry->off = dir->d.size - 4; +static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *regions) { + lfs_off_t oldoff = entry->off; + lfs_ssize_t diff = 0; + lfs_size_t len = 0; + struct lfs_region **tail = ®ions; + for (struct lfs_region *r = regions; r; r = r->next) { + diff += r->diff; + len += r->len; + tail = &r->next; + } - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, sizeof(entry->d), - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = data}, entry->d.nlen}}); - lfs_entry_fromle32(&entry->d); - return err; + // do we still fit? + if ((0x7fffffff & dir->d.size) + diff <= lfs->cfg->block_size) { + for (struct lfs_region *r = regions; r; r = r->next) { + r->off += entry->off; } - // we need to allocate a new dir block - if (!(0x80000000 & dir->d.size)) { - lfs_dir_t olddir = *dir; - int err = lfs_dir_alloc(lfs, dir); - if (err) { - return err; - } + lfs_entry_tole32(&entry->d); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, sizeof(entry->d), regions}); + lfs_entry_fromle32(&entry->d); + if (err) { + return err; + } + } else { + lfs_dir_t olddir = *dir; - dir->d.tail[0] = olddir.d.tail[0]; - dir->d.tail[1] = olddir.d.tail[1]; - entry->off = dir->d.size - 4; - lfs_entry_tole32(&entry->d); - err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, sizeof(entry->d), - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = data}, entry->d.nlen}}); - lfs_entry_fromle32(&entry->d); - if (err) { - return err; - } + // mark as moving + entry->d.type |= LFS_STRUCT_MOVED; + int err = lfs_dir_commit(lfs, &olddir, + &(struct lfs_region){ + LFS_FROM_MEM, oldoff, 0, + {.m.data = &entry->d.type}, 1}); + if (err) { + return err; + } - olddir.d.size |= 0x80000000; - olddir.d.tail[0] = dir->pair[0]; - olddir.d.tail[1] = dir->pair[1]; - return lfs_dir_commit(lfs, &olddir, NULL); + // append updated entry + entry->d.type &= LFS_STRUCT_MOVED; + lfs_size_t prefix = regions->off; + lfs_size_t suffix = lfs_entry_size(entry) - len - 4; + for (struct lfs_region *r = regions; r; r = r->next) { + r->off = 0; + r->diff = r->len; } - int err = lfs_dir_fetch(lfs, dir, dir->d.tail); + regions = &(struct lfs_region){ + LFS_FROM_DISK, 0, +prefix, + {.d.block = olddir.pair[0], .d.off = entry->off}, + prefix, regions}; + *tail = &(struct lfs_region){ + LFS_FROM_DISK, 0, +suffix, + {.d.block = olddir.pair[0], .d.off = entry->off+prefix+len}, + suffix}; + + // remove old entry + err = lfs_dir_commit(lfs, &olddir, + &(struct lfs_region){ + LFS_FROM_MEM, oldoff, -(lfs_entry_size(entry) - diff), + {.m.data = NULL}, 0}); if (err) { return err; } } + + // shift over any files/directories that are affected + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0) { + if (f->poff == oldoff) { + f->poff = entry->off; + } else if (f->poff > entry->off) { + f->poff += diff; + } + } + } + + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + if (d->off > entry->off) { + d->off += diff; + d->pos += diff; + } + } + } + + return 0; } static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { @@ -805,7 +828,7 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { // shift out the entry int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - LFS_FROM_MEM, entry->off, lfs_entry_size(entry), + LFS_FROM_MEM, entry->off, -lfs_entry_size(entry), {.m.data = NULL}, 0}); if (err) { return err; @@ -1026,12 +1049,12 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; - err = lfs_dir_append_(lfs, &cwd, &entry, + err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = (uint8_t*)&entry.d + 4}, sizeof(entry.d) - 4, + LFS_FROM_MEM, 0, +(sizeof(entry.d)-4), + {.m.data = (uint8_t*)&entry.d+4}, sizeof(entry.d)-4, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, + LFS_FROM_MEM, 0, +entry.d.nlen, {.m.data = path}, entry.d.nlen}}); if (err) { return err; @@ -1416,12 +1439,12 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.nlen = strlen(path); entry.d.u.file.head = 0xffffffff; entry.d.u.file.size = 0; - err = lfs_dir_append_(lfs, &cwd, &entry, + err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = (uint8_t*)&entry.d + 4}, sizeof(entry.d) - 4, + LFS_FROM_MEM, 0, +(sizeof(entry.d)-4), + {.m.data = (uint8_t*)&entry.d+4}, sizeof(entry.d)-4, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, + LFS_FROM_MEM, 0, +entry.d.nlen, {.m.data = path}, entry.d.nlen}}); if (err) { return err; @@ -1638,7 +1661,10 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; - err = lfs_dir_update(lfs, &cwd, &entry, NULL); + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (const uint8_t *)&entry.d+4}, sizeof(entry.d)-4}); if (err) { return err; } @@ -2077,17 +2103,25 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newentry.d.nlen = strlen(newpath); if (prevexists) { - err = lfs_dir_update(lfs, &newcwd, &newentry, newpath); + err = lfs_dir_update(lfs, &newcwd, &newentry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (const uint8_t*)&newentry.d+4}, + sizeof(newentry.d)-4, + &(struct lfs_region){ + LFS_FROM_MEM, sizeof(newentry.d)-4, 0, + {.m.data = newpath}, newentry.d.nlen}}); if (err) { return err; } } else { - err = lfs_dir_append_(lfs, &newcwd, &newentry, + err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = (uint8_t*)&newentry.d + 4}, sizeof(newentry.d) - 4, + LFS_FROM_MEM, 0, +(sizeof(newentry.d)-4), + {.m.data = (const uint8_t*)&newentry.d+4}, + sizeof(newentry.d)-4, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, + LFS_FROM_MEM, 0, +newentry.d.nlen, {.m.data = newpath}, newentry.d.nlen}}); if (err) { return err; @@ -2255,7 +2289,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { bool valid = false; for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ - LFS_FROM_MEM, sizeof(superdir.d), sizeof(superblock.d), + LFS_FROM_MEM, sizeof(superdir.d), 0, {.m.data = &superblock.d}, sizeof(superblock.d)}); if (err && err != LFS_ERR_CORRUPT) { return err; @@ -2520,7 +2554,10 @@ static int lfs_relocate(lfs_t *lfs, entry.d.u.dir[0] = newpair[0]; entry.d.u.dir[1] = newpair[1]; - int err = lfs_dir_update(lfs, &parent, &entry, NULL); + int err = lfs_dir_update(lfs, &parent, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (const uint8_t*)&entry.d+4}, sizeof(entry.d)-4}); if (err) { return err; } From ca3d6a52d27482a3520294f91626c78dc5051b0e Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Mar 2018 21:28:06 -0600 Subject: [PATCH 006/139] Made implicity tag updates explicit Before, tags were implicitly updated by the dir update functions, which have a strong understanding of the entry struct. However, most of the time the tag was already a part of the entry struct being committed. By making tag updates explicit, this does add cost to commits that now have to pass tag updates explicitly, but it reduces cost where that tag and entry update can be combined into one commit region. It also simplifies the dir update functions. --- lfs.c | 63 ++++++++++++++++++++++++----------------------------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/lfs.c b/lfs.c index cf8f2e11..0f4ab2c1 100644 --- a/lfs.c +++ b/lfs.c @@ -349,10 +349,11 @@ static void lfs_entry_fromle32(struct lfs_disk_entry *d) { d->u.dir[1] = lfs_fromle32(d->u.dir[1]); } -static void lfs_entry_tole32(struct lfs_disk_entry *d) { - d->u.dir[0] = lfs_tole32(d->u.dir[0]); - d->u.dir[1] = lfs_tole32(d->u.dir[1]); -} +// TODO +//static void lfs_entry_tole32(struct lfs_disk_entry *d) { +// d->u.dir[0] = lfs_tole32(d->u.dir[0]); +// d->u.dir[1] = lfs_tole32(d->u.dir[1]); +//} static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { d->root[0] = lfs_fromle32(d->root[0]); @@ -670,13 +671,7 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, +4, - {.m.data = &entry->d}, 4, regions}); - lfs_entry_fromle32(&entry->d); - return err; + return lfs_dir_commit(lfs, dir, regions); } // we need to allocate a new dir block @@ -694,12 +689,7 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } - lfs_entry_tole32(&entry->d); - err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, +4, - {.m.data = &entry->d}, 4, regions}); - lfs_entry_fromle32(&entry->d); + err = lfs_dir_commit(lfs, dir, regions); if (err) { return err; } @@ -735,12 +725,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, sizeof(entry->d), regions}); - lfs_entry_fromle32(&entry->d); + int err = lfs_dir_commit(lfs, dir, regions); if (err) { return err; } @@ -1051,8 +1036,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +(sizeof(entry.d)-4), - {.m.data = (uint8_t*)&entry.d+4}, sizeof(entry.d)-4, + LFS_FROM_MEM, 0, +sizeof(entry.d), + {.m.data = &entry.d}, sizeof(entry.d), &(struct lfs_region){ LFS_FROM_MEM, 0, +entry.d.nlen, {.m.data = path}, entry.d.nlen}}); @@ -1441,8 +1426,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.u.file.size = 0; err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +(sizeof(entry.d)-4), - {.m.data = (uint8_t*)&entry.d+4}, sizeof(entry.d)-4, + LFS_FROM_MEM, 0, +sizeof(entry.d), + {.m.data = &entry.d}, sizeof(entry.d), &(struct lfs_region){ LFS_FROM_MEM, 0, +entry.d.nlen, {.m.data = path}, entry.d.nlen}}); @@ -1664,7 +1649,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ LFS_FROM_MEM, 0, 0, - {.m.data = (const uint8_t *)&entry.d+4}, sizeof(entry.d)-4}); + {.m.data = &entry.d}, sizeof(entry.d)}); if (err) { return err; } @@ -2086,7 +2071,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving oldentry.d.type |= LFS_STRUCT_MOVED; - err = lfs_dir_update(lfs, &oldcwd, &oldentry, NULL); + err = lfs_dir_update(lfs, &oldcwd, &oldentry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = &oldentry.d}, sizeof(oldentry.d)}); if (err) { return err; } @@ -2106,10 +2094,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_update(lfs, &newcwd, &newentry, &(struct lfs_region){ LFS_FROM_MEM, 0, 0, - {.m.data = (const uint8_t*)&newentry.d+4}, - sizeof(newentry.d)-4, + {.m.data = &newentry.d}, sizeof(newentry.d), &(struct lfs_region){ - LFS_FROM_MEM, sizeof(newentry.d)-4, 0, + LFS_FROM_MEM, sizeof(newentry.d), 0, {.m.data = newpath}, newentry.d.nlen}}); if (err) { return err; @@ -2117,9 +2104,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } else { err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +(sizeof(newentry.d)-4), - {.m.data = (const uint8_t*)&newentry.d+4}, - sizeof(newentry.d)-4, + LFS_FROM_MEM, 0, +sizeof(newentry.d), + {.m.data = &newentry.d}, sizeof(newentry.d), &(struct lfs_region){ LFS_FROM_MEM, 0, +newentry.d.nlen, {.m.data = newpath}, newentry.d.nlen}}); @@ -2557,7 +2543,7 @@ static int lfs_relocate(lfs_t *lfs, int err = lfs_dir_update(lfs, &parent, &entry, &(struct lfs_region){ LFS_FROM_MEM, 0, 0, - {.m.data = (const uint8_t*)&entry.d+4}, sizeof(entry.d)-4}); + {.m.data = &entry.d}, sizeof(entry.d)}); if (err) { return err; } @@ -2681,7 +2667,10 @@ int lfs_deorphan(lfs_t *lfs) { LFS_DEBUG("Found partial move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); entry.d.type &= ~LFS_STRUCT_MOVED; - err = lfs_dir_update(lfs, &cwd, &entry, NULL); + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = &entry.d}, sizeof(entry.d)}); if (err) { return err; } From f30ab677a4553f6da441963444d1fec08c9df2b8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 11 Mar 2018 11:28:13 -0500 Subject: [PATCH 007/139] Traded enum-based DSL for full callback-based DSL Now, instead of passing an enum for mem/disk commits, we pass a function pointer that can specify any behaviour. This has the benefit of opening up the possibility to pass any sort of commit logic to the committers, and unused logic can be garbage-collected by the compiler if unused. The downside is that unfortunately compilers have a harder time optimizing around functions pointers than enums, and fitting the state into structs for the callbacks may be costly. --- lfs.c | 274 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 151 insertions(+), 123 deletions(-) diff --git a/lfs.c b/lfs.c index 0f4ab2c1..35bb1646 100644 --- a/lfs.c +++ b/lfs.c @@ -484,30 +484,85 @@ static int lfs_dir_fetch(lfs_t *lfs, return 0; } -struct lfs_region { - enum { - LFS_FROM_MEM, - LFS_FROM_DISK, - } source; +struct lfs_commit { + uint32_t crc; + lfs_block_t block; + lfs_off_t off; +}; +static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, const void *data, lfs_size_t size) { + lfs_crc(&c->crc, data, size); + int err = lfs_bd_prog(lfs, c->block, c->off, data, size); + c->off += size; + return err; +} + +struct lfs_region { lfs_off_t off; lfs_ssize_t diff; - union { - struct { - const void *data; - } m; - struct { - lfs_block_t block; - lfs_off_t off; - } d; - } u; - lfs_size_t len; - + int (*commit)(lfs_t *lfs, struct lfs_commit *c, const void *p); + const void *data; struct lfs_region *next; }; +struct lfs_commit_mem { + const void *data; + lfs_size_t size; +}; + +static int lfs_commit_mem(lfs_t *lfs, struct lfs_commit *c, const void *p) { + const struct lfs_commit_mem *m = p; + return lfs_commit(lfs, c, m->data, m->size); +} + +struct lfs_commit_disk { + lfs_block_t block; + lfs_off_t off; + lfs_size_t size; + struct lfs_region *regions; +}; + +static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, const void *p) { + const struct lfs_commit_disk *u = p; + + struct lfs_region *r = u->regions; + lfs_off_t off = 0; + while (true) { + if (r && r->off == off) { + lfs_off_t orig = c->off; + int err = r->commit(lfs, c, r->data); + if (err) { + return err; + } + + off += (c->off - orig) - r->diff; + r = r->next; + } else if (off < u->size) { + uint8_t data; + int err = lfs_bd_read(lfs, u->block, u->off + off, &data, 1); + if (err) { + return err; + } + + err = lfs_commit(lfs, c, &data, 1); + if (err) { + return err; + } + + off += 1; + } else { + return 0; + } + } +} + static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, struct lfs_region *regions) { + // state for copying over + const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; + lfs_size_t oldsize = (0x7fffffff & dir->d.size) - 4; + bool relocated = false; + // increment revision count dir->d.rev += 1; @@ -517,9 +572,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, dir->d.size += r->diff; } - const lfs_block_t oldpair[2] = {dir->pair[0], dir->pair[1]}; - bool relocated = false; - while (true) { if (true) { int err = lfs_bd_erase(lfs, dir->pair[0]); @@ -530,10 +582,20 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - uint32_t crc = 0xffffffff; + struct lfs_commit c = { + .crc = 0xffffffff, + .block = dir->pair[0], + .off = 0, + }; + lfs_dir_tole32(&dir->d); - lfs_crc(&crc, &dir->d, sizeof(dir->d)); - err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); + err = lfs_commit_disk(lfs, &c, &(struct lfs_commit_disk){ + oldpair[1], 0, oldsize, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &dir->d, sizeof(dir->d)}, + regions}}); lfs_dir_fromle32(&dir->d); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -542,54 +604,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - struct lfs_region *r = regions; - int j = 0; - lfs_off_t oldoff = sizeof(dir->d); - lfs_off_t newoff = sizeof(dir->d); - while (newoff < (0x7fffffff & dir->d.size)-4) { - while (r && r->off == oldoff && r->len == j) { - oldoff += r->len - r->diff; - r = r->next; - j = 0; - } - - uint8_t data; - if (r && r->off == oldoff) { - if (r->source == LFS_FROM_DISK) { - err = lfs_bd_read(lfs, r->u.d.block, - r->u.d.off + j, &data, 1); - if (err) { - return err; - } - } else { - data = ((const uint8_t *)r->u.m.data)[j]; - } - - j += 1; - } else { - err = lfs_bd_read(lfs, oldpair[1], oldoff, &data, 1); - if (err) { - return err; - } - - oldoff += 1; - } - - lfs_crc(&crc, &data, 1); - err = lfs_bd_prog(lfs, dir->pair[0], newoff, &data, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - newoff += 1; - } - - crc = lfs_tole32(crc); - err = lfs_bd_prog(lfs, dir->pair[0], newoff, &crc, 4); - crc = lfs_fromle32(crc); + c.crc = lfs_tole32(c.crc); + err = lfs_bd_prog(lfs, dir->pair[0], c.off, &c.crc, 4); + c.crc = lfs_fromle32(c.crc); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -613,12 +630,13 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (ncrc != crc) { + if (ncrc != c.crc) { goto relocate; } } break; + relocate: //commit was corrupted LFS_DEBUG("Bad block at %d", dir->pair[0]); @@ -709,14 +727,10 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { - lfs_off_t oldoff = entry->off; + lfs_off_t oldoff = entry->off; // <- TODO rm me? lfs_ssize_t diff = 0; - lfs_size_t len = 0; - struct lfs_region **tail = ®ions; for (struct lfs_region *r = regions; r; r = r->next) { diff += r->diff; - len += r->len; - tail = &r->next; } // do we still fit? @@ -731,45 +745,44 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } } else { lfs_dir_t olddir = *dir; + lfs_off_t oldoff = entry->off; + lfs_size_t oldsize = lfs_entry_size(entry) - diff; // mark as moving entry->d.type |= LFS_STRUCT_MOVED; int err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ - LFS_FROM_MEM, oldoff, 0, - {.m.data = &entry->d.type}, 1}); + oldoff, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry->d.type, 1}}); if (err) { return err; } // append updated entry entry->d.type &= LFS_STRUCT_MOVED; - lfs_size_t prefix = regions->off; - lfs_size_t suffix = lfs_entry_size(entry) - len - 4; - for (struct lfs_region *r = regions; r; r = r->next) { - r->off = 0; - r->diff = r->len; + err = lfs_dir_append(lfs, dir, entry, + &(struct lfs_region){ + 0, +lfs_entry_size(entry), + lfs_commit_disk, &(struct lfs_commit_disk){ + olddir.pair[0], entry->off, oldsize, regions}}); + if (err) { + return err; } - regions = &(struct lfs_region){ - LFS_FROM_DISK, 0, +prefix, - {.d.block = olddir.pair[0], .d.off = entry->off}, - prefix, regions}; - *tail = &(struct lfs_region){ - LFS_FROM_DISK, 0, +suffix, - {.d.block = olddir.pair[0], .d.off = entry->off+prefix+len}, - suffix}; - // remove old entry err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ - LFS_FROM_MEM, oldoff, -(lfs_entry_size(entry) - diff), - {.m.data = NULL}, 0}); + oldoff, -oldsize, + lfs_commit_mem, &(struct lfs_commit_mem){ + NULL, 0}}); if (err) { return err; } } + // TODO move to dir_commit? + // TODO this doesn't work... // shift over any files/directories that are affected for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { @@ -812,9 +825,11 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { } // shift out the entry - int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - LFS_FROM_MEM, entry->off, -lfs_entry_size(entry), - {.m.data = NULL}, 0}); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + entry->off, -lfs_entry_size(entry), + lfs_commit_mem, &(struct lfs_commit_mem){ + NULL, 0}}); if (err) { return err; } @@ -1036,11 +1051,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +sizeof(entry.d), - {.m.data = &entry.d}, sizeof(entry.d), + 0, +sizeof(entry.d), + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}, &(struct lfs_region){ - LFS_FROM_MEM, 0, +entry.d.nlen, - {.m.data = path}, entry.d.nlen}}); + 0, +entry.d.nlen, + lfs_commit_mem, &(struct lfs_commit_mem){ + path, entry.d.nlen}}}); if (err) { return err; } @@ -1426,11 +1443,13 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.u.file.size = 0; err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +sizeof(entry.d), - {.m.data = &entry.d}, sizeof(entry.d), + 0, +sizeof(entry.d), + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}, &(struct lfs_region){ - LFS_FROM_MEM, 0, +entry.d.nlen, - {.m.data = path}, entry.d.nlen}}); + 0, +entry.d.nlen, + lfs_commit_mem, &(struct lfs_commit_mem){ + path, entry.d.nlen}}}); if (err) { return err; } @@ -1648,8 +1667,9 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &entry.d}, sizeof(entry.d)}); + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}}); if (err) { return err; } @@ -2073,8 +2093,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { oldentry.d.type |= LFS_STRUCT_MOVED; err = lfs_dir_update(lfs, &oldcwd, &oldentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &oldentry.d}, sizeof(oldentry.d)}); + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &oldentry.d, sizeof(oldentry.d)}}); if (err) { return err; } @@ -2093,22 +2114,26 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (prevexists) { err = lfs_dir_update(lfs, &newcwd, &newentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &newentry.d}, sizeof(newentry.d), + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &newentry.d, sizeof(newentry.d)}, &(struct lfs_region){ - LFS_FROM_MEM, sizeof(newentry.d), 0, - {.m.data = newpath}, newentry.d.nlen}}); + sizeof(newentry.d), 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + newpath, newentry.d.nlen}}}); if (err) { return err; } } else { err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +sizeof(newentry.d), - {.m.data = &newentry.d}, sizeof(newentry.d), + 0, +sizeof(newentry.d), + lfs_commit_mem, &(struct lfs_commit_mem){ + &newentry.d, sizeof(newentry.d)}, &(struct lfs_region){ - LFS_FROM_MEM, 0, +newentry.d.nlen, - {.m.data = newpath}, newentry.d.nlen}}); + 0, +newentry.d.nlen, + lfs_commit_mem, &(struct lfs_commit_mem){ + newpath, newentry.d.nlen}}}); if (err) { return err; } @@ -2275,8 +2300,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { bool valid = false; for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ - LFS_FROM_MEM, sizeof(superdir.d), 0, - {.m.data = &superblock.d}, sizeof(superblock.d)}); + sizeof(superdir.d), 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &superblock.d, sizeof(superblock.d)}}); if (err && err != LFS_ERR_CORRUPT) { return err; } @@ -2542,8 +2568,9 @@ static int lfs_relocate(lfs_t *lfs, int err = lfs_dir_update(lfs, &parent, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &entry.d}, sizeof(entry.d)}); + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}}); if (err) { return err; } @@ -2669,8 +2696,9 @@ int lfs_deorphan(lfs_t *lfs) { entry.d.type &= ~LFS_STRUCT_MOVED; err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &entry.d}, sizeof(entry.d)}); + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}}); if (err) { return err; } From e4a0cd942d05778379b63f4c2c5c417d3cac4f74 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 11 Mar 2018 22:43:06 -0500 Subject: [PATCH 008/139] Take advantage of empty space early in dir search Before, when appending new entries to a directory, we try to find empty space in the last block of a directory chain. This has a nice side-effect that the order of directory entries is maintained. However, this isn't strictly necessary. We're already scanning the directory chain in order, so other than changes to directory order, there's no downside to taking advantage of any free space we come across. --- lfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lfs.c b/lfs.c index 35bb1646..5563b951 100644 --- a/lfs.c +++ b/lfs.c @@ -683,7 +683,8 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on while (true) { - if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { + if ((0x7fffffff & dir->d.size) + lfs_entry_size(entry) + <= lfs->cfg->block_size) { entry->off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { r->off += entry->off; From 362b0bbe45b803c38838b69c6a922476dc0957cd Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 12 Mar 2018 19:44:07 -0500 Subject: [PATCH 009/139] Minor improvement to from-memory commits Tweaked the commit callback to pass the arguments for from-memory commits explicitly, with non-from-memory commits still being able to hijack the opaque data pointer for additional state. The from-memory commits make up the vast majority of commits in littlefs, so this small change has a noticable impact. --- lfs.c | 88 ++++++++++++++++++++++++----------------------------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/lfs.c b/lfs.c index 5563b951..ba207102 100644 --- a/lfs.c +++ b/lfs.c @@ -490,7 +490,8 @@ struct lfs_commit { lfs_off_t off; }; -static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, const void *data, lfs_size_t size) { +static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, + const void *data, lfs_size_t size) { lfs_crc(&c->crc, data, size); int err = lfs_bd_prog(lfs, c->block, c->off, data, size); c->off += size; @@ -500,46 +501,44 @@ static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, const void *data, lfs_si struct lfs_region { lfs_off_t off; lfs_ssize_t diff; - int (*commit)(lfs_t *lfs, struct lfs_commit *c, const void *p); - const void *data; - struct lfs_region *next; -}; -struct lfs_commit_mem { + int (*commit)(lfs_t *lfs, struct lfs_commit *c, + const void *data, lfs_size_t size); const void *data; lfs_size_t size; + struct lfs_region *next; }; -static int lfs_commit_mem(lfs_t *lfs, struct lfs_commit *c, const void *p) { - const struct lfs_commit_mem *m = p; - return lfs_commit(lfs, c, m->data, m->size); +static int lfs_commit_mem(lfs_t *lfs, struct lfs_commit *c, + const void *data, lfs_size_t size) { + return lfs_commit(lfs, c, data, size); } struct lfs_commit_disk { lfs_block_t block; lfs_off_t off; - lfs_size_t size; struct lfs_region *regions; }; -static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, const void *p) { - const struct lfs_commit_disk *u = p; +static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, + const void *p, lfs_size_t size) { + const struct lfs_commit_disk *d = p; - struct lfs_region *r = u->regions; + struct lfs_region *r = d->regions; lfs_off_t off = 0; while (true) { if (r && r->off == off) { lfs_off_t orig = c->off; - int err = r->commit(lfs, c, r->data); + int err = r->commit(lfs, c, r->data, r->size); if (err) { return err; } off += (c->off - orig) - r->diff; r = r->next; - } else if (off < u->size) { + } else if (off < size) { uint8_t data; - int err = lfs_bd_read(lfs, u->block, u->off + off, &data, 1); + int err = lfs_bd_read(lfs, d->block, d->off + off, &data, 1); if (err) { return err; } @@ -590,12 +589,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, lfs_dir_tole32(&dir->d); err = lfs_commit_disk(lfs, &c, &(struct lfs_commit_disk){ - oldpair[1], 0, oldsize, + oldpair[1], 0, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &dir->d, sizeof(dir->d)}, - regions}}); + lfs_commit_mem, &dir->d, sizeof(dir->d), + regions}}, oldsize); lfs_dir_fromle32(&dir->d); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -754,8 +752,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, int err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ oldoff, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry->d.type, 1}}); + lfs_commit_mem, &entry->d.type, 1}); if (err) { return err; } @@ -766,7 +763,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, &(struct lfs_region){ 0, +lfs_entry_size(entry), lfs_commit_disk, &(struct lfs_commit_disk){ - olddir.pair[0], entry->off, oldsize, regions}}); + olddir.pair[0], entry->off, regions}, oldsize}); if (err) { return err; } @@ -775,8 +772,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ oldoff, -oldsize, - lfs_commit_mem, &(struct lfs_commit_mem){ - NULL, 0}}); + lfs_commit_mem, NULL, 0}); if (err) { return err; } @@ -829,8 +825,7 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ entry->off, -lfs_entry_size(entry), - lfs_commit_mem, &(struct lfs_commit_mem){ - NULL, 0}}); + lfs_commit_mem, NULL, 0}); if (err) { return err; } @@ -1053,12 +1048,10 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ 0, +sizeof(entry.d), - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}, + lfs_commit_mem, &entry.d, sizeof(entry.d), &(struct lfs_region){ 0, +entry.d.nlen, - lfs_commit_mem, &(struct lfs_commit_mem){ - path, entry.d.nlen}}}); + lfs_commit_mem, path, entry.d.nlen}}); if (err) { return err; } @@ -1445,12 +1438,10 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ 0, +sizeof(entry.d), - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}, + lfs_commit_mem, &entry.d, sizeof(entry.d), &(struct lfs_region){ 0, +entry.d.nlen, - lfs_commit_mem, &(struct lfs_commit_mem){ - path, entry.d.nlen}}}); + lfs_commit_mem, path, entry.d.nlen}}); if (err) { return err; } @@ -1669,8 +1660,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}}); + lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; } @@ -2095,8 +2085,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_update(lfs, &oldcwd, &oldentry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &oldentry.d, sizeof(oldentry.d)}}); + lfs_commit_mem, &oldentry.d, sizeof(oldentry.d)}); if (err) { return err; } @@ -2116,12 +2105,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_update(lfs, &newcwd, &newentry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &newentry.d, sizeof(newentry.d)}, + lfs_commit_mem, &newentry.d, sizeof(newentry.d), &(struct lfs_region){ sizeof(newentry.d), 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - newpath, newentry.d.nlen}}}); + lfs_commit_mem, newpath, newentry.d.nlen}}); if (err) { return err; } @@ -2129,12 +2116,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ 0, +sizeof(newentry.d), - lfs_commit_mem, &(struct lfs_commit_mem){ - &newentry.d, sizeof(newentry.d)}, + lfs_commit_mem, &newentry.d, sizeof(newentry.d), &(struct lfs_region){ 0, +newentry.d.nlen, - lfs_commit_mem, &(struct lfs_commit_mem){ - newpath, newentry.d.nlen}}}); + lfs_commit_mem, newpath, newentry.d.nlen}}); if (err) { return err; } @@ -2302,8 +2287,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ sizeof(superdir.d), 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &superblock.d, sizeof(superblock.d)}}); + lfs_commit_mem, &superblock.d, sizeof(superblock.d)}); if (err && err != LFS_ERR_CORRUPT) { return err; } @@ -2570,8 +2554,7 @@ static int lfs_relocate(lfs_t *lfs, int err = lfs_dir_update(lfs, &parent, &entry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}}); + lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; } @@ -2698,8 +2681,7 @@ int lfs_deorphan(lfs_t *lfs) { err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}}); + lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; } From 03b262b1e8f4fbaff9d3a2bc25e5c42aa7416c86 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 15 Mar 2018 20:32:00 -0500 Subject: [PATCH 010/139] Separated out version of dir remove/append for non-entries This allows updates to directories without needing to allocate an entry struct for every call. --- lfs.c | 177 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 91 insertions(+), 86 deletions(-) diff --git a/lfs.c b/lfs.c index ba207102..313198b9 100644 --- a/lfs.c +++ b/lfs.c @@ -677,15 +677,14 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *regions) { +static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, + lfs_off_t *off, lfs_size_t size, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on while (true) { - if ((0x7fffffff & dir->d.size) + lfs_entry_size(entry) - <= lfs->cfg->block_size) { - entry->off = dir->d.size - 4; + if ((0x7fffffff & dir->d.size) + size <= lfs->cfg->block_size) { + *off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; + r->off += *off; } return lfs_dir_commit(lfs, dir, regions); @@ -701,9 +700,9 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, dir->d.tail[0] = olddir.d.tail[0]; dir->d.tail[1] = olddir.d.tail[1]; - entry->off = dir->d.size - 4; + *off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; + r->off += *off; } err = lfs_dir_commit(lfs, dir, regions); @@ -724,9 +723,70 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, } } +static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, + lfs_off_t off, lfs_size_t size) { + // check if we should just drop the directory block + if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 + size) { + lfs_dir_t pdir; + int res = lfs_pred(lfs, dir->pair, &pdir); + if (res < 0) { + return res; + } + + if (pdir.d.size & 0x80000000) { + pdir.d.size &= dir->d.size | 0x7fffffff; + pdir.d.tail[0] = dir->d.tail[0]; + pdir.d.tail[1] = dir->d.tail[1]; + return lfs_dir_commit(lfs, &pdir, NULL); + } + } + + // shift out the entry + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + off, -size, + lfs_commit_mem, NULL, 0}); + if (err) { + return err; + } + + // shift over any files/directories that are affected + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0) { + if (f->poff == off) { + f->pair[0] = 0xffffffff; + f->pair[1] = 0xffffffff; + } else if (f->poff > off) { + f->poff -= size; + } + } + } + + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + if (d->off > off) { + d->off -= size; + d->pos -= size; + } + } + } + + return 0; +} + +static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *regions) { + return lfs_dir_append_(lfs, dir, + &entry->off, lfs_entry_size(entry), regions); +} + +static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { + return lfs_dir_remove_(lfs, dir, + entry->off, lfs_entry_size(entry)); +} + static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { - lfs_off_t oldoff = entry->off; // <- TODO rm me? lfs_ssize_t diff = 0; for (struct lfs_region *r = regions; r; r = r->next) { diff += r->diff; @@ -742,6 +802,24 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, if (err) { return err; } + + // shift over any files/directories that are affected + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0) { + if (f->poff > entry->off) { + f->poff += diff; + } + } + } + + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + if (d->off > entry->off) { + d->off += diff; + d->pos += diff; + } + } + } } else { lfs_dir_t olddir = *dir; lfs_off_t oldoff = entry->off; @@ -756,9 +834,9 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, if (err) { return err; } + entry->d.type &= LFS_STRUCT_MOVED; // append updated entry - entry->d.type &= LFS_STRUCT_MOVED; err = lfs_dir_append(lfs, dir, entry, &(struct lfs_region){ 0, +lfs_entry_size(entry), @@ -769,86 +847,13 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } // remove old entry - err = lfs_dir_commit(lfs, &olddir, - &(struct lfs_region){ - oldoff, -oldsize, - lfs_commit_mem, NULL, 0}); + err = lfs_dir_remove_(lfs, dir, oldoff, oldsize); if (err) { return err; } - } - // TODO move to dir_commit? - // TODO this doesn't work... - // shift over any files/directories that are affected - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == oldoff) { - f->poff = entry->off; - } else if (f->poff > entry->off) { - f->poff += diff; - } - } - } - - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > entry->off) { - d->off += diff; - d->pos += diff; - } - } - } - - return 0; -} - -static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { - // check if we should just drop the directory block - if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 - + lfs_entry_size(entry)) { - lfs_dir_t pdir; - int res = lfs_pred(lfs, dir->pair, &pdir); - if (res < 0) { - return res; - } - - if (pdir.d.size & 0x80000000) { - pdir.d.size &= dir->d.size | 0x7fffffff; - pdir.d.tail[0] = dir->d.tail[0]; - pdir.d.tail[1] = dir->d.tail[1]; - return lfs_dir_commit(lfs, &pdir, NULL); - } - } - - // shift out the entry - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - entry->off, -lfs_entry_size(entry), - lfs_commit_mem, NULL, 0}); - if (err) { - return err; - } - - // shift over any files/directories that are affected - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == entry->off) { - f->pair[0] = 0xffffffff; - f->pair[1] = 0xffffffff; - } else if (f->poff > entry->off) { - f->poff -= lfs_entry_size(entry); - } - } - } - - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > entry->off) { - d->off -= lfs_entry_size(entry); - d->pos -= lfs_entry_size(entry); - } - } + // TODO remove file under file? + // TODO need to shift entries? } return 0; From 9273ac708b69a1eb53b1add1058af3e5526986a8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 15 Mar 2018 20:58:29 -0500 Subject: [PATCH 011/139] Added size field to entry structure The size field is redundant, since an entry's size can be determined from the nlen+elen+alen+4. However, as you may have guessed from that expression, calculating the size this way is a bit roundabout and inefficient. Despite its redundancy, it's cheaper to store the size in the entry, though with a minor RAM cost. Note, extra care must now be taken to make sure these size and len fields don't fall out of sync. --- lfs.c | 82 +++++++++++++++++++++++------------------------------------ lfs.h | 3 +-- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/lfs.c b/lfs.c index 313198b9..66c8163c 100644 --- a/lfs.c +++ b/lfs.c @@ -397,10 +397,6 @@ static inline bool lfs_pairsync( (paira[0] == pairb[1] && paira[1] == pairb[0]); } -static inline lfs_size_t lfs_entry_size(const lfs_entry_t *entry) { - return 4 + entry->d.elen + entry->d.alen + entry->d.nlen; -} - static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { // allocate pair of dir blocks for (int i = 0; i < 2; i++) { @@ -677,14 +673,14 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, - lfs_off_t *off, lfs_size_t size, struct lfs_region *regions) { +static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on while (true) { - if ((0x7fffffff & dir->d.size) + size <= lfs->cfg->block_size) { - *off = dir->d.size - 4; + if ((0x7fffffff & dir->d.size) + entry->size <= lfs->cfg->block_size) { + entry->off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { - r->off += *off; + r->off += entry->off; } return lfs_dir_commit(lfs, dir, regions); @@ -700,9 +696,9 @@ static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, dir->d.tail[0] = olddir.d.tail[0]; dir->d.tail[1] = olddir.d.tail[1]; - *off = dir->d.size - 4; + entry->off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { - r->off += *off; + r->off += entry->off; } err = lfs_dir_commit(lfs, dir, regions); @@ -723,10 +719,10 @@ static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, } } -static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, - lfs_off_t off, lfs_size_t size) { +static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, + const lfs_entry_t *entry) { // check if we should just drop the directory block - if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 + size) { + if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 + entry->size) { lfs_dir_t pdir; int res = lfs_pred(lfs, dir->pair, &pdir); if (res < 0) { @@ -744,7 +740,7 @@ static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, // shift out the entry int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - off, -size, + entry->off, -entry->size, lfs_commit_mem, NULL, 0}); if (err) { return err; @@ -753,20 +749,20 @@ static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, // shift over any files/directories that are affected for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == off) { + if (f->poff == entry->off) { f->pair[0] = 0xffffffff; f->pair[1] = 0xffffffff; - } else if (f->poff > off) { - f->poff -= size; + } else if (f->poff > entry->off) { + f->poff -= entry->size; } } } for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > off) { - d->off -= size; - d->pos -= size; + if (d->off > entry->off) { + d->off -= entry->size; + d->pos -= entry->size; } } } @@ -774,17 +770,6 @@ static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *regions) { - return lfs_dir_append_(lfs, dir, - &entry->off, lfs_entry_size(entry), regions); -} - -static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { - return lfs_dir_remove_(lfs, dir, - entry->off, lfs_entry_size(entry)); -} - static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { lfs_ssize_t diff = 0; @@ -823,7 +808,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } else { lfs_dir_t olddir = *dir; lfs_off_t oldoff = entry->off; - lfs_size_t oldsize = lfs_entry_size(entry) - diff; + lfs_size_t oldsize = entry->size - diff; // mark as moving entry->d.type |= LFS_STRUCT_MOVED; @@ -839,7 +824,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, // append updated entry err = lfs_dir_append(lfs, dir, entry, &(struct lfs_region){ - 0, +lfs_entry_size(entry), + 0, +entry->size, lfs_commit_disk, &(struct lfs_commit_disk){ olddir.pair[0], entry->off, regions}, oldsize}); if (err) { @@ -847,13 +832,10 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } // remove old entry - err = lfs_dir_remove_(lfs, dir, oldoff, oldsize); + err = lfs_dir_remove(lfs, dir, &(lfs_entry_t){oldoff, oldsize}); if (err) { return err; } - - // TODO remove file under file? - // TODO need to shift entries? } return 0; @@ -883,8 +865,9 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { } entry->off = dir->off; - dir->off += lfs_entry_size(entry); - dir->pos += lfs_entry_size(entry); + entry->size = 4 + entry->d.elen + entry->d.alen + entry->d.nlen; + dir->off += entry->size; + dir->pos += entry->size; return 0; } @@ -903,9 +886,6 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, if (pathname[0] == '\0') { *entry = (lfs_entry_t){ .d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR, - .d.elen = sizeof(entry->d) - 4, - .d.alen = 0, - .d.nlen = 0, .d.u.dir[0] = lfs->root[0], .d.u.dir[1] = lfs->root[1], }; @@ -960,7 +940,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } int res = lfs_bd_cmp(lfs, dir->pair[0], - entry->off + 4+entry->d.elen+entry->d.alen, + entry->off + entry->size - pathlen, pathname, pathlen); if (res < 0) { return res; @@ -1046,6 +1026,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { entry.d.nlen = strlen(path); entry.d.u.dir[0] = dir.pair[0]; entry.d.u.dir[1] = dir.pair[1]; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; @@ -1164,7 +1145,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { } int err = lfs_bd_read(lfs, dir->pair[0], - entry.off + 4+entry.d.elen+entry.d.alen, + entry.off + entry.size - entry.d.nlen, info->name, entry.d.nlen); if (err) { return err; @@ -1440,6 +1421,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.nlen = strlen(path); entry.d.u.file.head = 0xffffffff; entry.d.u.file.size = 0; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ 0, +sizeof(entry.d), @@ -1959,7 +1942,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { strcpy(info->name, "/"); } else { err = lfs_bd_read(lfs, cwd.pair[0], - entry.off + 4+entry.d.elen+entry.d.alen, + entry.off + entry.size - entry.d.nlen, info->name, entry.d.nlen); if (err) { return err; @@ -2272,7 +2255,6 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // write superblocks lfs_superblock_t superblock = { - .off = sizeof(superdir.d), .d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK, .d.elen = sizeof(superblock.d) - sizeof(superblock.d.magic) - 4, .d.nlen = sizeof(superblock.d.magic), @@ -2374,8 +2356,6 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } // iterate over metadata pairs - lfs_dir_t dir; - lfs_entry_t entry; lfs_block_t cwd[2] = {0, 1}; while (true) { @@ -2386,12 +2366,14 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } } + lfs_dir_t dir; int err = lfs_dir_fetch(lfs, &dir, cwd); if (err) { return err; } // iterate over contents + lfs_entry_t entry; while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { err = lfs_bd_read(lfs, dir.pair[0], dir.off, &entry.d, sizeof(entry.d)); @@ -2400,7 +2382,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { return err; } - dir.off += lfs_entry_size(&entry); + dir.off += 4 + entry.d.elen + entry.d.alen + entry.d.nlen; if ((0x70 & entry.d.type) == LFS_STRUCT_CTZ) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, entry.d.u.file.head, entry.d.u.file.size, cb, data); diff --git a/lfs.h b/lfs.h index 4a12c4a2..b35ad249 100644 --- a/lfs.h +++ b/lfs.h @@ -196,6 +196,7 @@ struct lfs_info { /// littlefs data structures /// typedef struct lfs_entry { lfs_off_t off; + lfs_size_t size; struct lfs_disk_entry { uint8_t type; @@ -249,8 +250,6 @@ typedef struct lfs_dir { } lfs_dir_t; typedef struct lfs_superblock { - lfs_off_t off; - struct lfs_disk_superblock { uint8_t type; uint8_t elen; From fb2304487206c6deed88f941da45c3e6113ad5f7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 15 Mar 2018 21:26:03 -0500 Subject: [PATCH 012/139] Fixed big-endian support for entry structures --- lfs.c | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/lfs.c b/lfs.c index 66c8163c..cfbc2b91 100644 --- a/lfs.c +++ b/lfs.c @@ -349,11 +349,10 @@ static void lfs_entry_fromle32(struct lfs_disk_entry *d) { d->u.dir[1] = lfs_fromle32(d->u.dir[1]); } -// TODO -//static void lfs_entry_tole32(struct lfs_disk_entry *d) { -// d->u.dir[0] = lfs_tole32(d->u.dir[0]); -// d->u.dir[1] = lfs_tole32(d->u.dir[1]); -//} +static void lfs_entry_tole32(struct lfs_disk_entry *d) { + d->u.dir[0] = lfs_tole32(d->u.dir[0]); + d->u.dir[1] = lfs_tole32(d->u.dir[1]); +} static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { d->root[0] = lfs_fromle32(d->root[0]); @@ -683,7 +682,10 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } - return lfs_dir_commit(lfs, dir, regions); + lfs_entry_tole32(&entry->d); + int err = lfs_dir_commit(lfs, dir, regions); + lfs_entry_fromle32(&entry->d); + return err; } // we need to allocate a new dir block @@ -701,7 +703,9 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } + lfs_entry_tole32(&entry->d); err = lfs_dir_commit(lfs, dir, regions); + lfs_entry_fromle32(&entry->d); if (err) { return err; } @@ -783,7 +787,9 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } + lfs_entry_tole32(&entry->d); int err = lfs_dir_commit(lfs, dir, regions); + lfs_entry_fromle32(&entry->d); if (err) { return err; } @@ -807,32 +813,35 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } } else { lfs_dir_t olddir = *dir; - lfs_off_t oldoff = entry->off; - lfs_size_t oldsize = entry->size - diff; + lfs_entry_t oldentry = { + .off = entry->off, + .size = entry->size - diff, + .d.type = entry->d.type | LFS_STRUCT_MOVED, + }; // mark as moving - entry->d.type |= LFS_STRUCT_MOVED; int err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ - oldoff, 0, - lfs_commit_mem, &entry->d.type, 1}); + oldentry.off, 0, + lfs_commit_mem, &oldentry.d.type, 1}); if (err) { return err; } - entry->d.type &= LFS_STRUCT_MOVED; // append updated entry + lfs_entry_tole32(&entry->d); err = lfs_dir_append(lfs, dir, entry, &(struct lfs_region){ 0, +entry->size, lfs_commit_disk, &(struct lfs_commit_disk){ - olddir.pair[0], entry->off, regions}, oldsize}); + olddir.pair[0], entry->off, regions}, oldentry.size}); + lfs_entry_fromle32(&entry->d); if (err) { return err; } // remove old entry - err = lfs_dir_remove(lfs, dir, &(lfs_entry_t){oldoff, oldsize}); + err = lfs_dir_remove(lfs, dir, &oldentry); if (err) { return err; } @@ -1645,10 +1654,12 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; + lfs_entry_tole32(&entry.d); err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ 0, 0, lfs_commit_mem, &entry.d, sizeof(entry.d)}); + lfs_entry_fromle32(&entry.d); if (err) { return err; } @@ -2073,7 +2084,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_update(lfs, &oldcwd, &oldentry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &oldentry.d, sizeof(oldentry.d)}); + lfs_commit_mem, &oldentry.d.type, 1}); if (err) { return err; } From 836e23895a0e9205e6b530112dcc9374306417d7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 17 Mar 2018 10:28:14 -0500 Subject: [PATCH 013/139] Shoehorned in hacky implementation of inline files Proof-of-concept implementation of inline files that stores the file's content directly in its parent's directory pair. Inline files are indicated by a different type stored in an entry's struct field, and take advantage of resizable entries. Where a normal file's entry would normally hold the reference to the CTZ skip-list, an inline file's entry contains the contents of the actual file. Unfortunately, storing the inline file on disk is the easy part. We also need to manage inline files in the internals of littlefs and provide the same operations that we do on normal files, all while reusing as much code as possible to avoid a significant increase in code cost. There is a relatively simple, though maybe a bit hacky, solution here. If a file fits entirely in a cache line, the file logic never actually has to go to disk. This means we can just give the file a "pretend" block (hopefully one that would assert if ever written to), and carry out file operations as normal, as long as we catch the file before it exceeds the cache line and write out the file to an actual disk. --- lfs.c | 163 ++++++++++++++++++++++++++++++++------------- lfs.h | 28 +++++--- tests/test_dirs.sh | 2 +- 3 files changed, 133 insertions(+), 60 deletions(-) diff --git a/lfs.c b/lfs.c index cfbc2b91..d0128636 100644 --- a/lfs.c +++ b/lfs.c @@ -24,7 +24,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, const lfs_cache_t *pcache, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block != 0xffffffff); while (size > 0) { if (pcache && block == pcache->block && off >= pcache->off && @@ -68,6 +68,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, } // load to cache, first condition can no longer fail + LFS_ASSERT(block < lfs->cfg->block_count); rcache->block = block; rcache->off = off - (off % lfs->cfg->read_size); int err = lfs->cfg->read(lfs->cfg, rcache->block, @@ -121,6 +122,7 @@ static int lfs_cache_crc(lfs_t *lfs, lfs_cache_t *rcache, static int lfs_cache_flush(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache) { if (pcache->block != 0xffffffff) { + LFS_ASSERT(pcache->block < lfs->cfg->block_count); int err = lfs->cfg->prog(lfs->cfg, pcache->block, pcache->off, pcache->buffer, lfs->cfg->prog_size); if (err) { @@ -149,7 +151,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block != 0xffffffff); while (size > 0) { if (block == pcache->block && off >= pcache->off && @@ -181,6 +183,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, if (off % lfs->cfg->prog_size == 0 && size >= lfs->cfg->prog_size) { // bypass pcache? + LFS_ASSERT(block < lfs->cfg->block_count); lfs_size_t diff = size - (size % lfs->cfg->prog_size); int err = lfs->cfg->prog(lfs->cfg, block, off, data, diff); if (err) { @@ -240,6 +243,7 @@ static int lfs_bd_crc(lfs_t *lfs, lfs_block_t block, } static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { + LFS_ASSERT(block < lfs->cfg->block_count); return lfs->cfg->erase(lfs->cfg, block); } @@ -851,7 +855,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { - while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) { + while (dir->off >= (0x7fffffff & dir->d.size)-4) { if (!(0x80000000 & dir->d.size)) { entry->off = dir->off; return LFS_ERR_NOENT; @@ -942,8 +946,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (((0x7f & entry->d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && - (0x7f & entry->d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) || + if (((0xf & entry->d.type) != LFS_TYPE_REG && + (0xf & entry->d.type) != LFS_TYPE_DIR) || entry->d.nlen != pathlen) { continue; } @@ -1126,8 +1130,8 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { return (err == LFS_ERR_NOENT) ? 0 : err; } - if ((0x7f & entry.d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && - (0x7f & entry.d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { + if ((0xf & entry.d.type) != LFS_TYPE_REG && + (0xf & entry.d.type) != LFS_TYPE_DIR) { continue; } @@ -1149,8 +1153,10 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { } info->type = 0xf & entry.d.type; - if (info->type == LFS_TYPE_REG) { + if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; + } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { + info->size = entry.d.elen; } int err = lfs_bd_read(lfs, dir->pair[0], @@ -1424,18 +1430,16 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // create entry to remember name - entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; - entry.d.elen = sizeof(entry.d) - 4; + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; + entry.d.elen = 0; entry.d.alen = 0; entry.d.nlen = strlen(path); - entry.d.u.file.head = 0xffffffff; - entry.d.u.file.size = 0; entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - 0, +sizeof(entry.d), - lfs_commit_mem, &entry.d, sizeof(entry.d), + 0, +4, + lfs_commit_mem, &entry.d, 4, &(struct lfs_region){ 0, +entry.d.nlen, lfs_commit_mem, path, entry.d.nlen}}); @@ -1481,6 +1485,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } + // load inline files + if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) { + file->head = 0xfffffffe; + file->size = entry.d.elen; + file->flags |= LFS_F_INLINE; + file->cache.block = file->head; + file->cache.off = 0; + err = lfs_bd_read(lfs, cwd.pair[0], + entry.off + 4, + file->cache.buffer, file->size); + if (err) { + lfs_free(file->cache.buffer); + return err; + } + } + // add to list of files file->next = lfs->files; lfs->files = file; @@ -1556,6 +1576,20 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_INLINE) { + // do nothing since we won't need the cache for anything else + if (file->flags & LFS_F_READING) { + file->flags &= ~LFS_F_READING; + } + + if (file->flags & LFS_F_WRITING) { + file->flags &= ~LFS_F_WRITING; + file->flags |= LFS_F_DIRTY; + } + + return 0; + } + if (file->flags & LFS_F_READING) { // just drop read cache file->cache.block = 0xffffffff; @@ -1642,6 +1676,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } + // TODO entry read? lfs_entry_t entry = {.off = file->poff}; err = lfs_bd_read(lfs, cwd.pair[0], entry.off, &entry.d, sizeof(entry.d)); @@ -1649,19 +1684,35 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { if (err) { return err; } + LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - LFS_ASSERT(entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)); - entry.d.u.file.head = file->head; - entry.d.u.file.size = file->size; + if (file->flags & LFS_F_INLINE) { + file->size = lfs_max(file->pos, file->size); + lfs_ssize_t diff = file->size - entry.d.elen; + entry.d.elen = file->size; - lfs_entry_tole32(&entry.d); - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); - lfs_entry_fromle32(&entry.d); - if (err) { - return err; + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &entry.d, 4, + &(struct lfs_region){ + 4, diff, + lfs_commit_mem, file->cache.buffer, file->size}}); + if (err) { + return err; + } + } else { + entry.d.u.file.head = file->head; + entry.d.u.file.size = file->size; + + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &entry.d, sizeof(entry.d)}); + if (err) { + return err; + } } file->flags &= ~LFS_F_DIRTY; @@ -1699,11 +1750,16 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { - int err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, - file->pos, &file->block, &file->off); - if (err) { - return err; + if (file->flags & LFS_F_INLINE) { + file->block = 0xfffffffe; + file->off = 0; + } else { + int err = lfs_ctz_find(lfs, &file->cache, NULL, + file->head, file->size, + file->pos, &file->block, &file->off); + if (err) { + return err; + } } file->flags |= LFS_F_READING; @@ -1761,31 +1817,42 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } while (nsize > 0) { + //printf("pos %d\n", file->pos + nsize); + // TODO combine with block allocation? + if (file->pos + nsize >= LFS_INLINE_MAX) { + file->flags &= ~LFS_F_INLINE; + } + // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { - if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { - // find out which block we're extending from - int err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, - file->pos-1, &file->block, &file->off); + if (file->flags & LFS_F_INLINE) { + file->block = 0xfffffffe; + file->off = 0; + } else { + if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { + // find out which block we're extending from + int err = lfs_ctz_find(lfs, &file->cache, NULL, + file->head, file->size, + file->pos-1, &file->block, &file->off); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + + // mark cache as dirty since we may have read data into it + file->cache.block = 0xffffffff; + } + + // extend file with new blocks + lfs_alloc_ack(lfs); + int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache, + file->block, file->pos, + &file->block, &file->off); if (err) { file->flags |= LFS_F_ERRED; return err; } - - // mark cache as dirty since we may have read data into it - file->cache.block = 0xffffffff; - } - - // extend file with new blocks - lfs_alloc_ack(lfs); - int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache, - file->block, file->pos, - &file->block, &file->off); - if (err) { - file->flags |= LFS_F_ERRED; - return err; } file->flags |= LFS_F_WRITING; diff --git a/lfs.h b/lfs.h index b35ad249..75033b4c 100644 --- a/lfs.h +++ b/lfs.h @@ -55,6 +55,10 @@ typedef uint32_t lfs_block_t; #define LFS_NAME_MAX 255 #endif +#ifndef LFS_INLINE_MAX +#define LFS_INLINE_MAX 255 +#endif + // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { @@ -82,25 +86,27 @@ enum lfs_type { // on disk structure LFS_STRUCT_CTZ = 0x10, LFS_STRUCT_DIR = 0x20, + LFS_STRUCT_INLINE = 0x30, LFS_STRUCT_MOVED = 0x80, }; // File open flags enum lfs_open_flags { // open flags - LFS_O_RDONLY = 1, // Open a file as read only - LFS_O_WRONLY = 2, // Open a file as write only - LFS_O_RDWR = 3, // Open a file as read and write - LFS_O_CREAT = 0x0100, // Create a file if it does not exist - LFS_O_EXCL = 0x0200, // Fail if a file already exists - LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size - LFS_O_APPEND = 0x0800, // Move to end of file on every write + LFS_O_RDONLY = 1, // Open a file as read only + LFS_O_WRONLY = 2, // Open a file as write only + LFS_O_RDWR = 3, // Open a file as read and write + LFS_O_CREAT = 0x0100, // Create a file if it does not exist + LFS_O_EXCL = 0x0200, // Fail if a file already exists + LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size + LFS_O_APPEND = 0x0800, // Move to end of file on every write // internally used flags - LFS_F_DIRTY = 0x10000, // File does not match storage - LFS_F_WRITING = 0x20000, // File has been written since last flush - LFS_F_READING = 0x40000, // File has been read since last flush - LFS_F_ERRED = 0x80000, // An error occured during write + LFS_F_DIRTY = 0x10000, // File does not match storage + LFS_F_WRITING = 0x20000, // File has been written since last flush + LFS_F_READING = 0x40000, // File has been read since last flush + LFS_F_ERRED = 0x80000, // An error occured during write + LFS_F_INLINE = 0x100000, // Currently inlined in directory entry }; // File seek flags diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 53d76f7a..11dc26ef 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -357,7 +357,7 @@ tests/test.py << TEST TEST echo "--- Multi-block directory with files ---" -tests/test.py << TEST +tests/test.py -s << TEST lfs_mount(&lfs, &cfg) => 0; lfs_mkdir(&lfs, "prickly-pear") => 0; for (int i = 0; i < $LARGESIZE; i++) { From d8cadecba63c41c71a6a1796a25a197e8b07020f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 17 Mar 2018 20:32:16 -0500 Subject: [PATCH 014/139] Better implementation of inline files, now with overflowing Now when a file overflows the max inline file size, it will be correctly written out to a proper block. Additionally, tweaked corner cases around inline file, however this still needs significant testing. A real neat part that surprised me is that littlefs _already_ contains the logic for writing out inline files: in lfs_file_relocate! With a bit of tweaking, littlefs can pull off both the overflow from inline to normal files _and_ the relocating of bad blocks in files with the same piece of logic. --- lfs.c | 91 ++++++++++++++++++++++++++++++---------------- tests/test_dirs.sh | 2 +- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/lfs.c b/lfs.c index d0128636..56559ccd 100644 --- a/lfs.c +++ b/lfs.c @@ -554,6 +554,7 @@ static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, } } +// TODO handle overflowing reads (zero?) static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, struct lfs_region *regions) { // state for copying over @@ -1152,6 +1153,8 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { break; } + // TODO common info constructor? + // TODO also used in lfs_stat info->type = 0xf & entry.d.type; if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; @@ -1452,23 +1455,6 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_EXIST; } - // setup file struct - file->pair[0] = cwd.pair[0]; - file->pair[1] = cwd.pair[1]; - file->poff = entry.off; - file->head = entry.d.u.file.head; - file->size = entry.d.u.file.size; - file->flags = flags; - file->pos = 0; - - if (flags & LFS_O_TRUNC) { - if (file->size != 0) { - file->flags |= LFS_F_DIRTY; - } - file->head = 0xffffffff; - file->size = 0; - } - // allocate buffer if needed file->cache.block = 0xffffffff; if (lfs->cfg->file_buffer) { @@ -1485,6 +1471,25 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } + // TODO combine these below? + // setup file struct + file->pair[0] = cwd.pair[0]; + file->pair[1] = cwd.pair[1]; + file->poff = entry.off; + file->head = entry.d.u.file.head; + file->size = entry.d.u.file.size; + file->flags = flags; + file->pos = 0; + + if (flags & LFS_O_TRUNC) { + if (file->size != 0) { + file->flags |= LFS_F_DIRTY; + } + + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; + entry.d.elen = 0; + } + // load inline files if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) { file->head = 0xfffffffe; @@ -1528,9 +1533,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { -relocate: - LFS_DEBUG("Bad block at %d", file->block); - +relocate:; // just relocate what exists into new block lfs_block_t nblock; int err = lfs_alloc(lfs, &nblock); @@ -1583,6 +1586,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { } if (file->flags & LFS_F_WRITING) { + file->size = lfs_max(file->pos, file->size); file->flags &= ~LFS_F_WRITING; file->flags |= LFS_F_DIRTY; } @@ -1642,6 +1646,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { break; relocate: + LFS_DEBUG("Bad block at %d", file->block); err = lfs_file_relocate(lfs, file); if (err) { return err; @@ -1676,7 +1681,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - // TODO entry read? + // TODO entry read function? lfs_entry_t entry = {.off = file->poff}; err = lfs_bd_read(lfs, cwd.pair[0], entry.off, &entry.d, sizeof(entry.d)); @@ -1685,12 +1690,12 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; if (file->flags & LFS_F_INLINE) { - file->size = lfs_max(file->pos, file->size); lfs_ssize_t diff = file->size - entry.d.elen; + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; entry.d.elen = file->size; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ @@ -1703,12 +1708,17 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } } else { + lfs_ssize_t diff = sizeof(entry.d) - entry.d.elen; + entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; + entry.d.elen = sizeof(entry.d); entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + // TODO combine up? err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - 0, 0, + 0, diff, lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; @@ -1747,12 +1757,13 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, nsize = size; while (nsize > 0) { + // TODO can this be collapsed? // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { if (file->flags & LFS_F_INLINE) { file->block = 0xfffffffe; - file->off = 0; + file->off = file->pos; } else { int err = lfs_ctz_find(lfs, &file->cache, NULL, file->head, file->size, @@ -1816,19 +1827,31 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } } - while (nsize > 0) { - //printf("pos %d\n", file->pos + nsize); - // TODO combine with block allocation? - if (file->pos + nsize >= LFS_INLINE_MAX) { - file->flags &= ~LFS_F_INLINE; + // TODO combine with block allocation? + // TODO need to move out if no longer fits in block also + // TODO store INLINE_MAX in superblock? + if ((file->pos + nsize >= LFS_INLINE_MAX) || + (file->pos + nsize >= lfs->cfg->read_size)) { + int err = lfs_file_relocate(lfs, file); + if (err) { + file->flags |= LFS_F_ERRED; + return err; } + file->flags &= ~LFS_F_INLINE; + file->flags |= LFS_F_WRITING; + } + + while (nsize > 0) { + // TODO can this be collapsed? + // TODO can we reduce this now that block 0 is never allocated? + // TODO actually, how does this behave if inline max == 0? // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { if (file->flags & LFS_F_INLINE) { file->block = 0xfffffffe; - file->off = 0; + file->off = file->pos; } else { if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { // find out which block we're extending from @@ -1920,6 +1943,8 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, return file->pos; } +// TODO handle inlining? +// TODO note at least needs tests for trunc on inlined file int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { if ((file->flags & 3) == LFS_O_RDONLY) { return LFS_ERR_BADF; @@ -2012,8 +2037,10 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { memset(info, 0, sizeof(*info)); info->type = 0xf & entry.d.type; - if (info->type == LFS_TYPE_REG) { + if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; + } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { + info->size = entry.d.elen; } if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) { diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 11dc26ef..53d76f7a 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -357,7 +357,7 @@ tests/test.py << TEST TEST echo "--- Multi-block directory with files ---" -tests/test.py -s << TEST +tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_mkdir(&lfs, "prickly-pear") => 0; for (int i = 0; i < $LARGESIZE; i++) { From 701e4fa4386924ca34c31f8a1fee7e2497e9494b Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 18 Mar 2018 20:36:48 -0500 Subject: [PATCH 015/139] Fixed a handful of bugs as result of testing --- lfs.c | 211 ++++++++++++++++++++++-------------------- tests/test_alloc.sh | 12 ++- tests/test_corrupt.sh | 2 +- 3 files changed, 123 insertions(+), 102 deletions(-) diff --git a/lfs.c b/lfs.c index 56559ccd..376e083a 100644 --- a/lfs.c +++ b/lfs.c @@ -821,7 +821,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t oldentry = { .off = entry->off, .size = entry->size - diff, - .d.type = entry->d.type | LFS_STRUCT_MOVED, + .d.type = entry->d.type | LFS_STRUCT_MOVED, // TODO FIXME when changing type?? }; // mark as moving @@ -846,7 +846,12 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } // remove old entry - err = lfs_dir_remove(lfs, dir, &oldentry); + err = lfs_dir_fetch(lfs, &olddir, olddir.pair); + if (err) { + return err; + } + + err = lfs_dir_remove(lfs, &olddir, &oldentry); if (err) { return err; } @@ -1579,78 +1584,69 @@ relocate:; } static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { - if (file->flags & LFS_F_INLINE) { - // do nothing since we won't need the cache for anything else - if (file->flags & LFS_F_READING) { - file->flags &= ~LFS_F_READING; - } - - if (file->flags & LFS_F_WRITING) { - file->size = lfs_max(file->pos, file->size); - file->flags &= ~LFS_F_WRITING; - file->flags |= LFS_F_DIRTY; - } - - return 0; - } - if (file->flags & LFS_F_READING) { - // just drop read cache - file->cache.block = 0xffffffff; + if (!(file->flags & LFS_F_INLINE)) { + // just drop read cache + file->cache.block = 0xffffffff; + } file->flags &= ~LFS_F_READING; } if (file->flags & LFS_F_WRITING) { lfs_off_t pos = file->pos; - // copy over anything after current branch - lfs_file_t orig = { - .head = file->head, - .size = file->size, - .flags = LFS_O_RDONLY, - .pos = file->pos, - .cache = lfs->rcache, - }; - lfs->rcache.block = 0xffffffff; + if (!(file->flags & LFS_F_INLINE)) { + // copy over anything after current branch + lfs_file_t orig = { + .head = file->head, + .size = file->size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; + lfs->rcache.block = 0xffffffff; - while (file->pos < file->size) { - // copy over a byte at a time, leave it up to caching - // to make this efficient - uint8_t data; - lfs_ssize_t res = lfs_file_read(lfs, &orig, &data, 1); - if (res < 0) { - return res; - } + while (file->pos < file->size) { + // copy over a byte at a time, leave it up to caching + // to make this efficient + uint8_t data; + lfs_ssize_t res = lfs_file_read(lfs, &orig, &data, 1); + if (res < 0) { + return res; + } - res = lfs_file_write(lfs, file, &data, 1); - if (res < 0) { - return res; - } + res = lfs_file_write(lfs, file, &data, 1); + if (res < 0) { + return res; + } - // keep our reference to the rcache in sync - if (lfs->rcache.block != 0xffffffff) { - orig.cache.block = 0xffffffff; - lfs->rcache.block = 0xffffffff; + // keep our reference to the rcache in sync + if (lfs->rcache.block != 0xffffffff) { + orig.cache.block = 0xffffffff; + lfs->rcache.block = 0xffffffff; + } } - } - // write out what we have - while (true) { - int err = lfs_cache_flush(lfs, &file->cache, &lfs->rcache); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + // write out what we have + while (true) { + int err = lfs_cache_flush(lfs, &file->cache, &lfs->rcache); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; } - return err; - } - break; + break; relocate: - LFS_DEBUG("Bad block at %d", file->block); - err = lfs_file_relocate(lfs, file); - if (err) { - return err; + LFS_DEBUG("Bad block at %d", file->block); + err = lfs_file_relocate(lfs, file); + if (err) { + return err; + } } + } else { + file->size = lfs_max(file->pos, file->size); } // actual file updates @@ -1691,35 +1687,35 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - if (file->flags & LFS_F_INLINE) { - lfs_ssize_t diff = file->size - entry.d.elen; - entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = file->size; + if (!(file->flags & LFS_F_INLINE)) { + lfs_ssize_t diff = sizeof(entry.d)-4 - entry.d.elen; + entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; + entry.d.elen = sizeof(entry.d)-4; + entry.d.u.file.head = file->head; + entry.d.u.file.size = file->size; entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + // TODO combine down? err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, 4, - &(struct lfs_region){ - 4, diff, - lfs_commit_mem, file->cache.buffer, file->size}}); + 0, diff, + lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; } } else { - lfs_ssize_t diff = sizeof(entry.d) - entry.d.elen; - entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; - entry.d.elen = sizeof(entry.d); - entry.d.u.file.head = file->head; - entry.d.u.file.size = file->size; + lfs_ssize_t diff = file->size - entry.d.elen; + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; + entry.d.elen = file->size; entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - // TODO combine up? err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - 0, diff, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); + 0, 0, + lfs_commit_mem, &entry.d, 4, + &(struct lfs_region){ + 4, diff, + lfs_commit_mem, file->cache.buffer, file->size}}); if (err) { return err; } @@ -1761,16 +1757,16 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { - if (file->flags & LFS_F_INLINE) { - file->block = 0xfffffffe; - file->off = file->pos; - } else { + if (!(file->flags & LFS_F_INLINE)) { int err = lfs_ctz_find(lfs, &file->cache, NULL, file->head, file->size, file->pos, &file->block, &file->off); if (err) { return err; } + } else { + file->block = 0xfffffffe; + file->off = file->pos; } file->flags |= LFS_F_READING; @@ -1830,8 +1826,14 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // TODO combine with block allocation? // TODO need to move out if no longer fits in block also // TODO store INLINE_MAX in superblock? - if ((file->pos + nsize >= LFS_INLINE_MAX) || - (file->pos + nsize >= lfs->cfg->read_size)) { + // TODO what if inline files is > block size (ie 128) + if ((file->flags & LFS_F_INLINE) && ( + (file->pos + nsize >= LFS_INLINE_MAX) || + (file->pos + nsize >= lfs->cfg->read_size))) { + file->block = 0xfffffffe; + file->off = file->pos; + + lfs_alloc_ack(lfs); int err = lfs_file_relocate(lfs, file); if (err) { file->flags |= LFS_F_ERRED; @@ -1849,10 +1851,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { - if (file->flags & LFS_F_INLINE) { - file->block = 0xfffffffe; - file->off = file->pos; - } else { + if (!(file->flags & LFS_F_INLINE)) { if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { // find out which block we're extending from int err = lfs_ctz_find(lfs, &file->cache, NULL, @@ -1876,6 +1875,9 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, file->flags |= LFS_F_ERRED; return err; } + } else { + file->block = 0xfffffffe; + file->off = file->pos; } file->flags |= LFS_F_WRITING; @@ -1944,7 +1946,6 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, } // TODO handle inlining? -// TODO note at least needs tests for trunc on inlined file int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { if ((file->flags & 3) == LFS_O_RDONLY) { return LFS_ERR_BADF; @@ -2179,6 +2180,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { &(struct lfs_region){ 0, 0, lfs_commit_mem, &oldentry.d.type, 1}); + oldentry.d.type &= ~LFS_STRUCT_MOVED; if (err) { return err; } @@ -2193,26 +2195,39 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newentry.d = oldentry.d; newentry.d.type &= ~LFS_STRUCT_MOVED; newentry.d.nlen = strlen(newpath); + newentry.size = 4 + newentry.d.elen + newentry.d.alen + newentry.d.nlen; if (prevexists) { err = lfs_dir_update(lfs, &newcwd, &newentry, &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &newentry.d, sizeof(newentry.d), - &(struct lfs_region){ - sizeof(newentry.d), 0, - lfs_commit_mem, newpath, newentry.d.nlen}}); + 0, newentry.size - preventry.size, + lfs_commit_disk, &(struct lfs_commit_disk){ + oldcwd.pair[0], oldentry.off, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &newentry.d, 4, + &(struct lfs_region){ + newentry.size - newentry.d.nlen, + +newentry.d.nlen-oldentry.d.nlen, + lfs_commit_mem, newpath, newentry.d.nlen}}}, + oldentry.size}); if (err) { return err; } } else { err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ - 0, +sizeof(newentry.d), - lfs_commit_mem, &newentry.d, sizeof(newentry.d), - &(struct lfs_region){ - 0, +newentry.d.nlen, - lfs_commit_mem, newpath, newentry.d.nlen}}); + 0, +newentry.size, + lfs_commit_disk, &(struct lfs_commit_disk){ + oldcwd.pair[0], oldentry.off, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &newentry.d, 4, + &(struct lfs_region){ + newentry.size - newentry.d.nlen, + +newentry.d.nlen-oldentry.d.nlen, + lfs_commit_mem, newpath, newentry.d.nlen}}}, + oldentry.size}); if (err) { return err; } @@ -2507,7 +2522,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { // iterate over any open files for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (f->flags & LFS_F_DIRTY) { + if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, f->head, f->size, cb, data); if (err) { @@ -2515,7 +2530,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } } - if (f->flags & LFS_F_WRITING) { + if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, f->block, f->pos, cb, data); if (err) { diff --git a/tests/test_alloc.sh b/tests/test_alloc.sh index 8c814908..d3c8da40 100755 --- a/tests/test_alloc.sh +++ b/tests/test_alloc.sh @@ -274,9 +274,12 @@ TEST tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - // create one block whole for half a directory + // create one block hole for half a directory lfs_file_open(&lfs, &file[0], "bump", LFS_O_WRONLY | LFS_O_CREAT) => 0; - lfs_file_write(&lfs, &file[0], (void*)"hi", 2) => 2; + for (lfs_size_t i = 0; i < cfg.block_size; i += 2) { + memcpy(&buffer[i], "hi", 2); + } + lfs_file_write(&lfs, &file[0], buffer, cfg.block_size) => cfg.block_size; lfs_file_close(&lfs, &file[0]) => 0; lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); @@ -295,7 +298,10 @@ tests/test.py << TEST lfs_mkdir(&lfs, "splitdir") => 0; lfs_file_open(&lfs, &file[0], "splitdir/bump", LFS_O_WRONLY | LFS_O_CREAT) => 0; - lfs_file_write(&lfs, &file[0], buffer, size) => LFS_ERR_NOSPC; + for (lfs_size_t i = 0; i < cfg.block_size; i += 2) { + memcpy(&buffer[i], "hi", 2); + } + lfs_file_write(&lfs, &file[0], buffer, cfg.block_size) => LFS_ERR_NOSPC; lfs_file_close(&lfs, &file[0]) => 0; lfs_unmount(&lfs) => 0; diff --git a/tests/test_corrupt.sh b/tests/test_corrupt.sh index 44f1caee..6af984ff 100755 --- a/tests/test_corrupt.sh +++ b/tests/test_corrupt.sh @@ -88,7 +88,7 @@ do rm -rf blocks mkdir blocks lfs_mktree - chmod a-w blocks/$(printf '%x' $i) + chmod a-w blocks/$(printf '%x' $i) || true lfs_mktree lfs_chktree done From d0e0453651fb054004ac2212d70d49784afef1c0 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 23 Mar 2018 16:11:36 -0500 Subject: [PATCH 016/139] Changed how we write out superblock to use append Making the superblock look like "just another entry" allows us to treat the superblock like "just another entry" and reuse a decent amount of logic that would otherwise only be used a format and mount time. In this case we can use append to write out the superblock like it was creating a new entry on the filesystem. --- lfs.c | 77 ++++++++++++++++++++++++------------------- lfs.h | 5 --- tests/test_corrupt.sh | 4 +-- tests/test_format.sh | 12 +------ 4 files changed, 46 insertions(+), 52 deletions(-) diff --git a/lfs.c b/lfs.c index 376e083a..4375ad98 100644 --- a/lfs.c +++ b/lfs.c @@ -2372,38 +2372,40 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[0] = root.pair[0]; lfs->root[1] = root.pair[1]; + superdir.d.tail[0] = lfs->root[0]; + superdir.d.tail[1] = lfs->root[1]; - // write superblocks - lfs_superblock_t superblock = { - .d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK, - .d.elen = sizeof(superblock.d) - sizeof(superblock.d.magic) - 4, - .d.nlen = sizeof(superblock.d.magic), - .d.version = LFS_DISK_VERSION, - .d.magic = {"littlefs"}, - .d.block_size = lfs->cfg->block_size, - .d.block_count = lfs->cfg->block_count, - .d.root = {lfs->root[0], lfs->root[1]}, - }; - superdir.d.tail[0] = root.pair[0]; - superdir.d.tail[1] = root.pair[1]; - superdir.d.size = sizeof(superdir.d) + sizeof(superblock.d) + 4; - - // write both pairs to be safe + // write one superblocks + lfs_superblock_t superblock; + superblock.d.version = LFS_DISK_VERSION, + superblock.d.root[0] = lfs->root[0]; + superblock.d.root[1] = lfs->root[1]; + superblock.d.block_size = lfs->cfg->block_size; + superblock.d.block_count = lfs->cfg->block_count; + + lfs_entry_t superentry; + superentry.d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK; + superentry.d.elen = sizeof(superblock.d); + superentry.d.alen = 0; + superentry.d.nlen = strlen("littlefs"); + superentry.off = sizeof(superdir.d); + superentry.size = 4 + superentry.d.elen + + superentry.d.alen + superentry.d.nlen; + + lfs_entry_tole32(&superentry.d); lfs_superblock_tole32(&superblock.d); - bool valid = false; - for (int i = 0; i < 2; i++) { - err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ - sizeof(superdir.d), 0, - lfs_commit_mem, &superblock.d, sizeof(superblock.d)}); - if (err && err != LFS_ERR_CORRUPT) { - return err; - } - - valid = valid || !err; - } - - if (!valid) { - return LFS_ERR_CORRUPT; + err = lfs_dir_append(lfs, &superdir, &superentry, + &(struct lfs_region){ + 0, +4, + lfs_commit_mem, &superentry.d, 4, + &(struct lfs_region){ + 0, +sizeof(superblock.d), + lfs_commit_mem, &superblock.d, sizeof(superblock.d), + &(struct lfs_region){ + 0, +superentry.d.nlen, + lfs_commit_mem, "littlefs", superentry.d.nlen}}}); + if (err) { + return err; } // sanity check that fetch works @@ -2412,7 +2414,6 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - lfs_alloc_ack(lfs); return lfs_deinit(lfs); } @@ -2431,25 +2432,33 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // load superblock lfs_dir_t dir; lfs_superblock_t superblock; + char magic[8]; err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err && err != LFS_ERR_CORRUPT) { return err; } if (!err) { - err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d), + err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d)+4, &superblock.d, sizeof(superblock.d)); lfs_superblock_fromle32(&superblock.d); if (err) { return err; } + err = lfs_bd_read(lfs, dir.pair[0], + sizeof(dir.d) + 4 + sizeof(superblock.d), + magic, sizeof(magic)); + if (err) { + return err; + } + lfs->root[0] = superblock.d.root[0]; lfs->root[1] = superblock.d.root[1]; } - if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) { - LFS_ERROR("Invalid superblock at %d %d", 0, 1); + if (err || memcmp(magic, "littlefs", 8) != 0) { + LFS_ERROR("Invalid superblock at %d %d", dir.pair[0], dir.pair[1]); return LFS_ERR_CORRUPT; } diff --git a/lfs.h b/lfs.h index 75033b4c..6acf5e75 100644 --- a/lfs.h +++ b/lfs.h @@ -257,15 +257,10 @@ typedef struct lfs_dir { typedef struct lfs_superblock { struct lfs_disk_superblock { - uint8_t type; - uint8_t elen; - uint8_t alen; - uint8_t nlen; lfs_block_t root[2]; uint32_t block_size; uint32_t block_count; uint32_t version; - char magic[8]; } d; } lfs_superblock_t; diff --git a/tests/test_corrupt.sh b/tests/test_corrupt.sh index 6af984ff..1491dac3 100755 --- a/tests/test_corrupt.sh +++ b/tests/test_corrupt.sh @@ -73,7 +73,7 @@ lfs_mktree lfs_chktree echo "--- Block corruption ---" -for i in {0..33} +for i in {2..33} do rm -rf blocks mkdir blocks @@ -83,7 +83,7 @@ do done echo "--- Block persistance ---" -for i in {0..33} +for i in {2..33} do rm -rf blocks mkdir blocks diff --git a/tests/test_format.sh b/tests/test_format.sh index b9071015..cff0baca 100755 --- a/tests/test_format.sh +++ b/tests/test_format.sh @@ -30,20 +30,10 @@ echo "--- Invalid mount ---" tests/test.py << TEST lfs_format(&lfs, &cfg) => 0; TEST -rm blocks/0 blocks/1 +rm -f blocks/0 blocks/1 tests/test.py << TEST lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; TEST -echo "--- Valid corrupt mount ---" -tests/test.py << TEST - lfs_format(&lfs, &cfg) => 0; -TEST -rm blocks/0 -tests/test.py << TEST - lfs_mount(&lfs, &cfg) => 0; - lfs_unmount(&lfs) => 0; -TEST - echo "--- Results ---" tests/stats.py From ad74825bcf9ba1f4ab4742f26515f6a2bb92d6d4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 23 Mar 2018 18:35:55 -0500 Subject: [PATCH 017/139] Added internal lfs_dir_get to consolidate logic for reading dir entries It's a relatively simple function but offers some code reuse as well as making the dir entry operations a bit more readable. --- lfs.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/lfs.c b/lfs.c index 4375ad98..fdeebe08 100644 --- a/lfs.c +++ b/lfs.c @@ -677,6 +677,12 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } +// TODO zeros? +static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, + lfs_off_t off, void *buffer, lfs_size_t size) { + return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); +} + static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on @@ -876,8 +882,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { dir->pos += sizeof(dir->d) + 4; } - int err = lfs_bd_read(lfs, dir->pair[0], dir->off, - &entry->d, sizeof(entry->d)); + int err = lfs_dir_get(lfs, dir, dir->off, &entry->d, sizeof(entry->d)); lfs_entry_fromle32(&entry->d); if (err) { return err; @@ -1167,7 +1172,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { info->size = entry.d.elen; } - int err = lfs_bd_read(lfs, dir->pair[0], + int err = lfs_dir_get(lfs, dir, entry.off + entry.size - entry.d.nlen, info->name, entry.d.nlen); if (err) { @@ -1502,7 +1507,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, file->flags |= LFS_F_INLINE; file->cache.block = file->head; file->cache.off = 0; - err = lfs_bd_read(lfs, cwd.pair[0], + err = lfs_dir_get(lfs, &cwd, entry.off + 4, file->cache.buffer, file->size); if (err) { @@ -1679,8 +1684,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // TODO entry read function? lfs_entry_t entry = {.off = file->poff}; - err = lfs_bd_read(lfs, cwd.pair[0], entry.off, - &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &cwd, + entry.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); if (err) { return err; @@ -2047,7 +2052,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) { strcpy(info->name, "/"); } else { - err = lfs_bd_read(lfs, cwd.pair[0], + err = lfs_dir_get(lfs, &cwd, entry.off + entry.size - entry.d.nlen, info->name, entry.d.nlen); if (err) { @@ -2439,14 +2444,14 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } if (!err) { - err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d)+4, - &superblock.d, sizeof(superblock.d)); + err = lfs_dir_get(lfs, &dir, + sizeof(dir.d)+4, &superblock.d, sizeof(superblock.d)); lfs_superblock_fromle32(&superblock.d); if (err) { return err; } - err = lfs_bd_read(lfs, dir.pair[0], + err = lfs_dir_get(lfs, &dir, sizeof(dir.d) + 4 + sizeof(superblock.d), magic, sizeof(magic)); if (err) { @@ -2504,8 +2509,8 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { // iterate over contents lfs_entry_t entry; while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { - err = lfs_bd_read(lfs, dir.pair[0], dir.off, - &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &dir, + dir.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); if (err) { return err; From 955545839bfdfde234bfaa366242fc943d7f5532 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 27 Mar 2018 17:57:07 -0500 Subject: [PATCH 018/139] Added internal lfs_dir_set, an umbrella to dir append/update/remove operations This move was surprisingly complex, but offers the ultimate opportunity for code reuse in terms of resizable entries. Instead of needing to provide separate functions for adding and removing entries, adding and removing entries can just be viewed as changing an entry's size to-and-from zero. Unfortunately, it's not _quite_ that simple, since append and remove hide some relatively complex operations for when directory blocks overflow or need to be cleaned up. However, with enough shoehorning, and a new committer type that allows specifying recursive commit lists (is this now a push-down automata?), it does seem to be possible to shove all of the entry update logic into a single function. Sidenote, I switched back to an enum-based DSL, since the addition of a recursive region opcode breaks the consistency of what needs to be passed to the DSL callback functions. It's much simpler to handle each opcode explicitly inside a recursive lfs_commit_region function. --- lfs.c | 555 ++++++++++++++++++++++++++-------------------------------- 1 file changed, 251 insertions(+), 304 deletions(-) diff --git a/lfs.c b/lfs.c index fdeebe08..8d3017c9 100644 --- a/lfs.c +++ b/lfs.c @@ -152,6 +152,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; LFS_ASSERT(block != 0xffffffff); + LFS_ASSERT(off + size <= lfs->cfg->block_size); while (size > 0) { if (block == pcache->block && off >= pcache->off && @@ -483,83 +484,93 @@ static int lfs_dir_fetch(lfs_t *lfs, return 0; } -struct lfs_commit { - uint32_t crc; - lfs_block_t block; - lfs_off_t off; -}; - -static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, - const void *data, lfs_size_t size) { - lfs_crc(&c->crc, data, size); - int err = lfs_bd_prog(lfs, c->block, c->off, data, size); - c->off += size; - return err; -} - struct lfs_region { - lfs_off_t off; - lfs_ssize_t diff; + enum { + LFS_FROM_DROP, + LFS_FROM_MEM, + LFS_FROM_REGION, + } type; - int (*commit)(lfs_t *lfs, struct lfs_commit *c, - const void *data, lfs_size_t size); - const void *data; - lfs_size_t size; - struct lfs_region *next; + lfs_off_t off; + const void *buffer; + lfs_ssize_t size; }; -static int lfs_commit_mem(lfs_t *lfs, struct lfs_commit *c, - const void *data, lfs_size_t size) { - return lfs_commit(lfs, c, data, size); -} - -struct lfs_commit_disk { +struct lfs_region_region { lfs_block_t block; lfs_off_t off; struct lfs_region *regions; + int count; }; -static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, - const void *p, lfs_size_t size) { - const struct lfs_commit_disk *d = p; - - struct lfs_region *r = d->regions; - lfs_off_t off = 0; - while (true) { - if (r && r->off == off) { - lfs_off_t orig = c->off; - int err = r->commit(lfs, c, r->data, r->size); - if (err) { - return err; +static int lfs_commit_region(lfs_t *lfs, + lfs_block_t oldblock, lfs_off_t oldoff, + lfs_block_t newblock, lfs_off_t newoff, + lfs_off_t regionoff, + const struct lfs_region *regions, int count, + lfs_size_t size, uint32_t *crc) { + int i = 0; + lfs_size_t end = newoff + size; + while (newoff < end) { + // commit from different types of regions + if (i < count && regions[i].off == oldoff - regionoff) { + switch (regions[i].type) { + case LFS_FROM_DROP: { + oldoff -= regions[i].size; + break; + } + case LFS_FROM_MEM: { + lfs_crc(crc, regions[i].buffer, regions[i].size); + int err = lfs_bd_prog(lfs, newblock, newoff, + regions[i].buffer, regions[i].size); + if (err) { + return err; + } + newoff += regions[i].size; + break; + } + case LFS_FROM_REGION: { + const struct lfs_region_region *disk = regions[i].buffer; + int err = lfs_commit_region(lfs, + disk->block, disk->off, + newblock, newoff, + disk->off, disk->regions, disk->count, + regions[i].size, crc); + if (err) { + return err; + } + newoff += regions[i].size; + break; + } } - off += (c->off - orig) - r->diff; - r = r->next; - } else if (off < size) { + i += 1; + } else { + // copy data from old block if not covered by region uint8_t data; - int err = lfs_bd_read(lfs, d->block, d->off + off, &data, 1); + int err = lfs_bd_read(lfs, oldblock, oldoff, &data, 1); if (err) { return err; } - err = lfs_commit(lfs, c, &data, 1); + lfs_crc(crc, &data, 1); + err = lfs_bd_prog(lfs, newblock, newoff, &data, 1); if (err) { return err; } - off += 1; - } else { - return 0; + oldoff += 1; + newoff += 1; } } + + return 0; } -// TODO handle overflowing reads (zero?) -static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, - struct lfs_region *regions) { +static int lfs_dif_commit(lfs_t *lfs, lfs_dir_t *dir, + const struct lfs_region *regions, int count) { // state for copying over const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; - lfs_size_t oldsize = (0x7fffffff & dir->d.size) - 4; bool relocated = false; // increment revision count @@ -567,8 +578,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, // keep pairs in order such that pair[0] is most recent lfs_pairswap(dir->pair); - for (struct lfs_region *r = regions; r; r = r->next) { - dir->d.size += r->diff; + for (int i = 0; i < count; i++) { + dir->d.size += regions[i].size; } while (true) { @@ -581,19 +592,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - struct lfs_commit c = { - .crc = 0xffffffff, - .block = dir->pair[0], - .off = 0, - }; - + // commit header + uint32_t crc = 0xffffffff; lfs_dir_tole32(&dir->d); - err = lfs_commit_disk(lfs, &c, &(struct lfs_commit_disk){ - oldpair[1], 0, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &dir->d, sizeof(dir->d), - regions}}, oldsize); + lfs_crc(&crc, &dir->d, sizeof(dir->d)); + err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); lfs_dir_fromle32(&dir->d); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -602,9 +605,25 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - c.crc = lfs_tole32(c.crc); - err = lfs_bd_prog(lfs, dir->pair[0], c.off, &c.crc, 4); - c.crc = lfs_fromle32(c.crc); + // commit region + err = lfs_commit_region(lfs, + dir->pair[1], sizeof(dir->d), + dir->pair[0], sizeof(dir->d), + 0, regions, count, + (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, + &crc); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // commit crc + crc = lfs_tole32(crc); + err = lfs_bd_prog(lfs, dir->pair[0], + (0x7fffffff & dir->d.size)-4, &crc, 4); + crc = lfs_fromle32(crc); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -628,7 +647,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (ncrc != c.crc) { + if (ncrc != crc) { goto relocate; } } @@ -683,61 +702,106 @@ static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *regions) { - // check if we fit, if top bit is set we do not and move on - while (true) { - if ((0x7fffffff & dir->d.size) + entry->size <= lfs->cfg->block_size) { - entry->off = dir->d.size - 4; - for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; - } +static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, + struct lfs_region *regions, int count) { + lfs_ssize_t diff = 0; + for (int i = 0; i < count; i++) { + diff += regions[i].size; + } - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, regions); - lfs_entry_fromle32(&entry->d); - return err; - } + lfs_size_t oldsize = entry->size; + if (entry->off == 0) { + entry->off = (0x7fffffff & dir->d.size) - 4; + } - // we need to allocate a new dir block - if (!(0x80000000 & dir->d.size)) { - lfs_dir_t olddir = *dir; - int err = lfs_dir_alloc(lfs, dir); + if ((0x7fffffff & dir->d.size) + diff > lfs->cfg->block_size) { + lfs_dir_t olddir = *dir; + lfs_off_t oldoff = entry->off; + + if (oldsize) { + // mark as moving + uint8_t type; + int err = lfs_dir_get(lfs, &olddir, oldoff, &type, 1); + if (err) { + return err; + } + + type |= LFS_STRUCT_MOVED; + err = lfs_dif_commit(lfs, &olddir, (struct lfs_region[]){ + {LFS_FROM_MEM, oldoff, &type, 1}, + {LFS_FROM_DROP, oldoff, NULL, -1}}, 2); if (err) { return err; } + } - dir->d.tail[0] = olddir.d.tail[0]; - dir->d.tail[1] = olddir.d.tail[1]; - entry->off = dir->d.size - 4; - for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; + lfs_dir_t pdir = olddir; + + // find available block or create a new one + while ((0x7fffffff & dir->d.size) + oldsize + diff + > lfs->cfg->block_size) { + // we need to allocate a new dir block + if (!(0x80000000 & dir->d.size)) { + pdir = *dir; + int err = lfs_dir_alloc(lfs, dir); + if (err) { + return err; + } + + dir->d.tail[0] = pdir.d.tail[0]; + dir->d.tail[1] = pdir.d.tail[1]; + + break; } - lfs_entry_tole32(&entry->d); - err = lfs_dir_commit(lfs, dir, regions); - lfs_entry_fromle32(&entry->d); + int err = lfs_dir_fetch(lfs, dir, dir->d.tail); if (err) { return err; } + } + // writing out new entry + entry->off = dir->d.size - 4; + entry->size += diff; + int err = lfs_dif_commit(lfs, dir, (struct lfs_region[]){ + {LFS_FROM_REGION, entry->off, &(struct lfs_region_region){ + olddir.pair[0], oldoff, + regions, count}, entry->size}}, 1); + if (err) { + return err; + } + + // update pred dir, unless pred == old we can coalesce + if (!oldsize || lfs_paircmp(pdir.pair, olddir.pair) != 0) { + pdir.d.size |= 0x80000000; + pdir.d.tail[0] = dir->pair[0]; + pdir.d.tail[1] = dir->pair[1]; + + err = lfs_dif_commit(lfs, &pdir, NULL, 0); + if (err) { + return err; + } + } else if (oldsize) { olddir.d.size |= 0x80000000; olddir.d.tail[0] = dir->pair[0]; olddir.d.tail[1] = dir->pair[1]; - return lfs_dir_commit(lfs, &olddir, NULL); } - int err = lfs_dir_fetch(lfs, dir, dir->d.tail); - if (err) { - return err; + // remove old entry + if (oldsize) { + lfs_entry_t oldentry; + oldentry.off = oldoff; + err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ + {LFS_FROM_DROP, 0, NULL, -oldsize}}, 1); + if (err) { + return err; + } } + + goto shift; } -} -static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, - const lfs_entry_t *entry) { - // check if we should just drop the directory block - if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 + entry->size) { + if ((0x7fffffff & dir->d.size) + diff == sizeof(dir->d)+4) { lfs_dir_t pdir; int res = lfs_pred(lfs, dir->pair, &pdir); if (res < 0) { @@ -748,27 +812,34 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, pdir.d.size &= dir->d.size | 0x7fffffff; pdir.d.tail[0] = dir->d.tail[0]; pdir.d.tail[1] = dir->d.tail[1]; - return lfs_dir_commit(lfs, &pdir, NULL); + int err = lfs_dif_commit(lfs, &pdir, NULL, 0); + if (err) { + return err; + } + goto shift; } } - // shift out the entry - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - entry->off, -entry->size, - lfs_commit_mem, NULL, 0}); + for (int i = 0; i < count; i++) { + regions[i].off += entry->off; + } + + int err = lfs_dif_commit(lfs, dir, regions, count); if (err) { return err; } + entry->size += diff; + +shift: // shift over any files/directories that are affected for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == entry->off) { + if (f->poff == entry->off && entry->size == 0) { f->pair[0] = 0xffffffff; f->pair[1] = 0xffffffff; } else if (f->poff > entry->off) { - f->poff -= entry->size; + f->poff += diff; } } } @@ -776,91 +847,10 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->pair, dir->pair) == 0) { if (d->off > entry->off) { - d->off -= entry->size; - d->pos -= entry->size; - } - } - } - - return 0; -} - -static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *regions) { - lfs_ssize_t diff = 0; - for (struct lfs_region *r = regions; r; r = r->next) { - diff += r->diff; - } - - // do we still fit? - if ((0x7fffffff & dir->d.size) + diff <= lfs->cfg->block_size) { - for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; - } - - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, regions); - lfs_entry_fromle32(&entry->d); - if (err) { - return err; - } - - // shift over any files/directories that are affected - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff > entry->off) { - f->poff += diff; - } - } - } - - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > entry->off) { - d->off += diff; - d->pos += diff; - } + d->off += diff; + d->pos += diff; } } - } else { - lfs_dir_t olddir = *dir; - lfs_entry_t oldentry = { - .off = entry->off, - .size = entry->size - diff, - .d.type = entry->d.type | LFS_STRUCT_MOVED, // TODO FIXME when changing type?? - }; - - // mark as moving - int err = lfs_dir_commit(lfs, &olddir, - &(struct lfs_region){ - oldentry.off, 0, - lfs_commit_mem, &oldentry.d.type, 1}); - if (err) { - return err; - } - - // append updated entry - lfs_entry_tole32(&entry->d); - err = lfs_dir_append(lfs, dir, entry, - &(struct lfs_region){ - 0, +entry->size, - lfs_commit_disk, &(struct lfs_commit_disk){ - olddir.pair[0], entry->off, regions}, oldentry.size}); - lfs_entry_fromle32(&entry->d); - if (err) { - return err; - } - - // remove old entry - err = lfs_dir_fetch(lfs, &olddir, olddir.pair); - if (err) { - return err; - } - - err = lfs_dir_remove(lfs, &olddir, &oldentry); - if (err) { - return err; - } } return 0; @@ -1039,7 +1029,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { dir.d.tail[0] = cwd.d.tail[0]; dir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dir_commit(lfs, &dir, NULL); + err = lfs_dif_commit(lfs, &dir, NULL, 0); if (err) { return err; } @@ -1050,18 +1040,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { entry.d.nlen = strlen(path); entry.d.u.dir[0] = dir.pair[0]; entry.d.u.dir[1] = dir.pair[1]; - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + entry.size = 0; cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; - - err = lfs_dir_append(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, +sizeof(entry.d), - lfs_commit_mem, &entry.d, sizeof(entry.d), - &(struct lfs_region){ - 0, +entry.d.nlen, - lfs_commit_mem, path, entry.d.nlen}}); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_MEM, 0, path, entry.d.nlen}}, 2); if (err) { return err; } @@ -1447,15 +1432,11 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.elen = 0; entry.d.alen = 0; entry.d.nlen = strlen(path); - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + entry.size = 0; - err = lfs_dir_append(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, +4, - lfs_commit_mem, &entry.d, 4, - &(struct lfs_region){ - 0, +entry.d.nlen, - lfs_commit_mem, path, entry.d.nlen}}); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, 4}, + {LFS_FROM_MEM, 0, path, entry.d.nlen}}, 2); if (err) { return err; } @@ -1684,43 +1665,37 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // TODO entry read function? lfs_entry_t entry = {.off = file->poff}; - err = lfs_dir_get(lfs, &cwd, - entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); if (err) { return err; } + LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); + lfs_size_t oldlen = entry.d.elen; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { - lfs_ssize_t diff = sizeof(entry.d)-4 - entry.d.elen; entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; entry.d.elen = sizeof(entry.d)-4; entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - // TODO combine down? - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, diff, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_DROP, 0, NULL, -oldlen-4}}, 2); if (err) { return err; } } else { - lfs_ssize_t diff = file->size - entry.d.elen; entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; entry.d.elen = file->size; - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, 4, - &(struct lfs_region){ - 4, diff, - lfs_commit_mem, file->cache.buffer, file->size}}); + + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, 4}, + {LFS_FROM_MEM, 0, file->cache.buffer, file->size}, + {LFS_FROM_DROP, 0, NULL, -oldlen-4}}, 3); if (err) { return err; } @@ -2098,7 +2073,8 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // remove the entry - err = lfs_dir_remove(lfs, &cwd, &entry); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_DROP, 0, NULL, -entry.size}}, 1); if (err) { return err; } @@ -2114,7 +2090,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.d.tail[0]; cwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dir_commit(lfs, &cwd, NULL); + err = lfs_dif_commit(lfs, &cwd, NULL, 0); if (err) { return err; } @@ -2181,10 +2157,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving oldentry.d.type |= LFS_STRUCT_MOVED; - err = lfs_dir_update(lfs, &oldcwd, &oldentry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &oldentry.d.type, 1}); + err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &oldentry.d.type, 1}, + {LFS_FROM_DROP, 0, NULL, -1}}, 2); oldentry.d.type &= ~LFS_STRUCT_MOVED; if (err) { return err; @@ -2200,42 +2175,22 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newentry.d = oldentry.d; newentry.d.type &= ~LFS_STRUCT_MOVED; newentry.d.nlen = strlen(newpath); - newentry.size = 4 + newentry.d.elen + newentry.d.alen + newentry.d.nlen; - - if (prevexists) { - err = lfs_dir_update(lfs, &newcwd, &newentry, - &(struct lfs_region){ - 0, newentry.size - preventry.size, - lfs_commit_disk, &(struct lfs_commit_disk){ - oldcwd.pair[0], oldentry.off, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &newentry.d, 4, - &(struct lfs_region){ - newentry.size - newentry.d.nlen, - +newentry.d.nlen-oldentry.d.nlen, - lfs_commit_mem, newpath, newentry.d.nlen}}}, - oldentry.size}); - if (err) { - return err; - } - } else { - err = lfs_dir_append(lfs, &newcwd, &newentry, - &(struct lfs_region){ - 0, +newentry.size, - lfs_commit_disk, &(struct lfs_commit_disk){ - oldcwd.pair[0], oldentry.off, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &newentry.d, 4, - &(struct lfs_region){ - newentry.size - newentry.d.nlen, - +newentry.d.nlen-oldentry.d.nlen, - lfs_commit_mem, newpath, newentry.d.nlen}}}, - oldentry.size}); - if (err) { - return err; - } + if (!prevexists) { + newentry.size = 0; + } + + lfs_size_t newsize = oldentry.size - oldentry.d.nlen + newentry.d.nlen; + err = lfs_dir_set(lfs, &newcwd, &newentry, (struct lfs_region[]){ + {LFS_FROM_REGION, 0, &(struct lfs_region_region){ + oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &newentry.d, 4}, + {LFS_FROM_DROP, 0, NULL, -4}, + {LFS_FROM_MEM, newsize - newentry.d.nlen, + newpath, newentry.d.nlen}}, 3}, + newsize}, + {LFS_FROM_DROP, 0, NULL, -preventry.size}}, prevexists ? 2 : 1); + if (err) { + return err; } // update pair if newcwd == oldcwd @@ -2244,7 +2199,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // remove old entry - err = lfs_dir_remove(lfs, &oldcwd, &oldentry); + err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ + {LFS_FROM_DROP, 0, NULL, -oldentry.size}}, 1); if (err) { return err; } @@ -2260,7 +2216,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newcwd.d.tail[0] = dir.d.tail[0]; newcwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dir_commit(lfs, &newcwd, NULL); + err = lfs_dif_commit(lfs, &newcwd, NULL, 0); if (err) { return err; } @@ -2370,7 +2326,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_commit(lfs, &root, NULL); + err = lfs_dif_commit(lfs, &root, NULL, 0); if (err) { return err; } @@ -2394,21 +2350,14 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { superentry.d.alen = 0; superentry.d.nlen = strlen("littlefs"); superentry.off = sizeof(superdir.d); - superentry.size = 4 + superentry.d.elen + - superentry.d.alen + superentry.d.nlen; + superentry.size = 0; lfs_entry_tole32(&superentry.d); lfs_superblock_tole32(&superblock.d); - err = lfs_dir_append(lfs, &superdir, &superentry, - &(struct lfs_region){ - 0, +4, - lfs_commit_mem, &superentry.d, 4, - &(struct lfs_region){ - 0, +sizeof(superblock.d), - lfs_commit_mem, &superblock.d, sizeof(superblock.d), - &(struct lfs_region){ - 0, +superentry.d.nlen, - lfs_commit_mem, "littlefs", superentry.d.nlen}}}); + err = lfs_dir_set(lfs, &superdir, &superentry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &superentry.d, 4}, + {LFS_FROM_MEM, 0, &superblock.d, sizeof(superblock.d)}, + {LFS_FROM_MEM, 0, "littlefs", superentry.d.nlen}}, 3); if (err) { return err; } @@ -2452,8 +2401,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } err = lfs_dir_get(lfs, &dir, - sizeof(dir.d) + 4 + sizeof(superblock.d), - magic, sizeof(magic)); + sizeof(dir.d)+4 + sizeof(superblock.d), magic, sizeof(magic)); if (err) { return err; } @@ -2671,11 +2619,9 @@ static int lfs_relocate(lfs_t *lfs, // update disk, this creates a desync entry.d.u.dir[0] = newpair[0]; entry.d.u.dir[1] = newpair[1]; - - int err = lfs_dir_update(lfs, &parent, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); + int err = lfs_dir_set(lfs, &parent, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_DROP, 0, NULL, (lfs_ssize_t)-sizeof(entry.d)}}, 2); if (err) { return err; } @@ -2702,7 +2648,7 @@ static int lfs_relocate(lfs_t *lfs, parent.d.tail[0] = newpair[0]; parent.d.tail[1] = newpair[1]; - return lfs_dir_commit(lfs, &parent, NULL); + return lfs_dif_commit(lfs, &parent, NULL, 0); } // couldn't find dir, must be new @@ -2744,7 +2690,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = cwd.d.tail[0]; pdir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dir_commit(lfs, &pdir, NULL); + err = lfs_dif_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -2760,7 +2706,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = entry.d.u.dir[0]; pdir.d.tail[1] = entry.d.u.dir[1]; - err = lfs_dir_commit(lfs, &pdir, NULL); + err = lfs_dif_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -2791,7 +2737,8 @@ int lfs_deorphan(lfs_t *lfs) { if (moved) { LFS_DEBUG("Found move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); - err = lfs_dir_remove(lfs, &cwd, &entry); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_DROP, 0, NULL, -entry.size}}, 1); if (err) { return err; } @@ -2799,10 +2746,10 @@ int lfs_deorphan(lfs_t *lfs) { LFS_DEBUG("Found partial move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); entry.d.type &= ~LFS_STRUCT_MOVED; - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_DROP, 0, NULL, + (lfs_ssize_t)-sizeof(entry.d)}}, 2); if (err) { return err; } From 6362afa8d06da19f9ce9b1b5b62100ec8a16031d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 1 Apr 2018 15:36:29 -0500 Subject: [PATCH 019/139] Added disk-backed limits on the name/attrs/inline sizes Being a portable, microcontroller-scale embedded filesystem, littlefs is presented with a relatively unique challenge. The amount of RAM available is on completely different scales from machine to machine, and what is normally a reasonable RAM assumption may break completely on an embedded system. A great example of this is file names. On almost every PC these days, the limit for a file name is 255 bytes. It's a very convenient limit for a number of reasons. However, on microcontrollers, allocating 255 bytes of RAM to do a file search can be unreasonable. The simplest solution (and one that has existing in littlefs for a while), is to let this limit be redefined to a smaller value on devices that need to save RAM. However, this presents an interesting portability issue. If these devices are plugged into a PC with relatively infinite RAM, nothing stops the PC from writing files with full 255-byte file names, which can't be read on the small device. One solution here is to store this limit on the superblock during format time. When mounting a disk, the filesystem implementation is responsible for checking this limit in the superblock. If it's larger than what can be read, raise an error. If it's smaller, respect the limit on the superblock and raise an error if the user attempts to exceed it. In this commit, this strategy is adopted for file names, inline files, and the size of all attributes, since these could impact the memory consumption of the filesystem. (Recording the attribute's limit is iffy, but is the only other arbitrary limit and could be used for disabling support of custom attributes). Note! This changes makes it very important to configure littlefs correctly at format time. If littlefs is formatted on a PC without changing the limits appropriately, it will be rejected by a smaller device. --- lfs.c | 146 +++++++++++++++++++++++++++++++++++++++++++++------------- lfs.h | 67 ++++++++++++++++++--------- 2 files changed, 159 insertions(+), 54 deletions(-) diff --git a/lfs.c b/lfs.c index 8d3017c9..ad3ad041 100644 --- a/lfs.c +++ b/lfs.c @@ -365,6 +365,9 @@ static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { d->block_size = lfs_fromle32(d->block_size); d->block_count = lfs_fromle32(d->block_count); d->version = lfs_fromle32(d->version); + d->inline_size = lfs_fromle32(d->inline_size); + d->attrs_size = lfs_fromle32(d->attrs_size); + d->name_size = lfs_fromle32(d->name_size); } static void lfs_superblock_tole32(struct lfs_disk_superblock *d) { @@ -373,6 +376,9 @@ static void lfs_superblock_tole32(struct lfs_disk_superblock *d) { d->block_size = lfs_tole32(d->block_size); d->block_count = lfs_tole32(d->block_count); d->version = lfs_tole32(d->version); + d->inline_size = lfs_tole32(d->inline_size); + d->attrs_size = lfs_tole32(d->attrs_size); + d->name_size = lfs_tole32(d->name_size); } @@ -1018,6 +1024,12 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err ? err : LFS_ERR_EXIST; } + // check that name fits + lfs_size_t nlen = strlen(path); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + // build up new directory lfs_alloc_ack(lfs); @@ -1037,7 +1049,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { entry.d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR; entry.d.elen = sizeof(entry.d) - 4; entry.d.alen = 0; - entry.d.nlen = strlen(path); + entry.d.nlen = nlen; entry.d.u.dir[0] = dir.pair[0]; entry.d.u.dir[1] = dir.pair[1]; entry.size = 0; @@ -1046,7 +1058,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[1] = dir.pair[1]; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_MEM, 0, path, entry.d.nlen}}, 2); + {LFS_FROM_MEM, 0, path, nlen}}, 2); if (err) { return err; } @@ -1427,16 +1439,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_NOENT; } + // check that name fits + lfs_size_t nlen = strlen(path); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + // create entry to remember name entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; entry.d.elen = 0; entry.d.alen = 0; - entry.d.nlen = strlen(path); + entry.d.nlen = nlen; entry.size = 0; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, path, entry.d.nlen}}, 2); + {LFS_FROM_MEM, 0, path, nlen}}, 2); if (err) { return err; } @@ -1571,10 +1589,6 @@ relocate:; static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { if (file->flags & LFS_F_READING) { - if (!(file->flags & LFS_F_INLINE)) { - // just drop read cache - file->cache.block = 0xffffffff; - } file->flags &= ~LFS_F_READING; } @@ -1807,9 +1821,8 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // TODO need to move out if no longer fits in block also // TODO store INLINE_MAX in superblock? // TODO what if inline files is > block size (ie 128) - if ((file->flags & LFS_F_INLINE) && ( - (file->pos + nsize >= LFS_INLINE_MAX) || - (file->pos + nsize >= lfs->cfg->read_size))) { + if ((file->flags & LFS_F_INLINE) && + file->pos + nsize >= lfs->inline_size) { file->block = 0xfffffffe; file->off = file->pos; @@ -2137,6 +2150,12 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { bool prevexists = (err != LFS_ERR_NOENT); bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); + // check that name fits + lfs_size_t nlen = strlen(newpath); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + // must have same type if (prevexists && preventry.d.type != oldentry.d.type) { return LFS_ERR_ISDIR; @@ -2174,7 +2193,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_entry_t newentry = preventry; newentry.d = oldentry.d; newentry.d.type &= ~LFS_STRUCT_MOVED; - newentry.d.nlen = strlen(newpath); + newentry.d.nlen = nlen; if (!prevexists) { newentry.size = 0; } @@ -2185,8 +2204,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ {LFS_FROM_MEM, 0, &newentry.d, 4}, {LFS_FROM_DROP, 0, NULL, -4}, - {LFS_FROM_MEM, newsize - newentry.d.nlen, - newpath, newentry.d.nlen}}, 3}, + {LFS_FROM_MEM, newsize - nlen, newpath, nlen}}, 3}, newsize}, {LFS_FROM_DROP, 0, NULL, -preventry.size}}, prevexists ? 2 : 1); if (err) { @@ -2272,6 +2290,26 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) <= lfs->cfg->block_size); + // check that the size limits are sane + LFS_ASSERT(lfs->cfg->inline_size <= LFS_INLINE_MAX); + LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->read_size); + lfs->inline_size = lfs->cfg->inline_size; + if (!lfs->inline_size) { + lfs->inline_size = lfs_min(LFS_INLINE_MAX, lfs->cfg->read_size); + } + + LFS_ASSERT(lfs->cfg->attrs_size <= LFS_ATTRS_MAX); + lfs->attrs_size = lfs->cfg->attrs_size; + if (!lfs->attrs_size) { + lfs->attrs_size = LFS_ATTRS_MAX; + } + + LFS_ASSERT(lfs->cfg->name_size <= LFS_NAME_MAX); + lfs->name_size = lfs->cfg->name_size; + if (!lfs->name_size) { + lfs->name_size = LFS_NAME_MAX; + } + // setup default state lfs->root[0] = 0xffffffff; lfs->root[1] = 0xffffffff; @@ -2336,13 +2374,16 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { superdir.d.tail[0] = lfs->root[0]; superdir.d.tail[1] = lfs->root[1]; - // write one superblocks + // write one superblock lfs_superblock_t superblock; superblock.d.version = LFS_DISK_VERSION, superblock.d.root[0] = lfs->root[0]; superblock.d.root[1] = lfs->root[1]; superblock.d.block_size = lfs->cfg->block_size; superblock.d.block_count = lfs->cfg->block_count; + superblock.d.inline_size = lfs->inline_size; + superblock.d.attrs_size = lfs->attrs_size; + superblock.d.name_size = lfs->name_size; lfs_entry_t superentry; superentry.d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK; @@ -2385,33 +2426,41 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // load superblock lfs_dir_t dir; + lfs_entry_t entry; lfs_superblock_t superblock; char magic[8]; + err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); - if (err && err != LFS_ERR_CORRUPT) { + if (err) { + if (err == LFS_ERR_CORRUPT) { + LFS_ERROR("Invalid superblock at %d %d", 0, 1); + } return err; } - if (!err) { - err = lfs_dir_get(lfs, &dir, - sizeof(dir.d)+4, &superblock.d, sizeof(superblock.d)); - lfs_superblock_fromle32(&superblock.d); - if (err) { - return err; - } + err = lfs_dir_get(lfs, &dir, sizeof(dir.d), &entry.d, sizeof(entry.d)); + if (err) { + return err; + } - err = lfs_dir_get(lfs, &dir, - sizeof(dir.d)+4 + sizeof(superblock.d), magic, sizeof(magic)); - if (err) { - return err; - } + memset(&superblock.d, 0, sizeof(superblock.d)); + err = lfs_dir_get(lfs, &dir, + sizeof(dir.d)+4, &superblock.d, + lfs_min(sizeof(superblock.d), entry.d.elen)); + lfs_superblock_fromle32(&superblock.d); + if (err) { + return err; + } - lfs->root[0] = superblock.d.root[0]; - lfs->root[1] = superblock.d.root[1]; + err = lfs_dir_get(lfs, &dir, + sizeof(dir.d)+4+entry.d.elen+entry.d.alen, magic, + lfs_min(sizeof(magic), entry.d.nlen)); + if (err) { + return err; } - if (err || memcmp(magic, "littlefs", 8) != 0) { - LFS_ERROR("Invalid superblock at %d %d", dir.pair[0], dir.pair[1]); + if (memcmp(magic, "littlefs", 8) != 0) { + LFS_ERROR("Invalid superblock at %d %d", 0, 1); return LFS_ERR_CORRUPT; } @@ -2423,6 +2472,39 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return LFS_ERR_INVAL; } + if (superblock.d.inline_size) { + if (superblock.d.inline_size > lfs->inline_size) { + LFS_ERROR("Unsupported inline size (%d > %d)", + superblock.d.inline_size, lfs->inline_size); + return LFS_ERR_INVAL; + } + + lfs->inline_size = superblock.d.inline_size; + } + + if (superblock.d.attrs_size) { + if (superblock.d.attrs_size > lfs->attrs_size) { + LFS_ERROR("Unsupported attrs size (%d > %d)", + superblock.d.attrs_size, lfs->attrs_size); + return LFS_ERR_INVAL; + } + + lfs->attrs_size = superblock.d.attrs_size; + } + + if (superblock.d.name_size) { + if (superblock.d.name_size > lfs->name_size) { + LFS_ERROR("Unsupported name size (%d > %d)", + superblock.d.name_size, lfs->name_size); + return LFS_ERR_INVAL; + } + + lfs->name_size = superblock.d.name_size; + } + + lfs->root[0] = superblock.d.root[0]; + lfs->root[1] = superblock.d.root[1]; + return 0; } diff --git a/lfs.h b/lfs.h index 6acf5e75..9a7b8c37 100644 --- a/lfs.h +++ b/lfs.h @@ -50,30 +50,37 @@ typedef int32_t lfs_soff_t; typedef uint32_t lfs_block_t; +// Maximum inline file size in bytes +#ifndef LFS_INLINE_MAX +#define LFS_INLINE_MAX 255 +#endif + +// Maximum size of all attributes per file in bytes +#ifndef LFS_ATTRS_MAX +#define LFS_ATTRS_MAX 255 +#endif + // Max name size in bytes #ifndef LFS_NAME_MAX #define LFS_NAME_MAX 255 #endif -#ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 255 -#endif - // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { - LFS_ERR_OK = 0, // No error - LFS_ERR_IO = -5, // Error during device operation - LFS_ERR_CORRUPT = -52, // Corrupted - LFS_ERR_NOENT = -2, // No directory entry - LFS_ERR_EXIST = -17, // Entry already exists - LFS_ERR_NOTDIR = -20, // Entry is not a dir - LFS_ERR_ISDIR = -21, // Entry is a dir - LFS_ERR_NOTEMPTY = -39, // Dir is not empty - LFS_ERR_BADF = -9, // Bad file number - LFS_ERR_INVAL = -22, // Invalid parameter - LFS_ERR_NOSPC = -28, // No space left on device - LFS_ERR_NOMEM = -12, // No more memory available + LFS_ERR_OK = 0, // No error + LFS_ERR_IO = -5, // Error during device operation + LFS_ERR_CORRUPT = -52, // Corrupted + LFS_ERR_NOENT = -2, // No directory entry + LFS_ERR_EXIST = -17, // Entry already exists + LFS_ERR_NOTDIR = -20, // Entry is not a dir + LFS_ERR_ISDIR = -21, // Entry is a dir + LFS_ERR_NOTEMPTY = -39, // Dir is not empty + LFS_ERR_BADF = -9, // Bad file number + LFS_ERR_INVAL = -22, // Invalid parameter + LFS_ERR_NOSPC = -28, // No space left on device + LFS_ERR_NOMEM = -12, // No more memory available + LFS_ERR_NAMETOOLONG = -36, // File name too long }; // File types @@ -102,10 +109,10 @@ enum lfs_open_flags { LFS_O_APPEND = 0x0800, // Move to end of file on every write // internally used flags - LFS_F_DIRTY = 0x10000, // File does not match storage - LFS_F_WRITING = 0x20000, // File has been written since last flush - LFS_F_READING = 0x40000, // File has been read since last flush - LFS_F_ERRED = 0x80000, // An error occured during write + LFS_F_DIRTY = 0x010000, // File does not match storage + LFS_F_WRITING = 0x020000, // File has been written since last flush + LFS_F_READING = 0x040000, // File has been read since last flush + LFS_F_ERRED = 0x080000, // An error occured during write LFS_F_INLINE = 0x100000, // Currently inlined in directory entry }; @@ -183,6 +190,13 @@ struct lfs_config { // Optional, statically allocated buffer for files. Must be program sized. // If enabled, only one file may be opened at a time. void *file_buffer; + + // Optional, + lfs_size_t inline_size; + // Optional, + lfs_size_t attrs_size; + // Optional, + lfs_size_t name_size; }; @@ -258,9 +272,14 @@ typedef struct lfs_dir { typedef struct lfs_superblock { struct lfs_disk_superblock { lfs_block_t root[2]; - uint32_t block_size; - uint32_t block_count; + + lfs_size_t block_size; + lfs_size_t block_count; uint32_t version; + + lfs_size_t inline_size; + lfs_size_t attrs_size; + lfs_size_t name_size; } d; } lfs_superblock_t; @@ -285,6 +304,10 @@ typedef struct lfs { lfs_free_t free; bool deorphaned; + + lfs_size_t inline_size; + lfs_size_t attrs_size; + lfs_size_t name_size; } lfs_t; From 67742761245597833bdad497083b3319ce5d71d4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 3 Apr 2018 08:28:09 -0500 Subject: [PATCH 020/139] Expanded inline files up to a limit of 1023 bytes One of the big benefits of inline files is that small files no longer need to take up a full block. This opens up an opportunity to provide much better support for storage devices with only a handful of very large blocks. Such as the internal flash found on most microcontrollers. After investigating some use cases for a filesystem on internal flash, it has become apparent that the 255-byte limit is going to be too restrictive to be useful in many cases. Most uses I found needed files ~4-64 bytes in size, but it wasn't uncommon to find files ~512 bytes in length. To try to remedy this, I've pushed the 255 byte limit up to 1023 bytes, by stealing some bits from the previously-unused attributes's size. Unfortunately this limits attributes to 63 bytes in total and has a minor code cost, but I'm not sure even 1023 bytes will be sufficient for a lot of cases. The littlefs will probably never be as efficient with internal flash as other filesystems such as SPIFFS, it just wasn't designed for this sort of limited geometry. However, this feature has been heavily requested, even with limitations, because of the opportunity for code reuse on microcontrollers with both internal and external flash. --- lfs.c | 103 ++++++++++++++++++++++++++++++++++++++-------------------- lfs.h | 7 ++-- 2 files changed, 71 insertions(+), 39 deletions(-) diff --git a/lfs.c b/lfs.c index ad3ad041..d5342c05 100644 --- a/lfs.c +++ b/lfs.c @@ -381,6 +381,26 @@ static void lfs_superblock_tole32(struct lfs_disk_superblock *d) { d->name_size = lfs_tole32(d->name_size); } +/// Other struct functions /// +static inline lfs_size_t lfs_entry_elen(const lfs_entry_t *entry) { + return (lfs_size_t)(entry->d.elen) | + ((lfs_size_t)(entry->d.alen & 0xc0) << 2); +} + +static inline lfs_size_t lfs_entry_alen(const lfs_entry_t *entry) { + return entry->d.alen & 0x3f; +} + +static inline lfs_size_t lfs_entry_nlen(const lfs_entry_t *entry) { + return entry->d.nlen; +} + +static inline lfs_size_t lfs_entry_size(const lfs_entry_t *entry) { + return 4 + lfs_entry_elen(entry) + + lfs_entry_alen(entry) + + lfs_entry_nlen(entry); +} + /// Metadata pair and directory operations /// static inline void lfs_pairswap(lfs_block_t pair[2]) { @@ -573,7 +593,7 @@ static int lfs_commit_region(lfs_t *lfs, return 0; } -static int lfs_dif_commit(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, const struct lfs_region *regions, int count) { // state for copying over const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; @@ -733,7 +753,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, } type |= LFS_STRUCT_MOVED; - err = lfs_dif_commit(lfs, &olddir, (struct lfs_region[]){ + err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ {LFS_FROM_MEM, oldoff, &type, 1}, {LFS_FROM_DROP, oldoff, NULL, -1}}, 2); if (err) { @@ -769,7 +789,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, // writing out new entry entry->off = dir->d.size - 4; entry->size += diff; - int err = lfs_dif_commit(lfs, dir, (struct lfs_region[]){ + int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ {LFS_FROM_REGION, entry->off, &(struct lfs_region_region){ olddir.pair[0], oldoff, regions, count}, entry->size}}, 1); @@ -783,7 +803,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, pdir.d.tail[0] = dir->pair[0]; pdir.d.tail[1] = dir->pair[1]; - err = lfs_dif_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -818,7 +838,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, pdir.d.size &= dir->d.size | 0x7fffffff; pdir.d.tail[0] = dir->d.tail[0]; pdir.d.tail[1] = dir->d.tail[1]; - int err = lfs_dif_commit(lfs, &pdir, NULL, 0); + int err = lfs_dir_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -830,7 +850,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, regions[i].off += entry->off; } - int err = lfs_dif_commit(lfs, dir, regions, count); + int err = lfs_dir_commit(lfs, dir, regions, count); if (err) { return err; } @@ -885,7 +905,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { } entry->off = dir->off; - entry->size = 4 + entry->d.elen + entry->d.alen + entry->d.nlen; + entry->size = lfs_entry_size(entry); dir->off += entry->size; dir->pos += entry->size; return 0; @@ -1041,7 +1061,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { dir.d.tail[0] = cwd.d.tail[0]; dir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dif_commit(lfs, &dir, NULL, 0); + err = lfs_dir_commit(lfs, &dir, NULL, 0); if (err) { return err; } @@ -1166,7 +1186,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = entry.d.elen; + info->size = lfs_entry_elen(&entry); } int err = lfs_dir_get(lfs, dir, @@ -1480,29 +1500,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } - // TODO combine these below? // setup file struct file->pair[0] = cwd.pair[0]; file->pair[1] = cwd.pair[1]; file->poff = entry.off; - file->head = entry.d.u.file.head; - file->size = entry.d.u.file.size; file->flags = flags; file->pos = 0; - if (flags & LFS_O_TRUNC) { - if (file->size != 0) { - file->flags |= LFS_F_DIRTY; - } - - entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = 0; - } + // calculate max inline size based on the size of the entry + file->inline_size = lfs_min(lfs->inline_size, + lfs->cfg->block_size - (sizeof(cwd.d)+4) - + (lfs_entry_size(&entry) - lfs_entry_elen(&entry))); - // load inline files if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) { + // load inline files file->head = 0xfffffffe; - file->size = entry.d.elen; + file->size = lfs_entry_elen(&entry); file->flags |= LFS_F_INLINE; file->cache.block = file->head; file->cache.off = 0; @@ -1513,6 +1526,23 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, lfs_free(file->cache.buffer); return err; } + } else { + // use ctz list from entry + file->head = entry.d.u.file.head; + file->size = entry.d.u.file.size; + } + + // truncate if requested + if (flags & LFS_O_TRUNC) { + if (file->size != 0) { + file->flags |= LFS_F_DIRTY; + } + + file->head = 0xfffffffe; + file->size = 0; + file->flags |= LFS_F_INLINE; + file->cache.block = file->head; + file->cache.off = 0; } // add to list of files @@ -1686,8 +1716,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - lfs_size_t oldlen = entry.d.elen; - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + lfs_size_t oldlen = lfs_entry_elen(&entry); + entry.size = lfs_entry_size(&entry); // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { @@ -1704,7 +1734,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } } else { entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = file->size; + entry.d.elen = file->size & 0xff; + entry.d.alen = (entry.d.alen & 0x3f) | ((file->size >> 2) & 0xc0); err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, &entry.d, 4}, @@ -1822,7 +1853,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // TODO store INLINE_MAX in superblock? // TODO what if inline files is > block size (ie 128) if ((file->flags & LFS_F_INLINE) && - file->pos + nsize >= lfs->inline_size) { + file->pos + nsize >= file->inline_size) { file->block = 0xfffffffe; file->off = file->pos; @@ -2034,7 +2065,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = entry.d.elen; + info->size = lfs_entry_elen(&entry); } if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) { @@ -2103,7 +2134,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.d.tail[0]; cwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dif_commit(lfs, &cwd, NULL, 0); + err = lfs_dir_commit(lfs, &cwd, NULL, 0); if (err) { return err; } @@ -2234,7 +2265,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newcwd.d.tail[0] = dir.d.tail[0]; newcwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dif_commit(lfs, &newcwd, NULL, 0); + err = lfs_dir_commit(lfs, &newcwd, NULL, 0); if (err) { return err; } @@ -2364,7 +2395,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dif_commit(lfs, &root, NULL, 0); + err = lfs_dir_commit(lfs, &root, NULL, 0); if (err) { return err; } @@ -2446,14 +2477,14 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { memset(&superblock.d, 0, sizeof(superblock.d)); err = lfs_dir_get(lfs, &dir, sizeof(dir.d)+4, &superblock.d, - lfs_min(sizeof(superblock.d), entry.d.elen)); + lfs_min(sizeof(superblock.d), lfs_entry_elen(&entry))); lfs_superblock_fromle32(&superblock.d); if (err) { return err; } err = lfs_dir_get(lfs, &dir, - sizeof(dir.d)+4+entry.d.elen+entry.d.alen, magic, + sizeof(dir.d)+lfs_entry_size(&entry)-entry.d.nlen, magic, lfs_min(sizeof(magic), entry.d.nlen)); if (err) { return err; @@ -2546,7 +2577,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { return err; } - dir.off += 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + dir.off += lfs_entry_size(&entry); if ((0x70 & entry.d.type) == LFS_STRUCT_CTZ) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, entry.d.u.file.head, entry.d.u.file.size, cb, data); @@ -2730,7 +2761,7 @@ static int lfs_relocate(lfs_t *lfs, parent.d.tail[0] = newpair[0]; parent.d.tail[1] = newpair[1]; - return lfs_dif_commit(lfs, &parent, NULL, 0); + return lfs_dir_commit(lfs, &parent, NULL, 0); } // couldn't find dir, must be new @@ -2772,7 +2803,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = cwd.d.tail[0]; pdir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dif_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -2788,7 +2819,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = entry.d.u.dir[0]; pdir.d.tail[1] = entry.d.u.dir[1]; - err = lfs_dif_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL, 0); if (err) { return err; } diff --git a/lfs.h b/lfs.h index 9a7b8c37..ca40e0f8 100644 --- a/lfs.h +++ b/lfs.h @@ -52,17 +52,17 @@ typedef uint32_t lfs_block_t; // Maximum inline file size in bytes #ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 255 +#define LFS_INLINE_MAX 0x3ff #endif // Maximum size of all attributes per file in bytes #ifndef LFS_ATTRS_MAX -#define LFS_ATTRS_MAX 255 +#define LFS_ATTRS_MAX 0x3f #endif // Max name size in bytes #ifndef LFS_NAME_MAX -#define LFS_NAME_MAX 255 +#define LFS_NAME_MAX 0xff #endif // Possible error codes, these are negative to allow @@ -248,6 +248,7 @@ typedef struct lfs_file { lfs_size_t size; uint32_t flags; + lfs_size_t inline_size; lfs_off_t pos; lfs_block_t block; lfs_off_t off; From 65ea6b3d0f6bda68a3139fae1b1c2eb0b7882cdb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 3 Apr 2018 08:29:28 -0500 Subject: [PATCH 021/139] Bumped versions, cleaned up some TODOs and missing comments --- lfs.c | 14 +------------- lfs.h | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/lfs.c b/lfs.c index d5342c05..d8693767 100644 --- a/lfs.c +++ b/lfs.c @@ -722,7 +722,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } -// TODO zeros? static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, lfs_off_t off, void *buffer, lfs_size_t size) { return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); @@ -1180,8 +1179,6 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { break; } - // TODO common info constructor? - // TODO also used in lfs_stat info->type = 0xf & entry.d.type; if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; @@ -1707,7 +1704,6 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - // TODO entry read function? lfs_entry_t entry = {.off = file->poff}; err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); @@ -1778,7 +1774,6 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, nsize = size; while (nsize > 0) { - // TODO can this be collapsed? // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { @@ -1848,12 +1843,9 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } } - // TODO combine with block allocation? - // TODO need to move out if no longer fits in block also - // TODO store INLINE_MAX in superblock? - // TODO what if inline files is > block size (ie 128) if ((file->flags & LFS_F_INLINE) && file->pos + nsize >= file->inline_size) { + // inline file doesn't fit anymore file->block = 0xfffffffe; file->off = file->pos; @@ -1869,9 +1861,6 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } while (nsize > 0) { - // TODO can this be collapsed? - // TODO can we reduce this now that block 0 is never allocated? - // TODO actually, how does this behave if inline max == 0? // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { @@ -1969,7 +1958,6 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, return file->pos; } -// TODO handle inlining? int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { if ((file->flags & 3) == LFS_O_RDONLY) { return LFS_ERR_BADF; diff --git a/lfs.h b/lfs.h index ca40e0f8..55148231 100644 --- a/lfs.h +++ b/lfs.h @@ -27,14 +27,14 @@ // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_VERSION 0x00010003 +#define LFS_VERSION 0x00010004 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16)) #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0)) // Version of On-disk data structures // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_DISK_VERSION 0x00010001 +#define LFS_DISK_VERSION 0x00010002 #define LFS_DISK_VERSION_MAJOR (0xffff & (LFS_DISK_VERSION >> 16)) #define LFS_DISK_VERSION_MINOR (0xffff & (LFS_DISK_VERSION >> 0)) @@ -50,17 +50,25 @@ typedef int32_t lfs_soff_t; typedef uint32_t lfs_block_t; -// Maximum inline file size in bytes +// Maximum inline file size in bytes. Large inline files require a larger +// read and prog cache, but if a file can be inline it does not need its own +// data block. LFS_ATTRS_MAX + LFS_INLINE_MAX must be <= 0xffff. Stored in +// superblock and must be respected by other littlefs drivers. #ifndef LFS_INLINE_MAX #define LFS_INLINE_MAX 0x3ff #endif -// Maximum size of all attributes per file in bytes +// Maximum size of all attributes per file in bytes, may be redefined but a +// a smaller LFS_ATTRS_MAX has no benefit. LFS_ATTRS_MAX + LFS_INLINE_MAX +// must be <= 0xffff. Stored in superblock and must be respected by other +// littlefs drivers. #ifndef LFS_ATTRS_MAX #define LFS_ATTRS_MAX 0x3f #endif -// Max name size in bytes +// Max name size in bytes, may be redefined to reduce the size of the +// info struct. Stored in superblock and must be respected by other +// littlefs drivers. #ifndef LFS_NAME_MAX #define LFS_NAME_MAX 0xff #endif @@ -191,11 +199,23 @@ struct lfs_config { // If enabled, only one file may be opened at a time. void *file_buffer; - // Optional, + // Optional upper limit on inlined files in bytes. Large inline files + // require a larger read and prog cache, but if a file can be inlined it + // does not need its own data block. Must be smaller than the read size + // and prog size. Defaults to min(LFS_INLINE_MAX, read_size) when zero. + // Stored in superblock and must be respected by other littlefs drivers. lfs_size_t inline_size; - // Optional, + + // Optional upper limit on attributes per file in bytes. No downside for + // larger attributes size but must be less than LFS_ATTRS_MAX. Defaults to + // LFS_ATTRS_MAX when zero.Stored in superblock and must be respected by + // other littlefs drivers. lfs_size_t attrs_size; - // Optional, + + // Optional upper limit on length of file names in bytes. No downside for + // larger names except the size of the info struct which is controlled by + // the LFS_NAME_MAX define. Defaults to LFS_NAME_MAX when zero. Stored in + // superblock and must be respected by other littlefs drivers. lfs_size_t name_size; }; From 6ffc8d3480b01ee36001826dbe0489ac7badb761 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 5 Apr 2018 19:03:58 -0500 Subject: [PATCH 022/139] Added simple custom attributes A much requested feature (mostly because of littlefs's notable lack of timestamps), this commits adds support for user-specified custom attributes. Planned (though underestimated) since v1, custom attributes provide a route for OSs and applications to provide their own metadata in littlefs, without limiting portability. However, unlike custom attributes that can be found on much more powerful PC filesystems, these custom attributes are very limited, intended for only a handful of bytes for very important metadata. Each attribute has only a single byte to identify the attribute, and the size of all attributes attached to a file is limited to 64 bytes. Custom attributes can be accessed through the lfs_getattr, lfs_setattr, and lfs_removeattr functions. --- lfs.c | 224 +++++++++++++++++++++++++++++++++++++++++++++++++--------- lfs.h | 32 +++++++++ 2 files changed, 224 insertions(+), 32 deletions(-) diff --git a/lfs.c b/lfs.c index d8693767..fa63e8ee 100644 --- a/lfs.c +++ b/lfs.c @@ -913,7 +913,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const char **path) { const char *pathname = *path; - size_t pathlen; + lfs_size_t pathlen; while (true) { nextname: @@ -940,7 +940,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, // skip if matched by '..' in name const char *suffix = pathname + pathlen; - size_t sufflen; + lfs_size_t sufflen; int depth = 1; while (true) { suffix += strspn(suffix, "/"); @@ -1019,6 +1019,142 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } } +/// Internal attribute operations /// +static int lfs_dir_getinfo(lfs_t *lfs, + lfs_dir_t *dir, const lfs_entry_t *entry, struct lfs_info *info) { + memset(info, 0, sizeof(*info)); + info->type = 0xf & entry->d.type; + if (entry->d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { + info->size = entry->d.u.file.size; + } else if (entry->d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { + info->size = lfs_entry_elen(entry); + } + + if (lfs_paircmp(entry->d.u.dir, lfs->root) == 0) { + strcpy(info->name, "/"); + } else { + int err = lfs_dir_get(lfs, dir, + entry->off + entry->size - entry->d.nlen, + info->name, entry->d.nlen); + if (err) { + return err; + } + } + + return 0; +} + +static int lfs_dir_getattr(lfs_t *lfs, + lfs_dir_t *dir, const lfs_entry_t *entry, + uint8_t type, void *buffer, lfs_size_t size) { + // search for attribute in attribute region + lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); + lfs_off_t i = 0; + while (i < lfs_entry_alen(entry)) { + lfs_attr_t attr; + int err = lfs_dir_get(lfs, dir, + entry->off+off+i, &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + if (attr.d.type != type) { + i += attr.d.len; + continue; + } + + if (attr.d.len > size) { + return LFS_ERR_RANGE; + } + + err = lfs_dir_get(lfs, dir, + entry->off+off+i+sizeof(attr.d), buffer, attr.d.len); + if (err) { + return err; + } + + return attr.d.len; + } + + return LFS_ERR_NODATA; +} + +static int lfs_dir_setattr(lfs_t *lfs, + lfs_dir_t *dir, lfs_entry_t *entry, + uint8_t type, const void *buffer, lfs_size_t size) { + // search for attribute in attribute region + lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); + lfs_off_t i = 0; + lfs_size_t oldlen = 0; + while (i < lfs_entry_alen(entry)) { + lfs_attr_t attr; + int err = lfs_dir_get(lfs, dir, + entry->off+off+i, &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + if (attr.d.type != type) { + i += attr.d.len; + continue; + } + + oldlen = attr.d.len; + break; + } + + // make sure the attribute fits + if (lfs_entry_elen(entry) - oldlen + size > lfs->attrs_size || + (0x7fffffff & dir->d.size) - oldlen + size > lfs->cfg->block_size) { + return LFS_ERR_NOSPC; + } + + lfs_attr_t attr; + attr.d.type = type; + attr.d.len = size; + int err = lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ + {LFS_FROM_MEM, off+i, &attr.d, sizeof(attr.d)}, + {LFS_FROM_MEM, off+i, buffer, size}, + {LFS_FROM_DROP, off+i, NULL, -oldlen}}, 3); + if (err) { + return err; + } + + return 0; +} + +static int lfs_dir_removeattr(lfs_t *lfs, + lfs_dir_t *dir, lfs_entry_t *entry, uint8_t type) { + // search for attribute in attribute region + lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); + lfs_off_t i = 0; + while (i < lfs_entry_alen(entry)) { + lfs_attr_t attr; + int err = lfs_dir_get(lfs, dir, + entry->off+off+i, &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + if (attr.d.type != type) { + i += attr.d.len; + continue; + } + + err = lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ + {LFS_FROM_DROP, off+i, + NULL, -(sizeof(attr.d)+attr.d.len)}}, 1); + if (err) { + return err; + } + + return 0; + } + + return LFS_ERR_NODATA; +} + + /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { @@ -1179,16 +1315,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { break; } - info->type = 0xf & entry.d.type; - if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { - info->size = entry.d.u.file.size; - } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = lfs_entry_elen(&entry); - } - - int err = lfs_dir_get(lfs, dir, - entry.off + entry.size - entry.d.nlen, - info->name, entry.d.nlen); + int err = lfs_dir_getinfo(lfs, dir, &entry, info); if (err) { return err; } @@ -2048,26 +2175,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { return err; } - memset(info, 0, sizeof(*info)); - info->type = 0xf & entry.d.type; - if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { - info->size = entry.d.u.file.size; - } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = lfs_entry_elen(&entry); - } - - if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) { - strcpy(info->name, "/"); - } else { - err = lfs_dir_get(lfs, &cwd, - entry.off + entry.size - entry.d.nlen, - info->name, entry.d.nlen); - if (err) { - return err; - } - } - - return 0; + return lfs_dir_getinfo(lfs, &cwd, &entry, info); } int lfs_remove(lfs_t *lfs, const char *path) { @@ -2263,6 +2371,58 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } +/// Attribute operations /// +int lfs_getattr(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err) { + return err; + } + + return lfs_dir_getattr(lfs, &cwd, &entry, type, buffer, size); +} + +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err) { + return err; + } + + return lfs_dir_setattr(lfs, &cwd, &entry, type, buffer, size); +} + +int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err) { + return err; + } + + return lfs_dir_removeattr(lfs, &cwd, &entry, type); +} + + /// Filesystem operations /// static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->cfg = cfg; diff --git a/lfs.h b/lfs.h index 55148231..adc1c02d 100644 --- a/lfs.h +++ b/lfs.h @@ -89,6 +89,8 @@ enum lfs_error { LFS_ERR_NOSPC = -28, // No space left on device LFS_ERR_NOMEM = -12, // No more memory available LFS_ERR_NAMETOOLONG = -36, // File name too long + LFS_ERR_NODATA = -61, // No data/attr available + LFS_ERR_RANGE = -34, // Result not representable }; // File types @@ -253,6 +255,13 @@ typedef struct lfs_entry { } d; } lfs_entry_t; +typedef struct lfs_attr { + struct lfs_disk_attr { + uint8_t type; + uint8_t len; + } d; +} lfs_attr_t; + typedef struct lfs_cache { lfs_block_t block; lfs_off_t off; @@ -379,6 +388,29 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath); // Returns a negative error code on failure. int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); +// Get a custom attribute +// +// Attributes are identified by an 8-bit type and are limited to at +// most LFS_ATTRS_SIZE bytes. +// Returns the size of the attribute, or a negative error code on failure. +int lfs_getattr(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size); + +// Set a custom attribute +// +// Attributes are identified by an 8-bit type and are limited to at +// most LFS_ATTRS_SIZE bytes. +// Returns a negative error code on failure. +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size); + +// Remove a custom attribute +// +// Attributes are identified by an 8-bit type and are limited to at +// most LFS_ATTRS_SIZE bytes. +// Returns a negative error code on failure. +int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type); + /// File operations /// From 6c754c802352d7c46b8b10ba4dd8088b2cea2d86 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 5 Apr 2018 23:23:14 -0500 Subject: [PATCH 023/139] Added support for atomically committing custom attributes Although it's simple and probably what most users expect, the previous custom attributes API suffered from one problem: the inability to update attributes atomically. If we consider our timestamp use case, updating a file would require: 1. Update the file 2. Update the timestamp If a power loss occurs during this sequence of updates, we could end up with a file with an incorrect timestamp. Is this a big deal? Probably not, but it could be a surprise only found after a power-loss. And littlefs was developed with the _specifically_ to avoid suprises during power-loss. The littlefs is perfectly capable of bundling multiple attribute updates in a single directory commit. That's kind of what it was designed to do. So all we need is a new committer opcode for list of attributes, and then poking that list of attributes through the API. We could provide the single-attribute functions, but don't, because the fewer functions makes for a smaller codebase, and these are already the more advanced functions so we can expect more from users. This also changes semantics about what happens when we don't find an attribute, since erroring would throw away all of the other attributes we're processing. To atomically commit both custom attributes and file updates, we need a new API, lfs_file_setattr. Unfortunately the semantics are a bit more confusing than lfs_setattr, since the attributes aren't written out immediately. --- lfs.c | 242 ++++++++++++++++++++++++++++++++++------------------------ lfs.h | 94 ++++++++++++++++++----- 2 files changed, 218 insertions(+), 118 deletions(-) diff --git a/lfs.c b/lfs.c index fa63e8ee..b079ef4a 100644 --- a/lfs.c +++ b/lfs.c @@ -515,6 +515,7 @@ struct lfs_region { LFS_FROM_DROP, LFS_FROM_MEM, LFS_FROM_REGION, + LFS_FROM_ATTRS, } type; lfs_off_t off; @@ -522,6 +523,12 @@ struct lfs_region { lfs_ssize_t size; }; +struct lfs_attrs_region { + const struct lfs_attr *attrs; + int count; + lfs_size_t len; +}; + struct lfs_region_region { lfs_block_t block; lfs_off_t off; @@ -568,6 +575,73 @@ static int lfs_commit_region(lfs_t *lfs, newoff += regions[i].size; break; } + case LFS_FROM_ATTRS: { + const struct lfs_attrs_region *attrs = regions[i].buffer; + + // order doesn't matter, so we write new attrs first. this + // is still O(n^2) but only O(n) disk access + for (int j = 0; j < attrs->count; j++) { + if (attrs->attrs[j].size == 0) { + continue; + } + + lfs_entry_attr_t attr = { + .d.type = attrs->attrs[j].type, + .d.len = attrs->attrs[j].size, + }; + + lfs_crc(crc, &attr.d, sizeof(attr.d)); + int err = lfs_bd_prog(lfs, newblock, newoff, + &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + lfs_crc(crc, + attrs->attrs[j].buffer, attrs->attrs[j].size); + err = lfs_bd_prog(lfs, newblock, newoff+sizeof(attr.d), + attrs->attrs[j].buffer, attrs->attrs[j].size); + if (err) { + return err; + } + + newoff += sizeof(attr.d) + attrs->attrs[j].size; + } + + // copy over attributes without updates + lfs_entry_attr_t attr; + for (lfs_off_t k = 0; k < attrs->len; k += 2+attr.d.len) { + int err = lfs_bd_read(lfs, oldblock, oldoff, + &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + bool updating = false; + for (int j = 0; j < attrs->count; j++) { + if (attr.d.type == attrs->attrs[j].type) { + updating = true; + } + } + + if (!updating) { + err = lfs_commit_region(lfs, + oldblock, oldoff, + newblock, newoff, + 0, NULL, 0, + attr.d.len, crc); + if (err) { + return err; + } + + newoff += 2+attr.d.len; + } + + oldoff += 2+attr.d.len; + } + + break; + } } i += 1; @@ -590,6 +664,8 @@ static int lfs_commit_region(lfs_t *lfs, } } + // sanity check our commit math + LFS_ASSERT(newoff == end); return 0; } @@ -1044,118 +1120,102 @@ static int lfs_dir_getinfo(lfs_t *lfs, return 0; } -static int lfs_dir_getattr(lfs_t *lfs, +static int lfs_dir_getattrs(lfs_t *lfs, lfs_dir_t *dir, const lfs_entry_t *entry, - uint8_t type, void *buffer, lfs_size_t size) { + const struct lfs_attr *attrs, int count) { + // set to zero in case we can't find the attributes or size mismatch + for (int j = 0; j < count; j++) { + memset(attrs[j].buffer, 0, attrs[j].size); + } + // search for attribute in attribute region - lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); - lfs_off_t i = 0; - while (i < lfs_entry_alen(entry)) { - lfs_attr_t attr; + lfs_off_t off = 4+lfs_entry_elen(entry); + lfs_entry_attr_t attr; + for (lfs_off_t i = 0; i < lfs_entry_alen(entry); i += 2+attr.d.len) { int err = lfs_dir_get(lfs, dir, entry->off+off+i, &attr.d, sizeof(attr.d)); if (err) { return err; } - if (attr.d.type != type) { - i += attr.d.len; - continue; - } - - if (attr.d.len > size) { - return LFS_ERR_RANGE; - } + for (int j = 0; j < count; j++) { + if (attr.d.type == attrs[j].type) { + if (attr.d.len > attrs[j].size) { + return LFS_ERR_RANGE; + } - err = lfs_dir_get(lfs, dir, - entry->off+off+i+sizeof(attr.d), buffer, attr.d.len); - if (err) { - return err; + err = lfs_dir_get(lfs, dir, + entry->off+off+i+sizeof(attr.d), + attrs[j].buffer, attr.d.len); + if (err) { + return err; + } + } } - - return attr.d.len; } - return LFS_ERR_NODATA; + return 0; } -static int lfs_dir_setattr(lfs_t *lfs, +static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, - uint8_t type, const void *buffer, lfs_size_t size) { - // search for attribute in attribute region - lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); - lfs_off_t i = 0; - lfs_size_t oldlen = 0; - while (i < lfs_entry_alen(entry)) { - lfs_attr_t attr; + const struct lfs_attr *attrs, int count) { + // check that attributes fit + lfs_size_t nsize = 0; + for (int j = 0; j < count; j++) { + nsize += 2+attrs[j].size; + } + + lfs_off_t off = 4+lfs_entry_elen(entry); + lfs_entry_attr_t attr; + for (lfs_off_t i = 0; i < lfs_entry_alen(entry); i += 2+attr.d.len) { int err = lfs_dir_get(lfs, dir, entry->off+off+i, &attr.d, sizeof(attr.d)); if (err) { return err; } - if (attr.d.type != type) { - i += attr.d.len; - continue; + bool updated = false; + for (int j = 0; j < count; j++) { + if (attr.d.type == attrs[j].type) { + updated = true; + } } - oldlen = attr.d.len; - break; + if (!updated) { + nsize += 2+attr.d.len; + } } - // make sure the attribute fits - if (lfs_entry_elen(entry) - oldlen + size > lfs->attrs_size || - (0x7fffffff & dir->d.size) - oldlen + size > lfs->cfg->block_size) { + if (nsize > lfs->attrs_size || ( + (0x7fffffff & dir->d.size) + lfs_entry_size(entry) - + lfs_entry_alen(entry) + nsize > lfs->cfg->block_size)) { return LFS_ERR_NOSPC; } - lfs_attr_t attr; - attr.d.type = type; - attr.d.len = size; - int err = lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ - {LFS_FROM_MEM, off+i, &attr.d, sizeof(attr.d)}, - {LFS_FROM_MEM, off+i, buffer, size}, - {LFS_FROM_DROP, off+i, NULL, -oldlen}}, 3); - if (err) { - return err; - } - - return 0; + return nsize; } -static int lfs_dir_removeattr(lfs_t *lfs, - lfs_dir_t *dir, lfs_entry_t *entry, uint8_t type) { - // search for attribute in attribute region - lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); - lfs_off_t i = 0; - while (i < lfs_entry_alen(entry)) { - lfs_attr_t attr; - int err = lfs_dir_get(lfs, dir, - entry->off+off+i, &attr.d, sizeof(attr.d)); - if (err) { - return err; - } - - if (attr.d.type != type) { - i += attr.d.len; - continue; - } - - err = lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ - {LFS_FROM_DROP, off+i, - NULL, -(sizeof(attr.d)+attr.d.len)}}, 1); - if (err) { - return err; - } - - return 0; +static int lfs_dir_setattrs(lfs_t *lfs, + lfs_dir_t *dir, lfs_entry_t *entry, + const struct lfs_attr *attrs, int count) { + // make sure attributes fit + lfs_ssize_t nsize = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); + if (nsize < 0) { + return nsize; } - return LFS_ERR_NODATA; + // commit to entry, majority of work is in LFS_FROM_ATTRS + lfs_size_t oldlen = lfs_entry_alen(entry); + entry->d.alen = (0xc0 & entry->d.alen) | nsize; + return lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry->d, 4}, + {LFS_FROM_DROP, 0, NULL, -4}, + {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), + &(struct lfs_attrs_region){attrs, count, oldlen}, nsize}}, 3); } - /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron @@ -2372,25 +2432,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { /// Attribute operations /// -int lfs_getattr(lfs_t *lfs, const char *path, - uint8_t type, void *buffer, lfs_size_t size) { - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t entry; - err = lfs_dir_find(lfs, &cwd, &entry, &path); - if (err) { - return err; - } - - return lfs_dir_getattr(lfs, &cwd, &entry, type, buffer, size); -} - -int lfs_setattr(lfs_t *lfs, const char *path, - uint8_t type, const void *buffer, lfs_size_t size) { +int lfs_getattrs(lfs_t *lfs, const char *path, + const struct lfs_attr *attrs, int count) { lfs_dir_t cwd; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); if (err) { @@ -2403,10 +2446,11 @@ int lfs_setattr(lfs_t *lfs, const char *path, return err; } - return lfs_dir_setattr(lfs, &cwd, &entry, type, buffer, size); + return lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); } -int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { +int lfs_setattrs(lfs_t *lfs, const char *path, + const struct lfs_attr *attrs, int count) { lfs_dir_t cwd; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); if (err) { @@ -2419,7 +2463,7 @@ int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { return err; } - return lfs_dir_removeattr(lfs, &cwd, &entry, type); + return lfs_dir_setattrs(lfs, &cwd, &entry, attrs, count); } diff --git a/lfs.h b/lfs.h index adc1c02d..9de7a3a6 100644 --- a/lfs.h +++ b/lfs.h @@ -234,6 +234,18 @@ struct lfs_info { char name[LFS_NAME_MAX+1]; }; +// Custom attribute structure +struct lfs_attr { + // Type of attribute, provided by user and used to identify the attribute + uint8_t type; + + // Pointer to buffer containing the attribute + void *buffer; + + // Size of attribute in bytes, limited to LFS_ATTRS_MAX + lfs_size_t size; +}; + /// littlefs data structures /// typedef struct lfs_entry { @@ -255,12 +267,12 @@ typedef struct lfs_entry { } d; } lfs_entry_t; -typedef struct lfs_attr { - struct lfs_disk_attr { +typedef struct lfs_entry_attr { + struct lfs_disk_entry_attr { uint8_t type; uint8_t len; } d; -} lfs_attr_t; +} lfs_entry_attr_t; typedef struct lfs_cache { lfs_block_t block; @@ -388,28 +400,25 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath); // Returns a negative error code on failure. int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); -// Get a custom attribute +// Get custom attributes // -// Attributes are identified by an 8-bit type and are limited to at -// most LFS_ATTRS_SIZE bytes. -// Returns the size of the attribute, or a negative error code on failure. -int lfs_getattr(lfs_t *lfs, const char *path, - uint8_t type, void *buffer, lfs_size_t size); - -// Set a custom attribute +// Attributes are looked up based on the type id. If the stored attribute is +// smaller than the buffer, it is padded with zeros. It the stored attribute +// is larger than the buffer, LFS_ERR_RANGE is returned. // -// Attributes are identified by an 8-bit type and are limited to at -// most LFS_ATTRS_SIZE bytes. // Returns a negative error code on failure. -int lfs_setattr(lfs_t *lfs, const char *path, - uint8_t type, const void *buffer, lfs_size_t size); +int lfs_getattrs(lfs_t *lfs, const char *path, + const struct lfs_attr *attrs, int count); -// Remove a custom attribute +// Set custom attributes +// +// The array of attributes will be used to update the attributes stored on +// disk based on their type id. Unspecified attributes are left unmodified. +// Specifying an attribute with zero size deletes the attribute. // -// Attributes are identified by an 8-bit type and are limited to at -// most LFS_ATTRS_SIZE bytes. // Returns a negative error code on failure. -int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type); +int lfs_setattrs(lfs_t *lfs, const char *path, + const struct lfs_attr *attrs, int count); /// File operations /// @@ -484,6 +493,30 @@ int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file); // Returns the size of the file, or a negative error code on failure. lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file); +// Get custom attributes attached to a file +// +// Attributes are looked up based on the type id. If the stored attribute is +// smaller than the buffer, it is padded with zeros. It the stored attribute +// is larger than the buffer, LFS_ERR_RANGE is returned. +// +// Returns a negative error code on failure. +int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, + const struct lfs_attr *attrs, int count); + +// Set custom attributes on a file +// +// The array of attributes will be used to update the attributes stored on +// disk based on their type id. Unspecified attributes are left unmodified. +// Specifying an attribute with zero size deletes the attribute. +// +// Note: Attributes are not written out until a call to lfs_file_sync +// or lfs_file_close and must be allocated until the file is closed or +// lfs_file_setattrs is called with a count of zero. +// +// Returns a negative error code on failure. +int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, + const struct lfs_attr *attrs, int count); + /// Directory operations /// @@ -532,6 +565,29 @@ lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir); int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); +/// Filesystem filesystem operations /// + +// Get custom attributes on the filesystem +// +// Attributes are looked up based on the type id. If the stored attribute is +// smaller than the buffer, it is padded with zeros. It the stored attribute +// is larger than the buffer, LFS_ERR_RANGE is returned. +// +// Returns a negative error code on failure. +int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); + +// Set custom attributes on the filesystem +// +// The array of attributes will be used to update the attributes stored on +// disk based on their type id. Unspecified attributes are left unmodified. +// Specifying an attribute with zero size deletes the attribute. +// +// Note: Filesystem level attributes are not available for wear-leveling +// +// Returns a negative error code on failure. +int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); + + /// Miscellaneous littlefs specific operations /// // Traverse through all blocks in use by the filesystem From 636c0ed3d1952cc2830e7ed695ebc492f84768d6 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 8 Apr 2018 04:23:23 -0500 Subject: [PATCH 024/139] Modified commit regions to work better with custom attributes Mostly just removed LFS_FROM_DROP and changed the DSL grammar a bit to allow drops to occur naturally through oldsize -> newsize diff expressed in the region struct. This prevents us from having to add a drop every time we want to update an entry in-place. --- lfs.c | 196 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 96 insertions(+), 100 deletions(-) diff --git a/lfs.c b/lfs.c index b079ef4a..e68d37f1 100644 --- a/lfs.c +++ b/lfs.c @@ -512,21 +512,20 @@ static int lfs_dir_fetch(lfs_t *lfs, struct lfs_region { enum { - LFS_FROM_DROP, LFS_FROM_MEM, LFS_FROM_REGION, LFS_FROM_ATTRS, } type; - lfs_off_t off; + lfs_off_t oldoff; + lfs_size_t oldsize; const void *buffer; - lfs_ssize_t size; + lfs_size_t newsize; }; -struct lfs_attrs_region { +struct lfs_region_attrs { const struct lfs_attr *attrs; int count; - lfs_size_t len; }; struct lfs_region_region { @@ -536,47 +535,44 @@ struct lfs_region_region { int count; }; -static int lfs_commit_region(lfs_t *lfs, +static int lfs_commit_region(lfs_t *lfs, uint32_t *crc, lfs_block_t oldblock, lfs_off_t oldoff, lfs_block_t newblock, lfs_off_t newoff, - lfs_off_t regionoff, - const struct lfs_region *regions, int count, - lfs_size_t size, uint32_t *crc) { + lfs_off_t regionoff, lfs_size_t regionsize, + const struct lfs_region *regions, int count) { int i = 0; - lfs_size_t end = newoff + size; - while (newoff < end) { + lfs_size_t newend = newoff + regionsize; + while (newoff < newend) { // commit from different types of regions - if (i < count && regions[i].off == oldoff - regionoff) { + if (i < count && regions[i].oldoff == oldoff - regionoff) { switch (regions[i].type) { - case LFS_FROM_DROP: { - oldoff -= regions[i].size; - break; - } case LFS_FROM_MEM: { - lfs_crc(crc, regions[i].buffer, regions[i].size); + lfs_crc(crc, regions[i].buffer, regions[i].newsize); int err = lfs_bd_prog(lfs, newblock, newoff, - regions[i].buffer, regions[i].size); + regions[i].buffer, regions[i].newsize); if (err) { return err; } - newoff += regions[i].size; + newoff += regions[i].newsize; + oldoff += regions[i].oldsize; break; } case LFS_FROM_REGION: { const struct lfs_region_region *disk = regions[i].buffer; - int err = lfs_commit_region(lfs, + int err = lfs_commit_region(lfs, crc, disk->block, disk->off, newblock, newoff, - disk->off, disk->regions, disk->count, - regions[i].size, crc); + disk->off, regions[i].newsize, + disk->regions, disk->count); if (err) { return err; } - newoff += regions[i].size; + newoff += regions[i].newsize; + oldoff -= regions[i].oldsize; break; } case LFS_FROM_ATTRS: { - const struct lfs_attrs_region *attrs = regions[i].buffer; + const struct lfs_region_attrs *attrs = regions[i].buffer; // order doesn't matter, so we write new attrs first. this // is still O(n^2) but only O(n) disk access @@ -585,10 +581,9 @@ static int lfs_commit_region(lfs_t *lfs, continue; } - lfs_entry_attr_t attr = { - .d.type = attrs->attrs[j].type, - .d.len = attrs->attrs[j].size, - }; + lfs_entry_attr_t attr; + attr.d.type = attrs->attrs[j].type; + attr.d.len = attrs->attrs[j].size; lfs_crc(crc, &attr.d, sizeof(attr.d)); int err = lfs_bd_prog(lfs, newblock, newoff, @@ -605,12 +600,13 @@ static int lfs_commit_region(lfs_t *lfs, return err; } - newoff += sizeof(attr.d) + attrs->attrs[j].size; + newoff += 2+attrs->attrs[j].size; } // copy over attributes without updates - lfs_entry_attr_t attr; - for (lfs_off_t k = 0; k < attrs->len; k += 2+attr.d.len) { + lfs_off_t oldend = oldoff + regions[i].oldsize; + while (oldoff < oldend) { + lfs_entry_attr_t attr; int err = lfs_bd_read(lfs, oldblock, oldoff, &attr.d, sizeof(attr.d)); if (err) { @@ -625,11 +621,11 @@ static int lfs_commit_region(lfs_t *lfs, } if (!updating) { - err = lfs_commit_region(lfs, + err = lfs_commit_region(lfs, crc, oldblock, oldoff, newblock, newoff, - 0, NULL, 0, - attr.d.len, crc); + 0, 2+attr.d.len, + NULL, 0); if (err) { return err; } @@ -665,7 +661,7 @@ static int lfs_commit_region(lfs_t *lfs, } // sanity check our commit math - LFS_ASSERT(newoff == end); + LFS_ASSERT(newoff == newend); return 0; } @@ -681,7 +677,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, // keep pairs in order such that pair[0] is most recent lfs_pairswap(dir->pair); for (int i = 0; i < count; i++) { - dir->d.size += regions[i].size; + dir->d.size += regions[i].newsize; + dir->d.size -= regions[i].oldsize; } while (true) { @@ -708,12 +705,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, } // commit region - err = lfs_commit_region(lfs, + err = lfs_commit_region(lfs, &crc, dir->pair[1], sizeof(dir->d), dir->pair[0], sizeof(dir->d), - 0, regions, count, - (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, - &crc); + 0, (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, + regions, count); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -807,7 +803,8 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions, int count) { lfs_ssize_t diff = 0; for (int i = 0; i < count; i++) { - diff += regions[i].size; + diff += regions[i].newsize; + diff -= regions[i].oldsize; } lfs_size_t oldsize = entry->size; @@ -829,8 +826,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, type |= LFS_STRUCT_MOVED; err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ - {LFS_FROM_MEM, oldoff, &type, 1}, - {LFS_FROM_DROP, oldoff, NULL, -1}}, 2); + {LFS_FROM_MEM, oldoff, 1, &type, 1}}, 1); if (err) { return err; } @@ -865,7 +861,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, entry->off = dir->d.size - 4; entry->size += diff; int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {LFS_FROM_REGION, entry->off, &(struct lfs_region_region){ + {LFS_FROM_REGION, entry->off, 0, &(struct lfs_region_region){ olddir.pair[0], oldoff, regions, count}, entry->size}}, 1); if (err) { @@ -893,7 +889,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, lfs_entry_t oldentry; oldentry.off = oldoff; err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ - {LFS_FROM_DROP, 0, NULL, -oldsize}}, 1); + {LFS_FROM_MEM, 0, oldsize, NULL, 0}}, 1); if (err) { return err; } @@ -922,7 +918,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, } for (int i = 0; i < count; i++) { - regions[i].off += entry->off; + regions[i].oldoff += entry->off; } int err = lfs_dir_commit(lfs, dir, regions, count); @@ -1129,11 +1125,11 @@ static int lfs_dir_getattrs(lfs_t *lfs, } // search for attribute in attribute region - lfs_off_t off = 4+lfs_entry_elen(entry); - lfs_entry_attr_t attr; - for (lfs_off_t i = 0; i < lfs_entry_alen(entry); i += 2+attr.d.len) { - int err = lfs_dir_get(lfs, dir, - entry->off+off+i, &attr.d, sizeof(attr.d)); + lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); + lfs_off_t end = off + lfs_entry_alen(entry); + while (off < end) { + lfs_entry_attr_t attr; + int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); if (err) { return err; } @@ -1144,14 +1140,15 @@ static int lfs_dir_getattrs(lfs_t *lfs, return LFS_ERR_RANGE; } - err = lfs_dir_get(lfs, dir, - entry->off+off+i+sizeof(attr.d), + err = lfs_dir_get(lfs, dir, off+sizeof(attr.d), attrs[j].buffer, attr.d.len); if (err) { return err; } } } + + off += 2+attr.d.len; } return 0; @@ -1161,16 +1158,19 @@ static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const struct lfs_attr *attrs, int count) { // check that attributes fit + // two separate passes so disk access is O(n) lfs_size_t nsize = 0; for (int j = 0; j < count; j++) { - nsize += 2+attrs[j].size; + if (attrs[j].size > 0) { + nsize += 2+attrs[j].size; + } } - lfs_off_t off = 4+lfs_entry_elen(entry); - lfs_entry_attr_t attr; - for (lfs_off_t i = 0; i < lfs_entry_alen(entry); i += 2+attr.d.len) { - int err = lfs_dir_get(lfs, dir, - entry->off+off+i, &attr.d, sizeof(attr.d)); + lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); + lfs_off_t end = off + lfs_entry_alen(entry); + while (off < end) { + lfs_entry_attr_t attr; + int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); if (err) { return err; } @@ -1185,6 +1185,8 @@ static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, if (!updated) { nsize += 2+attr.d.len; } + + off += 2+attr.d.len; } if (nsize > lfs->attrs_size || ( @@ -1200,19 +1202,18 @@ static int lfs_dir_setattrs(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const struct lfs_attr *attrs, int count) { // make sure attributes fit - lfs_ssize_t nsize = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); - if (nsize < 0) { - return nsize; + lfs_size_t oldlen = lfs_entry_alen(entry); + lfs_ssize_t newlen = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); + if (newlen < 0) { + return newlen; } // commit to entry, majority of work is in LFS_FROM_ATTRS - lfs_size_t oldlen = lfs_entry_alen(entry); - entry->d.alen = (0xc0 & entry->d.alen) | nsize; + entry->d.alen = (0xc0 & entry->d.alen) | newlen; return lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry->d, 4}, - {LFS_FROM_DROP, 0, NULL, -4}, - {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), - &(struct lfs_attrs_region){attrs, count, oldlen}, nsize}}, 3); + {LFS_FROM_MEM, 0, 4, &entry->d, 4}, + {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), oldlen, + &(struct lfs_region_attrs){attrs, count}, newlen}}, 2); } @@ -1272,8 +1273,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_MEM, 0, path, nlen}}, 2); + {LFS_FROM_MEM, 0, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); if (err) { return err; } @@ -1657,8 +1658,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.size = 0; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, path, nlen}}, 2); + {LFS_FROM_MEM, 0, 0, &entry.d, 4}, + {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); if (err) { return err; } @@ -1910,8 +1911,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.u.file.size = file->size; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_DROP, 0, NULL, -oldlen-4}}, 2); + {LFS_FROM_MEM, 0, 4+oldlen, + &entry.d, sizeof(entry.d)}}, 1); if (err) { return err; } @@ -1921,9 +1922,9 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.alen = (entry.d.alen & 0x3f) | ((file->size >> 2) & 0xc0); err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, file->cache.buffer, file->size}, - {LFS_FROM_DROP, 0, NULL, -oldlen-4}}, 3); + {LFS_FROM_MEM, 0, 0, &entry.d, 4}, + {LFS_FROM_MEM, 0, 4+oldlen, + file->cache.buffer, file->size}}, 2); if (err) { return err; } @@ -2274,7 +2275,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { // remove the entry err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_DROP, 0, NULL, -entry.size}}, 1); + {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); if (err) { return err; } @@ -2364,8 +2365,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving oldentry.d.type |= LFS_STRUCT_MOVED; err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &oldentry.d.type, 1}, - {LFS_FROM_DROP, 0, NULL, -1}}, 2); + {LFS_FROM_MEM, 0, 1, &oldentry.d.type, 1}}, 1); oldentry.d.type &= ~LFS_STRUCT_MOVED; if (err) { return err; @@ -2381,19 +2381,16 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newentry.d = oldentry.d; newentry.d.type &= ~LFS_STRUCT_MOVED; newentry.d.nlen = nlen; - if (!prevexists) { - newentry.size = 0; - } + newentry.size = prevexists ? preventry.size : 0; lfs_size_t newsize = oldentry.size - oldentry.d.nlen + newentry.d.nlen; err = lfs_dir_set(lfs, &newcwd, &newentry, (struct lfs_region[]){ - {LFS_FROM_REGION, 0, &(struct lfs_region_region){ - oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &newentry.d, 4}, - {LFS_FROM_DROP, 0, NULL, -4}, - {LFS_FROM_MEM, newsize - nlen, newpath, nlen}}, 3}, - newsize}, - {LFS_FROM_DROP, 0, NULL, -preventry.size}}, prevexists ? 2 : 1); + {LFS_FROM_REGION, 0, prevexists ? preventry.size : 0, + &(struct lfs_region_region){ + oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, 4, &newentry.d, 4}, + {LFS_FROM_MEM, newsize-nlen, 0, newpath, nlen}}, 2}, + newsize}}, 1); if (err) { return err; } @@ -2405,7 +2402,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // remove old entry err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ - {LFS_FROM_DROP, 0, NULL, -oldentry.size}}, 1); + {LFS_FROM_MEM, 0, oldentry.size, NULL, 0}}, 1); if (err) { return err; } @@ -2619,9 +2616,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_entry_tole32(&superentry.d); lfs_superblock_tole32(&superblock.d); err = lfs_dir_set(lfs, &superdir, &superentry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &superentry.d, 4}, - {LFS_FROM_MEM, 0, &superblock.d, sizeof(superblock.d)}, - {LFS_FROM_MEM, 0, "littlefs", superentry.d.nlen}}, 3); + {LFS_FROM_MEM, 0, 0, &superentry.d, 4}, + {LFS_FROM_MEM, 0, 0, &superblock.d, sizeof(superblock.d)}, + {LFS_FROM_MEM, 0, 0, "littlefs", superentry.d.nlen}}, 3); if (err) { return err; } @@ -2925,8 +2922,8 @@ static int lfs_relocate(lfs_t *lfs, entry.d.u.dir[0] = newpair[0]; entry.d.u.dir[1] = newpair[1]; int err = lfs_dir_set(lfs, &parent, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_DROP, 0, NULL, (lfs_ssize_t)-sizeof(entry.d)}}, 2); + {LFS_FROM_MEM, 0, sizeof(entry.d), + &entry.d, sizeof(entry.d)}}, 1); if (err) { return err; } @@ -3043,7 +3040,7 @@ int lfs_deorphan(lfs_t *lfs) { LFS_DEBUG("Found move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_DROP, 0, NULL, -entry.size}}, 1); + {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); if (err) { return err; } @@ -3052,9 +3049,8 @@ int lfs_deorphan(lfs_t *lfs) { entry.d.u.dir[0], entry.d.u.dir[1]); entry.d.type &= ~LFS_STRUCT_MOVED; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_DROP, 0, NULL, - (lfs_ssize_t)-sizeof(entry.d)}}, 2); + {LFS_FROM_MEM, 0, sizeof(entry.d), + &entry.d, sizeof(entry.d)}}, 1); if (err) { return err; } From 93244a3734df6bea67a3cbf7b0d8519859e6b434 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 8 Apr 2018 16:58:12 -0500 Subject: [PATCH 025/139] Added file-level and fs-level custom attribute APIs In the form of lfs_file_setattr, lfs_file_getattr, lfs_fs_setattr, lfs_fs_getattr. This enables atomic updates of custom attributes as described in 6c754c8, and provides a custom attribute API that allows custom attributes to be stored on the filesystem itself. --- lfs.c | 307 ++++++++++++++++++++++++++++++++++++++++------------------ lfs.h | 5 +- 2 files changed, 217 insertions(+), 95 deletions(-) diff --git a/lfs.c b/lfs.c index e68d37f1..efc8744c 100644 --- a/lfs.c +++ b/lfs.c @@ -801,7 +801,7 @@ static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions, int count) { - lfs_ssize_t diff = 0; + lfs_ssize_t diff = 0; for (int i = 0; i < count; i++) { diff += regions[i].newsize; diff -= regions[i].oldsize; @@ -932,11 +932,11 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, // shift over any files/directories that are affected for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == entry->off && entry->size == 0) { + if (f->pairoff == entry->off && entry->size == 0) { f->pair[0] = 0xffffffff; f->pair[1] = 0xffffffff; - } else if (f->poff > entry->off) { - f->poff += diff; + } else if (f->pairoff > entry->off) { + f->pairoff += diff; } } } @@ -1135,8 +1135,8 @@ static int lfs_dir_getattrs(lfs_t *lfs, } for (int j = 0; j < count; j++) { - if (attr.d.type == attrs[j].type) { - if (attr.d.len > attrs[j].size) { + if (attrs[j].type == attr.d.type) { + if (attrs[j].size < attr.d.len) { return LFS_ERR_RANGE; } @@ -1190,8 +1190,8 @@ static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, } if (nsize > lfs->attrs_size || ( - (0x7fffffff & dir->d.size) + lfs_entry_size(entry) - - lfs_entry_alen(entry) + nsize > lfs->cfg->block_size)) { + lfs_entry_size(entry) - lfs_entry_alen(entry) + nsize + > lfs->cfg->block_size)) { return LFS_ERR_NOSPC; } @@ -1688,7 +1688,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // setup file struct file->pair[0] = cwd.pair[0]; file->pair[1] = cwd.pair[1]; - file->poff = entry.off; + file->pairoff = entry.off; file->flags = flags; file->pos = 0; @@ -1738,87 +1738,87 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { - int err = lfs_file_sync(lfs, file); + int err = lfs_file_sync(lfs, file); - // remove from list of files - for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { - if (*p == file) { - *p = file->next; - break; + // remove from list of files + for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { + if (*p == file) { + *p = file->next; + break; + } } - } - - // clean up memory - if (!lfs->cfg->file_buffer) { - lfs_free(file->cache.buffer); - } - - return err; -} -static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { -relocate:; - // just relocate what exists into new block - lfs_block_t nblock; - int err = lfs_alloc(lfs, &nblock); - if (err) { - return err; - } - - err = lfs_bd_erase(lfs, nblock); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + // clean up memory + if (!lfs->cfg->file_buffer) { + lfs_free(file->cache.buffer); } + return err; } - // either read from dirty cache or disk - for (lfs_off_t i = 0; i < file->off; i++) { - uint8_t data; - err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, - file->block, i, &data, 1); + static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { + relocate:; + // just relocate what exists into new block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); if (err) { return err; } - err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, - nblock, i, &data, 1); + err = lfs_bd_erase(lfs, nblock); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } - } - // copy over new state of file - memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); - file->cache.block = lfs->pcache.block; - file->cache.off = lfs->pcache.off; - lfs->pcache.block = 0xffffffff; + // either read from dirty cache or disk + for (lfs_off_t i = 0; i < file->off; i++) { + uint8_t data; + err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, + file->block, i, &data, 1); + if (err) { + return err; + } - file->block = nblock; - return 0; -} + err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, + nblock, i, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } -static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { - if (file->flags & LFS_F_READING) { - file->flags &= ~LFS_F_READING; + // copy over new state of file + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); + file->cache.block = lfs->pcache.block; + file->cache.off = lfs->pcache.off; + lfs->pcache.block = 0xffffffff; + + file->block = nblock; + return 0; } - if (file->flags & LFS_F_WRITING) { - lfs_off_t pos = file->pos; + static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_READING) { + file->flags &= ~LFS_F_READING; + } - if (!(file->flags & LFS_F_INLINE)) { - // copy over anything after current branch - lfs_file_t orig = { - .head = file->head, - .size = file->size, - .flags = LFS_O_RDONLY, - .pos = file->pos, - .cache = lfs->rcache, - }; + if (file->flags & LFS_F_WRITING) { + lfs_off_t pos = file->pos; + + if (!(file->flags & LFS_F_INLINE)) { + // copy over anything after current branch + lfs_file_t orig = { + .head = file->head, + .size = file->size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; lfs->rcache.block = 0xffffffff; while (file->pos < file->size) { @@ -1892,42 +1892,54 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - lfs_entry_t entry = {.off = file->poff}; + lfs_entry_t entry = {.off = file->pairoff}; err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); if (err) { return err; } + entry.size = lfs_entry_size(&entry); LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - lfs_size_t oldlen = lfs_entry_elen(&entry); - entry.size = lfs_entry_size(&entry); + lfs_size_t oldelen = lfs_entry_elen(&entry); + lfs_size_t oldalen = lfs_entry_alen(&entry); + const void *buffer; + lfs_size_t size; // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; - entry.d.elen = sizeof(entry.d)-4; entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 4+oldlen, - &entry.d, sizeof(entry.d)}}, 1); - if (err) { - return err; - } + buffer = (const uint8_t *)&entry.d + 4; + size = sizeof(entry.d) - 4; } else { entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = file->size & 0xff; - entry.d.alen = (entry.d.alen & 0x3f) | ((file->size >> 2) & 0xc0); - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, 4+oldlen, - file->cache.buffer, file->size}}, 2); - if (err) { - return err; - } + buffer = file->cache.buffer; + size = file->size; + } + + // get new alen from disk + lfs_ssize_t newalen = lfs_dir_checkattrs(lfs, &cwd, &entry, + file->attrs, file->attrcount); + if (newalen < 0) { + return newalen; + } + + entry.d.elen = size & 0xff; + entry.d.alen = (newalen & 0x3f) | ((size >> 2) & 0xc0); + + // write out update + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, 4, &entry.d, 4}, + {LFS_FROM_MEM, 4, oldelen, buffer, size}, + {LFS_FROM_ATTRS, 4+oldelen, oldalen, + &(struct lfs_region_attrs){file->attrs, file->attrcount}, + newalen}}, 3); + if (err) { + return err; } file->flags &= ~LFS_F_DIRTY; @@ -2221,6 +2233,81 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { } } +int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, + const struct lfs_attr *attrs, int count) { + // set to null in case we can't find the attrs (missing file?) + for (int j = 0; j < count; j++) { + memset(attrs[j].buffer, 0, attrs[j].size); + } + + // load from disk if we haven't already been deleted + if (!lfs_pairisnull(file->pair)) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, file->pair); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = file->pairoff}; + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + entry.size = lfs_entry_size(&entry); + + err = lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); + if (err) { + return err; + } + } + + // override an attrs we have stored locally + for (int i = 0; i < file->attrcount; i++) { + for (int j = 0; j < count; j++) { + if (attrs[j].type == file->attrs[i].type) { + if (attrs[j].size < file->attrs[i].size) { + return LFS_ERR_RANGE; + } + + memcpy(attrs[j].buffer, + file->attrs[i].buffer, file->attrs[i].size); + } + } + } + + return 0; +} + +int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, + const struct lfs_attr *attrs, int count) { + // just tack to the file, will be written at sync time + file->attrs = attrs; + file->attrcount = count; + + // at least make sure attributes fit + if (!lfs_pairisnull(file->pair)) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, file->pair); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = file->pairoff}; + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + entry.size = lfs_entry_size(&entry); + + lfs_ssize_t res = lfs_dir_checkattrs(lfs, &cwd, &entry, attrs, count); + if (res < 0) { + return res; + } + } + + return 0; +} + /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { @@ -2427,8 +2514,6 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return 0; } - -/// Attribute operations /// int lfs_getattrs(lfs_t *lfs, const char *path, const struct lfs_attr *attrs, int count) { lfs_dir_t cwd; @@ -2646,10 +2731,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // load superblock lfs_dir_t dir; - lfs_entry_t entry; - lfs_superblock_t superblock; - char magic[8]; - err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -2658,11 +2739,13 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_get(lfs, &dir, sizeof(dir.d), &entry.d, sizeof(entry.d)); + lfs_entry_t entry = {.off = sizeof(dir.d)}; + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); if (err) { return err; } + lfs_superblock_t superblock; memset(&superblock.d, 0, sizeof(superblock.d)); err = lfs_dir_get(lfs, &dir, sizeof(dir.d)+4, &superblock.d, @@ -2672,6 +2755,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } + char magic[8]; err = lfs_dir_get(lfs, &dir, sizeof(dir.d)+lfs_entry_size(&entry)-entry.d.nlen, magic, lfs_min(sizeof(magic), entry.d.nlen)); @@ -2733,7 +2817,7 @@ int lfs_unmount(lfs_t *lfs) { } -/// Littlefs specific operations /// +/// Internal filesystem filesystem operations /// int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { if (lfs_pairisnull(lfs->root)) { return 0; @@ -3064,3 +3148,38 @@ int lfs_deorphan(lfs_t *lfs) { return 0; } + +/// External filesystem filesystem operations /// +int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { + lfs_dir_t dir; + int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = sizeof(dir.d)}; + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + entry.size = lfs_entry_size(&entry); + + return lfs_dir_getattrs(lfs, &dir, &entry, attrs, count); +} + +int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { + lfs_dir_t dir; + int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = sizeof(dir.d)}; + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + entry.size = lfs_entry_size(&entry); + + return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); +} diff --git a/lfs.h b/lfs.h index 9de7a3a6..1e81a38b 100644 --- a/lfs.h +++ b/lfs.h @@ -283,7 +283,7 @@ typedef struct lfs_cache { typedef struct lfs_file { struct lfs_file *next; lfs_block_t pair[2]; - lfs_off_t poff; + lfs_off_t pairoff; lfs_block_t head; lfs_size_t size; @@ -294,6 +294,9 @@ typedef struct lfs_file { lfs_block_t block; lfs_off_t off; lfs_cache_t cache; + + const struct lfs_attr *attrs; + int attrcount; } lfs_file_t; typedef struct lfs_dir { From 746b90965cfd5d81b91af6d6a63362adf0e08060 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 8 Apr 2018 22:25:58 -0500 Subject: [PATCH 026/139] Added lfs_fs_size for finding a count of used blocks This has existed for some time in the form of the lfs_traverse function, through which a user could provide a simple callback that would just count the number of blocks lfs_traverse finds. However, this approach is relatively unconventional and has proven to be confusing for most users. --- lfs.c | 16 ++++++++++++++++ lfs.h | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/lfs.c b/lfs.c index efc8744c..12c903f7 100644 --- a/lfs.c +++ b/lfs.c @@ -3183,3 +3183,19 @@ int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); } + +static int lfs_fs_size_count(void *p, lfs_block_t block) { + lfs_size_t *size = p; + *size += 1; + return 0; +} + +lfs_ssize_t lfs_fs_size(lfs_t *lfs) { + lfs_size_t size = 0; + int err = lfs_traverse(lfs, lfs_fs_size_count, &size); + if (err) { + return err; + } + + return size; +} diff --git a/lfs.h b/lfs.h index 1e81a38b..3fd3ba13 100644 --- a/lfs.h +++ b/lfs.h @@ -590,6 +590,14 @@ int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); // Returns a negative error code on failure. int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); +// Finds the current size of the filesystem +// +// Note: Result is best effort. If files share COW structures, the returned +// size may be larger than the filesystem actually is. +// +// Returns the number of allocated blocks, or a negative error code on failure. +lfs_ssize_t lfs_fs_size(lfs_t *lfs); + /// Miscellaneous littlefs specific operations /// From 2a8277bd4dafdec1d92f69e7992a339a45088c42 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 10 Apr 2018 16:15:37 -0500 Subject: [PATCH 027/139] Added test coverage for filesystems with no inline files --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5c0988b5..54a5771e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ script: - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_PROG_SIZE=512" - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048" + - make clean test QUIET=1 CFLAGS+="-DLFS_INLINE_MAX=0" - make clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS" # compile and find the code size with the smallest configuration From ea4ded420ced77503f51917b41cdd157384f4eb4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 10 Apr 2018 16:35:29 -0500 Subject: [PATCH 028/139] Fixed big-endian support again This is what I get for not runing CI on a local development branch. --- lfs.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lfs.c b/lfs.c index 12c903f7..627b9d8c 100644 --- a/lfs.c +++ b/lfs.c @@ -1272,6 +1272,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; + lfs_entry_tole32(&entry.d); err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, 0, &entry.d, sizeof(entry.d)}, {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); @@ -1893,8 +1894,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); - lfs_entry_fromle32(&entry.d); + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); if (err) { return err; } @@ -1912,6 +1912,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; + lfs_entry_tole32(&entry.d); buffer = (const uint8_t *)&entry.d + 4; size = sizeof(entry.d) - 4; } else { @@ -2249,7 +2250,7 @@ int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, } lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); if (err) { return err; } @@ -2293,7 +2294,7 @@ int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, } lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); if (err) { return err; } @@ -2740,7 +2741,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); if (err) { return err; } @@ -3005,6 +3006,7 @@ static int lfs_relocate(lfs_t *lfs, // update disk, this creates a desync entry.d.u.dir[0] = newpair[0]; entry.d.u.dir[1] = newpair[1]; + lfs_entry_tole32(&entry.d); int err = lfs_dir_set(lfs, &parent, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, sizeof(entry.d), &entry.d, sizeof(entry.d)}}, 1); @@ -3133,8 +3135,7 @@ int lfs_deorphan(lfs_t *lfs) { entry.d.u.dir[0], entry.d.u.dir[1]); entry.d.type &= ~LFS_STRUCT_MOVED; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, sizeof(entry.d), - &entry.d, sizeof(entry.d)}}, 1); + {LFS_FROM_MEM, 0, 1, &entry.d, 1}}, 1); if (err) { return err; } @@ -3158,7 +3159,7 @@ int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { } lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); if (err) { return err; } @@ -3175,7 +3176,7 @@ int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { } lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); if (err) { return err; } From 61f454b008e8c8a7208e8b74095da8501d62c621 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 10 Apr 2018 19:55:17 -0500 Subject: [PATCH 029/139] Added tests for resizable entries and custom attributes Also found some bugs. Should now have a good amount of confidence in these features. --- Makefile | 5 +- lfs.c | 132 ++++++++++--------- tests/test.py | 2 +- tests/test_attrs.sh | 292 ++++++++++++++++++++++++++++++++++++++++++ tests/test_entries.sh | 220 +++++++++++++++++++++++++++++++ tests/test_paths.sh | 18 +++ 6 files changed, 605 insertions(+), 64 deletions(-) create mode 100755 tests/test_attrs.sh create mode 100755 tests/test_entries.sh diff --git a/Makefile b/Makefile index c0933139..55c95dbd 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,9 @@ size: $(OBJ) $(SIZE) -t $^ .SUFFIXES: -test: test_format test_dirs test_files test_seek test_truncate \ - test_interspersed test_alloc test_paths test_orphan test_move test_corrupt +test: test_format test_dirs test_files test_seek test_truncate test_entries \ + test_interspersed test_alloc test_paths test_attrs \ + test_orphan test_move test_corrupt test_%: tests/test_%.sh ifdef QUIET @./$< | sed -n '/^[-=]/p' diff --git a/lfs.c b/lfs.c index 627b9d8c..49b4f3bc 100644 --- a/lfs.c +++ b/lfs.c @@ -1739,87 +1739,87 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { - int err = lfs_file_sync(lfs, file); + int err = lfs_file_sync(lfs, file); - // remove from list of files - for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { - if (*p == file) { - *p = file->next; - break; - } + // remove from list of files + for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { + if (*p == file) { + *p = file->next; + break; } + } - // clean up memory - if (!lfs->cfg->file_buffer) { - lfs_free(file->cache.buffer); - } + // clean up memory + if (!lfs->cfg->file_buffer) { + lfs_free(file->cache.buffer); + } + return err; +} + +static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { +relocate:; + // just relocate what exists into new block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); + if (err) { return err; } - static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { - relocate:; - // just relocate what exists into new block - lfs_block_t nblock; - int err = lfs_alloc(lfs, &nblock); + err = lfs_bd_erase(lfs, nblock); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // either read from dirty cache or disk + for (lfs_off_t i = 0; i < file->off; i++) { + uint8_t data; + err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, + file->block, i, &data, 1); if (err) { return err; } - err = lfs_bd_erase(lfs, nblock); + err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, + nblock, i, &data, 1); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } + } - // either read from dirty cache or disk - for (lfs_off_t i = 0; i < file->off; i++) { - uint8_t data; - err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, - file->block, i, &data, 1); - if (err) { - return err; - } - - err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, - nblock, i, &data, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - } + // copy over new state of file + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); + file->cache.block = lfs->pcache.block; + file->cache.off = lfs->pcache.off; + lfs->pcache.block = 0xffffffff; - // copy over new state of file - memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); - file->cache.block = lfs->pcache.block; - file->cache.off = lfs->pcache.off; - lfs->pcache.block = 0xffffffff; + file->block = nblock; + return 0; +} - file->block = nblock; - return 0; +static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_READING) { + file->flags &= ~LFS_F_READING; } - static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { - if (file->flags & LFS_F_READING) { - file->flags &= ~LFS_F_READING; - } - - if (file->flags & LFS_F_WRITING) { - lfs_off_t pos = file->pos; + if (file->flags & LFS_F_WRITING) { + lfs_off_t pos = file->pos; - if (!(file->flags & LFS_F_INLINE)) { - // copy over anything after current branch - lfs_file_t orig = { - .head = file->head, - .size = file->size, - .flags = LFS_O_RDONLY, - .pos = file->pos, - .cache = lfs->rcache, - }; + if (!(file->flags & LFS_F_INLINE)) { + // copy over anything after current branch + lfs_file_t orig = { + .head = file->head, + .size = file->size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; lfs->rcache.block = 0xffffffff; while (file->pos < file->size) { @@ -2270,6 +2270,7 @@ int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_RANGE; } + memset(attrs[j].buffer, 0, attrs[j].size); memcpy(attrs[j].buffer, file->attrs[i].buffer, file->attrs[i].size); } @@ -2281,9 +2282,9 @@ int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, const struct lfs_attr *attrs, int count) { - // just tack to the file, will be written at sync time - file->attrs = attrs; - file->attrcount = count; + if ((file->flags & 3) == LFS_O_RDONLY) { + return LFS_ERR_BADF; + } // at least make sure attributes fit if (!lfs_pairisnull(file->pair)) { @@ -2306,6 +2307,11 @@ int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, } } + // just tack to the file, will be written at sync time + file->attrs = attrs; + file->attrcount = count; + file->flags |= LFS_F_DIRTY; + return 0; } @@ -2432,6 +2438,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return LFS_ERR_NAMETOOLONG; } + if (oldentry.size - oldentry.d.nlen + nlen > lfs->cfg->block_size) { + return LFS_ERR_NOSPC; + } + // must have same type if (prevexists && preventry.d.type != oldentry.d.type) { return LFS_ERR_ISDIR; diff --git a/tests/test.py b/tests/test.py index 24b0d1a3..e93ccec6 100755 --- a/tests/test.py +++ b/tests/test.py @@ -10,7 +10,7 @@ def generate(test): template = file.read() lines = [] - for line in re.split('(?<=[;{}])\n', test.read()): + for line in re.split('(?<=(?:.;| [{}]))\n', test.read()): match = re.match('(?: *\n)*( *)(.*)=>(.*);', line, re.DOTALL | re.MULTILINE) if match: tab, test, expect = match.groups() diff --git a/tests/test_attrs.sh b/tests/test_attrs.sh new file mode 100755 index 00000000..286b909d --- /dev/null +++ b/tests/test_attrs.sh @@ -0,0 +1,292 @@ +#!/bin/bash +set -eu + +echo "=== Attr tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "hello") => 0; + lfs_file_open(&lfs, &file[0], "hello/hello", + LFS_O_WRONLY | LFS_O_CREAT) => 0; + lfs_file_write(&lfs, &file[0], "hello", strlen("hello")) + => strlen("hello"); + lfs_file_close(&lfs, &file[0]); + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Set/get attribute ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', "aaaa", 4}, + {'B', "bbbbbb", 6}, + {'C', "ccccc", 5}}, 3) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "bbbbbb", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'B', "", 0}}, 1) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'B', "dddddd", 6}}, 1) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "dddddd", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'B', "eee", 3}}, 1) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "eee\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, LFS_ATTRS_MAX+1}}, 1) => LFS_ERR_NOSPC; + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'B', "fffffffff", 9}}, 1) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 9}, + {'C', buffer+13, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "fffffffff", 9) => 0; + memcmp(buffer+13, "ccccc", 5) => 0; + + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], buffer, sizeof(buffer)) => strlen("hello"); + memcmp(buffer, "hello", strlen("hello")) => 0; + lfs_file_close(&lfs, &file[0]); + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Set/get fs attribute ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'A', "aaaa", 4}, + {'B', "bbbbbb", 6}, + {'C', "ccccc", 5}}, 3) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "bbbbbb", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'B', "", 0}}, 1) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'B', "dddddd", 6}}, 1) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "dddddd", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'B', "eee", 3}}, 1) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "eee\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, LFS_ATTRS_MAX+1}}, 1) => LFS_ERR_NOSPC; + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'B', "fffffffff", 9}}, 1) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 9}, + {'C', buffer+13, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "fffffffff", 9) => 0; + memcmp(buffer+13, "ccccc", 5) => 0; + + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], buffer, sizeof(buffer)) => strlen("hello"); + memcmp(buffer, "hello", strlen("hello")) => 0; + lfs_file_close(&lfs, &file[0]); + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Set/get file attribute ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_WRONLY) => 0; + + struct lfs_attr attr[3]; + attr[0] = (struct lfs_attr){'A', "aaaa", 4}; + attr[1] = (struct lfs_attr){'B', "bbbbbb", 6}; + attr[2] = (struct lfs_attr){'C', "ccccc", 5}; + lfs_file_setattrs(&lfs, &file[0], attr, 3) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "bbbbbb", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; + + attr[0] = (struct lfs_attr){'B', "", 0}; + lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; + + attr[0] = (struct lfs_attr){'B', "dddddd", 6}; + lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "dddddd", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; + + attr[0] = (struct lfs_attr){'B', "eee", 3}; + lfs_file_setattrs(&lfs, &file[0], attr, 1); + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "eee\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; + + attr[0] = (struct lfs_attr){'A', buffer, LFS_ATTRS_MAX+1}; + lfs_file_setattrs(&lfs, &file[0], attr, 1) => LFS_ERR_NOSPC; + attr[0] = (struct lfs_attr){'B', "fffffffff", 9}; + lfs_file_open(&lfs, &file[1], "hello/hello", LFS_O_RDONLY) => 0; + lfs_file_setattrs(&lfs, &file[1], attr, 1) => LFS_ERR_BADF; + lfs_file_close(&lfs, &file[1]) => 0; + lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 9}, + {'C', buffer+13, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "fffffffff", 9) => 0; + memcmp(buffer+13, "ccccc", 5) => 0; + + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], buffer, sizeof(buffer)) => strlen("hello"); + memcmp(buffer, "hello", strlen("hello")) => 0; + lfs_file_close(&lfs, &file[0]); + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Deferred file attributes ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDWR) => 0; + + struct lfs_attr attr[] = { + {'B', "gggg", 4}, + {'C', "", 0}, + {'D', "hhhh", 4}, + }; + + lfs_file_setattrs(&lfs, &file[0], attr, 3) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'B', buffer, 9}, + {'C', buffer+9, 9}, + {'D', buffer+18, 9}}, 3) => 0; + memcmp(buffer, "gggg\0\0\0\0\0", 9) => 0; + memcmp(buffer+9, "\0\0\0\0\0\0\0\0\0", 9) => 0; + memcmp(buffer+18, "hhhh\0\0\0\0\0", 9) => 0; + + lfs_getattrs(&lfs, "hello/hello", (struct lfs_attr[]){ + {'B', buffer, 9}, + {'C', buffer+9, 9}, + {'D', buffer+18, 9}}, 3) => 0; + memcmp(buffer, "fffffffff", 9) => 0; + memcmp(buffer+9, "ccccc\0\0\0\0", 9) => 0; + memcmp(buffer+18, "\0\0\0\0\0\0\0\0\0", 9) => 0; + + lfs_file_sync(&lfs, &file[0]) => 0; + lfs_getattrs(&lfs, "hello/hello", (struct lfs_attr[]){ + {'B', buffer, 9}, + {'C', buffer+9, 9}, + {'D', buffer+18, 9}}, 3) => 0; + memcmp(buffer, "gggg\0\0\0\0\0", 9) => 0; + memcmp(buffer+9, "\0\0\0\0\0\0\0\0\0", 9) => 0; + memcmp(buffer+18, "hhhh\0\0\0\0\0", 9) => 0; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/tests/test_entries.sh b/tests/test_entries.sh new file mode 100755 index 00000000..447d9bca --- /dev/null +++ b/tests/test_entries.sh @@ -0,0 +1,220 @@ +#!/bin/bash +set -eu + +# Note: These tests are intended for 512 byte inline size at different +# inline sizes they should still pass, but won't be testing anything + +echo "=== Directory tests ===" +function read_file { +cat << TEST + + size = $2; + lfs_file_open(&lfs, &file[0], "$1", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; +TEST +} + +function write_file { +cat << TEST + + size = $2; + lfs_file_open(&lfs, &file[0], "$1", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + memset(wbuffer, 'c', size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; +TEST +} + +echo "--- Entry grow test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 20) + $(write_file "hi1" 20) + $(write_file "hi2" 20) + $(write_file "hi3" 20) + + $(read_file "hi1" 20) + $(write_file "hi1" 200) + + $(read_file "hi0" 20) + $(read_file "hi1" 200) + $(read_file "hi2" 20) + $(read_file "hi3" 20) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry shrink test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 20) + $(write_file "hi1" 200) + $(write_file "hi2" 20) + $(write_file "hi3" 20) + + $(read_file "hi1" 200) + $(write_file "hi1" 20) + + $(read_file "hi0" 20) + $(read_file "hi1" 20) + $(read_file "hi2" 20) + $(read_file "hi3" 20) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry spill test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 200) + $(write_file "hi1" 200) + $(write_file "hi2" 200) + $(write_file "hi3" 200) + + $(read_file "hi0" 200) + $(read_file "hi1" 200) + $(read_file "hi2" 200) + $(read_file "hi3" 200) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry push spill test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 200) + $(write_file "hi1" 20) + $(write_file "hi2" 200) + $(write_file "hi3" 200) + + $(read_file "hi1" 20) + $(write_file "hi1" 200) + + $(read_file "hi0" 200) + $(read_file "hi1" 200) + $(read_file "hi2" 200) + $(read_file "hi3" 200) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry push spill two test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 200) + $(write_file "hi1" 20) + $(write_file "hi2" 200) + $(write_file "hi3" 200) + $(write_file "hi4" 200) + + $(read_file "hi1" 20) + $(write_file "hi1" 200) + + $(read_file "hi0" 200) + $(read_file "hi1" 200) + $(read_file "hi2" 200) + $(read_file "hi3" 200) + $(read_file "hi4" 200) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry drop test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 200) + $(write_file "hi1" 200) + $(write_file "hi2" 200) + $(write_file "hi3" 200) + + lfs_remove(&lfs, "hi1") => 0; + lfs_stat(&lfs, "hi1", &info) => LFS_ERR_NOENT; + $(read_file "hi0" 200) + $(read_file "hi2" 200) + $(read_file "hi3" 200) + + lfs_remove(&lfs, "hi2") => 0; + lfs_stat(&lfs, "hi2", &info) => LFS_ERR_NOENT; + $(read_file "hi0" 200) + $(read_file "hi3" 200) + + lfs_remove(&lfs, "hi3") => 0; + lfs_stat(&lfs, "hi3", &info) => LFS_ERR_NOENT; + $(read_file "hi0" 200) + + lfs_remove(&lfs, "hi0") => 0; + lfs_stat(&lfs, "hi0", &info) => LFS_ERR_NOENT; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Create too big ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + memset(buffer, 'm', 200); + buffer[200] = '\0'; + + size = 400; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + memset(wbuffer, 'c', size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; + + size = 400; + lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Resize too big ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + memset(buffer, 'm', 200); + buffer[200] = '\0'; + + size = 40; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + memset(wbuffer, 'c', size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; + + size = 40; + lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + + size = 400; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + memset(wbuffer, 'c', size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; + + size = 400; + lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/tests/test_paths.sh b/tests/test_paths.sh index f277e451..33843296 100755 --- a/tests/test_paths.sh +++ b/tests/test_paths.sh @@ -123,5 +123,23 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Max path test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + memset(buffer, 'w', LFS_NAME_MAX+1); + buffer[LFS_NAME_MAX+2] = '\0'; + lfs_mkdir(&lfs, (char*)buffer) => LFS_ERR_NAMETOOLONG; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_NAMETOOLONG; + + memcpy(buffer, "coffee/", strlen("coffee/")); + memset(buffer+strlen("coffee/"), 'w', LFS_NAME_MAX+1); + buffer[strlen("coffee/")+LFS_NAME_MAX+2] = '\0'; + lfs_mkdir(&lfs, (char*)buffer) => LFS_ERR_NAMETOOLONG; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_NAMETOOLONG; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Results ---" tests/stats.py From 8070abec34594c2fe9c8dc9bb4f8a9a2a250a79c Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 19 May 2018 18:25:47 -0500 Subject: [PATCH 030/139] Added rudimentary framework for journaling metadata pairs This is a big change stemming from the fact that resizable entries were surprisingly complicated to implement and came in with a sizable code cost. The theory is that the journalling has a comparable cost to resizable entries. Both need to handle overflowing blocks, and managing offsets is comparable to managing attribute IDs. But by jumping all the way to full journaling, we can statically wear-level the metadata written to metadata pairs. The idea of journaling littlefs's metadata has been mentioned several times in discussions and fits well into how littlefs works. You could even view the existing metadata log as a log of size 2. The downside of this approach is that changing the metadata in this way would break compatibility from the existing layout on disk. Something that resizable entries does not do. That being said, adopting journaling at the metadata layer offers a big improvement to littlefs's performance and wear-leveling, with very little cost (maybe even none or negative after resizable entries?). --- lfs.c | 909 ++++++++++++++++++++++++++++++++++++++++++++++++++++- lfs.h | 57 +++- lfs_util.h | 5 + 3 files changed, 966 insertions(+), 5 deletions(-) diff --git a/lfs.c b/lfs.c index 49b4f3bc..d054037f 100644 --- a/lfs.c +++ b/lfs.c @@ -221,9 +221,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, /// General lfs block device operations /// static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { - // if we ever do more than writes to alternating pairs, - // this may need to consider pcache - return lfs_cache_read(lfs, &lfs->rcache, NULL, + return lfs_cache_read(lfs, &lfs->rcache, &lfs->pcache, block, off, buffer, size); } @@ -427,6 +425,911 @@ static inline bool lfs_pairsync( (paira[0] == pairb[1] && paira[1] == pairb[0]); } +enum { + LFS_TAG_VALID = (int)0x80000000, + LFS_TAG_TYPE = (int)0x7fc00000, + LFS_TAG_ID = (int)0x001ff000, + LFS_TAG_SIZE = (int)0x00000fff, +}; + +static inline uint32_t lfs_mktag(uint16_t type, uint16_t id, uint16_t size) { + return (type << 22) | (id << 12) | size; +} + +static inline bool lfs_tag_valid(uint32_t tag) { + return !(tag & 0x80000000); +} + +static inline uint32_t lfs_tag_uid(uint32_t tag) { + return (tag & 0x7fdff000) >> 12; +} + +static inline uint16_t lfs_tag_type(uint32_t tag) { + return (tag & 0x7fc00000) >> 22; +} + +static inline uint16_t lfs_tag_id(uint32_t tag) { + return (tag & 0x001ff000) >> 12; +} + +static uint16_t lfs_tag_size(uint32_t tag) { + return tag & 0x00000fff; +} + +struct lfs_region__ { + uint32_t tag; + union { + void *buffer; + struct { + lfs_block_t block; + lfs_off_t off; + } d; + } u; +}; + +struct lfs_commit { + lfs_block_t block; + lfs_off_t off; + lfs_off_t begin; + lfs_off_t end; + + uint32_t ptag; + uint32_t crc; + + struct { + int16_t id; + uint16_t type; + } compact; +}; + +static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, + int (*cb)(lfs_t *lfs, void *data, struct lfs_region__ region), + void *data) { + // iterate over dir block backwards (for faster lookups) + lfs_block_t block = commit->block; + lfs_off_t off = commit->off; + uint32_t tag = commit->ptag; + + while (off != sizeof(uint32_t)) { + //printf("read %#010x at %x:%x\n", tag, block, off); + int err = cb(lfs, data, (struct lfs_region__){ + .tag=(0x80000000 | tag), + .u.d.block=block, + .u.d.off=off-lfs_tag_size(tag)}); + if (err) { + return err; + } + + LFS_ASSERT(off > sizeof(uint32_t)+lfs_tag_size(tag)); + off -= sizeof(uint32_t)+lfs_tag_size(tag); + + uint32_t ntag; + err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); + if (err) { + return err; + } + + tag ^= lfs_fromle32(ntag); + } + + return 0; +} + +static int lfs_commit_compactcheck(lfs_t *lfs, void *p, + struct lfs_region__ region) { + struct lfs_commit *commit = p; + if (lfs_tag_id(region.tag) != commit->compact.id) { + return 1; + } else if (lfs_tag_type(region.tag) == commit->compact.type) { + return 2; + } + + return 0; +} + +static int lfs_commit_commit(lfs_t *lfs, + struct lfs_commit *commit, struct lfs_region__ region) { + // request for compaction? + if (commit->compact.id >= 0) { + if (lfs_tag_id(region.tag) != commit->compact.id) { + // ignore non-matching ids + return 0; + } + + commit->compact.type = lfs_tag_type(region.tag); + int res = lfs_commit_traverse(lfs, commit, + lfs_commit_compactcheck, commit); + //printf("traverse(%d, %#010x) -> %d\n", commit->compact.id, region.tag, res); + if (res < 0) { + return res; + } + + if (res == 2) { + //printf("ignoring %#010x at %x:%x\n", region.tag, commit->block, commit->off); + // already committed + return 0; + } + } + + // check if we fit + lfs_size_t size = lfs_tag_size(region.tag); + //printf("writing %#010x at %x:%x\n", region.tag, commit->block, commit->off); + if (commit->off + sizeof(uint32_t)+size > commit->end) { + return LFS_ERR_NOSPC; + } + + // write out tag + uint32_t tag = lfs_tole32((region.tag & 0x7fffffff) ^ commit->ptag); + lfs_crc(&commit->crc, &tag, sizeof(tag)); + int err = lfs_bd_prog(lfs, commit->block, commit->off, &tag, sizeof(tag)); + if (err) { + return err; + } + commit->off += sizeof(tag); + + if (!(region.tag & 0x80000000)) { + // from memory + lfs_crc(&commit->crc, region.u.buffer, size); + err = lfs_bd_prog(lfs, commit->block, commit->off, + region.u.buffer, size); + if (err) { + return err; + } + } else { + // from disk + for (lfs_off_t i = 0; i < size; i++) { + uint8_t dat; + int err = lfs_bd_read(lfs, + region.u.d.block, region.u.d.off+i, &dat, 1); + if (err) { + return err; + } + + lfs_crc(&commit->crc, &dat, 1); + err = lfs_bd_prog(lfs, commit->block, commit->off+i, &dat, 1); + if (err) { + return err; + } + } + } + commit->off += size; + commit->ptag = region.tag; + + return 0; +} + +static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { + // align to program units + lfs_off_t noff = lfs_alignup( + commit->off + 2*sizeof(uint32_t), lfs->cfg->prog_size); + + // read erased state from next program unit + uint32_t tag; + int err = lfs_bd_read(lfs, commit->block, noff, &tag, sizeof(tag)); + if (err) { + return err; + } + + // build crc tag + tag = (0x80000000 & ~lfs_fromle32(tag)) | + lfs_mktag(LFS_TYPE_CRC_, 0x1ff, + noff - (commit->off+sizeof(uint32_t))); + + // write out crc + uint32_t footer[2]; + footer[0] = lfs_tole32(tag ^ commit->ptag); + lfs_crc(&commit->crc, &footer[0], sizeof(footer[0])); + footer[1] = lfs_tole32(commit->crc); + err = lfs_bd_prog(lfs, commit->block, commit->off, + footer, sizeof(footer)); + if (err) { + return err; + } + commit->off += sizeof(uint32_t)+lfs_tag_size(tag); + commit->ptag = tag; + + // flush buffers + err = lfs_bd_sync(lfs); + if (err) { + return err; + } + + // successful commit, check checksum to make sure + uint32_t crc = 0xffffffff; + err = lfs_bd_crc(lfs, commit->block, commit->begin, + commit->off-lfs_tag_size(tag) - commit->begin, &crc); + if (err) { + return err; + } + + if (crc != commit->crc) { + return LFS_ERR_CORRUPT; + } + + return 0; +} + +/*static*/ int lfs_dir_alloc_(lfs_t *lfs, lfs_dir_t_ *dir) { + // allocate pair of dir blocks (backwards, so we write to block 1 first) + for (int i = 0; i < 2; i++) { + int err = lfs_alloc(lfs, &dir->pair[(i+1)%2]); + if (err) { + return err; + } + } + + // rather than clobbering one of the blocks we just pretend + // the revision may be valid + int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->rev, 4); + dir->rev = lfs_fromle32(dir->rev); + if (err) { + return err; + } + + // set defaults + dir->off = sizeof(dir->rev); + dir->etag = 0; + dir->count = 0; + dir->erased = false; + dir->tail[0] = 0xffffffff; + dir->tail[1] = 0xffffffff; + + // don't write out yet, let caller take care of that + return 0; +} + +/*static*/ int lfs_dir_fetch_(lfs_t *lfs, + lfs_dir_t_ *dir, const lfs_block_t pair[2], + int (*cb)(lfs_t *lfs, void *data, struct lfs_region__ region), + void *data) { + dir->pair[0] = pair[0]; + dir->pair[1] = pair[1]; + + // find the block with the most recent revision + uint32_t rev[2]; + for (int i = 0; i < 2; i++) { + int err = lfs_bd_read(lfs, dir->pair[i], 0, &rev[i], sizeof(rev[i])); + rev[i] = lfs_fromle32(rev[i]); + if (err) { + return err; + } + } + + if (lfs_scmp(rev[1], rev[0]) > 0) { + lfs_pairswap(dir->pair); + lfs_pairswap(rev); + } + + // load blocks and check crc + for (int i = 0; i < 2; i++) { + lfs_off_t off = sizeof(dir->rev); + uint32_t ptag = 0; + uint32_t crc = 0xffffffff; + + dir->rev = lfs_tole32(rev[0]); + lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); + dir->rev = lfs_fromle32(dir->rev); + + while (true) { + // extract next tag + uint32_t tag; + int err = lfs_bd_read(lfs, dir->pair[0], off, &tag, sizeof(tag)); + if (err) { + return err; + } + + lfs_crc(&crc, &tag, sizeof(tag)); + tag = lfs_fromle32(tag) ^ ptag; + printf("tag %#010x (%x:%x)\n", tag, dir->pair[0], off); + + // next commit not yet programmed + if (lfs_tag_type(ptag) == LFS_TYPE_CRC_ && lfs_tag_valid(tag)) { + dir->erased = true; + return 0; + } + + // check we're in valid range + if (off + sizeof(uint32_t)+lfs_tag_size(tag) > + lfs->cfg->block_size - 2*sizeof(uint32_t)) { + break; + } + + if (lfs_tag_type(tag) == LFS_TYPE_CRC_) { + // check the crc entry + uint32_t dcrc; + int err = lfs_bd_read(lfs, dir->pair[0], + off+sizeof(uint32_t), &dcrc, sizeof(dcrc)); + if (err) { + return err; + } + + if (crc != lfs_fromle32(dcrc)) { + if (off == sizeof(dir->rev)) { + // try other block + break; + } else { + // consider what we have good enough + dir->erased = false; + return 0; + } + } + + dir->off = off + sizeof(uint32_t)+lfs_tag_size(tag); + dir->etag = tag; + crc = 0xffffffff; + } else { + err = lfs_bd_crc(lfs, dir->pair[0], + off+sizeof(uint32_t), lfs_tag_size(tag), &crc); + if (err) { + return err; + } + + if (cb) { + err = cb(lfs, data, (struct lfs_region__){ + .tag=(tag | 0x80000000), + .u.d.block=dir->pair[0], + .u.d.off=off+sizeof(uint32_t)}); + if (err) { + return err; + } + } + } + + ptag = tag; + off += sizeof(uint32_t)+lfs_tag_size(tag); + } + + // failed, try the other crc? + lfs_pairswap(dir->pair); + lfs_pairswap(rev); + } + + LFS_ERROR("Corrupted dir pair at %d %d", dir->pair[0], dir->pair[1]); + return LFS_ERR_CORRUPT; +} + +static int lfs_dir_traverse2_(lfs_t *lfs, lfs_dir_t_ *dir, + int (*cb)(lfs_t *lfs, void *data, struct lfs_region__ region), + void *data) { + return lfs_commit_traverse(lfs, &(struct lfs_commit){ + .block=dir->pair[0], .off=dir->off, .ptag=dir->etag}, + cb, data); +} + +struct lfs_dir_getter { + uint32_t tag; + uint32_t mask; + void *buffer; +}; + +static int lfs_dir_getter(lfs_t *lfs, void *p, struct lfs_region__ region) { + struct lfs_dir_getter *getter = p; + if ((region.tag & getter->mask) == (getter->tag & getter->mask)) { + lfs_size_t size = lfs_tag_size(getter->tag); + if (lfs_tag_size(region.tag) > size) { + return LFS_ERR_RANGE; + } + + int err = lfs_bd_read(lfs, region.u.d.block, region.u.d.off, + getter->buffer, size); + if (err) { + return err; + } + + memset((uint8_t*)getter->buffer + size, 0, + lfs_tag_size(region.tag) - size); + getter->tag |= region.tag & ~0x80000fff; + return true; + } + + return false; +} + +/*static*/ int32_t lfs_dir_get_(lfs_t *lfs, lfs_dir_t_ *dir, + uint32_t tag, uint32_t mask, void *buffer) { + struct lfs_dir_getter getter = {tag, mask, buffer}; + int res = lfs_dir_traverse2_(lfs, dir, lfs_dir_getter, &getter); + if (res < 0) { + return res; + } + + if (!res) { + return LFS_ERR_NOENT; + } + + return getter.tag; +} + +struct lfs_dir_mover { + // traversal things + lfs_dir_t_ *dir; + int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit); + void *data; + + // ids to iterate through + uint16_t begin; + uint16_t end; + uint16_t ack; +}; + +static int lfs_dir_mover_commit(lfs_t *lfs, void *p, + struct lfs_region__ region) { + return lfs_commit_commit(lfs, p, region); +} + +int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { + struct lfs_dir_mover *mover = p; + for (int i = mover->begin; i < mover->end; i++) { + // tell the committer to check for duplicates + uint16_t old = commit->compact.id; + if (commit->compact.id < 0) { + commit->compact.id = i; + } + + // commit pending commits + int err = mover->cb(lfs, mover->data, commit); + if (err) { + commit->compact.id = old; + return err; + } + + // iterate over on-disk regions + err = lfs_dir_traverse2_(lfs, mover->dir, + lfs_dir_mover_commit, commit); + if (err) { + commit->compact.id = old; + return err; + } + + mover->ack = i; + commit->compact.id = old; + } + + return 0; +} + +/*static*/ int lfs_dir_compact2_(lfs_t *lfs, lfs_dir_t_ *dir, + int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), + void *data) { + // save some state in case block is bad + const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; + bool relocated = false; + + // increment revision count + dir->rev += 1; + + while (true) { + // setup mover + struct lfs_dir_mover mover = { + .dir = dir, + .cb = cb, + .data = data, + + .begin = 0, + .end = dir->count, + .ack = 0, + }; + + if (true) { + // erase block to write to + int err = lfs_bd_erase(lfs, dir->pair[1]); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // write out header + uint32_t crc = 0xffffffff; + uint32_t rev = lfs_tole32(dir->rev); + lfs_crc(&crc, &rev, sizeof(rev)); + err = lfs_bd_prog(lfs, dir->pair[1], 0, &rev, sizeof(rev)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // setup compaction + struct lfs_commit commit = { + .block = dir->pair[1], + .off = sizeof(dir->rev), + // leave space for tail pointer + .begin = 0, + .end = lfs_min(lfs->cfg->block_size - 5*sizeof(uint32_t), + lfs_alignup(lfs->cfg->block_size / 2, + lfs->cfg->prog_size)), + .crc = crc, + .ptag = 0, + .compact.id = -1, + }; + + // run compaction over mover + err = lfs_dir_mover(lfs, &mover, &commit); + if (err) { + if (err == LFS_ERR_NOSPC) { + goto split; + } else if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + if (!lfs_pairisnull(dir->tail)) { + // TODO le32 + commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t), + err = lfs_commit_commit(lfs, &commit, (struct lfs_region__){ + .tag=lfs_mktag(LFS_TYPE_TAIL_, 0x1ff, + sizeof(dir->tail)), + .u.buffer=dir->tail}); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + err = lfs_commit_crc(lfs, &commit); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // successful compaction, swap dir pair to indicate most recent + lfs_pairswap(dir->pair); + dir->off = commit.off; + dir->etag = commit.ptag; + dir->erased = true; + } + break; + +split: + // commit no longer fits, need to split dir + dir->count = mover.ack; + mover.begin = mover.ack+1; + + // drop caches and create tail + lfs->pcache.block = 0xffffffff; + + lfs_dir_t_ tail; + int err = lfs_dir_alloc_(lfs, &tail); + if (err) { + return err; + } + + tail.tail[0] = dir->tail[0]; + tail.tail[1] = dir->tail[1]; + + err = lfs_dir_compact2_(lfs, &tail, lfs_dir_mover, &mover); + if (err) { + return err; + } + + dir->tail[0] = tail.pair[0]; + dir->tail[1] = tail.pair[1]; + continue; + +relocate: + //commit was corrupted + LFS_DEBUG("Bad block at %d", dir->pair[1]); + + // drop caches and prepare to relocate block + relocated = true; + lfs->pcache.block = 0xffffffff; + + // can't relocate superblock, filesystem is now frozen + if (lfs_paircmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { + LFS_WARN("Superblock %d has become unwritable", oldpair[1]); + return LFS_ERR_CORRUPT; + } + + // relocate half of pair + err = lfs_alloc(lfs, &dir->pair[1]); + if (err) { + return err; + } + + continue; + } + + if (relocated) { + // update references if we relocated + LFS_DEBUG("Relocating %d %d to %d %d", + oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); + int err = lfs_relocate(lfs, oldpair, dir->pair); + if (err) { + return err; + } + } + + // shift over any directories that are affected + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + d->pair[0] = dir->pair[0]; + d->pair[1] = dir->pair[1]; + } + } + + return 0; +} + +/*static*/ int lfs_dir_commit2_(lfs_t *lfs, lfs_dir_t_ *dir, + int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), + void *data) { + if (!dir->erased) { + // not erased, must compact + return lfs_dir_compact2_(lfs, dir, cb, data); + } + + struct lfs_commit commit = { + .block = dir->pair[0], + .begin = dir->off, + .off = dir->off, + .end = lfs->cfg->block_size - 2*sizeof(uint32_t), + .crc = 0xffffffff, + .ptag = dir->etag, + .compact.id = -1, + }; + + int err = cb(lfs, data, &commit); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + return lfs_dir_compact2_(lfs, dir, cb, data); + } + return err; + } + + err = lfs_commit_crc(lfs, &commit); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + return lfs_dir_compact2_(lfs, dir, cb, data); + } + return err; + } + + // successful commit, lets update dir + dir->off = commit.off; + dir->etag = commit.ptag; + return 0; +} + +struct lfs_dir_commit_regions { + const struct lfs_region__ *regions; + int count; +}; + +int lfs_dir_commit_regions(lfs_t *lfs, void *p, struct lfs_commit *commit) { + struct lfs_dir_commit_regions *region = p; + for (int i = 0; i < region->count; i++) { + int err = lfs_commit_commit(lfs, commit, region->regions[i]); + if (err) { + return err; + } + } + + return 0; +} + +// TODO rm me, just for testing +/*static*/ int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t_ *dir, + const struct lfs_region__ *regions, int count) { + return lfs_dir_compact2_(lfs, dir, lfs_dir_commit_regions, + &(struct lfs_dir_commit_regions){regions, count}); +} + +/*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, + const struct lfs_region__ *regions, int count) { + return lfs_dir_commit2_(lfs, dir, lfs_dir_commit_regions, + &(struct lfs_dir_commit_regions){regions, count}); +} + +/*static*/ int lfs_dir_add(lfs_t *lfs, lfs_dir_t_ *dir) { + uint16_t id = dir->count; + dir->count += 1; + return id; +} + +/*static*/ int lfs_dir_drop(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id) { + dir->count -= 1; + // TODO compact during traverse when compacting? + return lfs_dir_commit_(lfs, dir, (struct lfs_region__[]){{ + lfs_mktag(LFS_TYPE_DROP_, id, 0)}}, 1); +} + +struct lfs_dir_setter { + const struct lfs_region__ *regions; + int count; +}; + +int lfs_dir_setter(lfs_t *lfs, void *p, struct lfs_commit *commit) { + struct lfs_dir_setter *setter = p; + for (int i = 0; i < setter->count; i++) { + int err = lfs_commit_commit(lfs, commit, setter->regions[i]); + if (err) { + return err; + } + } + + return 0; +} + +/*static*/ int lfs_dir_set_(lfs_t *lfs, lfs_dir_t_ *dir, + const struct lfs_region__ *regions, int count) { + return lfs_dir_commit2_(lfs, dir, lfs_dir_setter, + &(struct lfs_dir_setter){regions, count}); +} + +struct lfs_dir_finder { + const char *name; + lfs_size_t len; + + int16_t id; + lfs_entry_t_ *entry; + lfs_block_t tail[2]; +}; + +static int lfs_dir_finder(lfs_t *lfs, void *p, struct lfs_region__ region) { + struct lfs_dir_finder *find = p; + + if (lfs_tag_type(region.tag) == LFS_TYPE_NAME_ && + lfs_tag_size(region.tag) == find->len) { + int res = lfs_bd_cmp(lfs, region.u.d.block, region.u.d.off, + find->name, find->len); + if (res < 0) { + return res; + } + + if (res) { + // found a match + find->id = lfs_tag_id(region.tag); + find->entry->tag = 0xffffffff; + } + } + + if (find->id >= 0 && lfs_tag_id(region.tag) == find->id && + (lfs_tag_type(region.tag) & 0x1f0) >= LFS_TYPE_REG_ && + (lfs_tag_type(region.tag) & 0x1f0) <= LFS_TYPE_DIR_) { + // TODO combine regions and entries? + find->entry->tag = ~0x80000000 & region.tag; + if (lfs_tag_type(region.tag) & 0x00f) { + int err = lfs_bd_read(lfs, region.u.d.block, region.u.d.off, + &find->entry->u, sizeof(find->entry->u)); + if (err) { + return err; + } + } else { + find->entry->u.d.block = region.u.d.block; + find->entry->u.d.off = region.u.d.off; + } + } + + if (lfs_tag_type(region.tag) == LFS_TYPE_TAIL_) { + int err = lfs_bd_read(lfs, region.u.d.block, region.u.d.off, + find->tail, sizeof(find->tail)); + if (err) { + return err; + } + } + + return 0; +} + +/*static*/ int32_t lfs_dir_find_(lfs_t *lfs, lfs_dir_t_ *dir, + lfs_entry_t_ *entry, const char **path) { + struct lfs_dir_finder find = { + .name = *path, + .entry = entry, + }; + + // TODO make superblock + entry->u.pair[0] = 4; + entry->u.pair[1] = 5; + + while (true) { + nextname: + // skip slashes + find.name += strspn(find.name, "/"); + find.len = strcspn(find.name, "/"); + + // special case for root dir + if (find.name[0] == '\0') { + // TODO set up root? + entry->tag = LFS_STRUCT_DIR | LFS_TYPE_DIR; + entry->u.pair[0] = lfs->root[0]; + entry->u.pair[1] = lfs->root[1]; + return lfs_mktag(LFS_TYPE_DIR_, 0x1ff, 0); + } + + // skip '.' and root '..' + if ((find.len == 1 && memcmp(find.name, ".", 1) == 0) || + (find.len == 2 && memcmp(find.name, "..", 2) == 0)) { + find.name += find.len; + goto nextname; + } + + // skip if matched by '..' in name + const char *suffix = find.name + find.len; + lfs_size_t sufflen; + int depth = 1; + while (true) { + suffix += strspn(suffix, "/"); + sufflen = strcspn(suffix, "/"); + if (sufflen == 0) { + break; + } + + if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { + depth -= 1; + if (depth == 0) { + find.name = suffix + sufflen; + goto nextname; + } + } else { + depth += 1; + } + + suffix += sufflen; + } + + // update what we've found + *path = find.name; + + // find path // TODO handle tails + while (true) { + find.id = -1; + find.tail[0] = 0xffffffff; + find.tail[1] = 0xffffffff; + int err = lfs_dir_fetch_(lfs, dir, entry->u.pair, + lfs_dir_finder, &find); + if (err) { + return err; + } + + if (find.id >= 0) { + // found it + break; + } + + if (lfs_pairisnull(find.tail)) { + return LFS_ERR_NOENT; + } + + entry->u.pair[0] = find.tail[0]; + entry->u.pair[1] = find.tail[1]; + } + +// TODO handle moves +// // check that entry has not been moved +// if (entry->d.type & LFS_STRUCT_MOVED) { +// int moved = lfs_moved(lfs, &entry->d.u); +// if (moved < 0 || moved) { +// return (moved < 0) ? moved : LFS_ERR_NOENT; +// } +// +// entry->d.type &= ~LFS_STRUCT_MOVED; +// } + + find.name += find.len; + find.name += strspn(find.name, "/"); + if (find.name[0] == '\0') { + return 0; + } + + // continue on if we hit a directory + // TODO update with what's on master? + if (lfs_tag_type(entry->tag) != LFS_TYPE_DIR_) { + return LFS_ERR_NOTDIR; + } + } +} + +////////////////////////////////////////////////////////// + static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { // allocate pair of dir blocks for (int i = 0; i < 2; i++) { diff --git a/lfs.h b/lfs.h index 3fd3ba13..b0046f0e 100644 --- a/lfs.h +++ b/lfs.h @@ -27,14 +27,14 @@ // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_VERSION 0x00010004 +#define LFS_VERSION 0x00020000 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16)) #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0)) // Version of On-disk data structures // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_DISK_VERSION 0x00010002 +#define LFS_DISK_VERSION 0x00020000 #define LFS_DISK_VERSION_MAJOR (0xffff & (LFS_DISK_VERSION >> 16)) #define LFS_DISK_VERSION_MINOR (0xffff & (LFS_DISK_VERSION >> 0)) @@ -105,6 +105,33 @@ enum lfs_type { LFS_STRUCT_DIR = 0x20, LFS_STRUCT_INLINE = 0x30, LFS_STRUCT_MOVED = 0x80, + + // file type + LFS_TYPE_REG_ = 0x020, + LFS_TYPE_DIR_ = 0x030, + + LFS_TYPE_NAME_ = 0x010, + LFS_TYPE_MOVE_ = 0x060, + LFS_TYPE_DROP_ = 0x070, + + LFS_TYPE_SUPERBLOCK_ = 0x0c0, + LFS_TYPE_TAIL_ = 0x0d0, + LFS_TYPE_CRC_ = 0x0e0, + + // on disk structure + LFS_STRUCT_ATTR_ = 0x100, + LFS_STRUCT_INLINE_ = 0x000, + LFS_STRUCT_CTZ_ = 0x00c, + LFS_STRUCT_DIR_ = 0x008, + +// LFS_TYPE_DIR_ = 0x002, +// LFS_TYPE_SUPERBLOCK_ = 0xff2, + +// LFS_MASK_ID_ = 0xff000000, +// LFS_MASK_TYPE_ = 0x00fff000, +// LFS_MASK_ATTR_ = 0x00ff0000, +// LFS_MASK_STRUCT_ = 0x0000f000, +// LFS_MASK_SIZE_ = 0x00000fff, }; // File open flags @@ -267,6 +294,21 @@ typedef struct lfs_entry { } d; } lfs_entry_t; +typedef struct lfs_entry_ { + uint32_t tag; + union { + lfs_block_t pair[2]; + struct { + lfs_block_t head; + lfs_size_t size; + } ctz; + struct { + lfs_block_t block; + lfs_off_t off; + } d; + } u; +} lfs_entry_t_; + typedef struct lfs_entry_attr { struct lfs_disk_entry_attr { uint8_t type; @@ -314,6 +356,17 @@ typedef struct lfs_dir { } d; } lfs_dir_t; +typedef struct lfs_dir_ { + lfs_block_t pair[2]; + lfs_block_t tail[2]; + + uint32_t rev; + lfs_off_t off; + uint32_t etag; + uint16_t count; + bool erased; +} lfs_dir_t_; + typedef struct lfs_superblock { struct lfs_disk_superblock { lfs_block_t root[2]; diff --git a/lfs_util.h b/lfs_util.h index 3527ce6c..ecd5dd30 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -161,6 +161,11 @@ static inline uint32_t lfs_tole32(uint32_t a) { return lfs_fromle32(a); } +// Align to nearest multiple of a size +static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) { + return (a + alignment-1) - ((a + alignment-1) % alignment); +} + // Calculate CRC-32 with polynomial = 0x04c11db7 void lfs_crc(uint32_t *crc, const void *buffer, size_t size); From 87f3e01a1743194960f92c43f721159d6dcf7128 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 21 May 2018 00:56:20 -0500 Subject: [PATCH 031/139] Progressed integration of journaling metadata pairs - Integrated journaling into lfs_dir_t_ struct and operations, duplicating functions where necessary - Added internal lfs_tag_t and lfs_stag_t - Consolidated lfs_region and lfs_entry structures --- lfs.c | 956 ++++++++++++++++++++++++++++++++++++++++++---------------- lfs.h | 35 ++- 2 files changed, 720 insertions(+), 271 deletions(-) diff --git a/lfs.c b/lfs.c index d054037f..cfc036bb 100644 --- a/lfs.c +++ b/lfs.c @@ -270,9 +270,7 @@ int lfs_deorphan(lfs_t *lfs); /// Block allocator /// -static int lfs_alloc_lookahead(void *p, lfs_block_t block) { - lfs_t *lfs = p; - +static int lfs_alloc_lookahead(lfs_t *lfs, void *p, lfs_block_t block) { lfs_block_t off = ((block - lfs->free.off) + lfs->cfg->block_count) % lfs->cfg->block_count; @@ -320,7 +318,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { // find mask of free blocks from tree memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); - int err = lfs_traverse(lfs, lfs_alloc_lookahead, lfs); + int err = lfs_traverse_(lfs, lfs_alloc_lookahead, NULL); if (err) { return err; } @@ -357,7 +355,7 @@ static void lfs_entry_tole32(struct lfs_disk_entry *d) { d->u.dir[1] = lfs_tole32(d->u.dir[1]); } -static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { +/*static*/ void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { d->root[0] = lfs_fromle32(d->root[0]); d->root[1] = lfs_fromle32(d->root[1]); d->block_size = lfs_fromle32(d->block_size); @@ -368,7 +366,7 @@ static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { d->name_size = lfs_fromle32(d->name_size); } -static void lfs_superblock_tole32(struct lfs_disk_superblock *d) { +/*static*/ void lfs_superblock_tole32(struct lfs_disk_superblock *d) { d->root[0] = lfs_tole32(d->root[0]); d->root[1] = lfs_tole32(d->root[1]); d->block_size = lfs_tole32(d->block_size); @@ -425,55 +423,47 @@ static inline bool lfs_pairsync( (paira[0] == pairb[1] && paira[1] == pairb[0]); } -enum { - LFS_TAG_VALID = (int)0x80000000, - LFS_TAG_TYPE = (int)0x7fc00000, - LFS_TAG_ID = (int)0x001ff000, - LFS_TAG_SIZE = (int)0x00000fff, -}; - -static inline uint32_t lfs_mktag(uint16_t type, uint16_t id, uint16_t size) { +/// Entry tag operations /// +static inline lfs_tag_t lfs_mktag( + uint16_t type, uint16_t id, lfs_size_t size) { return (type << 22) | (id << 12) | size; } -static inline bool lfs_tag_valid(uint32_t tag) { +static inline bool lfs_tag_valid(lfs_tag_t tag) { return !(tag & 0x80000000); } -static inline uint32_t lfs_tag_uid(uint32_t tag) { - return (tag & 0x7fdff000) >> 12; +static inline uint16_t lfs_tag_type(lfs_tag_t tag) { + return (tag & 0x7fc00000) >> 22; } -static inline uint16_t lfs_tag_type(uint32_t tag) { - return (tag & 0x7fc00000) >> 22; +static inline uint8_t lfs_tag_supertype(lfs_tag_t tag) { + return (tag & 0x70000000) >> 22; +} + +static inline uint8_t lfs_tag_subtype(lfs_tag_t tag) { + return (tag & 0x7c000000) >> 22; +} + +static inline uint8_t lfs_tag_struct(lfs_tag_t tag) { + return (tag & 0x03c00000) >> 22; } -static inline uint16_t lfs_tag_id(uint32_t tag) { +static inline uint16_t lfs_tag_id(lfs_tag_t tag) { return (tag & 0x001ff000) >> 12; } -static uint16_t lfs_tag_size(uint32_t tag) { +static inline lfs_size_t lfs_tag_size(lfs_tag_t tag) { return tag & 0x00000fff; } -struct lfs_region__ { - uint32_t tag; - union { - void *buffer; - struct { - lfs_block_t block; - lfs_off_t off; - } d; - } u; -}; - struct lfs_commit { lfs_block_t block; lfs_off_t off; lfs_off_t begin; lfs_off_t end; - uint32_t ptag; + lfs_tag_t ptag; uint32_t crc; struct { @@ -483,27 +473,27 @@ struct lfs_commit { }; static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, - int (*cb)(lfs_t *lfs, void *data, struct lfs_region__ region), + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), void *data) { // iterate over dir block backwards (for faster lookups) lfs_block_t block = commit->block; lfs_off_t off = commit->off; - uint32_t tag = commit->ptag; + lfs_tag_t tag = commit->ptag; while (off != sizeof(uint32_t)) { - //printf("read %#010x at %x:%x\n", tag, block, off); - int err = cb(lfs, data, (struct lfs_region__){ - .tag=(0x80000000 | tag), + printf("tag r %#010x (%x:%x)\n", tag, block, off-lfs_tag_size(tag)); + int err = cb(lfs, data, (lfs_entry_t_){ + (0x80000000 | tag), .u.d.block=block, .u.d.off=off-lfs_tag_size(tag)}); if (err) { return err; } - LFS_ASSERT(off > sizeof(uint32_t)+lfs_tag_size(tag)); - off -= sizeof(uint32_t)+lfs_tag_size(tag); + LFS_ASSERT(off > sizeof(tag)+lfs_tag_size(tag)); + off -= sizeof(tag)+lfs_tag_size(tag); - uint32_t ntag; + lfs_tag_t ntag; err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); if (err) { return err; @@ -515,12 +505,11 @@ static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commit_compactcheck(lfs_t *lfs, void *p, - struct lfs_region__ region) { +static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t_ entry) { struct lfs_commit *commit = p; - if (lfs_tag_id(region.tag) != commit->compact.id) { + if (lfs_tag_id(entry.tag) != commit->compact.id) { return 1; - } else if (lfs_tag_type(region.tag) == commit->compact.type) { + } else if (lfs_tag_type(entry.tag) == commit->compact.type) { return 2; } @@ -528,38 +517,36 @@ static int lfs_commit_compactcheck(lfs_t *lfs, void *p, } static int lfs_commit_commit(lfs_t *lfs, - struct lfs_commit *commit, struct lfs_region__ region) { + struct lfs_commit *commit, lfs_entry_t_ entry) { // request for compaction? if (commit->compact.id >= 0) { - if (lfs_tag_id(region.tag) != commit->compact.id) { + if (lfs_tag_id(entry.tag) != commit->compact.id) { // ignore non-matching ids return 0; } - commit->compact.type = lfs_tag_type(region.tag); + commit->compact.type = lfs_tag_type(entry.tag); int res = lfs_commit_traverse(lfs, commit, lfs_commit_compactcheck, commit); - //printf("traverse(%d, %#010x) -> %d\n", commit->compact.id, region.tag, res); if (res < 0) { return res; } if (res == 2) { - //printf("ignoring %#010x at %x:%x\n", region.tag, commit->block, commit->off); // already committed return 0; } } // check if we fit - lfs_size_t size = lfs_tag_size(region.tag); - //printf("writing %#010x at %x:%x\n", region.tag, commit->block, commit->off); - if (commit->off + sizeof(uint32_t)+size > commit->end) { + lfs_size_t size = lfs_tag_size(entry.tag); + if (commit->off + sizeof(lfs_tag_t)+size > commit->end) { return LFS_ERR_NOSPC; } // write out tag - uint32_t tag = lfs_tole32((region.tag & 0x7fffffff) ^ commit->ptag); + printf("tag w %#010x (%x:%x)\n", entry.tag, commit->block, commit->off+sizeof(lfs_tag_t)); + lfs_tag_t tag = lfs_tole32((entry.tag & 0x7fffffff) ^ commit->ptag); lfs_crc(&commit->crc, &tag, sizeof(tag)); int err = lfs_bd_prog(lfs, commit->block, commit->off, &tag, sizeof(tag)); if (err) { @@ -567,11 +554,11 @@ static int lfs_commit_commit(lfs_t *lfs, } commit->off += sizeof(tag); - if (!(region.tag & 0x80000000)) { + if (!(entry.tag & 0x80000000)) { // from memory - lfs_crc(&commit->crc, region.u.buffer, size); + lfs_crc(&commit->crc, entry.u.buffer, size); err = lfs_bd_prog(lfs, commit->block, commit->off, - region.u.buffer, size); + entry.u.buffer, size); if (err) { return err; } @@ -580,7 +567,7 @@ static int lfs_commit_commit(lfs_t *lfs, for (lfs_off_t i = 0; i < size; i++) { uint8_t dat; int err = lfs_bd_read(lfs, - region.u.d.block, region.u.d.off+i, &dat, 1); + entry.u.d.block, entry.u.d.off+i, &dat, 1); if (err) { return err; } @@ -593,7 +580,7 @@ static int lfs_commit_commit(lfs_t *lfs, } } commit->off += size; - commit->ptag = region.tag; + commit->ptag = entry.tag; return 0; } @@ -604,7 +591,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { commit->off + 2*sizeof(uint32_t), lfs->cfg->prog_size); // read erased state from next program unit - uint32_t tag; + lfs_tag_t tag; int err = lfs_bd_read(lfs, commit->block, noff, &tag, sizeof(tag)); if (err) { return err; @@ -616,6 +603,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { noff - (commit->off+sizeof(uint32_t))); // write out crc + printf("tag w %#010x (%x:%x)\n", tag, commit->block, commit->off+sizeof(tag)); uint32_t footer[2]; footer[0] = lfs_tole32(tag ^ commit->ptag); lfs_crc(&commit->crc, &footer[0], sizeof(footer[0])); @@ -625,7 +613,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { if (err) { return err; } - commit->off += sizeof(uint32_t)+lfs_tag_size(tag); + commit->off += sizeof(tag)+lfs_tag_size(tag); commit->ptag = tag; // flush buffers @@ -670,9 +658,10 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { dir->off = sizeof(dir->rev); dir->etag = 0; dir->count = 0; - dir->erased = false; dir->tail[0] = 0xffffffff; dir->tail[1] = 0xffffffff; + dir->erased = false; + dir->split = false; // don't write out yet, let caller take care of that return 0; @@ -680,7 +669,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { /*static*/ int lfs_dir_fetch_(lfs_t *lfs, lfs_dir_t_ *dir, const lfs_block_t pair[2], - int (*cb)(lfs_t *lfs, void *data, struct lfs_region__ region), + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), void *data) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; @@ -703,8 +692,12 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // load blocks and check crc for (int i = 0; i < 2; i++) { lfs_off_t off = sizeof(dir->rev); - uint32_t ptag = 0; + lfs_tag_t ptag = 0; uint32_t crc = 0xffffffff; + dir->tail[0] = 0xffffffff; + dir->tail[1] = 0xffffffff; + dir->count = 0; + dir->split = false; dir->rev = lfs_tole32(rev[0]); lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); @@ -712,7 +705,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { while (true) { // extract next tag - uint32_t tag; + lfs_tag_t tag; int err = lfs_bd_read(lfs, dir->pair[0], off, &tag, sizeof(tag)); if (err) { return err; @@ -720,25 +713,25 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { lfs_crc(&crc, &tag, sizeof(tag)); tag = lfs_fromle32(tag) ^ ptag; - printf("tag %#010x (%x:%x)\n", tag, dir->pair[0], off); // next commit not yet programmed - if (lfs_tag_type(ptag) == LFS_TYPE_CRC_ && lfs_tag_valid(tag)) { + if (lfs_tag_type(ptag) == LFS_TYPE_CRC_ && !lfs_tag_valid(tag)) { dir->erased = true; return 0; } // check we're in valid range - if (off + sizeof(uint32_t)+lfs_tag_size(tag) > + if (off + sizeof(tag)+lfs_tag_size(tag) > lfs->cfg->block_size - 2*sizeof(uint32_t)) { break; } + printf("tag r %#010x (%x:%x)\n", tag, dir->pair[0], off+sizeof(tag)); if (lfs_tag_type(tag) == LFS_TYPE_CRC_) { // check the crc entry uint32_t dcrc; int err = lfs_bd_read(lfs, dir->pair[0], - off+sizeof(uint32_t), &dcrc, sizeof(dcrc)); + off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { return err; } @@ -754,21 +747,32 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { } } - dir->off = off + sizeof(uint32_t)+lfs_tag_size(tag); + dir->off = off + sizeof(tag)+lfs_tag_size(tag); dir->etag = tag; crc = 0xffffffff; } else { err = lfs_bd_crc(lfs, dir->pair[0], - off+sizeof(uint32_t), lfs_tag_size(tag), &crc); + off+sizeof(tag), lfs_tag_size(tag), &crc); if (err) { return err; } - if (cb) { - err = cb(lfs, data, (struct lfs_region__){ - .tag=(tag | 0x80000000), + // TODO handle deletes and stuff + if (lfs_tag_id(tag) < 0x1ff && lfs_tag_id(tag) >= dir->count) { + dir->count = lfs_tag_id(tag)+1; + } + + if (lfs_tag_type(tag) == LFS_TYPE_TAIL_) { + err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), + dir->tail, sizeof(dir->tail)); + if (err) { + return err; + } + } else if (cb) { + err = cb(lfs, data, (lfs_entry_t_){ + (tag | 0x80000000), .u.d.block=dir->pair[0], - .u.d.off=off+sizeof(uint32_t)}); + .u.d.off=off+sizeof(tag)}); if (err) { return err; } @@ -776,7 +780,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { } ptag = tag; - off += sizeof(uint32_t)+lfs_tag_size(tag); + off += sizeof(tag)+lfs_tag_size(tag); } // failed, try the other crc? @@ -788,8 +792,8 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return LFS_ERR_CORRUPT; } -static int lfs_dir_traverse2_(lfs_t *lfs, lfs_dir_t_ *dir, - int (*cb)(lfs_t *lfs, void *data, struct lfs_region__ region), +static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), void *data) { return lfs_commit_traverse(lfs, &(struct lfs_commit){ .block=dir->pair[0], .off=dir->off, .ptag=dir->etag}, @@ -797,38 +801,37 @@ static int lfs_dir_traverse2_(lfs_t *lfs, lfs_dir_t_ *dir, } struct lfs_dir_getter { - uint32_t tag; uint32_t mask; - void *buffer; + lfs_entry_t_ entry; }; -static int lfs_dir_getter(lfs_t *lfs, void *p, struct lfs_region__ region) { - struct lfs_dir_getter *getter = p; - if ((region.tag & getter->mask) == (getter->tag & getter->mask)) { - lfs_size_t size = lfs_tag_size(getter->tag); - if (lfs_tag_size(region.tag) > size) { - return LFS_ERR_RANGE; - } - - int err = lfs_bd_read(lfs, region.u.d.block, region.u.d.off, - getter->buffer, size); +static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { + struct lfs_dir_getter *get = p; + if ((entry.tag & get->mask) == (get->entry.tag & get->mask)) { + int err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, + get->entry.u.buffer, lfs_min(lfs_tag_size(entry.tag), + lfs_tag_size(get->entry.tag))); if (err) { return err; } - memset((uint8_t*)getter->buffer + size, 0, - lfs_tag_size(region.tag) - size); - getter->tag |= region.tag & ~0x80000fff; - return true; + get->entry.tag |= entry.tag & ~0x80000fff; + if (lfs_tag_size(entry.tag) > lfs_tag_size(get->entry.tag)) { + return LFS_ERR_RANGE; + } else { + return true; + } } return false; } -/*static*/ int32_t lfs_dir_get_(lfs_t *lfs, lfs_dir_t_ *dir, - uint32_t tag, uint32_t mask, void *buffer) { - struct lfs_dir_getter getter = {tag, mask, buffer}; - int res = lfs_dir_traverse2_(lfs, dir, lfs_dir_getter, &getter); +/*static*/ int lfs_dir_get_(lfs_t *lfs, lfs_dir_t_ *dir, + uint32_t mask, lfs_entry_t_ *entry) { + struct lfs_dir_getter get = {mask, *entry}; + memset(get.entry.u.buffer, 0, lfs_tag_size(get.entry.tag)); + int res = lfs_dir_traverse_(lfs, dir, lfs_dir_getter, &get); + entry->tag = get.entry.tag; if (res < 0) { return res; } @@ -837,7 +840,7 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, struct lfs_region__ region) { return LFS_ERR_NOENT; } - return getter.tag; + return 0; } struct lfs_dir_mover { @@ -853,8 +856,8 @@ struct lfs_dir_mover { }; static int lfs_dir_mover_commit(lfs_t *lfs, void *p, - struct lfs_region__ region) { - return lfs_commit_commit(lfs, p, region); + lfs_entry_t_ entry) { + return lfs_commit_commit(lfs, p, entry); } int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { @@ -874,7 +877,7 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { } // iterate over on-disk regions - err = lfs_dir_traverse2_(lfs, mover->dir, + err = lfs_dir_traverse_(lfs, mover->dir, lfs_dir_mover_commit, commit); if (err) { commit->compact.id = old; @@ -888,7 +891,7 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -/*static*/ int lfs_dir_compact2_(lfs_t *lfs, lfs_dir_t_ *dir, +/*static*/ int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t_ *dir, int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), void *data) { // save some state in case block is bad @@ -960,9 +963,8 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { if (!lfs_pairisnull(dir->tail)) { // TODO le32 commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t), - err = lfs_commit_commit(lfs, &commit, (struct lfs_region__){ - .tag=lfs_mktag(LFS_TYPE_TAIL_, 0x1ff, - sizeof(dir->tail)), + err = lfs_commit_commit(lfs, &commit, (lfs_entry_t_){ + lfs_mktag(LFS_TYPE_TAIL_, 0x1ff, sizeof(dir->tail)), .u.buffer=dir->tail}); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1005,13 +1007,14 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { tail.tail[0] = dir->tail[0]; tail.tail[1] = dir->tail[1]; - err = lfs_dir_compact2_(lfs, &tail, lfs_dir_mover, &mover); + err = lfs_dir_compact_(lfs, &tail, lfs_dir_mover, &mover); if (err) { return err; } dir->tail[0] = tail.pair[0]; dir->tail[1] = tail.pair[1]; + dir->split = true; continue; relocate: @@ -1058,12 +1061,12 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -/*static*/ int lfs_dir_commit2_(lfs_t *lfs, lfs_dir_t_ *dir, +/*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), void *data) { if (!dir->erased) { // not erased, must compact - return lfs_dir_compact2_(lfs, dir, cb, data); + return lfs_dir_compact_(lfs, dir, cb, data); } struct lfs_commit commit = { @@ -1079,7 +1082,7 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { int err = cb(lfs, data, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - return lfs_dir_compact2_(lfs, dir, cb, data); + return lfs_dir_compact_(lfs, dir, cb, data); } return err; } @@ -1087,7 +1090,7 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { err = lfs_commit_crc(lfs, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - return lfs_dir_compact2_(lfs, dir, cb, data); + return lfs_dir_compact_(lfs, dir, cb, data); } return err; } @@ -1098,58 +1101,45 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -struct lfs_dir_commit_regions { - const struct lfs_region__ *regions; - int count; -}; - -int lfs_dir_commit_regions(lfs_t *lfs, void *p, struct lfs_commit *commit) { - struct lfs_dir_commit_regions *region = p; - for (int i = 0; i < region->count; i++) { - int err = lfs_commit_commit(lfs, commit, region->regions[i]); - if (err) { - return err; - } - } - - return 0; -} - +//struct lfs_dir_commit_regions { +// const lfs_entry_t_ *regions; +// int count; +//}; +// +//int lfs_dir_commit_regions(lfs_t *lfs, void *p, struct lfs_commit *commit) { +// struct lfs_dir_commit_regions *entry = p; +// for (int i = 0; i < entry->count; i++) { +// int err = lfs_commit_commit(lfs, commit, entry->regions[i]); +// if (err) { +// return err; +// } +// } +// +// return 0; +//} +// // TODO rm me, just for testing -/*static*/ int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t_ *dir, - const struct lfs_region__ *regions, int count) { - return lfs_dir_compact2_(lfs, dir, lfs_dir_commit_regions, - &(struct lfs_dir_commit_regions){regions, count}); -} - -/*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, - const struct lfs_region__ *regions, int count) { - return lfs_dir_commit2_(lfs, dir, lfs_dir_commit_regions, - &(struct lfs_dir_commit_regions){regions, count}); -} - -/*static*/ int lfs_dir_add(lfs_t *lfs, lfs_dir_t_ *dir) { - uint16_t id = dir->count; - dir->count += 1; - return id; -} - -/*static*/ int lfs_dir_drop(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id) { - dir->count -= 1; - // TODO compact during traverse when compacting? - return lfs_dir_commit_(lfs, dir, (struct lfs_region__[]){{ - lfs_mktag(LFS_TYPE_DROP_, id, 0)}}, 1); -} +///*static*/ int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t_ *dir, +// const lfs_entry_t_ *regions, int count) { +// return lfs_dir_compact_(lfs, dir, lfs_dir_commit_regions, +// &(struct lfs_dir_commit_regions){regions, count}); +//} +// +///*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, +// const lfs_entry_t_ *regions, int count) { +// return lfs_dir_commit_(lfs, dir, lfs_dir_commit_regions, +// &(struct lfs_dir_commit_regions){regions, count}); +//} struct lfs_dir_setter { - const struct lfs_region__ *regions; + const lfs_entry_t_ *regions; int count; }; int lfs_dir_setter(lfs_t *lfs, void *p, struct lfs_commit *commit) { - struct lfs_dir_setter *setter = p; - for (int i = 0; i < setter->count; i++) { - int err = lfs_commit_commit(lfs, commit, setter->regions[i]); + struct lfs_dir_setter *set = p; + for (int i = 0; i < set->count; i++) { + int err = lfs_commit_commit(lfs, commit, set->regions[i]); if (err) { return err; } @@ -1159,8 +1149,8 @@ int lfs_dir_setter(lfs_t *lfs, void *p, struct lfs_commit *commit) { } /*static*/ int lfs_dir_set_(lfs_t *lfs, lfs_dir_t_ *dir, - const struct lfs_region__ *regions, int count) { - return lfs_dir_commit2_(lfs, dir, lfs_dir_setter, + const lfs_entry_t_ *regions, int count) { + return lfs_dir_commit_(lfs, dir, lfs_dir_setter, &(struct lfs_dir_setter){regions, count}); } @@ -1170,15 +1160,27 @@ struct lfs_dir_finder { int16_t id; lfs_entry_t_ *entry; - lfs_block_t tail[2]; }; -static int lfs_dir_finder(lfs_t *lfs, void *p, struct lfs_region__ region) { +/*static*/ int lfs_dir_add(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t *id) { + *id = dir->count; + dir->count += 1; + return 0; +} + +/*static*/ int lfs_dir_drop(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id) { + dir->count -= 1; + // TODO compact during traverse when compacting? + return lfs_dir_set_(lfs, dir, (lfs_entry_t_[]){{ + lfs_mktag(LFS_TYPE_DROP_, id, 0)}}, 1); +} + +static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { struct lfs_dir_finder *find = p; - if (lfs_tag_type(region.tag) == LFS_TYPE_NAME_ && - lfs_tag_size(region.tag) == find->len) { - int res = lfs_bd_cmp(lfs, region.u.d.block, region.u.d.off, + if (lfs_tag_type(entry.tag) == LFS_TYPE_NAME_ && + lfs_tag_size(entry.tag) == find->len) { + int res = lfs_bd_cmp(lfs, entry.u.d.block, entry.u.d.off, find->name, find->len); if (res < 0) { return res; @@ -1186,40 +1188,31 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, struct lfs_region__ region) { if (res) { // found a match - find->id = lfs_tag_id(region.tag); + find->id = lfs_tag_id(entry.tag); find->entry->tag = 0xffffffff; } } - if (find->id >= 0 && lfs_tag_id(region.tag) == find->id && - (lfs_tag_type(region.tag) & 0x1f0) >= LFS_TYPE_REG_ && - (lfs_tag_type(region.tag) & 0x1f0) <= LFS_TYPE_DIR_) { + if (find->id >= 0 && lfs_tag_id(entry.tag) == find->id && + (lfs_tag_type(entry.tag) & 0x1c0) == LFS_TYPE_REG_) { // TODO combine regions and entries? - find->entry->tag = ~0x80000000 & region.tag; - if (lfs_tag_type(region.tag) & 0x00f) { - int err = lfs_bd_read(lfs, region.u.d.block, region.u.d.off, + find->entry->tag = ~0x80000000 & entry.tag; + if (lfs_tag_type(entry.tag) & 0x00f) { + int err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, &find->entry->u, sizeof(find->entry->u)); if (err) { return err; } } else { - find->entry->u.d.block = region.u.d.block; - find->entry->u.d.off = region.u.d.off; - } - } - - if (lfs_tag_type(region.tag) == LFS_TYPE_TAIL_) { - int err = lfs_bd_read(lfs, region.u.d.block, region.u.d.off, - find->tail, sizeof(find->tail)); - if (err) { - return err; + find->entry->u.d.block = entry.u.d.block; + find->entry->u.d.off = entry.u.d.off; } } return 0; } -/*static*/ int32_t lfs_dir_find_(lfs_t *lfs, lfs_dir_t_ *dir, +/*static*/ int lfs_dir_find_(lfs_t *lfs, lfs_dir_t_ *dir, lfs_entry_t_ *entry, const char **path) { struct lfs_dir_finder find = { .name = *path, @@ -1227,8 +1220,8 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, struct lfs_region__ region) { }; // TODO make superblock - entry->u.pair[0] = 4; - entry->u.pair[1] = 5; + entry->u.pair[0] = lfs->root[0]; + entry->u.pair[1] = lfs->root[1]; while (true) { nextname: @@ -1279,11 +1272,9 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, struct lfs_region__ region) { // update what we've found *path = find.name; - // find path // TODO handle tails + // find path while (true) { find.id = -1; - find.tail[0] = 0xffffffff; - find.tail[1] = 0xffffffff; int err = lfs_dir_fetch_(lfs, dir, entry->u.pair, lfs_dir_finder, &find); if (err) { @@ -1295,12 +1286,12 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, struct lfs_region__ region) { break; } - if (lfs_pairisnull(find.tail)) { + if (lfs_pairisnull(dir->tail)) { return LFS_ERR_NOENT; } - entry->u.pair[0] = find.tail[0]; - entry->u.pair[1] = find.tail[1]; + entry->u.pair[0] = dir->tail[0]; + entry->u.pair[1] = dir->tail[1]; } // TODO handle moves @@ -1328,6 +1319,7 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, struct lfs_region__ region) { } } + ////////////////////////////////////////////////////////// static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { @@ -1545,7 +1537,7 @@ static int lfs_commit_region(lfs_t *lfs, uint32_t *crc, i += 1; } else { - // copy data from old block if not covered by region + // copy data from old block if not covered by entry uint8_t data; int err = lfs_bd_read(lfs, oldblock, oldoff, &data, 1); if (err) { @@ -1607,7 +1599,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - // commit region + // commit entry err = lfs_commit_region(lfs, &crc, dir->pair[1], sizeof(dir->d), dir->pair[0], sizeof(dir->d), @@ -2027,7 +2019,7 @@ static int lfs_dir_getattrs(lfs_t *lfs, memset(attrs[j].buffer, 0, attrs[j].size); } - // search for attribute in attribute region + // search for attribute in attribute entry lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); lfs_off_t end = off + lfs_entry_alen(entry); while (off < end) { @@ -2121,6 +2113,66 @@ static int lfs_dir_setattrs(lfs_t *lfs, /// Top level directory operations /// +int lfs_mkdir(lfs_t *lfs, const char *path) { + // deorphan if we haven't yet, needed at most once after poweron + if (!lfs->deorphaned) { + int err = lfs_deorphan_(lfs); + if (err) { + return err; + } + } + + // fetch parent directory + lfs_dir_t_ cwd; + lfs_entry_t_ entry; + int err = lfs_dir_find_(lfs, &cwd, &entry, &path); + if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { + if (!err) { + return LFS_ERR_EXIST; + } + return err; + } + + // check that name fits + lfs_size_t nlen = strlen(path); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + + // build up new directory + lfs_alloc_ack(lfs); + + lfs_dir_t_ dir; + err = lfs_dir_alloc_(lfs, &dir); // TODO add tail to this function? + if (err) { + return err; + } + dir.tail[0] = cwd.tail[0]; + dir.tail[1] = cwd.tail[1]; + + err = lfs_dir_set_(lfs, &dir, NULL, 0); + if (err) { + return err; + } + + // get next slot and commit + uint16_t id; + err = lfs_dir_add(lfs, &cwd, &id); + if (err) { + return err; + } + + err = lfs_dir_set_(lfs, &cwd, (lfs_entry_t_[]){ + {lfs_mktag(LFS_TYPE_NAME_, id, nlen), .u.buffer=path}, + {lfs_mktag(LFS_TYPE_DIR_ | LFS_STRUCT_DIR_, id, + sizeof(dir.pair)), .u.buffer=dir.pair}}, 2); + + // TODO need ack here? + lfs_alloc_ack(lfs); + return 0; +} + +#if 0 int lfs_mkdir(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron if (!lfs->deorphaned) { @@ -2186,6 +2238,121 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_alloc_ack(lfs); return 0; } +#endif + +int lfs_dir_open_(lfs_t *lfs, lfs_dir_t_ *dir, const char *path) { + lfs_entry_t_ entry; + int err = lfs_dir_find_(lfs, dir, &entry, &path); + if (err) { + return err; + } + + if ((lfs_tag_type(entry.tag) & 0x1f0) != LFS_TYPE_DIR_) { + return LFS_ERR_NOTDIR; + } + + err = lfs_dir_fetch_(lfs, dir, entry.u.pair, NULL, NULL); + if (err) { + return err; + } + + // setup head dir + dir->head[0] = dir->pair[0]; + dir->head[1] = dir->pair[1]; + dir->pos = 0; + dir->id = 0; + + // add to list of directories + dir->next = lfs->dirs; + lfs->dirs = dir; + + return 0; +} + +int lfs_dir_close_(lfs_t *lfs, lfs_dir_t_ *dir) { + // remove from list of directories + for (lfs_dir_t_ **p = &lfs->dirs; *p; p = &(*p)->next) { + if (*p == dir) { + *p = dir->next; + break; + } + } + + return 0; +} + +// TODO move me? +static int lfs_dir_getinfo_(lfs_t *lfs, lfs_dir_t_ *dir, + uint16_t id, struct lfs_info *info) { + lfs_entry_t_ entry = {lfs_mktag(LFS_TYPE_REG, id, 8), .u.buffer=&entry.u}; + int err = lfs_dir_get_(lfs, dir, 0x701ff000, &entry); + if (err && err != LFS_ERR_RANGE) { + return err; + } + + info->type = lfs_tag_subtype(entry.tag); + if (lfs_tag_type(entry.tag) == + (LFS_TYPE_REG_ | LFS_STRUCT_CTZ_)) { + info->size = entry.u.ctz.size; + } else if (lfs_tag_type(entry.tag) == + (LFS_TYPE_REG_ | LFS_STRUCT_INLINE_)) { + info->size = lfs_tag_size(entry.tag); + } + + err = lfs_dir_get_(lfs, dir, 0x7ffff000, &(lfs_entry_t_){ + lfs_mktag(LFS_TYPE_NAME_, id, lfs->cfg->name_size), + .u.buffer=info->name}); + if (err) { + return err; + } + + return 0; +} + +int lfs_dir_read_(lfs_t *lfs, lfs_dir_t_ *dir, struct lfs_info *info) { + memset(info, 0, sizeof(*info)); + + // special offset for '.' and '..' + if (dir->pos == 0) { + info->type = LFS_TYPE_DIR; + strcpy(info->name, "."); + dir->pos += 1; + return 1; + } else if (dir->pos == 1) { + info->type = LFS_TYPE_DIR; + strcpy(info->name, ".."); + dir->pos += 1; + return 1; + } + + while (true) { + if (dir->id == dir->count) { + if (!dir->split) { + return false; + } + + int err = lfs_dir_fetch_(lfs, dir, dir->tail, NULL, NULL); + if (err) { + return err; + } + + dir->id = 0; + } + + int err = lfs_dir_getinfo_(lfs, dir, dir->id, info); + if (err != LFS_ERR_NOENT) { + if (!err) { + break; + } + return err; + } + + dir->id += 1; + } + + dir->pos += 1; + return true; +} int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { dir->pair[0] = lfs->root[0]; @@ -3570,60 +3737,54 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_alloc_ack(lfs); // create superblock dir - lfs_dir_t superdir; - err = lfs_dir_alloc(lfs, &superdir); + lfs_dir_t_ dir; + err = lfs_dir_alloc_(lfs, &dir); if (err) { return err; } // write root directory - lfs_dir_t root; - err = lfs_dir_alloc(lfs, &root); + lfs_dir_t_ root; + err = lfs_dir_alloc_(lfs, &root); if (err) { return err; } - err = lfs_dir_commit(lfs, &root, NULL, 0); + err = lfs_dir_commit_(lfs, &root, NULL, 0); if (err) { return err; } lfs->root[0] = root.pair[0]; lfs->root[1] = root.pair[1]; - superdir.d.tail[0] = lfs->root[0]; - superdir.d.tail[1] = lfs->root[1]; + dir.tail[0] = lfs->root[0]; + dir.tail[1] = lfs->root[1]; // write one superblock - lfs_superblock_t superblock; - superblock.d.version = LFS_DISK_VERSION, - superblock.d.root[0] = lfs->root[0]; - superblock.d.root[1] = lfs->root[1]; - superblock.d.block_size = lfs->cfg->block_size; - superblock.d.block_count = lfs->cfg->block_count; - superblock.d.inline_size = lfs->inline_size; - superblock.d.attrs_size = lfs->attrs_size; - superblock.d.name_size = lfs->name_size; - - lfs_entry_t superentry; - superentry.d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK; - superentry.d.elen = sizeof(superblock.d); - superentry.d.alen = 0; - superentry.d.nlen = strlen("littlefs"); - superentry.off = sizeof(superdir.d); - superentry.size = 0; - - lfs_entry_tole32(&superentry.d); - lfs_superblock_tole32(&superblock.d); - err = lfs_dir_set(lfs, &superdir, &superentry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 0, &superentry.d, 4}, - {LFS_FROM_MEM, 0, 0, &superblock.d, sizeof(superblock.d)}, - {LFS_FROM_MEM, 0, 0, "littlefs", superentry.d.nlen}}, 3); + lfs_superblock_t_ superblock = { + .root[0] = lfs->root[0], + .root[1] = lfs->root[1], + .magic = {"littlefs"}, + .version = LFS_DISK_VERSION, + + .block_size = lfs->cfg->block_size, + .block_count = lfs->cfg->block_count, + .inline_size = lfs->cfg->inline_size, + .attrs_size = lfs->cfg->attrs_size, + .name_size = lfs->cfg->name_size, + }; + + dir.count += 1; + err = lfs_dir_set_(lfs, &dir, (lfs_entry_t_[]){ + {lfs_mktag(LFS_TYPE_SUPERBLOCK_, 0, sizeof(superblock)), + .u.buffer=&superblock}}, 1); if (err) { return err; } // sanity check that fetch works - err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); + err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}, + NULL, NULL); if (err) { return err; } @@ -3644,8 +3805,9 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_alloc_ack(lfs); // load superblock - lfs_dir_t dir; - err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + lfs_dir_t_ dir; + err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}, + NULL, NULL); if (err) { if (err == LFS_ERR_CORRUPT) { LFS_ERROR("Invalid superblock at %d %d", 0, 1); @@ -3653,75 +3815,59 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); - if (err) { + lfs_superblock_t_ superblock; + err = lfs_dir_get_(lfs, &dir, 0x7ffff000, &(lfs_entry_t_){ + lfs_mktag(LFS_TYPE_SUPERBLOCK_, 0, sizeof(superblock)), + .u.buffer=&superblock}); + if (err && err != LFS_ERR_RANGE) { return err; } - lfs_superblock_t superblock; - memset(&superblock.d, 0, sizeof(superblock.d)); - err = lfs_dir_get(lfs, &dir, - sizeof(dir.d)+4, &superblock.d, - lfs_min(sizeof(superblock.d), lfs_entry_elen(&entry))); - lfs_superblock_fromle32(&superblock.d); - if (err) { - return err; - } - - char magic[8]; - err = lfs_dir_get(lfs, &dir, - sizeof(dir.d)+lfs_entry_size(&entry)-entry.d.nlen, magic, - lfs_min(sizeof(magic), entry.d.nlen)); - if (err) { - return err; - } - - if (memcmp(magic, "littlefs", 8) != 0) { + if (memcmp(superblock.magic, "littlefs", 8) != 0) { LFS_ERROR("Invalid superblock at %d %d", 0, 1); return LFS_ERR_CORRUPT; } - uint16_t major_version = (0xffff & (superblock.d.version >> 16)); - uint16_t minor_version = (0xffff & (superblock.d.version >> 0)); + uint16_t major_version = (0xffff & (superblock.version >> 16)); + uint16_t minor_version = (0xffff & (superblock.version >> 0)); if ((major_version != LFS_DISK_VERSION_MAJOR || minor_version > LFS_DISK_VERSION_MINOR)) { LFS_ERROR("Invalid version %d.%d", major_version, minor_version); return LFS_ERR_INVAL; } - if (superblock.d.inline_size) { - if (superblock.d.inline_size > lfs->inline_size) { + if (superblock.inline_size) { + if (superblock.inline_size > lfs->inline_size) { LFS_ERROR("Unsupported inline size (%d > %d)", - superblock.d.inline_size, lfs->inline_size); + superblock.inline_size, lfs->inline_size); return LFS_ERR_INVAL; } - lfs->inline_size = superblock.d.inline_size; + lfs->inline_size = superblock.inline_size; } - if (superblock.d.attrs_size) { - if (superblock.d.attrs_size > lfs->attrs_size) { + if (superblock.attrs_size) { + if (superblock.attrs_size > lfs->attrs_size) { LFS_ERROR("Unsupported attrs size (%d > %d)", - superblock.d.attrs_size, lfs->attrs_size); + superblock.attrs_size, lfs->attrs_size); return LFS_ERR_INVAL; } - lfs->attrs_size = superblock.d.attrs_size; + lfs->attrs_size = superblock.attrs_size; } - if (superblock.d.name_size) { - if (superblock.d.name_size > lfs->name_size) { + if (superblock.name_size) { + if (superblock.name_size > lfs->name_size) { LFS_ERROR("Unsupported name size (%d > %d)", - superblock.d.name_size, lfs->name_size); + superblock.name_size, lfs->name_size); return LFS_ERR_INVAL; } - lfs->name_size = superblock.d.name_size; + lfs->name_size = superblock.name_size; } - lfs->root[0] = superblock.d.root[0]; - lfs->root[1] = superblock.d.root[1]; + lfs->root[0] = superblock.root[0]; + lfs->root[1] = superblock.root[1]; return 0; } @@ -3732,6 +3878,73 @@ int lfs_unmount(lfs_t *lfs) { /// Internal filesystem filesystem operations /// +int lfs_traverse_(lfs_t *lfs, + int (*cb)(lfs_t *lfs, void *data, lfs_block_t block), void *data) { + if (lfs_pairisnull(lfs->root)) { + return 0; + } + + // iterate over metadata pairs + lfs_dir_t_ dir = {.tail = {0, 1}}; + while (!lfs_pairisnull(dir.tail)) { + for (int i = 0; i < 2; i++) { + int err = cb(lfs, data, dir.tail[i]); + if (err) { + return err; + } + } + + // iterate through ids in directory + int err = lfs_dir_fetch_(lfs, &dir, dir.tail, NULL, NULL); + if (err) { + return err; + } + + for (int i = 0; i < dir.count; i++) { + lfs_entry_t_ entry = {lfs_mktag(LFS_TYPE_REG_, i, 8), + .u.buffer=entry.u.pair}; + int err = lfs_dir_get_(lfs, &dir, 0x701ff000, &entry); + if (err) { + if (err == LFS_ERR_NOENT) { + continue; + } + return err; + } + + if (lfs_tag_struct(entry.tag) == LFS_STRUCT_CTZ_) { +// TODO +// err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, +// entry.d.u.file.head, entry.d.u.file.size, cb, data); +// if (err) { +// return err; +// } + } + + } + } + + // iterate over any open files + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { +// int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, +// f->head, f->size, cb, data); +// if (err) { +// return err; +// } + } + + if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { +// int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, +// f->block, f->pos, cb, data); +// if (err) { +// return err; +// } + } + } + + return 0; +} + int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { if (lfs_pairisnull(lfs->root)) { return 0; @@ -3804,12 +4017,31 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { return 0; } +static int lfs_pred_(lfs_t *lfs, const lfs_block_t pair[2], lfs_dir_t_ *pdir) { + pdir->tail[0] = 0; + pdir->tail[1] = 1; + + // iterate over all directory directory entries + while (!lfs_pairisnull(pdir->tail)) { + if (lfs_paircmp(pdir->tail, pair) == 0) { + return true; + } + + int err = lfs_dir_fetch_(lfs, pdir, pdir->tail, NULL, NULL); + if (err) { + return err; + } + } + + return false; +} + static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir) { if (lfs_pairisnull(lfs->root)) { return 0; } - // iterate over all directory directory entries + // iterate directories int err = lfs_dir_fetch(lfs, pdir, (const lfs_block_t[2]){0, 1}); if (err) { return err; @@ -3829,6 +4061,38 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir) { return false; } +static int lfs_parent_(lfs_t *lfs, const lfs_block_t pair[2], + lfs_dir_t_ *parent, lfs_entry_t_ *entry) { + parent->tail[0] = 0; + parent->tail[1] = 1; + + // iterate over all directory directory entries + while (!lfs_pairisnull(parent->tail)) { + int err = lfs_dir_fetch_(lfs, parent, parent->tail, NULL, NULL); + if (err) { + return err; + } + + for (int i = 0; i < parent->count; i++) { + entry->tag = lfs_mktag(LFS_STRUCT_DIR_, i, 8); + entry->u.buffer = &entry->u; + int err = lfs_dir_get_(lfs, parent, 0x43dff000, entry); + if (err && err != LFS_ERR_RANGE) { + if (err == LFS_ERR_NOENT) { + continue; + } + return err; + } + + if (lfs_paircmp(entry->u.pair, pair) == 0) { + return true; + } + } + } + + return false; +} + static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *parent, lfs_entry_t *entry) { if (lfs_pairisnull(lfs->root)) { @@ -3865,6 +4129,51 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], return false; } +static int lfs_moved_(lfs_t *lfs, const lfs_block_t pair[2]) { + // skip superblock + lfs_dir_t_ dir; + int err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}, + NULL, NULL); + if (err) { + return err; + } + + // iterate over all directory directory entries + while (!lfs_pairisnull(dir.tail)) { + int err = lfs_dir_fetch_(lfs, &dir, dir.tail, NULL, NULL); + if (err) { + return err; + } + + for (int i = 0; i < dir.count; i++) { + lfs_entry_t_ entry = {lfs_mktag(LFS_STRUCT_DIR_, i, 8), + .u.buffer=&entry.u}; + int err = lfs_dir_get_(lfs, &dir, 0x43dff000, &entry); + if (err) { + if (err == LFS_ERR_NOENT) { + continue; + } + return err; + } + + err = lfs_dir_get_(lfs, &dir, 0x7ffff000, &(lfs_entry_t_){ + lfs_mktag(LFS_TYPE_MOVE_, i, 0)}); + if (err != LFS_ERR_NOENT) { + if (!err) { + continue; + } + return err; + } + + if (lfs_paircmp(entry.u.pair, pair) == 0) { + return true; + } + } + } + + return false; +} + static int lfs_moved(lfs_t *lfs, const void *e) { if (lfs_pairisnull(lfs->root)) { return 0; @@ -3956,6 +4265,123 @@ static int lfs_relocate(lfs_t *lfs, return 0; } +int lfs_deorphan_check(lfs_t *lfs, void *p, lfs_entry_t_ entry) { + int16_t *id = p; + + // TODO this fine for only grabbing the last one? + // TODO should I also grab deletes? I should, move will always be last yay + if (lfs_tag_type(entry.tag) == LFS_TYPE_MOVE_) { + *id = lfs_tag_id(entry.tag); + } + + // TODO handle unrelated deletes + if (lfs_tag_type(entry.tag) == LFS_TYPE_DROP_ && + lfs_tag_id(entry.tag) == *id) { + *id = -1; + } + + return 0; +} + +int lfs_deorphan_(lfs_t *lfs) { + lfs->deorphaned = true; + if (lfs_pairisnull(lfs->root)) { + return 0; + } + + lfs_dir_t_ pdir = {.split = true}; + lfs_dir_t_ dir = {.tail = {0, 1}}; + + // iterate over all directory directory entries + while (!lfs_pairisnull(dir.tail)) { + int16_t moveid = -1; + int err = lfs_dir_fetch_(lfs, &dir, dir.tail, + lfs_deorphan_check, &moveid); + if (err) { + return err; + } + + // check head blocks for orphans + if (!pdir.split) { + // check if we have a parent + lfs_dir_t_ parent; + lfs_entry_t_ entry; + int res = lfs_parent_(lfs, pdir.tail, &parent, &entry); + if (res < 0) { + return res; + } + + if (!res) { + // we are an orphan + LFS_DEBUG("Found orphan %d %d", + pdir.tail[0], pdir.tail[1]); + + pdir.tail[0] = dir.tail[0]; + pdir.tail[1] = dir.tail[1]; + err = lfs_dir_set_(lfs, &pdir, &(lfs_entry_t_){ + lfs_mktag(LFS_TYPE_TAIL_, 0x1ff, sizeof(pdir.tail)), + .u.buffer=pdir.tail}, 1); + if (err) { + return err; + } + + break; + } + + if (!lfs_pairsync(entry.u.pair, pdir.tail)) { + // we have desynced + LFS_DEBUG("Found desync %d %d", + entry.u.pair[0], entry.u.pair[1]); + + pdir.tail[0] = entry.u.pair[0]; + pdir.tail[1] = entry.u.pair[1]; + err = lfs_dir_set_(lfs, &pdir, &(lfs_entry_t_){ + lfs_mktag(LFS_TYPE_TAIL_, 0x1ff, sizeof(pdir.tail)), + .u.buffer=pdir.tail}, 1); + if (err) { + return err; + } + + break; + } + } + + // check entries for moves + if (moveid >= 0) { +// TODO moves and stuff + // TODO need to load entry to find it +// // found moved entry +// int moved = lfs_moved(lfs, &entry.u); +// if (moved < 0) { +// return moved; +// } +// +// if (moved) { +// LFS_DEBUG("Found move %d %d", +// entry.d.u.dir[0], entry.d.u.dir[1]); +// err = lfs_dir_set(lfs, &dir, &entry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); +// if (err) { +// return err; +// } +// } else { +// LFS_DEBUG("Found partial move %d %d", +// entry.d.u.dir[0], entry.d.u.dir[1]); +// entry.d.type &= ~LFS_STRUCT_MOVED; +// err = lfs_dir_set(lfs, &dir, &entry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, 1, &entry.d, 1}}, 1); +// if (err) { +// return err; +// } +// } + } + + memcpy(&pdir, &dir, sizeof(pdir)); + } + + return 0; +} + int lfs_deorphan(lfs_t *lfs) { lfs->deorphaned = true; diff --git a/lfs.h b/lfs.h index b0046f0e..5e963f25 100644 --- a/lfs.h +++ b/lfs.h @@ -107,14 +107,14 @@ enum lfs_type { LFS_STRUCT_MOVED = 0x80, // file type - LFS_TYPE_REG_ = 0x020, - LFS_TYPE_DIR_ = 0x030, + LFS_TYPE_REG_ = 0x040, + LFS_TYPE_DIR_ = 0x050, LFS_TYPE_NAME_ = 0x010, - LFS_TYPE_MOVE_ = 0x060, - LFS_TYPE_DROP_ = 0x070, + LFS_TYPE_MOVE_ = 0x080, + LFS_TYPE_DROP_ = 0x090, - LFS_TYPE_SUPERBLOCK_ = 0x0c0, + LFS_TYPE_SUPERBLOCK_ = 0x0c8, LFS_TYPE_TAIL_ = 0x0d0, LFS_TYPE_CRC_ = 0x0e0, @@ -294,9 +294,13 @@ typedef struct lfs_entry { } d; } lfs_entry_t; +typedef uint32_t lfs_tag_t; +typedef int32_t lfs_stag_t; + typedef struct lfs_entry_ { uint32_t tag; union { + void *buffer; lfs_block_t pair[2]; struct { lfs_block_t head; @@ -357,14 +361,20 @@ typedef struct lfs_dir { } lfs_dir_t; typedef struct lfs_dir_ { + struct lfs_dir_ *next; lfs_block_t pair[2]; - lfs_block_t tail[2]; + lfs_block_t tail[2]; uint32_t rev; lfs_off_t off; uint32_t etag; uint16_t count; bool erased; + bool split; + + uint16_t id; + lfs_block_t head[2]; + lfs_off_t pos; } lfs_dir_t_; typedef struct lfs_superblock { @@ -381,6 +391,19 @@ typedef struct lfs_superblock { } d; } lfs_superblock_t; +typedef struct lfs_superblock_ { + lfs_block_t root[2]; + char magic[8]; + uint32_t version; + + lfs_size_t block_size; + lfs_size_t block_count; + + lfs_size_t inline_size; + lfs_size_t attrs_size; + lfs_size_t name_size; +} lfs_superblock_t_; + typedef struct lfs_free { lfs_block_t off; lfs_block_t size; From fe553e8af4530188bf6e63a3efcad6e96258992a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 22 May 2018 17:43:39 -0500 Subject: [PATCH 032/139] More progress integrating journaling - Integrated into lfs_file_t_, duplicating functions where necessary - Added lfs_dir_fetchwith_ as common parent to both lfs_dir_fetch_ and lfs_dir_find_ - Added similar parent with lfs_dir_commitwith_ - Made matching find/get operations with getbuffer/getentry and findbuffer/findentry - lfs_dir_alloc now populates tail, since almost all directory block allocations need to populate tail --- lfs.c | 640 +++++++++++++++++++++++++++++++++++++++++++--------------- lfs.h | 37 ++-- 2 files changed, 496 insertions(+), 181 deletions(-) diff --git a/lfs.c b/lfs.c index cfc036bb..6e236ebb 100644 --- a/lfs.c +++ b/lfs.c @@ -637,7 +637,8 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return 0; } -/*static*/ int lfs_dir_alloc_(lfs_t *lfs, lfs_dir_t_ *dir) { +/*static*/ int lfs_dir_alloc_(lfs_t *lfs, lfs_dir_t_ *dir, + const lfs_block_t tail[2]) { // allocate pair of dir blocks (backwards, so we write to block 1 first) for (int i = 0; i < 2; i++) { int err = lfs_alloc(lfs, &dir->pair[(i+1)%2]); @@ -658,8 +659,8 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { dir->off = sizeof(dir->rev); dir->etag = 0; dir->count = 0; - dir->tail[0] = 0xffffffff; - dir->tail[1] = 0xffffffff; + dir->tail[0] = tail[0]; + dir->tail[1] = tail[1]; dir->erased = false; dir->split = false; @@ -667,7 +668,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return 0; } -/*static*/ int lfs_dir_fetch_(lfs_t *lfs, +/*static*/ int lfs_dir_fetchwith_(lfs_t *lfs, lfs_dir_t_ *dir, const lfs_block_t pair[2], int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), void *data) { @@ -762,7 +763,9 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { dir->count = lfs_tag_id(tag)+1; } - if (lfs_tag_type(tag) == LFS_TYPE_TAIL_) { + if (lfs_tag_type(tag) == LFS_TYPE_SOFTTAIL_ || + lfs_tag_type(tag) == LFS_TYPE_HARDTAIL_) { + dir->split = lfs_tag_type(tag) == LFS_TYPE_HARDTAIL_; err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), dir->tail, sizeof(dir->tail)); if (err) { @@ -792,6 +795,11 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return LFS_ERR_CORRUPT; } +/*static*/ int lfs_dir_fetch_(lfs_t *lfs, + lfs_dir_t_ *dir, const lfs_block_t pair[2]) { + return lfs_dir_fetchwith_(lfs, dir, pair, NULL, NULL); +} + static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), void *data) { @@ -800,49 +808,6 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, cb, data); } -struct lfs_dir_getter { - uint32_t mask; - lfs_entry_t_ entry; -}; - -static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { - struct lfs_dir_getter *get = p; - if ((entry.tag & get->mask) == (get->entry.tag & get->mask)) { - int err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, - get->entry.u.buffer, lfs_min(lfs_tag_size(entry.tag), - lfs_tag_size(get->entry.tag))); - if (err) { - return err; - } - - get->entry.tag |= entry.tag & ~0x80000fff; - if (lfs_tag_size(entry.tag) > lfs_tag_size(get->entry.tag)) { - return LFS_ERR_RANGE; - } else { - return true; - } - } - - return false; -} - -/*static*/ int lfs_dir_get_(lfs_t *lfs, lfs_dir_t_ *dir, - uint32_t mask, lfs_entry_t_ *entry) { - struct lfs_dir_getter get = {mask, *entry}; - memset(get.entry.u.buffer, 0, lfs_tag_size(get.entry.tag)); - int res = lfs_dir_traverse_(lfs, dir, lfs_dir_getter, &get); - entry->tag = get.entry.tag; - if (res < 0) { - return res; - } - - if (!res) { - return LFS_ERR_NOENT; - } - - return 0; -} - struct lfs_dir_mover { // traversal things lfs_dir_t_ *dir; @@ -964,7 +929,8 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { // TODO le32 commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t), err = lfs_commit_commit(lfs, &commit, (lfs_entry_t_){ - lfs_mktag(LFS_TYPE_TAIL_, 0x1ff, sizeof(dir->tail)), + lfs_mktag(LFS_TYPE_SOFTTAIL_ + dir->split, + 0x1ff, sizeof(dir->tail)), .u.buffer=dir->tail}); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -999,14 +965,11 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { lfs->pcache.block = 0xffffffff; lfs_dir_t_ tail; - int err = lfs_dir_alloc_(lfs, &tail); + int err = lfs_dir_alloc_(lfs, &tail, dir->tail); if (err) { return err; } - tail.tail[0] = dir->tail[0]; - tail.tail[1] = dir->tail[1]; - err = lfs_dir_compact_(lfs, &tail, lfs_dir_mover, &mover); if (err) { return err; @@ -1061,7 +1024,7 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -/*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, +/*static*/ int lfs_dir_commitwith_(lfs_t *lfs, lfs_dir_t_ *dir, int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), void *data) { if (!dir->erased) { @@ -1101,43 +1064,13 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -//struct lfs_dir_commit_regions { -// const lfs_entry_t_ *regions; -// int count; -//}; -// -//int lfs_dir_commit_regions(lfs_t *lfs, void *p, struct lfs_commit *commit) { -// struct lfs_dir_commit_regions *entry = p; -// for (int i = 0; i < entry->count; i++) { -// int err = lfs_commit_commit(lfs, commit, entry->regions[i]); -// if (err) { -// return err; -// } -// } -// -// return 0; -//} -// -// TODO rm me, just for testing -///*static*/ int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t_ *dir, -// const lfs_entry_t_ *regions, int count) { -// return lfs_dir_compact_(lfs, dir, lfs_dir_commit_regions, -// &(struct lfs_dir_commit_regions){regions, count}); -//} -// -///*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, -// const lfs_entry_t_ *regions, int count) { -// return lfs_dir_commit_(lfs, dir, lfs_dir_commit_regions, -// &(struct lfs_dir_commit_regions){regions, count}); -//} - -struct lfs_dir_setter { +struct lfs_dir_committer { const lfs_entry_t_ *regions; int count; }; -int lfs_dir_setter(lfs_t *lfs, void *p, struct lfs_commit *commit) { - struct lfs_dir_setter *set = p; +int lfs_dir_committer(lfs_t *lfs, void *p, struct lfs_commit *commit) { + struct lfs_dir_committer *set = p; for (int i = 0; i < set->count; i++) { int err = lfs_commit_commit(lfs, commit, set->regions[i]); if (err) { @@ -1148,20 +1081,12 @@ int lfs_dir_setter(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -/*static*/ int lfs_dir_set_(lfs_t *lfs, lfs_dir_t_ *dir, +/*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, const lfs_entry_t_ *regions, int count) { - return lfs_dir_commit_(lfs, dir, lfs_dir_setter, - &(struct lfs_dir_setter){regions, count}); + return lfs_dir_commitwith_(lfs, dir, lfs_dir_committer, + &(struct lfs_dir_committer){regions, count}); } -struct lfs_dir_finder { - const char *name; - lfs_size_t len; - - int16_t id; - lfs_entry_t_ *entry; -}; - /*static*/ int lfs_dir_add(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t *id) { *id = dir->count; dir->count += 1; @@ -1171,10 +1096,80 @@ struct lfs_dir_finder { /*static*/ int lfs_dir_drop(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id) { dir->count -= 1; // TODO compact during traverse when compacting? - return lfs_dir_set_(lfs, dir, (lfs_entry_t_[]){{ + return lfs_dir_commit_(lfs, dir, (lfs_entry_t_[]){{ lfs_mktag(LFS_TYPE_DROP_, id, 0)}}, 1); } +struct lfs_dir_getter { + uint32_t mask; + lfs_tag_t tag; + lfs_entry_t_ *entry; +}; + +static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { + struct lfs_dir_getter *get = p; + if ((entry.tag & get->mask) == (get->tag & get->mask)) { + if (get->entry) { + *get->entry = entry; + } + return true; + } + + return false; +} + +/*static*/ int lfs_dir_get_(lfs_t *lfs, lfs_dir_t_ *dir, + uint32_t mask, lfs_tag_t tag, lfs_entry_t_ *entry) { + int res = lfs_dir_traverse_(lfs, dir, lfs_dir_getter, + &(struct lfs_dir_getter){mask, tag, entry}); + if (res < 0) { + return res; + } + + if (!res) { + return LFS_ERR_NOENT; + } + + return 0; +} + +/*static*/ int lfs_dir_getbuffer_(lfs_t *lfs, lfs_dir_t_ *dir, + uint32_t mask, lfs_tag_t tag, lfs_entry_t_ *entry) { + void *buffer = entry->u.buffer; + lfs_size_t size = lfs_tag_size(tag); + int err = lfs_dir_get_(lfs, dir, mask, tag, entry); + if (err) { + return err; + } + + lfs_size_t diff = lfs_min(size, lfs_tag_size(entry->tag)); + memset((uint8_t*)buffer + diff, 0, size - diff); + err = lfs_bd_read(lfs, entry->u.d.block, entry->u.d.off, buffer, diff); + if (err) { + return err; + } + + if (lfs_tag_size(entry->tag) > size) { + return LFS_ERR_RANGE; + } + + return 0; +} + +/*static*/ int lfs_dir_getentry_(lfs_t *lfs, lfs_dir_t_ *dir, + uint32_t mask, lfs_tag_t tag, lfs_entry_t_ *entry) { + entry->u.buffer = &entry->u; + return lfs_dir_getbuffer_(lfs, dir, mask, tag, entry); +} + +struct lfs_dir_finder { + const char *name; + lfs_size_t len; + + int16_t id; + lfs_entry_t_ *entry; +}; + static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { struct lfs_dir_finder *find = p; @@ -1194,26 +1189,15 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { } if (find->id >= 0 && lfs_tag_id(entry.tag) == find->id && - (lfs_tag_type(entry.tag) & 0x1c0) == LFS_TYPE_REG_) { - // TODO combine regions and entries? - find->entry->tag = ~0x80000000 & entry.tag; - if (lfs_tag_type(entry.tag) & 0x00f) { - int err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, - &find->entry->u, sizeof(find->entry->u)); - if (err) { - return err; - } - } else { - find->entry->u.d.block = entry.u.d.block; - find->entry->u.d.off = entry.u.d.off; - } + lfs_tag_supertype(entry.tag) == LFS_TYPE_REG_) { + *find->entry = entry; } return 0; } /*static*/ int lfs_dir_find_(lfs_t *lfs, lfs_dir_t_ *dir, - lfs_entry_t_ *entry, const char **path) { + const char **path, lfs_entry_t_ *entry) { struct lfs_dir_finder find = { .name = *path, .entry = entry, @@ -1275,7 +1259,7 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { // find path while (true) { find.id = -1; - int err = lfs_dir_fetch_(lfs, dir, entry->u.pair, + int err = lfs_dir_fetchwith_(lfs, dir, entry->u.pair, lfs_dir_finder, &find); if (err) { return err; @@ -1319,6 +1303,36 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { } } +/*static*/ int lfs_dir_findbuffer_(lfs_t *lfs, lfs_dir_t_ *dir, + const char **path, lfs_entry_t_ *entry) { + void *buffer = entry->u.buffer; + lfs_size_t size = lfs_tag_size(entry->tag); + int err = lfs_dir_find_(lfs, dir, path, entry); + if (err) { + return err; + } + + lfs_size_t diff = lfs_min(size, lfs_tag_size(entry->tag)); + memset((uint8_t*)buffer + diff, 0, size - diff); + err = lfs_bd_read(lfs, entry->u.d.block, entry->u.d.off, buffer, diff); + if (err) { + return err; + } + + if (lfs_tag_size(entry->tag) > size) { + return LFS_ERR_RANGE; + } + + return 0; +} + +/*static*/ int lfs_dir_findentry_(lfs_t *lfs, lfs_dir_t_ *dir, + const char **path, lfs_entry_t_ *entry) { + entry->tag = sizeof(entry->u); + entry->u.buffer = &entry->u; + return lfs_dir_findbuffer_(lfs, dir, path, entry); +} + ////////////////////////////////////////////////////////// @@ -2124,8 +2138,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // fetch parent directory lfs_dir_t_ cwd; - lfs_entry_t_ entry; - int err = lfs_dir_find_(lfs, &cwd, &entry, &path); + int err = lfs_dir_findentry_(lfs, &cwd, &path, &(lfs_entry_t_){0}); if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { if (!err) { return LFS_ERR_EXIST; @@ -2143,14 +2156,12 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_alloc_ack(lfs); lfs_dir_t_ dir; - err = lfs_dir_alloc_(lfs, &dir); // TODO add tail to this function? + err = lfs_dir_alloc_(lfs, &dir, cwd.tail); if (err) { return err; } - dir.tail[0] = cwd.tail[0]; - dir.tail[1] = cwd.tail[1]; - err = lfs_dir_set_(lfs, &dir, NULL, 0); + err = lfs_dir_commit_(lfs, &dir, NULL, 0); if (err) { return err; } @@ -2162,8 +2173,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } - err = lfs_dir_set_(lfs, &cwd, (lfs_entry_t_[]){ - {lfs_mktag(LFS_TYPE_NAME_, id, nlen), .u.buffer=path}, + err = lfs_dir_commit_(lfs, &cwd, (lfs_entry_t_[]){ + {lfs_mktag(LFS_TYPE_NAME_, id, nlen), .u.buffer=(void*)path}, {lfs_mktag(LFS_TYPE_DIR_ | LFS_STRUCT_DIR_, id, sizeof(dir.pair)), .u.buffer=dir.pair}}, 2); @@ -2242,7 +2253,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { int lfs_dir_open_(lfs_t *lfs, lfs_dir_t_ *dir, const char *path) { lfs_entry_t_ entry; - int err = lfs_dir_find_(lfs, dir, &entry, &path); + int err = lfs_dir_findentry_(lfs, dir, &path, &entry); if (err) { return err; } @@ -2251,7 +2262,7 @@ int lfs_dir_open_(lfs_t *lfs, lfs_dir_t_ *dir, const char *path) { return LFS_ERR_NOTDIR; } - err = lfs_dir_fetch_(lfs, dir, entry.u.pair, NULL, NULL); + err = lfs_dir_fetch_(lfs, dir, entry.u.pair); if (err) { return err; } @@ -2284,24 +2295,23 @@ int lfs_dir_close_(lfs_t *lfs, lfs_dir_t_ *dir) { // TODO move me? static int lfs_dir_getinfo_(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id, struct lfs_info *info) { - lfs_entry_t_ entry = {lfs_mktag(LFS_TYPE_REG, id, 8), .u.buffer=&entry.u}; - int err = lfs_dir_get_(lfs, dir, 0x701ff000, &entry); + lfs_entry_t_ entry; + int err = lfs_dir_getentry_(lfs, dir, + 0x701ff000, lfs_mktag(LFS_TYPE_REG, id, 8), &entry); if (err && err != LFS_ERR_RANGE) { return err; } info->type = lfs_tag_subtype(entry.tag); - if (lfs_tag_type(entry.tag) == - (LFS_TYPE_REG_ | LFS_STRUCT_CTZ_)) { + if (lfs_tag_type(entry.tag) == (LFS_TYPE_REG_ | LFS_STRUCT_CTZ_)) { info->size = entry.u.ctz.size; - } else if (lfs_tag_type(entry.tag) == - (LFS_TYPE_REG_ | LFS_STRUCT_INLINE_)) { + } else if (lfs_tag_type(entry.tag) == (LFS_TYPE_REG_ | LFS_STRUCT_INLINE_)) { info->size = lfs_tag_size(entry.tag); } - err = lfs_dir_get_(lfs, dir, 0x7ffff000, &(lfs_entry_t_){ - lfs_mktag(LFS_TYPE_NAME_, id, lfs->cfg->name_size), - .u.buffer=info->name}); + err = lfs_dir_getbuffer_(lfs, dir, + 0x7ffff000, lfs_mktag(LFS_TYPE_NAME_, id, lfs->cfg->name_size+1), + &(lfs_entry_t_){.u.buffer=info->name}); if (err) { return err; } @@ -2331,7 +2341,7 @@ int lfs_dir_read_(lfs_t *lfs, lfs_dir_t_ *dir, struct lfs_info *info) { return false; } - int err = lfs_dir_fetch_(lfs, dir, dir->tail, NULL, NULL); + int err = lfs_dir_fetch_(lfs, dir, dir->tail); if (err) { return err; } @@ -2687,6 +2697,119 @@ static int lfs_ctz_traverse(lfs_t *lfs, /// Top level file operations /// +int lfs_file_open_(lfs_t *lfs, lfs_file_t_ *file, + const char *path, int flags) { + // deorphan if we haven't yet, needed at most once after poweron + if ((flags & 3) != LFS_O_RDONLY && !lfs->deorphaned) { + int err = lfs_deorphan_(lfs); + if (err) { + return err; + } + } + + // allocate entry for file if it doesn't exist + lfs_dir_t_ cwd; + lfs_entry_t_ entry; + int err = lfs_dir_find_(lfs, &cwd, &path, &entry); + if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { + return err; + } + + if (err == LFS_ERR_NOENT) { + if (!(flags & LFS_O_CREAT)) { + return LFS_ERR_NOENT; + } + + // check that name fits + lfs_size_t nlen = strlen(path); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + + // get next slot and create entry to remember name + uint16_t id; + err = lfs_dir_add(lfs, &cwd, &id); + if (err) { + return err; + } + + err = lfs_dir_commit_(lfs, &cwd, (lfs_entry_t_[]){ + {lfs_mktag(LFS_TYPE_NAME_, id, nlen), .u.buffer=(void*)path}, + {lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_INLINE_, id, 0)}}, 2); + if (err) { + return err; + } + } else if (lfs_tag_subtype(entry.tag) != LFS_TYPE_REG_) { + return LFS_ERR_ISDIR; + } else if (flags & LFS_O_EXCL) { + return LFS_ERR_EXIST; + } + + // allocate buffer if needed + file->cache.block = 0xffffffff; + if (lfs->cfg->file_buffer) { + file->cache.buffer = lfs->cfg->file_buffer; + } else if ((file->flags & 3) == LFS_O_RDONLY) { + file->cache.buffer = lfs_malloc(lfs->cfg->read_size); + if (!file->cache.buffer) { + return LFS_ERR_NOMEM; + } + } else { + file->cache.buffer = lfs_malloc(lfs->cfg->prog_size); + if (!file->cache.buffer) { + return LFS_ERR_NOMEM; + } + } + + // setup file struct + file->pair[0] = cwd.pair[0]; + file->pair[1] = cwd.pair[1]; + file->id = lfs_tag_id(entry.tag); + file->flags = flags; + file->pos = 0; + + if (lfs_tag_struct(entry.tag) == LFS_STRUCT_INLINE_) { + // load inline files + file->head = 0xfffffffe; + file->size = lfs_tag_size(entry.tag); + file->flags |= LFS_F_INLINE; + file->cache.block = file->head; + file->cache.off = 0; + err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, + file->cache.buffer, file->size); + if (err) { + lfs_free(file->cache.buffer); + return err; + } + } else { + // use ctz list from entry + err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, + &entry.u, sizeof(entry.u)); + // TODO move to disk struct directly? + file->head = entry.u.ctz.head; + file->size = entry.u.ctz.size; + } + + // truncate if requested + if (flags & LFS_O_TRUNC) { + if (file->size != 0) { + file->flags |= LFS_F_DIRTY; + } + + file->head = 0xfffffffe; + file->size = 0; + file->flags |= LFS_F_INLINE; + file->cache.block = file->head; + file->cache.off = 0; + } + + // add to list of files + file->next = lfs->files; + lfs->files = file; + + return 0; +} + int lfs_file_open(lfs_t *lfs, lfs_file_t *file, const char *path, int flags) { // deorphan if we haven't yet, needed at most once after poweron @@ -2827,6 +2950,52 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { return err; } +static int lfs_file_relocate_(lfs_t *lfs, lfs_file_t_ *file) { +relocate:; + // just relocate what exists into new block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); + if (err) { + return err; + } + + err = lfs_bd_erase(lfs, nblock); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // either read from dirty cache or disk + for (lfs_off_t i = 0; i < file->off; i++) { + uint8_t data; + err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, + file->block, i, &data, 1); + if (err) { + return err; + } + + err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, + nblock, i, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + // copy over new state of file + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); + file->cache.block = lfs->pcache.block; + file->cache.off = lfs->pcache.off; + lfs->pcache.block = 0xffffffff; + + file->block = nblock; + return 0; +} + static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { relocate:; // just relocate what exists into new block @@ -2873,6 +3042,80 @@ relocate:; return 0; } +static int lfs_file_flush_(lfs_t *lfs, lfs_file_t_ *file) { + if (file->flags & LFS_F_READING) { + file->flags &= ~LFS_F_READING; + } + + if (file->flags & LFS_F_WRITING) { + lfs_off_t pos = file->pos; + + if (!(file->flags & LFS_F_INLINE)) { + // copy over anything after current branch + lfs_file_t_ orig = { + .head = file->head, + .size = file->size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; + lfs->rcache.block = 0xffffffff; + + while (file->pos < file->size) { + // copy over a byte at a time, leave it up to caching + // to make this efficient + uint8_t data; + lfs_ssize_t res = lfs_file_read(lfs, &orig, &data, 1); + if (res < 0) { + return res; + } + + res = lfs_file_write(lfs, file, &data, 1); + if (res < 0) { + return res; + } + + // keep our reference to the rcache in sync + if (lfs->rcache.block != 0xffffffff) { + orig.cache.block = 0xffffffff; + lfs->rcache.block = 0xffffffff; + } + } + + // write out what we have + while (true) { + int err = lfs_cache_flush(lfs, &file->cache, &lfs->rcache); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + break; +relocate: + LFS_DEBUG("Bad block at %d", file->block); + err = lfs_file_relocate_(lfs, file); + if (err) { + return err; + } + } + } else { + file->size = lfs_max(file->pos, file->size); + } + + // actual file updates + file->head = file->block; + file->size = file->pos; + file->flags &= ~LFS_F_WRITING; + file->flags |= LFS_F_DIRTY; + + file->pos = pos; + } + + return 0; +} + static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { if (file->flags & LFS_F_READING) { file->flags &= ~LFS_F_READING; @@ -2947,6 +3190,68 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { return 0; } +int lfs_file_sync_(lfs_t *lfs, lfs_file_t_ *file) { + int err = lfs_file_flush_(lfs, file); + if (err) { + return err; + } + + if ((file->flags & LFS_F_DIRTY) && + !(file->flags & LFS_F_ERRED) && + !lfs_pairisnull(file->pair)) { + // update dir entry + // TODO keep list of dirs including these guys for no + // need of another reload? + lfs_dir_t_ cwd; + err = lfs_dir_fetch_(lfs, &cwd, file->pair); + if (err) { + return err; + } + + // either update the references or inline the whole file + // TODO handle attributes +// if (!(file->flags & LFS_F_INLINE)) { +// entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; +// entry.d.u.file.head = file->head; +// entry.d.u.file.size = file->size; +// +// lfs_entry_tole32(&entry.d); +// buffer = (const uint8_t *)&entry.d + 4; +// size = sizeof(entry.d) - 4; +// } else { +// entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; +// +// buffer = file->cache.buffer; +// size = file->size; +// } +// +// // get new alen from disk +// lfs_ssize_t newalen = lfs_dir_checkattrs(lfs, &cwd, &entry, +// file->attrs, file->attrcount); +// if (newalen < 0) { +// return newalen; +// } +// +// entry.d.elen = size & 0xff; +// entry.d.alen = (newalen & 0x3f) | ((size >> 2) & 0xc0); +// +// // write out update +// err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, 4, &entry.d, 4}, +// {LFS_FROM_MEM, 4, oldelen, buffer, size}, +// {LFS_FROM_ATTRS, 4+oldelen, oldalen, +// &(struct lfs_region_attrs){file->attrs, file->attrcount}, +// newalen}}, 3); +// if (err) { +// return err; +// } + + file->flags &= ~LFS_F_DIRTY; + } + + return 0; +} + int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { int err = lfs_file_flush(lfs, file); if (err) { @@ -3738,14 +4043,16 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // create superblock dir lfs_dir_t_ dir; - err = lfs_dir_alloc_(lfs, &dir); + err = lfs_dir_alloc_(lfs, &dir, + (const lfs_block_t[2]){0xffffffff, 0xffffffff}); if (err) { return err; } // write root directory lfs_dir_t_ root; - err = lfs_dir_alloc_(lfs, &root); + err = lfs_dir_alloc_(lfs, &root, + (const lfs_block_t[2]){0xffffffff, 0xffffffff}); if (err) { return err; } @@ -3775,16 +4082,15 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; dir.count += 1; - err = lfs_dir_set_(lfs, &dir, (lfs_entry_t_[]){ - {lfs_mktag(LFS_TYPE_SUPERBLOCK_, 0, sizeof(superblock)), - .u.buffer=&superblock}}, 1); + err = lfs_dir_commit_(lfs, &dir, (lfs_entry_t_[]){ + {lfs_mktag(LFS_TYPE_SUPERBLOCK_ | LFS_STRUCT_DIR_, 0, + sizeof(superblock)), .u.buffer=&superblock}}, 1); if (err) { return err; } // sanity check that fetch works - err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}, - NULL, NULL); + err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { return err; } @@ -3806,8 +4112,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // load superblock lfs_dir_t_ dir; - err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}, - NULL, NULL); + err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { if (err == LFS_ERR_CORRUPT) { LFS_ERROR("Invalid superblock at %d %d", 0, 1); @@ -3816,9 +4121,9 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_superblock_t_ superblock; - err = lfs_dir_get_(lfs, &dir, 0x7ffff000, &(lfs_entry_t_){ - lfs_mktag(LFS_TYPE_SUPERBLOCK_, 0, sizeof(superblock)), - .u.buffer=&superblock}); + err = lfs_dir_getbuffer_(lfs, &dir, + 0x7ffff000, lfs_mktag(LFS_TYPE_SUPERBLOCK_ | LFS_STRUCT_DIR_, 0, + sizeof(superblock)), &(lfs_entry_t_){.u.buffer=&superblock}); if (err && err != LFS_ERR_RANGE) { return err; } @@ -3895,15 +4200,15 @@ int lfs_traverse_(lfs_t *lfs, } // iterate through ids in directory - int err = lfs_dir_fetch_(lfs, &dir, dir.tail, NULL, NULL); + int err = lfs_dir_fetch_(lfs, &dir, dir.tail); if (err) { return err; } for (int i = 0; i < dir.count; i++) { - lfs_entry_t_ entry = {lfs_mktag(LFS_TYPE_REG_, i, 8), - .u.buffer=entry.u.pair}; - int err = lfs_dir_get_(lfs, &dir, 0x701ff000, &entry); + lfs_entry_t_ entry; + int err = lfs_dir_getentry_(lfs, &dir, + 0x701ff000, lfs_mktag(LFS_TYPE_REG_, i, 8), &entry); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -4027,7 +4332,7 @@ static int lfs_pred_(lfs_t *lfs, const lfs_block_t pair[2], lfs_dir_t_ *pdir) { return true; } - int err = lfs_dir_fetch_(lfs, pdir, pdir->tail, NULL, NULL); + int err = lfs_dir_fetch_(lfs, pdir, pdir->tail); if (err) { return err; } @@ -4068,15 +4373,14 @@ static int lfs_parent_(lfs_t *lfs, const lfs_block_t pair[2], // iterate over all directory directory entries while (!lfs_pairisnull(parent->tail)) { - int err = lfs_dir_fetch_(lfs, parent, parent->tail, NULL, NULL); + int err = lfs_dir_fetch_(lfs, parent, parent->tail); if (err) { return err; } for (int i = 0; i < parent->count; i++) { - entry->tag = lfs_mktag(LFS_STRUCT_DIR_, i, 8); - entry->u.buffer = &entry->u; - int err = lfs_dir_get_(lfs, parent, 0x43dff000, entry); + int err = lfs_dir_getentry_(lfs, parent, + 0x43dff000, lfs_mktag(LFS_STRUCT_DIR_, i, 8), entry); if (err && err != LFS_ERR_RANGE) { if (err == LFS_ERR_NOENT) { continue; @@ -4132,23 +4436,22 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], static int lfs_moved_(lfs_t *lfs, const lfs_block_t pair[2]) { // skip superblock lfs_dir_t_ dir; - int err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}, - NULL, NULL); + int err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { return err; } // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { - int err = lfs_dir_fetch_(lfs, &dir, dir.tail, NULL, NULL); + int err = lfs_dir_fetch_(lfs, &dir, dir.tail); if (err) { return err; } for (int i = 0; i < dir.count; i++) { - lfs_entry_t_ entry = {lfs_mktag(LFS_STRUCT_DIR_, i, 8), - .u.buffer=&entry.u}; - int err = lfs_dir_get_(lfs, &dir, 0x43dff000, &entry); + lfs_entry_t_ entry; + int err = lfs_dir_getentry_(lfs, &dir, + 0x43dff000, lfs_mktag(LFS_STRUCT_DIR_, i, 8), &entry); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -4156,8 +4459,8 @@ static int lfs_moved_(lfs_t *lfs, const lfs_block_t pair[2]) { return err; } - err = lfs_dir_get_(lfs, &dir, 0x7ffff000, &(lfs_entry_t_){ - lfs_mktag(LFS_TYPE_MOVE_, i, 0)}); + err = lfs_dir_get_(lfs, &dir, + 0x7ffff000, lfs_mktag(LFS_TYPE_MOVE_, i, 0), NULL); if (err != LFS_ERR_NOENT) { if (!err) { continue; @@ -4265,6 +4568,7 @@ static int lfs_relocate(lfs_t *lfs, return 0; } +// TODO use this in lfs_move? int lfs_deorphan_check(lfs_t *lfs, void *p, lfs_entry_t_ entry) { int16_t *id = p; @@ -4295,7 +4599,7 @@ int lfs_deorphan_(lfs_t *lfs) { // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { int16_t moveid = -1; - int err = lfs_dir_fetch_(lfs, &dir, dir.tail, + int err = lfs_dir_fetchwith_(lfs, &dir, dir.tail, lfs_deorphan_check, &moveid); if (err) { return err; @@ -4318,8 +4622,8 @@ int lfs_deorphan_(lfs_t *lfs) { pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; - err = lfs_dir_set_(lfs, &pdir, &(lfs_entry_t_){ - lfs_mktag(LFS_TYPE_TAIL_, 0x1ff, sizeof(pdir.tail)), + err = lfs_dir_commit_(lfs, &pdir, &(lfs_entry_t_){ + lfs_mktag(LFS_TYPE_SOFTTAIL_, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}, 1); if (err) { return err; @@ -4335,8 +4639,8 @@ int lfs_deorphan_(lfs_t *lfs) { pdir.tail[0] = entry.u.pair[0]; pdir.tail[1] = entry.u.pair[1]; - err = lfs_dir_set_(lfs, &pdir, &(lfs_entry_t_){ - lfs_mktag(LFS_TYPE_TAIL_, 0x1ff, sizeof(pdir.tail)), + err = lfs_dir_commit_(lfs, &pdir, &(lfs_entry_t_){ + lfs_mktag(LFS_TYPE_SOFTTAIL_, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}, 1); if (err) { return err; diff --git a/lfs.h b/lfs.h index 5e963f25..276650f1 100644 --- a/lfs.h +++ b/lfs.h @@ -106,32 +106,25 @@ enum lfs_type { LFS_STRUCT_INLINE = 0x30, LFS_STRUCT_MOVED = 0x80, - // file type + // file types LFS_TYPE_REG_ = 0x040, LFS_TYPE_DIR_ = 0x050, + // internally used types LFS_TYPE_NAME_ = 0x010, LFS_TYPE_MOVE_ = 0x080, LFS_TYPE_DROP_ = 0x090, - LFS_TYPE_SUPERBLOCK_ = 0x0c8, - LFS_TYPE_TAIL_ = 0x0d0, + LFS_TYPE_SUPERBLOCK_ = 0x0a0, + LFS_TYPE_SOFTTAIL_ = 0x0c0, + LFS_TYPE_HARDTAIL_ = 0x0d0, LFS_TYPE_CRC_ = 0x0e0, // on disk structure LFS_STRUCT_ATTR_ = 0x100, LFS_STRUCT_INLINE_ = 0x000, - LFS_STRUCT_CTZ_ = 0x00c, + LFS_STRUCT_CTZ_ = 0x004, LFS_STRUCT_DIR_ = 0x008, - -// LFS_TYPE_DIR_ = 0x002, -// LFS_TYPE_SUPERBLOCK_ = 0xff2, - -// LFS_MASK_ID_ = 0xff000000, -// LFS_MASK_TYPE_ = 0x00fff000, -// LFS_MASK_ATTR_ = 0x00ff0000, -// LFS_MASK_STRUCT_ = 0x0000f000, -// LFS_MASK_SIZE_ = 0x00000fff, }; // File open flags @@ -345,6 +338,24 @@ typedef struct lfs_file { int attrcount; } lfs_file_t; +typedef struct lfs_file_ { + struct lfs_file *next; + lfs_block_t pair[2]; + uint16_t id; + + lfs_block_t head; + lfs_size_t size; + + uint32_t flags; + lfs_off_t pos; + lfs_block_t block; + lfs_off_t off; + lfs_cache_t cache; + + const struct lfs_attr *attrs; + int attrcount; +} lfs_file_t_; + typedef struct lfs_dir { struct lfs_dir *next; lfs_block_t pair[2]; From 0695862b38402512904ee2a15f938ffd87bfe387 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 22 May 2018 23:57:19 -0500 Subject: [PATCH 033/139] Completed transition of files with journalling metadata This was the simpler part of transitioning since file operations only interact with metadata at sync time. Also switched from array to linked-list of entries. --- lfs.c | 908 +++++++++++++++++++--------------------------------------- lfs.h | 29 +- 2 files changed, 303 insertions(+), 634 deletions(-) diff --git a/lfs.c b/lfs.c index 6e236ebb..8026a888 100644 --- a/lfs.c +++ b/lfs.c @@ -1064,15 +1064,9 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -struct lfs_dir_committer { - const lfs_entry_t_ *regions; - int count; -}; - -int lfs_dir_committer(lfs_t *lfs, void *p, struct lfs_commit *commit) { - struct lfs_dir_committer *set = p; - for (int i = 0; i < set->count; i++) { - int err = lfs_commit_commit(lfs, commit, set->regions[i]); +int lfs_dir_commitregions(lfs_t *lfs, void *p, struct lfs_commit *commit) { + for (lfs_entrylist_t *regions = p; regions; regions = regions->next) { + int err = lfs_commit_commit(lfs, commit, regions->e); if (err) { return err; } @@ -1082,9 +1076,8 @@ int lfs_dir_committer(lfs_t *lfs, void *p, struct lfs_commit *commit) { } /*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, - const lfs_entry_t_ *regions, int count) { - return lfs_dir_commitwith_(lfs, dir, lfs_dir_committer, - &(struct lfs_dir_committer){regions, count}); + const lfs_entrylist_t *regions) { + return lfs_dir_commitwith_(lfs, dir, lfs_dir_commitregions, regions); } /*static*/ int lfs_dir_add(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t *id) { @@ -1096,8 +1089,8 @@ int lfs_dir_committer(lfs_t *lfs, void *p, struct lfs_commit *commit) { /*static*/ int lfs_dir_drop(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id) { dir->count -= 1; // TODO compact during traverse when compacting? - return lfs_dir_commit_(lfs, dir, (lfs_entry_t_[]){{ - lfs_mktag(LFS_TYPE_DROP_, id, 0)}}, 1); + return lfs_dir_commit_(lfs, dir, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_DROP_, id, 0)}}); } struct lfs_dir_getter { @@ -1710,156 +1703,157 @@ static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions, int count) { - lfs_ssize_t diff = 0; - for (int i = 0; i < count; i++) { - diff += regions[i].newsize; - diff -= regions[i].oldsize; - } - - lfs_size_t oldsize = entry->size; - if (entry->off == 0) { - entry->off = (0x7fffffff & dir->d.size) - 4; - } - - if ((0x7fffffff & dir->d.size) + diff > lfs->cfg->block_size) { - lfs_dir_t olddir = *dir; - lfs_off_t oldoff = entry->off; - - if (oldsize) { - // mark as moving - uint8_t type; - int err = lfs_dir_get(lfs, &olddir, oldoff, &type, 1); - if (err) { - return err; - } - - type |= LFS_STRUCT_MOVED; - err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ - {LFS_FROM_MEM, oldoff, 1, &type, 1}}, 1); - if (err) { - return err; - } - } - - lfs_dir_t pdir = olddir; - - // find available block or create a new one - while ((0x7fffffff & dir->d.size) + oldsize + diff - > lfs->cfg->block_size) { - // we need to allocate a new dir block - if (!(0x80000000 & dir->d.size)) { - pdir = *dir; - int err = lfs_dir_alloc(lfs, dir); - if (err) { - return err; - } - - dir->d.tail[0] = pdir.d.tail[0]; - dir->d.tail[1] = pdir.d.tail[1]; - - break; - } - - int err = lfs_dir_fetch(lfs, dir, dir->d.tail); - if (err) { - return err; - } - } - - // writing out new entry - entry->off = dir->d.size - 4; - entry->size += diff; - int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {LFS_FROM_REGION, entry->off, 0, &(struct lfs_region_region){ - olddir.pair[0], oldoff, - regions, count}, entry->size}}, 1); - if (err) { - return err; - } - - // update pred dir, unless pred == old we can coalesce - if (!oldsize || lfs_paircmp(pdir.pair, olddir.pair) != 0) { - pdir.d.size |= 0x80000000; - pdir.d.tail[0] = dir->pair[0]; - pdir.d.tail[1] = dir->pair[1]; - - err = lfs_dir_commit(lfs, &pdir, NULL, 0); - if (err) { - return err; - } - } else if (oldsize) { - olddir.d.size |= 0x80000000; - olddir.d.tail[0] = dir->pair[0]; - olddir.d.tail[1] = dir->pair[1]; - } - - // remove old entry - if (oldsize) { - lfs_entry_t oldentry; - oldentry.off = oldoff; - err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, oldsize, NULL, 0}}, 1); - if (err) { - return err; - } - } - - goto shift; - } - - if ((0x7fffffff & dir->d.size) + diff == sizeof(dir->d)+4) { - lfs_dir_t pdir; - int res = lfs_pred(lfs, dir->pair, &pdir); - if (res < 0) { - return res; - } - - if (pdir.d.size & 0x80000000) { - pdir.d.size &= dir->d.size | 0x7fffffff; - pdir.d.tail[0] = dir->d.tail[0]; - pdir.d.tail[1] = dir->d.tail[1]; - int err = lfs_dir_commit(lfs, &pdir, NULL, 0); - if (err) { - return err; - } - goto shift; - } - } - - for (int i = 0; i < count; i++) { - regions[i].oldoff += entry->off; - } - - int err = lfs_dir_commit(lfs, dir, regions, count); - if (err) { - return err; - } - - entry->size += diff; - -shift: - // shift over any files/directories that are affected - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->pairoff == entry->off && entry->size == 0) { - f->pair[0] = 0xffffffff; - f->pair[1] = 0xffffffff; - } else if (f->pairoff > entry->off) { - f->pairoff += diff; - } - } - } - - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > entry->off) { - d->off += diff; - d->pos += diff; - } - } - } - - return 0; + return -999; +// lfs_ssize_t diff = 0; +// for (int i = 0; i < count; i++) { +// diff += regions[i].newsize; +// diff -= regions[i].oldsize; +// } +// +// lfs_size_t oldsize = entry->size; +// if (entry->off == 0) { +// entry->off = (0x7fffffff & dir->d.size) - 4; +// } +// +// if ((0x7fffffff & dir->d.size) + diff > lfs->cfg->block_size) { +// lfs_dir_t olddir = *dir; +// lfs_off_t oldoff = entry->off; +// +// if (oldsize) { +// // mark as moving +// uint8_t type; +// int err = lfs_dir_get(lfs, &olddir, oldoff, &type, 1); +// if (err) { +// return err; +// } +// +// type |= LFS_STRUCT_MOVED; +// err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ +// {LFS_FROM_MEM, oldoff, 1, &type, 1}}, 1); +// if (err) { +// return err; +// } +// } +// +// lfs_dir_t pdir = olddir; +// +// // find available block or create a new one +// while ((0x7fffffff & dir->d.size) + oldsize + diff +// > lfs->cfg->block_size) { +// // we need to allocate a new dir block +// if (!(0x80000000 & dir->d.size)) { +// pdir = *dir; +// int err = lfs_dir_alloc(lfs, dir); +// if (err) { +// return err; +// } +// +// dir->d.tail[0] = pdir.d.tail[0]; +// dir->d.tail[1] = pdir.d.tail[1]; +// +// break; +// } +// +// int err = lfs_dir_fetch(lfs, dir, dir->d.tail); +// if (err) { +// return err; +// } +// } +// +// // writing out new entry +// entry->off = dir->d.size - 4; +// entry->size += diff; +// int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ +// {LFS_FROM_REGION, entry->off, 0, &(struct lfs_region_region){ +// olddir.pair[0], oldoff, +// regions, count}, entry->size}}, 1); +// if (err) { +// return err; +// } +// +// // update pred dir, unless pred == old we can coalesce +// if (!oldsize || lfs_paircmp(pdir.pair, olddir.pair) != 0) { +// pdir.d.size |= 0x80000000; +// pdir.d.tail[0] = dir->pair[0]; +// pdir.d.tail[1] = dir->pair[1]; +// +// err = lfs_dir_commit(lfs, &pdir, NULL, 0); +// if (err) { +// return err; +// } +// } else if (oldsize) { +// olddir.d.size |= 0x80000000; +// olddir.d.tail[0] = dir->pair[0]; +// olddir.d.tail[1] = dir->pair[1]; +// } +// +// // remove old entry +// if (oldsize) { +// lfs_entry_t oldentry; +// oldentry.off = oldoff; +// err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, oldsize, NULL, 0}}, 1); +// if (err) { +// return err; +// } +// } +// +// goto shift; +// } +// +// if ((0x7fffffff & dir->d.size) + diff == sizeof(dir->d)+4) { +// lfs_dir_t pdir; +// int res = lfs_pred(lfs, dir->pair, &pdir); +// if (res < 0) { +// return res; +// } +// +// if (pdir.d.size & 0x80000000) { +// pdir.d.size &= dir->d.size | 0x7fffffff; +// pdir.d.tail[0] = dir->d.tail[0]; +// pdir.d.tail[1] = dir->d.tail[1]; +// int err = lfs_dir_commit(lfs, &pdir, NULL, 0); +// if (err) { +// return err; +// } +// goto shift; +// } +// } +// +// for (int i = 0; i < count; i++) { +// regions[i].oldoff += entry->off; +// } +// +// int err = lfs_dir_commit(lfs, dir, regions, count); +// if (err) { +// return err; +// } +// +// entry->size += diff; +// +//shift: +// // shift over any files/directories that are affected +// for (lfs_file_t *f = lfs->files; f; f = f->next) { +// if (lfs_paircmp(f->pair, dir->pair) == 0) { +// if (f->pairoff == entry->off && entry->size == 0) { +// f->pair[0] = 0xffffffff; +// f->pair[1] = 0xffffffff; +// } else if (f->pairoff > entry->off) { +// f->pairoff += diff; +// } +// } +// } +// +// for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { +// if (lfs_paircmp(d->pair, dir->pair) == 0) { +// if (d->off > entry->off) { +// d->off += diff; +// d->pos += diff; +// } +// } +// } +// +// return 0; } static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { @@ -2161,7 +2155,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } - err = lfs_dir_commit_(lfs, &dir, NULL, 0); + err = lfs_dir_commit_(lfs, &dir, NULL); if (err) { return err; } @@ -2173,10 +2167,11 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } - err = lfs_dir_commit_(lfs, &cwd, (lfs_entry_t_[]){ - {lfs_mktag(LFS_TYPE_NAME_, id, nlen), .u.buffer=(void*)path}, - {lfs_mktag(LFS_TYPE_DIR_ | LFS_STRUCT_DIR_, id, - sizeof(dir.pair)), .u.buffer=dir.pair}}, 2); + err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_NAME_, id, nlen), + .u.buffer=(void*)path}, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_DIR_ | LFS_STRUCT_DIR_, id, sizeof(dir.pair)), + .u.buffer=dir.pair}}}); // TODO need ack here? lfs_alloc_ack(lfs); @@ -2697,7 +2692,7 @@ static int lfs_ctz_traverse(lfs_t *lfs, /// Top level file operations /// -int lfs_file_open_(lfs_t *lfs, lfs_file_t_ *file, +int lfs_file_open(lfs_t *lfs, lfs_file_t *file, const char *path, int flags) { // deorphan if we haven't yet, needed at most once after poweron if ((flags & 3) != LFS_O_RDONLY && !lfs->deorphaned) { @@ -2733,135 +2728,29 @@ int lfs_file_open_(lfs_t *lfs, lfs_file_t_ *file, return err; } - err = lfs_dir_commit_(lfs, &cwd, (lfs_entry_t_[]){ - {lfs_mktag(LFS_TYPE_NAME_, id, nlen), .u.buffer=(void*)path}, - {lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_INLINE_, id, 0)}}, 2); + err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_NAME_, id, nlen), + .u.buffer=(void*)path}, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_INLINE_, id, 0)}}}); if (err) { return err; } + + // TODO, used later, clean this up? + entry.tag = lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_INLINE_, id, 0); } else if (lfs_tag_subtype(entry.tag) != LFS_TYPE_REG_) { return LFS_ERR_ISDIR; } else if (flags & LFS_O_EXCL) { return LFS_ERR_EXIST; } - // allocate buffer if needed - file->cache.block = 0xffffffff; - if (lfs->cfg->file_buffer) { - file->cache.buffer = lfs->cfg->file_buffer; - } else if ((file->flags & 3) == LFS_O_RDONLY) { - file->cache.buffer = lfs_malloc(lfs->cfg->read_size); - if (!file->cache.buffer) { - return LFS_ERR_NOMEM; - } - } else { - file->cache.buffer = lfs_malloc(lfs->cfg->prog_size); - if (!file->cache.buffer) { - return LFS_ERR_NOMEM; - } - } - // setup file struct file->pair[0] = cwd.pair[0]; file->pair[1] = cwd.pair[1]; file->id = lfs_tag_id(entry.tag); file->flags = flags; file->pos = 0; - - if (lfs_tag_struct(entry.tag) == LFS_STRUCT_INLINE_) { - // load inline files - file->head = 0xfffffffe; - file->size = lfs_tag_size(entry.tag); - file->flags |= LFS_F_INLINE; - file->cache.block = file->head; - file->cache.off = 0; - err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, - file->cache.buffer, file->size); - if (err) { - lfs_free(file->cache.buffer); - return err; - } - } else { - // use ctz list from entry - err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, - &entry.u, sizeof(entry.u)); - // TODO move to disk struct directly? - file->head = entry.u.ctz.head; - file->size = entry.u.ctz.size; - } - - // truncate if requested - if (flags & LFS_O_TRUNC) { - if (file->size != 0) { - file->flags |= LFS_F_DIRTY; - } - - file->head = 0xfffffffe; - file->size = 0; - file->flags |= LFS_F_INLINE; - file->cache.block = file->head; - file->cache.off = 0; - } - - // add to list of files - file->next = lfs->files; - lfs->files = file; - - return 0; -} - -int lfs_file_open(lfs_t *lfs, lfs_file_t *file, - const char *path, int flags) { - // deorphan if we haven't yet, needed at most once after poweron - if ((flags & 3) != LFS_O_RDONLY && !lfs->deorphaned) { - int err = lfs_deorphan(lfs); - if (err) { - return err; - } - } - - // allocate entry for file if it doesn't exist - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t entry; - err = lfs_dir_find(lfs, &cwd, &entry, &path); - if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { - return err; - } - - if (err == LFS_ERR_NOENT) { - if (!(flags & LFS_O_CREAT)) { - return LFS_ERR_NOENT; - } - - // check that name fits - lfs_size_t nlen = strlen(path); - if (nlen > lfs->name_size) { - return LFS_ERR_NAMETOOLONG; - } - - // create entry to remember name - entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = 0; - entry.d.alen = 0; - entry.d.nlen = nlen; - entry.size = 0; - - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); - if (err) { - return err; - } - } else if ((0xf & entry.d.type) == LFS_TYPE_DIR) { - return LFS_ERR_ISDIR; - } else if (flags & LFS_O_EXCL) { - return LFS_ERR_EXIST; - } + file->attrs = NULL; // allocate buffer if needed file->cache.block = 0xffffffff; @@ -2879,27 +2768,14 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } - // setup file struct - file->pair[0] = cwd.pair[0]; - file->pair[1] = cwd.pair[1]; - file->pairoff = entry.off; - file->flags = flags; - file->pos = 0; - - // calculate max inline size based on the size of the entry - file->inline_size = lfs_min(lfs->inline_size, - lfs->cfg->block_size - (sizeof(cwd.d)+4) - - (lfs_entry_size(&entry) - lfs_entry_elen(&entry))); - - if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) { + if (lfs_tag_struct(entry.tag) == LFS_STRUCT_INLINE_) { // load inline files file->head = 0xfffffffe; - file->size = lfs_entry_elen(&entry); + file->size = lfs_tag_size(entry.tag); file->flags |= LFS_F_INLINE; file->cache.block = file->head; file->cache.off = 0; - err = lfs_dir_get(lfs, &cwd, - entry.off + 4, + err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, file->cache.buffer, file->size); if (err) { lfs_free(file->cache.buffer); @@ -2907,8 +2783,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } else { // use ctz list from entry - file->head = entry.d.u.file.head; - file->size = entry.d.u.file.size; + err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, + &file->head, 2*sizeof(uint32_t)); } // truncate if requested @@ -2931,7 +2807,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return 0; } -int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { +int lfs_file_close_(lfs_t *lfs, lfs_file_t *file) { int err = lfs_file_sync(lfs, file); // remove from list of files @@ -2950,50 +2826,23 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { return err; } -static int lfs_file_relocate_(lfs_t *lfs, lfs_file_t_ *file) { -relocate:; - // just relocate what exists into new block - lfs_block_t nblock; - int err = lfs_alloc(lfs, &nblock); - if (err) { - return err; - } +int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { + int err = lfs_file_sync(lfs, file); - err = lfs_bd_erase(lfs, nblock); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + // remove from list of files + for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { + if (*p == file) { + *p = file->next; + break; } - return err; } - // either read from dirty cache or disk - for (lfs_off_t i = 0; i < file->off; i++) { - uint8_t data; - err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, - file->block, i, &data, 1); - if (err) { - return err; - } - - err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, - nblock, i, &data, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } + // clean up memory + if (!lfs->cfg->file_buffer) { + lfs_free(file->cache.buffer); } - // copy over new state of file - memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); - file->cache.block = lfs->pcache.block; - file->cache.off = lfs->pcache.off; - lfs->pcache.block = 0xffffffff; - - file->block = nblock; - return 0; + return err; } static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { @@ -3042,80 +2891,6 @@ relocate:; return 0; } -static int lfs_file_flush_(lfs_t *lfs, lfs_file_t_ *file) { - if (file->flags & LFS_F_READING) { - file->flags &= ~LFS_F_READING; - } - - if (file->flags & LFS_F_WRITING) { - lfs_off_t pos = file->pos; - - if (!(file->flags & LFS_F_INLINE)) { - // copy over anything after current branch - lfs_file_t_ orig = { - .head = file->head, - .size = file->size, - .flags = LFS_O_RDONLY, - .pos = file->pos, - .cache = lfs->rcache, - }; - lfs->rcache.block = 0xffffffff; - - while (file->pos < file->size) { - // copy over a byte at a time, leave it up to caching - // to make this efficient - uint8_t data; - lfs_ssize_t res = lfs_file_read(lfs, &orig, &data, 1); - if (res < 0) { - return res; - } - - res = lfs_file_write(lfs, file, &data, 1); - if (res < 0) { - return res; - } - - // keep our reference to the rcache in sync - if (lfs->rcache.block != 0xffffffff) { - orig.cache.block = 0xffffffff; - lfs->rcache.block = 0xffffffff; - } - } - - // write out what we have - while (true) { - int err = lfs_cache_flush(lfs, &file->cache, &lfs->rcache); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - break; -relocate: - LFS_DEBUG("Bad block at %d", file->block); - err = lfs_file_relocate_(lfs, file); - if (err) { - return err; - } - } - } else { - file->size = lfs_max(file->pos, file->size); - } - - // actual file updates - file->head = file->block; - file->size = file->pos; - file->flags &= ~LFS_F_WRITING; - file->flags |= LFS_F_DIRTY; - - file->pos = pos; - } - - return 0; -} - static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { if (file->flags & LFS_F_READING) { file->flags &= ~LFS_F_READING; @@ -3190,8 +2965,8 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { return 0; } -int lfs_file_sync_(lfs_t *lfs, lfs_file_t_ *file) { - int err = lfs_file_flush_(lfs, file); +int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { + int err = lfs_file_flush(lfs, file); if (err) { return err; } @@ -3208,114 +2983,23 @@ int lfs_file_sync_(lfs_t *lfs, lfs_file_t_ *file) { return err; } - // either update the references or inline the whole file - // TODO handle attributes -// if (!(file->flags & LFS_F_INLINE)) { -// entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; -// entry.d.u.file.head = file->head; -// entry.d.u.file.size = file->size; -// -// lfs_entry_tole32(&entry.d); -// buffer = (const uint8_t *)&entry.d + 4; -// size = sizeof(entry.d) - 4; -// } else { -// entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; -// -// buffer = file->cache.buffer; -// size = file->size; -// } -// -// // get new alen from disk -// lfs_ssize_t newalen = lfs_dir_checkattrs(lfs, &cwd, &entry, -// file->attrs, file->attrcount); -// if (newalen < 0) { -// return newalen; -// } -// -// entry.d.elen = size & 0xff; -// entry.d.alen = (newalen & 0x3f) | ((size >> 2) & 0xc0); -// -// // write out update -// err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, 4, &entry.d, 4}, -// {LFS_FROM_MEM, 4, oldelen, buffer, size}, -// {LFS_FROM_ATTRS, 4+oldelen, oldalen, -// &(struct lfs_region_attrs){file->attrs, file->attrcount}, -// newalen}}, 3); -// if (err) { -// return err; -// } - - file->flags &= ~LFS_F_DIRTY; - } - - return 0; -} - -int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { - int err = lfs_file_flush(lfs, file); - if (err) { - return err; - } - - if ((file->flags & LFS_F_DIRTY) && - !(file->flags & LFS_F_ERRED) && - !lfs_pairisnull(file->pair)) { - // update dir entry - lfs_dir_t cwd; - err = lfs_dir_fetch(lfs, &cwd, file->pair); - if (err) { - return err; - } - - lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); - if (err) { - return err; - } - entry.size = lfs_entry_size(&entry); - - LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - lfs_size_t oldelen = lfs_entry_elen(&entry); - lfs_size_t oldalen = lfs_entry_alen(&entry); - const void *buffer; - lfs_size_t size; - // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { - entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; - entry.d.u.file.head = file->head; - entry.d.u.file.size = file->size; - - lfs_entry_tole32(&entry.d); - buffer = (const uint8_t *)&entry.d + 4; - size = sizeof(entry.d) - 4; + int err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_CTZ_, file->id, + 2*sizeof(uint32_t)), .u.buffer=&file->head}, + file->attrs}); + if (err) { + return err; + } } else { - entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - - buffer = file->cache.buffer; - size = file->size; - } - - // get new alen from disk - lfs_ssize_t newalen = lfs_dir_checkattrs(lfs, &cwd, &entry, - file->attrs, file->attrcount); - if (newalen < 0) { - return newalen; - } - - entry.d.elen = size & 0xff; - entry.d.alen = (newalen & 0x3f) | ((size >> 2) & 0xc0); - - // write out update - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 4, &entry.d, 4}, - {LFS_FROM_MEM, 4, oldelen, buffer, size}, - {LFS_FROM_ATTRS, 4+oldelen, oldalen, - &(struct lfs_region_attrs){file->attrs, file->attrcount}, - newalen}}, 3); - if (err) { - return err; + int err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_INLINE_, file->id, + file->size), .u.buffer=file->cache.buffer}, + file->attrs}); + if (err) { + return err; + } } file->flags &= ~LFS_F_DIRTY; @@ -3420,7 +3104,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } if ((file->flags & LFS_F_INLINE) && - file->pos + nsize >= file->inline_size) { + file->pos + nsize >= lfs->cfg->inline_size) { // inline file doesn't fit anymore file->block = 0xfffffffe; file->off = file->pos; @@ -3609,86 +3293,86 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { } } -int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, - const struct lfs_attr *attrs, int count) { - // set to null in case we can't find the attrs (missing file?) - for (int j = 0; j < count; j++) { - memset(attrs[j].buffer, 0, attrs[j].size); - } - - // load from disk if we haven't already been deleted - if (!lfs_pairisnull(file->pair)) { - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, file->pair); - if (err) { - return err; - } - - lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); - if (err) { - return err; - } - entry.size = lfs_entry_size(&entry); - - err = lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); - if (err) { - return err; - } - } - - // override an attrs we have stored locally - for (int i = 0; i < file->attrcount; i++) { - for (int j = 0; j < count; j++) { - if (attrs[j].type == file->attrs[i].type) { - if (attrs[j].size < file->attrs[i].size) { - return LFS_ERR_RANGE; - } - - memset(attrs[j].buffer, 0, attrs[j].size); - memcpy(attrs[j].buffer, - file->attrs[i].buffer, file->attrs[i].size); - } - } - } - - return 0; -} - -int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, - const struct lfs_attr *attrs, int count) { - if ((file->flags & 3) == LFS_O_RDONLY) { - return LFS_ERR_BADF; - } - - // at least make sure attributes fit - if (!lfs_pairisnull(file->pair)) { - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, file->pair); - if (err) { - return err; - } - - lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); - if (err) { - return err; - } - entry.size = lfs_entry_size(&entry); - - lfs_ssize_t res = lfs_dir_checkattrs(lfs, &cwd, &entry, attrs, count); - if (res < 0) { - return res; - } - } - - // just tack to the file, will be written at sync time - file->attrs = attrs; - file->attrcount = count; - file->flags |= LFS_F_DIRTY; - - return 0; -} +//int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, +// const struct lfs_attr *attrs, int count) { +// // set to null in case we can't find the attrs (missing file?) +// for (int j = 0; j < count; j++) { +// memset(attrs[j].buffer, 0, attrs[j].size); +// } +// +// // load from disk if we haven't already been deleted +// if (!lfs_pairisnull(file->pair)) { +// lfs_dir_t cwd; +// int err = lfs_dir_fetch(lfs, &cwd, file->pair); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry = {.off = file->pairoff}; +// err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); +// if (err) { +// return err; +// } +// entry.size = lfs_entry_size(&entry); +// +// err = lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); +// if (err) { +// return err; +// } +// } +// +// // override an attrs we have stored locally +// for (int i = 0; i < file->attrcount; i++) { +// for (int j = 0; j < count; j++) { +// if (attrs[j].type == file->attrs[i].type) { +// if (attrs[j].size < file->attrs[i].size) { +// return LFS_ERR_RANGE; +// } +// +// memset(attrs[j].buffer, 0, attrs[j].size); +// memcpy(attrs[j].buffer, +// file->attrs[i].buffer, file->attrs[i].size); +// } +// } +// } +// +// return 0; +//} + +//int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, +// const struct lfs_attr *attrs, int count) { +// if ((file->flags & 3) == LFS_O_RDONLY) { +// return LFS_ERR_BADF; +// } +// +// // at least make sure attributes fit +// if (!lfs_pairisnull(file->pair)) { +// lfs_dir_t cwd; +// int err = lfs_dir_fetch(lfs, &cwd, file->pair); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry = {.off = file->pairoff}; +// err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); +// if (err) { +// return err; +// } +// entry.size = lfs_entry_size(&entry); +// +// lfs_ssize_t res = lfs_dir_checkattrs(lfs, &cwd, &entry, attrs, count); +// if (res < 0) { +// return res; +// } +// } +// +// // just tack to the file, will be written at sync time +// file->attrs = attrs; +// file->attrcount = count; +// file->flags |= LFS_F_DIRTY; +// +// return 0; +//} /// General fs operations /// @@ -4057,7 +3741,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_commit_(lfs, &root, NULL, 0); + err = lfs_dir_commit_(lfs, &root, NULL); if (err) { return err; } @@ -4082,9 +3766,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; dir.count += 1; - err = lfs_dir_commit_(lfs, &dir, (lfs_entry_t_[]){ + err = lfs_dir_commit_(lfs, &dir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SUPERBLOCK_ | LFS_STRUCT_DIR_, 0, - sizeof(superblock)), .u.buffer=&superblock}}, 1); + sizeof(superblock)), .u.buffer=&superblock}}); if (err) { return err; } @@ -4622,9 +4306,9 @@ int lfs_deorphan_(lfs_t *lfs) { pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit_(lfs, &pdir, &(lfs_entry_t_){ - lfs_mktag(LFS_TYPE_SOFTTAIL_, 0x1ff, sizeof(pdir.tail)), - .u.buffer=pdir.tail}, 1); + err = lfs_dir_commit_(lfs, &pdir, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL_, 0x1ff, + sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { return err; } @@ -4639,9 +4323,9 @@ int lfs_deorphan_(lfs_t *lfs) { pdir.tail[0] = entry.u.pair[0]; pdir.tail[1] = entry.u.pair[1]; - err = lfs_dir_commit_(lfs, &pdir, &(lfs_entry_t_){ - lfs_mktag(LFS_TYPE_SOFTTAIL_, 0x1ff, sizeof(pdir.tail)), - .u.buffer=pdir.tail}, 1); + err = lfs_dir_commit_(lfs, &pdir, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL_, 0x1ff, + sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { return err; } diff --git a/lfs.h b/lfs.h index 276650f1..cb14eb3a 100644 --- a/lfs.h +++ b/lfs.h @@ -306,6 +306,11 @@ typedef struct lfs_entry_ { } u; } lfs_entry_t_; +typedef struct lfs_entry_list_ { + lfs_entry_t_ e; + struct lfs_entry_list_ *next; +} lfs_entrylist_t; + typedef struct lfs_entry_attr { struct lfs_disk_entry_attr { uint8_t type; @@ -319,25 +324,6 @@ typedef struct lfs_cache { uint8_t *buffer; } lfs_cache_t; -typedef struct lfs_file { - struct lfs_file *next; - lfs_block_t pair[2]; - lfs_off_t pairoff; - - lfs_block_t head; - lfs_size_t size; - - uint32_t flags; - lfs_size_t inline_size; - lfs_off_t pos; - lfs_block_t block; - lfs_off_t off; - lfs_cache_t cache; - - const struct lfs_attr *attrs; - int attrcount; -} lfs_file_t; - typedef struct lfs_file_ { struct lfs_file *next; lfs_block_t pair[2]; @@ -352,9 +338,8 @@ typedef struct lfs_file_ { lfs_off_t off; lfs_cache_t cache; - const struct lfs_attr *attrs; - int attrcount; -} lfs_file_t_; + lfs_entrylist_t *attrs; +} lfs_file_t; typedef struct lfs_dir { struct lfs_dir *next; From a3c67d9697cd7f1a804620dcc7d8bfa6012f5072 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 25 May 2018 19:04:01 -0500 Subject: [PATCH 034/139] Reorganized the internal operations to make more sense Also refactored lfs_dir_compact a bit, adding begin and end as arguments since they simplify a bit of the logic and can be found out much easier earlier in the commit logic. Also changed add -> append and drop -> delete and cleaned up some of the logic around there. --- lfs.c | 1421 ++++++++++++++++++++++++++++++++------------------------- lfs.h | 3 +- 2 files changed, 798 insertions(+), 626 deletions(-) diff --git a/lfs.c b/lfs.c index 8026a888..b35cba03 100644 --- a/lfs.c +++ b/lfs.c @@ -467,14 +467,15 @@ struct lfs_commit { uint32_t crc; struct { - int16_t id; - uint16_t type; - } compact; + uint16_t begin; + uint16_t end; + } filter; }; static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), void *data) { + // TODO duplicate this? move it to dir? // iterate over dir block backwards (for faster lookups) lfs_block_t block = commit->block; lfs_off_t off = commit->off; @@ -505,38 +506,47 @@ static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t_ entry) { - struct lfs_commit *commit = p; - if (lfs_tag_id(entry.tag) != commit->compact.id) { - return 1; - } else if (lfs_tag_type(entry.tag) == commit->compact.type) { - return 2; - } - - return 0; -} - +//static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t_ entry) { +// struct lfs_commit *commit = p; +// if (lfs_tag_id(entry.tag) != commit->compact.id) { +// return 1; +// } else if (lfs_tag_type(entry.tag) == commit->compact.type) { +// return 2; +// } +// +// return 0; +//} +// static int lfs_commit_commit(lfs_t *lfs, struct lfs_commit *commit, lfs_entry_t_ entry) { - // request for compaction? - if (commit->compact.id >= 0) { - if (lfs_tag_id(entry.tag) != commit->compact.id) { - // ignore non-matching ids - return 0; - } - - commit->compact.type = lfs_tag_type(entry.tag); - int res = lfs_commit_traverse(lfs, commit, - lfs_commit_compactcheck, commit); - if (res < 0) { - return res; - } - - if (res == 2) { - // already committed - return 0; - } +// // request for compaction? +// if (commit->compact.id >= 0) { +// if (lfs_tag_id(entry.tag) != commit->compact.id) { +// // ignore non-matching ids +// return 0; +// } +// +// commit->compact.type = lfs_tag_type(entry.tag); +// int res = lfs_commit_traverse(lfs, commit, +// lfs_commit_compactcheck, commit); +// if (res < 0) { +// return res; +// } +// +// if (res == 2) { +// // already committed +// return 0; +// } +// } +// + // filter out ids + if (lfs_tag_id(entry.tag) != 0x1ff && ( + lfs_tag_id(entry.tag) < commit->filter.begin || + lfs_tag_id(entry.tag) >= commit->filter.end)) { + return 0; } + uint16_t id = lfs_tag_id(entry.tag) - commit->filter.begin; + entry.tag = (id << 12) | (entry.tag & 0xffe00fff); // check if we fit lfs_size_t size = lfs_tag_size(entry.tag); @@ -637,8 +647,97 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return 0; } +// committer for regions +static int lfs_commit_regions(lfs_t *lfs, void *p, + struct lfs_commit *commit) { + for (lfs_entrylist_t *regions = p; regions; regions = regions->next) { + int err = lfs_commit_commit(lfs, commit, regions->e); + if (err) { + return err; + } + } + + return 0; +} + +// TODO redeclare +static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), + void *data); + +// committer for moves +struct lfs_commit_move { + lfs_dir_t_ *dir; + struct { + uint16_t from; + uint16_t to; + } id; + uint16_t type; + + struct lfs_commit *commit; +}; + +static int lfs_commit_movecheck(lfs_t *lfs, void *p, lfs_entry_t_ entry) { + struct lfs_commit_move *move = p; + + if (lfs_tag_id(entry.tag) != move->id.from) { + return 1; + } else if (lfs_tag_type(entry.tag) == move->type) { + return 2; + } + + return 0; +} + +static int lfs_commit_moveiter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { + struct lfs_commit_move *move = p; + + if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE_) { + if (lfs_tag_id(entry.tag) == move->id.from) { + // we done, bail + return 1; + } else if (lfs_tag_id(entry.tag) < move->id.from) { + // something was deleted, we need to move around it + move->id.from += 1; + } + } + + if (lfs_tag_id(entry.tag) != move->id.from) { + // ignore non-matching ids + return 0; + } + + move->type = lfs_tag_type(entry.tag); + int res = lfs_commit_traverse(lfs, move->commit, + lfs_commit_movecheck, move); + if (res < 0) { + return res; + } + + if (res == 2) { + // already committed + return 0; + } + + // update id and commit, as we are currently unique + entry.tag = (move->id.to << 12) | (entry.tag & 0xffe00fff); + return lfs_commit_commit(lfs, move->commit, entry); +} + +static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { + struct lfs_commit_move *move = p; + move->commit = commit; + + int err = lfs_dir_traverse_(lfs, move->dir, lfs_commit_moveiter, move); + if (err < 0) { + return err; + } + + return 0; +} + /*static*/ int lfs_dir_alloc_(lfs_t *lfs, lfs_dir_t_ *dir, - const lfs_block_t tail[2]) { + bool split, const lfs_block_t tail[2]) { // allocate pair of dir blocks (backwards, so we write to block 1 first) for (int i = 0; i < 2; i++) { int err = lfs_alloc(lfs, &dir->pair[(i+1)%2]); @@ -662,7 +761,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { dir->tail[0] = tail[0]; dir->tail[1] = tail[1]; dir->erased = false; - dir->split = false; + dir->split = split; // don't write out yet, let caller take care of that return 0; @@ -699,6 +798,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { dir->tail[1] = 0xffffffff; dir->count = 0; dir->split = false; + dir->moveid = -1; dir->rev = lfs_tole32(rev[0]); lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); @@ -758,11 +858,6 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return err; } - // TODO handle deletes and stuff - if (lfs_tag_id(tag) < 0x1ff && lfs_tag_id(tag) >= dir->count) { - dir->count = lfs_tag_id(tag)+1; - } - if (lfs_tag_type(tag) == LFS_TYPE_SOFTTAIL_ || lfs_tag_type(tag) == LFS_TYPE_HARDTAIL_) { dir->split = lfs_tag_type(tag) == LFS_TYPE_HARDTAIL_; @@ -771,13 +866,32 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { if (err) { return err; } - } else if (cb) { - err = cb(lfs, data, (lfs_entry_t_){ - (tag | 0x80000000), - .u.d.block=dir->pair[0], - .u.d.off=off+sizeof(tag)}); - if (err) { - return err; + } else if (lfs_tag_type(tag) == LFS_TYPE_MOVE_) { + dir->moveid = lfs_tag_id(tag); + } else { + // TODO handle deletes and stuff + if (lfs_tag_id(tag) < 0x1ff && + lfs_tag_id(tag) >= dir->count) { + dir->count = lfs_tag_id(tag)+1; + } + + if (lfs_tag_type(tag) == LFS_TYPE_DELETE_) { + dir->count -= 1; + if (lfs_tag_id(tag) == dir->moveid) { + dir->moveid = -1; + } else if (lfs_tag_id(tag) < dir->moveid) { + dir->moveid -= 1; + } + } + + if (cb) { + err = cb(lfs, data, (lfs_entry_t_){ + (tag | 0x80000000), + .u.d.block=dir->pair[0], + .u.d.off=off+sizeof(tag)}); + if (err) { + return err; + } } } } @@ -808,57 +922,57 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, cb, data); } -struct lfs_dir_mover { - // traversal things - lfs_dir_t_ *dir; - int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit); - void *data; - - // ids to iterate through - uint16_t begin; - uint16_t end; - uint16_t ack; -}; - -static int lfs_dir_mover_commit(lfs_t *lfs, void *p, - lfs_entry_t_ entry) { - return lfs_commit_commit(lfs, p, entry); -} - -int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { - struct lfs_dir_mover *mover = p; - for (int i = mover->begin; i < mover->end; i++) { - // tell the committer to check for duplicates - uint16_t old = commit->compact.id; - if (commit->compact.id < 0) { - commit->compact.id = i; - } - - // commit pending commits - int err = mover->cb(lfs, mover->data, commit); - if (err) { - commit->compact.id = old; - return err; - } - - // iterate over on-disk regions - err = lfs_dir_traverse_(lfs, mover->dir, - lfs_dir_mover_commit, commit); - if (err) { - commit->compact.id = old; - return err; - } - - mover->ack = i; - commit->compact.id = old; - } - - return 0; -} +//struct lfs_dir_commitmove { +// // traversal things +// lfs_dir_t_ *dir; +// int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit); +// void *data; +// +// // ids to iterate through +// uint16_t begin; +// uint16_t end; +// uint16_t ack; +//}; +// +//static int lfs_dir_commitmove_commit(lfs_t *lfs, void *p, +// lfs_entry_t_ entry) { +// return lfs_commit_commit(lfs, p, entry); +//} +// +//int lfs_dir_commitmove(lfs_t *lfs, void *p, struct lfs_commit *commit) { +// struct lfs_dir_commitmove *move = p; +// for (int i = move->begin; i < move->end; i++) { +// // tell the committer to check for duplicates +// uint16_t old = commit->compact.id; +// if (commit->compact.id < 0) { +// commit->compact.id = i; +// } +// +// // commit pending commits +// int err = move->cb(lfs, move->data, commit); +// if (err) { +// commit->compact.id = old; +// return err; +// } +// +// // iterate over on-disk regions +// err = lfs_dir_traverse_(lfs, move->dir, +// lfs_dir_commitmove_commit, commit); +// if (err) { +// commit->compact.id = old; +// return err; +// } +// +// move->ack = i; +// commit->compact.id = old; +// } +// +// return 0; +//} /*static*/ int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t_ *dir, int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), - void *data) { + void *data, lfs_dir_t_ *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; bool relocated = false; @@ -867,16 +981,9 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { dir->rev += 1; while (true) { - // setup mover - struct lfs_dir_mover mover = { - .dir = dir, - .cb = cb, - .data = data, - - .begin = 0, - .end = dir->count, - .ack = 0, - }; + // last complete id + int16_t ack = -1; + dir->count = end - begin; if (true) { // erase block to write to @@ -904,18 +1011,24 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { struct lfs_commit commit = { .block = dir->pair[1], .off = sizeof(dir->rev), - // leave space for tail pointer + + // space is complicated, we need room for tail, crc, + // and we keep cap at around half a block .begin = 0, - .end = lfs_min(lfs->cfg->block_size - 5*sizeof(uint32_t), + .end = lfs_min( lfs_alignup(lfs->cfg->block_size / 2, - lfs->cfg->prog_size)), + lfs->cfg->prog_size), + lfs->cfg->block_size - 5*sizeof(uint32_t)), .crc = crc, .ptag = 0, - .compact.id = -1, + + // filter out ids + .filter.begin = begin, + .filter.end = end, }; - // run compaction over mover - err = lfs_dir_mover(lfs, &mover, &commit); + // commit regions which can't fend for themselves + err = cb(lfs, data, &commit); if (err) { if (err == LFS_ERR_NOSPC) { goto split; @@ -925,9 +1038,27 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { return err; } + // move over other commits, leaving it up to lfs_commit_move to + // filter out duplicates, and the commit filtering to reassign ids + for (uint16_t id = begin; id < end; id++) { + err = lfs_commit_move(lfs, + &(struct lfs_commit_move){.dir=source, .id={id, id}}, + &commit); + if (err) { + if (err == LFS_ERR_NOSPC) { + goto split; + } else if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + ack = id; + } + + commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); if (!lfs_pairisnull(dir->tail)) { // TODO le32 - commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t), err = lfs_commit_commit(lfs, &commit, (lfs_entry_t_){ lfs_mktag(LFS_TYPE_SOFTTAIL_ + dir->split, 0x1ff, sizeof(dir->tail)), @@ -957,24 +1088,22 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { break; split: - // commit no longer fits, need to split dir - dir->count = mover.ack; - mover.begin = mover.ack+1; - + // commit no longer fits, need to split dir, // drop caches and create tail lfs->pcache.block = 0xffffffff; lfs_dir_t_ tail; - int err = lfs_dir_alloc_(lfs, &tail, dir->tail); + int err = lfs_dir_alloc_(lfs, &tail, dir->split, dir->tail); if (err) { return err; } - err = lfs_dir_compact_(lfs, &tail, lfs_dir_mover, &mover); + err = lfs_dir_compact_(lfs, &tail, cb, data, dir, ack+1, end); if (err) { return err; } + end = ack+1; dir->tail[0] = tail.pair[0]; dir->tail[1] = tail.pair[1]; dir->split = true; @@ -1029,7 +1158,7 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { void *data) { if (!dir->erased) { // not erased, must compact - return lfs_dir_compact_(lfs, dir, cb, data); + return lfs_dir_compact_(lfs, dir, cb, data, dir, 0, dir->count); } struct lfs_commit commit = { @@ -1039,13 +1168,14 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { .end = lfs->cfg->block_size - 2*sizeof(uint32_t), .crc = 0xffffffff, .ptag = dir->etag, - .compact.id = -1, + .filter.begin = 0, + .filter.end = 0x1ff, }; int err = cb(lfs, data, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - return lfs_dir_compact_(lfs, dir, cb, data); + return lfs_dir_compact_(lfs, dir, cb, data, dir, 0, dir->count); } return err; } @@ -1053,7 +1183,7 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { err = lfs_commit_crc(lfs, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - return lfs_dir_compact_(lfs, dir, cb, data); + return lfs_dir_compact_(lfs, dir, cb, data, dir, 0, dir->count); } return err; } @@ -1064,33 +1194,22 @@ int lfs_dir_mover(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -int lfs_dir_commitregions(lfs_t *lfs, void *p, struct lfs_commit *commit) { - for (lfs_entrylist_t *regions = p; regions; regions = regions->next) { - int err = lfs_commit_commit(lfs, commit, regions->e); - if (err) { - return err; - } - } - - return 0; -} - /*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, const lfs_entrylist_t *regions) { - return lfs_dir_commitwith_(lfs, dir, lfs_dir_commitregions, regions); + return lfs_dir_commitwith_(lfs, dir, lfs_commit_regions, regions); } -/*static*/ int lfs_dir_add(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t *id) { +/*static*/ int lfs_dir_append_(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t *id) { *id = dir->count; dir->count += 1; return 0; } -/*static*/ int lfs_dir_drop(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id) { +/*static*/ int lfs_dir_delete(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id) { dir->count -= 1; // TODO compact during traverse when compacting? return lfs_dir_commit_(lfs, dir, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_DROP_, id, 0)}}); + {lfs_mktag(LFS_TYPE_DELETE_, id, 0)}}); } struct lfs_dir_getter { @@ -1157,10 +1276,8 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { struct lfs_dir_finder { const char *name; - lfs_size_t len; - + uint16_t len; int16_t id; - lfs_entry_t_ *entry; }; static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { @@ -1177,13 +1294,13 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { if (res) { // found a match find->id = lfs_tag_id(entry.tag); - find->entry->tag = 0xffffffff; } - } - - if (find->id >= 0 && lfs_tag_id(entry.tag) == find->id && - lfs_tag_supertype(entry.tag) == LFS_TYPE_REG_) { - *find->entry = entry; + } else if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE_) { + if (lfs_tag_id(entry.tag) == find->id) { + find->id = -1; + } else if (lfs_tag_id(entry.tag) < find->id) { + find->id -= 1; + } } return 0; @@ -1193,7 +1310,6 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { const char **path, lfs_entry_t_ *entry) { struct lfs_dir_finder find = { .name = *path, - .entry = entry, }; // TODO make superblock @@ -1282,6 +1398,15 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { // entry->d.type &= ~LFS_STRUCT_MOVED; // } + // TODO optimize grab for inline files and like? + // TODO would this mean more code? + // grab the entry data + int err = lfs_dir_getentry_(lfs, dir, 0x701ff000, + lfs_mktag(LFS_TYPE_REG_, find.id, 0), entry); + if (err && err != LFS_ERR_RANGE) { + return err; + } + find.name += find.len; find.name += strspn(find.name, "/"); if (find.name[0] == '\0') { @@ -2132,6 +2257,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // fetch parent directory lfs_dir_t_ cwd; + // TODO remove this? int err = lfs_dir_findentry_(lfs, &cwd, &path, &(lfs_entry_t_){0}); if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { if (!err) { @@ -2150,7 +2276,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_alloc_ack(lfs); lfs_dir_t_ dir; - err = lfs_dir_alloc_(lfs, &dir, cwd.tail); + err = lfs_dir_alloc_(lfs, &dir, false, cwd.tail); if (err) { return err; } @@ -2162,7 +2288,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // get next slot and commit uint16_t id; - err = lfs_dir_add(lfs, &cwd, &id); + err = lfs_dir_append_(lfs, &cwd, &id); if (err) { return err; } @@ -2178,73 +2304,71 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return 0; } -#if 0 -int lfs_mkdir(lfs_t *lfs, const char *path) { - // deorphan if we haven't yet, needed at most once after poweron - if (!lfs->deorphaned) { - int err = lfs_deorphan(lfs); - if (err) { - return err; - } - } - - // fetch parent directory - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t entry; - err = lfs_dir_find(lfs, &cwd, &entry, &path); - if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { - return err ? err : LFS_ERR_EXIST; - } - - // check that name fits - lfs_size_t nlen = strlen(path); - if (nlen > lfs->name_size) { - return LFS_ERR_NAMETOOLONG; - } - - // build up new directory - lfs_alloc_ack(lfs); - - lfs_dir_t dir; - err = lfs_dir_alloc(lfs, &dir); - if (err) { - return err; - } - dir.d.tail[0] = cwd.d.tail[0]; - dir.d.tail[1] = cwd.d.tail[1]; - - err = lfs_dir_commit(lfs, &dir, NULL, 0); - if (err) { - return err; - } - - entry.d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR; - entry.d.elen = sizeof(entry.d) - 4; - entry.d.alen = 0; - entry.d.nlen = nlen; - entry.d.u.dir[0] = dir.pair[0]; - entry.d.u.dir[1] = dir.pair[1]; - entry.size = 0; - - cwd.d.tail[0] = dir.pair[0]; - cwd.d.tail[1] = dir.pair[1]; - lfs_entry_tole32(&entry.d); - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); - if (err) { - return err; - } - - lfs_alloc_ack(lfs); - return 0; -} -#endif +//int lfs_mkdir(lfs_t *lfs, const char *path) { +// // deorphan if we haven't yet, needed at most once after poweron +// if (!lfs->deorphaned) { +// int err = lfs_deorphan(lfs); +// if (err) { +// return err; +// } +// } +// +// // fetch parent directory +// lfs_dir_t cwd; +// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry; +// err = lfs_dir_find(lfs, &cwd, &entry, &path); +// if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { +// return err ? err : LFS_ERR_EXIST; +// } +// +// // check that name fits +// lfs_size_t nlen = strlen(path); +// if (nlen > lfs->name_size) { +// return LFS_ERR_NAMETOOLONG; +// } +// +// // build up new directory +// lfs_alloc_ack(lfs); +// +// lfs_dir_t dir; +// err = lfs_dir_alloc(lfs, &dir); +// if (err) { +// return err; +// } +// dir.d.tail[0] = cwd.d.tail[0]; +// dir.d.tail[1] = cwd.d.tail[1]; +// +// err = lfs_dir_commit(lfs, &dir, NULL, 0); +// if (err) { +// return err; +// } +// +// entry.d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR; +// entry.d.elen = sizeof(entry.d) - 4; +// entry.d.alen = 0; +// entry.d.nlen = nlen; +// entry.d.u.dir[0] = dir.pair[0]; +// entry.d.u.dir[1] = dir.pair[1]; +// entry.size = 0; +// +// cwd.d.tail[0] = dir.pair[0]; +// cwd.d.tail[1] = dir.pair[1]; +// lfs_entry_tole32(&entry.d); +// err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, 0, &entry.d, sizeof(entry.d)}, +// {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); +// if (err) { +// return err; +// } +// +// lfs_alloc_ack(lfs); +// return 0; +//} int lfs_dir_open_(lfs_t *lfs, lfs_dir_t_ *dir, const char *path) { lfs_entry_t_ entry; @@ -2359,151 +2483,204 @@ int lfs_dir_read_(lfs_t *lfs, lfs_dir_t_ *dir, struct lfs_info *info) { return true; } -int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { - dir->pair[0] = lfs->root[0]; - dir->pair[1] = lfs->root[1]; - - int err = lfs_dir_fetch(lfs, dir, dir->pair); +// TODO does this work? +int lfs_dir_seek_(lfs_t *lfs, lfs_dir_t_ *dir, lfs_off_t off) { + // simply walk from head dir + int err = lfs_dir_rewind_(lfs, dir); if (err) { return err; } - lfs_entry_t entry; - err = lfs_dir_find(lfs, dir, &entry, &path); - if (err) { - return err; - } else if (entry.d.type != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { - return LFS_ERR_NOTDIR; - } + // first two for ./.. + dir->id = lfs_min(2 + dir->count, off); + dir->pos += dir->id; + off -= dir->id; - err = lfs_dir_fetch(lfs, dir, entry.d.u.dir); - if (err) { - return err; - } + while (off != 0) { + if (dir->id == dir->count) { + if (!dir->split) { + return LFS_ERR_INVAL; + } - // setup head dir - // special offset for '.' and '..' - dir->head[0] = dir->pair[0]; - dir->head[1] = dir->pair[1]; - dir->pos = sizeof(dir->d) - 2; - dir->off = sizeof(dir->d); + int err = lfs_dir_fetch_(lfs, dir, dir->tail); + if (err) { + return err; + } + } - // add to list of directories - dir->next = lfs->dirs; - lfs->dirs = dir; - - return 0; -} - -int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir) { - // remove from list of directories - for (lfs_dir_t **p = &lfs->dirs; *p; p = &(*p)->next) { - if (*p == dir) { - *p = dir->next; - break; - } - } - - return 0; -} - -int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { - memset(info, 0, sizeof(*info)); - - // special offset for '.' and '..' - if (dir->pos == sizeof(dir->d) - 2) { - info->type = LFS_TYPE_DIR; - strcpy(info->name, "."); - dir->pos += 1; - return 1; - } else if (dir->pos == sizeof(dir->d) - 1) { - info->type = LFS_TYPE_DIR; - strcpy(info->name, ".."); - dir->pos += 1; - return 1; - } - - lfs_entry_t entry; - while (true) { - int err = lfs_dir_next(lfs, dir, &entry); - if (err) { - return (err == LFS_ERR_NOENT) ? 0 : err; - } - - if ((0xf & entry.d.type) != LFS_TYPE_REG && - (0xf & entry.d.type) != LFS_TYPE_DIR) { - continue; - } - - // check that entry has not been moved - if (entry.d.type & LFS_STRUCT_MOVED) { - int moved = lfs_moved(lfs, &entry.d.u); - if (moved < 0) { - return moved; - } - - if (moved) { - continue; - } - - entry.d.type &= ~LFS_STRUCT_MOVED; - } - - break; - } - - int err = lfs_dir_getinfo(lfs, dir, &entry, info); - if (err) { - return err; - } - - return 1; -} - -int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { - // simply walk from head dir - int err = lfs_dir_rewind(lfs, dir); - if (err) { - return err; + dir->id = lfs_min(dir->count, off); + dir->pos += dir->id; + off -= dir->id; } - dir->pos = off; - while (off > (0x7fffffff & dir->d.size)) { - off -= 0x7fffffff & dir->d.size; - if (!(0x80000000 & dir->d.size)) { - return LFS_ERR_INVAL; - } - - err = lfs_dir_fetch(lfs, dir, dir->d.tail); - if (err) { - return err; - } - } - - dir->off = off; return 0; } -lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir) { +lfs_soff_t lfs_dir_tell_(lfs_t *lfs, lfs_dir_t *dir) { (void)lfs; return dir->pos; } -int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { +int lfs_dir_rewind_(lfs_t *lfs, lfs_dir_t_ *dir) { // reload the head dir - int err = lfs_dir_fetch(lfs, dir, dir->head); + int err = lfs_dir_fetch_(lfs, dir, dir->head); if (err) { return err; } dir->pair[0] = dir->head[0]; dir->pair[1] = dir->head[1]; - dir->pos = sizeof(dir->d) - 2; - dir->off = sizeof(dir->d); + dir->pos = 0; + dir->id = 0; return 0; } +//int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { +// dir->pair[0] = lfs->root[0]; +// dir->pair[1] = lfs->root[1]; +// +// int err = lfs_dir_fetch(lfs, dir, dir->pair); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry; +// err = lfs_dir_find(lfs, dir, &entry, &path); +// if (err) { +// return err; +// } else if (entry.d.type != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { +// return LFS_ERR_NOTDIR; +// } +// +// err = lfs_dir_fetch(lfs, dir, entry.d.u.dir); +// if (err) { +// return err; +// } +// +// // setup head dir +// // special offset for '.' and '..' +// dir->head[0] = dir->pair[0]; +// dir->head[1] = dir->pair[1]; +// dir->pos = sizeof(dir->d) - 2; +// dir->off = sizeof(dir->d); +// +// // add to list of directories +// dir->next = lfs->dirs; +// lfs->dirs = dir; +// +// return 0; +//} +// +//int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir) { +// // remove from list of directories +// for (lfs_dir_t **p = &lfs->dirs; *p; p = &(*p)->next) { +// if (*p == dir) { +// *p = dir->next; +// break; +// } +// } +// +// return 0; +//} +// +//int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { +// memset(info, 0, sizeof(*info)); +// +// // special offset for '.' and '..' +// if (dir->pos == sizeof(dir->d) - 2) { +// info->type = LFS_TYPE_DIR; +// strcpy(info->name, "."); +// dir->pos += 1; +// return 1; +// } else if (dir->pos == sizeof(dir->d) - 1) { +// info->type = LFS_TYPE_DIR; +// strcpy(info->name, ".."); +// dir->pos += 1; +// return 1; +// } +// +// lfs_entry_t entry; +// while (true) { +// int err = lfs_dir_next(lfs, dir, &entry); +// if (err) { +// return (err == LFS_ERR_NOENT) ? 0 : err; +// } +// +// if ((0xf & entry.d.type) != LFS_TYPE_REG && +// (0xf & entry.d.type) != LFS_TYPE_DIR) { +// continue; +// } +// +// // check that entry has not been moved +// if (entry.d.type & LFS_STRUCT_MOVED) { +// int moved = lfs_moved(lfs, &entry.d.u); +// if (moved < 0) { +// return moved; +// } +// +// if (moved) { +// continue; +// } +// +// entry.d.type &= ~LFS_STRUCT_MOVED; +// } +// +// break; +// } +// +// int err = lfs_dir_getinfo(lfs, dir, &entry, info); +// if (err) { +// return err; +// } +// +// return 1; +//} +// +//int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { +// // simply walk from head dir +// int err = lfs_dir_rewind(lfs, dir); +// if (err) { +// return err; +// } +// dir->pos = off; +// +// while (off > (0x7fffffff & dir->d.size)) { +// off -= 0x7fffffff & dir->d.size; +// if (!(0x80000000 & dir->d.size)) { +// return LFS_ERR_INVAL; +// } +// +// err = lfs_dir_fetch(lfs, dir, dir->d.tail); +// if (err) { +// return err; +// } +// } +// +// dir->off = off; +// return 0; +//} +// +//lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir) { +// (void)lfs; +// return dir->pos; +//} +// +//int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { +// // reload the head dir +// int err = lfs_dir_fetch(lfs, dir, dir->head); +// if (err) { +// return err; +// } +// +// dir->pair[0] = dir->head[0]; +// dir->pair[1] = dir->head[1]; +// dir->pos = sizeof(dir->d) - 2; +// dir->off = sizeof(dir->d); +// return 0; +//} + + /// File index list operations /// static int lfs_ctz_index(lfs_t *lfs, lfs_off_t *off) { lfs_off_t size = *off; @@ -2723,7 +2900,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // get next slot and create entry to remember name uint16_t id; - err = lfs_dir_add(lfs, &cwd, &id); + err = lfs_dir_append_(lfs, &cwd, &id); if (err) { return err; } @@ -3377,246 +3554,241 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t entry; - err = lfs_dir_find(lfs, &cwd, &entry, &path); - if (err) { - return err; - } - - return lfs_dir_getinfo(lfs, &cwd, &entry, info); -} - -int lfs_remove(lfs_t *lfs, const char *path) { - // deorphan if we haven't yet, needed at most once after poweron - if (!lfs->deorphaned) { - int err = lfs_deorphan(lfs); - if (err) { - return err; - } - } - - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t entry; - err = lfs_dir_find(lfs, &cwd, &entry, &path); - if (err) { - return err; - } - - lfs_dir_t dir; - if ((0xf & entry.d.type) == LFS_TYPE_DIR) { - // must be empty before removal, checking size - // without masking top bit checks for any case where - // dir is not empty - err = lfs_dir_fetch(lfs, &dir, entry.d.u.dir); - if (err) { - return err; - } else if (dir.d.size != sizeof(dir.d)+4) { - return LFS_ERR_NOTEMPTY; - } - } - - // remove the entry - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); - if (err) { - return err; - } - - // if we were a directory, find pred, replace tail - if ((0xf & entry.d.type) == LFS_TYPE_DIR) { - int res = lfs_pred(lfs, dir.pair, &cwd); - if (res < 0) { - return res; - } - - LFS_ASSERT(res); // must have pred - cwd.d.tail[0] = dir.d.tail[0]; - cwd.d.tail[1] = dir.d.tail[1]; - - err = lfs_dir_commit(lfs, &cwd, NULL, 0); - if (err) { - return err; - } - } - - return 0; -} - -int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { - // deorphan if we haven't yet, needed at most once after poweron - if (!lfs->deorphaned) { - int err = lfs_deorphan(lfs); - if (err) { - return err; - } - } - - // find old entry - lfs_dir_t oldcwd; - int err = lfs_dir_fetch(lfs, &oldcwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t oldentry; - err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath); - if (err) { - return err; - } - - // allocate new entry - lfs_dir_t newcwd; - err = lfs_dir_fetch(lfs, &newcwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t preventry; - err = lfs_dir_find(lfs, &newcwd, &preventry, &newpath); - if (err && (err != LFS_ERR_NOENT || strchr(newpath, '/') != NULL)) { - return err; - } - - bool prevexists = (err != LFS_ERR_NOENT); - bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); - - // check that name fits - lfs_size_t nlen = strlen(newpath); - if (nlen > lfs->name_size) { - return LFS_ERR_NAMETOOLONG; - } - - if (oldentry.size - oldentry.d.nlen + nlen > lfs->cfg->block_size) { - return LFS_ERR_NOSPC; - } - - // must have same type - if (prevexists && preventry.d.type != oldentry.d.type) { - return LFS_ERR_ISDIR; - } - - lfs_dir_t dir; - if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { - // must be empty before removal, checking size - // without masking top bit checks for any case where - // dir is not empty - err = lfs_dir_fetch(lfs, &dir, preventry.d.u.dir); - if (err) { - return err; - } else if (dir.d.size != sizeof(dir.d)+4) { - return LFS_ERR_NOTEMPTY; - } - } - - // mark as moving - oldentry.d.type |= LFS_STRUCT_MOVED; - err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 1, &oldentry.d.type, 1}}, 1); - oldentry.d.type &= ~LFS_STRUCT_MOVED; - if (err) { - return err; - } - - // update pair if newcwd == oldcwd - if (samepair) { - newcwd = oldcwd; - } - - // move to new location - lfs_entry_t newentry = preventry; - newentry.d = oldentry.d; - newentry.d.type &= ~LFS_STRUCT_MOVED; - newentry.d.nlen = nlen; - newentry.size = prevexists ? preventry.size : 0; - - lfs_size_t newsize = oldentry.size - oldentry.d.nlen + newentry.d.nlen; - err = lfs_dir_set(lfs, &newcwd, &newentry, (struct lfs_region[]){ - {LFS_FROM_REGION, 0, prevexists ? preventry.size : 0, - &(struct lfs_region_region){ - oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 4, &newentry.d, 4}, - {LFS_FROM_MEM, newsize-nlen, 0, newpath, nlen}}, 2}, - newsize}}, 1); - if (err) { - return err; - } - - // update pair if newcwd == oldcwd - if (samepair) { - oldcwd = newcwd; - } - - // remove old entry - err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, oldentry.size, NULL, 0}}, 1); - if (err) { - return err; - } - - // if we were a directory, find pred, replace tail - if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { - int res = lfs_pred(lfs, dir.pair, &newcwd); - if (res < 0) { - return res; - } - - LFS_ASSERT(res); // must have pred - newcwd.d.tail[0] = dir.d.tail[0]; - newcwd.d.tail[1] = dir.d.tail[1]; - - err = lfs_dir_commit(lfs, &newcwd, NULL, 0); - if (err) { - return err; - } - } - - return 0; -} - -int lfs_getattrs(lfs_t *lfs, const char *path, - const struct lfs_attr *attrs, int count) { - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t entry; - err = lfs_dir_find(lfs, &cwd, &entry, &path); + lfs_dir_t_ cwd; + lfs_entry_t_ entry; + int err = lfs_dir_find_(lfs, &cwd, &entry, &path); if (err) { return err; } - return lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); + return lfs_dir_getinfo_(lfs, &cwd, lfs_tag_id(entry.tag), info); } -int lfs_setattrs(lfs_t *lfs, const char *path, - const struct lfs_attr *attrs, int count) { - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t entry; - err = lfs_dir_find(lfs, &cwd, &entry, &path); - if (err) { - return err; - } +//int lfs_remove(lfs_t *lfs, const char *path) { +// // deorphan if we haven't yet, needed at most once after poweron +// if (!lfs->deorphaned) { +// int err = lfs_deorphan(lfs); +// if (err) { +// return err; +// } +// } +// +// lfs_dir_t cwd; +// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry; +// err = lfs_dir_find(lfs, &cwd, &entry, &path); +// if (err) { +// return err; +// } +// +// lfs_dir_t dir; +// if ((0xf & entry.d.type) == LFS_TYPE_DIR) { +// // must be empty before removal, checking size +// // without masking top bit checks for any case where +// // dir is not empty +// err = lfs_dir_fetch(lfs, &dir, entry.d.u.dir); +// if (err) { +// return err; +// } else if (dir.d.size != sizeof(dir.d)+4) { +// return LFS_ERR_NOTEMPTY; +// } +// } +// +// // remove the entry +// err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); +// if (err) { +// return err; +// } +// +// // if we were a directory, find pred, replace tail +// if ((0xf & entry.d.type) == LFS_TYPE_DIR) { +// int res = lfs_pred(lfs, dir.pair, &cwd); +// if (res < 0) { +// return res; +// } +// +// LFS_ASSERT(res); // must have pred +// cwd.d.tail[0] = dir.d.tail[0]; +// cwd.d.tail[1] = dir.d.tail[1]; +// +// err = lfs_dir_commit(lfs, &cwd, NULL, 0); +// if (err) { +// return err; +// } +// } +// +// return 0; +//} +// +//int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { +// // deorphan if we haven't yet, needed at most once after poweron +// if (!lfs->deorphaned) { +// int err = lfs_deorphan(lfs); +// if (err) { +// return err; +// } +// } +// +// // find old entry +// lfs_dir_t oldcwd; +// int err = lfs_dir_fetch(lfs, &oldcwd, lfs->root); +// if (err) { +// return err; +// } +// +// lfs_entry_t oldentry; +// err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath); +// if (err) { +// return err; +// } +// +// // allocate new entry +// lfs_dir_t newcwd; +// err = lfs_dir_fetch(lfs, &newcwd, lfs->root); +// if (err) { +// return err; +// } +// +// lfs_entry_t preventry; +// err = lfs_dir_find(lfs, &newcwd, &preventry, &newpath); +// if (err && (err != LFS_ERR_NOENT || strchr(newpath, '/') != NULL)) { +// return err; +// } +// +// bool prevexists = (err != LFS_ERR_NOENT); +// bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); +// +// // check that name fits +// lfs_size_t nlen = strlen(newpath); +// if (nlen > lfs->name_size) { +// return LFS_ERR_NAMETOOLONG; +// } +// +// if (oldentry.size - oldentry.d.nlen + nlen > lfs->cfg->block_size) { +// return LFS_ERR_NOSPC; +// } +// +// // must have same type +// if (prevexists && preventry.d.type != oldentry.d.type) { +// return LFS_ERR_ISDIR; +// } +// +// lfs_dir_t dir; +// if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { +// // must be empty before removal, checking size +// // without masking top bit checks for any case where +// // dir is not empty +// err = lfs_dir_fetch(lfs, &dir, preventry.d.u.dir); +// if (err) { +// return err; +// } else if (dir.d.size != sizeof(dir.d)+4) { +// return LFS_ERR_NOTEMPTY; +// } +// } +// +// // mark as moving +// oldentry.d.type |= LFS_STRUCT_MOVED; +// err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, 1, &oldentry.d.type, 1}}, 1); +// oldentry.d.type &= ~LFS_STRUCT_MOVED; +// if (err) { +// return err; +// } +// +// // update pair if newcwd == oldcwd +// if (samepair) { +// newcwd = oldcwd; +// } +// +// // move to new location +// lfs_entry_t newentry = preventry; +// newentry.d = oldentry.d; +// newentry.d.type &= ~LFS_STRUCT_MOVED; +// newentry.d.nlen = nlen; +// newentry.size = prevexists ? preventry.size : 0; +// +// lfs_size_t newsize = oldentry.size - oldentry.d.nlen + newentry.d.nlen; +// err = lfs_dir_set(lfs, &newcwd, &newentry, (struct lfs_region[]){ +// {LFS_FROM_REGION, 0, prevexists ? preventry.size : 0, +// &(struct lfs_region_region){ +// oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, 4, &newentry.d, 4}, +// {LFS_FROM_MEM, newsize-nlen, 0, newpath, nlen}}, 2}, +// newsize}}, 1); +// if (err) { +// return err; +// } +// +// // update pair if newcwd == oldcwd +// if (samepair) { +// oldcwd = newcwd; +// } +// +// // remove old entry +// err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, oldentry.size, NULL, 0}}, 1); +// if (err) { +// return err; +// } +// +// // if we were a directory, find pred, replace tail +// if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { +// int res = lfs_pred(lfs, dir.pair, &newcwd); +// if (res < 0) { +// return res; +// } +// +// LFS_ASSERT(res); // must have pred +// newcwd.d.tail[0] = dir.d.tail[0]; +// newcwd.d.tail[1] = dir.d.tail[1]; +// +// err = lfs_dir_commit(lfs, &newcwd, NULL, 0); +// if (err) { +// return err; +// } +// } +// +// return 0; +//} - return lfs_dir_setattrs(lfs, &cwd, &entry, attrs, count); -} +//int lfs_getattrs(lfs_t *lfs, const char *path, +// const struct lfs_attr *attrs, int count) { +// lfs_dir_t cwd; +// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry; +// err = lfs_dir_find(lfs, &cwd, &entry, &path); +// if (err) { +// return err; +// } +// +// return lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); +//} +// +//int lfs_setattrs(lfs_t *lfs, const char *path, +// const struct lfs_attr *attrs, int count) { +// lfs_dir_t cwd; +// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry; +// err = lfs_dir_find(lfs, &cwd, &entry, &path); +// if (err) { +// return err; +// } +// +// return lfs_dir_setattrs(lfs, &cwd, &entry, attrs, count); +//} /// Filesystem operations /// @@ -3727,7 +3899,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // create superblock dir lfs_dir_t_ dir; - err = lfs_dir_alloc_(lfs, &dir, + err = lfs_dir_alloc_(lfs, &dir, false, (const lfs_block_t[2]){0xffffffff, 0xffffffff}); if (err) { return err; @@ -3735,7 +3907,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // write root directory lfs_dir_t_ root; - err = lfs_dir_alloc_(lfs, &root, + err = lfs_dir_alloc_(lfs, &root, false, (const lfs_block_t[2]){0xffffffff, 0xffffffff}); if (err) { return err; @@ -4252,25 +4424,6 @@ static int lfs_relocate(lfs_t *lfs, return 0; } -// TODO use this in lfs_move? -int lfs_deorphan_check(lfs_t *lfs, void *p, lfs_entry_t_ entry) { - int16_t *id = p; - - // TODO this fine for only grabbing the last one? - // TODO should I also grab deletes? I should, move will always be last yay - if (lfs_tag_type(entry.tag) == LFS_TYPE_MOVE_) { - *id = lfs_tag_id(entry.tag); - } - - // TODO handle unrelated deletes - if (lfs_tag_type(entry.tag) == LFS_TYPE_DROP_ && - lfs_tag_id(entry.tag) == *id) { - *id = -1; - } - - return 0; -} - int lfs_deorphan_(lfs_t *lfs) { lfs->deorphaned = true; if (lfs_pairisnull(lfs->root)) { @@ -4282,9 +4435,7 @@ int lfs_deorphan_(lfs_t *lfs) { // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { - int16_t moveid = -1; - int err = lfs_dir_fetchwith_(lfs, &dir, dir.tail, - lfs_deorphan_check, &moveid); + int err = lfs_dir_fetch_(lfs, &dir, dir.tail); if (err) { return err; } @@ -4335,7 +4486,7 @@ int lfs_deorphan_(lfs_t *lfs) { } // check entries for moves - if (moveid >= 0) { + if (dir.moveid >= 0) { // TODO moves and stuff // TODO need to load entry to find it // // found moved entry @@ -4478,39 +4629,59 @@ int lfs_deorphan(lfs_t *lfs) { /// External filesystem filesystem operations /// -int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { - lfs_dir_t dir; - int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); - if (err) { - return err; - } - entry.size = lfs_entry_size(&entry); - - return lfs_dir_getattrs(lfs, &dir, &entry, attrs, count); -} - -int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { - lfs_dir_t dir; - int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); - if (err) { - return err; - } - entry.size = lfs_entry_size(&entry); - - return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); -} +//int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { +// lfs_dir_t dir; +// int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry = {.off = sizeof(dir.d)}; +// err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); +// if (err) { +// return err; +// } +// entry.size = lfs_entry_size(&entry); +// +// if (err != LFS_ERR_NOENT) { +// if (!err) { +// break; +// } +// return err; +// } +// +// lfs_dir_t cwd; +// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry; +// err = lfs_dir_find(lfs, &cwd, &entry, &path); +// if (err) { +// return err; +// } +// +// return lfs_dir_getinfo(lfs, &cwd, &entry, info); +// return lfs_dir_getattrs(lfs, &dir, &entry, attrs, count); +//} +// +//int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { +// lfs_dir_t dir; +// int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); +// if (err) { +// return err; +// } +// +// lfs_entry_t entry = {.off = sizeof(dir.d)}; +// err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); +// if (err) { +// return err; +// } +// entry.size = lfs_entry_size(&entry); +// +// return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); +//} static int lfs_fs_size_count(void *p, lfs_block_t block) { lfs_size_t *size = p; diff --git a/lfs.h b/lfs.h index cb14eb3a..d344ce04 100644 --- a/lfs.h +++ b/lfs.h @@ -113,7 +113,7 @@ enum lfs_type { // internally used types LFS_TYPE_NAME_ = 0x010, LFS_TYPE_MOVE_ = 0x080, - LFS_TYPE_DROP_ = 0x090, + LFS_TYPE_DELETE_ = 0x090, LFS_TYPE_SUPERBLOCK_ = 0x0a0, LFS_TYPE_SOFTTAIL_ = 0x0c0, @@ -367,6 +367,7 @@ typedef struct lfs_dir_ { uint16_t count; bool erased; bool split; + int16_t moveid; uint16_t id; lfs_block_t head[2]; From 0405ceb1710cb5112feeb587241dc862375f40c8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 26 May 2018 13:50:06 -0500 Subject: [PATCH 035/139] Cleaned up enough things to pass basic file testing --- lfs.c | 2341 ++++++++++++++++++++++---------------------- lfs.h | 112 +-- tests/template.fmt | 2 + tests/test_dirs.sh | 2 + 4 files changed, 1199 insertions(+), 1258 deletions(-) diff --git a/lfs.c b/lfs.c index b35cba03..aad21524 100644 --- a/lfs.c +++ b/lfs.c @@ -259,11 +259,12 @@ static int lfs_bd_sync(lfs_t *lfs) { /// Internal operations predeclared here /// -int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); +int lfs_fs_traverse(lfs_t *lfs, + int (*cb)(lfs_t*, void*, lfs_block_t), void *data); static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir); static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *parent, lfs_entry_t *entry); -static int lfs_moved(lfs_t *lfs, const void *e); +static int lfs_moved(lfs_t *lfs, const lfs_block_t pair[2]); static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); int lfs_deorphan(lfs_t *lfs); @@ -318,7 +319,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { // find mask of free blocks from tree memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); - int err = lfs_traverse_(lfs, lfs_alloc_lookahead, NULL); + int err = lfs_fs_traverse(lfs, lfs_alloc_lookahead, NULL); if (err) { return err; } @@ -331,71 +332,71 @@ static void lfs_alloc_ack(lfs_t *lfs) { /// Endian swapping functions /// -static void lfs_dir_fromle32(struct lfs_disk_dir *d) { - d->rev = lfs_fromle32(d->rev); - d->size = lfs_fromle32(d->size); - d->tail[0] = lfs_fromle32(d->tail[0]); - d->tail[1] = lfs_fromle32(d->tail[1]); -} - -static void lfs_dir_tole32(struct lfs_disk_dir *d) { - d->rev = lfs_tole32(d->rev); - d->size = lfs_tole32(d->size); - d->tail[0] = lfs_tole32(d->tail[0]); - d->tail[1] = lfs_tole32(d->tail[1]); -} - -static void lfs_entry_fromle32(struct lfs_disk_entry *d) { - d->u.dir[0] = lfs_fromle32(d->u.dir[0]); - d->u.dir[1] = lfs_fromle32(d->u.dir[1]); -} - -static void lfs_entry_tole32(struct lfs_disk_entry *d) { - d->u.dir[0] = lfs_tole32(d->u.dir[0]); - d->u.dir[1] = lfs_tole32(d->u.dir[1]); -} - -/*static*/ void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { - d->root[0] = lfs_fromle32(d->root[0]); - d->root[1] = lfs_fromle32(d->root[1]); - d->block_size = lfs_fromle32(d->block_size); - d->block_count = lfs_fromle32(d->block_count); - d->version = lfs_fromle32(d->version); - d->inline_size = lfs_fromle32(d->inline_size); - d->attrs_size = lfs_fromle32(d->attrs_size); - d->name_size = lfs_fromle32(d->name_size); -} +//static void lfs_dir_fromle32(struct lfs_disk_dir *d) { +// d->rev = lfs_fromle32(d->rev); +// d->size = lfs_fromle32(d->size); +// d->tail[0] = lfs_fromle32(d->tail[0]); +// d->tail[1] = lfs_fromle32(d->tail[1]); +//} +// +//static void lfs_dir_tole32(struct lfs_disk_dir *d) { +// d->rev = lfs_tole32(d->rev); +// d->size = lfs_tole32(d->size); +// d->tail[0] = lfs_tole32(d->tail[0]); +// d->tail[1] = lfs_tole32(d->tail[1]); +//} +// +//static void lfs_entry_fromle32(struct lfs_disk_entry *d) { +// d->u.dir[0] = lfs_fromle32(d->u.dir[0]); +// d->u.dir[1] = lfs_fromle32(d->u.dir[1]); +//} +// +//static void lfs_entry_tole32(struct lfs_disk_entry *d) { +// d->u.dir[0] = lfs_tole32(d->u.dir[0]); +// d->u.dir[1] = lfs_tole32(d->u.dir[1]); +//} -/*static*/ void lfs_superblock_tole32(struct lfs_disk_superblock *d) { - d->root[0] = lfs_tole32(d->root[0]); - d->root[1] = lfs_tole32(d->root[1]); - d->block_size = lfs_tole32(d->block_size); - d->block_count = lfs_tole32(d->block_count); - d->version = lfs_tole32(d->version); - d->inline_size = lfs_tole32(d->inline_size); - d->attrs_size = lfs_tole32(d->attrs_size); - d->name_size = lfs_tole32(d->name_size); -} +///*static*/ void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { +// d->root[0] = lfs_fromle32(d->root[0]); +// d->root[1] = lfs_fromle32(d->root[1]); +// d->block_size = lfs_fromle32(d->block_size); +// d->block_count = lfs_fromle32(d->block_count); +// d->version = lfs_fromle32(d->version); +// d->inline_size = lfs_fromle32(d->inline_size); +// d->attrs_size = lfs_fromle32(d->attrs_size); +// d->name_size = lfs_fromle32(d->name_size); +//} +// +///*static*/ void lfs_superblock_tole32(struct lfs_disk_superblock *d) { +// d->root[0] = lfs_tole32(d->root[0]); +// d->root[1] = lfs_tole32(d->root[1]); +// d->block_size = lfs_tole32(d->block_size); +// d->block_count = lfs_tole32(d->block_count); +// d->version = lfs_tole32(d->version); +// d->inline_size = lfs_tole32(d->inline_size); +// d->attrs_size = lfs_tole32(d->attrs_size); +// d->name_size = lfs_tole32(d->name_size); +//} /// Other struct functions /// -static inline lfs_size_t lfs_entry_elen(const lfs_entry_t *entry) { - return (lfs_size_t)(entry->d.elen) | - ((lfs_size_t)(entry->d.alen & 0xc0) << 2); -} - -static inline lfs_size_t lfs_entry_alen(const lfs_entry_t *entry) { - return entry->d.alen & 0x3f; -} - -static inline lfs_size_t lfs_entry_nlen(const lfs_entry_t *entry) { - return entry->d.nlen; -} - -static inline lfs_size_t lfs_entry_size(const lfs_entry_t *entry) { - return 4 + lfs_entry_elen(entry) + - lfs_entry_alen(entry) + - lfs_entry_nlen(entry); -} +//static inline lfs_size_t lfs_entry_elen(const lfs_entry_t *entry) { +// return (lfs_size_t)(entry->d.elen) | +// ((lfs_size_t)(entry->d.alen & 0xc0) << 2); +//} +// +//static inline lfs_size_t lfs_entry_alen(const lfs_entry_t *entry) { +// return entry->d.alen & 0x3f; +//} +// +//static inline lfs_size_t lfs_entry_nlen(const lfs_entry_t *entry) { +// return entry->d.nlen; +//} +// +//static inline lfs_size_t lfs_entry_size(const lfs_entry_t *entry) { +// return 4 + lfs_entry_elen(entry) + +// lfs_entry_alen(entry) + +// lfs_entry_nlen(entry); +//} /// Metadata pair and directory operations /// @@ -473,8 +474,7 @@ struct lfs_commit { }; static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), - void *data) { + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data) { // TODO duplicate this? move it to dir? // iterate over dir block backwards (for faster lookups) lfs_block_t block = commit->block; @@ -482,8 +482,9 @@ static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, lfs_tag_t tag = commit->ptag; while (off != sizeof(uint32_t)) { - printf("tag r %#010x (%x:%x)\n", tag, block, off-lfs_tag_size(tag)); - int err = cb(lfs, data, (lfs_entry_t_){ + // TODO rm me + printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, block, off-lfs_tag_size(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); + int err = cb(lfs, data, (lfs_entry_t){ (0x80000000 | tag), .u.d.block=block, .u.d.off=off-lfs_tag_size(tag)}); @@ -506,7 +507,7 @@ static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, return 0; } -//static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t_ entry) { +//static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t entry) { // struct lfs_commit *commit = p; // if (lfs_tag_id(entry.tag) != commit->compact.id) { // return 1; @@ -518,27 +519,7 @@ static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, //} // static int lfs_commit_commit(lfs_t *lfs, - struct lfs_commit *commit, lfs_entry_t_ entry) { -// // request for compaction? -// if (commit->compact.id >= 0) { -// if (lfs_tag_id(entry.tag) != commit->compact.id) { -// // ignore non-matching ids -// return 0; -// } -// -// commit->compact.type = lfs_tag_type(entry.tag); -// int res = lfs_commit_traverse(lfs, commit, -// lfs_commit_compactcheck, commit); -// if (res < 0) { -// return res; -// } -// -// if (res == 2) { -// // already committed -// return 0; -// } -// } -// + struct lfs_commit *commit, lfs_entry_t entry) { // filter out ids if (lfs_tag_id(entry.tag) != 0x1ff && ( lfs_tag_id(entry.tag) < commit->filter.begin || @@ -555,7 +536,8 @@ static int lfs_commit_commit(lfs_t *lfs, } // write out tag - printf("tag w %#010x (%x:%x)\n", entry.tag, commit->block, commit->off+sizeof(lfs_tag_t)); + // TODO rm me + printf("tag w %#010x (%x:%x %03x %03x %03x)\n", entry.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tag_type(entry.tag), lfs_tag_id(entry.tag), lfs_tag_size(entry.tag)); lfs_tag_t tag = lfs_tole32((entry.tag & 0x7fffffff) ^ commit->ptag); lfs_crc(&commit->crc, &tag, sizeof(tag)); int err = lfs_bd_prog(lfs, commit->block, commit->off, &tag, sizeof(tag)); @@ -609,11 +591,11 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // build crc tag tag = (0x80000000 & ~lfs_fromle32(tag)) | - lfs_mktag(LFS_TYPE_CRC_, 0x1ff, + lfs_mktag(LFS_TYPE_CRC, 0x1ff, noff - (commit->off+sizeof(uint32_t))); // write out crc - printf("tag w %#010x (%x:%x)\n", tag, commit->block, commit->off+sizeof(tag)); + printf("tag w %#010x (%x:%x %03x %03x %03x)\n", tag, commit->block, commit->off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); uint32_t footer[2]; footer[0] = lfs_tole32(tag ^ commit->ptag); lfs_crc(&commit->crc, &footer[0], sizeof(footer[0])); @@ -648,8 +630,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { } // committer for regions -static int lfs_commit_regions(lfs_t *lfs, void *p, - struct lfs_commit *commit) { +static int lfs_commit_regions(lfs_t *lfs, void *p, struct lfs_commit *commit) { for (lfs_entrylist_t *regions = p; regions; regions = regions->next) { int err = lfs_commit_commit(lfs, commit, regions->e); if (err) { @@ -661,13 +642,13 @@ static int lfs_commit_regions(lfs_t *lfs, void *p, } // TODO redeclare -static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), +static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data); // committer for moves struct lfs_commit_move { - lfs_dir_t_ *dir; + lfs_dir_t *dir; struct { uint16_t from; uint16_t to; @@ -677,26 +658,32 @@ struct lfs_commit_move { struct lfs_commit *commit; }; -static int lfs_commit_movecheck(lfs_t *lfs, void *p, lfs_entry_t_ entry) { +static int lfs_commit_movecheck(lfs_t *lfs, void *p, lfs_entry_t entry) { struct lfs_commit_move *move = p; - if (lfs_tag_id(entry.tag) != move->id.from) { + if (lfs_tag_id(entry.tag) != move->id.to - move->commit->filter.begin) { return 1; - } else if (lfs_tag_type(entry.tag) == move->type) { - return 2; + } + + if (lfs_tag_type(entry.tag) & 0x100) { + if (lfs_tag_type(entry.tag) == move->type) { + return 2; + } + } else { + // TODO hmm + if (lfs_tag_subtype(entry.tag) == (move->type & 0x1f0)) { + return 2; + } } return 0; } -static int lfs_commit_moveiter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { +static int lfs_commit_moveiter(lfs_t *lfs, void *p, lfs_entry_t entry) { struct lfs_commit_move *move = p; - if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE_) { - if (lfs_tag_id(entry.tag) == move->id.from) { - // we done, bail - return 1; - } else if (lfs_tag_id(entry.tag) < move->id.from) { + if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE) { + if (lfs_tag_id(entry.tag) <= move->id.from) { // something was deleted, we need to move around it move->id.from += 1; } @@ -728,7 +715,7 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { struct lfs_commit_move *move = p; move->commit = commit; - int err = lfs_dir_traverse_(lfs, move->dir, lfs_commit_moveiter, move); + int err = lfs_dir_traverse(lfs, move->dir, lfs_commit_moveiter, move); if (err < 0) { return err; } @@ -736,7 +723,7 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -/*static*/ int lfs_dir_alloc_(lfs_t *lfs, lfs_dir_t_ *dir, +static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir, bool split, const lfs_block_t tail[2]) { // allocate pair of dir blocks (backwards, so we write to block 1 first) for (int i = 0; i < 2; i++) { @@ -767,10 +754,9 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -/*static*/ int lfs_dir_fetchwith_(lfs_t *lfs, - lfs_dir_t_ *dir, const lfs_block_t pair[2], - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), - void *data) { +static int lfs_dir_fetchwith(lfs_t *lfs, + lfs_dir_t *dir, const lfs_block_t pair[2], + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; @@ -816,19 +802,18 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { tag = lfs_fromle32(tag) ^ ptag; // next commit not yet programmed - if (lfs_tag_type(ptag) == LFS_TYPE_CRC_ && !lfs_tag_valid(tag)) { + if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_valid(tag)) { dir->erased = true; return 0; } // check we're in valid range - if (off + sizeof(tag)+lfs_tag_size(tag) > - lfs->cfg->block_size - 2*sizeof(uint32_t)) { + if (off + sizeof(tag)+lfs_tag_size(tag) > lfs->cfg->block_size) { break; } - printf("tag r %#010x (%x:%x)\n", tag, dir->pair[0], off+sizeof(tag)); - if (lfs_tag_type(tag) == LFS_TYPE_CRC_) { + printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, dir->pair[0], off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); + if (lfs_tag_type(tag) == LFS_TYPE_CRC) { // check the crc entry uint32_t dcrc; int err = lfs_bd_read(lfs, dir->pair[0], @@ -858,24 +843,24 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { return err; } - if (lfs_tag_type(tag) == LFS_TYPE_SOFTTAIL_ || - lfs_tag_type(tag) == LFS_TYPE_HARDTAIL_) { - dir->split = lfs_tag_type(tag) == LFS_TYPE_HARDTAIL_; + if (lfs_tag_type(tag) == LFS_TYPE_SOFTTAIL || + lfs_tag_type(tag) == LFS_TYPE_HARDTAIL) { + dir->split = lfs_tag_type(tag) == LFS_TYPE_HARDTAIL; err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), dir->tail, sizeof(dir->tail)); if (err) { return err; } - } else if (lfs_tag_type(tag) == LFS_TYPE_MOVE_) { + } else if (lfs_tag_type(tag) == LFS_TYPE_MOVE) { + // TODO handle moves correctly? dir->moveid = lfs_tag_id(tag); } else { - // TODO handle deletes and stuff if (lfs_tag_id(tag) < 0x1ff && lfs_tag_id(tag) >= dir->count) { dir->count = lfs_tag_id(tag)+1; } - if (lfs_tag_type(tag) == LFS_TYPE_DELETE_) { + if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { dir->count -= 1; if (lfs_tag_id(tag) == dir->moveid) { dir->moveid = -1; @@ -885,7 +870,7 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { } if (cb) { - err = cb(lfs, data, (lfs_entry_t_){ + err = cb(lfs, data, (lfs_entry_t){ (tag | 0x80000000), .u.d.block=dir->pair[0], .u.d.off=off+sizeof(tag)}); @@ -909,14 +894,13 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { return LFS_ERR_CORRUPT; } -/*static*/ int lfs_dir_fetch_(lfs_t *lfs, - lfs_dir_t_ *dir, const lfs_block_t pair[2]) { - return lfs_dir_fetchwith_(lfs, dir, pair, NULL, NULL); +static int lfs_dir_fetch(lfs_t *lfs, + lfs_dir_t *dir, const lfs_block_t pair[2]) { + return lfs_dir_fetchwith(lfs, dir, pair, NULL, NULL); } -static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t_ entry), - void *data) { +static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data) { return lfs_commit_traverse(lfs, &(struct lfs_commit){ .block=dir->pair[0], .off=dir->off, .ptag=dir->etag}, cb, data); @@ -924,7 +908,7 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, //struct lfs_dir_commitmove { // // traversal things -// lfs_dir_t_ *dir; +// lfs_dir_t *dir; // int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit); // void *data; // @@ -935,7 +919,7 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, //}; // //static int lfs_dir_commitmove_commit(lfs_t *lfs, void *p, -// lfs_entry_t_ entry) { +// lfs_entry_t entry) { // return lfs_commit_commit(lfs, p, entry); //} // @@ -956,7 +940,7 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, // } // // // iterate over on-disk regions -// err = lfs_dir_traverse_(lfs, move->dir, +// err = lfs_dir_traverse(lfs, move->dir, // lfs_dir_commitmove_commit, commit); // if (err) { // commit->compact.id = old; @@ -970,9 +954,9 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, // return 0; //} -/*static*/ int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t_ *dir, +static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), - void *data, lfs_dir_t_ *source, uint16_t begin, uint16_t end) { + void *data, lfs_dir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; bool relocated = false; @@ -1059,8 +1043,8 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); if (!lfs_pairisnull(dir->tail)) { // TODO le32 - err = lfs_commit_commit(lfs, &commit, (lfs_entry_t_){ - lfs_mktag(LFS_TYPE_SOFTTAIL_ + dir->split, + err = lfs_commit_commit(lfs, &commit, (lfs_entry_t){ + lfs_mktag(LFS_TYPE_SOFTTAIL + dir->split*0x10, 0x1ff, sizeof(dir->tail)), .u.buffer=dir->tail}); if (err) { @@ -1092,13 +1076,13 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, // drop caches and create tail lfs->pcache.block = 0xffffffff; - lfs_dir_t_ tail; - int err = lfs_dir_alloc_(lfs, &tail, dir->split, dir->tail); + lfs_dir_t tail; + int err = lfs_dir_alloc(lfs, &tail, dir->split, dir->tail); if (err) { return err; } - err = lfs_dir_compact_(lfs, &tail, cb, data, dir, ack+1, end); + err = lfs_dir_compact(lfs, &tail, cb, data, dir, ack+1, end); if (err) { return err; } @@ -1153,12 +1137,12 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, return 0; } -/*static*/ int lfs_dir_commitwith_(lfs_t *lfs, lfs_dir_t_ *dir, +static int lfs_dir_commitwith(lfs_t *lfs, lfs_dir_t *dir, int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), void *data) { if (!dir->erased) { // not erased, must compact - return lfs_dir_compact_(lfs, dir, cb, data, dir, 0, dir->count); + return lfs_dir_compact(lfs, dir, cb, data, dir, 0, dir->count); } struct lfs_commit commit = { @@ -1175,7 +1159,7 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, int err = cb(lfs, data, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - return lfs_dir_compact_(lfs, dir, cb, data, dir, 0, dir->count); + return lfs_dir_compact(lfs, dir, cb, data, dir, 0, dir->count); } return err; } @@ -1183,7 +1167,7 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, err = lfs_commit_crc(lfs, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - return lfs_dir_compact_(lfs, dir, cb, data, dir, 0, dir->count); + return lfs_dir_compact(lfs, dir, cb, data, dir, 0, dir->count); } return err; } @@ -1194,31 +1178,30 @@ static int lfs_dir_traverse_(lfs_t *lfs, lfs_dir_t_ *dir, return 0; } -/*static*/ int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t_ *dir, +static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, const lfs_entrylist_t *regions) { - return lfs_dir_commitwith_(lfs, dir, lfs_commit_regions, regions); + return lfs_dir_commitwith(lfs, dir, lfs_commit_regions, (void*)regions); } -/*static*/ int lfs_dir_append_(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t *id) { +static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, uint16_t *id) { *id = dir->count; dir->count += 1; return 0; } -/*static*/ int lfs_dir_delete(lfs_t *lfs, lfs_dir_t_ *dir, uint16_t id) { +static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { dir->count -= 1; - // TODO compact during traverse when compacting? - return lfs_dir_commit_(lfs, dir, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_DELETE_, id, 0)}}); + return lfs_dir_commit(lfs, dir, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_DELETE, id, 0)}}); } struct lfs_dir_getter { uint32_t mask; lfs_tag_t tag; - lfs_entry_t_ *entry; + lfs_entry_t *entry; }; -static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { +static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t entry) { struct lfs_dir_getter *get = p; if ((entry.tag & get->mask) == (get->tag & get->mask)) { if (get->entry) { @@ -1230,10 +1213,10 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { return false; } -/*static*/ int lfs_dir_get_(lfs_t *lfs, lfs_dir_t_ *dir, - uint32_t mask, lfs_tag_t tag, lfs_entry_t_ *entry) { - int res = lfs_dir_traverse_(lfs, dir, lfs_dir_getter, - &(struct lfs_dir_getter){mask, tag, entry}); +static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, + uint32_t mask, lfs_entry_t *entry) { + int res = lfs_dir_traverse(lfs, dir, lfs_dir_getter, + &(struct lfs_dir_getter){mask, entry->tag, entry}); if (res < 0) { return res; } @@ -1245,11 +1228,11 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { return 0; } -/*static*/ int lfs_dir_getbuffer_(lfs_t *lfs, lfs_dir_t_ *dir, - uint32_t mask, lfs_tag_t tag, lfs_entry_t_ *entry) { +static int lfs_dir_getbuffer(lfs_t *lfs, lfs_dir_t *dir, + uint32_t mask, lfs_entry_t *entry) { void *buffer = entry->u.buffer; - lfs_size_t size = lfs_tag_size(tag); - int err = lfs_dir_get_(lfs, dir, mask, tag, entry); + lfs_size_t size = lfs_tag_size(entry->tag); + int err = lfs_dir_get(lfs, dir, mask, entry); if (err) { return err; } @@ -1268,10 +1251,49 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t_ entry) { return 0; } -/*static*/ int lfs_dir_getentry_(lfs_t *lfs, lfs_dir_t_ *dir, - uint32_t mask, lfs_tag_t tag, lfs_entry_t_ *entry) { +static int lfs_dir_getentry(lfs_t *lfs, lfs_dir_t *dir, + uint32_t mask, lfs_tag_t tag, lfs_entry_t *entry) { + entry->tag = tag | sizeof(entry->u); entry->u.buffer = &entry->u; - return lfs_dir_getbuffer_(lfs, dir, mask, tag, entry); + int err = lfs_dir_getbuffer(lfs, dir, mask, entry); + if (err && err != LFS_ERR_RANGE) { + return err; + } + + return 0; +} + +static int lfs_dir_getinfo(lfs_t *lfs, lfs_dir_t *dir, + int16_t id, struct lfs_info *info) { + if (id < 0) { + // special case for root + strcpy(info->name, "/"); + info->type = LFS_TYPE_DIR; + return 0; + } + + lfs_entry_t entry; + int err = lfs_dir_getentry(lfs, dir, 0x701ff000, + lfs_mktag(LFS_TYPE_REG, id, 0), &entry); + if (err) { + return err; + } + + info->type = lfs_tag_subtype(entry.tag); + if (lfs_tag_type(entry.tag) == (LFS_TYPE_REG | LFS_STRUCT_CTZ)) { + info->size = entry.u.ctz.size; + } else if (lfs_tag_type(entry.tag) == (LFS_TYPE_REG | LFS_STRUCT_INLINE)) { + info->size = lfs_tag_size(entry.tag); + } + + err = lfs_dir_getbuffer(lfs, dir, 0x7ffff000, &(lfs_entry_t){ + lfs_mktag(LFS_TYPE_NAME, id, lfs->cfg->name_size+1), + .u.buffer=info->name}); + if (err) { + return err; + } + + return 0; } struct lfs_dir_finder { @@ -1280,10 +1302,10 @@ struct lfs_dir_finder { int16_t id; }; -static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { +static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t entry) { struct lfs_dir_finder *find = p; - if (lfs_tag_type(entry.tag) == LFS_TYPE_NAME_ && + if (lfs_tag_type(entry.tag) == LFS_TYPE_NAME && lfs_tag_size(entry.tag) == find->len) { int res = lfs_bd_cmp(lfs, entry.u.d.block, entry.u.d.off, find->name, find->len); @@ -1295,7 +1317,7 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { // found a match find->id = lfs_tag_id(entry.tag); } - } else if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE_) { + } else if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE) { if (lfs_tag_id(entry.tag) == find->id) { find->id = -1; } else if (lfs_tag_id(entry.tag) < find->id) { @@ -1306,16 +1328,18 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { return 0; } -/*static*/ int lfs_dir_find_(lfs_t *lfs, lfs_dir_t_ *dir, - const char **path, lfs_entry_t_ *entry) { +// TODO drop others, make this only return id, also make get take in only entry to populate (with embedded tag) +static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, + const char **path, int16_t *id) { + lfs_entry_t entry = { + .u.pair[0] = lfs->root[0], + .u.pair[1] = lfs->root[1], + }; + struct lfs_dir_finder find = { .name = *path, }; - // TODO make superblock - entry->u.pair[0] = lfs->root[0]; - entry->u.pair[1] = lfs->root[1]; - while (true) { nextname: // skip slashes @@ -1325,10 +1349,8 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { // special case for root dir if (find.name[0] == '\0') { // TODO set up root? - entry->tag = LFS_STRUCT_DIR | LFS_TYPE_DIR; - entry->u.pair[0] = lfs->root[0]; - entry->u.pair[1] = lfs->root[1]; - return lfs_mktag(LFS_TYPE_DIR_, 0x1ff, 0); + *id = -1; + return 0; } // skip '.' and root '..' @@ -1368,7 +1390,7 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { // find path while (true) { find.id = -1; - int err = lfs_dir_fetchwith_(lfs, dir, entry->u.pair, + int err = lfs_dir_fetchwith(lfs, dir, entry.u.pair, lfs_dir_finder, &find); if (err) { return err; @@ -1383,8 +1405,8 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { return LFS_ERR_NOENT; } - entry->u.pair[0] = dir->tail[0]; - entry->u.pair[1] = dir->tail[1]; + entry.u.pair[0] = dir->tail[0]; + entry.u.pair[1] = dir->tail[1]; } // TODO handle moves @@ -1398,867 +1420,871 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t_ entry) { // entry->d.type &= ~LFS_STRUCT_MOVED; // } - // TODO optimize grab for inline files and like? - // TODO would this mean more code? - // grab the entry data - int err = lfs_dir_getentry_(lfs, dir, 0x701ff000, - lfs_mktag(LFS_TYPE_REG_, find.id, 0), entry); - if (err && err != LFS_ERR_RANGE) { - return err; - } - + *id = find.id; find.name += find.len; find.name += strspn(find.name, "/"); if (find.name[0] == '\0') { return 0; } - // continue on if we hit a directory - // TODO update with what's on master? - if (lfs_tag_type(entry->tag) != LFS_TYPE_DIR_) { - return LFS_ERR_NOTDIR; - } - } -} - -/*static*/ int lfs_dir_findbuffer_(lfs_t *lfs, lfs_dir_t_ *dir, - const char **path, lfs_entry_t_ *entry) { - void *buffer = entry->u.buffer; - lfs_size_t size = lfs_tag_size(entry->tag); - int err = lfs_dir_find_(lfs, dir, path, entry); - if (err) { - return err; - } - - lfs_size_t diff = lfs_min(size, lfs_tag_size(entry->tag)); - memset((uint8_t*)buffer + diff, 0, size - diff); - err = lfs_bd_read(lfs, entry->u.d.block, entry->u.d.off, buffer, diff); - if (err) { - return err; - } - - if (lfs_tag_size(entry->tag) > size) { - return LFS_ERR_RANGE; - } - - return 0; -} - -/*static*/ int lfs_dir_findentry_(lfs_t *lfs, lfs_dir_t_ *dir, - const char **path, lfs_entry_t_ *entry) { - entry->tag = sizeof(entry->u); - entry->u.buffer = &entry->u; - return lfs_dir_findbuffer_(lfs, dir, path, entry); -} - - -////////////////////////////////////////////////////////// - -static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { - // allocate pair of dir blocks - for (int i = 0; i < 2; i++) { - int err = lfs_alloc(lfs, &dir->pair[i]); - if (err) { - return err; - } - } - - // rather than clobbering one of the blocks we just pretend - // the revision may be valid - int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->d.rev, 4); - dir->d.rev = lfs_fromle32(dir->d.rev); - if (err) { - return err; - } - - // set defaults - dir->d.rev += 1; - dir->d.size = sizeof(dir->d)+4; - dir->d.tail[0] = 0xffffffff; - dir->d.tail[1] = 0xffffffff; - dir->off = sizeof(dir->d); - - // don't write out yet, let caller take care of that - return 0; -} - -static int lfs_dir_fetch(lfs_t *lfs, - lfs_dir_t *dir, const lfs_block_t pair[2]) { - // copy out pair, otherwise may be aliasing dir - const lfs_block_t tpair[2] = {pair[0], pair[1]}; - bool valid = false; - - // check both blocks for the most recent revision - for (int i = 0; i < 2; i++) { - struct lfs_disk_dir test; - int err = lfs_bd_read(lfs, tpair[i], 0, &test, sizeof(test)); - lfs_dir_fromle32(&test); - if (err) { - return err; - } - - if (valid && lfs_scmp(test.rev, dir->d.rev) < 0) { - continue; - } - - if ((0x7fffffff & test.size) < sizeof(test)+4 || - (0x7fffffff & test.size) > lfs->cfg->block_size) { - continue; - } - - uint32_t crc = 0xffffffff; - lfs_dir_tole32(&test); - lfs_crc(&crc, &test, sizeof(test)); - lfs_dir_fromle32(&test); - err = lfs_bd_crc(lfs, tpair[i], sizeof(test), - (0x7fffffff & test.size) - sizeof(test), &crc); + // TODO optimize grab for inline files and like? + // TODO would this mean more code? + // grab the entry data + int err = lfs_dir_getentry(lfs, dir, 0x701ff000, + lfs_mktag(LFS_TYPE_REG, find.id, 0), &entry); if (err) { return err; } - if (crc != 0) { - continue; - } - - valid = true; - - // setup dir in case it's valid - dir->pair[0] = tpair[(i+0) % 2]; - dir->pair[1] = tpair[(i+1) % 2]; - dir->off = sizeof(dir->d); - dir->d = test; - } - - if (!valid) { - LFS_ERROR("Corrupted dir pair at %d %d", tpair[0], tpair[1]); - return LFS_ERR_CORRUPT; - } - - return 0; -} - -struct lfs_region { - enum { - LFS_FROM_MEM, - LFS_FROM_REGION, - LFS_FROM_ATTRS, - } type; - - lfs_off_t oldoff; - lfs_size_t oldsize; - const void *buffer; - lfs_size_t newsize; -}; - -struct lfs_region_attrs { - const struct lfs_attr *attrs; - int count; -}; - -struct lfs_region_region { - lfs_block_t block; - lfs_off_t off; - struct lfs_region *regions; - int count; -}; - -static int lfs_commit_region(lfs_t *lfs, uint32_t *crc, - lfs_block_t oldblock, lfs_off_t oldoff, - lfs_block_t newblock, lfs_off_t newoff, - lfs_off_t regionoff, lfs_size_t regionsize, - const struct lfs_region *regions, int count) { - int i = 0; - lfs_size_t newend = newoff + regionsize; - while (newoff < newend) { - // commit from different types of regions - if (i < count && regions[i].oldoff == oldoff - regionoff) { - switch (regions[i].type) { - case LFS_FROM_MEM: { - lfs_crc(crc, regions[i].buffer, regions[i].newsize); - int err = lfs_bd_prog(lfs, newblock, newoff, - regions[i].buffer, regions[i].newsize); - if (err) { - return err; - } - newoff += regions[i].newsize; - oldoff += regions[i].oldsize; - break; - } - case LFS_FROM_REGION: { - const struct lfs_region_region *disk = regions[i].buffer; - int err = lfs_commit_region(lfs, crc, - disk->block, disk->off, - newblock, newoff, - disk->off, regions[i].newsize, - disk->regions, disk->count); - if (err) { - return err; - } - newoff += regions[i].newsize; - oldoff -= regions[i].oldsize; - break; - } - case LFS_FROM_ATTRS: { - const struct lfs_region_attrs *attrs = regions[i].buffer; - - // order doesn't matter, so we write new attrs first. this - // is still O(n^2) but only O(n) disk access - for (int j = 0; j < attrs->count; j++) { - if (attrs->attrs[j].size == 0) { - continue; - } - - lfs_entry_attr_t attr; - attr.d.type = attrs->attrs[j].type; - attr.d.len = attrs->attrs[j].size; - - lfs_crc(crc, &attr.d, sizeof(attr.d)); - int err = lfs_bd_prog(lfs, newblock, newoff, - &attr.d, sizeof(attr.d)); - if (err) { - return err; - } - - lfs_crc(crc, - attrs->attrs[j].buffer, attrs->attrs[j].size); - err = lfs_bd_prog(lfs, newblock, newoff+sizeof(attr.d), - attrs->attrs[j].buffer, attrs->attrs[j].size); - if (err) { - return err; - } - - newoff += 2+attrs->attrs[j].size; - } - - // copy over attributes without updates - lfs_off_t oldend = oldoff + regions[i].oldsize; - while (oldoff < oldend) { - lfs_entry_attr_t attr; - int err = lfs_bd_read(lfs, oldblock, oldoff, - &attr.d, sizeof(attr.d)); - if (err) { - return err; - } - - bool updating = false; - for (int j = 0; j < attrs->count; j++) { - if (attr.d.type == attrs->attrs[j].type) { - updating = true; - } - } - - if (!updating) { - err = lfs_commit_region(lfs, crc, - oldblock, oldoff, - newblock, newoff, - 0, 2+attr.d.len, - NULL, 0); - if (err) { - return err; - } - - newoff += 2+attr.d.len; - } - - oldoff += 2+attr.d.len; - } - - break; - } - } - - i += 1; - } else { - // copy data from old block if not covered by entry - uint8_t data; - int err = lfs_bd_read(lfs, oldblock, oldoff, &data, 1); - if (err) { - return err; - } - - lfs_crc(crc, &data, 1); - err = lfs_bd_prog(lfs, newblock, newoff, &data, 1); - if (err) { - return err; - } - - oldoff += 1; - newoff += 1; + // continue on if we hit a directory + // TODO update with what's on master? + if (lfs_tag_subtype(entry.tag) != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; } } - - // sanity check our commit math - LFS_ASSERT(newoff == newend); - return 0; } -static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, - const struct lfs_region *regions, int count) { - // state for copying over - const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; - bool relocated = false; +//static int lfs_dir_findbuffer(lfs_t *lfs, lfs_dir_t *dir, +// const char **path, lfs_entry_t *entry) { +// void *buffer = entry->u.buffer; +// lfs_size_t size = lfs_tag_size(entry->tag); +// int err = lfs_dir_find(lfs, dir, path, entry); +// if (err) { +// return err; +// } +// +// lfs_size_t diff = lfs_min(size, lfs_tag_size(entry->tag)); +// memset((uint8_t*)buffer + diff, 0, size - diff); +// // TODO hmm +// if (lfs_tag_valid(entry->tag)) { +// memcpy(buffer, entry->u.buffer, diff); +// } else { +// err = lfs_bd_read(lfs, entry->u.d.block, entry->u.d.off, buffer, diff); +// if (err) { +// return err; +// } +// } +// +// if (lfs_tag_size(entry->tag) > size) { +// return LFS_ERR_RANGE; +// } +// +// return 0; +//} +// +//static int lfs_dir_findentry(lfs_t *lfs, lfs_dir_t *dir, +// const char **path, lfs_entry_t *entry) { +// entry->tag = sizeof(entry->u); +// entry->u.buffer = &entry->u; +// return lfs_dir_findbuffer(lfs, dir, path, entry); +//} - // increment revision count - dir->d.rev += 1; - // keep pairs in order such that pair[0] is most recent - lfs_pairswap(dir->pair); - for (int i = 0; i < count; i++) { - dir->d.size += regions[i].newsize; - dir->d.size -= regions[i].oldsize; - } +////////////////////////////////////////////////////////// - while (true) { - if (true) { - int err = lfs_bd_erase(lfs, dir->pair[0]); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // commit header - uint32_t crc = 0xffffffff; - lfs_dir_tole32(&dir->d); - lfs_crc(&crc, &dir->d, sizeof(dir->d)); - err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); - lfs_dir_fromle32(&dir->d); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // commit entry - err = lfs_commit_region(lfs, &crc, - dir->pair[1], sizeof(dir->d), - dir->pair[0], sizeof(dir->d), - 0, (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, - regions, count); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // commit crc - crc = lfs_tole32(crc); - err = lfs_bd_prog(lfs, dir->pair[0], - (0x7fffffff & dir->d.size)-4, &crc, 4); - crc = lfs_fromle32(crc); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - err = lfs_bd_sync(lfs); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // successful commit, check checksum to make sure - uint32_t ncrc = 0xffffffff; - err = lfs_bd_crc(lfs, dir->pair[0], 0, - (0x7fffffff & dir->d.size)-4, &ncrc); - if (err) { - return err; - } - - if (ncrc != crc) { - goto relocate; - } - } - - break; - -relocate: - //commit was corrupted - LFS_DEBUG("Bad block at %d", dir->pair[0]); - - // drop caches and prepare to relocate block - relocated = true; - lfs->pcache.block = 0xffffffff; - - // can't relocate superblock, filesystem is now frozen - if (lfs_paircmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { - LFS_WARN("Superblock %d has become unwritable", oldpair[0]); - return LFS_ERR_CORRUPT; - } - - // relocate half of pair - int err = lfs_alloc(lfs, &dir->pair[0]); - if (err) { - return err; - } - } - - if (relocated) { - // update references if we relocated - LFS_DEBUG("Relocating %d %d to %d %d", - oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); - int err = lfs_relocate(lfs, oldpair, dir->pair); - if (err) { - return err; - } - } - - // shift over any directories that are affected - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - d->pair[0] = dir->pair[0]; - d->pair[1] = dir->pair[1]; - } - } - - return 0; -} - -static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, - lfs_off_t off, void *buffer, lfs_size_t size) { - return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); -} - -static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, - struct lfs_region *regions, int count) { - return -999; -// lfs_ssize_t diff = 0; -// for (int i = 0; i < count; i++) { -// diff += regions[i].newsize; -// diff -= regions[i].oldsize; +//static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { +// // allocate pair of dir blocks +// for (int i = 0; i < 2; i++) { +// int err = lfs_alloc(lfs, &dir->pair[i]); +// if (err) { +// return err; +// } +// } +// +// // rather than clobbering one of the blocks we just pretend +// // the revision may be valid +// int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->d.rev, 4); +// dir->d.rev = lfs_fromle32(dir->d.rev); +// if (err) { +// return err; +// } +// +// // set defaults +// dir->d.rev += 1; +// dir->d.size = sizeof(dir->d)+4; +// dir->d.tail[0] = 0xffffffff; +// dir->d.tail[1] = 0xffffffff; +// dir->off = sizeof(dir->d); +// +// // don't write out yet, let caller take care of that +// return 0; +//} +// +//static int lfs_dir_fetch(lfs_t *lfs, +// lfs_dir_t *dir, const lfs_block_t pair[2]) { +// // copy out pair, otherwise may be aliasing dir +// const lfs_block_t tpair[2] = {pair[0], pair[1]}; +// bool valid = false; +// +// // check both blocks for the most recent revision +// for (int i = 0; i < 2; i++) { +// struct lfs_disk_dir test; +// int err = lfs_bd_read(lfs, tpair[i], 0, &test, sizeof(test)); +// lfs_dir_fromle32(&test); +// if (err) { +// return err; +// } +// +// if (valid && lfs_scmp(test.rev, dir->d.rev) < 0) { +// continue; +// } +// +// if ((0x7fffffff & test.size) < sizeof(test)+4 || +// (0x7fffffff & test.size) > lfs->cfg->block_size) { +// continue; +// } +// +// uint32_t crc = 0xffffffff; +// lfs_dir_tole32(&test); +// lfs_crc(&crc, &test, sizeof(test)); +// lfs_dir_fromle32(&test); +// err = lfs_bd_crc(lfs, tpair[i], sizeof(test), +// (0x7fffffff & test.size) - sizeof(test), &crc); +// if (err) { +// return err; +// } +// +// if (crc != 0) { +// continue; +// } +// +// valid = true; +// +// // setup dir in case it's valid +// dir->pair[0] = tpair[(i+0) % 2]; +// dir->pair[1] = tpair[(i+1) % 2]; +// dir->off = sizeof(dir->d); +// dir->d = test; // } // -// lfs_size_t oldsize = entry->size; -// if (entry->off == 0) { -// entry->off = (0x7fffffff & dir->d.size) - 4; +// if (!valid) { +// LFS_ERROR("Corrupted dir pair at %d %d", tpair[0], tpair[1]); +// return LFS_ERR_CORRUPT; // } // -// if ((0x7fffffff & dir->d.size) + diff > lfs->cfg->block_size) { -// lfs_dir_t olddir = *dir; -// lfs_off_t oldoff = entry->off; +// return 0; +//} +// +//struct lfs_region { +// enum { +// LFS_FROM_MEM, +// LFS_FROM_REGION, +// LFS_FROM_ATTRS, +// } type; +// +// lfs_off_t oldoff; +// lfs_size_t oldsize; +// const void *buffer; +// lfs_size_t newsize; +//}; +// +//struct lfs_region_attrs { +// const struct lfs_attr *attrs; +// int count; +//}; +// +//struct lfs_region_region { +// lfs_block_t block; +// lfs_off_t off; +// struct lfs_region *regions; +// int count; +//}; +// +//static int lfs_commit_region(lfs_t *lfs, uint32_t *crc, +// lfs_block_t oldblock, lfs_off_t oldoff, +// lfs_block_t newblock, lfs_off_t newoff, +// lfs_off_t regionoff, lfs_size_t regionsize, +// const struct lfs_region *regions, int count) { +// int i = 0; +// lfs_size_t newend = newoff + regionsize; +// while (newoff < newend) { +// // commit from different types of regions +// if (i < count && regions[i].oldoff == oldoff - regionoff) { +// switch (regions[i].type) { +// case LFS_FROM_MEM: { +// lfs_crc(crc, regions[i].buffer, regions[i].newsize); +// int err = lfs_bd_prog(lfs, newblock, newoff, +// regions[i].buffer, regions[i].newsize); +// if (err) { +// return err; +// } +// newoff += regions[i].newsize; +// oldoff += regions[i].oldsize; +// break; +// } +// case LFS_FROM_REGION: { +// const struct lfs_region_region *disk = regions[i].buffer; +// int err = lfs_commit_region(lfs, crc, +// disk->block, disk->off, +// newblock, newoff, +// disk->off, regions[i].newsize, +// disk->regions, disk->count); +// if (err) { +// return err; +// } +// newoff += regions[i].newsize; +// oldoff -= regions[i].oldsize; +// break; +// } +// case LFS_FROM_ATTRS: { +// const struct lfs_region_attrs *attrs = regions[i].buffer; +// +// // order doesn't matter, so we write new attrs first. this +// // is still O(n^2) but only O(n) disk access +// for (int j = 0; j < attrs->count; j++) { +// if (attrs->attrs[j].size == 0) { +// continue; +// } +// +// lfs_entry_attr_t attr; +// attr.d.type = attrs->attrs[j].type; +// attr.d.len = attrs->attrs[j].size; +// +// lfs_crc(crc, &attr.d, sizeof(attr.d)); +// int err = lfs_bd_prog(lfs, newblock, newoff, +// &attr.d, sizeof(attr.d)); +// if (err) { +// return err; +// } +// +// lfs_crc(crc, +// attrs->attrs[j].buffer, attrs->attrs[j].size); +// err = lfs_bd_prog(lfs, newblock, newoff+sizeof(attr.d), +// attrs->attrs[j].buffer, attrs->attrs[j].size); +// if (err) { +// return err; +// } +// +// newoff += 2+attrs->attrs[j].size; +// } +// +// // copy over attributes without updates +// lfs_off_t oldend = oldoff + regions[i].oldsize; +// while (oldoff < oldend) { +// lfs_entry_attr_t attr; +// int err = lfs_bd_read(lfs, oldblock, oldoff, +// &attr.d, sizeof(attr.d)); +// if (err) { +// return err; +// } +// +// bool updating = false; +// for (int j = 0; j < attrs->count; j++) { +// if (attr.d.type == attrs->attrs[j].type) { +// updating = true; +// } +// } +// +// if (!updating) { +// err = lfs_commit_region(lfs, crc, +// oldblock, oldoff, +// newblock, newoff, +// 0, 2+attr.d.len, +// NULL, 0); +// if (err) { +// return err; +// } +// +// newoff += 2+attr.d.len; +// } +// +// oldoff += 2+attr.d.len; +// } // -// if (oldsize) { -// // mark as moving -// uint8_t type; -// int err = lfs_dir_get(lfs, &olddir, oldoff, &type, 1); +// break; +// } +// } +// +// i += 1; +// } else { +// // copy data from old block if not covered by entry +// uint8_t data; +// int err = lfs_bd_read(lfs, oldblock, oldoff, &data, 1); // if (err) { // return err; // } // -// type |= LFS_STRUCT_MOVED; -// err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ -// {LFS_FROM_MEM, oldoff, 1, &type, 1}}, 1); +// lfs_crc(crc, &data, 1); +// err = lfs_bd_prog(lfs, newblock, newoff, &data, 1); // if (err) { // return err; // } +// +// oldoff += 1; +// newoff += 1; // } +// } +// +// // sanity check our commit math +// LFS_ASSERT(newoff == newend); +// return 0; +//} // -// lfs_dir_t pdir = olddir; +//static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, +// const struct lfs_region *regions, int count) { +// // state for copying over +// const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; +// bool relocated = false; // -// // find available block or create a new one -// while ((0x7fffffff & dir->d.size) + oldsize + diff -// > lfs->cfg->block_size) { -// // we need to allocate a new dir block -// if (!(0x80000000 & dir->d.size)) { -// pdir = *dir; -// int err = lfs_dir_alloc(lfs, dir); -// if (err) { -// return err; -// } +// // increment revision count +// dir->d.rev += 1; // -// dir->d.tail[0] = pdir.d.tail[0]; -// dir->d.tail[1] = pdir.d.tail[1]; +// // keep pairs in order such that pair[0] is most recent +// lfs_pairswap(dir->pair); +// for (int i = 0; i < count; i++) { +// dir->d.size += regions[i].newsize; +// dir->d.size -= regions[i].oldsize; +// } // -// break; +// while (true) { +// if (true) { +// int err = lfs_bd_erase(lfs, dir->pair[0]); +// if (err) { +// if (err == LFS_ERR_CORRUPT) { +// goto relocate; +// } +// return err; // } // -// int err = lfs_dir_fetch(lfs, dir, dir->d.tail); +// // commit header +// uint32_t crc = 0xffffffff; +// lfs_dir_tole32(&dir->d); +// lfs_crc(&crc, &dir->d, sizeof(dir->d)); +// err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); +// lfs_dir_fromle32(&dir->d); // if (err) { +// if (err == LFS_ERR_CORRUPT) { +// goto relocate; +// } // return err; // } -// } // -// // writing out new entry -// entry->off = dir->d.size - 4; -// entry->size += diff; -// int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ -// {LFS_FROM_REGION, entry->off, 0, &(struct lfs_region_region){ -// olddir.pair[0], oldoff, -// regions, count}, entry->size}}, 1); -// if (err) { -// return err; -// } +// // commit entry +// err = lfs_commit_region(lfs, &crc, +// dir->pair[1], sizeof(dir->d), +// dir->pair[0], sizeof(dir->d), +// 0, (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, +// regions, count); +// if (err) { +// if (err == LFS_ERR_CORRUPT) { +// goto relocate; +// } +// return err; +// } // -// // update pred dir, unless pred == old we can coalesce -// if (!oldsize || lfs_paircmp(pdir.pair, olddir.pair) != 0) { -// pdir.d.size |= 0x80000000; -// pdir.d.tail[0] = dir->pair[0]; -// pdir.d.tail[1] = dir->pair[1]; +// // commit crc +// crc = lfs_tole32(crc); +// err = lfs_bd_prog(lfs, dir->pair[0], +// (0x7fffffff & dir->d.size)-4, &crc, 4); +// crc = lfs_fromle32(crc); +// if (err) { +// if (err == LFS_ERR_CORRUPT) { +// goto relocate; +// } +// return err; +// } // -// err = lfs_dir_commit(lfs, &pdir, NULL, 0); +// err = lfs_bd_sync(lfs); // if (err) { +// if (err == LFS_ERR_CORRUPT) { +// goto relocate; +// } // return err; // } -// } else if (oldsize) { -// olddir.d.size |= 0x80000000; -// olddir.d.tail[0] = dir->pair[0]; -// olddir.d.tail[1] = dir->pair[1]; -// } // -// // remove old entry -// if (oldsize) { -// lfs_entry_t oldentry; -// oldentry.off = oldoff; -// err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, oldsize, NULL, 0}}, 1); +// // successful commit, check checksum to make sure +// uint32_t ncrc = 0xffffffff; +// err = lfs_bd_crc(lfs, dir->pair[0], 0, +// (0x7fffffff & dir->d.size)-4, &ncrc); // if (err) { // return err; // } +// +// if (ncrc != crc) { +// goto relocate; +// } // } // -// goto shift; +// break; +// +//relocate: +// //commit was corrupted +// LFS_DEBUG("Bad block at %d", dir->pair[0]); +// +// // drop caches and prepare to relocate block +// relocated = true; +// lfs->pcache.block = 0xffffffff; +// +// // can't relocate superblock, filesystem is now frozen +// if (lfs_paircmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { +// LFS_WARN("Superblock %d has become unwritable", oldpair[0]); +// return LFS_ERR_CORRUPT; +// } +// +// // relocate half of pair +// int err = lfs_alloc(lfs, &dir->pair[0]); +// if (err) { +// return err; +// } // } // -// if ((0x7fffffff & dir->d.size) + diff == sizeof(dir->d)+4) { -// lfs_dir_t pdir; -// int res = lfs_pred(lfs, dir->pair, &pdir); -// if (res < 0) { -// return res; +// if (relocated) { +// // update references if we relocated +// LFS_DEBUG("Relocating %d %d to %d %d", +// oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); +// int err = lfs_relocate(lfs, oldpair, dir->pair); +// if (err) { +// return err; // } +// } // -// if (pdir.d.size & 0x80000000) { -// pdir.d.size &= dir->d.size | 0x7fffffff; -// pdir.d.tail[0] = dir->d.tail[0]; -// pdir.d.tail[1] = dir->d.tail[1]; -// int err = lfs_dir_commit(lfs, &pdir, NULL, 0); -// if (err) { -// return err; -// } -// goto shift; +// // shift over any directories that are affected +// for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { +// if (lfs_paircmp(d->pair, dir->pair) == 0) { +// d->pair[0] = dir->pair[0]; +// d->pair[1] = dir->pair[1]; // } // } // -// for (int i = 0; i < count; i++) { -// regions[i].oldoff += entry->off; +// return 0; +//} +// +//static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, +// lfs_off_t off, void *buffer, lfs_size_t size) { +// return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); +//} +// +//static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, +// struct lfs_region *regions, int count) { +// return -999; +//// lfs_ssize_t diff = 0; +//// for (int i = 0; i < count; i++) { +//// diff += regions[i].newsize; +//// diff -= regions[i].oldsize; +//// } +//// +//// lfs_size_t oldsize = entry->size; +//// if (entry->off == 0) { +//// entry->off = (0x7fffffff & dir->d.size) - 4; +//// } +//// +//// if ((0x7fffffff & dir->d.size) + diff > lfs->cfg->block_size) { +//// lfs_dir_t olddir = *dir; +//// lfs_off_t oldoff = entry->off; +//// +//// if (oldsize) { +//// // mark as moving +//// uint8_t type; +//// int err = lfs_dir_get(lfs, &olddir, oldoff, &type, 1); +//// if (err) { +//// return err; +//// } +//// +//// type |= LFS_STRUCT_MOVED; +//// err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ +//// {LFS_FROM_MEM, oldoff, 1, &type, 1}}, 1); +//// if (err) { +//// return err; +//// } +//// } +//// +//// lfs_dir_t pdir = olddir; +//// +//// // find available block or create a new one +//// while ((0x7fffffff & dir->d.size) + oldsize + diff +//// > lfs->cfg->block_size) { +//// // we need to allocate a new dir block +//// if (!(0x80000000 & dir->d.size)) { +//// pdir = *dir; +//// int err = lfs_dir_alloc(lfs, dir); +//// if (err) { +//// return err; +//// } +//// +//// dir->d.tail[0] = pdir.d.tail[0]; +//// dir->d.tail[1] = pdir.d.tail[1]; +//// +//// break; +//// } +//// +//// int err = lfs_dir_fetch(lfs, dir, dir->d.tail); +//// if (err) { +//// return err; +//// } +//// } +//// +//// // writing out new entry +//// entry->off = dir->d.size - 4; +//// entry->size += diff; +//// int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ +//// {LFS_FROM_REGION, entry->off, 0, &(struct lfs_region_region){ +//// olddir.pair[0], oldoff, +//// regions, count}, entry->size}}, 1); +//// if (err) { +//// return err; +//// } +//// +//// // update pred dir, unless pred == old we can coalesce +//// if (!oldsize || lfs_paircmp(pdir.pair, olddir.pair) != 0) { +//// pdir.d.size |= 0x80000000; +//// pdir.d.tail[0] = dir->pair[0]; +//// pdir.d.tail[1] = dir->pair[1]; +//// +//// err = lfs_dir_commit(lfs, &pdir, NULL, 0); +//// if (err) { +//// return err; +//// } +//// } else if (oldsize) { +//// olddir.d.size |= 0x80000000; +//// olddir.d.tail[0] = dir->pair[0]; +//// olddir.d.tail[1] = dir->pair[1]; +//// } +//// +//// // remove old entry +//// if (oldsize) { +//// lfs_entry_t oldentry; +//// oldentry.off = oldoff; +//// err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ +//// {LFS_FROM_MEM, 0, oldsize, NULL, 0}}, 1); +//// if (err) { +//// return err; +//// } +//// } +//// +//// goto shift; +//// } +//// +//// if ((0x7fffffff & dir->d.size) + diff == sizeof(dir->d)+4) { +//// lfs_dir_t pdir; +//// int res = lfs_pred(lfs, dir->pair, &pdir); +//// if (res < 0) { +//// return res; +//// } +//// +//// if (pdir.d.size & 0x80000000) { +//// pdir.d.size &= dir->d.size | 0x7fffffff; +//// pdir.d.tail[0] = dir->d.tail[0]; +//// pdir.d.tail[1] = dir->d.tail[1]; +//// int err = lfs_dir_commit(lfs, &pdir, NULL, 0); +//// if (err) { +//// return err; +//// } +//// goto shift; +//// } +//// } +//// +//// for (int i = 0; i < count; i++) { +//// regions[i].oldoff += entry->off; +//// } +//// +//// int err = lfs_dir_commit(lfs, dir, regions, count); +//// if (err) { +//// return err; +//// } +//// +//// entry->size += diff; +//// +////shift: +//// // shift over any files/directories that are affected +//// for (lfs_file_t *f = lfs->files; f; f = f->next) { +//// if (lfs_paircmp(f->pair, dir->pair) == 0) { +//// if (f->pairoff == entry->off && entry->size == 0) { +//// f->pair[0] = 0xffffffff; +//// f->pair[1] = 0xffffffff; +//// } else if (f->pairoff > entry->off) { +//// f->pairoff += diff; +//// } +//// } +//// } +//// +//// for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { +//// if (lfs_paircmp(d->pair, dir->pair) == 0) { +//// if (d->off > entry->off) { +//// d->off += diff; +//// d->pos += diff; +//// } +//// } +//// } +//// +//// return 0; +//} +// +//static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { +// while (dir->off >= (0x7fffffff & dir->d.size)-4) { +// if (!(0x80000000 & dir->d.size)) { +// entry->off = dir->off; +// return LFS_ERR_NOENT; +// } +// +// int err = lfs_dir_fetch(lfs, dir, dir->d.tail); +// if (err) { +// return err; +// } +// +// dir->off = sizeof(dir->d); +// dir->pos += sizeof(dir->d) + 4; // } // -// int err = lfs_dir_commit(lfs, dir, regions, count); +// int err = lfs_dir_get(lfs, dir, dir->off, &entry->d, sizeof(entry->d)); +// lfs_entry_fromle32(&entry->d); // if (err) { // return err; // } // -// entry->size += diff; -// -//shift: -// // shift over any files/directories that are affected -// for (lfs_file_t *f = lfs->files; f; f = f->next) { -// if (lfs_paircmp(f->pair, dir->pair) == 0) { -// if (f->pairoff == entry->off && entry->size == 0) { -// f->pair[0] = 0xffffffff; -// f->pair[1] = 0xffffffff; -// } else if (f->pairoff > entry->off) { -// f->pairoff += diff; +// entry->off = dir->off; +// entry->size = lfs_entry_size(entry); +// dir->off += entry->size; +// dir->pos += entry->size; +// return 0; +//} +// +//static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, +// lfs_entry_t *entry, const char **path) { +// const char *pathname = *path; +// lfs_size_t pathlen; +// +// while (true) { +// nextname: +// // skip slashes +// pathname += strspn(pathname, "/"); +// pathlen = strcspn(pathname, "/"); +// +// // special case for root dir +// if (pathname[0] == '\0') { +// *entry = (lfs_entry_t){ +// .d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR, +// .d.u.dir[0] = lfs->root[0], +// .d.u.dir[1] = lfs->root[1], +// }; +// return 0; +// } +// +// // skip '.' and root '..' +// if ((pathlen == 1 && memcmp(pathname, ".", 1) == 0) || +// (pathlen == 2 && memcmp(pathname, "..", 2) == 0)) { +// pathname += pathlen; +// goto nextname; +// } +// +// // skip if matched by '..' in name +// const char *suffix = pathname + pathlen; +// lfs_size_t sufflen; +// int depth = 1; +// while (true) { +// suffix += strspn(suffix, "/"); +// sufflen = strcspn(suffix, "/"); +// if (sufflen == 0) { +// break; +// } +// +// if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { +// depth -= 1; +// if (depth == 0) { +// pathname = suffix + sufflen; +// goto nextname; +// } +// } else { +// depth += 1; // } +// +// suffix += sufflen; // } -// } // -// for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { -// if (lfs_paircmp(d->pair, dir->pair) == 0) { -// if (d->off > entry->off) { -// d->off += diff; -// d->pos += diff; +// // update what we've found +// *path = pathname; +// +// // find path +// while (true) { +// int err = lfs_dir_next(lfs, dir, entry); +// if (err) { +// return err; +// } +// +// if (((0xf & entry->d.type) != LFS_TYPE_REG && +// (0xf & entry->d.type) != LFS_TYPE_DIR) || +// entry->d.nlen != pathlen) { +// continue; +// } +// +// int res = lfs_bd_cmp(lfs, dir->pair[0], +// entry->off + entry->size - pathlen, +// pathname, pathlen); +// if (res < 0) { +// return res; +// } +// +// // found match +// if (res) { +// break; // } // } -// } // -// return 0; -} - -static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { - while (dir->off >= (0x7fffffff & dir->d.size)-4) { - if (!(0x80000000 & dir->d.size)) { - entry->off = dir->off; - return LFS_ERR_NOENT; - } - - int err = lfs_dir_fetch(lfs, dir, dir->d.tail); - if (err) { - return err; - } - - dir->off = sizeof(dir->d); - dir->pos += sizeof(dir->d) + 4; - } - - int err = lfs_dir_get(lfs, dir, dir->off, &entry->d, sizeof(entry->d)); - lfs_entry_fromle32(&entry->d); - if (err) { - return err; - } - - entry->off = dir->off; - entry->size = lfs_entry_size(entry); - dir->off += entry->size; - dir->pos += entry->size; - return 0; -} - -static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, const char **path) { - const char *pathname = *path; - lfs_size_t pathlen; - - while (true) { - nextname: - // skip slashes - pathname += strspn(pathname, "/"); - pathlen = strcspn(pathname, "/"); - - // special case for root dir - if (pathname[0] == '\0') { - *entry = (lfs_entry_t){ - .d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR, - .d.u.dir[0] = lfs->root[0], - .d.u.dir[1] = lfs->root[1], - }; - return 0; - } - - // skip '.' and root '..' - if ((pathlen == 1 && memcmp(pathname, ".", 1) == 0) || - (pathlen == 2 && memcmp(pathname, "..", 2) == 0)) { - pathname += pathlen; - goto nextname; - } - - // skip if matched by '..' in name - const char *suffix = pathname + pathlen; - lfs_size_t sufflen; - int depth = 1; - while (true) { - suffix += strspn(suffix, "/"); - sufflen = strcspn(suffix, "/"); - if (sufflen == 0) { - break; - } - - if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { - depth -= 1; - if (depth == 0) { - pathname = suffix + sufflen; - goto nextname; - } - } else { - depth += 1; - } - - suffix += sufflen; - } - - // update what we've found - *path = pathname; - - // find path - while (true) { - int err = lfs_dir_next(lfs, dir, entry); - if (err) { - return err; - } - - if (((0xf & entry->d.type) != LFS_TYPE_REG && - (0xf & entry->d.type) != LFS_TYPE_DIR) || - entry->d.nlen != pathlen) { - continue; - } - - int res = lfs_bd_cmp(lfs, dir->pair[0], - entry->off + entry->size - pathlen, - pathname, pathlen); - if (res < 0) { - return res; - } - - // found match - if (res) { - break; - } - } - - // check that entry has not been moved - if (entry->d.type & LFS_STRUCT_MOVED) { - int moved = lfs_moved(lfs, &entry->d.u); - if (moved < 0 || moved) { - return (moved < 0) ? moved : LFS_ERR_NOENT; - } - - entry->d.type &= ~LFS_STRUCT_MOVED; - } - - pathname += pathlen; - pathname += strspn(pathname, "/"); - if (pathname[0] == '\0') { - return 0; - } - - // continue on if we hit a directory - if ((0xf & entry->d.type) != LFS_TYPE_DIR) { - return LFS_ERR_NOTDIR; - } - - int err = lfs_dir_fetch(lfs, dir, entry->d.u.dir); - if (err) { - return err; - } - } -} - +// // check that entry has not been moved +// if (entry->d.type & LFS_STRUCT_MOVED) { +// int moved = lfs_moved(lfs, &entry->d.u); +// if (moved < 0 || moved) { +// return (moved < 0) ? moved : LFS_ERR_NOENT; +// } +// +// entry->d.type &= ~LFS_STRUCT_MOVED; +// } +// +// pathname += pathlen; +// pathname += strspn(pathname, "/"); +// if (pathname[0] == '\0') { +// return 0; +// } +// +// // continue on if we hit a directory +// if ((0xf & entry->d.type) != LFS_TYPE_DIR) { +// return LFS_ERR_NOTDIR; +// } +// +// int err = lfs_dir_fetch(lfs, dir, entry->d.u.dir); +// if (err) { +// return err; +// } +// } +//} +// /// Internal attribute operations /// -static int lfs_dir_getinfo(lfs_t *lfs, - lfs_dir_t *dir, const lfs_entry_t *entry, struct lfs_info *info) { - memset(info, 0, sizeof(*info)); - info->type = 0xf & entry->d.type; - if (entry->d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { - info->size = entry->d.u.file.size; - } else if (entry->d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = lfs_entry_elen(entry); - } - - if (lfs_paircmp(entry->d.u.dir, lfs->root) == 0) { - strcpy(info->name, "/"); - } else { - int err = lfs_dir_get(lfs, dir, - entry->off + entry->size - entry->d.nlen, - info->name, entry->d.nlen); - if (err) { - return err; - } - } - - return 0; -} - -static int lfs_dir_getattrs(lfs_t *lfs, - lfs_dir_t *dir, const lfs_entry_t *entry, - const struct lfs_attr *attrs, int count) { - // set to zero in case we can't find the attributes or size mismatch - for (int j = 0; j < count; j++) { - memset(attrs[j].buffer, 0, attrs[j].size); - } - - // search for attribute in attribute entry - lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); - lfs_off_t end = off + lfs_entry_alen(entry); - while (off < end) { - lfs_entry_attr_t attr; - int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); - if (err) { - return err; - } - - for (int j = 0; j < count; j++) { - if (attrs[j].type == attr.d.type) { - if (attrs[j].size < attr.d.len) { - return LFS_ERR_RANGE; - } - - err = lfs_dir_get(lfs, dir, off+sizeof(attr.d), - attrs[j].buffer, attr.d.len); - if (err) { - return err; - } - } - } - - off += 2+attr.d.len; - } - - return 0; -} - -static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, - lfs_dir_t *dir, lfs_entry_t *entry, - const struct lfs_attr *attrs, int count) { - // check that attributes fit - // two separate passes so disk access is O(n) - lfs_size_t nsize = 0; - for (int j = 0; j < count; j++) { - if (attrs[j].size > 0) { - nsize += 2+attrs[j].size; - } - } - - lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); - lfs_off_t end = off + lfs_entry_alen(entry); - while (off < end) { - lfs_entry_attr_t attr; - int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); - if (err) { - return err; - } - - bool updated = false; - for (int j = 0; j < count; j++) { - if (attr.d.type == attrs[j].type) { - updated = true; - } - } - - if (!updated) { - nsize += 2+attr.d.len; - } - - off += 2+attr.d.len; - } - - if (nsize > lfs->attrs_size || ( - lfs_entry_size(entry) - lfs_entry_alen(entry) + nsize - > lfs->cfg->block_size)) { - return LFS_ERR_NOSPC; - } - - return nsize; -} - -static int lfs_dir_setattrs(lfs_t *lfs, - lfs_dir_t *dir, lfs_entry_t *entry, - const struct lfs_attr *attrs, int count) { - // make sure attributes fit - lfs_size_t oldlen = lfs_entry_alen(entry); - lfs_ssize_t newlen = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); - if (newlen < 0) { - return newlen; - } - - // commit to entry, majority of work is in LFS_FROM_ATTRS - entry->d.alen = (0xc0 & entry->d.alen) | newlen; - return lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 4, &entry->d, 4}, - {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), oldlen, - &(struct lfs_region_attrs){attrs, count}, newlen}}, 2); -} - +//static int lfs_dir_getinfo(lfs_t *lfs, +// lfs_dir_t *dir, const lfs_entry_t *entry, struct lfs_info *info) { +// memset(info, 0, sizeof(*info)); +// info->type = 0xf & entry->d.type; +// if (entry->d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { +// info->size = entry->d.u.file.size; +// } else if (entry->d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { +// info->size = lfs_entry_elen(entry); +// } +// +// if (lfs_paircmp(entry->d.u.dir, lfs->root) == 0) { +// strcpy(info->name, "/"); +// } else { +// int err = lfs_dir_get(lfs, dir, +// entry->off + entry->size - entry->d.nlen, +// info->name, entry->d.nlen); +// if (err) { +// return err; +// } +// } +// +// return 0; +//} +// +//static int lfs_dir_getattrs(lfs_t *lfs, +// lfs_dir_t *dir, const lfs_entry_t *entry, +// const struct lfs_attr *attrs, int count) { +// // set to zero in case we can't find the attributes or size mismatch +// for (int j = 0; j < count; j++) { +// memset(attrs[j].buffer, 0, attrs[j].size); +// } +// +// // search for attribute in attribute entry +// lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); +// lfs_off_t end = off + lfs_entry_alen(entry); +// while (off < end) { +// lfs_entry_attr_t attr; +// int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); +// if (err) { +// return err; +// } +// +// for (int j = 0; j < count; j++) { +// if (attrs[j].type == attr.d.type) { +// if (attrs[j].size < attr.d.len) { +// return LFS_ERR_RANGE; +// } +// +// err = lfs_dir_get(lfs, dir, off+sizeof(attr.d), +// attrs[j].buffer, attr.d.len); +// if (err) { +// return err; +// } +// } +// } +// +// off += 2+attr.d.len; +// } +// +// return 0; +//} +// +//static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, +// lfs_dir_t *dir, lfs_entry_t *entry, +// const struct lfs_attr *attrs, int count) { +// // check that attributes fit +// // two separate passes so disk access is O(n) +// lfs_size_t nsize = 0; +// for (int j = 0; j < count; j++) { +// if (attrs[j].size > 0) { +// nsize += 2+attrs[j].size; +// } +// } +// +// lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); +// lfs_off_t end = off + lfs_entry_alen(entry); +// while (off < end) { +// lfs_entry_attr_t attr; +// int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); +// if (err) { +// return err; +// } +// +// bool updated = false; +// for (int j = 0; j < count; j++) { +// if (attr.d.type == attrs[j].type) { +// updated = true; +// } +// } +// +// if (!updated) { +// nsize += 2+attr.d.len; +// } +// +// off += 2+attr.d.len; +// } +// +// if (nsize > lfs->attrs_size || ( +// lfs_entry_size(entry) - lfs_entry_alen(entry) + nsize +// > lfs->cfg->block_size)) { +// return LFS_ERR_NOSPC; +// } +// +// return nsize; +//} +// +//static int lfs_dir_setattrs(lfs_t *lfs, +// lfs_dir_t *dir, lfs_entry_t *entry, +// const struct lfs_attr *attrs, int count) { +// // make sure attributes fit +// lfs_size_t oldlen = lfs_entry_alen(entry); +// lfs_ssize_t newlen = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); +// if (newlen < 0) { +// return newlen; +// } +// +// // commit to entry, majority of work is in LFS_FROM_ATTRS +// entry->d.alen = (0xc0 & entry->d.alen) | newlen; +// return lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ +// {LFS_FROM_MEM, 0, 4, &entry->d, 4}, +// {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), oldlen, +// &(struct lfs_region_attrs){attrs, count}, newlen}}, 2); +//} +// /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron if (!lfs->deorphaned) { - int err = lfs_deorphan_(lfs); + int err = lfs_deorphan(lfs); if (err) { return err; } } - // fetch parent directory - lfs_dir_t_ cwd; - // TODO remove this? - int err = lfs_dir_findentry_(lfs, &cwd, &path, &(lfs_entry_t_){0}); + lfs_dir_t cwd; + int err = lfs_dir_find(lfs, &cwd, &path, &(int16_t){0}); if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { if (!err) { return LFS_ERR_EXIST; @@ -2275,28 +2301,28 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // build up new directory lfs_alloc_ack(lfs); - lfs_dir_t_ dir; - err = lfs_dir_alloc_(lfs, &dir, false, cwd.tail); + lfs_dir_t dir; + err = lfs_dir_alloc(lfs, &dir, false, cwd.tail); if (err) { return err; } - err = lfs_dir_commit_(lfs, &dir, NULL); + err = lfs_dir_commit(lfs, &dir, NULL); if (err) { return err; } // get next slot and commit uint16_t id; - err = lfs_dir_append_(lfs, &cwd, &id); + err = lfs_dir_append(lfs, &cwd, &id); if (err) { return err; } - err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_NAME_, id, nlen), + err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_NAME, id, nlen), .u.buffer=(void*)path}, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_DIR_ | LFS_STRUCT_DIR_, id, sizeof(dir.pair)), + {lfs_mktag(LFS_TYPE_DIR | LFS_STRUCT_DIR, id, sizeof(dir.pair)), .u.buffer=dir.pair}}}); // TODO need ack here? @@ -2370,18 +2396,33 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // return 0; //} -int lfs_dir_open_(lfs_t *lfs, lfs_dir_t_ *dir, const char *path) { - lfs_entry_t_ entry; - int err = lfs_dir_findentry_(lfs, dir, &path, &entry); +int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { + int16_t id; + int err = lfs_dir_find(lfs, dir, &path, &id); if (err) { return err; } - if ((lfs_tag_type(entry.tag) & 0x1f0) != LFS_TYPE_DIR_) { - return LFS_ERR_NOTDIR; + lfs_entry_t entry; + if (id < 0) { + // handle root dir separately + entry.u.pair[0] = lfs->root[0]; + entry.u.pair[1] = lfs->root[1]; + } else { + // get dir pair from parent + err = lfs_dir_getentry(lfs, dir, 0x701ff000, + lfs_mktag(LFS_TYPE_REG, id, 0), &entry); + if (err) { + return err; + } + + if (lfs_tag_subtype(entry.tag) != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } } - err = lfs_dir_fetch_(lfs, dir, entry.u.pair); + // fetch first pair + err = lfs_dir_fetch(lfs, dir, entry.u.pair); if (err) { return err; } @@ -2399,9 +2440,9 @@ int lfs_dir_open_(lfs_t *lfs, lfs_dir_t_ *dir, const char *path) { return 0; } -int lfs_dir_close_(lfs_t *lfs, lfs_dir_t_ *dir) { +int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir) { // remove from list of directories - for (lfs_dir_t_ **p = &lfs->dirs; *p; p = &(*p)->next) { + for (lfs_dir_t **p = &lfs->dirs; *p; p = &(*p)->next) { if (*p == dir) { *p = dir->next; break; @@ -2411,34 +2452,7 @@ int lfs_dir_close_(lfs_t *lfs, lfs_dir_t_ *dir) { return 0; } -// TODO move me? -static int lfs_dir_getinfo_(lfs_t *lfs, lfs_dir_t_ *dir, - uint16_t id, struct lfs_info *info) { - lfs_entry_t_ entry; - int err = lfs_dir_getentry_(lfs, dir, - 0x701ff000, lfs_mktag(LFS_TYPE_REG, id, 8), &entry); - if (err && err != LFS_ERR_RANGE) { - return err; - } - - info->type = lfs_tag_subtype(entry.tag); - if (lfs_tag_type(entry.tag) == (LFS_TYPE_REG_ | LFS_STRUCT_CTZ_)) { - info->size = entry.u.ctz.size; - } else if (lfs_tag_type(entry.tag) == (LFS_TYPE_REG_ | LFS_STRUCT_INLINE_)) { - info->size = lfs_tag_size(entry.tag); - } - - err = lfs_dir_getbuffer_(lfs, dir, - 0x7ffff000, lfs_mktag(LFS_TYPE_NAME_, id, lfs->cfg->name_size+1), - &(lfs_entry_t_){.u.buffer=info->name}); - if (err) { - return err; - } - - return 0; -} - -int lfs_dir_read_(lfs_t *lfs, lfs_dir_t_ *dir, struct lfs_info *info) { +int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { memset(info, 0, sizeof(*info)); // special offset for '.' and '..' @@ -2460,7 +2474,7 @@ int lfs_dir_read_(lfs_t *lfs, lfs_dir_t_ *dir, struct lfs_info *info) { return false; } - int err = lfs_dir_fetch_(lfs, dir, dir->tail); + int err = lfs_dir_fetch(lfs, dir, dir->tail); if (err) { return err; } @@ -2468,15 +2482,15 @@ int lfs_dir_read_(lfs_t *lfs, lfs_dir_t_ *dir, struct lfs_info *info) { dir->id = 0; } - int err = lfs_dir_getinfo_(lfs, dir, dir->id, info); - if (err != LFS_ERR_NOENT) { - if (!err) { - break; - } + int err = lfs_dir_getinfo(lfs, dir, dir->id, info); + if (err && err != LFS_ERR_NOENT) { return err; } dir->id += 1; + if (err != LFS_ERR_NOENT) { + break; + } } dir->pos += 1; @@ -2484,9 +2498,9 @@ int lfs_dir_read_(lfs_t *lfs, lfs_dir_t_ *dir, struct lfs_info *info) { } // TODO does this work? -int lfs_dir_seek_(lfs_t *lfs, lfs_dir_t_ *dir, lfs_off_t off) { +int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { // simply walk from head dir - int err = lfs_dir_rewind_(lfs, dir); + int err = lfs_dir_rewind(lfs, dir); if (err) { return err; } @@ -2502,7 +2516,7 @@ int lfs_dir_seek_(lfs_t *lfs, lfs_dir_t_ *dir, lfs_off_t off) { return LFS_ERR_INVAL; } - int err = lfs_dir_fetch_(lfs, dir, dir->tail); + int err = lfs_dir_fetch(lfs, dir, dir->tail); if (err) { return err; } @@ -2516,14 +2530,14 @@ int lfs_dir_seek_(lfs_t *lfs, lfs_dir_t_ *dir, lfs_off_t off) { return 0; } -lfs_soff_t lfs_dir_tell_(lfs_t *lfs, lfs_dir_t *dir) { +lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir) { (void)lfs; return dir->pos; } -int lfs_dir_rewind_(lfs_t *lfs, lfs_dir_t_ *dir) { +int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { // reload the head dir - int err = lfs_dir_fetch_(lfs, dir, dir->head); + int err = lfs_dir_fetch(lfs, dir, dir->head); if (err) { return err; } @@ -2829,7 +2843,7 @@ static int lfs_ctz_extend(lfs_t *lfs, static int lfs_ctz_traverse(lfs_t *lfs, lfs_cache_t *rcache, const lfs_cache_t *pcache, lfs_block_t head, lfs_size_t size, - int (*cb)(void*, lfs_block_t), void *data) { + int (*cb)(lfs_t*, void*, lfs_block_t), void *data) { if (size == 0) { return 0; } @@ -2837,7 +2851,7 @@ static int lfs_ctz_traverse(lfs_t *lfs, lfs_off_t index = lfs_ctz_index(lfs, &(lfs_off_t){size-1}); while (true) { - int err = cb(data, head); + int err = cb(lfs, data, head); if (err) { return err; } @@ -2856,7 +2870,7 @@ static int lfs_ctz_traverse(lfs_t *lfs, } for (int i = 0; i < count-1; i++) { - err = cb(data, heads[i]); + err = cb(lfs, data, heads[i]); if (err) { return err; } @@ -2873,20 +2887,21 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, const char *path, int flags) { // deorphan if we haven't yet, needed at most once after poweron if ((flags & 3) != LFS_O_RDONLY && !lfs->deorphaned) { - int err = lfs_deorphan_(lfs); + int err = lfs_deorphan(lfs); if (err) { return err; } } // allocate entry for file if it doesn't exist - lfs_dir_t_ cwd; - lfs_entry_t_ entry; - int err = lfs_dir_find_(lfs, &cwd, &path, &entry); + lfs_dir_t cwd; + int16_t id; + int err = lfs_dir_find(lfs, &cwd, &path, &id); if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { return err; } + lfs_entry_t entry; if (err == LFS_ERR_NOENT) { if (!(flags & LFS_O_CREAT)) { return LFS_ERR_NOENT; @@ -2899,32 +2914,40 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // get next slot and create entry to remember name - uint16_t id; - err = lfs_dir_append_(lfs, &cwd, &id); + err = lfs_dir_append(lfs, &cwd, &id); if (err) { return err; } - err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_NAME_, id, nlen), - .u.buffer=(void*)path}, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_INLINE_, id, 0)}}}); + err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_NAME, id, nlen), + .u.buffer=(void*)path}, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0)}}}); + if (err) { + return err; + } + + entry.tag = lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0); + } else { + if (flags & LFS_O_EXCL) { + return LFS_ERR_EXIST; + } + + entry.tag = lfs_mktag(LFS_TYPE_REG, id, 0); + err = lfs_dir_get(lfs, &cwd, 0x701ff000, &entry); if (err) { return err; } - // TODO, used later, clean this up? - entry.tag = lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_INLINE_, id, 0); - } else if (lfs_tag_subtype(entry.tag) != LFS_TYPE_REG_) { - return LFS_ERR_ISDIR; - } else if (flags & LFS_O_EXCL) { - return LFS_ERR_EXIST; + if (lfs_tag_subtype(entry.tag) != LFS_TYPE_REG) { + return LFS_ERR_ISDIR; + } } // setup file struct file->pair[0] = cwd.pair[0]; file->pair[1] = cwd.pair[1]; - file->id = lfs_tag_id(entry.tag); + file->id = id; file->flags = flags; file->pos = 0; file->attrs = NULL; @@ -2945,18 +2968,21 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } - if (lfs_tag_struct(entry.tag) == LFS_STRUCT_INLINE_) { + if (lfs_tag_struct(entry.tag) == LFS_STRUCT_INLINE) { // load inline files file->head = 0xfffffffe; file->size = lfs_tag_size(entry.tag); file->flags |= LFS_F_INLINE; file->cache.block = file->head; file->cache.off = 0; - err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, - file->cache.buffer, file->size); - if (err) { - lfs_free(file->cache.buffer); - return err; + // don't always read (may be new file) + if (file->size > 0) { + err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, + file->cache.buffer, file->size); + if (err) { + lfs_free(file->cache.buffer); + return err; + } } } else { // use ctz list from entry @@ -2984,25 +3010,6 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return 0; } -int lfs_file_close_(lfs_t *lfs, lfs_file_t *file) { - int err = lfs_file_sync(lfs, file); - - // remove from list of files - for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { - if (*p == file) { - *p = file->next; - break; - } - } - - // clean up memory - if (!lfs->cfg->file_buffer) { - lfs_free(file->cache.buffer); - } - - return err; -} - int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { int err = lfs_file_sync(lfs, file); @@ -3154,24 +3161,24 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // update dir entry // TODO keep list of dirs including these guys for no // need of another reload? - lfs_dir_t_ cwd; - err = lfs_dir_fetch_(lfs, &cwd, file->pair); + lfs_dir_t cwd; + err = lfs_dir_fetch(lfs, &cwd, file->pair); if (err) { return err; } // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { - int err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_CTZ_, file->id, + int err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_CTZ, file->id, 2*sizeof(uint32_t)), .u.buffer=&file->head}, file->attrs}); if (err) { return err; } } else { - int err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_REG_ | LFS_STRUCT_INLINE_, file->id, + int err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, file->id, file->size), .u.buffer=file->cache.buffer}, file->attrs}); if (err) { @@ -3554,16 +3561,17 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { - lfs_dir_t_ cwd; - lfs_entry_t_ entry; - int err = lfs_dir_find_(lfs, &cwd, &entry, &path); + lfs_dir_t cwd; + int16_t id; + int err = lfs_dir_find(lfs, &cwd, &path, &id); if (err) { return err; } - return lfs_dir_getinfo_(lfs, &cwd, lfs_tag_id(entry.tag), info); + return lfs_dir_getinfo(lfs, &cwd, id, info); } +// //int lfs_remove(lfs_t *lfs, const char *path) { // // deorphan if we haven't yet, needed at most once after poweron // if (!lfs->deorphaned) { @@ -3898,22 +3906,22 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_alloc_ack(lfs); // create superblock dir - lfs_dir_t_ dir; - err = lfs_dir_alloc_(lfs, &dir, false, + lfs_dir_t dir; + err = lfs_dir_alloc(lfs, &dir, false, (const lfs_block_t[2]){0xffffffff, 0xffffffff}); if (err) { return err; } // write root directory - lfs_dir_t_ root; - err = lfs_dir_alloc_(lfs, &root, false, + lfs_dir_t root; + err = lfs_dir_alloc(lfs, &root, false, (const lfs_block_t[2]){0xffffffff, 0xffffffff}); if (err) { return err; } - err = lfs_dir_commit_(lfs, &root, NULL); + err = lfs_dir_commit(lfs, &root, NULL); if (err) { return err; } @@ -3924,7 +3932,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { dir.tail[1] = lfs->root[1]; // write one superblock - lfs_superblock_t_ superblock = { + lfs_superblock_t superblock = { .root[0] = lfs->root[0], .root[1] = lfs->root[1], .magic = {"littlefs"}, @@ -3938,15 +3946,15 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; dir.count += 1; - err = lfs_dir_commit_(lfs, &dir, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_SUPERBLOCK_ | LFS_STRUCT_DIR_, 0, + err = lfs_dir_commit(lfs, &dir, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, 0, sizeof(superblock)), .u.buffer=&superblock}}); if (err) { return err; } // sanity check that fetch works - err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}); + err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { return err; } @@ -3967,8 +3975,8 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_alloc_ack(lfs); // load superblock - lfs_dir_t_ dir; - err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}); + lfs_dir_t dir; + err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { if (err == LFS_ERR_CORRUPT) { LFS_ERROR("Invalid superblock at %d %d", 0, 1); @@ -3976,10 +3984,11 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - lfs_superblock_t_ superblock; - err = lfs_dir_getbuffer_(lfs, &dir, - 0x7ffff000, lfs_mktag(LFS_TYPE_SUPERBLOCK_ | LFS_STRUCT_DIR_, 0, - sizeof(superblock)), &(lfs_entry_t_){.u.buffer=&superblock}); + lfs_superblock_t superblock; + err = lfs_dir_getbuffer(lfs, &dir, 0x7ffff000, &(lfs_entry_t){ + lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, + 0, sizeof(superblock)), + .u.buffer=&superblock}); if (err && err != LFS_ERR_RANGE) { return err; } @@ -4039,14 +4048,14 @@ int lfs_unmount(lfs_t *lfs) { /// Internal filesystem filesystem operations /// -int lfs_traverse_(lfs_t *lfs, +int lfs_fs_traverse(lfs_t *lfs, int (*cb)(lfs_t *lfs, void *data, lfs_block_t block), void *data) { if (lfs_pairisnull(lfs->root)) { return 0; } // iterate over metadata pairs - lfs_dir_t_ dir = {.tail = {0, 1}}; + lfs_dir_t dir = {.tail = {0, 1}}; while (!lfs_pairisnull(dir.tail)) { for (int i = 0; i < 2; i++) { int err = cb(lfs, data, dir.tail[i]); @@ -4056,15 +4065,15 @@ int lfs_traverse_(lfs_t *lfs, } // iterate through ids in directory - int err = lfs_dir_fetch_(lfs, &dir, dir.tail); + int err = lfs_dir_fetch(lfs, &dir, dir.tail); if (err) { return err; } - for (int i = 0; i < dir.count; i++) { - lfs_entry_t_ entry; - int err = lfs_dir_getentry_(lfs, &dir, - 0x701ff000, lfs_mktag(LFS_TYPE_REG_, i, 8), &entry); + for (uint16_t id = 0; id < dir.count; id++) { + lfs_entry_t entry; + int err = lfs_dir_getentry(lfs, &dir, 0x701ff000, + lfs_mktag(LFS_TYPE_REG, id, 0), &entry); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -4072,13 +4081,12 @@ int lfs_traverse_(lfs_t *lfs, return err; } - if (lfs_tag_struct(entry.tag) == LFS_STRUCT_CTZ_) { -// TODO -// err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, -// entry.d.u.file.head, entry.d.u.file.size, cb, data); -// if (err) { -// return err; -// } + if (lfs_tag_struct(entry.tag) == LFS_STRUCT_CTZ) { + err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, + entry.u.ctz.head, entry.u.ctz.size, cb, data); + if (err) { + return err; + } } } @@ -4087,26 +4095,26 @@ int lfs_traverse_(lfs_t *lfs, // iterate over any open files for (lfs_file_t *f = lfs->files; f; f = f->next) { if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { -// int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, -// f->head, f->size, cb, data); -// if (err) { -// return err; -// } + int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, + f->head, f->size, cb, data); + if (err) { + return err; + } } if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { -// int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, -// f->block, f->pos, cb, data); -// if (err) { -// return err; -// } + int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, + f->block, f->pos, cb, data); + if (err) { + return err; + } } } return 0; } - -int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { +/* +int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { if (lfs_pairisnull(lfs->root)) { return 0; } @@ -4177,18 +4185,17 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { return 0; } - -static int lfs_pred_(lfs_t *lfs, const lfs_block_t pair[2], lfs_dir_t_ *pdir) { +*/ +static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_dir_t *pdir) { + // iterate over all directory directory entries pdir->tail[0] = 0; pdir->tail[1] = 1; - - // iterate over all directory directory entries while (!lfs_pairisnull(pdir->tail)) { if (lfs_paircmp(pdir->tail, pair) == 0) { return true; } - int err = lfs_dir_fetch_(lfs, pdir, pdir->tail); + int err = lfs_dir_fetch(lfs, pdir, pdir->tail); if (err) { return err; } @@ -4196,7 +4203,7 @@ static int lfs_pred_(lfs_t *lfs, const lfs_block_t pair[2], lfs_dir_t_ *pdir) { return false; } - +/* static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir) { if (lfs_pairisnull(lfs->root)) { return 0; @@ -4221,23 +4228,22 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir) { return false; } - -static int lfs_parent_(lfs_t *lfs, const lfs_block_t pair[2], - lfs_dir_t_ *parent, lfs_entry_t_ *entry) { +*/ +static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], + lfs_dir_t *parent, lfs_entry_t *entry) { + // iterate over all directory directory entries parent->tail[0] = 0; parent->tail[1] = 1; - - // iterate over all directory directory entries while (!lfs_pairisnull(parent->tail)) { - int err = lfs_dir_fetch_(lfs, parent, parent->tail); + int err = lfs_dir_fetch(lfs, parent, parent->tail); if (err) { return err; } - for (int i = 0; i < parent->count; i++) { - int err = lfs_dir_getentry_(lfs, parent, - 0x43dff000, lfs_mktag(LFS_STRUCT_DIR_, i, 8), entry); - if (err && err != LFS_ERR_RANGE) { + for (uint16_t id = 0; id < parent->count; id++) { + int err = lfs_dir_getentry(lfs, parent, 0x43dff000, + lfs_mktag(LFS_STRUCT_DIR, id, 0), entry); + if (err) { if (err == LFS_ERR_NOENT) { continue; } @@ -4252,7 +4258,7 @@ static int lfs_parent_(lfs_t *lfs, const lfs_block_t pair[2], return false; } - +/* static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *parent, lfs_entry_t *entry) { if (lfs_pairisnull(lfs->root)) { @@ -4288,26 +4294,26 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], return false; } - -static int lfs_moved_(lfs_t *lfs, const lfs_block_t pair[2]) { +*/ +static int lfs_moved(lfs_t *lfs, const lfs_block_t pair[2]) { // skip superblock - lfs_dir_t_ dir; - int err = lfs_dir_fetch_(lfs, &dir, (const lfs_block_t[2]){0, 1}); + lfs_dir_t dir; + int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { return err; } // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { - int err = lfs_dir_fetch_(lfs, &dir, dir.tail); + int err = lfs_dir_fetch(lfs, &dir, dir.tail); if (err) { return err; } - for (int i = 0; i < dir.count; i++) { - lfs_entry_t_ entry; - int err = lfs_dir_getentry_(lfs, &dir, - 0x43dff000, lfs_mktag(LFS_STRUCT_DIR_, i, 8), &entry); + for (int id = 0; id < dir.count; id++) { + lfs_entry_t entry; + int err = lfs_dir_getentry(lfs, &dir, 0x43dff000, + lfs_mktag(LFS_STRUCT_DIR, id, 0), &entry); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -4315,8 +4321,8 @@ static int lfs_moved_(lfs_t *lfs, const lfs_block_t pair[2]) { return err; } - err = lfs_dir_get_(lfs, &dir, - 0x7ffff000, lfs_mktag(LFS_TYPE_MOVE_, i, 0), NULL); + err = lfs_dir_get(lfs, &dir, 0x7ffff000, &(lfs_entry_t){ + lfs_mktag(LFS_TYPE_MOVE, id, 0)}); if (err != LFS_ERR_NOENT) { if (!err) { continue; @@ -4332,7 +4338,7 @@ static int lfs_moved_(lfs_t *lfs, const lfs_block_t pair[2]) { return false; } - +/* static int lfs_moved(lfs_t *lfs, const void *e) { if (lfs_pairisnull(lfs->root)) { return 0; @@ -4372,7 +4378,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) { return false; } - +*/ static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]) { // find parent @@ -4385,12 +4391,9 @@ static int lfs_relocate(lfs_t *lfs, if (res) { // update disk, this creates a desync - entry.d.u.dir[0] = newpair[0]; - entry.d.u.dir[1] = newpair[1]; - lfs_entry_tole32(&entry.d); - int err = lfs_dir_set(lfs, &parent, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, sizeof(entry.d), - &entry.d, sizeof(entry.d)}}, 1); + entry.u.pair[0] = newpair[0]; + entry.u.pair[1] = newpair[1]; + int err = lfs_dir_commit(lfs, &parent, &(lfs_entrylist_t){entry}); if (err) { return err; } @@ -4414,28 +4417,30 @@ static int lfs_relocate(lfs_t *lfs, if (res) { // just replace bad pair, no desync can occur - parent.d.tail[0] = newpair[0]; - parent.d.tail[1] = newpair[1]; - - return lfs_dir_commit(lfs, &parent, NULL, 0); + parent.tail[0] = newpair[0]; + parent.tail[1] = newpair[1]; + return lfs_dir_commit(lfs, &parent, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL + parent.split*0x10, // TODO hm + 0x1ff, sizeof(newpair)), + .u.pair[0]=newpair[0], .u.pair[1]=newpair[1]}}); } // couldn't find dir, must be new return 0; } -int lfs_deorphan_(lfs_t *lfs) { +int lfs_deorphan(lfs_t *lfs) { lfs->deorphaned = true; if (lfs_pairisnull(lfs->root)) { return 0; } - lfs_dir_t_ pdir = {.split = true}; - lfs_dir_t_ dir = {.tail = {0, 1}}; + lfs_dir_t pdir = {.split = true}; + lfs_dir_t dir = {.tail = {0, 1}}; // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { - int err = lfs_dir_fetch_(lfs, &dir, dir.tail); + int err = lfs_dir_fetch(lfs, &dir, dir.tail); if (err) { return err; } @@ -4443,9 +4448,9 @@ int lfs_deorphan_(lfs_t *lfs) { // check head blocks for orphans if (!pdir.split) { // check if we have a parent - lfs_dir_t_ parent; - lfs_entry_t_ entry; - int res = lfs_parent_(lfs, pdir.tail, &parent, &entry); + lfs_dir_t parent; + lfs_entry_t entry; + int res = lfs_parent(lfs, pdir.tail, &parent, &entry); if (res < 0) { return res; } @@ -4457,8 +4462,8 @@ int lfs_deorphan_(lfs_t *lfs) { pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit_(lfs, &pdir, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL_, 0x1ff, + err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { return err; @@ -4474,8 +4479,8 @@ int lfs_deorphan_(lfs_t *lfs) { pdir.tail[0] = entry.u.pair[0]; pdir.tail[1] = entry.u.pair[1]; - err = lfs_dir_commit_(lfs, &pdir, &(lfs_entrylist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL_, 0x1ff, + err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { return err; @@ -4520,7 +4525,7 @@ int lfs_deorphan_(lfs_t *lfs) { return 0; } - +/* int lfs_deorphan(lfs_t *lfs) { lfs->deorphaned = true; @@ -4626,7 +4631,7 @@ int lfs_deorphan(lfs_t *lfs) { return 0; } - +*/ /// External filesystem filesystem operations /// //int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { @@ -4691,7 +4696,7 @@ static int lfs_fs_size_count(void *p, lfs_block_t block) { lfs_ssize_t lfs_fs_size(lfs_t *lfs) { lfs_size_t size = 0; - int err = lfs_traverse(lfs, lfs_fs_size_count, &size); + int err = lfs_fs_traverse(lfs, lfs_fs_size_count, &size); if (err) { return err; } diff --git a/lfs.h b/lfs.h index d344ce04..a0891252 100644 --- a/lfs.h +++ b/lfs.h @@ -95,36 +95,25 @@ enum lfs_error { // File types enum lfs_type { - // file type - LFS_TYPE_REG = 0x01, - LFS_TYPE_DIR = 0x02, - LFS_TYPE_SUPERBLOCK = 0x0e, - - // on disk structure - LFS_STRUCT_CTZ = 0x10, - LFS_STRUCT_DIR = 0x20, - LFS_STRUCT_INLINE = 0x30, - LFS_STRUCT_MOVED = 0x80, - // file types - LFS_TYPE_REG_ = 0x040, - LFS_TYPE_DIR_ = 0x050, + LFS_TYPE_REG = 0x040, + LFS_TYPE_DIR = 0x050, // internally used types - LFS_TYPE_NAME_ = 0x010, - LFS_TYPE_MOVE_ = 0x080, - LFS_TYPE_DELETE_ = 0x090, + LFS_TYPE_NAME = 0x010, + LFS_TYPE_MOVE = 0x080, + LFS_TYPE_DELETE = 0x090, - LFS_TYPE_SUPERBLOCK_ = 0x0a0, - LFS_TYPE_SOFTTAIL_ = 0x0c0, - LFS_TYPE_HARDTAIL_ = 0x0d0, - LFS_TYPE_CRC_ = 0x0e0, + LFS_TYPE_SUPERBLOCK = 0x0a0, + LFS_TYPE_SOFTTAIL = 0x0c0, + LFS_TYPE_HARDTAIL = 0x0d0, + LFS_TYPE_CRC = 0x0e0, // on disk structure - LFS_STRUCT_ATTR_ = 0x100, - LFS_STRUCT_INLINE_ = 0x000, - LFS_STRUCT_CTZ_ = 0x004, - LFS_STRUCT_DIR_ = 0x008, + LFS_STRUCT_ATTR = 0x100, + LFS_STRUCT_INLINE = 0x000, + LFS_STRUCT_CTZ = 0x004, + LFS_STRUCT_DIR = 0x008, }; // File open flags @@ -268,30 +257,9 @@ struct lfs_attr { /// littlefs data structures /// -typedef struct lfs_entry { - lfs_off_t off; - lfs_size_t size; - - struct lfs_disk_entry { - uint8_t type; - uint8_t elen; - uint8_t alen; - uint8_t nlen; - union { - struct { - lfs_block_t head; - lfs_size_t size; - } file; - lfs_block_t dir[2]; - } u; - } d; -} lfs_entry_t; - typedef uint32_t lfs_tag_t; -typedef int32_t lfs_stag_t; - -typedef struct lfs_entry_ { - uint32_t tag; +typedef struct lfs_entry { + lfs_tag_t tag; union { void *buffer; lfs_block_t pair[2]; @@ -304,27 +272,20 @@ typedef struct lfs_entry_ { lfs_off_t off; } d; } u; -} lfs_entry_t_; +} lfs_entry_t; -typedef struct lfs_entry_list_ { - lfs_entry_t_ e; - struct lfs_entry_list_ *next; +typedef struct lfs_entrylist { + lfs_entry_t e; + struct lfs_entrylist *next; } lfs_entrylist_t; -typedef struct lfs_entry_attr { - struct lfs_disk_entry_attr { - uint8_t type; - uint8_t len; - } d; -} lfs_entry_attr_t; - typedef struct lfs_cache { lfs_block_t block; lfs_off_t off; uint8_t *buffer; } lfs_cache_t; -typedef struct lfs_file_ { +typedef struct lfs_file { struct lfs_file *next; lfs_block_t pair[2]; uint16_t id; @@ -344,21 +305,6 @@ typedef struct lfs_file_ { typedef struct lfs_dir { struct lfs_dir *next; lfs_block_t pair[2]; - lfs_off_t off; - - lfs_block_t head[2]; - lfs_off_t pos; - - struct lfs_disk_dir { - uint32_t rev; - lfs_size_t size; - lfs_block_t tail[2]; - } d; -} lfs_dir_t; - -typedef struct lfs_dir_ { - struct lfs_dir_ *next; - lfs_block_t pair[2]; lfs_block_t tail[2]; uint32_t rev; @@ -372,23 +318,9 @@ typedef struct lfs_dir_ { uint16_t id; lfs_block_t head[2]; lfs_off_t pos; -} lfs_dir_t_; +} lfs_dir_t; typedef struct lfs_superblock { - struct lfs_disk_superblock { - lfs_block_t root[2]; - - lfs_size_t block_size; - lfs_size_t block_count; - uint32_t version; - - lfs_size_t inline_size; - lfs_size_t attrs_size; - lfs_size_t name_size; - } d; -} lfs_superblock_t; - -typedef struct lfs_superblock_ { lfs_block_t root[2]; char magic[8]; uint32_t version; @@ -399,7 +331,7 @@ typedef struct lfs_superblock_ { lfs_size_t inline_size; lfs_size_t attrs_size; lfs_size_t name_size; -} lfs_superblock_t_; +} lfs_superblock_t; typedef struct lfs_free { lfs_block_t off; diff --git a/tests/template.fmt b/tests/template.fmt index a53f0c7e..8f1e8efa 100644 --- a/tests/template.fmt +++ b/tests/template.fmt @@ -88,6 +88,8 @@ const struct lfs_config cfg = {{ .erase = &lfs_emubd_erase, .sync = &lfs_emubd_sync, + .name_size = 255, + .read_size = LFS_READ_SIZE, .prog_size = LFS_PROG_SIZE, .block_size = LFS_BLOCK_SIZE, diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 53d76f7a..d5303cc2 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -117,6 +117,8 @@ tests/test.py << TEST for (int i = 0; i < $LARGESIZE; i++) { sprintf((char*)buffer, "test%d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; + printf("nameee %s\n", info.name); + printf("expect %s\n", (char*)buffer); strcmp(info.name, (char*)buffer) => 0; info.type => LFS_TYPE_DIR; } From 0bdaeb7f8b3948e8b20e4c81c6e160d5b1e2eae8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 27 May 2018 10:15:28 -0500 Subject: [PATCH 036/139] More testing progress, combined dir/commit traversal Passing more tests now with the journalling change, but still have more work to do. The most humorous bug was a bug where during the three step move process, the entry move logic would dumbly copy over any tags associated with the moving entry, including the tag used to temporarily mark the entry as "moving". Also combined dir and commit traversal using a "stop_at_commit" flag in directory struct as a short-term hack to combine the code paths. --- lfs.c | 643 ++++++++++++++++++++++++--------------------- lfs.h | 6 + tests/test_dirs.sh | 4 +- 3 files changed, 350 insertions(+), 303 deletions(-) diff --git a/lfs.c b/lfs.c index aad21524..d4e6dc5b 100644 --- a/lfs.c +++ b/lfs.c @@ -264,7 +264,7 @@ int lfs_fs_traverse(lfs_t *lfs, static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir); static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *parent, lfs_entry_t *entry); -static int lfs_moved(lfs_t *lfs, const lfs_block_t pair[2]); +static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid); static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); int lfs_deorphan(lfs_t *lfs); @@ -473,40 +473,6 @@ struct lfs_commit { } filter; }; -static int lfs_commit_traverse(lfs_t *lfs, struct lfs_commit *commit, - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data) { - // TODO duplicate this? move it to dir? - // iterate over dir block backwards (for faster lookups) - lfs_block_t block = commit->block; - lfs_off_t off = commit->off; - lfs_tag_t tag = commit->ptag; - - while (off != sizeof(uint32_t)) { - // TODO rm me - printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, block, off-lfs_tag_size(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); - int err = cb(lfs, data, (lfs_entry_t){ - (0x80000000 | tag), - .u.d.block=block, - .u.d.off=off-lfs_tag_size(tag)}); - if (err) { - return err; - } - - LFS_ASSERT(off > sizeof(tag)+lfs_tag_size(tag)); - off -= sizeof(tag)+lfs_tag_size(tag); - - lfs_tag_t ntag; - err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); - if (err) { - return err; - } - - tag ^= lfs_fromle32(ntag); - } - - return 0; -} - //static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t entry) { // struct lfs_commit *commit = p; // if (lfs_tag_id(entry.tag) != commit->compact.id) { @@ -527,7 +493,7 @@ static int lfs_commit_commit(lfs_t *lfs, return 0; } uint16_t id = lfs_tag_id(entry.tag) - commit->filter.begin; - entry.tag = (id << 12) | (entry.tag & 0xffe00fff); + entry.tag = lfs_mktag(0, id, 0) | (entry.tag & 0xffe00fff); // check if we fit lfs_size_t size = lfs_tag_size(entry.tag); @@ -537,7 +503,7 @@ static int lfs_commit_commit(lfs_t *lfs, // write out tag // TODO rm me - printf("tag w %#010x (%x:%x %03x %03x %03x)\n", entry.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tag_type(entry.tag), lfs_tag_id(entry.tag), lfs_tag_size(entry.tag)); + //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", entry.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tag_type(entry.tag), lfs_tag_id(entry.tag), lfs_tag_size(entry.tag)); lfs_tag_t tag = lfs_tole32((entry.tag & 0x7fffffff) ^ commit->ptag); lfs_crc(&commit->crc, &tag, sizeof(tag)); int err = lfs_bd_prog(lfs, commit->block, commit->off, &tag, sizeof(tag)); @@ -572,7 +538,7 @@ static int lfs_commit_commit(lfs_t *lfs, } } commit->off += size; - commit->ptag = entry.tag; + commit->ptag = entry.tag & 0x7fffffff; // TODO do this once return 0; } @@ -595,7 +561,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { noff - (commit->off+sizeof(uint32_t))); // write out crc - printf("tag w %#010x (%x:%x %03x %03x %03x)\n", tag, commit->block, commit->off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); + //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", tag, commit->block, commit->off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); uint32_t footer[2]; footer[0] = lfs_tole32(tag ^ commit->ptag); lfs_crc(&commit->crc, &footer[0], sizeof(footer[0])); @@ -641,10 +607,6 @@ static int lfs_commit_regions(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } -// TODO redeclare -static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), - void *data); // committer for moves struct lfs_commit_move { @@ -653,40 +615,31 @@ struct lfs_commit_move { uint16_t from; uint16_t to; } id; - uint16_t type; struct lfs_commit *commit; }; -static int lfs_commit_movecheck(lfs_t *lfs, void *p, lfs_entry_t entry) { - struct lfs_commit_move *move = p; +// TODO redeclare +static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, + int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), + void *data); +static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, + uint32_t mask, lfs_entry_t *entry); - if (lfs_tag_id(entry.tag) != move->id.to - move->commit->filter.begin) { - return 1; - } +static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_entry_t entry) { + struct lfs_commit_move *move = p; - if (lfs_tag_type(entry.tag) & 0x100) { - if (lfs_tag_type(entry.tag) == move->type) { - return 2; - } - } else { - // TODO hmm - if (lfs_tag_subtype(entry.tag) == (move->type & 0x1f0)) { - return 2; - } + if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE && + lfs_tag_id(entry.tag) <= move->id.from) { + // something was deleted, we need to move around it + move->id.from += 1; + return 0; } - - return 0; -} - -static int lfs_commit_moveiter(lfs_t *lfs, void *p, lfs_entry_t entry) { - struct lfs_commit_move *move = p; - if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(entry.tag) <= move->id.from) { - // something was deleted, we need to move around it - move->id.from += 1; - } + if (lfs_tag_type(entry.tag) == LFS_TYPE_MOVE) { + // TODO need this? + // ignore moves + return 0; } if (lfs_tag_id(entry.tag) != move->id.from) { @@ -694,28 +647,41 @@ static int lfs_commit_moveiter(lfs_t *lfs, void *p, lfs_entry_t entry) { return 0; } - move->type = lfs_tag_type(entry.tag); - int res = lfs_commit_traverse(lfs, move->commit, - lfs_commit_movecheck, move); - if (res < 0) { - return res; + // check if type has already been committed + int err = lfs_dir_get(lfs, + &(lfs_dir_t){ + .pair[0]=move->commit->block, + .off=move->commit->off, + .etag=move->commit->ptag, + .stop_at_commit=true}, + lfs_tag_type(entry.tag) & 0x100 ? 0x7ffff000 : 0x7c1ff000, + &(lfs_entry_t){ + lfs_mktag(lfs_tag_type(entry.tag), move->id.to, 0)}); + if (err && err != LFS_ERR_NOENT) { + return err; } - if (res == 2) { + if (err != LFS_ERR_NOENT) { // already committed return 0; } // update id and commit, as we are currently unique - entry.tag = (move->id.to << 12) | (entry.tag & 0xffe00fff); + entry.tag = lfs_mktag(0, move->id.to, 0) | (entry.tag & 0xffe00fff); return lfs_commit_commit(lfs, move->commit, entry); } +// TODO change this to be special RAM-side type in linked list of commits? static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { struct lfs_commit_move *move = p; move->commit = commit; + if (move->id.to < commit->filter.begin || + move->id.to >= commit->filter.end) { + // skip if not in filter + return 0; + } - int err = lfs_dir_traverse(lfs, move->dir, lfs_commit_moveiter, move); + int err = lfs_dir_traverse(lfs, move->dir, lfs_commit_movescan, move); if (err < 0) { return err; } @@ -759,6 +725,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; + dir->stop_at_commit = false; // find the block with the most recent revision uint32_t rev[2]; @@ -812,7 +779,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, break; } - printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, dir->pair[0], off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); + //printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, dir->pair[0], off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); if (lfs_tag_type(tag) == LFS_TYPE_CRC) { // check the crc entry uint32_t dcrc; @@ -862,6 +829,9 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { dir->count -= 1; + if (dir->moveid != -1) { + //printf("RENAME DEL %d (%d)\n", lfs_tag_id(tag), dir->moveid); + } if (lfs_tag_id(tag) == dir->moveid) { dir->moveid = -1; } else if (lfs_tag_id(tag) < dir->moveid) { @@ -901,9 +871,41 @@ static int lfs_dir_fetch(lfs_t *lfs, static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data) { - return lfs_commit_traverse(lfs, &(struct lfs_commit){ - .block=dir->pair[0], .off=dir->off, .ptag=dir->etag}, - cb, data); + // iterate over dir block backwards (for faster lookups) + lfs_block_t block = dir->pair[0]; + lfs_off_t off = dir->off; + lfs_tag_t tag = dir->etag; + + while (off != sizeof(uint32_t)) { + // TODO rm me + //printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, block, off-lfs_tag_size(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); + + // TODO hmm + if (dir->stop_at_commit && lfs_tag_type(tag) == LFS_TYPE_CRC) { + break; + } + + int err = cb(lfs, data, (lfs_entry_t){ + (0x80000000 | tag), + .u.d.block=block, + .u.d.off=off-lfs_tag_size(tag)}); + if (err) { + return err; + } + + LFS_ASSERT(off > sizeof(tag)+lfs_tag_size(tag)); + off -= sizeof(tag)+lfs_tag_size(tag); + + lfs_tag_t ntag; + err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); + if (err) { + return err; + } + + tag ^= lfs_fromle32(ntag); + } + + return 0; } //struct lfs_dir_commitmove { @@ -1159,6 +1161,7 @@ static int lfs_dir_commitwith(lfs_t *lfs, lfs_dir_t *dir, int err = cb(lfs, data, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + lfs->pcache.block = 0xffffffff; return lfs_dir_compact(lfs, dir, cb, data, dir, 0, dir->count); } return err; @@ -1167,6 +1170,7 @@ static int lfs_dir_commitwith(lfs_t *lfs, lfs_dir_t *dir, err = lfs_commit_crc(lfs, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + lfs->pcache.block = 0xffffffff; return lfs_dir_compact(lfs, dir, cb, data, dir, 0, dir->count); } return err; @@ -1203,11 +1207,14 @@ struct lfs_dir_getter { static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t entry) { struct lfs_dir_getter *get = p; + if ((entry.tag & get->mask) == (get->tag & get->mask)) { - if (get->entry) { - *get->entry = entry; - } + *get->entry = entry; return true; + } else if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE) { + if (lfs_tag_id(entry.tag) <= lfs_tag_id(get->tag)) { + get->tag += lfs_mktag(0, 1, 0); + } } return false; @@ -1215,6 +1222,7 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t entry) { static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, uint32_t mask, lfs_entry_t *entry) { + uint16_t id = lfs_tag_id(entry->tag); int res = lfs_dir_traverse(lfs, dir, lfs_dir_getter, &(struct lfs_dir_getter){mask, entry->tag, entry}); if (res < 0) { @@ -1225,6 +1233,18 @@ static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, return LFS_ERR_NOENT; } + if (id == dir->moveid) { + int moved = lfs_moved(lfs, dir, dir->moveid); + if (moved < 0) { + return moved; + } + + if (moved) { + return LFS_ERR_NOENT; + } + } + + entry->tag = lfs_mktag(0, id, 0) | (entry->tag & 0xffe00fff); return 0; } @@ -1287,7 +1307,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_dir_t *dir, } err = lfs_dir_getbuffer(lfs, dir, 0x7ffff000, &(lfs_entry_t){ - lfs_mktag(LFS_TYPE_NAME, id, lfs->cfg->name_size+1), + lfs_mktag(LFS_TYPE_NAME, id, lfs->name_size+1), .u.buffer=info->name}); if (err) { return err; @@ -1401,24 +1421,24 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, break; } - if (lfs_pairisnull(dir->tail)) { + if (!dir->split) { return LFS_ERR_NOENT; } entry.u.pair[0] = dir->tail[0]; entry.u.pair[1] = dir->tail[1]; } - -// TODO handle moves -// // check that entry has not been moved -// if (entry->d.type & LFS_STRUCT_MOVED) { -// int moved = lfs_moved(lfs, &entry->d.u); -// if (moved < 0 || moved) { -// return (moved < 0) ? moved : LFS_ERR_NOENT; -// } -// -// entry->d.type &= ~LFS_STRUCT_MOVED; -// } + + if (find.id == dir->moveid) { + int moved = lfs_moved(lfs, dir, dir->moveid); + if (moved < 0) { + return moved; + } + + if (moved) { + return LFS_ERR_NOENT; + } + } *id = find.id; find.name += find.len; @@ -2319,11 +2339,15 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } + cwd.tail[0] = dir.pair[0]; + cwd.tail[1] = dir.pair[1]; err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_NAME, id, nlen), .u.buffer=(void*)path}, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_DIR | LFS_STRUCT_DIR, id, sizeof(dir.pair)), - .u.buffer=dir.pair}}}); + .u.buffer=dir.pair}, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(cwd.tail)), + .u.buffer=cwd.tail}}}}); // TODO need ack here? lfs_alloc_ack(lfs); @@ -3571,198 +3595,209 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { return lfs_dir_getinfo(lfs, &cwd, id, info); } -// -//int lfs_remove(lfs_t *lfs, const char *path) { -// // deorphan if we haven't yet, needed at most once after poweron -// if (!lfs->deorphaned) { -// int err = lfs_deorphan(lfs); -// if (err) { -// return err; -// } -// } -// -// lfs_dir_t cwd; -// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); -// if (err) { -// return err; -// } -// -// lfs_entry_t entry; -// err = lfs_dir_find(lfs, &cwd, &entry, &path); -// if (err) { -// return err; -// } -// -// lfs_dir_t dir; -// if ((0xf & entry.d.type) == LFS_TYPE_DIR) { -// // must be empty before removal, checking size -// // without masking top bit checks for any case where -// // dir is not empty -// err = lfs_dir_fetch(lfs, &dir, entry.d.u.dir); -// if (err) { -// return err; -// } else if (dir.d.size != sizeof(dir.d)+4) { -// return LFS_ERR_NOTEMPTY; -// } -// } -// -// // remove the entry -// err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); -// if (err) { -// return err; -// } -// -// // if we were a directory, find pred, replace tail -// if ((0xf & entry.d.type) == LFS_TYPE_DIR) { +int lfs_remove(lfs_t *lfs, const char *path) { + // deorphan if we haven't yet, needed at most once after poweron + if (!lfs->deorphaned) { + int err = lfs_deorphan(lfs); + if (err) { + return err; + } + } + + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + int16_t id; + err = lfs_dir_find(lfs, &cwd, &path, &id); + if (err) { + return err; + } + + // grab entry to see if we're dealing with a dir + lfs_entry_t entry; + err = lfs_dir_getentry(lfs, &cwd, 0x701ff000, + lfs_mktag(LFS_TYPE_REG, id, 0), &entry); + if (err) { + return err; + } + + if (lfs_tag_subtype(entry.tag) == LFS_TYPE_DIR) { + lfs_dir_t dir; + // must be empty before removal + err = lfs_dir_fetch(lfs, &dir, entry.u.pair); + if (err) { + return err; + } + + if (dir.count > 0 || dir.split) { + return LFS_ERR_NOTEMPTY; + } + } + + // delete the entry + err = lfs_dir_delete(lfs, &cwd, id); + if (err) { + return err; + } + + // if we were a directory, find pred, replace tail + // TODO can this just deorphan? + if (lfs_tag_subtype(entry.tag) == LFS_TYPE_DIR) { + err = lfs_deorphan(lfs); + if (err) { + return err; + } + } + +// if (lfs_tag_subtype(entry.tag) == LFS_TYPE_DIR) { // int res = lfs_pred(lfs, dir.pair, &cwd); // if (res < 0) { // return res; // } // // LFS_ASSERT(res); // must have pred -// cwd.d.tail[0] = dir.d.tail[0]; -// cwd.d.tail[1] = dir.d.tail[1]; +// cwd.tail[0] = dir.tail[0]; +// cwd.tail[1] = dir.tail[1]; // // err = lfs_dir_commit(lfs, &cwd, NULL, 0); // if (err) { // return err; // } // } -// -// return 0; -//} -// -//int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { -// // deorphan if we haven't yet, needed at most once after poweron -// if (!lfs->deorphaned) { -// int err = lfs_deorphan(lfs); -// if (err) { -// return err; -// } -// } -// -// // find old entry -// lfs_dir_t oldcwd; -// int err = lfs_dir_fetch(lfs, &oldcwd, lfs->root); -// if (err) { -// return err; -// } -// -// lfs_entry_t oldentry; -// err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath); -// if (err) { -// return err; -// } -// -// // allocate new entry -// lfs_dir_t newcwd; -// err = lfs_dir_fetch(lfs, &newcwd, lfs->root); -// if (err) { -// return err; -// } -// -// lfs_entry_t preventry; -// err = lfs_dir_find(lfs, &newcwd, &preventry, &newpath); -// if (err && (err != LFS_ERR_NOENT || strchr(newpath, '/') != NULL)) { -// return err; -// } -// -// bool prevexists = (err != LFS_ERR_NOENT); -// bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); -// -// // check that name fits -// lfs_size_t nlen = strlen(newpath); -// if (nlen > lfs->name_size) { -// return LFS_ERR_NAMETOOLONG; -// } -// -// if (oldentry.size - oldentry.d.nlen + nlen > lfs->cfg->block_size) { -// return LFS_ERR_NOSPC; -// } -// -// // must have same type -// if (prevexists && preventry.d.type != oldentry.d.type) { -// return LFS_ERR_ISDIR; -// } -// -// lfs_dir_t dir; -// if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { -// // must be empty before removal, checking size -// // without masking top bit checks for any case where -// // dir is not empty -// err = lfs_dir_fetch(lfs, &dir, preventry.d.u.dir); -// if (err) { -// return err; -// } else if (dir.d.size != sizeof(dir.d)+4) { -// return LFS_ERR_NOTEMPTY; -// } -// } -// -// // mark as moving -// oldentry.d.type |= LFS_STRUCT_MOVED; -// err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, 1, &oldentry.d.type, 1}}, 1); -// oldentry.d.type &= ~LFS_STRUCT_MOVED; -// if (err) { -// return err; -// } -// -// // update pair if newcwd == oldcwd -// if (samepair) { -// newcwd = oldcwd; -// } -// -// // move to new location -// lfs_entry_t newentry = preventry; -// newentry.d = oldentry.d; -// newentry.d.type &= ~LFS_STRUCT_MOVED; -// newentry.d.nlen = nlen; -// newentry.size = prevexists ? preventry.size : 0; -// -// lfs_size_t newsize = oldentry.size - oldentry.d.nlen + newentry.d.nlen; -// err = lfs_dir_set(lfs, &newcwd, &newentry, (struct lfs_region[]){ -// {LFS_FROM_REGION, 0, prevexists ? preventry.size : 0, -// &(struct lfs_region_region){ -// oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, 4, &newentry.d, 4}, -// {LFS_FROM_MEM, newsize-nlen, 0, newpath, nlen}}, 2}, -// newsize}}, 1); -// if (err) { -// return err; -// } -// -// // update pair if newcwd == oldcwd -// if (samepair) { -// oldcwd = newcwd; -// } -// -// // remove old entry -// err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, oldentry.size, NULL, 0}}, 1); -// if (err) { -// return err; -// } -// -// // if we were a directory, find pred, replace tail -// if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { -// int res = lfs_pred(lfs, dir.pair, &newcwd); -// if (res < 0) { -// return res; -// } -// -// LFS_ASSERT(res); // must have pred -// newcwd.d.tail[0] = dir.d.tail[0]; -// newcwd.d.tail[1] = dir.d.tail[1]; -// -// err = lfs_dir_commit(lfs, &newcwd, NULL, 0); -// if (err) { -// return err; -// } -// } -// -// return 0; -//} + + return 0; +} + +int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { + // deorphan if we haven't yet, needed at most once after poweron + if (!lfs->deorphaned) { + int err = lfs_deorphan(lfs); + if (err) { + return err; + } + } + + // find old entry + lfs_dir_t oldcwd; + int16_t oldid; + int err = lfs_dir_find(lfs, &oldcwd, &oldpath, &oldid); + if (err) { + return err; + } + + lfs_entry_t oldentry; + err = lfs_dir_getentry(lfs, &oldcwd, 0x701ff000, + lfs_mktag(LFS_TYPE_REG, oldid, 0), &oldentry); + if (err) { + return err; + } + + // find new entry + lfs_dir_t newcwd; + int16_t newid; + err = lfs_dir_find(lfs, &newcwd, &newpath, &newid); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + bool prevexists = (err != LFS_ERR_NOENT); + bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); + lfs_entry_t preventry; + if (prevexists) { + // get prev entry, check that we have same type + err = lfs_dir_getentry(lfs, &newcwd, 0x701ff000, + lfs_mktag(LFS_TYPE_REG, newid, 0), &preventry); + if (err) { + return err; + } + + if (lfs_tag_subtype(preventry.tag) != lfs_tag_subtype(oldentry.tag)) { + return LFS_ERR_ISDIR; + } + + if (lfs_tag_subtype(preventry.tag) == LFS_TYPE_DIR) { + lfs_dir_t prevdir; + // must be empty before removal + err = lfs_dir_fetch(lfs, &prevdir, preventry.u.pair); + if (err) { + return err; + } + + if (prevdir.count > 0 || prevdir.split) { + return LFS_ERR_NOTEMPTY; + } + } + } else { + // check that name fits + lfs_size_t nlen = strlen(newpath); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + + // get next id + err = lfs_dir_append(lfs, &newcwd, &newid); + if (err) { + return err; + } + } + + // mark as moving + //printf("RENAME MOVE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); + err = lfs_dir_commit(lfs, &oldcwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_MOVE, oldid, 0)}}); + if (err) { + return err; + } + + if (samepair) { + // update pair if newcwd == oldcwd + newcwd = oldcwd; + } + + // move to new location + // TODO NAME????? + // TODO HAH, move doesn't want to override things (due + // to its use in compaction), but that's _exactly_ what we want here + err = lfs_dir_commitwith(lfs, &newcwd, lfs_commit_move, + &(struct lfs_commit_move){.dir=&oldcwd, .id={oldid, newid}}); + if (err) { + return err; + } + // TODO NONONONONO + // TODO also don't call strlen twice (see prev name check) + err = lfs_dir_commit(lfs, &newcwd, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), + .u.buffer=(void*)newpath}}); + if (err) { + return err; + } + + if (samepair) { + // update pair if newcwd == oldcwd + oldcwd = newcwd; + } + + // remove old entry + //printf("RENAME DELETE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); + err = lfs_dir_delete(lfs, &oldcwd, oldid); + if (err) { + return err; + } + + // if we were a directory, find pred, replace tail + // TODO can this just deorphan? + if (prevexists && lfs_tag_subtype(preventry.tag) == LFS_TYPE_DIR) { + err = lfs_deorphan(lfs); + if (err) { + return err; + } + } + + return 0; +} //int lfs_getattrs(lfs_t *lfs, const char *path, // const struct lfs_attr *attrs, int count) { @@ -3940,9 +3975,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .block_size = lfs->cfg->block_size, .block_count = lfs->cfg->block_count, - .inline_size = lfs->cfg->inline_size, - .attrs_size = lfs->cfg->attrs_size, - .name_size = lfs->cfg->name_size, + .inline_size = lfs->inline_size, + .attrs_size = lfs->attrs_size, + .name_size = lfs->name_size, }; dir.count += 1; @@ -4088,7 +4123,6 @@ int lfs_fs_traverse(lfs_t *lfs, return err; } } - } } @@ -4240,6 +4274,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], return err; } + // TODO make this O(n) by using fetchwith to match the pointers for (uint16_t id = 0; id < parent->count; id++) { int err = lfs_dir_getentry(lfs, parent, 0x43dff000, lfs_mktag(LFS_STRUCT_DIR, id, 0), entry); @@ -4295,42 +4330,48 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], return false; } */ -static int lfs_moved(lfs_t *lfs, const lfs_block_t pair[2]) { +static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid) { + // grab entry pair we're looking for + fromdir->moveid = -1; + lfs_entry_t fromentry; + int err = lfs_dir_getentry(lfs, fromdir, 0x43dff000, + lfs_mktag(LFS_STRUCT_DIR, fromid, 0), &fromentry); + fromdir->moveid = fromid; + if (err) { + return err; + } + // skip superblock - lfs_dir_t dir; - int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + lfs_dir_t todir; + err = lfs_dir_fetch(lfs, &todir, (const lfs_block_t[2]){0, 1}); if (err) { return err; } // iterate over all directory directory entries - while (!lfs_pairisnull(dir.tail)) { - int err = lfs_dir_fetch(lfs, &dir, dir.tail); + while (!lfs_pairisnull(todir.tail)) { + int err = lfs_dir_fetch(lfs, &todir, todir.tail); if (err) { return err; } - for (int id = 0; id < dir.count; id++) { - lfs_entry_t entry; - int err = lfs_dir_getentry(lfs, &dir, 0x43dff000, - lfs_mktag(LFS_STRUCT_DIR, id, 0), &entry); - if (err) { - if (err == LFS_ERR_NOENT) { - continue; - } - return err; + for (int toid = 0; toid < todir.count; toid++) { + if (lfs_paircmp(todir.pair, fromdir->pair) == 0 && + toid == fromid) { + continue; } - err = lfs_dir_get(lfs, &dir, 0x7ffff000, &(lfs_entry_t){ - lfs_mktag(LFS_TYPE_MOVE, id, 0)}); - if (err != LFS_ERR_NOENT) { - if (!err) { + lfs_entry_t toentry; + int err = lfs_dir_getentry(lfs, &todir, 0x43dff000, + lfs_mktag(LFS_STRUCT_DIR, toid, 0), &toentry); + if (err) { + if (err == LFS_ERR_NOENT) { continue; } return err; } - if (lfs_paircmp(entry.u.pair, pair) == 0) { + if (lfs_paircmp(toentry.u.pair, fromentry.u.pair) == 0) { return true; } } diff --git a/lfs.h b/lfs.h index a0891252..5cb83779 100644 --- a/lfs.h +++ b/lfs.h @@ -114,6 +114,11 @@ enum lfs_type { LFS_STRUCT_INLINE = 0x000, LFS_STRUCT_CTZ = 0x004, LFS_STRUCT_DIR = 0x008, + + // internal sources + LFS_FROM_REGION = 0x000, + LFS_FROM_DISK = 0x001, + LFS_FROM_MOVE = 0x002, }; // File open flags @@ -313,6 +318,7 @@ typedef struct lfs_dir { uint16_t count; bool erased; bool split; + bool stop_at_commit; // TODO hmmm int16_t moveid; uint16_t id; diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index d5303cc2..c9cc6694 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -117,8 +117,6 @@ tests/test.py << TEST for (int i = 0; i < $LARGESIZE; i++) { sprintf((char*)buffer, "test%d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; - printf("nameee %s\n", info.name); - printf("expect %s\n", (char*)buffer); strcmp(info.name, (char*)buffer) => 0; info.type => LFS_TYPE_DIR; } @@ -154,6 +152,8 @@ tests/test.py << TEST strcmp(info.name, "..") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; + printf("nameee \"%s\"\n", info.name); + printf("expect \"%s\"\n", "burito"); strcmp(info.name, "burito") => 0; info.type => LFS_TYPE_REG; lfs_dir_read(&lfs, &dir[0], &info) => 1; From 11a3c8d0623053a42957a135bf67708c3992d3e2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 28 May 2018 02:08:16 -0500 Subject: [PATCH 037/139] Continued progress with reintroducing testing on the new metadata logging Now with some tweaks to commit/compact, and a committers for entrylists and moves specifically. No longer relying on a commitwith callback, the types of commits are now infered from their tags. This means we can now commit things atomically with special commits, such as moves. Now lfs_rename can move entries to new names correctly. --- lfs.c | 332 +++++++++++++++++++++++++++++++++++++++++---- lfs.h | 5 +- tests/test_dirs.sh | 2 +- 3 files changed, 308 insertions(+), 31 deletions(-) diff --git a/lfs.c b/lfs.c index d4e6dc5b..d670bb37 100644 --- a/lfs.c +++ b/lfs.c @@ -473,6 +473,10 @@ struct lfs_commit { } filter; }; +// TODO predelcare +static int lfs_commit_move_(lfs_t *lfs, struct lfs_commit *commit, + lfs_entry_t entry); + //static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t entry) { // struct lfs_commit *commit = p; // if (lfs_tag_id(entry.tag) != commit->compact.id) { @@ -492,6 +496,12 @@ static int lfs_commit_commit(lfs_t *lfs, lfs_tag_id(entry.tag) >= commit->filter.end)) { return 0; } + + // special cases + if ((lfs_tag_type(entry.tag) & 0x103) == LFS_FROM_MOVE) { + return lfs_commit_move_(lfs, commit, entry); + } + uint16_t id = lfs_tag_id(entry.tag) - commit->filter.begin; entry.tag = lfs_mktag(0, id, 0) | (entry.tag & 0xffe00fff); @@ -607,8 +617,21 @@ static int lfs_commit_regions(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } +static int lfs_commit_list(lfs_t *lfs, struct lfs_commit *commit, + lfs_entrylist_t *list) { + for (; list; list = list->next) { + int err = lfs_commit_commit(lfs, commit, list->e); + if (err) { + return err; + } + } + + return 0; +} + // committer for moves +// TODO rename? struct lfs_commit_move { lfs_dir_t *dir; struct { @@ -619,6 +642,7 @@ struct lfs_commit_move { struct lfs_commit *commit; }; + // TODO redeclare static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), @@ -689,6 +713,23 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { return 0; } +static int lfs_commit_move_(lfs_t *lfs, struct lfs_commit *commit, + lfs_entry_t entry) { + struct lfs_commit_move move = { + .dir = entry.u.dir, + .id.to = lfs_tag_id(entry.tag), + .id.from = lfs_tag_size(entry.tag), + .commit = commit, + }; + + int err = lfs_dir_traverse(lfs, entry.u.dir, lfs_commit_movescan, &move); + if (err) { + return err; + } + + return 0; +} + static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir, bool split, const lfs_block_t tail[2]) { // allocate pair of dir blocks (backwards, so we write to block 1 first) @@ -1139,6 +1180,228 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, return 0; } +static int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, + lfs_dir_t *source, uint16_t begin, uint16_t end) { + // save some state in case block is bad + const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; + bool relocated = false; + + // increment revision count + dir->rev += 1; + + while (true) { + // last complete id + int16_t ack = -1; + dir->count = end - begin; + + if (true) { + // erase block to write to + int err = lfs_bd_erase(lfs, dir->pair[1]); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // write out header + uint32_t crc = 0xffffffff; + uint32_t rev = lfs_tole32(dir->rev); + lfs_crc(&crc, &rev, sizeof(rev)); + err = lfs_bd_prog(lfs, dir->pair[1], 0, &rev, sizeof(rev)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // setup compaction + struct lfs_commit commit = { + .block = dir->pair[1], + .off = sizeof(dir->rev), + + // space is complicated, we need room for tail, crc, + // and we keep cap at around half a block + .begin = 0, + .end = lfs_min( + lfs_alignup(lfs->cfg->block_size / 2, + lfs->cfg->prog_size), + lfs->cfg->block_size - 5*sizeof(uint32_t)), + .crc = crc, + .ptag = 0, + + // filter out ids + .filter.begin = begin, + .filter.end = end, + }; + + // commit regions which can't fend for themselves + err = lfs_commit_list(lfs, &commit, list); + if (err) { + if (err == LFS_ERR_NOSPC) { + goto split; + } else if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // move over other commits, leaving it up to lfs_commit_move to + // filter out duplicates, and the commit filtering to reassign ids + for (uint16_t id = begin; id < end; id++) { + err = lfs_commit_commit(lfs, &commit, (lfs_entry_t){ + lfs_mktag(LFS_FROM_MOVE, id, id), .u.dir=source}); + if (err) { + if (err == LFS_ERR_NOSPC) { + goto split; + } else if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + ack = id; + } + + commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); + if (!lfs_pairisnull(dir->tail)) { + // TODO le32 + err = lfs_commit_commit(lfs, &commit, (lfs_entry_t){ + lfs_mktag(LFS_TYPE_SOFTTAIL + dir->split*0x10, + 0x1ff, sizeof(dir->tail)), + .u.buffer=dir->tail}); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + err = lfs_commit_crc(lfs, &commit); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // successful compaction, swap dir pair to indicate most recent + lfs_pairswap(dir->pair); + dir->off = commit.off; + dir->etag = commit.ptag; + dir->erased = true; + } + break; + +split: + // commit no longer fits, need to split dir, + // drop caches and create tail + lfs->pcache.block = 0xffffffff; + + lfs_dir_t tail; + int err = lfs_dir_alloc(lfs, &tail, dir->split, dir->tail); + if (err) { + return err; + } + + err = lfs_dir_compact_(lfs, &tail, list, dir, ack+1, end); + if (err) { + return err; + } + + end = ack+1; + dir->tail[0] = tail.pair[0]; + dir->tail[1] = tail.pair[1]; + dir->split = true; + continue; + +relocate: + //commit was corrupted + LFS_DEBUG("Bad block at %d", dir->pair[1]); + + // drop caches and prepare to relocate block + relocated = true; + lfs->pcache.block = 0xffffffff; + + // can't relocate superblock, filesystem is now frozen + if (lfs_paircmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { + LFS_WARN("Superblock %d has become unwritable", oldpair[1]); + return LFS_ERR_CORRUPT; + } + + // relocate half of pair + err = lfs_alloc(lfs, &dir->pair[1]); + if (err) { + return err; + } + + continue; + } + + if (relocated) { + // update references if we relocated + LFS_DEBUG("Relocating %d %d to %d %d", + oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); + int err = lfs_relocate(lfs, oldpair, dir->pair); + if (err) { + return err; + } + } + + // shift over any directories that are affected + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + d->pair[0] = dir->pair[0]; + d->pair[1] = dir->pair[1]; + } + } + + return 0; +} + +static int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list) { + if (!dir->erased) { + // not erased, must compact + return lfs_dir_compact_(lfs, dir, list, dir, 0, dir->count); + } + + struct lfs_commit commit = { + .block = dir->pair[0], + .begin = dir->off, + .off = dir->off, + .end = lfs->cfg->block_size - 2*sizeof(uint32_t), + .crc = 0xffffffff, + .ptag = dir->etag, + .filter.begin = 0, + .filter.end = 0x1ff, + }; + + int err = lfs_commit_list(lfs, &commit, list); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + lfs->pcache.block = 0xffffffff; + return lfs_dir_compact_(lfs, dir, list, dir, 0, dir->count); + } + return err; + } + + err = lfs_commit_crc(lfs, &commit); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + lfs->pcache.block = 0xffffffff; + return lfs_dir_compact_(lfs, dir, list, dir, 0, dir->count); + } + return err; + } + + // successful commit, lets update dir + dir->off = commit.off; + dir->etag = commit.ptag; + return 0; +} + static int lfs_dir_commitwith(lfs_t *lfs, lfs_dir_t *dir, int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), void *data) { @@ -1195,7 +1458,7 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, uint16_t *id) { static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { dir->count -= 1; - return lfs_dir_commit(lfs, dir, &(lfs_entrylist_t){ + return lfs_dir_commit_(lfs, dir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_DELETE, id, 0)}}); } @@ -1233,7 +1496,9 @@ static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, return LFS_ERR_NOENT; } - if (id == dir->moveid) { + // TODO hmm, stop at commit? maybe we need to handle this elsewhere? + // Should commit get be its own thing? commit traverse? + if (id == dir->moveid && !dir->stop_at_commit) { int moved = lfs_moved(lfs, dir, dir->moveid); if (moved < 0) { return moved; @@ -1409,6 +1674,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, // find path while (true) { + printf("checking %d %d for %s\n", entry.u.pair[0], entry.u.pair[1], *path); find.id = -1; int err = lfs_dir_fetchwith(lfs, dir, entry.u.pair, lfs_dir_finder, &find); @@ -2327,7 +2593,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } - err = lfs_dir_commit(lfs, &dir, NULL); + err = lfs_dir_commit_(lfs, &dir, NULL); if (err) { return err; } @@ -2341,7 +2607,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; - err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_NAME, id, nlen), .u.buffer=(void*)path}, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_DIR | LFS_STRUCT_DIR, id, sizeof(dir.pair)), @@ -2943,7 +3209,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return err; } - err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_NAME, id, nlen), .u.buffer=(void*)path}, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0)}}}); @@ -3193,7 +3459,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { - int err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + int err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_CTZ, file->id, 2*sizeof(uint32_t)), .u.buffer=&file->head}, file->attrs}); @@ -3201,7 +3467,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } } else { - int err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + int err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, file->id, file->size), .u.buffer=file->cache.buffer}, file->attrs}); @@ -3746,7 +4012,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving //printf("RENAME MOVE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); - err = lfs_dir_commit(lfs, &oldcwd, &(lfs_entrylist_t){ + err = lfs_dir_commit_(lfs, &oldcwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_MOVE, oldid, 0)}}); if (err) { return err; @@ -3757,20 +4023,30 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newcwd = oldcwd; } - // move to new location - // TODO NAME????? - // TODO HAH, move doesn't want to override things (due - // to its use in compaction), but that's _exactly_ what we want here - err = lfs_dir_commitwith(lfs, &newcwd, lfs_commit_move, - &(struct lfs_commit_move){.dir=&oldcwd, .id={oldid, newid}}); - if (err) { - return err; - } - // TODO NONONONONO - // TODO also don't call strlen twice (see prev name check) - err = lfs_dir_commit(lfs, &newcwd, &(lfs_entrylist_t){ +// TODO check that all complaints are fixed +// // move to new location +// // TODO NAME????? +// // TODO HAH, move doesn't want to override things (due +// // to its use in compaction), but that's _exactly_ what we want here +// err = lfs_dir_commitwith(lfs, &newcwd, lfs_commit_move, +// &(struct lfs_commit_move){.dir=&oldcwd, .id={oldid, newid}}); +// if (err) { +// return err; +// } +// // TODO NONONONONO +// // TODO also don't call strlen twice (see prev name check) +// err = lfs_dir_commit_(lfs, &newcwd, &(lfs_entrylist_t){ +// {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), +// .u.buffer=(void*)newpath}}); +// if (err) { +// return err; +// } + + err = lfs_dir_commit_(lfs, &newcwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), - .u.buffer=(void*)newpath}}); + .u.buffer=(void*)newpath}, &(lfs_entrylist_t){ + {lfs_mktag(LFS_FROM_MOVE, newid, oldid), + .u.dir=&oldcwd}}}); if (err) { return err; } @@ -3956,7 +4232,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_commit(lfs, &root, NULL); + err = lfs_dir_commit_(lfs, &root, NULL); if (err) { return err; } @@ -3981,7 +4257,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; dir.count += 1; - err = lfs_dir_commit(lfs, &dir, &(lfs_entrylist_t){ + err = lfs_dir_commit_(lfs, &dir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, 0, sizeof(superblock)), .u.buffer=&superblock}}); if (err) { @@ -4434,7 +4710,7 @@ static int lfs_relocate(lfs_t *lfs, // update disk, this creates a desync entry.u.pair[0] = newpair[0]; entry.u.pair[1] = newpair[1]; - int err = lfs_dir_commit(lfs, &parent, &(lfs_entrylist_t){entry}); + int err = lfs_dir_commit_(lfs, &parent, &(lfs_entrylist_t){entry}); if (err) { return err; } @@ -4460,9 +4736,9 @@ static int lfs_relocate(lfs_t *lfs, // just replace bad pair, no desync can occur parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; - return lfs_dir_commit(lfs, &parent, &(lfs_entrylist_t){ + return lfs_dir_commit_(lfs, &parent, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL + parent.split*0x10, // TODO hm - 0x1ff, sizeof(newpair)), + 0x1ff, sizeof(lfs_block_t[2])), .u.pair[0]=newpair[0], .u.pair[1]=newpair[1]}}); } @@ -4503,7 +4779,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ + err = lfs_dir_commit_(lfs, &pdir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { @@ -4520,7 +4796,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = entry.u.pair[0]; pdir.tail[1] = entry.u.pair[1]; - err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ + err = lfs_dir_commit_(lfs, &pdir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { diff --git a/lfs.h b/lfs.h index 5cb83779..b4bed046 100644 --- a/lfs.h +++ b/lfs.h @@ -117,8 +117,8 @@ enum lfs_type { // internal sources LFS_FROM_REGION = 0x000, - LFS_FROM_DISK = 0x001, - LFS_FROM_MOVE = 0x002, + LFS_FROM_DISK = 0x200, + LFS_FROM_MOVE = 0x001, }; // File open flags @@ -276,6 +276,7 @@ typedef struct lfs_entry { lfs_block_t block; lfs_off_t off; } d; + struct lfs_dir *dir; } u; } lfs_entry_t; diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index c9cc6694..26e88561 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -329,7 +329,7 @@ tests/test.py << TEST TEST echo "--- Multi-block remove ---" -tests/test.py << TEST +tests/test.py -s << TEST lfs_mount(&lfs, &cfg) => 0; lfs_remove(&lfs, "cactus") => LFS_ERR_NOTEMPTY; From 483d41c545783d58a954912b1253e9cea7dbdaea Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 28 May 2018 09:17:44 -0500 Subject: [PATCH 038/139] Passing all of the basic functionality tests Integration with the new journaling metadata has now progressed to the point where all of the basic functionality tests are passing. This includes: - test_format - test_dirs - test_files - test_seek - test_truncate - test_interspersed - test_paths Some of the fixes: - Modified move to correctly change entry ids - Called lfs_commit_move directly from compact, avoiding commit parsing logic during a compact - Opened up commit filters to be passed down from compact for moves - Added correct drop logic to lfs_dir_delete - Updated lfs_dir_seek to use ids instead of offsets - Caught id updates manually where possible (this needs to be fixed) --- lfs.c | 140 +++++++++++++++++++++++++++++++++------------ tests/test_dirs.sh | 2 +- 2 files changed, 106 insertions(+), 36 deletions(-) diff --git a/lfs.c b/lfs.c index d670bb37..5a038152 100644 --- a/lfs.c +++ b/lfs.c @@ -475,7 +475,8 @@ struct lfs_commit { // TODO predelcare static int lfs_commit_move_(lfs_t *lfs, struct lfs_commit *commit, - lfs_entry_t entry); + uint16_t fromid, uint16_t toid, + lfs_dir_t *dir, lfs_entrylist_t *list); //static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t entry) { // struct lfs_commit *commit = p; @@ -499,7 +500,9 @@ static int lfs_commit_commit(lfs_t *lfs, // special cases if ((lfs_tag_type(entry.tag) & 0x103) == LFS_FROM_MOVE) { - return lfs_commit_move_(lfs, commit, entry); + return lfs_commit_move_(lfs, commit, + lfs_tag_size(entry.tag), lfs_tag_id(entry.tag), + entry.u.dir, NULL); } uint16_t id = lfs_tag_id(entry.tag) - commit->filter.begin; @@ -633,7 +636,7 @@ static int lfs_commit_list(lfs_t *lfs, struct lfs_commit *commit, // committer for moves // TODO rename? struct lfs_commit_move { - lfs_dir_t *dir; + lfs_dir_t *dir; // TODO need dir? struct { uint16_t from; uint16_t to; @@ -680,7 +683,8 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_entry_t entry) { .stop_at_commit=true}, lfs_tag_type(entry.tag) & 0x100 ? 0x7ffff000 : 0x7c1ff000, &(lfs_entry_t){ - lfs_mktag(lfs_tag_type(entry.tag), move->id.to, 0)}); + lfs_mktag(lfs_tag_type(entry.tag), + move->id.to - move->commit->filter.begin, 0)}); // TODO can all these filter adjustments be consolidated? if (err && err != LFS_ERR_NOENT) { return err; } @@ -714,15 +718,22 @@ static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { } static int lfs_commit_move_(lfs_t *lfs, struct lfs_commit *commit, - lfs_entry_t entry) { + uint16_t fromid, uint16_t toid, + lfs_dir_t *dir, lfs_entrylist_t *list) { struct lfs_commit_move move = { - .dir = entry.u.dir, - .id.to = lfs_tag_id(entry.tag), - .id.from = lfs_tag_size(entry.tag), + .id.from = fromid, + .id.to = toid, .commit = commit, }; - int err = lfs_dir_traverse(lfs, entry.u.dir, lfs_commit_movescan, &move); + for (; list; list = list->next) { + int err = lfs_commit_movescan(lfs, &move, list->e); + if (err) { + return err; + } + } + + int err = lfs_dir_traverse(lfs, dir, lfs_commit_movescan, &move); if (err) { return err; } @@ -1185,6 +1196,7 @@ static int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; bool relocated = false; + lfs_dir_t tail; // increment revision count dir->rev += 1; @@ -1236,22 +1248,23 @@ static int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, .filter.end = end, }; - // commit regions which can't fend for themselves - err = lfs_commit_list(lfs, &commit, list); - if (err) { - if (err == LFS_ERR_NOSPC) { - goto split; - } else if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // move over other commits, leaving it up to lfs_commit_move to - // filter out duplicates, and the commit filtering to reassign ids +// // commit regions which can't fend for themselves +// err = lfs_commit_list(lfs, &commit, list); +// if (err) { +// if (err == LFS_ERR_NOSPC) { +// goto split; +// } else if (err == LFS_ERR_CORRUPT) { +// goto relocate; +// } +// return err; +// } +// +// // move over other commits, leaving it up to lfs_commit_move to +// // filter out duplicates, and the commit filtering to reassign ids for (uint16_t id = begin; id < end; id++) { - err = lfs_commit_commit(lfs, &commit, (lfs_entry_t){ - lfs_mktag(LFS_FROM_MOVE, id, id), .u.dir=source}); + err = lfs_commit_move_(lfs, &commit, id, id, source, list); +// err = lfs_commit_commit(lfs, &commit, (lfs_entry_t){ +// lfs_mktag(LFS_FROM_MOVE, id, id), .u.dir=source}); if (err) { if (err == LFS_ERR_NOSPC) { goto split; @@ -1458,8 +1471,55 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, uint16_t *id) { static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { dir->count -= 1; - return lfs_dir_commit_(lfs, dir, &(lfs_entrylist_t){ + + // check if we should drop the directory block + if (dir->count == 0) { + lfs_dir_t pdir; + int res = lfs_pred(lfs, dir->pair, &pdir); + if (res < 0) { + return res; + } + + if (res && pdir.split) { + pdir.split = dir->split; + pdir.tail[0] = dir->tail[0]; + pdir.tail[1] = dir->tail[1]; + int err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL + pdir.split*0x10, + 0x1ff, sizeof(pdir.tail)), + .u.buffer=pdir.tail}}); + return err; + } + } + + int err = lfs_dir_commit_(lfs, dir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_DELETE, id, 0)}}); + if (err) { + return err; + } + + // shift over any files that are affected + // TODO move this to dir_commit? + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0) { + if (f->id == id) { + f->pair[0] = 0xffffffff; + f->pair[1] = 0xffffffff; + } else if (f->id > id) { + f->id -= 1; + } + } + } + +// TODO ? +// for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { +// if (lfs_paircmp(d->pair, dir->pair) == 0) { +// if (d->id > id) { +// d->id -= 1; +// d->pos -= 1; +// } +// } +// } } struct lfs_dir_getter { @@ -1674,7 +1734,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, // find path while (true) { - printf("checking %d %d for %s\n", entry.u.pair[0], entry.u.pair[1], *path); + //printf("checking %d %d for %s\n", entry.u.pair[0], entry.u.pair[1], *path); find.id = -1; int err = lfs_dir_fetchwith(lfs, dir, entry.u.pair, lfs_dir_finder, &find); @@ -2796,11 +2856,14 @@ int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { } // first two for ./.. - dir->id = lfs_min(2 + dir->count, off); - dir->pos += dir->id; - off -= dir->id; + dir->pos = lfs_min(2, off); + off -= dir->pos; while (off != 0) { + dir->id = lfs_min(dir->count, off); + dir->pos += dir->id; + off -= dir->id; + if (dir->id == dir->count) { if (!dir->split) { return LFS_ERR_INVAL; @@ -2811,10 +2874,6 @@ int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { return err; } } - - dir->id = lfs_min(dir->count, off); - dir->pos += dir->id; - off -= dir->id; } return 0; @@ -3209,6 +3268,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return err; } + // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_NAME, id, nlen), .u.buffer=(void*)path}, &(lfs_entrylist_t){ @@ -3217,9 +3277,19 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return err; } + // TODO eh + if (id >= cwd.count) { + // catch updates from a compact in the above commit + id -= cwd.count; + cwd.pair[0] = cwd.tail[0]; + cwd.pair[1] = cwd.tail[1]; + } + entry.tag = lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0); } else { - if (flags & LFS_O_EXCL) { + if (id == -1) { + return LFS_ERR_ISDIR; + } else if (flags & LFS_O_EXCL) { return LFS_ERR_EXIST; } @@ -4502,7 +4572,7 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_dir_t *pdir) { pdir->tail[1] = 1; while (!lfs_pairisnull(pdir->tail)) { if (lfs_paircmp(pdir->tail, pair) == 0) { - return true; + return true; // TODO should we return true only if pred is part of dir? } int err = lfs_dir_fetch(lfs, pdir, pdir->tail); diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 26e88561..c9cc6694 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -329,7 +329,7 @@ tests/test.py << TEST TEST echo "--- Multi-block remove ---" -tests/test.py -s << TEST +tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_remove(&lfs, "cactus") => LFS_ERR_NOTEMPTY; From 85a9638d9f01434178af1ca07a07ac7fa97fe61a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 28 May 2018 17:46:32 -0500 Subject: [PATCH 039/139] Fixed issues discovered around testing moves lfs_dir_fetchwith did not recover from failed dir fetches correctly, added a temporary dir variable to hold dir contents while being populated, allowing us to fall back to a known good dir state if a commit is corrupted. There is a RAM cost, but the upside is that our lfs_dir_fetchwith actually works. Also added better handling of move ids during some get functions. --- lfs.c | 61 +++++++++++++++++++++++++++++----------------- tests/test_move.sh | 6 ++--- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/lfs.c b/lfs.c index 5a038152..cf67f768 100644 --- a/lfs.c +++ b/lfs.c @@ -809,10 +809,12 @@ static int lfs_dir_fetchwith(lfs_t *lfs, lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); dir->rev = lfs_fromle32(dir->rev); + lfs_dir_t temp = *dir; + while (true) { // extract next tag lfs_tag_t tag; - int err = lfs_bd_read(lfs, dir->pair[0], off, &tag, sizeof(tag)); + int err = lfs_bd_read(lfs, temp.pair[0], off, &tag, sizeof(tag)); if (err) { return err; } @@ -831,18 +833,18 @@ static int lfs_dir_fetchwith(lfs_t *lfs, break; } - //printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, dir->pair[0], off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); + //printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, temp.pair[0], off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); if (lfs_tag_type(tag) == LFS_TYPE_CRC) { // check the crc entry uint32_t dcrc; - int err = lfs_bd_read(lfs, dir->pair[0], + int err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { return err; } if (crc != lfs_fromle32(dcrc)) { - if (off == sizeof(dir->rev)) { + if (off == sizeof(temp.rev)) { // try other block break; } else { @@ -852,11 +854,12 @@ static int lfs_dir_fetchwith(lfs_t *lfs, } } - dir->off = off + sizeof(tag)+lfs_tag_size(tag); - dir->etag = tag; + temp.off = off + sizeof(tag)+lfs_tag_size(tag); + temp.etag = tag; crc = 0xffffffff; + *dir = temp; } else { - err = lfs_bd_crc(lfs, dir->pair[0], + err = lfs_bd_crc(lfs, temp.pair[0], off+sizeof(tag), lfs_tag_size(tag), &crc); if (err) { return err; @@ -864,37 +867,37 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (lfs_tag_type(tag) == LFS_TYPE_SOFTTAIL || lfs_tag_type(tag) == LFS_TYPE_HARDTAIL) { - dir->split = lfs_tag_type(tag) == LFS_TYPE_HARDTAIL; - err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), - dir->tail, sizeof(dir->tail)); + temp.split = lfs_tag_type(tag) == LFS_TYPE_HARDTAIL; + err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), + temp.tail, sizeof(temp.tail)); if (err) { return err; } } else if (lfs_tag_type(tag) == LFS_TYPE_MOVE) { // TODO handle moves correctly? - dir->moveid = lfs_tag_id(tag); + temp.moveid = lfs_tag_id(tag); } else { if (lfs_tag_id(tag) < 0x1ff && - lfs_tag_id(tag) >= dir->count) { - dir->count = lfs_tag_id(tag)+1; + lfs_tag_id(tag) >= temp.count) { + temp.count = lfs_tag_id(tag)+1; } if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { - dir->count -= 1; - if (dir->moveid != -1) { - //printf("RENAME DEL %d (%d)\n", lfs_tag_id(tag), dir->moveid); + temp.count -= 1; + if (temp.moveid != -1) { + //printf("RENAME DEL %d (%d)\n", lfs_tag_id(tag), temp.moveid); } - if (lfs_tag_id(tag) == dir->moveid) { - dir->moveid = -1; - } else if (lfs_tag_id(tag) < dir->moveid) { - dir->moveid -= 1; + if (lfs_tag_id(tag) == temp.moveid) { + temp.moveid = -1; + } else if (lfs_tag_id(tag) < temp.moveid) { + temp.moveid -= 1; } } if (cb) { err = cb(lfs, data, (lfs_entry_t){ (tag | 0x80000000), - .u.d.block=dir->pair[0], + .u.d.block=temp.pair[0], .u.d.off=off+sizeof(tag)}); if (err) { return err; @@ -1617,6 +1620,17 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_dir_t *dir, return 0; } + if (id == dir->moveid) { + int moved = lfs_moved(lfs, dir, dir->moveid); + if (moved < 0) { + return moved; + } + + if (moved) { + return LFS_ERR_NOENT; + } + } + lfs_entry_t entry; int err = lfs_dir_getentry(lfs, dir, 0x701ff000, lfs_mktag(LFS_TYPE_REG, id, 0), &entry); @@ -4680,8 +4694,9 @@ static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid) { // grab entry pair we're looking for fromdir->moveid = -1; lfs_entry_t fromentry; - int err = lfs_dir_getentry(lfs, fromdir, 0x43dff000, - lfs_mktag(LFS_STRUCT_DIR, fromid, 0), &fromentry); + // TODO what about inline files? + int err = lfs_dir_getentry(lfs, fromdir, 0x401ff000, + lfs_mktag(LFS_TYPE_REG, fromid, 0), &fromentry); fromdir->moveid = fromid; if (err) { return err; diff --git a/tests/test_move.sh b/tests/test_move.sh index 9e5ababf..ff02553d 100755 --- a/tests/test_move.sh +++ b/tests/test_move.sh @@ -59,7 +59,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; TEST -rm -v blocks/7 +truncate -s-7 blocks/6 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -86,8 +86,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hello", "d/hello") => 0; lfs_unmount(&lfs) => 0; TEST -rm -v blocks/8 -rm -v blocks/a +truncate -s-7 blocks/8 +truncate -s-7 blocks/a tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; From 9278b175374f9de9adebc520b7b06ff981201f03 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 28 May 2018 19:49:20 -0500 Subject: [PATCH 040/139] Trimmed old names and functions from the code base I've found using temporary names to duplicate functions temporarily (lfs_dir_commit + lfs_dir_commit_) is a great way to introduce sweeping changes while keeping the code base functional and (mostly) passing tests. It does mean at some point I need to deduplicate all these functions. --- lfs.c | 1431 ++------------------------------------------------------- 1 file changed, 29 insertions(+), 1402 deletions(-) diff --git a/lfs.c b/lfs.c index cf67f768..7c3ef0f6 100644 --- a/lfs.c +++ b/lfs.c @@ -474,21 +474,10 @@ struct lfs_commit { }; // TODO predelcare -static int lfs_commit_move_(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, lfs_dir_t *dir, lfs_entrylist_t *list); -//static int lfs_commit_compactcheck(lfs_t *lfs, void *p, lfs_entry_t entry) { -// struct lfs_commit *commit = p; -// if (lfs_tag_id(entry.tag) != commit->compact.id) { -// return 1; -// } else if (lfs_tag_type(entry.tag) == commit->compact.type) { -// return 2; -// } -// -// return 0; -//} -// static int lfs_commit_commit(lfs_t *lfs, struct lfs_commit *commit, lfs_entry_t entry) { // filter out ids @@ -500,7 +489,7 @@ static int lfs_commit_commit(lfs_t *lfs, // special cases if ((lfs_tag_type(entry.tag) & 0x103) == LFS_FROM_MOVE) { - return lfs_commit_move_(lfs, commit, + return lfs_commit_move(lfs, commit, lfs_tag_size(entry.tag), lfs_tag_id(entry.tag), entry.u.dir, NULL); } @@ -608,18 +597,6 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return 0; } -// committer for regions -static int lfs_commit_regions(lfs_t *lfs, void *p, struct lfs_commit *commit) { - for (lfs_entrylist_t *regions = p; regions; regions = regions->next) { - int err = lfs_commit_commit(lfs, commit, regions->e); - if (err) { - return err; - } - } - - return 0; -} - static int lfs_commit_list(lfs_t *lfs, struct lfs_commit *commit, lfs_entrylist_t *list) { for (; list; list = list->next) { @@ -699,25 +676,7 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_entry_t entry) { return lfs_commit_commit(lfs, move->commit, entry); } -// TODO change this to be special RAM-side type in linked list of commits? -static int lfs_commit_move(lfs_t *lfs, void *p, struct lfs_commit *commit) { - struct lfs_commit_move *move = p; - move->commit = commit; - if (move->id.to < commit->filter.begin || - move->id.to >= commit->filter.end) { - // skip if not in filter - return 0; - } - - int err = lfs_dir_traverse(lfs, move->dir, lfs_commit_movescan, move); - if (err < 0) { - return err; - } - - return 0; -} - -static int lfs_commit_move_(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, lfs_dir_t *dir, lfs_entrylist_t *list) { struct lfs_commit_move move = { @@ -963,243 +922,11 @@ static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, return 0; } -//struct lfs_dir_commitmove { -// // traversal things -// lfs_dir_t *dir; -// int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit); -// void *data; -// -// // ids to iterate through -// uint16_t begin; -// uint16_t end; -// uint16_t ack; -//}; -// -//static int lfs_dir_commitmove_commit(lfs_t *lfs, void *p, -// lfs_entry_t entry) { -// return lfs_commit_commit(lfs, p, entry); -//} -// -//int lfs_dir_commitmove(lfs_t *lfs, void *p, struct lfs_commit *commit) { -// struct lfs_dir_commitmove *move = p; -// for (int i = move->begin; i < move->end; i++) { -// // tell the committer to check for duplicates -// uint16_t old = commit->compact.id; -// if (commit->compact.id < 0) { -// commit->compact.id = i; -// } -// -// // commit pending commits -// int err = move->cb(lfs, move->data, commit); -// if (err) { -// commit->compact.id = old; -// return err; -// } -// -// // iterate over on-disk regions -// err = lfs_dir_traverse(lfs, move->dir, -// lfs_dir_commitmove_commit, commit); -// if (err) { -// commit->compact.id = old; -// return err; -// } -// -// move->ack = i; -// commit->compact.id = old; -// } -// -// return 0; -//} - -static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, - int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), - void *data, lfs_dir_t *source, uint16_t begin, uint16_t end) { - // save some state in case block is bad - const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; - bool relocated = false; - - // increment revision count - dir->rev += 1; - - while (true) { - // last complete id - int16_t ack = -1; - dir->count = end - begin; - - if (true) { - // erase block to write to - int err = lfs_bd_erase(lfs, dir->pair[1]); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // write out header - uint32_t crc = 0xffffffff; - uint32_t rev = lfs_tole32(dir->rev); - lfs_crc(&crc, &rev, sizeof(rev)); - err = lfs_bd_prog(lfs, dir->pair[1], 0, &rev, sizeof(rev)); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // setup compaction - struct lfs_commit commit = { - .block = dir->pair[1], - .off = sizeof(dir->rev), - - // space is complicated, we need room for tail, crc, - // and we keep cap at around half a block - .begin = 0, - .end = lfs_min( - lfs_alignup(lfs->cfg->block_size / 2, - lfs->cfg->prog_size), - lfs->cfg->block_size - 5*sizeof(uint32_t)), - .crc = crc, - .ptag = 0, - - // filter out ids - .filter.begin = begin, - .filter.end = end, - }; - - // commit regions which can't fend for themselves - err = cb(lfs, data, &commit); - if (err) { - if (err == LFS_ERR_NOSPC) { - goto split; - } else if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // move over other commits, leaving it up to lfs_commit_move to - // filter out duplicates, and the commit filtering to reassign ids - for (uint16_t id = begin; id < end; id++) { - err = lfs_commit_move(lfs, - &(struct lfs_commit_move){.dir=source, .id={id, id}}, - &commit); - if (err) { - if (err == LFS_ERR_NOSPC) { - goto split; - } else if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - ack = id; - } - - commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); - if (!lfs_pairisnull(dir->tail)) { - // TODO le32 - err = lfs_commit_commit(lfs, &commit, (lfs_entry_t){ - lfs_mktag(LFS_TYPE_SOFTTAIL + dir->split*0x10, - 0x1ff, sizeof(dir->tail)), - .u.buffer=dir->tail}); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - } - - err = lfs_commit_crc(lfs, &commit); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // successful compaction, swap dir pair to indicate most recent - lfs_pairswap(dir->pair); - dir->off = commit.off; - dir->etag = commit.ptag; - dir->erased = true; - } - break; - -split: - // commit no longer fits, need to split dir, - // drop caches and create tail - lfs->pcache.block = 0xffffffff; - - lfs_dir_t tail; - int err = lfs_dir_alloc(lfs, &tail, dir->split, dir->tail); - if (err) { - return err; - } - - err = lfs_dir_compact(lfs, &tail, cb, data, dir, ack+1, end); - if (err) { - return err; - } - - end = ack+1; - dir->tail[0] = tail.pair[0]; - dir->tail[1] = tail.pair[1]; - dir->split = true; - continue; - -relocate: - //commit was corrupted - LFS_DEBUG("Bad block at %d", dir->pair[1]); - - // drop caches and prepare to relocate block - relocated = true; - lfs->pcache.block = 0xffffffff; - - // can't relocate superblock, filesystem is now frozen - if (lfs_paircmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { - LFS_WARN("Superblock %d has become unwritable", oldpair[1]); - return LFS_ERR_CORRUPT; - } - - // relocate half of pair - err = lfs_alloc(lfs, &dir->pair[1]); - if (err) { - return err; - } - - continue; - } - - if (relocated) { - // update references if we relocated - LFS_DEBUG("Relocating %d %d to %d %d", - oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); - int err = lfs_relocate(lfs, oldpair, dir->pair); - if (err) { - return err; - } - } - - // shift over any directories that are affected - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - d->pair[0] = dir->pair[0]; - d->pair[1] = dir->pair[1]; - } - } - - return 0; -} - -static int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, +static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, lfs_dir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; bool relocated = false; - lfs_dir_t tail; // increment revision count dir->rev += 1; @@ -1251,23 +978,9 @@ static int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, .filter.end = end, }; -// // commit regions which can't fend for themselves -// err = lfs_commit_list(lfs, &commit, list); -// if (err) { -// if (err == LFS_ERR_NOSPC) { -// goto split; -// } else if (err == LFS_ERR_CORRUPT) { -// goto relocate; -// } -// return err; -// } -// -// // move over other commits, leaving it up to lfs_commit_move to -// // filter out duplicates, and the commit filtering to reassign ids + // commit with a move for (uint16_t id = begin; id < end; id++) { - err = lfs_commit_move_(lfs, &commit, id, id, source, list); -// err = lfs_commit_commit(lfs, &commit, (lfs_entry_t){ -// lfs_mktag(LFS_FROM_MOVE, id, id), .u.dir=source}); + err = lfs_commit_move(lfs, &commit, id, id, source, list); if (err) { if (err == LFS_ERR_NOSPC) { goto split; @@ -1322,7 +1035,7 @@ static int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, return err; } - err = lfs_dir_compact_(lfs, &tail, list, dir, ack+1, end); + err = lfs_dir_compact(lfs, &tail, list, dir, ack+1, end); if (err) { return err; } @@ -1377,10 +1090,10 @@ static int lfs_dir_compact_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, return 0; } -static int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list) { +static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list) { if (!dir->erased) { // not erased, must compact - return lfs_dir_compact_(lfs, dir, list, dir, 0, dir->count); + return lfs_dir_compact(lfs, dir, list, dir, 0, dir->count); } struct lfs_commit commit = { @@ -1398,50 +1111,7 @@ static int lfs_dir_commit_(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list) { if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { lfs->pcache.block = 0xffffffff; - return lfs_dir_compact_(lfs, dir, list, dir, 0, dir->count); - } - return err; - } - - err = lfs_commit_crc(lfs, &commit); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - lfs->pcache.block = 0xffffffff; - return lfs_dir_compact_(lfs, dir, list, dir, 0, dir->count); - } - return err; - } - - // successful commit, lets update dir - dir->off = commit.off; - dir->etag = commit.ptag; - return 0; -} - -static int lfs_dir_commitwith(lfs_t *lfs, lfs_dir_t *dir, - int (*cb)(lfs_t *lfs, void *data, struct lfs_commit *commit), - void *data) { - if (!dir->erased) { - // not erased, must compact - return lfs_dir_compact(lfs, dir, cb, data, dir, 0, dir->count); - } - - struct lfs_commit commit = { - .block = dir->pair[0], - .begin = dir->off, - .off = dir->off, - .end = lfs->cfg->block_size - 2*sizeof(uint32_t), - .crc = 0xffffffff, - .ptag = dir->etag, - .filter.begin = 0, - .filter.end = 0x1ff, - }; - - int err = cb(lfs, data, &commit); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - lfs->pcache.block = 0xffffffff; - return lfs_dir_compact(lfs, dir, cb, data, dir, 0, dir->count); + return lfs_dir_compact(lfs, dir, list, dir, 0, dir->count); } return err; } @@ -1450,7 +1120,7 @@ static int lfs_dir_commitwith(lfs_t *lfs, lfs_dir_t *dir, if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { lfs->pcache.block = 0xffffffff; - return lfs_dir_compact(lfs, dir, cb, data, dir, 0, dir->count); + return lfs_dir_compact(lfs, dir, list, dir, 0, dir->count); } return err; } @@ -1461,11 +1131,6 @@ static int lfs_dir_commitwith(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, - const lfs_entrylist_t *regions) { - return lfs_dir_commitwith(lfs, dir, lfs_commit_regions, (void*)regions); -} - static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, uint16_t *id) { *id = dir->count; dir->count += 1; @@ -1495,7 +1160,7 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { } } - int err = lfs_dir_commit_(lfs, dir, &(lfs_entrylist_t){ + int err = lfs_dir_commit(lfs, dir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_DELETE, id, 0)}}); if (err) { return err; @@ -1523,6 +1188,8 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { // } // } // } + + return 0; } struct lfs_dir_getter { @@ -1804,835 +1471,6 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } } -//static int lfs_dir_findbuffer(lfs_t *lfs, lfs_dir_t *dir, -// const char **path, lfs_entry_t *entry) { -// void *buffer = entry->u.buffer; -// lfs_size_t size = lfs_tag_size(entry->tag); -// int err = lfs_dir_find(lfs, dir, path, entry); -// if (err) { -// return err; -// } -// -// lfs_size_t diff = lfs_min(size, lfs_tag_size(entry->tag)); -// memset((uint8_t*)buffer + diff, 0, size - diff); -// // TODO hmm -// if (lfs_tag_valid(entry->tag)) { -// memcpy(buffer, entry->u.buffer, diff); -// } else { -// err = lfs_bd_read(lfs, entry->u.d.block, entry->u.d.off, buffer, diff); -// if (err) { -// return err; -// } -// } -// -// if (lfs_tag_size(entry->tag) > size) { -// return LFS_ERR_RANGE; -// } -// -// return 0; -//} -// -//static int lfs_dir_findentry(lfs_t *lfs, lfs_dir_t *dir, -// const char **path, lfs_entry_t *entry) { -// entry->tag = sizeof(entry->u); -// entry->u.buffer = &entry->u; -// return lfs_dir_findbuffer(lfs, dir, path, entry); -//} - - -////////////////////////////////////////////////////////// - -//static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { -// // allocate pair of dir blocks -// for (int i = 0; i < 2; i++) { -// int err = lfs_alloc(lfs, &dir->pair[i]); -// if (err) { -// return err; -// } -// } -// -// // rather than clobbering one of the blocks we just pretend -// // the revision may be valid -// int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->d.rev, 4); -// dir->d.rev = lfs_fromle32(dir->d.rev); -// if (err) { -// return err; -// } -// -// // set defaults -// dir->d.rev += 1; -// dir->d.size = sizeof(dir->d)+4; -// dir->d.tail[0] = 0xffffffff; -// dir->d.tail[1] = 0xffffffff; -// dir->off = sizeof(dir->d); -// -// // don't write out yet, let caller take care of that -// return 0; -//} -// -//static int lfs_dir_fetch(lfs_t *lfs, -// lfs_dir_t *dir, const lfs_block_t pair[2]) { -// // copy out pair, otherwise may be aliasing dir -// const lfs_block_t tpair[2] = {pair[0], pair[1]}; -// bool valid = false; -// -// // check both blocks for the most recent revision -// for (int i = 0; i < 2; i++) { -// struct lfs_disk_dir test; -// int err = lfs_bd_read(lfs, tpair[i], 0, &test, sizeof(test)); -// lfs_dir_fromle32(&test); -// if (err) { -// return err; -// } -// -// if (valid && lfs_scmp(test.rev, dir->d.rev) < 0) { -// continue; -// } -// -// if ((0x7fffffff & test.size) < sizeof(test)+4 || -// (0x7fffffff & test.size) > lfs->cfg->block_size) { -// continue; -// } -// -// uint32_t crc = 0xffffffff; -// lfs_dir_tole32(&test); -// lfs_crc(&crc, &test, sizeof(test)); -// lfs_dir_fromle32(&test); -// err = lfs_bd_crc(lfs, tpair[i], sizeof(test), -// (0x7fffffff & test.size) - sizeof(test), &crc); -// if (err) { -// return err; -// } -// -// if (crc != 0) { -// continue; -// } -// -// valid = true; -// -// // setup dir in case it's valid -// dir->pair[0] = tpair[(i+0) % 2]; -// dir->pair[1] = tpair[(i+1) % 2]; -// dir->off = sizeof(dir->d); -// dir->d = test; -// } -// -// if (!valid) { -// LFS_ERROR("Corrupted dir pair at %d %d", tpair[0], tpair[1]); -// return LFS_ERR_CORRUPT; -// } -// -// return 0; -//} -// -//struct lfs_region { -// enum { -// LFS_FROM_MEM, -// LFS_FROM_REGION, -// LFS_FROM_ATTRS, -// } type; -// -// lfs_off_t oldoff; -// lfs_size_t oldsize; -// const void *buffer; -// lfs_size_t newsize; -//}; -// -//struct lfs_region_attrs { -// const struct lfs_attr *attrs; -// int count; -//}; -// -//struct lfs_region_region { -// lfs_block_t block; -// lfs_off_t off; -// struct lfs_region *regions; -// int count; -//}; -// -//static int lfs_commit_region(lfs_t *lfs, uint32_t *crc, -// lfs_block_t oldblock, lfs_off_t oldoff, -// lfs_block_t newblock, lfs_off_t newoff, -// lfs_off_t regionoff, lfs_size_t regionsize, -// const struct lfs_region *regions, int count) { -// int i = 0; -// lfs_size_t newend = newoff + regionsize; -// while (newoff < newend) { -// // commit from different types of regions -// if (i < count && regions[i].oldoff == oldoff - regionoff) { -// switch (regions[i].type) { -// case LFS_FROM_MEM: { -// lfs_crc(crc, regions[i].buffer, regions[i].newsize); -// int err = lfs_bd_prog(lfs, newblock, newoff, -// regions[i].buffer, regions[i].newsize); -// if (err) { -// return err; -// } -// newoff += regions[i].newsize; -// oldoff += regions[i].oldsize; -// break; -// } -// case LFS_FROM_REGION: { -// const struct lfs_region_region *disk = regions[i].buffer; -// int err = lfs_commit_region(lfs, crc, -// disk->block, disk->off, -// newblock, newoff, -// disk->off, regions[i].newsize, -// disk->regions, disk->count); -// if (err) { -// return err; -// } -// newoff += regions[i].newsize; -// oldoff -= regions[i].oldsize; -// break; -// } -// case LFS_FROM_ATTRS: { -// const struct lfs_region_attrs *attrs = regions[i].buffer; -// -// // order doesn't matter, so we write new attrs first. this -// // is still O(n^2) but only O(n) disk access -// for (int j = 0; j < attrs->count; j++) { -// if (attrs->attrs[j].size == 0) { -// continue; -// } -// -// lfs_entry_attr_t attr; -// attr.d.type = attrs->attrs[j].type; -// attr.d.len = attrs->attrs[j].size; -// -// lfs_crc(crc, &attr.d, sizeof(attr.d)); -// int err = lfs_bd_prog(lfs, newblock, newoff, -// &attr.d, sizeof(attr.d)); -// if (err) { -// return err; -// } -// -// lfs_crc(crc, -// attrs->attrs[j].buffer, attrs->attrs[j].size); -// err = lfs_bd_prog(lfs, newblock, newoff+sizeof(attr.d), -// attrs->attrs[j].buffer, attrs->attrs[j].size); -// if (err) { -// return err; -// } -// -// newoff += 2+attrs->attrs[j].size; -// } -// -// // copy over attributes without updates -// lfs_off_t oldend = oldoff + regions[i].oldsize; -// while (oldoff < oldend) { -// lfs_entry_attr_t attr; -// int err = lfs_bd_read(lfs, oldblock, oldoff, -// &attr.d, sizeof(attr.d)); -// if (err) { -// return err; -// } -// -// bool updating = false; -// for (int j = 0; j < attrs->count; j++) { -// if (attr.d.type == attrs->attrs[j].type) { -// updating = true; -// } -// } -// -// if (!updating) { -// err = lfs_commit_region(lfs, crc, -// oldblock, oldoff, -// newblock, newoff, -// 0, 2+attr.d.len, -// NULL, 0); -// if (err) { -// return err; -// } -// -// newoff += 2+attr.d.len; -// } -// -// oldoff += 2+attr.d.len; -// } -// -// break; -// } -// } -// -// i += 1; -// } else { -// // copy data from old block if not covered by entry -// uint8_t data; -// int err = lfs_bd_read(lfs, oldblock, oldoff, &data, 1); -// if (err) { -// return err; -// } -// -// lfs_crc(crc, &data, 1); -// err = lfs_bd_prog(lfs, newblock, newoff, &data, 1); -// if (err) { -// return err; -// } -// -// oldoff += 1; -// newoff += 1; -// } -// } -// -// // sanity check our commit math -// LFS_ASSERT(newoff == newend); -// return 0; -//} -// -//static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, -// const struct lfs_region *regions, int count) { -// // state for copying over -// const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; -// bool relocated = false; -// -// // increment revision count -// dir->d.rev += 1; -// -// // keep pairs in order such that pair[0] is most recent -// lfs_pairswap(dir->pair); -// for (int i = 0; i < count; i++) { -// dir->d.size += regions[i].newsize; -// dir->d.size -= regions[i].oldsize; -// } -// -// while (true) { -// if (true) { -// int err = lfs_bd_erase(lfs, dir->pair[0]); -// if (err) { -// if (err == LFS_ERR_CORRUPT) { -// goto relocate; -// } -// return err; -// } -// -// // commit header -// uint32_t crc = 0xffffffff; -// lfs_dir_tole32(&dir->d); -// lfs_crc(&crc, &dir->d, sizeof(dir->d)); -// err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); -// lfs_dir_fromle32(&dir->d); -// if (err) { -// if (err == LFS_ERR_CORRUPT) { -// goto relocate; -// } -// return err; -// } -// -// // commit entry -// err = lfs_commit_region(lfs, &crc, -// dir->pair[1], sizeof(dir->d), -// dir->pair[0], sizeof(dir->d), -// 0, (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, -// regions, count); -// if (err) { -// if (err == LFS_ERR_CORRUPT) { -// goto relocate; -// } -// return err; -// } -// -// // commit crc -// crc = lfs_tole32(crc); -// err = lfs_bd_prog(lfs, dir->pair[0], -// (0x7fffffff & dir->d.size)-4, &crc, 4); -// crc = lfs_fromle32(crc); -// if (err) { -// if (err == LFS_ERR_CORRUPT) { -// goto relocate; -// } -// return err; -// } -// -// err = lfs_bd_sync(lfs); -// if (err) { -// if (err == LFS_ERR_CORRUPT) { -// goto relocate; -// } -// return err; -// } -// -// // successful commit, check checksum to make sure -// uint32_t ncrc = 0xffffffff; -// err = lfs_bd_crc(lfs, dir->pair[0], 0, -// (0x7fffffff & dir->d.size)-4, &ncrc); -// if (err) { -// return err; -// } -// -// if (ncrc != crc) { -// goto relocate; -// } -// } -// -// break; -// -//relocate: -// //commit was corrupted -// LFS_DEBUG("Bad block at %d", dir->pair[0]); -// -// // drop caches and prepare to relocate block -// relocated = true; -// lfs->pcache.block = 0xffffffff; -// -// // can't relocate superblock, filesystem is now frozen -// if (lfs_paircmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { -// LFS_WARN("Superblock %d has become unwritable", oldpair[0]); -// return LFS_ERR_CORRUPT; -// } -// -// // relocate half of pair -// int err = lfs_alloc(lfs, &dir->pair[0]); -// if (err) { -// return err; -// } -// } -// -// if (relocated) { -// // update references if we relocated -// LFS_DEBUG("Relocating %d %d to %d %d", -// oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); -// int err = lfs_relocate(lfs, oldpair, dir->pair); -// if (err) { -// return err; -// } -// } -// -// // shift over any directories that are affected -// for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { -// if (lfs_paircmp(d->pair, dir->pair) == 0) { -// d->pair[0] = dir->pair[0]; -// d->pair[1] = dir->pair[1]; -// } -// } -// -// return 0; -//} -// -//static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, -// lfs_off_t off, void *buffer, lfs_size_t size) { -// return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); -//} -// -//static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, -// struct lfs_region *regions, int count) { -// return -999; -//// lfs_ssize_t diff = 0; -//// for (int i = 0; i < count; i++) { -//// diff += regions[i].newsize; -//// diff -= regions[i].oldsize; -//// } -//// -//// lfs_size_t oldsize = entry->size; -//// if (entry->off == 0) { -//// entry->off = (0x7fffffff & dir->d.size) - 4; -//// } -//// -//// if ((0x7fffffff & dir->d.size) + diff > lfs->cfg->block_size) { -//// lfs_dir_t olddir = *dir; -//// lfs_off_t oldoff = entry->off; -//// -//// if (oldsize) { -//// // mark as moving -//// uint8_t type; -//// int err = lfs_dir_get(lfs, &olddir, oldoff, &type, 1); -//// if (err) { -//// return err; -//// } -//// -//// type |= LFS_STRUCT_MOVED; -//// err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ -//// {LFS_FROM_MEM, oldoff, 1, &type, 1}}, 1); -//// if (err) { -//// return err; -//// } -//// } -//// -//// lfs_dir_t pdir = olddir; -//// -//// // find available block or create a new one -//// while ((0x7fffffff & dir->d.size) + oldsize + diff -//// > lfs->cfg->block_size) { -//// // we need to allocate a new dir block -//// if (!(0x80000000 & dir->d.size)) { -//// pdir = *dir; -//// int err = lfs_dir_alloc(lfs, dir); -//// if (err) { -//// return err; -//// } -//// -//// dir->d.tail[0] = pdir.d.tail[0]; -//// dir->d.tail[1] = pdir.d.tail[1]; -//// -//// break; -//// } -//// -//// int err = lfs_dir_fetch(lfs, dir, dir->d.tail); -//// if (err) { -//// return err; -//// } -//// } -//// -//// // writing out new entry -//// entry->off = dir->d.size - 4; -//// entry->size += diff; -//// int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ -//// {LFS_FROM_REGION, entry->off, 0, &(struct lfs_region_region){ -//// olddir.pair[0], oldoff, -//// regions, count}, entry->size}}, 1); -//// if (err) { -//// return err; -//// } -//// -//// // update pred dir, unless pred == old we can coalesce -//// if (!oldsize || lfs_paircmp(pdir.pair, olddir.pair) != 0) { -//// pdir.d.size |= 0x80000000; -//// pdir.d.tail[0] = dir->pair[0]; -//// pdir.d.tail[1] = dir->pair[1]; -//// -//// err = lfs_dir_commit(lfs, &pdir, NULL, 0); -//// if (err) { -//// return err; -//// } -//// } else if (oldsize) { -//// olddir.d.size |= 0x80000000; -//// olddir.d.tail[0] = dir->pair[0]; -//// olddir.d.tail[1] = dir->pair[1]; -//// } -//// -//// // remove old entry -//// if (oldsize) { -//// lfs_entry_t oldentry; -//// oldentry.off = oldoff; -//// err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ -//// {LFS_FROM_MEM, 0, oldsize, NULL, 0}}, 1); -//// if (err) { -//// return err; -//// } -//// } -//// -//// goto shift; -//// } -//// -//// if ((0x7fffffff & dir->d.size) + diff == sizeof(dir->d)+4) { -//// lfs_dir_t pdir; -//// int res = lfs_pred(lfs, dir->pair, &pdir); -//// if (res < 0) { -//// return res; -//// } -//// -//// if (pdir.d.size & 0x80000000) { -//// pdir.d.size &= dir->d.size | 0x7fffffff; -//// pdir.d.tail[0] = dir->d.tail[0]; -//// pdir.d.tail[1] = dir->d.tail[1]; -//// int err = lfs_dir_commit(lfs, &pdir, NULL, 0); -//// if (err) { -//// return err; -//// } -//// goto shift; -//// } -//// } -//// -//// for (int i = 0; i < count; i++) { -//// regions[i].oldoff += entry->off; -//// } -//// -//// int err = lfs_dir_commit(lfs, dir, regions, count); -//// if (err) { -//// return err; -//// } -//// -//// entry->size += diff; -//// -////shift: -//// // shift over any files/directories that are affected -//// for (lfs_file_t *f = lfs->files; f; f = f->next) { -//// if (lfs_paircmp(f->pair, dir->pair) == 0) { -//// if (f->pairoff == entry->off && entry->size == 0) { -//// f->pair[0] = 0xffffffff; -//// f->pair[1] = 0xffffffff; -//// } else if (f->pairoff > entry->off) { -//// f->pairoff += diff; -//// } -//// } -//// } -//// -//// for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { -//// if (lfs_paircmp(d->pair, dir->pair) == 0) { -//// if (d->off > entry->off) { -//// d->off += diff; -//// d->pos += diff; -//// } -//// } -//// } -//// -//// return 0; -//} -// -//static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { -// while (dir->off >= (0x7fffffff & dir->d.size)-4) { -// if (!(0x80000000 & dir->d.size)) { -// entry->off = dir->off; -// return LFS_ERR_NOENT; -// } -// -// int err = lfs_dir_fetch(lfs, dir, dir->d.tail); -// if (err) { -// return err; -// } -// -// dir->off = sizeof(dir->d); -// dir->pos += sizeof(dir->d) + 4; -// } -// -// int err = lfs_dir_get(lfs, dir, dir->off, &entry->d, sizeof(entry->d)); -// lfs_entry_fromle32(&entry->d); -// if (err) { -// return err; -// } -// -// entry->off = dir->off; -// entry->size = lfs_entry_size(entry); -// dir->off += entry->size; -// dir->pos += entry->size; -// return 0; -//} -// -//static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, -// lfs_entry_t *entry, const char **path) { -// const char *pathname = *path; -// lfs_size_t pathlen; -// -// while (true) { -// nextname: -// // skip slashes -// pathname += strspn(pathname, "/"); -// pathlen = strcspn(pathname, "/"); -// -// // special case for root dir -// if (pathname[0] == '\0') { -// *entry = (lfs_entry_t){ -// .d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR, -// .d.u.dir[0] = lfs->root[0], -// .d.u.dir[1] = lfs->root[1], -// }; -// return 0; -// } -// -// // skip '.' and root '..' -// if ((pathlen == 1 && memcmp(pathname, ".", 1) == 0) || -// (pathlen == 2 && memcmp(pathname, "..", 2) == 0)) { -// pathname += pathlen; -// goto nextname; -// } -// -// // skip if matched by '..' in name -// const char *suffix = pathname + pathlen; -// lfs_size_t sufflen; -// int depth = 1; -// while (true) { -// suffix += strspn(suffix, "/"); -// sufflen = strcspn(suffix, "/"); -// if (sufflen == 0) { -// break; -// } -// -// if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { -// depth -= 1; -// if (depth == 0) { -// pathname = suffix + sufflen; -// goto nextname; -// } -// } else { -// depth += 1; -// } -// -// suffix += sufflen; -// } -// -// // update what we've found -// *path = pathname; -// -// // find path -// while (true) { -// int err = lfs_dir_next(lfs, dir, entry); -// if (err) { -// return err; -// } -// -// if (((0xf & entry->d.type) != LFS_TYPE_REG && -// (0xf & entry->d.type) != LFS_TYPE_DIR) || -// entry->d.nlen != pathlen) { -// continue; -// } -// -// int res = lfs_bd_cmp(lfs, dir->pair[0], -// entry->off + entry->size - pathlen, -// pathname, pathlen); -// if (res < 0) { -// return res; -// } -// -// // found match -// if (res) { -// break; -// } -// } -// -// // check that entry has not been moved -// if (entry->d.type & LFS_STRUCT_MOVED) { -// int moved = lfs_moved(lfs, &entry->d.u); -// if (moved < 0 || moved) { -// return (moved < 0) ? moved : LFS_ERR_NOENT; -// } -// -// entry->d.type &= ~LFS_STRUCT_MOVED; -// } -// -// pathname += pathlen; -// pathname += strspn(pathname, "/"); -// if (pathname[0] == '\0') { -// return 0; -// } -// -// // continue on if we hit a directory -// if ((0xf & entry->d.type) != LFS_TYPE_DIR) { -// return LFS_ERR_NOTDIR; -// } -// -// int err = lfs_dir_fetch(lfs, dir, entry->d.u.dir); -// if (err) { -// return err; -// } -// } -//} -// -/// Internal attribute operations /// -//static int lfs_dir_getinfo(lfs_t *lfs, -// lfs_dir_t *dir, const lfs_entry_t *entry, struct lfs_info *info) { -// memset(info, 0, sizeof(*info)); -// info->type = 0xf & entry->d.type; -// if (entry->d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { -// info->size = entry->d.u.file.size; -// } else if (entry->d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { -// info->size = lfs_entry_elen(entry); -// } -// -// if (lfs_paircmp(entry->d.u.dir, lfs->root) == 0) { -// strcpy(info->name, "/"); -// } else { -// int err = lfs_dir_get(lfs, dir, -// entry->off + entry->size - entry->d.nlen, -// info->name, entry->d.nlen); -// if (err) { -// return err; -// } -// } -// -// return 0; -//} -// -//static int lfs_dir_getattrs(lfs_t *lfs, -// lfs_dir_t *dir, const lfs_entry_t *entry, -// const struct lfs_attr *attrs, int count) { -// // set to zero in case we can't find the attributes or size mismatch -// for (int j = 0; j < count; j++) { -// memset(attrs[j].buffer, 0, attrs[j].size); -// } -// -// // search for attribute in attribute entry -// lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); -// lfs_off_t end = off + lfs_entry_alen(entry); -// while (off < end) { -// lfs_entry_attr_t attr; -// int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); -// if (err) { -// return err; -// } -// -// for (int j = 0; j < count; j++) { -// if (attrs[j].type == attr.d.type) { -// if (attrs[j].size < attr.d.len) { -// return LFS_ERR_RANGE; -// } -// -// err = lfs_dir_get(lfs, dir, off+sizeof(attr.d), -// attrs[j].buffer, attr.d.len); -// if (err) { -// return err; -// } -// } -// } -// -// off += 2+attr.d.len; -// } -// -// return 0; -//} -// -//static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, -// lfs_dir_t *dir, lfs_entry_t *entry, -// const struct lfs_attr *attrs, int count) { -// // check that attributes fit -// // two separate passes so disk access is O(n) -// lfs_size_t nsize = 0; -// for (int j = 0; j < count; j++) { -// if (attrs[j].size > 0) { -// nsize += 2+attrs[j].size; -// } -// } -// -// lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); -// lfs_off_t end = off + lfs_entry_alen(entry); -// while (off < end) { -// lfs_entry_attr_t attr; -// int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); -// if (err) { -// return err; -// } -// -// bool updated = false; -// for (int j = 0; j < count; j++) { -// if (attr.d.type == attrs[j].type) { -// updated = true; -// } -// } -// -// if (!updated) { -// nsize += 2+attr.d.len; -// } -// -// off += 2+attr.d.len; -// } -// -// if (nsize > lfs->attrs_size || ( -// lfs_entry_size(entry) - lfs_entry_alen(entry) + nsize -// > lfs->cfg->block_size)) { -// return LFS_ERR_NOSPC; -// } -// -// return nsize; -//} -// -//static int lfs_dir_setattrs(lfs_t *lfs, -// lfs_dir_t *dir, lfs_entry_t *entry, -// const struct lfs_attr *attrs, int count) { -// // make sure attributes fit -// lfs_size_t oldlen = lfs_entry_alen(entry); -// lfs_ssize_t newlen = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); -// if (newlen < 0) { -// return newlen; -// } -// -// // commit to entry, majority of work is in LFS_FROM_ATTRS -// entry->d.alen = (0xc0 & entry->d.alen) | newlen; -// return lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, 4, &entry->d, 4}, -// {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), oldlen, -// &(struct lfs_region_attrs){attrs, count}, newlen}}, 2); -//} -// - /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron @@ -2667,7 +1505,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } - err = lfs_dir_commit_(lfs, &dir, NULL); + err = lfs_dir_commit(lfs, &dir, NULL); if (err) { return err; } @@ -2681,7 +1519,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; - err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_NAME, id, nlen), .u.buffer=(void*)path}, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_DIR | LFS_STRUCT_DIR, id, sizeof(dir.pair)), @@ -2694,72 +1532,6 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return 0; } -//int lfs_mkdir(lfs_t *lfs, const char *path) { -// // deorphan if we haven't yet, needed at most once after poweron -// if (!lfs->deorphaned) { -// int err = lfs_deorphan(lfs); -// if (err) { -// return err; -// } -// } -// -// // fetch parent directory -// lfs_dir_t cwd; -// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); -// if (err) { -// return err; -// } -// -// lfs_entry_t entry; -// err = lfs_dir_find(lfs, &cwd, &entry, &path); -// if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { -// return err ? err : LFS_ERR_EXIST; -// } -// -// // check that name fits -// lfs_size_t nlen = strlen(path); -// if (nlen > lfs->name_size) { -// return LFS_ERR_NAMETOOLONG; -// } -// -// // build up new directory -// lfs_alloc_ack(lfs); -// -// lfs_dir_t dir; -// err = lfs_dir_alloc(lfs, &dir); -// if (err) { -// return err; -// } -// dir.d.tail[0] = cwd.d.tail[0]; -// dir.d.tail[1] = cwd.d.tail[1]; -// -// err = lfs_dir_commit(lfs, &dir, NULL, 0); -// if (err) { -// return err; -// } -// -// entry.d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR; -// entry.d.elen = sizeof(entry.d) - 4; -// entry.d.alen = 0; -// entry.d.nlen = nlen; -// entry.d.u.dir[0] = dir.pair[0]; -// entry.d.u.dir[1] = dir.pair[1]; -// entry.size = 0; -// -// cwd.d.tail[0] = dir.pair[0]; -// cwd.d.tail[1] = dir.pair[1]; -// lfs_entry_tole32(&entry.d); -// err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, 0, &entry.d, sizeof(entry.d)}, -// {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); -// if (err) { -// return err; -// } -// -// lfs_alloc_ack(lfs); -// return 0; -//} - int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { int16_t id; int err = lfs_dir_find(lfs, dir, &path, &id); @@ -2913,151 +1685,6 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { } -//int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { -// dir->pair[0] = lfs->root[0]; -// dir->pair[1] = lfs->root[1]; -// -// int err = lfs_dir_fetch(lfs, dir, dir->pair); -// if (err) { -// return err; -// } -// -// lfs_entry_t entry; -// err = lfs_dir_find(lfs, dir, &entry, &path); -// if (err) { -// return err; -// } else if (entry.d.type != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { -// return LFS_ERR_NOTDIR; -// } -// -// err = lfs_dir_fetch(lfs, dir, entry.d.u.dir); -// if (err) { -// return err; -// } -// -// // setup head dir -// // special offset for '.' and '..' -// dir->head[0] = dir->pair[0]; -// dir->head[1] = dir->pair[1]; -// dir->pos = sizeof(dir->d) - 2; -// dir->off = sizeof(dir->d); -// -// // add to list of directories -// dir->next = lfs->dirs; -// lfs->dirs = dir; -// -// return 0; -//} -// -//int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir) { -// // remove from list of directories -// for (lfs_dir_t **p = &lfs->dirs; *p; p = &(*p)->next) { -// if (*p == dir) { -// *p = dir->next; -// break; -// } -// } -// -// return 0; -//} -// -//int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { -// memset(info, 0, sizeof(*info)); -// -// // special offset for '.' and '..' -// if (dir->pos == sizeof(dir->d) - 2) { -// info->type = LFS_TYPE_DIR; -// strcpy(info->name, "."); -// dir->pos += 1; -// return 1; -// } else if (dir->pos == sizeof(dir->d) - 1) { -// info->type = LFS_TYPE_DIR; -// strcpy(info->name, ".."); -// dir->pos += 1; -// return 1; -// } -// -// lfs_entry_t entry; -// while (true) { -// int err = lfs_dir_next(lfs, dir, &entry); -// if (err) { -// return (err == LFS_ERR_NOENT) ? 0 : err; -// } -// -// if ((0xf & entry.d.type) != LFS_TYPE_REG && -// (0xf & entry.d.type) != LFS_TYPE_DIR) { -// continue; -// } -// -// // check that entry has not been moved -// if (entry.d.type & LFS_STRUCT_MOVED) { -// int moved = lfs_moved(lfs, &entry.d.u); -// if (moved < 0) { -// return moved; -// } -// -// if (moved) { -// continue; -// } -// -// entry.d.type &= ~LFS_STRUCT_MOVED; -// } -// -// break; -// } -// -// int err = lfs_dir_getinfo(lfs, dir, &entry, info); -// if (err) { -// return err; -// } -// -// return 1; -//} -// -//int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { -// // simply walk from head dir -// int err = lfs_dir_rewind(lfs, dir); -// if (err) { -// return err; -// } -// dir->pos = off; -// -// while (off > (0x7fffffff & dir->d.size)) { -// off -= 0x7fffffff & dir->d.size; -// if (!(0x80000000 & dir->d.size)) { -// return LFS_ERR_INVAL; -// } -// -// err = lfs_dir_fetch(lfs, dir, dir->d.tail); -// if (err) { -// return err; -// } -// } -// -// dir->off = off; -// return 0; -//} -// -//lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir) { -// (void)lfs; -// return dir->pos; -//} -// -//int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { -// // reload the head dir -// int err = lfs_dir_fetch(lfs, dir, dir->head); -// if (err) { -// return err; -// } -// -// dir->pair[0] = dir->head[0]; -// dir->pair[1] = dir->head[1]; -// dir->pos = sizeof(dir->d) - 2; -// dir->off = sizeof(dir->d); -// return 0; -//} - - /// File index list operations /// static int lfs_ctz_index(lfs_t *lfs, lfs_off_t *off) { lfs_off_t size = *off; @@ -3283,7 +1910,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change - err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_NAME, id, nlen), .u.buffer=(void*)path}, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0)}}}); @@ -3543,7 +2170,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { - int err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ + int err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_CTZ, file->id, 2*sizeof(uint32_t)), .u.buffer=&file->head}, file->attrs}); @@ -3551,7 +2178,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } } else { - int err = lfs_dir_commit_(lfs, &cwd, &(lfs_entrylist_t){ + int err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, file->id, file->size), .u.buffer=file->cache.buffer}, file->attrs}); @@ -4096,7 +2723,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving //printf("RENAME MOVE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); - err = lfs_dir_commit_(lfs, &oldcwd, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &oldcwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_MOVE, oldid, 0)}}); if (err) { return err; @@ -4111,7 +2738,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // // move to new location // // TODO NAME????? // // TODO HAH, move doesn't want to override things (due -// // to its use in compaction), but that's _exactly_ what we want here +// // to its use in compaction), but that's _exactly what we want here // err = lfs_dir_commitwith(lfs, &newcwd, lfs_commit_move, // &(struct lfs_commit_move){.dir=&oldcwd, .id={oldid, newid}}); // if (err) { @@ -4119,14 +2746,14 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // } // // TODO NONONONONO // // TODO also don't call strlen twice (see prev name check) -// err = lfs_dir_commit_(lfs, &newcwd, &(lfs_entrylist_t){ +// err = lfs_dir_commit(lfs, &newcwd, &(lfs_entrylist_t){ // {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), // .u.buffer=(void*)newpath}}); // if (err) { // return err; // } - err = lfs_dir_commit_(lfs, &newcwd, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &newcwd, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), .u.buffer=(void*)newpath}, &(lfs_entrylist_t){ {lfs_mktag(LFS_FROM_MOVE, newid, oldid), @@ -4316,7 +2943,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_commit_(lfs, &root, NULL); + err = lfs_dir_commit(lfs, &root, NULL); if (err) { return err; } @@ -4341,7 +2968,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; dir.count += 1; - err = lfs_dir_commit_(lfs, &dir, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &dir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, 0, sizeof(superblock)), .u.buffer=&superblock}}); if (err) { @@ -4795,7 +3422,7 @@ static int lfs_relocate(lfs_t *lfs, // update disk, this creates a desync entry.u.pair[0] = newpair[0]; entry.u.pair[1] = newpair[1]; - int err = lfs_dir_commit_(lfs, &parent, &(lfs_entrylist_t){entry}); + int err = lfs_dir_commit(lfs, &parent, &(lfs_entrylist_t){entry}); if (err) { return err; } @@ -4821,7 +3448,7 @@ static int lfs_relocate(lfs_t *lfs, // just replace bad pair, no desync can occur parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; - return lfs_dir_commit_(lfs, &parent, &(lfs_entrylist_t){ + return lfs_dir_commit(lfs, &parent, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL + parent.split*0x10, // TODO hm 0x1ff, sizeof(lfs_block_t[2])), .u.pair[0]=newpair[0], .u.pair[1]=newpair[1]}}); @@ -4864,7 +3491,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit_(lfs, &pdir, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { @@ -4881,7 +3508,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = entry.u.pair[0]; pdir.tail[1] = entry.u.pair[1]; - err = lfs_dir_commit_(lfs, &pdir, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { From eaa9220aad45c6bde17280e22bccc716c6c5c404 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 29 May 2018 00:50:47 -0500 Subject: [PATCH 041/139] Renamed lfs_entry_t -> lfs_mattr_t Attributes are used to describe more than just entries, so calling these list of attributes "entries" was inaccurate. However, the name "attributes" would conflict with "user attributes", user-facing attributes with a very similar purpose. "user attributes" must be kept distinct due to differences in binary layout (internal attributes can use a more compact tag+buffer representation, but expecting users to jump through hoops to get their data to look like that isn't very user-friendly). Decided to go with "mattr" as shorthand for "meta-attributes", similar to "metadata". --- lfs.c | 495 +++++++++++++++++++++++++++++++--------------------------- lfs.h | 22 ++- 2 files changed, 277 insertions(+), 240 deletions(-) diff --git a/lfs.c b/lfs.c index 7c3ef0f6..ca554eb3 100644 --- a/lfs.c +++ b/lfs.c @@ -263,7 +263,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(lfs_t*, void*, lfs_block_t), void *data); static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir); static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], - lfs_dir_t *parent, lfs_entry_t *entry); + lfs_dir_t *parent, lfs_mattr_t *attr); static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid); static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); @@ -379,23 +379,23 @@ static void lfs_alloc_ack(lfs_t *lfs) { //} /// Other struct functions /// -//static inline lfs_size_t lfs_entry_elen(const lfs_entry_t *entry) { -// return (lfs_size_t)(entry->d.elen) | -// ((lfs_size_t)(entry->d.alen & 0xc0) << 2); +//static inline lfs_size_t lfs_entry_elen(const lfs_mattr_t *attr) { +// return (lfs_size_t)(attr->d.elen) | +// ((lfs_size_t)(attr->d.alen & 0xc0) << 2); //} // -//static inline lfs_size_t lfs_entry_alen(const lfs_entry_t *entry) { -// return entry->d.alen & 0x3f; +//static inline lfs_size_t lfs_entry_alen(const lfs_mattr_t *attr) { +// return attr->d.alen & 0x3f; //} // -//static inline lfs_size_t lfs_entry_nlen(const lfs_entry_t *entry) { -// return entry->d.nlen; +//static inline lfs_size_t lfs_entry_nlen(const lfs_mattr_t *attr) { +// return attr->d.nlen; //} // -//static inline lfs_size_t lfs_entry_size(const lfs_entry_t *entry) { -// return 4 + lfs_entry_elen(entry) + -// lfs_entry_alen(entry) + -// lfs_entry_nlen(entry); +//static inline lfs_size_t lfs_entry_size(const lfs_mattr_t *attr) { +// return 4 + lfs_entry_elen(attr) + +// lfs_entry_alen(attr) + +// lfs_entry_nlen(attr); //} @@ -476,37 +476,37 @@ struct lfs_commit { // TODO predelcare static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, - lfs_dir_t *dir, lfs_entrylist_t *list); + lfs_dir_t *dir, lfs_mattrlist_t *list); static int lfs_commit_commit(lfs_t *lfs, - struct lfs_commit *commit, lfs_entry_t entry) { + struct lfs_commit *commit, lfs_mattr_t attr) { // filter out ids - if (lfs_tag_id(entry.tag) != 0x1ff && ( - lfs_tag_id(entry.tag) < commit->filter.begin || - lfs_tag_id(entry.tag) >= commit->filter.end)) { + if (lfs_tag_id(attr.tag) != 0x1ff && ( + lfs_tag_id(attr.tag) < commit->filter.begin || + lfs_tag_id(attr.tag) >= commit->filter.end)) { return 0; } // special cases - if ((lfs_tag_type(entry.tag) & 0x103) == LFS_FROM_MOVE) { + if ((lfs_tag_type(attr.tag) & 0x103) == LFS_FROM_MOVE) { return lfs_commit_move(lfs, commit, - lfs_tag_size(entry.tag), lfs_tag_id(entry.tag), - entry.u.dir, NULL); + lfs_tag_size(attr.tag), lfs_tag_id(attr.tag), + attr.u.dir, NULL); } - uint16_t id = lfs_tag_id(entry.tag) - commit->filter.begin; - entry.tag = lfs_mktag(0, id, 0) | (entry.tag & 0xffe00fff); + uint16_t id = lfs_tag_id(attr.tag) - commit->filter.begin; + attr.tag = lfs_mktag(0, id, 0) | (attr.tag & 0xffe00fff); // check if we fit - lfs_size_t size = lfs_tag_size(entry.tag); + lfs_size_t size = lfs_tag_size(attr.tag); if (commit->off + sizeof(lfs_tag_t)+size > commit->end) { return LFS_ERR_NOSPC; } // write out tag // TODO rm me - //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", entry.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tag_type(entry.tag), lfs_tag_id(entry.tag), lfs_tag_size(entry.tag)); - lfs_tag_t tag = lfs_tole32((entry.tag & 0x7fffffff) ^ commit->ptag); + //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", attr.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tag_type(attr.tag), lfs_tag_id(attr.tag), lfs_tag_size(attr.tag)); + lfs_tag_t tag = lfs_tole32((attr.tag & 0x7fffffff) ^ commit->ptag); lfs_crc(&commit->crc, &tag, sizeof(tag)); int err = lfs_bd_prog(lfs, commit->block, commit->off, &tag, sizeof(tag)); if (err) { @@ -514,11 +514,11 @@ static int lfs_commit_commit(lfs_t *lfs, } commit->off += sizeof(tag); - if (!(entry.tag & 0x80000000)) { + if (!(attr.tag & 0x80000000)) { // from memory - lfs_crc(&commit->crc, entry.u.buffer, size); + lfs_crc(&commit->crc, attr.u.buffer, size); err = lfs_bd_prog(lfs, commit->block, commit->off, - entry.u.buffer, size); + attr.u.buffer, size); if (err) { return err; } @@ -527,7 +527,7 @@ static int lfs_commit_commit(lfs_t *lfs, for (lfs_off_t i = 0; i < size; i++) { uint8_t dat; int err = lfs_bd_read(lfs, - entry.u.d.block, entry.u.d.off+i, &dat, 1); + attr.u.d.block, attr.u.d.off+i, &dat, 1); if (err) { return err; } @@ -540,7 +540,7 @@ static int lfs_commit_commit(lfs_t *lfs, } } commit->off += size; - commit->ptag = entry.tag & 0x7fffffff; // TODO do this once + commit->ptag = attr.tag & 0x7fffffff; // TODO do this once return 0; } @@ -598,7 +598,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { } static int lfs_commit_list(lfs_t *lfs, struct lfs_commit *commit, - lfs_entrylist_t *list) { + lfs_mattrlist_t *list) { for (; list; list = list->next) { int err = lfs_commit_commit(lfs, commit, list->e); if (err) { @@ -625,28 +625,28 @@ struct lfs_commit_move { // TODO redeclare static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), + int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data); static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, - uint32_t mask, lfs_entry_t *entry); + uint32_t mask, lfs_mattr_t *attr); -static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_entry_t entry) { +static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_commit_move *move = p; - if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE && - lfs_tag_id(entry.tag) <= move->id.from) { + if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE && + lfs_tag_id(attr.tag) <= move->id.from) { // something was deleted, we need to move around it move->id.from += 1; return 0; } - if (lfs_tag_type(entry.tag) == LFS_TYPE_MOVE) { + if (lfs_tag_type(attr.tag) == LFS_TYPE_MOVE) { // TODO need this? // ignore moves return 0; } - if (lfs_tag_id(entry.tag) != move->id.from) { + if (lfs_tag_id(attr.tag) != move->id.from) { // ignore non-matching ids return 0; } @@ -658,9 +658,9 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_entry_t entry) { .off=move->commit->off, .etag=move->commit->ptag, .stop_at_commit=true}, - lfs_tag_type(entry.tag) & 0x100 ? 0x7ffff000 : 0x7c1ff000, - &(lfs_entry_t){ - lfs_mktag(lfs_tag_type(entry.tag), + lfs_tag_type(attr.tag) & 0x100 ? 0x7ffff000 : 0x7c1ff000, + &(lfs_mattr_t){ + lfs_mktag(lfs_tag_type(attr.tag), move->id.to - move->commit->filter.begin, 0)}); // TODO can all these filter adjustments be consolidated? if (err && err != LFS_ERR_NOENT) { return err; @@ -672,13 +672,13 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_entry_t entry) { } // update id and commit, as we are currently unique - entry.tag = lfs_mktag(0, move->id.to, 0) | (entry.tag & 0xffe00fff); - return lfs_commit_commit(lfs, move->commit, entry); + attr.tag = lfs_mktag(0, move->id.to, 0) | (attr.tag & 0xffe00fff); + return lfs_commit_commit(lfs, move->commit, attr); } static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, - lfs_dir_t *dir, lfs_entrylist_t *list) { + lfs_dir_t *dir, lfs_mattrlist_t *list) { struct lfs_commit_move move = { .id.from = fromid, .id.to = toid, @@ -733,7 +733,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir, static int lfs_dir_fetchwith(lfs_t *lfs, lfs_dir_t *dir, const lfs_block_t pair[2], - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data) { + int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; dir->stop_at_commit = false; @@ -794,7 +794,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, //printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, temp.pair[0], off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); if (lfs_tag_type(tag) == LFS_TYPE_CRC) { - // check the crc entry + // check the crc attr uint32_t dcrc; int err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); @@ -854,7 +854,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, } if (cb) { - err = cb(lfs, data, (lfs_entry_t){ + err = cb(lfs, data, (lfs_mattr_t){ (tag | 0x80000000), .u.d.block=temp.pair[0], .u.d.off=off+sizeof(tag)}); @@ -884,7 +884,7 @@ static int lfs_dir_fetch(lfs_t *lfs, } static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, - int (*cb)(lfs_t *lfs, void *data, lfs_entry_t entry), void *data) { + int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data) { // iterate over dir block backwards (for faster lookups) lfs_block_t block = dir->pair[0]; lfs_off_t off = dir->off; @@ -899,7 +899,7 @@ static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, break; } - int err = cb(lfs, data, (lfs_entry_t){ + int err = cb(lfs, data, (lfs_mattr_t){ (0x80000000 | tag), .u.d.block=block, .u.d.off=off-lfs_tag_size(tag)}); @@ -922,7 +922,7 @@ static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, +static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, lfs_mattrlist_t *list, lfs_dir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; @@ -996,7 +996,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); if (!lfs_pairisnull(dir->tail)) { // TODO le32 - err = lfs_commit_commit(lfs, &commit, (lfs_entry_t){ + err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ lfs_mktag(LFS_TYPE_SOFTTAIL + dir->split*0x10, 0x1ff, sizeof(dir->tail)), .u.buffer=dir->tail}); @@ -1079,55 +1079,75 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list, } } - // shift over any directories that are affected - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - d->pair[0] = dir->pair[0]; - d->pair[1] = dir->pair[1]; - } - } - return 0; } -static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, lfs_entrylist_t *list) { - if (!dir->erased) { - // not erased, must compact - return lfs_dir_compact(lfs, dir, list, dir, 0, dir->count); - } +static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, lfs_mattrlist_t *list) { + while (true) { + if (!dir->erased) { + // not erased, must compact + goto compact; + } + + struct lfs_commit commit = { + .block = dir->pair[0], + .begin = dir->off, + .off = dir->off, + .end = lfs->cfg->block_size - 2*sizeof(uint32_t), + .crc = 0xffffffff, + .ptag = dir->etag, + .filter.begin = 0, + .filter.end = 0x1ff, + }; + + int err = lfs_commit_list(lfs, &commit, list); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; + } - struct lfs_commit commit = { - .block = dir->pair[0], - .begin = dir->off, - .off = dir->off, - .end = lfs->cfg->block_size - 2*sizeof(uint32_t), - .crc = 0xffffffff, - .ptag = dir->etag, - .filter.begin = 0, - .filter.end = 0x1ff, - }; + err = lfs_commit_crc(lfs, &commit); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; + } - int err = lfs_commit_list(lfs, &commit, list); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - lfs->pcache.block = 0xffffffff; - return lfs_dir_compact(lfs, dir, list, dir, 0, dir->count); + // successful commit, lets update dir + dir->off = commit.off; + dir->etag = commit.ptag; + break; + +compact: + lfs->pcache.block = 0xffffffff; + err = lfs_dir_compact(lfs, dir, list, dir, 0, dir->count); + if (err) { + return err; } - return err; + break; } - err = lfs_commit_crc(lfs, &commit); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - lfs->pcache.block = 0xffffffff; - return lfs_dir_compact(lfs, dir, list, dir, 0, dir->count); + // TODO combine with above? + // update any directories that are affected + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + d->pair[0] = dir->pair[0]; + d->pair[1] = dir->pair[1]; + d->tail[0] = dir->tail[0]; + d->tail[1] = dir->tail[1]; + d->rev = dir->rev; + d->off = dir->off; + d->etag = dir->etag; + d->count = dir->count; + d->erased = dir->erased; + d->split = dir->split; + d->moveid = dir->moveid; // TODO hm this is a bit much } - return err; } - // successful commit, lets update dir - dir->off = commit.off; - dir->etag = commit.ptag; return 0; } @@ -1152,7 +1172,7 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { pdir.split = dir->split; pdir.tail[0] = dir->tail[0]; pdir.tail[1] = dir->tail[1]; - int err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ + int err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL + pdir.split*0x10, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); @@ -1160,7 +1180,7 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { } } - int err = lfs_dir_commit(lfs, dir, &(lfs_entrylist_t){ + int err = lfs_dir_commit(lfs, dir, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_DELETE, id, 0)}}); if (err) { return err; @@ -1168,6 +1188,16 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { // shift over any files that are affected // TODO move this to dir_commit? + // TODO use entries??? + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + if (d->id > id) { + d->id -= 1; + d->pos -= 1; + } + } + } + for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { if (f->id == id) { @@ -1179,33 +1209,23 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { } } -// TODO ? -// for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { -// if (lfs_paircmp(d->pair, dir->pair) == 0) { -// if (d->id > id) { -// d->id -= 1; -// d->pos -= 1; -// } -// } -// } - return 0; } struct lfs_dir_getter { uint32_t mask; lfs_tag_t tag; - lfs_entry_t *entry; + lfs_mattr_t *attr; }; -static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t entry) { +static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_dir_getter *get = p; - if ((entry.tag & get->mask) == (get->tag & get->mask)) { - *get->entry = entry; + if ((attr.tag & get->mask) == (get->tag & get->mask)) { + *get->attr = attr; return true; - } else if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(entry.tag) <= lfs_tag_id(get->tag)) { + } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { + if (lfs_tag_id(attr.tag) <= lfs_tag_id(get->tag)) { get->tag += lfs_mktag(0, 1, 0); } } @@ -1214,10 +1234,10 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_entry_t entry) { } static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, - uint32_t mask, lfs_entry_t *entry) { - uint16_t id = lfs_tag_id(entry->tag); + uint32_t mask, lfs_mattr_t *attr) { + uint16_t id = lfs_tag_id(attr->tag); int res = lfs_dir_traverse(lfs, dir, lfs_dir_getter, - &(struct lfs_dir_getter){mask, entry->tag, entry}); + &(struct lfs_dir_getter){mask, attr->tag, attr}); if (res < 0) { return res; } @@ -1239,27 +1259,27 @@ static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, } } - entry->tag = lfs_mktag(0, id, 0) | (entry->tag & 0xffe00fff); + attr->tag = lfs_mktag(0, id, 0) | (attr->tag & 0xffe00fff); return 0; } static int lfs_dir_getbuffer(lfs_t *lfs, lfs_dir_t *dir, - uint32_t mask, lfs_entry_t *entry) { - void *buffer = entry->u.buffer; - lfs_size_t size = lfs_tag_size(entry->tag); - int err = lfs_dir_get(lfs, dir, mask, entry); + uint32_t mask, lfs_mattr_t *attr) { + void *buffer = attr->u.buffer; + lfs_size_t size = lfs_tag_size(attr->tag); + int err = lfs_dir_get(lfs, dir, mask, attr); if (err) { return err; } - lfs_size_t diff = lfs_min(size, lfs_tag_size(entry->tag)); + lfs_size_t diff = lfs_min(size, lfs_tag_size(attr->tag)); memset((uint8_t*)buffer + diff, 0, size - diff); - err = lfs_bd_read(lfs, entry->u.d.block, entry->u.d.off, buffer, diff); + err = lfs_bd_read(lfs, attr->u.d.block, attr->u.d.off, buffer, diff); if (err) { return err; } - if (lfs_tag_size(entry->tag) > size) { + if (lfs_tag_size(attr->tag) > size) { return LFS_ERR_RANGE; } @@ -1267,10 +1287,10 @@ static int lfs_dir_getbuffer(lfs_t *lfs, lfs_dir_t *dir, } static int lfs_dir_getentry(lfs_t *lfs, lfs_dir_t *dir, - uint32_t mask, lfs_tag_t tag, lfs_entry_t *entry) { - entry->tag = tag | sizeof(entry->u); - entry->u.buffer = &entry->u; - int err = lfs_dir_getbuffer(lfs, dir, mask, entry); + uint32_t mask, lfs_tag_t tag, lfs_mattr_t *attr) { + attr->tag = tag | sizeof(attr->u); + attr->u.buffer = &attr->u; + int err = lfs_dir_getbuffer(lfs, dir, mask, attr); if (err && err != LFS_ERR_RANGE) { return err; } @@ -1298,21 +1318,21 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_dir_t *dir, } } - lfs_entry_t entry; + lfs_mattr_t attr; int err = lfs_dir_getentry(lfs, dir, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, id, 0), &entry); + lfs_mktag(LFS_TYPE_REG, id, 0), &attr); if (err) { return err; } - info->type = lfs_tag_subtype(entry.tag); - if (lfs_tag_type(entry.tag) == (LFS_TYPE_REG | LFS_STRUCT_CTZ)) { - info->size = entry.u.ctz.size; - } else if (lfs_tag_type(entry.tag) == (LFS_TYPE_REG | LFS_STRUCT_INLINE)) { - info->size = lfs_tag_size(entry.tag); + info->type = lfs_tag_subtype(attr.tag); + if (lfs_tag_type(attr.tag) == (LFS_TYPE_REG | LFS_STRUCT_CTZ)) { + info->size = attr.u.ctz.size; + } else if (lfs_tag_type(attr.tag) == (LFS_TYPE_REG | LFS_STRUCT_INLINE)) { + info->size = lfs_tag_size(attr.tag); } - err = lfs_dir_getbuffer(lfs, dir, 0x7ffff000, &(lfs_entry_t){ + err = lfs_dir_getbuffer(lfs, dir, 0x7ffff000, &(lfs_mattr_t){ lfs_mktag(LFS_TYPE_NAME, id, lfs->name_size+1), .u.buffer=info->name}); if (err) { @@ -1328,12 +1348,12 @@ struct lfs_dir_finder { int16_t id; }; -static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t entry) { +static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_dir_finder *find = p; - if (lfs_tag_type(entry.tag) == LFS_TYPE_NAME && - lfs_tag_size(entry.tag) == find->len) { - int res = lfs_bd_cmp(lfs, entry.u.d.block, entry.u.d.off, + if (lfs_tag_type(attr.tag) == LFS_TYPE_NAME && + lfs_tag_size(attr.tag) == find->len) { + int res = lfs_bd_cmp(lfs, attr.u.d.block, attr.u.d.off, find->name, find->len); if (res < 0) { return res; @@ -1341,12 +1361,12 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t entry) { if (res) { // found a match - find->id = lfs_tag_id(entry.tag); + find->id = lfs_tag_id(attr.tag); } - } else if (lfs_tag_type(entry.tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(entry.tag) == find->id) { + } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { + if (lfs_tag_id(attr.tag) == find->id) { find->id = -1; - } else if (lfs_tag_id(entry.tag) < find->id) { + } else if (lfs_tag_id(attr.tag) < find->id) { find->id -= 1; } } @@ -1357,7 +1377,7 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_entry_t entry) { // TODO drop others, make this only return id, also make get take in only entry to populate (with embedded tag) static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, const char **path, int16_t *id) { - lfs_entry_t entry = { + lfs_mattr_t attr = { .u.pair[0] = lfs->root[0], .u.pair[1] = lfs->root[1], }; @@ -1415,9 +1435,9 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, // find path while (true) { - //printf("checking %d %d for %s\n", entry.u.pair[0], entry.u.pair[1], *path); + //printf("checking %d %d for %s\n", attr.u.pair[0], attr.u.pair[1], *path); find.id = -1; - int err = lfs_dir_fetchwith(lfs, dir, entry.u.pair, + int err = lfs_dir_fetchwith(lfs, dir, attr.u.pair, lfs_dir_finder, &find); if (err) { return err; @@ -1432,8 +1452,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, return LFS_ERR_NOENT; } - entry.u.pair[0] = dir->tail[0]; - entry.u.pair[1] = dir->tail[1]; + attr.u.pair[0] = dir->tail[0]; + attr.u.pair[1] = dir->tail[1]; } if (find.id == dir->moveid) { @@ -1458,14 +1478,14 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, // TODO would this mean more code? // grab the entry data int err = lfs_dir_getentry(lfs, dir, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, find.id, 0), &entry); + lfs_mktag(LFS_TYPE_REG, find.id, 0), &attr); if (err) { return err; } // continue on if we hit a directory // TODO update with what's on master? - if (lfs_tag_subtype(entry.tag) != LFS_TYPE_DIR) { + if (lfs_tag_subtype(attr.tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } } @@ -1519,11 +1539,11 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; - err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_NAME, id, nlen), - .u.buffer=(void*)path}, &(lfs_entrylist_t){ + .u.buffer=(void*)path}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_DIR | LFS_STRUCT_DIR, id, sizeof(dir.pair)), - .u.buffer=dir.pair}, &(lfs_entrylist_t){ + .u.buffer=dir.pair}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(cwd.tail)), .u.buffer=cwd.tail}}}}); @@ -1539,35 +1559,35 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { return err; } - lfs_entry_t entry; + lfs_mattr_t attr; if (id < 0) { // handle root dir separately - entry.u.pair[0] = lfs->root[0]; - entry.u.pair[1] = lfs->root[1]; + attr.u.pair[0] = lfs->root[0]; + attr.u.pair[1] = lfs->root[1]; } else { // get dir pair from parent err = lfs_dir_getentry(lfs, dir, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, id, 0), &entry); + lfs_mktag(LFS_TYPE_REG, id, 0), &attr); if (err) { return err; } - if (lfs_tag_subtype(entry.tag) != LFS_TYPE_DIR) { + if (lfs_tag_subtype(attr.tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } } // fetch first pair - err = lfs_dir_fetch(lfs, dir, entry.u.pair); + err = lfs_dir_fetch(lfs, dir, attr.u.pair); if (err) { return err; } - // setup head dir + // setup entry dir->head[0] = dir->pair[0]; dir->head[1] = dir->pair[1]; - dir->pos = 0; dir->id = 0; + dir->pos = 0; // add to list of directories dir->next = lfs->dirs; @@ -1679,8 +1699,8 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { dir->pair[0] = dir->head[0]; dir->pair[1] = dir->head[1]; - dir->pos = 0; dir->id = 0; + dir->pos = 0; return 0; } @@ -1891,7 +1911,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return err; } - lfs_entry_t entry; + lfs_mattr_t attr; if (err == LFS_ERR_NOENT) { if (!(flags & LFS_O_CREAT)) { return LFS_ERR_NOENT; @@ -1910,9 +1930,9 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change - err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_NAME, id, nlen), - .u.buffer=(void*)path}, &(lfs_entrylist_t){ + .u.buffer=(void*)path}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0)}}}); if (err) { return err; @@ -1926,7 +1946,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, cwd.pair[1] = cwd.tail[1]; } - entry.tag = lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0); + attr.tag = lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0); } else { if (id == -1) { return LFS_ERR_ISDIR; @@ -1934,13 +1954,13 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_EXIST; } - entry.tag = lfs_mktag(LFS_TYPE_REG, id, 0); - err = lfs_dir_get(lfs, &cwd, 0x701ff000, &entry); + attr.tag = lfs_mktag(LFS_TYPE_REG, id, 0); + err = lfs_dir_get(lfs, &cwd, 0x701ff000, &attr); if (err) { return err; } - if (lfs_tag_subtype(entry.tag) != LFS_TYPE_REG) { + if (lfs_tag_subtype(attr.tag) != LFS_TYPE_REG) { return LFS_ERR_ISDIR; } } @@ -1969,16 +1989,16 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } - if (lfs_tag_struct(entry.tag) == LFS_STRUCT_INLINE) { + if (lfs_tag_struct(attr.tag) == LFS_STRUCT_INLINE) { // load inline files file->head = 0xfffffffe; - file->size = lfs_tag_size(entry.tag); + file->size = lfs_tag_size(attr.tag); file->flags |= LFS_F_INLINE; file->cache.block = file->head; file->cache.off = 0; // don't always read (may be new file) if (file->size > 0) { - err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, + err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, file->cache.buffer, file->size); if (err) { lfs_free(file->cache.buffer); @@ -1987,7 +2007,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } else { // use ctz list from entry - err = lfs_bd_read(lfs, entry.u.d.block, entry.u.d.off, + err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, &file->head, 2*sizeof(uint32_t)); } @@ -2170,7 +2190,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { - int err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + int err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_CTZ, file->id, 2*sizeof(uint32_t)), .u.buffer=&file->head}, file->attrs}); @@ -2178,7 +2198,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } } else { - int err = lfs_dir_commit(lfs, &cwd, &(lfs_entrylist_t){ + int err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, file->id, file->size), .u.buffer=file->cache.buffer}, file->attrs}); @@ -2493,7 +2513,7 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { // return err; // } // -// lfs_entry_t entry = {.off = file->pairoff}; +// lfs_mattr_t entry = {.off = file->pairoff}; // err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); // if (err) { // return err; @@ -2538,7 +2558,7 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { // return err; // } // -// lfs_entry_t entry = {.off = file->pairoff}; +// lfs_mattr_t entry = {.off = file->pairoff}; // err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); // if (err) { // return err; @@ -2594,17 +2614,17 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // grab entry to see if we're dealing with a dir - lfs_entry_t entry; + lfs_mattr_t attr; err = lfs_dir_getentry(lfs, &cwd, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, id, 0), &entry); + lfs_mktag(LFS_TYPE_REG, id, 0), &attr); if (err) { return err; } - if (lfs_tag_subtype(entry.tag) == LFS_TYPE_DIR) { + if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { lfs_dir_t dir; // must be empty before removal - err = lfs_dir_fetch(lfs, &dir, entry.u.pair); + err = lfs_dir_fetch(lfs, &dir, attr.u.pair); if (err) { return err; } @@ -2622,14 +2642,14 @@ int lfs_remove(lfs_t *lfs, const char *path) { // if we were a directory, find pred, replace tail // TODO can this just deorphan? - if (lfs_tag_subtype(entry.tag) == LFS_TYPE_DIR) { + if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { err = lfs_deorphan(lfs); if (err) { return err; } } -// if (lfs_tag_subtype(entry.tag) == LFS_TYPE_DIR) { +// if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { // int res = lfs_pred(lfs, dir.pair, &cwd); // if (res < 0) { // return res; @@ -2665,9 +2685,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - lfs_entry_t oldentry; + // add to list so we catch updates if directories overlap + lfs_mattr_t oldattr; err = lfs_dir_getentry(lfs, &oldcwd, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, oldid, 0), &oldentry); + lfs_mktag(LFS_TYPE_REG, oldid, 0), &oldattr); if (err) { return err; } @@ -2682,23 +2703,23 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { bool prevexists = (err != LFS_ERR_NOENT); bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); - lfs_entry_t preventry; + lfs_mattr_t prevattr; if (prevexists) { // get prev entry, check that we have same type err = lfs_dir_getentry(lfs, &newcwd, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, newid, 0), &preventry); + lfs_mktag(LFS_TYPE_REG, newid, 0), &prevattr); if (err) { return err; } - if (lfs_tag_subtype(preventry.tag) != lfs_tag_subtype(oldentry.tag)) { + if (lfs_tag_subtype(prevattr.tag) != lfs_tag_subtype(oldattr.tag)) { return LFS_ERR_ISDIR; } - if (lfs_tag_subtype(preventry.tag) == LFS_TYPE_DIR) { + if (lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { lfs_dir_t prevdir; // must be empty before removal - err = lfs_dir_fetch(lfs, &prevdir, preventry.u.pair); + err = lfs_dir_fetch(lfs, &prevdir, prevattr.u.pair); if (err) { return err; } @@ -2723,7 +2744,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving //printf("RENAME MOVE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); - err = lfs_dir_commit(lfs, &oldcwd, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &oldcwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_MOVE, oldid, 0)}}); if (err) { return err; @@ -2746,16 +2767,16 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // } // // TODO NONONONONO // // TODO also don't call strlen twice (see prev name check) -// err = lfs_dir_commit(lfs, &newcwd, &(lfs_entrylist_t){ +// err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ // {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), // .u.buffer=(void*)newpath}}); // if (err) { // return err; // } - err = lfs_dir_commit(lfs, &newcwd, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), - .u.buffer=(void*)newpath}, &(lfs_entrylist_t){ + .u.buffer=(void*)newpath}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_FROM_MOVE, newid, oldid), .u.dir=&oldcwd}}}); if (err) { @@ -2776,7 +2797,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // if we were a directory, find pred, replace tail // TODO can this just deorphan? - if (prevexists && lfs_tag_subtype(preventry.tag) == LFS_TYPE_DIR) { + if (prevexists && lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { err = lfs_deorphan(lfs); if (err) { return err; @@ -2794,7 +2815,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // return err; // } // -// lfs_entry_t entry; +// lfs_mattr_t entry; // err = lfs_dir_find(lfs, &cwd, &entry, &path); // if (err) { // return err; @@ -2811,7 +2832,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // return err; // } // -// lfs_entry_t entry; +// lfs_mattr_t entry; // err = lfs_dir_find(lfs, &cwd, &entry, &path); // if (err) { // return err; @@ -2968,7 +2989,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; dir.count += 1; - err = lfs_dir_commit(lfs, &dir, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &dir, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, 0, sizeof(superblock)), .u.buffer=&superblock}}); if (err) { @@ -3007,7 +3028,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_superblock_t superblock; - err = lfs_dir_getbuffer(lfs, &dir, 0x7ffff000, &(lfs_entry_t){ + err = lfs_dir_getbuffer(lfs, &dir, 0x7ffff000, &(lfs_mattr_t){ lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, 0, sizeof(superblock)), .u.buffer=&superblock}); @@ -3093,9 +3114,9 @@ int lfs_fs_traverse(lfs_t *lfs, } for (uint16_t id = 0; id < dir.count; id++) { - lfs_entry_t entry; + lfs_mattr_t attr; int err = lfs_dir_getentry(lfs, &dir, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, id, 0), &entry); + lfs_mktag(LFS_TYPE_REG, id, 0), &attr); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -3103,9 +3124,9 @@ int lfs_fs_traverse(lfs_t *lfs, return err; } - if (lfs_tag_struct(entry.tag) == LFS_STRUCT_CTZ) { + if (lfs_tag_struct(attr.tag) == LFS_STRUCT_CTZ) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, - entry.u.ctz.head, entry.u.ctz.size, cb, data); + attr.u.ctz.head, attr.u.ctz.size, cb, data); if (err) { return err; } @@ -3158,7 +3179,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } // iterate over contents - lfs_entry_t entry; + lfs_mattr_t entry; while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { err = lfs_dir_get(lfs, &dir, dir.off, &entry.d, sizeof(entry.d)); @@ -3251,7 +3272,7 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir) { } */ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], - lfs_dir_t *parent, lfs_entry_t *entry) { + lfs_dir_t *parent, lfs_mattr_t *attr) { // iterate over all directory directory entries parent->tail[0] = 0; parent->tail[1] = 1; @@ -3264,7 +3285,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], // TODO make this O(n) by using fetchwith to match the pointers for (uint16_t id = 0; id < parent->count; id++) { int err = lfs_dir_getentry(lfs, parent, 0x43dff000, - lfs_mktag(LFS_STRUCT_DIR, id, 0), entry); + lfs_mktag(LFS_STRUCT_DIR, id, 0), attr); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -3272,7 +3293,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], return err; } - if (lfs_paircmp(entry->u.pair, pair) == 0) { + if (lfs_paircmp(attr->u.pair, pair) == 0) { return true; } } @@ -3282,7 +3303,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], } /* static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], - lfs_dir_t *parent, lfs_entry_t *entry) { + lfs_dir_t *parent, lfs_mattr_t *attr) { if (lfs_pairisnull(lfs->root)) { return 0; } @@ -3298,7 +3319,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], } while (true) { - err = lfs_dir_next(lfs, parent, entry); + err = lfs_dir_next(lfs, parent, attr); if (err && err != LFS_ERR_NOENT) { return err; } @@ -3307,8 +3328,8 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], break; } - if (((0x70 & entry->d.type) == LFS_STRUCT_DIR) && - lfs_paircmp(entry->d.u.dir, dir) == 0) { + if (((0x70 & attr->d.type) == LFS_STRUCT_DIR) && + lfs_paircmp(attr->d.u.dir, dir) == 0) { return true; } } @@ -3320,7 +3341,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid) { // grab entry pair we're looking for fromdir->moveid = -1; - lfs_entry_t fromentry; + lfs_mattr_t fromentry; // TODO what about inline files? int err = lfs_dir_getentry(lfs, fromdir, 0x401ff000, lfs_mktag(LFS_TYPE_REG, fromid, 0), &fromentry); @@ -3349,7 +3370,7 @@ static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid) { continue; } - lfs_entry_t toentry; + lfs_mattr_t toentry; int err = lfs_dir_getentry(lfs, &todir, 0x43dff000, lfs_mktag(LFS_STRUCT_DIR, toid, 0), &toentry); if (err) { @@ -3381,7 +3402,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) { } // iterate over all directory directory entries - lfs_entry_t entry; + lfs_mattr_t entry; while (!lfs_pairisnull(cwd.d.tail)) { err = lfs_dir_fetch(lfs, &cwd, cwd.d.tail); if (err) { @@ -3408,21 +3429,23 @@ static int lfs_moved(lfs_t *lfs, const void *e) { return false; } */ + +// TODO rename to lfs_dir_relocate? static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]) { // find parent lfs_dir_t parent; - lfs_entry_t entry; - int res = lfs_parent(lfs, oldpair, &parent, &entry); + lfs_mattr_t attr; + int res = lfs_parent(lfs, oldpair, &parent, &attr); if (res < 0) { return res; } if (res) { // update disk, this creates a desync - entry.u.pair[0] = newpair[0]; - entry.u.pair[1] = newpair[1]; - int err = lfs_dir_commit(lfs, &parent, &(lfs_entrylist_t){entry}); + attr.u.pair[0] = newpair[0]; + attr.u.pair[1] = newpair[1]; + int err = lfs_dir_commit(lfs, &parent, &(lfs_mattrlist_t){attr}); if (err) { return err; } @@ -3434,6 +3457,8 @@ static int lfs_relocate(lfs_t *lfs, lfs->root[1] = newpair[1]; } + // TODO update dir list!!? + // clean up bad block, which should now be a desync return lfs_deorphan(lfs); } @@ -3448,10 +3473,24 @@ static int lfs_relocate(lfs_t *lfs, // just replace bad pair, no desync can occur parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; - return lfs_dir_commit(lfs, &parent, &(lfs_entrylist_t){ + int err = lfs_dir_commit(lfs, &parent, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL + parent.split*0x10, // TODO hm 0x1ff, sizeof(lfs_block_t[2])), .u.pair[0]=newpair[0], .u.pair[1]=newpair[1]}}); + if (err) { + return err; + } + } + + // shift over any dirs/files that are affected + for (int i = 0; i < 2; i++) { + for (lfs_dir_t *d = ((void*[2]){lfs->dirs, lfs->files})[i]; + d; d = d->next) { + if (lfs_paircmp(d->pair, oldpair) == 0) { + d->pair[0] = newpair[0]; + d->pair[1] = newpair[1]; + } + } } // couldn't find dir, must be new @@ -3478,8 +3517,8 @@ int lfs_deorphan(lfs_t *lfs) { if (!pdir.split) { // check if we have a parent lfs_dir_t parent; - lfs_entry_t entry; - int res = lfs_parent(lfs, pdir.tail, &parent, &entry); + lfs_mattr_t attr; + int res = lfs_parent(lfs, pdir.tail, &parent, &attr); if (res < 0) { return res; } @@ -3491,7 +3530,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ + err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { @@ -3501,14 +3540,14 @@ int lfs_deorphan(lfs_t *lfs) { break; } - if (!lfs_pairsync(entry.u.pair, pdir.tail)) { + if (!lfs_pairsync(attr.u.pair, pdir.tail)) { // we have desynced LFS_DEBUG("Found desync %d %d", - entry.u.pair[0], entry.u.pair[1]); + attr.u.pair[0], attr.u.pair[1]); - pdir.tail[0] = entry.u.pair[0]; - pdir.tail[1] = entry.u.pair[1]; - err = lfs_dir_commit(lfs, &pdir, &(lfs_entrylist_t){ + pdir.tail[0] = attr.u.pair[0]; + pdir.tail[1] = attr.u.pair[1]; + err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); if (err) { @@ -3576,7 +3615,7 @@ int lfs_deorphan(lfs_t *lfs) { if (!(0x80000000 & pdir.d.size)) { // check if we have a parent lfs_dir_t parent; - lfs_entry_t entry; + lfs_mattr_t entry; int res = lfs_parent(lfs, pdir.d.tail, &parent, &entry); if (res < 0) { return res; @@ -3616,7 +3655,7 @@ int lfs_deorphan(lfs_t *lfs) { } // check entries for moves - lfs_entry_t entry; + lfs_mattr_t entry; while (true) { err = lfs_dir_next(lfs, &cwd, &entry); if (err && err != LFS_ERR_NOENT) { @@ -3670,7 +3709,7 @@ int lfs_deorphan(lfs_t *lfs) { // return err; // } // -// lfs_entry_t entry = {.off = sizeof(dir.d)}; +// lfs_mattr_t entry = {.off = sizeof(dir.d)}; // err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); // if (err) { // return err; @@ -3690,7 +3729,7 @@ int lfs_deorphan(lfs_t *lfs) { // return err; // } // -// lfs_entry_t entry; +// lfs_mattr_t entry; // err = lfs_dir_find(lfs, &cwd, &entry, &path); // if (err) { // return err; @@ -3707,7 +3746,7 @@ int lfs_deorphan(lfs_t *lfs) { // return err; // } // -// lfs_entry_t entry = {.off = sizeof(dir.d)}; +// lfs_mattr_t entry = {.off = sizeof(dir.d)}; // err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); // if (err) { // return err; diff --git a/lfs.h b/lfs.h index b4bed046..62aaeb44 100644 --- a/lfs.h +++ b/lfs.h @@ -263,7 +263,7 @@ struct lfs_attr { /// littlefs data structures /// typedef uint32_t lfs_tag_t; -typedef struct lfs_entry { +typedef struct lfs_mattr { lfs_tag_t tag; union { void *buffer; @@ -278,12 +278,12 @@ typedef struct lfs_entry { } d; struct lfs_dir *dir; } u; -} lfs_entry_t; +} lfs_mattr_t; -typedef struct lfs_entrylist { - lfs_entry_t e; - struct lfs_entrylist *next; -} lfs_entrylist_t; +typedef struct lfs_mattrlist { + lfs_mattr_t e; + struct lfs_mattrlist *next; +} lfs_mattrlist_t; typedef struct lfs_cache { lfs_block_t block; @@ -295,7 +295,6 @@ typedef struct lfs_file { struct lfs_file *next; lfs_block_t pair[2]; uint16_t id; - lfs_block_t head; lfs_size_t size; @@ -305,12 +304,15 @@ typedef struct lfs_file { lfs_off_t off; lfs_cache_t cache; - lfs_entrylist_t *attrs; + lfs_mattrlist_t *attrs; } lfs_file_t; typedef struct lfs_dir { struct lfs_dir *next; lfs_block_t pair[2]; + uint16_t id; + uint16_t pos; + lfs_block_t head[2]; lfs_block_t tail[2]; uint32_t rev; @@ -321,10 +323,6 @@ typedef struct lfs_dir { bool split; bool stop_at_commit; // TODO hmmm int16_t moveid; - - uint16_t id; - lfs_block_t head[2]; - lfs_off_t pos; } lfs_dir_t; typedef struct lfs_superblock { From f458da4b7c7d87066c930bee5dc61c5af5751141 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 29 May 2018 01:11:26 -0500 Subject: [PATCH 042/139] Added the internal meta-directory structure Similarly to the internal "meta-attributes", I was finding quite a bit of need for an internal structure that mirrors the user-facing directory structure for when I need to do an operation on a metadata-pair, but don't need all of the state associated with a fully iterable directory chain. lfs_mdir_t - meta-directory, describes a single metadata-pair lfs_dir_t - directory, describes an iterable directory chain While it may seem complex to have all these structures lying around, they only complicate the code at compile time. To the machine, any number of nested structures all looks the same. --- lfs.c | 184 +++++++++++++++++++++++++++------------------------------- lfs.h | 18 +++--- 2 files changed, 98 insertions(+), 104 deletions(-) diff --git a/lfs.c b/lfs.c index ca554eb3..ca557b53 100644 --- a/lfs.c +++ b/lfs.c @@ -261,10 +261,10 @@ static int lfs_bd_sync(lfs_t *lfs) { /// Internal operations predeclared here /// int lfs_fs_traverse(lfs_t *lfs, int (*cb)(lfs_t*, void*, lfs_block_t), void *data); -static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir); +static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], - lfs_dir_t *parent, lfs_mattr_t *attr); -static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid); + lfs_mdir_t *parent, lfs_mattr_t *attr); +static int lfs_moved(lfs_t *lfs, lfs_mdir_t *fromdir, uint16_t fromid); static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); int lfs_deorphan(lfs_t *lfs); @@ -339,7 +339,7 @@ static void lfs_alloc_ack(lfs_t *lfs) { // d->tail[1] = lfs_fromle32(d->tail[1]); //} // -//static void lfs_dir_tole32(struct lfs_disk_dir *d) { +//static void lfs_mdir_tole32(struct lfs_disk_dir *d) { // d->rev = lfs_tole32(d->rev); // d->size = lfs_tole32(d->size); // d->tail[0] = lfs_tole32(d->tail[0]); @@ -476,7 +476,7 @@ struct lfs_commit { // TODO predelcare static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, - lfs_dir_t *dir, lfs_mattrlist_t *list); + lfs_mdir_t *dir, lfs_mattrlist_t *list); static int lfs_commit_commit(lfs_t *lfs, struct lfs_commit *commit, lfs_mattr_t attr) { @@ -613,7 +613,7 @@ static int lfs_commit_list(lfs_t *lfs, struct lfs_commit *commit, // committer for moves // TODO rename? struct lfs_commit_move { - lfs_dir_t *dir; // TODO need dir? + lfs_mdir_t *dir; // TODO need dir? struct { uint16_t from; uint16_t to; @@ -624,10 +624,10 @@ struct lfs_commit_move { // TODO redeclare -static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_mdir_traverse(lfs_t *lfs, lfs_mdir_t *dir, int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data); -static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t mask, lfs_mattr_t *attr); static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { @@ -653,7 +653,7 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { // check if type has already been committed int err = lfs_dir_get(lfs, - &(lfs_dir_t){ + &(lfs_mdir_t){ .pair[0]=move->commit->block, .off=move->commit->off, .etag=move->commit->ptag, @@ -678,7 +678,7 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, - lfs_dir_t *dir, lfs_mattrlist_t *list) { + lfs_mdir_t *dir, lfs_mattrlist_t *list) { struct lfs_commit_move move = { .id.from = fromid, .id.to = toid, @@ -692,7 +692,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, } } - int err = lfs_dir_traverse(lfs, dir, lfs_commit_movescan, &move); + int err = lfs_mdir_traverse(lfs, dir, lfs_commit_movescan, &move); if (err) { return err; } @@ -700,7 +700,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, bool split, const lfs_block_t tail[2]) { // allocate pair of dir blocks (backwards, so we write to block 1 first) for (int i = 0; i < 2; i++) { @@ -732,7 +732,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir, } static int lfs_dir_fetchwith(lfs_t *lfs, - lfs_dir_t *dir, const lfs_block_t pair[2], + lfs_mdir_t *dir, const lfs_block_t pair[2], int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; @@ -768,7 +768,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); dir->rev = lfs_fromle32(dir->rev); - lfs_dir_t temp = *dir; + lfs_mdir_t temp = *dir; while (true) { // extract next tag @@ -879,11 +879,11 @@ static int lfs_dir_fetchwith(lfs_t *lfs, } static int lfs_dir_fetch(lfs_t *lfs, - lfs_dir_t *dir, const lfs_block_t pair[2]) { + lfs_mdir_t *dir, const lfs_block_t pair[2]) { return lfs_dir_fetchwith(lfs, dir, pair, NULL, NULL); } -static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_mdir_traverse(lfs_t *lfs, lfs_mdir_t *dir, int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data) { // iterate over dir block backwards (for faster lookups) lfs_block_t block = dir->pair[0]; @@ -922,8 +922,8 @@ static int lfs_dir_traverse(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, lfs_mattrlist_t *list, - lfs_dir_t *source, uint16_t begin, uint16_t end) { +static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, + lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; bool relocated = false; @@ -1029,7 +1029,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, lfs_mattrlist_t *list, // drop caches and create tail lfs->pcache.block = 0xffffffff; - lfs_dir_t tail; + lfs_mdir_t tail; int err = lfs_dir_alloc(lfs, &tail, dir->split, dir->tail); if (err) { return err; @@ -1082,7 +1082,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_dir_t *dir, lfs_mattrlist_t *list, return 0; } -static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, lfs_mattrlist_t *list) { +static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { while (true) { if (!dir->erased) { // not erased, must compact @@ -1133,36 +1133,26 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, lfs_mattrlist_t *list) { // TODO combine with above? // update any directories that are affected for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - d->pair[0] = dir->pair[0]; - d->pair[1] = dir->pair[1]; - d->tail[0] = dir->tail[0]; - d->tail[1] = dir->tail[1]; - d->rev = dir->rev; - d->off = dir->off; - d->etag = dir->etag; - d->count = dir->count; - d->erased = dir->erased; - d->split = dir->split; - d->moveid = dir->moveid; // TODO hm this is a bit much + if (lfs_paircmp(d->m.pair, dir->pair) == 0) { + d->m = *dir; } } return 0; } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, uint16_t *id) { +static int lfs_dir_append(lfs_t *lfs, lfs_mdir_t *dir, uint16_t *id) { *id = dir->count; dir->count += 1; return 0; } -static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { +static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { dir->count -= 1; // check if we should drop the directory block if (dir->count == 0) { - lfs_dir_t pdir; + lfs_mdir_t pdir; int res = lfs_pred(lfs, dir->pair, &pdir); if (res < 0) { return res; @@ -1190,7 +1180,7 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_dir_t *dir, uint16_t id) { // TODO move this to dir_commit? // TODO use entries??? for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { + if (lfs_paircmp(d->m.pair, dir->pair) == 0) { if (d->id > id) { d->id -= 1; d->pos -= 1; @@ -1233,10 +1223,10 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_mattr_t attr) { return false; } -static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t mask, lfs_mattr_t *attr) { uint16_t id = lfs_tag_id(attr->tag); - int res = lfs_dir_traverse(lfs, dir, lfs_dir_getter, + int res = lfs_mdir_traverse(lfs, dir, lfs_dir_getter, &(struct lfs_dir_getter){mask, attr->tag, attr}); if (res < 0) { return res; @@ -1263,7 +1253,7 @@ static int lfs_dir_get(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_getbuffer(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_getbuffer(lfs_t *lfs, lfs_mdir_t *dir, uint32_t mask, lfs_mattr_t *attr) { void *buffer = attr->u.buffer; lfs_size_t size = lfs_tag_size(attr->tag); @@ -1286,7 +1276,7 @@ static int lfs_dir_getbuffer(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_getentry(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_getentry(lfs_t *lfs, lfs_mdir_t *dir, uint32_t mask, lfs_tag_t tag, lfs_mattr_t *attr) { attr->tag = tag | sizeof(attr->u); attr->u.buffer = &attr->u; @@ -1298,7 +1288,7 @@ static int lfs_dir_getentry(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_getinfo(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { if (id < 0) { // special case for root @@ -1375,7 +1365,7 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_mattr_t attr) { } // TODO drop others, make this only return id, also make get take in only entry to populate (with embedded tag) -static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, const char **path, int16_t *id) { lfs_mattr_t attr = { .u.pair[0] = lfs->root[0], @@ -1501,7 +1491,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } } - lfs_dir_t cwd; + lfs_mdir_t cwd; int err = lfs_dir_find(lfs, &cwd, &path, &(int16_t){0}); if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { if (!err) { @@ -1519,7 +1509,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // build up new directory lfs_alloc_ack(lfs); - lfs_dir_t dir; + lfs_mdir_t dir; err = lfs_dir_alloc(lfs, &dir, false, cwd.tail); if (err) { return err; @@ -1554,7 +1544,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { int16_t id; - int err = lfs_dir_find(lfs, dir, &path, &id); + int err = lfs_dir_find(lfs, &dir->m, &path, &id); if (err) { return err; } @@ -1566,7 +1556,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { attr.u.pair[1] = lfs->root[1]; } else { // get dir pair from parent - err = lfs_dir_getentry(lfs, dir, 0x701ff000, + err = lfs_dir_getentry(lfs, &dir->m, 0x701ff000, lfs_mktag(LFS_TYPE_REG, id, 0), &attr); if (err) { return err; @@ -1578,14 +1568,14 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { } // fetch first pair - err = lfs_dir_fetch(lfs, dir, attr.u.pair); + err = lfs_dir_fetch(lfs, &dir->m, attr.u.pair); if (err) { return err; } // setup entry - dir->head[0] = dir->pair[0]; - dir->head[1] = dir->pair[1]; + dir->head[0] = dir->m.pair[0]; + dir->head[1] = dir->m.pair[1]; dir->id = 0; dir->pos = 0; @@ -1625,12 +1615,12 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { } while (true) { - if (dir->id == dir->count) { - if (!dir->split) { + if (dir->id == dir->m.count) { + if (!dir->m.split) { return false; } - int err = lfs_dir_fetch(lfs, dir, dir->tail); + int err = lfs_dir_fetch(lfs, &dir->m, dir->m.tail); if (err) { return err; } @@ -1638,7 +1628,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { dir->id = 0; } - int err = lfs_dir_getinfo(lfs, dir, dir->id, info); + int err = lfs_dir_getinfo(lfs, &dir->m, dir->id, info); if (err && err != LFS_ERR_NOENT) { return err; } @@ -1666,16 +1656,16 @@ int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { off -= dir->pos; while (off != 0) { - dir->id = lfs_min(dir->count, off); + dir->id = lfs_min(dir->m.count, off); dir->pos += dir->id; off -= dir->id; - if (dir->id == dir->count) { - if (!dir->split) { + if (dir->id == dir->m.count) { + if (!dir->m.split) { return LFS_ERR_INVAL; } - int err = lfs_dir_fetch(lfs, dir, dir->tail); + int err = lfs_dir_fetch(lfs, &dir->m, dir->m.tail); if (err) { return err; } @@ -1692,13 +1682,13 @@ lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir) { int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { // reload the head dir - int err = lfs_dir_fetch(lfs, dir, dir->head); + int err = lfs_dir_fetch(lfs, &dir->m, dir->head); if (err) { return err; } - dir->pair[0] = dir->head[0]; - dir->pair[1] = dir->head[1]; + dir->m.pair[0] = dir->head[0]; + dir->m.pair[1] = dir->head[1]; dir->id = 0; dir->pos = 0; return 0; @@ -1904,7 +1894,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // allocate entry for file if it doesn't exist - lfs_dir_t cwd; + lfs_mdir_t cwd; int16_t id; int err = lfs_dir_find(lfs, &cwd, &path, &id); if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { @@ -2182,7 +2172,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // update dir entry // TODO keep list of dirs including these guys for no // need of another reload? - lfs_dir_t cwd; + lfs_mdir_t cwd; err = lfs_dir_fetch(lfs, &cwd, file->pair); if (err) { return err; @@ -2507,7 +2497,7 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { // // // load from disk if we haven't already been deleted // if (!lfs_pairisnull(file->pair)) { -// lfs_dir_t cwd; +// lfs_mdir_t cwd; // int err = lfs_dir_fetch(lfs, &cwd, file->pair); // if (err) { // return err; @@ -2552,7 +2542,7 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { // // // at least make sure attributes fit // if (!lfs_pairisnull(file->pair)) { -// lfs_dir_t cwd; +// lfs_mdir_t cwd; // int err = lfs_dir_fetch(lfs, &cwd, file->pair); // if (err) { // return err; @@ -2582,7 +2572,7 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { - lfs_dir_t cwd; + lfs_mdir_t cwd; int16_t id; int err = lfs_dir_find(lfs, &cwd, &path, &id); if (err) { @@ -2601,7 +2591,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } } - lfs_dir_t cwd; + lfs_mdir_t cwd; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); if (err) { return err; @@ -2622,7 +2612,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { - lfs_dir_t dir; + lfs_mdir_t dir; // must be empty before removal err = lfs_dir_fetch(lfs, &dir, attr.u.pair); if (err) { @@ -2678,7 +2668,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // find old entry - lfs_dir_t oldcwd; + lfs_mdir_t oldcwd; int16_t oldid; int err = lfs_dir_find(lfs, &oldcwd, &oldpath, &oldid); if (err) { @@ -2694,7 +2684,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // find new entry - lfs_dir_t newcwd; + lfs_mdir_t newcwd; int16_t newid; err = lfs_dir_find(lfs, &newcwd, &newpath, &newid); if (err && err != LFS_ERR_NOENT) { @@ -2717,7 +2707,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } if (lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { - lfs_dir_t prevdir; + lfs_mdir_t prevdir; // must be empty before removal err = lfs_dir_fetch(lfs, &prevdir, prevattr.u.pair); if (err) { @@ -2809,7 +2799,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { //int lfs_getattrs(lfs_t *lfs, const char *path, // const struct lfs_attr *attrs, int count) { -// lfs_dir_t cwd; +// lfs_mdir_t cwd; // int err = lfs_dir_fetch(lfs, &cwd, lfs->root); // if (err) { // return err; @@ -2826,7 +2816,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // //int lfs_setattrs(lfs_t *lfs, const char *path, // const struct lfs_attr *attrs, int count) { -// lfs_dir_t cwd; +// lfs_mdir_t cwd; // int err = lfs_dir_fetch(lfs, &cwd, lfs->root); // if (err) { // return err; @@ -2949,7 +2939,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_alloc_ack(lfs); // create superblock dir - lfs_dir_t dir; + lfs_mdir_t dir; err = lfs_dir_alloc(lfs, &dir, false, (const lfs_block_t[2]){0xffffffff, 0xffffffff}); if (err) { @@ -2957,7 +2947,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { } // write root directory - lfs_dir_t root; + lfs_mdir_t root; err = lfs_dir_alloc(lfs, &root, false, (const lfs_block_t[2]){0xffffffff, 0xffffffff}); if (err) { @@ -3018,7 +3008,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_alloc_ack(lfs); // load superblock - lfs_dir_t dir; + lfs_mdir_t dir; err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -3098,7 +3088,7 @@ int lfs_fs_traverse(lfs_t *lfs, } // iterate over metadata pairs - lfs_dir_t dir = {.tail = {0, 1}}; + lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pairisnull(dir.tail)) { for (int i = 0; i < 2; i++) { int err = cb(lfs, data, dir.tail[i]); @@ -3172,7 +3162,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } } - lfs_dir_t dir; + lfs_mdir_t dir; int err = lfs_dir_fetch(lfs, &dir, cwd); if (err) { return err; @@ -3228,7 +3218,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { return 0; } */ -static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_dir_t *pdir) { +static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { // iterate over all directory directory entries pdir->tail[0] = 0; pdir->tail[1] = 1; @@ -3246,7 +3236,7 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_dir_t *pdir) { return false; } /* -static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir) { +static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir) { if (lfs_pairisnull(lfs->root)) { return 0; } @@ -3272,7 +3262,7 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_dir_t *pdir) { } */ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], - lfs_dir_t *parent, lfs_mattr_t *attr) { + lfs_mdir_t *parent, lfs_mattr_t *attr) { // iterate over all directory directory entries parent->tail[0] = 0; parent->tail[1] = 1; @@ -3303,7 +3293,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], } /* static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], - lfs_dir_t *parent, lfs_mattr_t *attr) { + lfs_mdir_t *parent, lfs_mattr_t *attr) { if (lfs_pairisnull(lfs->root)) { return 0; } @@ -3338,7 +3328,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], return false; } */ -static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid) { +static int lfs_moved(lfs_t *lfs, lfs_mdir_t *fromdir, uint16_t fromid) { // grab entry pair we're looking for fromdir->moveid = -1; lfs_mattr_t fromentry; @@ -3351,7 +3341,7 @@ static int lfs_moved(lfs_t *lfs, lfs_dir_t *fromdir, uint16_t fromid) { } // skip superblock - lfs_dir_t todir; + lfs_mdir_t todir; err = lfs_dir_fetch(lfs, &todir, (const lfs_block_t[2]){0, 1}); if (err) { return err; @@ -3395,7 +3385,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) { } // skip superblock - lfs_dir_t cwd; + lfs_mdir_t cwd; int err = lfs_dir_fetch(lfs, &cwd, (const lfs_block_t[2]){0, 1}); if (err) { return err; @@ -3434,7 +3424,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) { static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]) { // find parent - lfs_dir_t parent; + lfs_mdir_t parent; lfs_mattr_t attr; int res = lfs_parent(lfs, oldpair, &parent, &attr); if (res < 0) { @@ -3486,9 +3476,9 @@ static int lfs_relocate(lfs_t *lfs, for (int i = 0; i < 2; i++) { for (lfs_dir_t *d = ((void*[2]){lfs->dirs, lfs->files})[i]; d; d = d->next) { - if (lfs_paircmp(d->pair, oldpair) == 0) { - d->pair[0] = newpair[0]; - d->pair[1] = newpair[1]; + if (lfs_paircmp(d->m.pair, oldpair) == 0) { + d->m.pair[0] = newpair[0]; + d->m.pair[1] = newpair[1]; } } } @@ -3503,8 +3493,8 @@ int lfs_deorphan(lfs_t *lfs) { return 0; } - lfs_dir_t pdir = {.split = true}; - lfs_dir_t dir = {.tail = {0, 1}}; + lfs_mdir_t pdir = {.split = true}; + lfs_mdir_t dir = {.tail = {0, 1}}; // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { @@ -3516,7 +3506,7 @@ int lfs_deorphan(lfs_t *lfs) { // check head blocks for orphans if (!pdir.split) { // check if we have a parent - lfs_dir_t parent; + lfs_mdir_t parent; lfs_mattr_t attr; int res = lfs_parent(lfs, pdir.tail, &parent, &attr); if (res < 0) { @@ -3601,8 +3591,8 @@ int lfs_deorphan(lfs_t *lfs) { return 0; } - lfs_dir_t pdir = {.d.size = 0x80000000}; - lfs_dir_t cwd = {.d.tail[0] = 0, .d.tail[1] = 1}; + lfs_mdir_t pdir = {.d.size = 0x80000000}; + lfs_mdir_t cwd = {.d.tail[0] = 0, .d.tail[1] = 1}; // iterate over all directory directory entries while (!lfs_pairisnull(cwd.d.tail)) { @@ -3614,7 +3604,7 @@ int lfs_deorphan(lfs_t *lfs) { // check head blocks for orphans if (!(0x80000000 & pdir.d.size)) { // check if we have a parent - lfs_dir_t parent; + lfs_mdir_t parent; lfs_mattr_t entry; int res = lfs_parent(lfs, pdir.d.tail, &parent, &entry); if (res < 0) { @@ -3703,7 +3693,7 @@ int lfs_deorphan(lfs_t *lfs) { /// External filesystem filesystem operations /// //int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { -// lfs_dir_t dir; +// lfs_mdir_t dir; // int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); // if (err) { // return err; @@ -3723,7 +3713,7 @@ int lfs_deorphan(lfs_t *lfs) { // return err; // } // -// lfs_dir_t cwd; +// lfs_mdir_t cwd; // int err = lfs_dir_fetch(lfs, &cwd, lfs->root); // if (err) { // return err; @@ -3740,7 +3730,7 @@ int lfs_deorphan(lfs_t *lfs) { //} // //int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { -// lfs_dir_t dir; +// lfs_mdir_t dir; // int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); // if (err) { // return err; diff --git a/lfs.h b/lfs.h index 62aaeb44..59304848 100644 --- a/lfs.h +++ b/lfs.h @@ -276,7 +276,7 @@ typedef struct lfs_mattr { lfs_block_t block; lfs_off_t off; } d; - struct lfs_dir *dir; + struct lfs_mdir *dir; } u; } lfs_mattr_t; @@ -307,13 +307,8 @@ typedef struct lfs_file { lfs_mattrlist_t *attrs; } lfs_file_t; -typedef struct lfs_dir { - struct lfs_dir *next; +typedef struct lfs_mdir { lfs_block_t pair[2]; - uint16_t id; - uint16_t pos; - lfs_block_t head[2]; - lfs_block_t tail[2]; uint32_t rev; lfs_off_t off; @@ -323,6 +318,15 @@ typedef struct lfs_dir { bool split; bool stop_at_commit; // TODO hmmm int16_t moveid; +} lfs_mdir_t; + +typedef struct lfs_dir { + struct lfs_dir *next; + struct lfs_mdir m; + + lfs_block_t head[2]; + uint16_t id; + lfs_off_t pos; } lfs_dir_t; typedef struct lfs_superblock { From 116c1e76deb28c30715de63d313500053ba4fffa Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 29 May 2018 01:21:55 -0500 Subject: [PATCH 043/139] Adopted EISDIR as internal error for root path as argument Unfortunately, it's hard to make directory lookups for root not a special case, and we don't like special cases when we're trying to keep code size small. Since there are a handful of code paths where opening root should return EISDIR (such as lfs_file_open("/")), using EISDIR to note that the argument is in fact a path to the root. This is needed because we no longer look up an entries contents in lfs_dir_find for free, since entries are more ephemeral. --- lfs.c | 73 +++++++++++++++++++++++++++++------------------------------ 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/lfs.c b/lfs.c index ca557b53..2cf951d7 100644 --- a/lfs.c +++ b/lfs.c @@ -1290,13 +1290,6 @@ static int lfs_dir_getentry(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { - if (id < 0) { - // special case for root - strcpy(info->name, "/"); - info->type = LFS_TYPE_DIR; - return 0; - } - if (id == dir->moveid) { int moved = lfs_moved(lfs, dir, dir->moveid); if (moved < 0) { @@ -1366,7 +1359,7 @@ static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_mattr_t attr) { // TODO drop others, make this only return id, also make get take in only entry to populate (with embedded tag) static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, - const char **path, int16_t *id) { + const char **path, uint16_t *id) { lfs_mattr_t attr = { .u.pair[0] = lfs->root[0], .u.pair[1] = lfs->root[1], @@ -1384,9 +1377,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, // special case for root dir if (find.name[0] == '\0') { - // TODO set up root? - *id = -1; - return 0; + // Return ISDIR when we hit root + return LFS_ERR_ISDIR; } // skip '.' and root '..' @@ -1492,9 +1484,9 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } lfs_mdir_t cwd; - int err = lfs_dir_find(lfs, &cwd, &path, &(int16_t){0}); + int err = lfs_dir_find(lfs, &cwd, &path, &(uint16_t){0}); if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { - if (!err) { + if (!err || err == LFS_ERR_ISDIR) { return LFS_ERR_EXIST; } return err; @@ -1543,14 +1535,14 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { - int16_t id; + uint16_t id; int err = lfs_dir_find(lfs, &dir->m, &path, &id); - if (err) { + if (err && err != LFS_ERR_ISDIR) { return err; } lfs_mattr_t attr; - if (id < 0) { + if (err == LFS_ERR_ISDIR) { // handle root dir separately attr.u.pair[0] = lfs->root[0]; attr.u.pair[1] = lfs->root[1]; @@ -1895,7 +1887,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // allocate entry for file if it doesn't exist lfs_mdir_t cwd; - int16_t id; + uint16_t id; int err = lfs_dir_find(lfs, &cwd, &path, &id); if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { return err; @@ -2573,12 +2565,19 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { lfs_mdir_t cwd; - int16_t id; + uint16_t id; int err = lfs_dir_find(lfs, &cwd, &path, &id); - if (err) { + if (err && err != LFS_ERR_ISDIR) { return err; } + if (err == LFS_ERR_ISDIR) { + // special case for root + strcpy(info->name, "/"); + info->type = LFS_TYPE_DIR; + return 0; + } + return lfs_dir_getinfo(lfs, &cwd, id, info); } @@ -2597,7 +2596,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } - int16_t id; + uint16_t id; err = lfs_dir_find(lfs, &cwd, &path, &id); if (err) { return err; @@ -2669,7 +2668,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find old entry lfs_mdir_t oldcwd; - int16_t oldid; + uint16_t oldid; int err = lfs_dir_find(lfs, &oldcwd, &oldpath, &oldid); if (err) { return err; @@ -2685,7 +2684,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find new entry lfs_mdir_t newcwd; - int16_t newid; + uint16_t newid; err = lfs_dir_find(lfs, &newcwd, &newpath, &newid); if (err && err != LFS_ERR_NOENT) { return err; @@ -3746,18 +3745,18 @@ int lfs_deorphan(lfs_t *lfs) { // return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); //} -static int lfs_fs_size_count(void *p, lfs_block_t block) { - lfs_size_t *size = p; - *size += 1; - return 0; -} - -lfs_ssize_t lfs_fs_size(lfs_t *lfs) { - lfs_size_t size = 0; - int err = lfs_fs_traverse(lfs, lfs_fs_size_count, &size); - if (err) { - return err; - } - - return size; -} +//static int lfs_fs_size_count(void *p, lfs_block_t block) { +// lfs_size_t *size = p; +// *size += 1; +// return 0; +//} +// +//lfs_ssize_t lfs_fs_size(lfs_t *lfs) { +// lfs_size_t size = 0; +// int err = lfs_fs_traverse(lfs, lfs_fs_size_count, &size); +// if (err) { +// return err; +// } +// +// return size; +//} From e39f7e99d1714d8299157e3773d265657f5fc889 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 29 May 2018 12:35:23 -0500 Subject: [PATCH 044/139] Introduced xored-globals logic to fix fundamental problem with moves This was a big roadblock for a while: with the new feature of inlined files, the existing move logic was fundamentally flawed. To pull off atomic moves between two different metadata-pairs, littlefs uses a simple, if a bit clumsy trick. 1. Marks entry as "moving" 2. Copies entry to new metadata-pair 3. Deletes old entry If power is lost before the move operation is completed, we will find the "moving" tag. This means there may or may not be an incomplete move on the filesystem. In this case, we simply search for the moved entry, if we find it, we remove the old entry, otherwise we just remove the "moving" tag. This worked perfectly, until we introduced inlined files. See, unlike the existing directory and ctz entries, inlined files have no guarantee they are unique. There is nothing we can search for that will allow us to find a moved file unless we assign entries globally-unique ids. (note that moves are fundamentally rename operations, so searching for names does not make sense). --- Solving this problem required completely restructuring how littlefs handled moves and pulled out a really old idea that had been left in the cutting room floor back when littlefs was going through many designs: xored-globals. The problem xored-globals solves is the need to maintain some global state via commits to these distributed, independent metadata-pairs. The idea is that we can use some sort of symmetric operation, such as xor, to introduces deltas of the global state that can be committed atomically along with any other info to these metadata-pairs. This means that to figure out our global state, we xor together the global delta stored in every metadata-pair. Which means any commit can update the global state atomically, opening up a whole new set atomic possibilities. There is a couple of downsides. These globals may end up with deltas on every single metadata-pair, effectively duplicating the data for each block. Additionally, these globals need to have multiple copies in RAM. This means and globals need to be a bounded size and very small, since even small globals will have a large footprint. --- On top of xored-globals, it's trivial to fix our move logic. Here we've added an indirect delete tag which allows us to atomically specify a delete of any entry on the filesystem. Our move operation is now: 1. Copy entry to new metadata-pair and atomically xor globals to indirectly delete our original entry. 2. Delete the original entry and xor globals to remove the indirect delete. Extra exciting is that this now takes our relatively clumsy move operation into a sexy guaranteed O(1) move operation with no searching necessary (though we do need to xor globals during mount). Also reintroduced entry struct, now with a specific purpose to describe the metadata-pair + id combo needed by indirect deletes to locate an entry. --- lfs.c | 206 ++++++++++++++++++++++++++++++++++++++++++++++------------ lfs.h | 39 ++++++----- 2 files changed, 190 insertions(+), 55 deletions(-) diff --git a/lfs.c b/lfs.c index 2cf951d7..efd8e83f 100644 --- a/lfs.c +++ b/lfs.c @@ -624,7 +624,7 @@ struct lfs_commit_move { // TODO redeclare -static int lfs_mdir_traverse(lfs_t *lfs, lfs_mdir_t *dir, +static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data); static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, @@ -692,7 +692,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, } } - int err = lfs_mdir_traverse(lfs, dir, lfs_commit_movescan, &move); + int err = lfs_dir_traverse(lfs, dir, lfs_commit_movescan, &move); if (err) { return err; } @@ -726,6 +726,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, dir->tail[1] = tail[1]; dir->erased = false; dir->split = split; + dir->idelete = (lfs_entry_t){{0, 0}, 0}; // don't write out yet, let caller take care of that return 0; @@ -762,6 +763,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, dir->tail[1] = 0xffffffff; dir->count = 0; dir->split = false; + dir->idelete = (lfs_entry_t){{0, 0}, 0}; dir->moveid = -1; dir->rev = lfs_tole32(rev[0]); @@ -783,6 +785,14 @@ static int lfs_dir_fetchwith(lfs_t *lfs, // next commit not yet programmed if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_valid(tag)) { + if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0 && cb) { + int err = cb(lfs, data, (lfs_mattr_t){ + lfs_mktag(LFS_TYPE_DELETE, lfs->idelete.id, 0)}); + if (err) { + return err; + } + } + dir->erased = true; return 0; } @@ -807,6 +817,17 @@ static int lfs_dir_fetchwith(lfs_t *lfs, // try other block break; } else { + // TODO combine with above? + if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0 + && cb) { + int err = cb(lfs, data, (lfs_mattr_t){ + lfs_mktag(LFS_TYPE_DELETE, + lfs->idelete.id, 0)}); + if (err) { + return err; + } + } + // consider what we have good enough dir->erased = false; return 0; @@ -818,6 +839,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, crc = 0xffffffff; *dir = temp; } else { + // TODO crc before callback??? err = lfs_bd_crc(lfs, temp.pair[0], off+sizeof(tag), lfs_tag_size(tag), &crc); if (err) { @@ -832,6 +854,12 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (err) { return err; } + } else if (lfs_tag_type(tag) == LFS_TYPE_IDELETE) { + err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), + &temp.idelete, sizeof(temp.idelete)); + if (err) { + return err; + } } else if (lfs_tag_type(tag) == LFS_TYPE_MOVE) { // TODO handle moves correctly? temp.moveid = lfs_tag_id(tag); @@ -883,35 +911,45 @@ static int lfs_dir_fetch(lfs_t *lfs, return lfs_dir_fetchwith(lfs, dir, pair, NULL, NULL); } -static int lfs_mdir_traverse(lfs_t *lfs, lfs_mdir_t *dir, +static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data) { // iterate over dir block backwards (for faster lookups) lfs_block_t block = dir->pair[0]; lfs_off_t off = dir->off; lfs_tag_t tag = dir->etag; + if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0) { + int err = cb(lfs, data, (lfs_mattr_t){ + lfs_mktag(LFS_TYPE_DELETE, lfs->idelete.id, 0)}); + if (err) { + return err; + } + } + while (off != sizeof(uint32_t)) { // TODO rm me //printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, block, off-lfs_tag_size(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); // TODO hmm - if (dir->stop_at_commit && lfs_tag_type(tag) == LFS_TYPE_CRC) { - break; - } - - int err = cb(lfs, data, (lfs_mattr_t){ - (0x80000000 | tag), - .u.d.block=block, - .u.d.off=off-lfs_tag_size(tag)}); - if (err) { - return err; + if (lfs_tag_type(tag) == LFS_TYPE_CRC) { + if (dir->stop_at_commit) { + break; + } + } else { + int err = cb(lfs, data, (lfs_mattr_t){ + (0x80000000 | tag), + .u.d.block=block, + .u.d.off=off-lfs_tag_size(tag)}); + if (err) { + return err; + } } LFS_ASSERT(off > sizeof(tag)+lfs_tag_size(tag)); off -= sizeof(tag)+lfs_tag_size(tag); lfs_tag_t ntag; - err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); + int err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); if (err) { return err; } @@ -963,7 +1001,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, .block = dir->pair[1], .off = sizeof(dir->rev), - // space is complicated, we need room for tail, crc, + // space is complicated, we need room for tail, crc, idelete, // and we keep cap at around half a block .begin = 0, .end = lfs_min( @@ -1008,6 +1046,22 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, } } + if (!(dir->idelete.pair[0] == 0 && + dir->idelete.pair[0] == 0 && + dir->idelete.id == 0)) { + // TODO le32 + err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ + lfs_mktag(LFS_TYPE_IDELETE, + 0x1ff, sizeof(dir->idelete)), + .u.buffer=&dir->idelete}); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + err = lfs_commit_crc(lfs, &commit); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1130,7 +1184,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { break; } - // TODO combine with above? // update any directories that are affected for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->m.pair, dir->pair) == 0) { @@ -1176,9 +1229,7 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { return err; } - // shift over any files that are affected - // TODO move this to dir_commit? - // TODO use entries??? + // shift over any dirs/files that are affected for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->m.pair, dir->pair) == 0) { if (d->id > id) { @@ -1202,14 +1253,14 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { return 0; } -struct lfs_dir_getter { +struct lfs_dir_get { uint32_t mask; lfs_tag_t tag; lfs_mattr_t *attr; }; -static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_mattr_t attr) { - struct lfs_dir_getter *get = p; +static int lfs_dir_getscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { + struct lfs_dir_get *get = p; if ((attr.tag & get->mask) == (get->tag & get->mask)) { *get->attr = attr; @@ -1226,8 +1277,8 @@ static int lfs_dir_getter(lfs_t *lfs, void *p, lfs_mattr_t attr) { static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t mask, lfs_mattr_t *attr) { uint16_t id = lfs_tag_id(attr->tag); - int res = lfs_mdir_traverse(lfs, dir, lfs_dir_getter, - &(struct lfs_dir_getter){mask, attr->tag, attr}); + int res = lfs_dir_traverse(lfs, dir, lfs_dir_getscan, + &(struct lfs_dir_get){mask, attr->tag, attr}); if (res < 0) { return res; } @@ -1325,14 +1376,14 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, return 0; } -struct lfs_dir_finder { +struct lfs_dir_find { const char *name; uint16_t len; int16_t id; }; -static int lfs_dir_finder(lfs_t *lfs, void *p, lfs_mattr_t attr) { - struct lfs_dir_finder *find = p; +static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { + struct lfs_dir_find *find = p; if (lfs_tag_type(attr.tag) == LFS_TYPE_NAME && lfs_tag_size(attr.tag) == find->len) { @@ -1365,7 +1416,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, .u.pair[1] = lfs->root[1], }; - struct lfs_dir_finder find = { + struct lfs_dir_find find = { .name = *path, }; @@ -1420,7 +1471,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, //printf("checking %d %d for %s\n", attr.u.pair[0], attr.u.pair[1], *path); find.id = -1; int err = lfs_dir_fetchwith(lfs, dir, attr.u.pair, - lfs_dir_finder, &find); + lfs_dir_findscan, &find); if (err) { return err; } @@ -2903,6 +2954,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->files = NULL; lfs->dirs = NULL; lfs->deorphaned = false; + lfs->idelete = (lfs_entry_t){{0xffffffff, 0xffffffff}, 0xffff}; return 0; } @@ -3260,36 +3312,100 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir) { return false; } */ + + +struct lfs_dir_parentscan { + lfs_block_t pair[2]; + int16_t id; +}; + +static int lfs_parentscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { + struct lfs_dir_parentscan *parentscan = p; + + if ((lfs_tag_type(attr.tag) & 0x10f) == LFS_STRUCT_DIR) { + int err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, + &attr.u, sizeof(attr.u)); + if (err) { + return err; + } + + if (lfs_paircmp(attr.u.pair, parentscan->pair) == 0) { + // found a match + parentscan->id = lfs_tag_id(attr.tag); + } + } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { + if (lfs_tag_id(attr.tag) == parentscan->id) { + parentscan->id = -1; + } else if (lfs_tag_id(attr.tag) < parentscan->id) { + parentscan->id -= 1; + } + } + + return 0; +} + static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *parent, lfs_mattr_t *attr) { // iterate over all directory directory entries parent->tail[0] = 0; parent->tail[1] = 1; while (!lfs_pairisnull(parent->tail)) { - int err = lfs_dir_fetch(lfs, parent, parent->tail); + struct lfs_dir_parentscan parentscan = { + .pair[0] = pair[0], + .pair[1] = pair[1], + .id = -1 + }; + + int err = lfs_dir_fetchwith(lfs, parent, parent->tail, + lfs_parentscan, &parentscan); if (err) { return err; } - // TODO make this O(n) by using fetchwith to match the pointers - for (uint16_t id = 0; id < parent->count; id++) { + if (parentscan.id != -1) { int err = lfs_dir_getentry(lfs, parent, 0x43dff000, - lfs_mktag(LFS_STRUCT_DIR, id, 0), attr); + lfs_mktag(LFS_STRUCT_DIR, parentscan.id, 0), attr); if (err) { - if (err == LFS_ERR_NOENT) { - continue; - } return err; } - if (lfs_paircmp(attr->u.pair, pair) == 0) { - return true; - } + return true; } } return false; } +// +//static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], +// lfs_mdir_t *parent, lfs_mattr_t *attr) { +// // iterate over all directory directory entries +// parent->tail[0] = 0; +// parent->tail[1] = 1; +// while (!lfs_pairisnull(parent->tail)) { +// int err = lfs_dir_fetch(lfs, parent, parent->tail); +// if (err) { +// return err; +// } +// +// // TODO make this O(n) by using fetchwith to match the pointers +// for (uint16_t id = 0; id < parent->count; id++) { +// int err = lfs_dir_getentry(lfs, parent, 0x43dff000, +// lfs_mktag(LFS_STRUCT_DIR, id, 0), attr); +// if (err) { +// if (err == LFS_ERR_NOENT) { +// continue; +// } +// return err; +// } +// +// if (lfs_paircmp(attr->u.pair, pair) == 0) { +// return true; +// } +// } +// } +// +// return false; +//} /* static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent, lfs_mattr_t *attr) { @@ -3494,6 +3610,7 @@ int lfs_deorphan(lfs_t *lfs) { lfs_mdir_t pdir = {.split = true}; lfs_mdir_t dir = {.tail = {0, 1}}; + lfs_entry_t idelete = {{0xffffffff, 0xffffffff}, 0xffff}; // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { @@ -3502,6 +3619,11 @@ int lfs_deorphan(lfs_t *lfs) { return err; } + // xor together indirect deletes + idelete.pair[0] ^= dir.idelete.pair[0]; + idelete.pair[1] ^= dir.idelete.pair[1]; + idelete.id ^= dir.idelete.id; + // check head blocks for orphans if (!pdir.split) { // check if we have a parent @@ -3531,7 +3653,7 @@ int lfs_deorphan(lfs_t *lfs) { if (!lfs_pairsync(attr.u.pair, pdir.tail)) { // we have desynced - LFS_DEBUG("Found desync %d %d", + LFS_DEBUG("Found half-orphan %d %d", attr.u.pair[0], attr.u.pair[1]); pdir.tail[0] = attr.u.pair[0]; @@ -3580,6 +3702,10 @@ int lfs_deorphan(lfs_t *lfs) { memcpy(&pdir, &dir, sizeof(pdir)); } + // update littlefs with current move + // TODO do this here? needs to be before reads also + lfs->idelete = idelete; + return 0; } /* diff --git a/lfs.h b/lfs.h index 59304848..f763d0bb 100644 --- a/lfs.h +++ b/lfs.h @@ -102,9 +102,10 @@ enum lfs_type { // internally used types LFS_TYPE_NAME = 0x010, LFS_TYPE_MOVE = 0x080, - LFS_TYPE_DELETE = 0x090, + LFS_TYPE_DELETE = 0x020, - LFS_TYPE_SUPERBLOCK = 0x0a0, + LFS_TYPE_SUPERBLOCK = 0x030, + LFS_TYPE_IDELETE = 0x0b0, LFS_TYPE_SOFTTAIL = 0x0c0, LFS_TYPE_HARDTAIL = 0x0d0, LFS_TYPE_CRC = 0x0e0, @@ -285,6 +286,25 @@ typedef struct lfs_mattrlist { struct lfs_mattrlist *next; } lfs_mattrlist_t; +typedef struct lfs_entry { + lfs_block_t pair[2]; + uint16_t id; +} lfs_entry_t; + +typedef struct lfs_mdir { + lfs_block_t pair[2]; + lfs_block_t tail[2]; + uint32_t rev; + lfs_off_t off; + uint32_t etag; + uint16_t count; + bool erased; + bool split; + lfs_entry_t idelete; + bool stop_at_commit; // TODO hmmm + uint16_t moveid; // TODO rm me +} lfs_mdir_t; + typedef struct lfs_cache { lfs_block_t block; lfs_off_t off; @@ -307,19 +327,6 @@ typedef struct lfs_file { lfs_mattrlist_t *attrs; } lfs_file_t; -typedef struct lfs_mdir { - lfs_block_t pair[2]; - lfs_block_t tail[2]; - uint32_t rev; - lfs_off_t off; - uint32_t etag; - uint16_t count; - bool erased; - bool split; - bool stop_at_commit; // TODO hmmm - int16_t moveid; -} lfs_mdir_t; - typedef struct lfs_dir { struct lfs_dir *next; struct lfs_mdir m; @@ -363,6 +370,8 @@ typedef struct lfs { lfs_free_t free; bool deorphaned; + lfs_entry_t idelete; + lfs_entry_t diff; lfs_size_t inline_size; lfs_size_t attrs_size; From 3ffcedb95b330f53f1caa505f112a2729f90aa6d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 29 May 2018 20:08:42 -0500 Subject: [PATCH 045/139] Restructured tags to better support xored-globals 32-bit tag structure: [--- 32 ---] [1|- 9 -|- 10 -|-- 12 --] ^ ^ ^ ^- entry length | | \--------- file id | \--------------- tag type \------------------- valid In this tag, the type decomposes into some more information: [--- 9 ---] [1|- 2 -|- 3 -|- 3 -] ^ ^ ^ ^- struct | | \------- type | \------------- scope \----------------- user The change in this encoding is the addition of a global scope: LFS_SCOPE_STRUCT = 0 00 xxx xxx LFS_SCOPE_ENTRY = 0 01 xxx xxx LFS_SCOPE_DIR = 0 10 xxx xxx LFS_SCOPE_FS = 0 11 xxx xxx LFS_SCOPE_USER = 1 xx xxx xxx --- lfs.c | 179 +++++++++++++++++++++++++++++----------------------------- lfs.h | 46 +++++++++------ 2 files changed, 118 insertions(+), 107 deletions(-) diff --git a/lfs.c b/lfs.c index efd8e83f..8386238d 100644 --- a/lfs.c +++ b/lfs.c @@ -438,20 +438,20 @@ static inline uint16_t lfs_tag_type(lfs_tag_t tag) { return (tag & 0x7fc00000) >> 22; } -static inline uint8_t lfs_tag_supertype(lfs_tag_t tag) { +static inline uint16_t lfs_tag_scope(lfs_tag_t tag) { return (tag & 0x70000000) >> 22; } -static inline uint8_t lfs_tag_subtype(lfs_tag_t tag) { - return (tag & 0x7c000000) >> 22; +static inline uint16_t lfs_tag_subtype(lfs_tag_t tag) { + return (tag & 0x7e000000) >> 22; } -static inline uint8_t lfs_tag_struct(lfs_tag_t tag) { - return (tag & 0x03c00000) >> 22; +static inline uint16_t lfs_tag_struct(lfs_tag_t tag) { + return (tag & 0x71c00000) >> 22; } static inline uint16_t lfs_tag_id(lfs_tag_t tag) { - return (tag & 0x001ff000) >> 12; + return (tag & 0x003ff000) >> 12; } static inline lfs_size_t lfs_tag_size(lfs_tag_t tag) { @@ -481,21 +481,21 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, static int lfs_commit_commit(lfs_t *lfs, struct lfs_commit *commit, lfs_mattr_t attr) { // filter out ids - if (lfs_tag_id(attr.tag) != 0x1ff && ( + if (lfs_tag_scope(attr.tag) <= LFS_SCOPE_ENTRY && ( lfs_tag_id(attr.tag) < commit->filter.begin || lfs_tag_id(attr.tag) >= commit->filter.end)) { return 0; } // special cases - if ((lfs_tag_type(attr.tag) & 0x103) == LFS_FROM_MOVE) { + if (lfs_tag_type(attr.tag) == LFS_FROM_MOVE) { return lfs_commit_move(lfs, commit, lfs_tag_size(attr.tag), lfs_tag_id(attr.tag), attr.u.dir, NULL); } uint16_t id = lfs_tag_id(attr.tag) - commit->filter.begin; - attr.tag = lfs_mktag(0, id, 0) | (attr.tag & 0xffe00fff); + attr.tag = lfs_mktag(0, id, 0) | (attr.tag & 0xffc00fff); // check if we fit lfs_size_t size = lfs_tag_size(attr.tag); @@ -559,7 +559,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // build crc tag tag = (0x80000000 & ~lfs_fromle32(tag)) | - lfs_mktag(LFS_TYPE_CRC, 0x1ff, + lfs_mktag(LFS_TYPE_CRC, 0x3ff, noff - (commit->off+sizeof(uint32_t))); // write out crc @@ -633,14 +633,14 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_commit_move *move = p; - if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE && + if (lfs_tag_struct(attr.tag) == LFS_STRUCT_DELETE && lfs_tag_id(attr.tag) <= move->id.from) { // something was deleted, we need to move around it move->id.from += 1; return 0; } - if (lfs_tag_type(attr.tag) == LFS_TYPE_MOVE) { + if (lfs_tag_type(attr.tag) == LFS_STRUCT_MOVE) { // TODO need this? // ignore moves return 0; @@ -658,7 +658,8 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { .off=move->commit->off, .etag=move->commit->ptag, .stop_at_commit=true}, - lfs_tag_type(attr.tag) & 0x100 ? 0x7ffff000 : 0x7c1ff000, + (lfs_tag_scope(attr.tag) == LFS_SCOPE_STRUCT) ? // TODO also entry? + 0x7e3ff000 : 0x7ffff000, &(lfs_mattr_t){ lfs_mktag(lfs_tag_type(attr.tag), move->id.to - move->commit->filter.begin, 0)}); // TODO can all these filter adjustments be consolidated? @@ -672,7 +673,7 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { } // update id and commit, as we are currently unique - attr.tag = lfs_mktag(0, move->id.to, 0) | (attr.tag & 0xffe00fff); + attr.tag = lfs_mktag(0, move->id.to, 0) | (attr.tag & 0xffc00fff); return lfs_commit_commit(lfs, move->commit, attr); } @@ -787,7 +788,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_valid(tag)) { if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0 && cb) { int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_DELETE, lfs->idelete.id, 0)}); + lfs_mktag(LFS_STRUCT_DELETE, lfs->idelete.id, 0)}); if (err) { return err; } @@ -821,7 +822,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0 && cb) { int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_DELETE, + lfs_mktag(LFS_STRUCT_DELETE, lfs->idelete.id, 0)}); if (err) { return err; @@ -846,8 +847,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, return err; } - if (lfs_tag_type(tag) == LFS_TYPE_SOFTTAIL || - lfs_tag_type(tag) == LFS_TYPE_HARDTAIL) { + if (lfs_tag_struct(tag) == LFS_STRUCT_TAIL) { temp.split = lfs_tag_type(tag) == LFS_TYPE_HARDTAIL; err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), temp.tail, sizeof(temp.tail)); @@ -860,16 +860,16 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (err) { return err; } - } else if (lfs_tag_type(tag) == LFS_TYPE_MOVE) { + } else if (lfs_tag_type(tag) == LFS_STRUCT_MOVE) { // TODO handle moves correctly? temp.moveid = lfs_tag_id(tag); } else { - if (lfs_tag_id(tag) < 0x1ff && + if (lfs_tag_id(tag) < 0x3ff && lfs_tag_id(tag) >= temp.count) { temp.count = lfs_tag_id(tag)+1; } - if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { + if (lfs_tag_struct(tag) == LFS_STRUCT_DELETE) { temp.count -= 1; if (temp.moveid != -1) { //printf("RENAME DEL %d (%d)\n", lfs_tag_id(tag), temp.moveid); @@ -920,7 +920,7 @@ static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0) { int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_DELETE, lfs->idelete.id, 0)}); + lfs_mktag(LFS_STRUCT_DELETE, lfs->idelete.id, 0)}); if (err) { return err; } @@ -1035,8 +1035,8 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, if (!lfs_pairisnull(dir->tail)) { // TODO le32 err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_SOFTTAIL + dir->split*0x10, - 0x1ff, sizeof(dir->tail)), + lfs_mktag(LFS_STRUCT_TAIL + dir->split*0x8, + 0x3ff, sizeof(dir->tail)), .u.buffer=dir->tail}); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1052,7 +1052,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, // TODO le32 err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ lfs_mktag(LFS_TYPE_IDELETE, - 0x1ff, sizeof(dir->idelete)), + 0x3ff, sizeof(dir->idelete)), .u.buffer=&dir->idelete}); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1151,7 +1151,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { .crc = 0xffffffff, .ptag = dir->etag, .filter.begin = 0, - .filter.end = 0x1ff, + .filter.end = 0x3ff, }; int err = lfs_commit_list(lfs, &commit, list); @@ -1216,15 +1216,15 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { pdir.tail[0] = dir->tail[0]; pdir.tail[1] = dir->tail[1]; int err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL + pdir.split*0x10, - 0x1ff, sizeof(pdir.tail)), + {lfs_mktag(LFS_STRUCT_TAIL + pdir.split*0x8, + 0x3ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); return err; } } int err = lfs_dir_commit(lfs, dir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_DELETE, id, 0)}}); + {lfs_mktag(LFS_STRUCT_DELETE, id, 0)}}); if (err) { return err; } @@ -1265,7 +1265,7 @@ static int lfs_dir_getscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { if ((attr.tag & get->mask) == (get->tag & get->mask)) { *get->attr = attr; return true; - } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { + } else if (lfs_tag_type(attr.tag) == LFS_STRUCT_DELETE) { if (lfs_tag_id(attr.tag) <= lfs_tag_id(get->tag)) { get->tag += lfs_mktag(0, 1, 0); } @@ -1300,7 +1300,7 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, } } - attr->tag = lfs_mktag(0, id, 0) | (attr->tag & 0xffe00fff); + attr->tag = lfs_mktag(0, id, 0) | (attr->tag & 0xffc00fff); return 0; } @@ -1353,21 +1353,21 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, } lfs_mattr_t attr; - int err = lfs_dir_getentry(lfs, dir, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, id, 0), &attr); + int err = lfs_dir_getentry(lfs, dir, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); if (err) { return err; } info->type = lfs_tag_subtype(attr.tag); - if (lfs_tag_type(attr.tag) == (LFS_TYPE_REG | LFS_STRUCT_CTZ)) { + if (lfs_tag_struct(attr.tag) == LFS_STRUCT_CTZ) { info->size = attr.u.ctz.size; - } else if (lfs_tag_type(attr.tag) == (LFS_TYPE_REG | LFS_STRUCT_INLINE)) { + } else if (lfs_tag_struct(attr.tag) == LFS_STRUCT_INLINE) { info->size = lfs_tag_size(attr.tag); } - err = lfs_dir_getbuffer(lfs, dir, 0x7ffff000, &(lfs_mattr_t){ - lfs_mktag(LFS_TYPE_NAME, id, lfs->name_size+1), + err = lfs_dir_getbuffer(lfs, dir, 0x71fff000, &(lfs_mattr_t){ + lfs_mktag(LFS_STRUCT_NAME, id, lfs->name_size+1), .u.buffer=info->name}); if (err) { return err; @@ -1385,7 +1385,7 @@ struct lfs_dir_find { static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_dir_find *find = p; - if (lfs_tag_type(attr.tag) == LFS_TYPE_NAME && + if (lfs_tag_struct(attr.tag) == LFS_STRUCT_NAME && lfs_tag_size(attr.tag) == find->len) { int res = lfs_bd_cmp(lfs, attr.u.d.block, attr.u.d.off, find->name, find->len); @@ -1397,7 +1397,7 @@ static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { // found a match find->id = lfs_tag_id(attr.tag); } - } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { + } else if (lfs_tag_type(attr.tag) == LFS_STRUCT_DELETE) { if (lfs_tag_id(attr.tag) == find->id) { find->id = -1; } else if (lfs_tag_id(attr.tag) < find->id) { @@ -1510,8 +1510,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, // TODO optimize grab for inline files and like? // TODO would this mean more code? // grab the entry data - int err = lfs_dir_getentry(lfs, dir, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, find.id, 0), &attr); + int err = lfs_dir_getentry(lfs, dir, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, find.id, 0), &attr); if (err) { return err; } @@ -1573,11 +1573,11 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_NAME, id, nlen), + {lfs_mktag(LFS_STRUCT_NAME | LFS_TYPE_DIR, id, nlen), .u.buffer=(void*)path}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_DIR | LFS_STRUCT_DIR, id, sizeof(dir.pair)), + {lfs_mktag(LFS_STRUCT_DIR | LFS_TYPE_DIR, id, sizeof(dir.pair)), .u.buffer=dir.pair}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, sizeof(cwd.tail)), + {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(cwd.tail)), .u.buffer=cwd.tail}}}}); // TODO need ack here? @@ -1599,8 +1599,8 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { attr.u.pair[1] = lfs->root[1]; } else { // get dir pair from parent - err = lfs_dir_getentry(lfs, &dir->m, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, id, 0), &attr); + err = lfs_dir_getentry(lfs, &dir->m, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); if (err) { return err; } @@ -1964,9 +1964,9 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_NAME, id, nlen), - .u.buffer=(void*)path}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0)}}}); + {lfs_mktag(LFS_STRUCT_NAME | LFS_TYPE_REG, id, nlen), + .u.buffer=(void*)path}, &(lfs_mattrlist_t){ + {lfs_mktag(LFS_STRUCT_INLINE | LFS_TYPE_REG, id, 0)}}}); if (err) { return err; } @@ -1979,7 +1979,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, cwd.pair[1] = cwd.tail[1]; } - attr.tag = lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, id, 0); + attr.tag = lfs_mktag(LFS_STRUCT_INLINE | LFS_TYPE_REG, id, 0); } else { if (id == -1) { return LFS_ERR_ISDIR; @@ -1987,8 +1987,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_EXIST; } - attr.tag = lfs_mktag(LFS_TYPE_REG, id, 0); - err = lfs_dir_get(lfs, &cwd, 0x701ff000, &attr); + attr.tag = lfs_mktag(LFS_SCOPE_STRUCT, id, 0); + err = lfs_dir_get(lfs, &cwd, 0x703ff000, &attr); if (err) { return err; } @@ -2224,17 +2224,17 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { int err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_CTZ, file->id, - 2*sizeof(uint32_t)), .u.buffer=&file->head}, - file->attrs}); + {lfs_mktag(LFS_STRUCT_CTZ | LFS_TYPE_REG, + file->id, 2*sizeof(uint32_t)), .u.buffer=&file->head}, + file->attrs}); if (err) { return err; } } else { int err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_REG | LFS_STRUCT_INLINE, file->id, - file->size), .u.buffer=file->cache.buffer}, - file->attrs}); + {lfs_mktag(LFS_STRUCT_INLINE | LFS_TYPE_REG, + file->id, file->size), .u.buffer=file->cache.buffer}, + file->attrs}); if (err) { return err; } @@ -2655,8 +2655,8 @@ int lfs_remove(lfs_t *lfs, const char *path) { // grab entry to see if we're dealing with a dir lfs_mattr_t attr; - err = lfs_dir_getentry(lfs, &cwd, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, id, 0), &attr); + err = lfs_dir_getentry(lfs, &cwd, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); if (err) { return err; } @@ -2725,10 +2725,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - // add to list so we catch updates if directories overlap lfs_mattr_t oldattr; - err = lfs_dir_getentry(lfs, &oldcwd, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, oldid, 0), &oldattr); + err = lfs_dir_getentry(lfs, &oldcwd, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, oldid, 0), &oldattr); if (err) { return err; } @@ -2746,8 +2745,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_mattr_t prevattr; if (prevexists) { // get prev entry, check that we have same type - err = lfs_dir_getentry(lfs, &newcwd, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, newid, 0), &prevattr); + err = lfs_dir_getentry(lfs, &newcwd, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, newid, 0), &prevattr); if (err) { return err; } @@ -2785,7 +2784,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving //printf("RENAME MOVE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); err = lfs_dir_commit(lfs, &oldcwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_MOVE, oldid, 0)}}); + {lfs_mktag(LFS_STRUCT_MOVE, oldid, 0)}}); if (err) { return err; } @@ -2808,17 +2807,18 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // // TODO NONONONONO // // TODO also don't call strlen twice (see prev name check) // err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ -// {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), +// {lfs_mktag(LFS_STRUCT_NAME, newid, strlen(newpath)), // .u.buffer=(void*)newpath}}); // if (err) { // return err; // } err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_NAME, newid, strlen(newpath)), - .u.buffer=(void*)newpath}, &(lfs_mattrlist_t){ + {lfs_mktag(LFS_STRUCT_NAME | lfs_tag_subtype(oldattr.tag), + newid, strlen(newpath)), + .u.buffer=(void*)newpath}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_FROM_MOVE, newid, oldid), - .u.dir=&oldcwd}}}); + .u.dir=&oldcwd}}}); if (err) { return err; } @@ -3031,8 +3031,8 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { dir.count += 1; err = lfs_dir_commit(lfs, &dir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, 0, - sizeof(superblock)), .u.buffer=&superblock}}); + {lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, + 0, sizeof(superblock)), .u.buffer=&superblock}}); if (err) { return err; } @@ -3069,9 +3069,8 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_superblock_t superblock; - err = lfs_dir_getbuffer(lfs, &dir, 0x7ffff000, &(lfs_mattr_t){ - lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, - 0, sizeof(superblock)), + err = lfs_dir_getbuffer(lfs, &dir, 0x7e3ff000, &(lfs_mattr_t){ + lfs_mktag(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), .u.buffer=&superblock}); if (err && err != LFS_ERR_RANGE) { return err; @@ -3156,8 +3155,8 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { lfs_mattr_t attr; - int err = lfs_dir_getentry(lfs, &dir, 0x701ff000, - lfs_mktag(LFS_TYPE_REG, id, 0), &attr); + int err = lfs_dir_getentry(lfs, &dir, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -3322,7 +3321,7 @@ struct lfs_dir_parentscan { static int lfs_parentscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_dir_parentscan *parentscan = p; - if ((lfs_tag_type(attr.tag) & 0x10f) == LFS_STRUCT_DIR) { + if (lfs_tag_struct(attr.tag) == LFS_STRUCT_DIR) { int err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, &attr.u, sizeof(attr.u)); if (err) { @@ -3333,7 +3332,7 @@ static int lfs_parentscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { // found a match parentscan->id = lfs_tag_id(attr.tag); } - } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { + } else if (lfs_tag_struct(attr.tag) == LFS_STRUCT_DELETE) { if (lfs_tag_id(attr.tag) == parentscan->id) { parentscan->id = -1; } else if (lfs_tag_id(attr.tag) < parentscan->id) { @@ -3363,7 +3362,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], } if (parentscan.id != -1) { - int err = lfs_dir_getentry(lfs, parent, 0x43dff000, + int err = lfs_dir_getentry(lfs, parent, 0x71fff000, lfs_mktag(LFS_STRUCT_DIR, parentscan.id, 0), attr); if (err) { return err; @@ -3448,8 +3447,8 @@ static int lfs_moved(lfs_t *lfs, lfs_mdir_t *fromdir, uint16_t fromid) { fromdir->moveid = -1; lfs_mattr_t fromentry; // TODO what about inline files? - int err = lfs_dir_getentry(lfs, fromdir, 0x401ff000, - lfs_mktag(LFS_TYPE_REG, fromid, 0), &fromentry); + int err = lfs_dir_getentry(lfs, fromdir, 0x71fff000, + lfs_mktag(LFS_STRUCT_DIR, fromid, 0), &fromentry); fromdir->moveid = fromid; if (err) { return err; @@ -3476,7 +3475,7 @@ static int lfs_moved(lfs_t *lfs, lfs_mdir_t *fromdir, uint16_t fromid) { } lfs_mattr_t toentry; - int err = lfs_dir_getentry(lfs, &todir, 0x43dff000, + int err = lfs_dir_getentry(lfs, &todir, 0x71fff000, lfs_mktag(LFS_STRUCT_DIR, toid, 0), &toentry); if (err) { if (err == LFS_ERR_NOENT) { @@ -3579,8 +3578,8 @@ static int lfs_relocate(lfs_t *lfs, parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; int err = lfs_dir_commit(lfs, &parent, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL + parent.split*0x10, // TODO hm - 0x1ff, sizeof(lfs_block_t[2])), + {lfs_mktag(LFS_STRUCT_TAIL + parent.split*0x8, // TODO hm + 0x3ff, sizeof(lfs_block_t[2])), .u.pair[0]=newpair[0], .u.pair[1]=newpair[1]}}); if (err) { return err; @@ -3642,8 +3641,9 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, - sizeof(pdir.tail)), .u.buffer=pdir.tail}}); + {lfs_mktag(LFS_TYPE_SOFTTAIL, + 0x3ff, sizeof(pdir.tail)), + .u.buffer=pdir.tail}}); if (err) { return err; } @@ -3659,8 +3659,9 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = attr.u.pair[0]; pdir.tail[1] = attr.u.pair[1]; err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x1ff, - sizeof(pdir.tail)), .u.buffer=pdir.tail}}); + {lfs_mktag(LFS_TYPE_SOFTTAIL, + 0x3ff, sizeof(pdir.tail)), + .u.buffer=pdir.tail}}); if (err) { return err; } diff --git a/lfs.h b/lfs.h index f763d0bb..fdde961a 100644 --- a/lfs.h +++ b/lfs.h @@ -96,30 +96,40 @@ enum lfs_error { // File types enum lfs_type { // file types - LFS_TYPE_REG = 0x040, - LFS_TYPE_DIR = 0x050, + LFS_TYPE_REG = 0x008, + LFS_TYPE_DIR = 0x010, + LFS_TYPE_SUPERBLOCK = 0x038, + + // internally used entry structures + LFS_STRUCT_CTZ = 0x001, + LFS_STRUCT_DIR = 0x002, + LFS_STRUCT_INLINE = 0x003, + + LFS_STRUCT_NAME = 0x041, + LFS_STRUCT_DELETE = 0x047, + LFS_STRUCT_MOVE = 0x046, // TODO rm me + + LFS_STRUCT_TAIL = 0x081, + LFS_STRUCT_CRC = 0x087, + LFS_STRUCT_IDELETE = 0x0c1, // internally used types - LFS_TYPE_NAME = 0x010, - LFS_TYPE_MOVE = 0x080, - LFS_TYPE_DELETE = 0x020, - - LFS_TYPE_SUPERBLOCK = 0x030, - LFS_TYPE_IDELETE = 0x0b0, - LFS_TYPE_SOFTTAIL = 0x0c0, - LFS_TYPE_HARDTAIL = 0x0d0, - LFS_TYPE_CRC = 0x0e0, - - // on disk structure - LFS_STRUCT_ATTR = 0x100, - LFS_STRUCT_INLINE = 0x000, - LFS_STRUCT_CTZ = 0x004, - LFS_STRUCT_DIR = 0x008, + LFS_TYPE_SOFTTAIL = 0x081, + LFS_TYPE_HARDTAIL = 0x089, + LFS_TYPE_CRC = 0x087, + LFS_TYPE_IDELETE = 0x0c1, + + // internally used scopes + LFS_SCOPE_STRUCT = 0x000, + LFS_SCOPE_ENTRY = 0x040, + LFS_SCOPE_DIR = 0x080, + LFS_SCOPE_FS = 0x0c0, + LFS_SCOPE_USER = 0x100, // internal sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x001, + LFS_FROM_MOVE = 0x004, }; // File open flags From cebf7aa0fec09c04a4b9b2ddacec50809269e37d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 1 Jul 2018 22:29:42 -0500 Subject: [PATCH 046/139] Switched back to simple deorphan-step on directory remove Originally I tried to reuse the indirect delete to accomplish truely atomic directory removes, however this fell apart when it came to implementing directory removes as a side-effect of renames. A single indirect-delete simply can't handle renames with removes as a side effects. When copying an entry to its destination, we need to atomically delete both the old entry, and the source of our copy. We can't delete both with only a single indirect-delete. It is possible to accomplish this with two indirect-deletes, but this is such an uncommon case that it's really not worth supporting efficiently due to how expensive globals are. I also dropped indirect-deletes for normal directory removes. I may add it back later, but at the moment it's extra code cost for that's not traveled very often. As a result, restructured the indirect delete handling to be a bit more generic, now with a multipurpose lfs_globals_t struct instead of the delete specific lfs_entry_t struct. Also worked on integrating xored-globals, now with several primitive global operations to manage fetching/updating globals on disk. --- lfs.c | 362 ++++++++++++++++++++++++++++++++++----------- lfs.h | 23 ++- tests/test_move.sh | 32 +++- 3 files changed, 322 insertions(+), 95 deletions(-) diff --git a/lfs.c b/lfs.c index 8386238d..6680e0ea 100644 --- a/lfs.c +++ b/lfs.c @@ -267,6 +267,8 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], static int lfs_moved(lfs_t *lfs, lfs_mdir_t *fromdir, uint16_t fromid); static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); +int lfs_scan(lfs_t *lfs); +int lfs_fixmove(lfs_t *lfs); int lfs_deorphan(lfs_t *lfs); @@ -458,6 +460,22 @@ static inline lfs_size_t lfs_tag_size(lfs_tag_t tag) { return tag & 0x00000fff; } +// operations on globals +static lfs_globals_t lfs_globals_xor( + const lfs_globals_t *a, const lfs_globals_t *b) { + lfs_globals_t res; + res.move.pair[0] = a->move.pair[0] ^ b->move.pair[0]; + res.move.pair[1] = a->move.pair[1] ^ b->move.pair[1]; + res.move.id = a->move.id ^ b->move.id; + return res; +} + +static bool lfs_globals_iszero(const lfs_globals_t *a) { + return (a->move.pair[0] == 0 && a->move.pair[1] == 0 && a->move.id == 0); +} + + +// commit logic struct lfs_commit { lfs_block_t block; lfs_off_t off; @@ -701,6 +719,23 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, return 0; } +static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, + const lfs_globals_t *source, const lfs_globals_t *diff) { + lfs_globals_t res = lfs_globals_xor(source, diff); + + if (!lfs_globals_iszero(&res)) { + int err = lfs_commit_commit(lfs, commit, (lfs_mattr_t){ + lfs_mktag(LFS_TYPE_IDELETE, + res.move.id, sizeof(res.move.pair)), + .u.buffer=res.move.pair}); + if (err) { + return err; + } + } + + return 0; +} + static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, bool split, const lfs_block_t tail[2]) { // allocate pair of dir blocks (backwards, so we write to block 1 first) @@ -727,7 +762,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, dir->tail[1] = tail[1]; dir->erased = false; dir->split = split; - dir->idelete = (lfs_entry_t){{0, 0}, 0}; + dir->globals = (lfs_globals_t){0}; // don't write out yet, let caller take care of that return 0; @@ -764,7 +799,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, dir->tail[1] = 0xffffffff; dir->count = 0; dir->split = false; - dir->idelete = (lfs_entry_t){{0, 0}, 0}; + dir->globals = (lfs_globals_t){0}; dir->moveid = -1; dir->rev = lfs_tole32(rev[0]); @@ -786,9 +821,12 @@ static int lfs_dir_fetchwith(lfs_t *lfs, // next commit not yet programmed if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_valid(tag)) { - if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0 && cb) { + // synthetic move + if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 + && cb) { int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_STRUCT_DELETE, lfs->idelete.id, 0)}); + lfs_mktag(LFS_STRUCT_DELETE, + lfs->globals.move.id, 0)}); if (err) { return err; } @@ -818,12 +856,13 @@ static int lfs_dir_fetchwith(lfs_t *lfs, // try other block break; } else { + // snythetic move // TODO combine with above? - if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0 + if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 && cb) { int err = cb(lfs, data, (lfs_mattr_t){ lfs_mktag(LFS_STRUCT_DELETE, - lfs->idelete.id, 0)}); + lfs->globals.move.id, 0)}); if (err) { return err; } @@ -855,8 +894,10 @@ static int lfs_dir_fetchwith(lfs_t *lfs, return err; } } else if (lfs_tag_type(tag) == LFS_TYPE_IDELETE) { + temp.globals.move.id = lfs_tag_id(tag); err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), - &temp.idelete, sizeof(temp.idelete)); + &temp.globals.move.pair, + sizeof(temp.globals.move.pair)); if (err) { return err; } @@ -864,7 +905,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, // TODO handle moves correctly? temp.moveid = lfs_tag_id(tag); } else { - if (lfs_tag_id(tag) < 0x3ff && + if (lfs_tag_scope(tag) <= LFS_SCOPE_ENTRY && lfs_tag_id(tag) >= temp.count) { temp.count = lfs_tag_id(tag)+1; } @@ -918,9 +959,10 @@ static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, lfs_off_t off = dir->off; lfs_tag_t tag = dir->etag; - if (lfs_paircmp(dir->pair, lfs->idelete.pair) == 0) { + // synthetic move + if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_STRUCT_DELETE, lfs->idelete.id, 0)}); + lfs_mktag(LFS_STRUCT_DELETE, lfs->globals.move.id, 0)}); if (err) { return err; } @@ -1031,29 +1073,29 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, ack = id; } + // reopen the reserved space at the end + // TODO can I just commit these first? commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); - if (!lfs_pairisnull(dir->tail)) { - // TODO le32 - err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ - lfs_mktag(LFS_STRUCT_TAIL + dir->split*0x8, - 0x3ff, sizeof(dir->tail)), - .u.buffer=dir->tail}); + + if (!relocated) { + err = lfs_commit_globals(lfs, &commit, + &dir->globals, &lfs->diff); if (err) { - if (err == LFS_ERR_CORRUPT) { + if (err == LFS_ERR_NOSPC) { + goto split; + } else if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } } - if (!(dir->idelete.pair[0] == 0 && - dir->idelete.pair[0] == 0 && - dir->idelete.id == 0)) { + if (!lfs_pairisnull(dir->tail)) { // TODO le32 err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_IDELETE, - 0x3ff, sizeof(dir->idelete)), - .u.buffer=&dir->idelete}); + lfs_mktag(LFS_STRUCT_TAIL + dir->split*0x8, + 0x3ff, sizeof(dir->tail)), + .u.buffer=dir->tail}); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1131,6 +1173,9 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, if (err) { return err; } + } else { + lfs->globals = lfs_globals_xor(&lfs->globals, &lfs->diff); + lfs->diff = (lfs_globals_t){0}; } return 0; @@ -1162,6 +1207,16 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { return err; } + if (!lfs_globals_iszero(&lfs->diff)) { + err = lfs_commit_globals(lfs, &commit, &dir->globals, &lfs->diff); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; + } + } + err = lfs_commit_crc(lfs, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { @@ -1173,6 +1228,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { // successful commit, lets update dir dir->off = commit.off; dir->etag = commit.ptag; + lfs->globals = lfs_globals_xor(&lfs->globals, &lfs->diff); + lfs->diff = (lfs_globals_t){0}; break; compact: @@ -1212,9 +1269,13 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { } if (res && pdir.split) { + // steal tail, and global state pdir.split = dir->split; pdir.tail[0] = dir->tail[0]; pdir.tail[1] = dir->tail[1]; + lfs->diff = dir->globals; + lfs->globals = lfs_globals_xor(&lfs->globals, &dir->globals); + int err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ {lfs_mktag(LFS_STRUCT_TAIL + pdir.split*0x8, 0x3ff, sizeof(pdir.tail)), @@ -2661,17 +2722,50 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } + lfs_mdir_t dir; if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { - lfs_mdir_t dir; // must be empty before removal err = lfs_dir_fetch(lfs, &dir, attr.u.pair); if (err) { return err; } + // TODO lfs_dir_empty? if (dir.count > 0 || dir.split) { return LFS_ERR_NOTEMPTY; } +// +// // unlink from tail chain and create move to fix +// lfs->diff.move.pair[0] = cwd.pair[0] ^ lfs->globals.move.pair[0]; +// lfs->diff.move.pair[1] = cwd.pair[1] ^ lfs->globals.move.pair[1]; +// lfs->diff.move.id = id ^ lfs->globals.move.id; +// +// // xor over our child's global state +// lfs->diff = lfs_globals_xor(&lfs->diff, &dir.globals); +// lfs->globals = lfs_globals_xor(&lfs->globals, &dir.globals); +// +// // find pred and remove +// // TODO handle dropped block? +// lfs_mdir_t pred; +// int res = lfs_pred(lfs, dir.pair, &pred); +// if (res < 0) { +// return res; +// } +// +// LFS_ASSERT(res); // must have pred +// pred.tail[0] = dir.tail[0]; +// pred.tail[1] = dir.tail[1]; +// err = lfs_dir_commit(lfs, &pred, &(lfs_mattrlist_t){ +// {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(pred.tail)), +// .u.buffer=pred.tail}}); +// if (err) { +// return err; +// } +// +// // mark global state to clear move entry +// lfs->diff.move.pair[0] = 0xffffffff ^ lfs->globals.move.pair[0]; +// lfs->diff.move.pair[1] = 0xffffffff ^ lfs->globals.move.pair[1]; +// lfs->diff.move.id = 0x3ff ^ lfs->globals.move.id; } // delete the entry @@ -2680,31 +2774,28 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } - // if we were a directory, find pred, replace tail - // TODO can this just deorphan? - if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { - err = lfs_deorphan(lfs); - if (err) { - return err; - } - } - +// // if we were a directory, fix the move we just created // if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { -// int res = lfs_pred(lfs, dir.pair, &cwd); -// if (res < 0) { -// return res; -// } -// -// LFS_ASSERT(res); // must have pred -// cwd.tail[0] = dir.tail[0]; -// cwd.tail[1] = dir.tail[1]; -// -// err = lfs_dir_commit(lfs, &cwd, NULL, 0); +// err = lfs_deorphan(lfs); // if (err) { // return err; // } // } + if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { + int res = lfs_pred(lfs, dir.pair, &cwd); + if (res < 0) { + return res; + } + + LFS_ASSERT(res); // must have pred + cwd.tail[0] = dir.tail[0]; + cwd.tail[1] = dir.tail[1]; + err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(cwd.tail)), + .u.buffer=cwd.tail}}); + } + return 0; } @@ -2741,8 +2832,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } bool prevexists = (err != LFS_ERR_NOENT); - bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); + //bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); + lfs_mattr_t prevattr; + lfs_mdir_t prevdir; if (prevexists) { // get prev entry, check that we have same type err = lfs_dir_getentry(lfs, &newcwd, 0x703ff000, @@ -2756,7 +2849,6 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } if (lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { - lfs_mdir_t prevdir; // must be empty before removal err = lfs_dir_fetch(lfs, &prevdir, prevattr.u.pair); if (err) { @@ -2781,19 +2873,24 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } } - // mark as moving - //printf("RENAME MOVE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); - err = lfs_dir_commit(lfs, &oldcwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_MOVE, oldid, 0)}}); - if (err) { - return err; - } - - if (samepair) { - // update pair if newcwd == oldcwd - newcwd = oldcwd; - } + // create move to fix later + lfs->diff.move.pair[0] = oldcwd.pair[0] ^ lfs->globals.move.pair[0]; + lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; + lfs->diff.move.id = oldid ^ lfs->globals.move.id; +// // mark as moving +// //printf("RENAME MOVE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); +// err = lfs_dir_commit(lfs, &oldcwd, &(lfs_mattrlist_t){ +// {lfs_mktag(LFS_STRUCT_MOVE, oldid, 0)}}); +// if (err) { +// return err; +// } +// +// if (samepair) { +// // update pair if newcwd == oldcwd +// newcwd = oldcwd; +// } +// // TODO check that all complaints are fixed // // move to new location // // TODO NAME????? @@ -2813,6 +2910,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // return err; // } + // move over all attributes err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_STRUCT_NAME | lfs_tag_subtype(oldattr.tag), newid, strlen(newpath)), @@ -2823,28 +2921,53 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - if (samepair) { - // update pair if newcwd == oldcwd - oldcwd = newcwd; - } - - // remove old entry - //printf("RENAME DELETE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); - err = lfs_dir_delete(lfs, &oldcwd, oldid); + // clean up after ourselves + err = lfs_fixmove(lfs); if (err) { return err; } - // if we were a directory, find pred, replace tail - // TODO can this just deorphan? if (prevexists && lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { - err = lfs_deorphan(lfs); - if (err) { - return err; + int res = lfs_pred(lfs, prevdir.pair, &newcwd); + if (res < 0) { + return res; } + + LFS_ASSERT(res); // must have pred + newcwd.tail[0] = prevdir.tail[0]; + newcwd.tail[1] = prevdir.tail[1]; + err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ + {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(newcwd.tail)), + .u.buffer=newcwd.tail}}); } return 0; + + +// if (samepair) { +// // update pair if newcwd == oldcwd +// oldcwd = newcwd; +// } +// +// err = fix +// +// // remove old entry +// //printf("RENAME DELETE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); +// err = lfs_dir_delete(lfs, &oldcwd, oldid); +// if (err) { +// return err; +// } +// +// // if we were a directory, find pred, replace tail +// // TODO can this just deorphan? +// if (prevexists && lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { +// err = lfs_deorphan(lfs); +// if (err) { +// return err; +// } +// } +// + return 0; } //int lfs_getattrs(lfs_t *lfs, const char *path, @@ -2954,7 +3077,17 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->files = NULL; lfs->dirs = NULL; lfs->deorphaned = false; - lfs->idelete = (lfs_entry_t){{0xffffffff, 0xffffffff}, 0xffff}; + lfs->globals.move.pair[0] = 0xffffffff; + lfs->globals.move.pair[1] = 0xffffffff; + lfs->globals.move.id = 0x3ff; + lfs->diff = (lfs_globals_t){0}; + + // scan for any global updates + // TODO rm me? need to grab any inits + int err = lfs_scan(lfs); + if (err) { + return err; + } return 0; } @@ -3122,6 +3255,11 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[0] = superblock.root[0]; lfs->root[1] = superblock.root[1]; + err = lfs_scan(lfs); + if (err) { + return err; + } + return 0; } @@ -3601,15 +3739,80 @@ static int lfs_relocate(lfs_t *lfs, return 0; } +int lfs_scan(lfs_t *lfs) { + if (lfs_pairisnull(lfs->root)) { // TODO rm me + return 0; + } + + lfs_mdir_t dir = {.tail = {0, 1}}; + lfs_globals_t globals = {{{0xffffffff, 0xffffffff}, 0x3ff}}; + + // iterate over all directory directory entries + while (!lfs_pairisnull(dir.tail)) { + int err = lfs_dir_fetch(lfs, &dir, dir.tail); + if (err) { + return err; + } + + // xor together indirect deletes + globals = lfs_globals_xor(&globals, &dir.globals); + } + + // update littlefs with globals + lfs->globals = globals; + lfs->diff = (lfs_globals_t){0}; + if (!lfs_pairisnull(lfs->globals.move.pair)) { + LFS_DEBUG("Found move %d %d %d", + lfs->globals.move.pair[0], + lfs->globals.move.pair[1], + lfs->globals.move.id); + } + + return 0; +} + +int lfs_fixmove(lfs_t *lfs) { + LFS_DEBUG("Fixing move %d %d %d", // TODO move to just deorphan + lfs->globals.move.pair[0], + lfs->globals.move.pair[1], + lfs->globals.move.id); + + // mark global state to clear move entry + lfs->diff.move.pair[0] = 0xffffffff ^ lfs->globals.move.pair[0]; + lfs->diff.move.pair[1] = 0xffffffff ^ lfs->globals.move.pair[1]; + lfs->diff.move.id = 0x3ff ^ lfs->globals.move.id; + + // fetch and delete the moved entry + lfs_mdir_t movedir; + int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.move.pair); + if (err) { + return err; + } + + err = lfs_dir_delete(lfs, &movedir, lfs->globals.move.id); + if (err) { + return err; + } + + return 0; +} + int lfs_deorphan(lfs_t *lfs) { lfs->deorphaned = true; - if (lfs_pairisnull(lfs->root)) { + if (lfs_pairisnull(lfs->root)) { // TODO rm me? return 0; } + // Fix bad moves + if (!lfs_pairisnull(lfs->globals.move.pair)) { + int err = lfs_fixmove(lfs); + if (err) { + return err; + } + } + lfs_mdir_t pdir = {.split = true}; lfs_mdir_t dir = {.tail = {0, 1}}; - lfs_entry_t idelete = {{0xffffffff, 0xffffffff}, 0xffff}; // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { @@ -3618,11 +3821,6 @@ int lfs_deorphan(lfs_t *lfs) { return err; } - // xor together indirect deletes - idelete.pair[0] ^= dir.idelete.pair[0]; - idelete.pair[1] ^= dir.idelete.pair[1]; - idelete.id ^= dir.idelete.id; - // check head blocks for orphans if (!pdir.split) { // check if we have a parent @@ -3671,7 +3869,7 @@ int lfs_deorphan(lfs_t *lfs) { } // check entries for moves - if (dir.moveid >= 0) { + //if (dir.moveid >= 0) { // TODO moves and stuff // TODO need to load entry to find it // // found moved entry @@ -3698,15 +3896,11 @@ int lfs_deorphan(lfs_t *lfs) { // return err; // } // } - } + //} memcpy(&pdir, &dir, sizeof(pdir)); } - // update littlefs with current move - // TODO do this here? needs to be before reads also - lfs->idelete = idelete; - return 0; } /* diff --git a/lfs.h b/lfs.h index fdde961a..1a9e2dd8 100644 --- a/lfs.h +++ b/lfs.h @@ -129,7 +129,7 @@ enum lfs_type { // internal sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x004, + LFS_FROM_MOVE = 0x0ff, }; // File open flags @@ -296,10 +296,17 @@ typedef struct lfs_mattrlist { struct lfs_mattrlist *next; } lfs_mattrlist_t; -typedef struct lfs_entry { - lfs_block_t pair[2]; - uint16_t id; -} lfs_entry_t; +//typedef struct lfs_entry { +// lfs_block_t pair[2]; +// uint16_t id; +//} lfs_entry_t; + +typedef struct lfs_globals { + struct lfs_move { + lfs_block_t pair[2]; + uint16_t id; + } move; +} lfs_globals_t; typedef struct lfs_mdir { lfs_block_t pair[2]; @@ -310,7 +317,7 @@ typedef struct lfs_mdir { uint16_t count; bool erased; bool split; - lfs_entry_t idelete; + lfs_globals_t globals; bool stop_at_commit; // TODO hmmm uint16_t moveid; // TODO rm me } lfs_mdir_t; @@ -380,8 +387,8 @@ typedef struct lfs { lfs_free_t free; bool deorphaned; - lfs_entry_t idelete; - lfs_entry_t diff; + lfs_globals_t globals; + lfs_globals_t diff; lfs_size_t inline_size; lfs_size_t attrs_size; diff --git a/tests/test_move.sh b/tests/test_move.sh index ff02553d..11dfecc1 100755 --- a/tests/test_move.sh +++ b/tests/test_move.sh @@ -59,7 +59,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-7 blocks/6 +truncate -s-11 blocks/6 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -86,8 +86,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hello", "d/hello") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-7 blocks/8 -truncate -s-7 blocks/a +truncate -s-11 blocks/8 +truncate -s-11 blocks/a tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; @@ -108,6 +108,32 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Move file after corrupt ---" +tests/test.py -s << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_rename(&lfs, "c/hello", "d/hello") => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "c") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_dir_open(&lfs, &dir[0], "d") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "hello") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Move dir ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; From b46fcac58511333cef79b388c53889d865ddd72e Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 4 Jul 2018 01:35:04 -0500 Subject: [PATCH 047/139] Fixed issues with finding wrong ids after bad commits Unfortunately, the behaviour needed of lfs_dir_fetchwith is as subtle as it is important. When fetching from a block corrupted by power-loss, lfs_dir_fetch must be able to rewind any state it picks up to before the corruption. This is not limited to the directory state, but includes find results and other side-effects. This gets a bit complicated when trying to generalize littlefs's fetchwith mechanics. Being able to scan a directory block during a fetch greatly impacts the runtime of littlefs operations, but if the state is generic how do we know what to rollback to? The fix here is to leave the management of rolling back state to the fetchwith match functions, and transparently pass a CRC tag to indicate the temporary state can be saved. --- lfs.c | 80 ++++++++++++++++++++++++++++++---------------- tests/test_dirs.sh | 2 -- 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/lfs.c b/lfs.c index 6680e0ea..44f99b12 100644 --- a/lfs.c +++ b/lfs.c @@ -721,16 +721,18 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, const lfs_globals_t *source, const lfs_globals_t *diff) { - lfs_globals_t res = lfs_globals_xor(source, diff); + if (lfs_globals_iszero(diff)) { + return 0; + } - if (!lfs_globals_iszero(&res)) { - int err = lfs_commit_commit(lfs, commit, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_IDELETE, - res.move.id, sizeof(res.move.pair)), - .u.buffer=res.move.pair}); - if (err) { - return err; - } + // TODO check performance/complexity of different strategies here + lfs_globals_t res = lfs_globals_xor(source, diff); + int err = lfs_commit_commit(lfs, commit, (lfs_mattr_t){ + lfs_mktag(LFS_TYPE_IDELETE, + res.move.id, sizeof(res.move.pair)), + .u.buffer=res.move.pair}); + if (err) { + return err; } return 0; @@ -878,6 +880,17 @@ static int lfs_dir_fetchwith(lfs_t *lfs, temp.etag = tag; crc = 0xffffffff; *dir = temp; + + // TODO simplify this? + if (cb) { + err = cb(lfs, data, (lfs_mattr_t){ + (tag | 0x80000000), + .u.d.block=temp.pair[0], + .u.d.off=off+sizeof(tag)}); + if (err) { + return err; + } + } } else { // TODO crc before callback??? err = lfs_bd_crc(lfs, temp.pair[0], @@ -1008,6 +1021,11 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; bool relocated = false; + // There's nothing special about our global delta, so feed it back + // into the global global delta + lfs->diff = lfs_globals_xor(&lfs->diff, &dir->globals); + dir->globals = (lfs_globals_t){0}; + // increment revision count dir->rev += 1; @@ -1207,14 +1225,12 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { return err; } - if (!lfs_globals_iszero(&lfs->diff)) { - err = lfs_commit_globals(lfs, &commit, &dir->globals, &lfs->diff); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - goto compact; - } - return err; + err = lfs_commit_globals(lfs, &commit, &dir->globals, &lfs->diff); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; } + return err; } err = lfs_commit_crc(lfs, &commit); @@ -1242,6 +1258,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { } // update any directories that are affected + // TODO what about pairs? what if we're splitting?? for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->m.pair, dir->pair) == 0) { d->m = *dir; @@ -1441,6 +1458,7 @@ struct lfs_dir_find { const char *name; uint16_t len; int16_t id; + int16_t tempid; }; static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { @@ -1456,14 +1474,16 @@ static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { if (res) { // found a match - find->id = lfs_tag_id(attr.tag); + find->tempid = lfs_tag_id(attr.tag); } } else if (lfs_tag_type(attr.tag) == LFS_STRUCT_DELETE) { - if (lfs_tag_id(attr.tag) == find->id) { - find->id = -1; - } else if (lfs_tag_id(attr.tag) < find->id) { - find->id -= 1; + if (lfs_tag_id(attr.tag) == find->tempid) { + find->tempid = -1; + } else if (lfs_tag_id(attr.tag) < find->tempid) { + find->tempid -= 1; } + } else if (lfs_tag_type(attr.tag) == LFS_TYPE_CRC) { + find->id = find->tempid; } return 0; @@ -1531,6 +1551,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, while (true) { //printf("checking %d %d for %s\n", attr.u.pair[0], attr.u.pair[1], *path); find.id = -1; + find.tempid = -1; int err = lfs_dir_fetchwith(lfs, dir, attr.u.pair, lfs_dir_findscan, &find); if (err) { @@ -3451,9 +3472,11 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir) { */ +// TODO combine parentscan and findscan? struct lfs_dir_parentscan { lfs_block_t pair[2]; int16_t id; + int16_t tempid; }; static int lfs_parentscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { @@ -3468,14 +3491,16 @@ static int lfs_parentscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { if (lfs_paircmp(attr.u.pair, parentscan->pair) == 0) { // found a match - parentscan->id = lfs_tag_id(attr.tag); + parentscan->tempid = lfs_tag_id(attr.tag); } } else if (lfs_tag_struct(attr.tag) == LFS_STRUCT_DELETE) { - if (lfs_tag_id(attr.tag) == parentscan->id) { - parentscan->id = -1; - } else if (lfs_tag_id(attr.tag) < parentscan->id) { - parentscan->id -= 1; + if (lfs_tag_id(attr.tag) == parentscan->tempid) { + parentscan->tempid = -1; + } else if (lfs_tag_id(attr.tag) < parentscan->tempid) { + parentscan->tempid -= 1; } + } else if (lfs_tag_type(attr.tag) == LFS_TYPE_CRC) { + parentscan->id = parentscan->tempid; } return 0; @@ -3490,7 +3515,8 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], struct lfs_dir_parentscan parentscan = { .pair[0] = pair[0], .pair[1] = pair[1], - .id = -1 + .id = -1, + .tempid = -1, }; int err = lfs_dir_fetchwith(lfs, parent, parent->tail, diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index c9cc6694..53d76f7a 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -152,8 +152,6 @@ tests/test.py << TEST strcmp(info.name, "..") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - printf("nameee \"%s\"\n", info.name); - printf("expect \"%s\"\n", "burito"); strcmp(info.name, "burito") => 0; info.type => LFS_TYPE_REG; lfs_dir_read(&lfs, &dir[0], &info) => 1; From 2ff32d2dfb879e38cc99f4c759683e95bd11faef Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 8 Jul 2018 14:21:29 -0500 Subject: [PATCH 048/139] Fixed bug where globals were poisoning move commits The issue lies in the reuse of the id field for globals. Before globals, the only tags with a non-null (0x3ff) id field were names, structs, and other file-specific metadata. But globals are also using this field for the indirect delete, since otherwise the globals structure would be very unaligned (74-bits long). To make matters worse, the id field for globals contains the delta used to reconstruct the globals at mount time. Which means the id field could take on very absurd values and break the dir fetch logic if we're not careful. Solution is to use the scope portion of the type field where necessary, although unforunately this does add some code cost. --- lfs.c | 18 +++++++++++++++++ lfs.h | 2 +- tests/test_move.sh | 48 +++++++++++++++++++++++++++++++++++----------- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/lfs.c b/lfs.c index 44f99b12..ba3cfb1f 100644 --- a/lfs.c +++ b/lfs.c @@ -664,6 +664,13 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { return 0; } + // TODO AHHHH, scopes, what about user scope above? + if (lfs_tag_scope(attr.tag) == LFS_SCOPE_FS || + lfs_tag_scope(attr.tag) == LFS_SCOPE_DIR) { + // ignore non-matching ids + return 0; + } + if (lfs_tag_id(attr.tag) != move->id.from) { // ignore non-matching ids return 0; @@ -1023,6 +1030,9 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, // There's nothing special about our global delta, so feed it back // into the global global delta + // TODO IMMENSE HMM globals get bleed into from above, need to be fixed after commits due to potential moves + lfs_globals_t gtemp = dir->globals; // TODO hmm, why did we have different variables then? + lfs->diff = lfs_globals_xor(&lfs->diff, &dir->globals); dir->globals = (lfs_globals_t){0}; @@ -1196,6 +1206,8 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, lfs->diff = (lfs_globals_t){0}; } + lfs->globals = lfs_globals_xor(&lfs->globals, >emp); // TODO hmm, why did we have different variables then? + return 0; } @@ -1244,6 +1256,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { // successful commit, lets update dir dir->off = commit.off; dir->etag = commit.ptag; +// // TODO hm +// dir->globals = lfs_globals_xor(&dir->globals, &lfs->diff); lfs->globals = lfs_globals_xor(&lfs->globals, &lfs->diff); lfs->diff = (lfs_globals_t){0}; break; @@ -2954,6 +2968,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return res; } + // TODO test for global state stealing? + // steal global state + lfs->globals = lfs_globals_xor(&lfs->globals, &prevdir.globals); + LFS_ASSERT(res); // must have pred newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; diff --git a/lfs.h b/lfs.h index 1a9e2dd8..ff29e84b 100644 --- a/lfs.h +++ b/lfs.h @@ -129,7 +129,7 @@ enum lfs_type { // internal sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x0ff, + LFS_FROM_MOVE = 0x03f, }; // File open flags diff --git a/tests/test_move.sh b/tests/test_move.sh index 11dfecc1..5b851c5d 100755 --- a/tests/test_move.sh +++ b/tests/test_move.sh @@ -109,7 +109,7 @@ tests/test.py << TEST TEST echo "--- Move file after corrupt ---" -tests/test.py -s << TEST +tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_rename(&lfs, "c/hello", "d/hello") => 0; lfs_unmount(&lfs) => 0; @@ -166,7 +166,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; TEST -rm -v blocks/7 +truncate -s-11 blocks/7 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -182,8 +182,6 @@ tests/test.py << TEST lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "..") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "hello") => 0; - lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "hi") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_unmount(&lfs) => 0; @@ -195,8 +193,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hi", "d/hi") => 0; lfs_unmount(&lfs) => 0; TEST -rm -v blocks/9 -rm -v blocks/a +truncate -s-11 blocks/9 +truncate -s-11 blocks/b tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; @@ -205,9 +203,33 @@ tests/test.py << TEST lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "..") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "hi") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + lfs_dir_open(&lfs, &dir[0], "d") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "hello") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Move dir after corrupt ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_rename(&lfs, "c/hi", "d/hi") => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "c") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "hi") => 0; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; lfs_dir_open(&lfs, &dir[0], "d") => 0; @@ -215,6 +237,10 @@ tests/test.py << TEST strcmp(info.name, ".") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "..") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "hello") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "hi") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_unmount(&lfs) => 0; TEST @@ -225,9 +251,9 @@ tests/test.py << TEST lfs_dir_open(&lfs, &dir[0], "a/hi") => LFS_ERR_NOENT; lfs_dir_open(&lfs, &dir[0], "b/hi") => LFS_ERR_NOENT; - lfs_dir_open(&lfs, &dir[0], "d/hi") => LFS_ERR_NOENT; + lfs_dir_open(&lfs, &dir[0], "c/hi") => LFS_ERR_NOENT; - lfs_dir_open(&lfs, &dir[0], "c/hi") => 0; + lfs_dir_open(&lfs, &dir[0], "d/hi") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, ".") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; @@ -243,9 +269,9 @@ tests/test.py << TEST lfs_dir_open(&lfs, &dir[0], "a/hello") => LFS_ERR_NOENT; lfs_dir_open(&lfs, &dir[0], "b/hello") => LFS_ERR_NOENT; - lfs_dir_open(&lfs, &dir[0], "d/hello") => LFS_ERR_NOENT; + lfs_dir_open(&lfs, &dir[0], "c/hello") => LFS_ERR_NOENT; - lfs_file_open(&lfs, &file[0], "c/hello", LFS_O_RDONLY) => 0; + lfs_file_open(&lfs, &file[0], "d/hello", LFS_O_RDONLY) => 0; lfs_file_read(&lfs, &file[0], buffer, 5) => 5; memcmp(buffer, "hola\n", 5) => 0; lfs_file_read(&lfs, &file[0], buffer, 8) => 8; From d7b065293689658d30a214b269bbc950d2c7db39 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 9 Jul 2018 11:47:04 -0500 Subject: [PATCH 049/139] Removed old move logic, now passing move tests The introduction of xored-globals required quite a bit of work to integrate. But now that that is working, we can strip out the old move logic. It's worth noting that the xored-globals integration with commits is relatively complex and subtle. --- lfs.c | 339 +++------------------------------------------------------- lfs.h | 2 - 2 files changed, 13 insertions(+), 328 deletions(-) diff --git a/lfs.c b/lfs.c index ba3cfb1f..4c5379d9 100644 --- a/lfs.c +++ b/lfs.c @@ -264,7 +264,6 @@ int lfs_fs_traverse(lfs_t *lfs, static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent, lfs_mattr_t *attr); -static int lfs_moved(lfs_t *lfs, lfs_mdir_t *fromdir, uint16_t fromid); static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); int lfs_scan(lfs_t *lfs); @@ -658,12 +657,6 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { return 0; } - if (lfs_tag_type(attr.tag) == LFS_STRUCT_MOVE) { - // TODO need this? - // ignore moves - return 0; - } - // TODO AHHHH, scopes, what about user scope above? if (lfs_tag_scope(attr.tag) == LFS_SCOPE_FS || lfs_tag_scope(attr.tag) == LFS_SCOPE_DIR) { @@ -809,7 +802,6 @@ static int lfs_dir_fetchwith(lfs_t *lfs, dir->count = 0; dir->split = false; dir->globals = (lfs_globals_t){0}; - dir->moveid = -1; dir->rev = lfs_tole32(rev[0]); lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); @@ -921,9 +913,6 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (err) { return err; } - } else if (lfs_tag_type(tag) == LFS_STRUCT_MOVE) { - // TODO handle moves correctly? - temp.moveid = lfs_tag_id(tag); } else { if (lfs_tag_scope(tag) <= LFS_SCOPE_ENTRY && lfs_tag_id(tag) >= temp.count) { @@ -932,14 +921,6 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (lfs_tag_struct(tag) == LFS_STRUCT_DELETE) { temp.count -= 1; - if (temp.moveid != -1) { - //printf("RENAME DEL %d (%d)\n", lfs_tag_id(tag), temp.moveid); - } - if (lfs_tag_id(tag) == temp.moveid) { - temp.moveid = -1; - } else if (lfs_tag_id(tag) < temp.moveid) { - temp.moveid -= 1; - } } if (cb) { @@ -1086,9 +1067,9 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, .filter.end = end, }; - // commit with a move - for (uint16_t id = begin; id < end; id++) { - err = lfs_commit_move(lfs, &commit, id, id, source, list); + if (!relocated) { + err = lfs_commit_globals(lfs, &commit, + &dir->globals, &lfs->diff); if (err) { if (err == LFS_ERR_NOSPC) { goto split; @@ -1097,17 +1078,11 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, } return err; } - - ack = id; } - // reopen the reserved space at the end - // TODO can I just commit these first? - commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); - - if (!relocated) { - err = lfs_commit_globals(lfs, &commit, - &dir->globals, &lfs->diff); + // commit with a move + for (uint16_t id = begin; id < end; id++) { + err = lfs_commit_move(lfs, &commit, id, id, source, list); if (err) { if (err == LFS_ERR_NOSPC) { goto split; @@ -1116,9 +1091,15 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, } return err; } + + ack = id; } + // reopen reserved space at the end + commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); + if (!lfs_pairisnull(dir->tail)) { + // commit tail, which may be new after last size check // TODO le32 err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ lfs_mktag(LFS_STRUCT_TAIL + dir->split*0x8, @@ -1279,6 +1260,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { } } + // TODO what if we relocated the block containing the move? return 0; } @@ -1379,19 +1361,6 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, return LFS_ERR_NOENT; } - // TODO hmm, stop at commit? maybe we need to handle this elsewhere? - // Should commit get be its own thing? commit traverse? - if (id == dir->moveid && !dir->stop_at_commit) { - int moved = lfs_moved(lfs, dir, dir->moveid); - if (moved < 0) { - return moved; - } - - if (moved) { - return LFS_ERR_NOENT; - } - } - attr->tag = lfs_mktag(0, id, 0) | (attr->tag & 0xffc00fff); return 0; } @@ -1433,17 +1402,6 @@ static int lfs_dir_getentry(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { - if (id == dir->moveid) { - int moved = lfs_moved(lfs, dir, dir->moveid); - if (moved < 0) { - return moved; - } - - if (moved) { - return LFS_ERR_NOENT; - } - } - lfs_mattr_t attr; int err = lfs_dir_getentry(lfs, dir, 0x703ff000, lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); @@ -1585,17 +1543,6 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, attr.u.pair[1] = dir->tail[1]; } - if (find.id == dir->moveid) { - int moved = lfs_moved(lfs, dir, dir->moveid); - if (moved < 0) { - return moved; - } - - if (moved) { - return LFS_ERR_NOENT; - } - } - *id = find.id; find.name += find.len; find.name += strspn(find.name, "/"); @@ -2913,38 +2860,6 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; lfs->diff.move.id = oldid ^ lfs->globals.move.id; -// // mark as moving -// //printf("RENAME MOVE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); -// err = lfs_dir_commit(lfs, &oldcwd, &(lfs_mattrlist_t){ -// {lfs_mktag(LFS_STRUCT_MOVE, oldid, 0)}}); -// if (err) { -// return err; -// } -// -// if (samepair) { -// // update pair if newcwd == oldcwd -// newcwd = oldcwd; -// } -// -// TODO check that all complaints are fixed -// // move to new location -// // TODO NAME????? -// // TODO HAH, move doesn't want to override things (due -// // to its use in compaction), but that's _exactly what we want here -// err = lfs_dir_commitwith(lfs, &newcwd, lfs_commit_move, -// &(struct lfs_commit_move){.dir=&oldcwd, .id={oldid, newid}}); -// if (err) { -// return err; -// } -// // TODO NONONONONO -// // TODO also don't call strlen twice (see prev name check) -// err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ -// {lfs_mktag(LFS_STRUCT_NAME, newid, strlen(newpath)), -// .u.buffer=(void*)newpath}}); -// if (err) { -// return err; -// } - // move over all attributes err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_STRUCT_NAME | lfs_tag_subtype(oldattr.tag), @@ -3624,97 +3539,6 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], return false; } */ -static int lfs_moved(lfs_t *lfs, lfs_mdir_t *fromdir, uint16_t fromid) { - // grab entry pair we're looking for - fromdir->moveid = -1; - lfs_mattr_t fromentry; - // TODO what about inline files? - int err = lfs_dir_getentry(lfs, fromdir, 0x71fff000, - lfs_mktag(LFS_STRUCT_DIR, fromid, 0), &fromentry); - fromdir->moveid = fromid; - if (err) { - return err; - } - - // skip superblock - lfs_mdir_t todir; - err = lfs_dir_fetch(lfs, &todir, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - // iterate over all directory directory entries - while (!lfs_pairisnull(todir.tail)) { - int err = lfs_dir_fetch(lfs, &todir, todir.tail); - if (err) { - return err; - } - - for (int toid = 0; toid < todir.count; toid++) { - if (lfs_paircmp(todir.pair, fromdir->pair) == 0 && - toid == fromid) { - continue; - } - - lfs_mattr_t toentry; - int err = lfs_dir_getentry(lfs, &todir, 0x71fff000, - lfs_mktag(LFS_STRUCT_DIR, toid, 0), &toentry); - if (err) { - if (err == LFS_ERR_NOENT) { - continue; - } - return err; - } - - if (lfs_paircmp(toentry.u.pair, fromentry.u.pair) == 0) { - return true; - } - } - } - - return false; -} -/* -static int lfs_moved(lfs_t *lfs, const void *e) { - if (lfs_pairisnull(lfs->root)) { - return 0; - } - - // skip superblock - lfs_mdir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - // iterate over all directory directory entries - lfs_mattr_t entry; - while (!lfs_pairisnull(cwd.d.tail)) { - err = lfs_dir_fetch(lfs, &cwd, cwd.d.tail); - if (err) { - return err; - } - - while (true) { - err = lfs_dir_next(lfs, &cwd, &entry); - if (err && err != LFS_ERR_NOENT) { - return err; - } - - if (err == LFS_ERR_NOENT) { - break; - } - - if (!(LFS_STRUCT_MOVED & entry.d.type) && - memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) { - return true; - } - } - } - - return false; -} -*/ // TODO rename to lfs_dir_relocate? static int lfs_relocate(lfs_t *lfs, @@ -3912,148 +3736,11 @@ int lfs_deorphan(lfs_t *lfs) { } } - // check entries for moves - //if (dir.moveid >= 0) { -// TODO moves and stuff - // TODO need to load entry to find it -// // found moved entry -// int moved = lfs_moved(lfs, &entry.u); -// if (moved < 0) { -// return moved; -// } -// -// if (moved) { -// LFS_DEBUG("Found move %d %d", -// entry.d.u.dir[0], entry.d.u.dir[1]); -// err = lfs_dir_set(lfs, &dir, &entry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); -// if (err) { -// return err; -// } -// } else { -// LFS_DEBUG("Found partial move %d %d", -// entry.d.u.dir[0], entry.d.u.dir[1]); -// entry.d.type &= ~LFS_STRUCT_MOVED; -// err = lfs_dir_set(lfs, &dir, &entry, (struct lfs_region[]){ -// {LFS_FROM_MEM, 0, 1, &entry.d, 1}}, 1); -// if (err) { -// return err; -// } -// } - //} - memcpy(&pdir, &dir, sizeof(pdir)); } return 0; } -/* -int lfs_deorphan(lfs_t *lfs) { - lfs->deorphaned = true; - - if (lfs_pairisnull(lfs->root)) { - return 0; - } - - lfs_mdir_t pdir = {.d.size = 0x80000000}; - lfs_mdir_t cwd = {.d.tail[0] = 0, .d.tail[1] = 1}; - - // iterate over all directory directory entries - while (!lfs_pairisnull(cwd.d.tail)) { - int err = lfs_dir_fetch(lfs, &cwd, cwd.d.tail); - if (err) { - return err; - } - - // check head blocks for orphans - if (!(0x80000000 & pdir.d.size)) { - // check if we have a parent - lfs_mdir_t parent; - lfs_mattr_t entry; - int res = lfs_parent(lfs, pdir.d.tail, &parent, &entry); - if (res < 0) { - return res; - } - - if (!res) { - // we are an orphan - LFS_DEBUG("Found orphan %d %d", - pdir.d.tail[0], pdir.d.tail[1]); - - pdir.d.tail[0] = cwd.d.tail[0]; - pdir.d.tail[1] = cwd.d.tail[1]; - - err = lfs_dir_commit(lfs, &pdir, NULL, 0); - if (err) { - return err; - } - - break; - } - - if (!lfs_pairsync(entry.d.u.dir, pdir.d.tail)) { - // we have desynced - LFS_DEBUG("Found desync %d %d", - entry.d.u.dir[0], entry.d.u.dir[1]); - - pdir.d.tail[0] = entry.d.u.dir[0]; - pdir.d.tail[1] = entry.d.u.dir[1]; - - err = lfs_dir_commit(lfs, &pdir, NULL, 0); - if (err) { - return err; - } - - break; - } - } - - // check entries for moves - lfs_mattr_t entry; - while (true) { - err = lfs_dir_next(lfs, &cwd, &entry); - if (err && err != LFS_ERR_NOENT) { - return err; - } - - if (err == LFS_ERR_NOENT) { - break; - } - - // found moved entry - if (entry.d.type & LFS_STRUCT_MOVED) { - int moved = lfs_moved(lfs, &entry.d.u); - if (moved < 0) { - return moved; - } - - if (moved) { - LFS_DEBUG("Found move %d %d", - entry.d.u.dir[0], entry.d.u.dir[1]); - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); - if (err) { - return err; - } - } else { - LFS_DEBUG("Found partial move %d %d", - entry.d.u.dir[0], entry.d.u.dir[1]); - entry.d.type &= ~LFS_STRUCT_MOVED; - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 1, &entry.d, 1}}, 1); - if (err) { - return err; - } - } - } - } - - memcpy(&pdir, &cwd, sizeof(pdir)); - } - - return 0; -} -*/ /// External filesystem filesystem operations /// //int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { diff --git a/lfs.h b/lfs.h index ff29e84b..3ea3b5b0 100644 --- a/lfs.h +++ b/lfs.h @@ -107,7 +107,6 @@ enum lfs_type { LFS_STRUCT_NAME = 0x041, LFS_STRUCT_DELETE = 0x047, - LFS_STRUCT_MOVE = 0x046, // TODO rm me LFS_STRUCT_TAIL = 0x081, LFS_STRUCT_CRC = 0x087, @@ -319,7 +318,6 @@ typedef struct lfs_mdir { bool split; lfs_globals_t globals; bool stop_at_commit; // TODO hmmm - uint16_t moveid; // TODO rm me } lfs_mdir_t; typedef struct lfs_cache { From c1103efb536ec237a9b61ef40fe3617e63bb2e07 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 9 Jul 2018 12:51:31 -0500 Subject: [PATCH 050/139] Changed type info to be retrieved from name tag instead of struct tag Originally, I had type info encoded in the struct tag. This initially made sense because the type info only directly impacts the struct tag. However this was a case of focusing too much on the details instead of the bigger picture. A more file operations need to figure out the type of a file, but it's only actually a small number of file operations that need to interact with the file's structure. For the common case, providing the type of the file early shortens operations by a full tag access. Additionally, but storing the type in the file name tag, this opens up the struct tag to use those bits for storing more struct descriptions. --- lfs.c | 203 +++++++++++++++++++++++++--------------------------------- lfs.h | 5 +- 2 files changed, 88 insertions(+), 120 deletions(-) diff --git a/lfs.c b/lfs.c index 4c5379d9..c23d07f9 100644 --- a/lfs.c +++ b/lfs.c @@ -1402,27 +1402,30 @@ static int lfs_dir_getentry(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { - lfs_mattr_t attr; - int err = lfs_dir_getentry(lfs, dir, 0x703ff000, + lfs_mattr_t attr = { + lfs_mktag(LFS_STRUCT_NAME, id, lfs->name_size+1), + .u.buffer=info->name, + }; + + int err = lfs_dir_getbuffer(lfs, dir, 0x71fff000, &attr); + if (err) { + return err; + } + + info->type = 0x38 & lfs_tag_subtype(attr.tag); + + err = lfs_dir_getentry(lfs, dir, 0x703ff000, lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); if (err) { return err; } - info->type = lfs_tag_subtype(attr.tag); if (lfs_tag_struct(attr.tag) == LFS_STRUCT_CTZ) { info->size = attr.u.ctz.size; } else if (lfs_tag_struct(attr.tag) == LFS_STRUCT_INLINE) { info->size = lfs_tag_size(attr.tag); } - err = lfs_dir_getbuffer(lfs, dir, 0x71fff000, &(lfs_mattr_t){ - lfs_mktag(LFS_STRUCT_NAME, id, lfs->name_size+1), - .u.buffer=info->name}); - if (err) { - return err; - } - return 0; } @@ -1431,6 +1434,8 @@ struct lfs_dir_find { uint16_t len; int16_t id; int16_t tempid; + uint8_t findtype; + uint8_t tempfindtype; }; static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { @@ -1447,6 +1452,7 @@ static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { if (res) { // found a match find->tempid = lfs_tag_id(attr.tag); + find->tempfindtype = 0x38 & lfs_tag_subtype(attr.tag); } } else if (lfs_tag_type(attr.tag) == LFS_STRUCT_DELETE) { if (lfs_tag_id(attr.tag) == find->tempid) { @@ -1456,6 +1462,7 @@ static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { } } else if (lfs_tag_type(attr.tag) == LFS_TYPE_CRC) { find->id = find->tempid; + find->findtype = find->tempfindtype; } return 0; @@ -1463,7 +1470,7 @@ static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { // TODO drop others, make this only return id, also make get take in only entry to populate (with embedded tag) static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, - const char **path, uint16_t *id) { + const char **path, uint16_t *id, uint8_t *type) { lfs_mattr_t attr = { .u.pair[0] = lfs->root[0], .u.pair[1] = lfs->root[1], @@ -1482,6 +1489,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, // special case for root dir if (find.name[0] == '\0') { // Return ISDIR when we hit root + // TODO change this to -1 or 0x3ff? + *type = LFS_TYPE_DIR; return LFS_ERR_ISDIR; } @@ -1544,12 +1553,19 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, } *id = find.id; + *type = find.findtype; find.name += find.len; find.name += strspn(find.name, "/"); if (find.name[0] == '\0') { return 0; } + // don't continue on if we didn't hit a directory + // TODO update with what's on master? + if (find.findtype != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + // TODO optimize grab for inline files and like? // TODO would this mean more code? // grab the entry data @@ -1558,12 +1574,6 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, if (err) { return err; } - - // continue on if we hit a directory - // TODO update with what's on master? - if (lfs_tag_subtype(attr.tag) != LFS_TYPE_DIR) { - return LFS_ERR_NOTDIR; - } } } @@ -1578,7 +1588,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } lfs_mdir_t cwd; - int err = lfs_dir_find(lfs, &cwd, &path, &(uint16_t){0}); + int err = lfs_dir_find(lfs, &cwd, &path, &(uint16_t){0}, &(uint8_t){0}); if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { if (!err || err == LFS_ERR_ISDIR) { return LFS_ERR_EXIST; @@ -1618,7 +1628,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_STRUCT_NAME | LFS_TYPE_DIR, id, nlen), .u.buffer=(void*)path}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_DIR | LFS_TYPE_DIR, id, sizeof(dir.pair)), + {lfs_mktag(LFS_STRUCT_DIR, id, sizeof(dir.pair)), .u.buffer=dir.pair}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(cwd.tail)), .u.buffer=cwd.tail}}}}); @@ -1630,11 +1640,16 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { uint16_t id; - int err = lfs_dir_find(lfs, &dir->m, &path, &id); + uint8_t type; + int err = lfs_dir_find(lfs, &dir->m, &path, &id, &type); if (err && err != LFS_ERR_ISDIR) { return err; } + if (type != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + lfs_mattr_t attr; if (err == LFS_ERR_ISDIR) { // handle root dir separately @@ -1647,10 +1662,6 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { if (err) { return err; } - - if (lfs_tag_subtype(attr.tag) != LFS_TYPE_DIR) { - return LFS_ERR_NOTDIR; - } } // fetch first pair @@ -1982,8 +1993,10 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // allocate entry for file if it doesn't exist lfs_mdir_t cwd; uint16_t id; - int err = lfs_dir_find(lfs, &cwd, &path, &id); - if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { + uint8_t type; + int err = lfs_dir_find(lfs, &cwd, &path, &id, &type); + if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) && + err != LFS_ERR_ISDIR) { return err; } @@ -2009,7 +2022,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ {lfs_mktag(LFS_STRUCT_NAME | LFS_TYPE_REG, id, nlen), .u.buffer=(void*)path}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_INLINE | LFS_TYPE_REG, id, 0)}}}); + {lfs_mktag(LFS_STRUCT_INLINE, id, 0)}}}); if (err) { return err; } @@ -2022,9 +2035,9 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, cwd.pair[1] = cwd.tail[1]; } - attr.tag = lfs_mktag(LFS_STRUCT_INLINE | LFS_TYPE_REG, id, 0); + attr.tag = lfs_mktag(LFS_STRUCT_INLINE, id, 0); } else { - if (id == -1) { + if (type != LFS_TYPE_REG) { return LFS_ERR_ISDIR; } else if (flags & LFS_O_EXCL) { return LFS_ERR_EXIST; @@ -2035,10 +2048,6 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, if (err) { return err; } - - if (lfs_tag_subtype(attr.tag) != LFS_TYPE_REG) { - return LFS_ERR_ISDIR; - } } // setup file struct @@ -2267,7 +2276,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { int err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_CTZ | LFS_TYPE_REG, + {lfs_mktag(LFS_STRUCT_CTZ, file->id, 2*sizeof(uint32_t)), .u.buffer=&file->head}, file->attrs}); if (err) { @@ -2275,7 +2284,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } } else { int err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_INLINE | LFS_TYPE_REG, + {lfs_mktag(LFS_STRUCT_INLINE, file->id, file->size), .u.buffer=file->cache.buffer}, file->attrs}); if (err) { @@ -2660,7 +2669,8 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { lfs_mdir_t cwd; uint16_t id; - int err = lfs_dir_find(lfs, &cwd, &path, &id); + // TODO pass to getinfo? + int err = lfs_dir_find(lfs, &cwd, &path, &id, &(uint8_t){0}); if (err && err != LFS_ERR_ISDIR) { return err; } @@ -2691,22 +2701,22 @@ int lfs_remove(lfs_t *lfs, const char *path) { } uint16_t id; - err = lfs_dir_find(lfs, &cwd, &path, &id); - if (err) { - return err; - } - - // grab entry to see if we're dealing with a dir - lfs_mattr_t attr; - err = lfs_dir_getentry(lfs, &cwd, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); + uint8_t type; + err = lfs_dir_find(lfs, &cwd, &path, &id, &type); if (err) { return err; } lfs_mdir_t dir; - if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { + if (type == LFS_TYPE_DIR) { // must be empty before removal + lfs_mattr_t attr; + err = lfs_dir_getentry(lfs, &cwd, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); + if (err) { + return err; + } + err = lfs_dir_fetch(lfs, &dir, attr.u.pair); if (err) { return err; @@ -2716,38 +2726,6 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (dir.count > 0 || dir.split) { return LFS_ERR_NOTEMPTY; } -// -// // unlink from tail chain and create move to fix -// lfs->diff.move.pair[0] = cwd.pair[0] ^ lfs->globals.move.pair[0]; -// lfs->diff.move.pair[1] = cwd.pair[1] ^ lfs->globals.move.pair[1]; -// lfs->diff.move.id = id ^ lfs->globals.move.id; -// -// // xor over our child's global state -// lfs->diff = lfs_globals_xor(&lfs->diff, &dir.globals); -// lfs->globals = lfs_globals_xor(&lfs->globals, &dir.globals); -// -// // find pred and remove -// // TODO handle dropped block? -// lfs_mdir_t pred; -// int res = lfs_pred(lfs, dir.pair, &pred); -// if (res < 0) { -// return res; -// } -// -// LFS_ASSERT(res); // must have pred -// pred.tail[0] = dir.tail[0]; -// pred.tail[1] = dir.tail[1]; -// err = lfs_dir_commit(lfs, &pred, &(lfs_mattrlist_t){ -// {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(pred.tail)), -// .u.buffer=pred.tail}}); -// if (err) { -// return err; -// } -// -// // mark global state to clear move entry -// lfs->diff.move.pair[0] = 0xffffffff ^ lfs->globals.move.pair[0]; -// lfs->diff.move.pair[1] = 0xffffffff ^ lfs->globals.move.pair[1]; -// lfs->diff.move.id = 0x3ff ^ lfs->globals.move.id; } // delete the entry @@ -2756,15 +2734,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } -// // if we were a directory, fix the move we just created -// if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { -// err = lfs_deorphan(lfs); -// if (err) { -// return err; -// } -// } - - if (lfs_tag_subtype(attr.tag) == LFS_TYPE_DIR) { + if (type == LFS_TYPE_DIR) { int res = lfs_pred(lfs, dir.pair, &cwd); if (res < 0) { return res; @@ -2793,14 +2763,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find old entry lfs_mdir_t oldcwd; uint16_t oldid; - int err = lfs_dir_find(lfs, &oldcwd, &oldpath, &oldid); - if (err) { - return err; - } - - lfs_mattr_t oldattr; - err = lfs_dir_getentry(lfs, &oldcwd, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, oldid, 0), &oldattr); + uint8_t oldtype; + int err = lfs_dir_find(lfs, &oldcwd, &oldpath, &oldid, &oldtype); if (err) { return err; } @@ -2808,7 +2772,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find new entry lfs_mdir_t newcwd; uint16_t newid; - err = lfs_dir_find(lfs, &newcwd, &newpath, &newid); + uint8_t prevtype; + err = lfs_dir_find(lfs, &newcwd, &newpath, &newid, &prevtype); if (err && err != LFS_ERR_NOENT) { return err; } @@ -2816,21 +2781,22 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { bool prevexists = (err != LFS_ERR_NOENT); //bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); - lfs_mattr_t prevattr; lfs_mdir_t prevdir; if (prevexists) { - // get prev entry, check that we have same type - err = lfs_dir_getentry(lfs, &newcwd, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, newid, 0), &prevattr); - if (err) { - return err; - } - - if (lfs_tag_subtype(prevattr.tag) != lfs_tag_subtype(oldattr.tag)) { + // check that we have same type + if (prevtype != oldtype) { return LFS_ERR_ISDIR; } - if (lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { + if (prevtype == LFS_TYPE_DIR) { + // must be empty before removal + lfs_mattr_t prevattr; + err = lfs_dir_getentry(lfs, &newcwd, 0x703ff000, + lfs_mktag(LFS_SCOPE_STRUCT, newid, 0), &prevattr); + if (err) { + return err; + } + // must be empty before removal err = lfs_dir_fetch(lfs, &prevdir, prevattr.u.pair); if (err) { @@ -2862,8 +2828,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // move over all attributes err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_NAME | lfs_tag_subtype(oldattr.tag), - newid, strlen(newpath)), + {lfs_mktag(LFS_STRUCT_NAME | oldtype, newid, strlen(newpath)), .u.buffer=(void*)newpath}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_FROM_MOVE, newid, oldid), .u.dir=&oldcwd}}}); @@ -2877,7 +2842,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - if (prevexists && lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { + if (prevexists && prevtype == LFS_TYPE_DIR) { int res = lfs_pred(lfs, prevdir.pair, &newcwd); if (res < 0) { return res; @@ -3104,8 +3069,6 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // write one superblock lfs_superblock_t superblock = { - .root[0] = lfs->root[0], - .root[1] = lfs->root[1], .magic = {"littlefs"}, .version = LFS_DISK_VERSION, @@ -3118,8 +3081,10 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { dir.count += 1; err = lfs_dir_commit(lfs, &dir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SUPERBLOCK | LFS_STRUCT_DIR, - 0, sizeof(superblock)), .u.buffer=&superblock}}); + {lfs_mktag(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), + .u.buffer=&superblock}, &(lfs_mattrlist_t){ + {lfs_mktag(LFS_STRUCT_DIR, 0, sizeof(lfs->root)), + .u.buffer=lfs->root}}}); if (err) { return err; } @@ -3156,7 +3121,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_superblock_t superblock; - err = lfs_dir_getbuffer(lfs, &dir, 0x7e3ff000, &(lfs_mattr_t){ + err = lfs_dir_getbuffer(lfs, &dir, 0x7ffff000, &(lfs_mattr_t){ lfs_mktag(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), .u.buffer=&superblock}); if (err && err != LFS_ERR_RANGE) { @@ -3176,6 +3141,13 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return LFS_ERR_INVAL; } + err = lfs_dir_getbuffer(lfs, &dir, 0x7ffff000, &(lfs_mattr_t){ + lfs_mktag(LFS_STRUCT_DIR, 0, sizeof(lfs->root)), + .u.buffer=lfs->root}); + if (err) { + return err; + } + if (superblock.inline_size) { if (superblock.inline_size > lfs->inline_size) { LFS_ERROR("Unsupported inline size (%d > %d)", @@ -3206,9 +3178,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->name_size = superblock.name_size; } - lfs->root[0] = superblock.root[0]; - lfs->root[1] = superblock.root[1]; - err = lfs_scan(lfs); if (err) { return err; diff --git a/lfs.h b/lfs.h index 3ea3b5b0..55d72cfa 100644 --- a/lfs.h +++ b/lfs.h @@ -98,7 +98,7 @@ enum lfs_type { // file types LFS_TYPE_REG = 0x008, LFS_TYPE_DIR = 0x010, - LFS_TYPE_SUPERBLOCK = 0x038, + LFS_TYPE_SUPERBLOCK = 0x042, // internally used entry structures LFS_STRUCT_CTZ = 0x001, @@ -316,8 +316,8 @@ typedef struct lfs_mdir { uint16_t count; bool erased; bool split; - lfs_globals_t globals; bool stop_at_commit; // TODO hmmm + lfs_globals_t globals; } lfs_mdir_t; typedef struct lfs_cache { @@ -352,7 +352,6 @@ typedef struct lfs_dir { } lfs_dir_t; typedef struct lfs_superblock { - lfs_block_t root[2]; char magic[8]; uint32_t version; From b7bd34f4618fcaf4369cdf13fac743a6911684e7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 9 Jul 2018 14:13:31 -0500 Subject: [PATCH 051/139] Restructured types to use a more flexible bit encoding Recall that the 32-bit tag structure contains a 9-bit type. The type structure then decomposes into a bit more information: [--- 9 ---] [1|- 4 -|- 4 -] ^ ^ ^- specific type | \------- subtype \----------- user bit The main change is an observation from moving type info to the name tag from the struct tag. Since we don't need the type info in the struct tag, we can significantly simplify the type structure. --- lfs.c | 126 +++++++++++++++++++++++++++------------------------------- lfs.h | 48 +++++++++------------- 2 files changed, 78 insertions(+), 96 deletions(-) diff --git a/lfs.c b/lfs.c index c23d07f9..708ae36a 100644 --- a/lfs.c +++ b/lfs.c @@ -431,24 +431,24 @@ static inline lfs_tag_t lfs_mktag( return (type << 22) | (id << 12) | size; } -static inline bool lfs_tag_valid(lfs_tag_t tag) { +static inline bool lfs_tag_isvalid(lfs_tag_t tag) { return !(tag & 0x80000000); } -static inline uint16_t lfs_tag_type(lfs_tag_t tag) { - return (tag & 0x7fc00000) >> 22; +static inline bool lfs_tag_isuser(lfs_tag_t tag) { + return (tag & 0x40000000); } -static inline uint16_t lfs_tag_scope(lfs_tag_t tag) { - return (tag & 0x70000000) >> 22; +static inline bool lfs_tag_hasid(lfs_tag_t tag) { + return (tag & 0x60000000) != 0x20000000; } -static inline uint16_t lfs_tag_subtype(lfs_tag_t tag) { - return (tag & 0x7e000000) >> 22; +static inline uint16_t lfs_tag_type(lfs_tag_t tag) { + return (tag & 0x7fc00000) >> 22; } -static inline uint16_t lfs_tag_struct(lfs_tag_t tag) { - return (tag & 0x71c00000) >> 22; +static inline uint16_t lfs_tag_subtype(lfs_tag_t tag) { + return (tag & 0x7c000000) >> 22; } static inline uint16_t lfs_tag_id(lfs_tag_t tag) { @@ -498,7 +498,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, static int lfs_commit_commit(lfs_t *lfs, struct lfs_commit *commit, lfs_mattr_t attr) { // filter out ids - if (lfs_tag_scope(attr.tag) <= LFS_SCOPE_ENTRY && ( + if (lfs_tag_hasid(attr.tag) && ( lfs_tag_id(attr.tag) < commit->filter.begin || lfs_tag_id(attr.tag) >= commit->filter.end)) { return 0; @@ -650,21 +650,15 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_commit_move *move = p; - if (lfs_tag_struct(attr.tag) == LFS_STRUCT_DELETE && + if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE && lfs_tag_id(attr.tag) <= move->id.from) { // something was deleted, we need to move around it move->id.from += 1; return 0; } - // TODO AHHHH, scopes, what about user scope above? - if (lfs_tag_scope(attr.tag) == LFS_SCOPE_FS || - lfs_tag_scope(attr.tag) == LFS_SCOPE_DIR) { - // ignore non-matching ids - return 0; - } - - if (lfs_tag_id(attr.tag) != move->id.from) { + // TODO need this if 0x3ff is required? + if (!lfs_tag_hasid(attr.tag) || lfs_tag_id(attr.tag) != move->id.from) { // ignore non-matching ids return 0; } @@ -676,8 +670,7 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { .off=move->commit->off, .etag=move->commit->ptag, .stop_at_commit=true}, - (lfs_tag_scope(attr.tag) == LFS_SCOPE_STRUCT) ? // TODO also entry? - 0x7e3ff000 : 0x7ffff000, + lfs_tag_isuser(attr.tag) ? 0x7ffff000 : 0x7c3ff000, &(lfs_mattr_t){ lfs_mktag(lfs_tag_type(attr.tag), move->id.to - move->commit->filter.begin, 0)}); // TODO can all these filter adjustments be consolidated? @@ -728,7 +721,7 @@ static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, // TODO check performance/complexity of different strategies here lfs_globals_t res = lfs_globals_xor(source, diff); int err = lfs_commit_commit(lfs, commit, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_IDELETE, + lfs_mktag(LFS_TYPE_GLOBALS, res.move.id, sizeof(res.move.pair)), .u.buffer=res.move.pair}); if (err) { @@ -821,12 +814,12 @@ static int lfs_dir_fetchwith(lfs_t *lfs, tag = lfs_fromle32(tag) ^ ptag; // next commit not yet programmed - if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_valid(tag)) { + if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_isvalid(tag)) { // synthetic move if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 && cb) { int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_STRUCT_DELETE, + lfs_mktag(LFS_TYPE_DELETE, lfs->globals.move.id, 0)}); if (err) { return err; @@ -862,7 +855,7 @@ static int lfs_dir_fetchwith(lfs_t *lfs, if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 && cb) { int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_STRUCT_DELETE, + lfs_mktag(LFS_TYPE_DELETE, lfs->globals.move.id, 0)}); if (err) { return err; @@ -898,14 +891,14 @@ static int lfs_dir_fetchwith(lfs_t *lfs, return err; } - if (lfs_tag_struct(tag) == LFS_STRUCT_TAIL) { - temp.split = lfs_tag_type(tag) == LFS_TYPE_HARDTAIL; + if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { + temp.split = (lfs_tag_type(tag) & 1); err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), temp.tail, sizeof(temp.tail)); if (err) { return err; } - } else if (lfs_tag_type(tag) == LFS_TYPE_IDELETE) { + } else if (lfs_tag_type(tag) == LFS_TYPE_GLOBALS) { temp.globals.move.id = lfs_tag_id(tag); err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), &temp.globals.move.pair, @@ -914,12 +907,11 @@ static int lfs_dir_fetchwith(lfs_t *lfs, return err; } } else { - if (lfs_tag_scope(tag) <= LFS_SCOPE_ENTRY && - lfs_tag_id(tag) >= temp.count) { + if (lfs_tag_hasid(tag) && lfs_tag_id(tag) >= temp.count) { temp.count = lfs_tag_id(tag)+1; } - if (lfs_tag_struct(tag) == LFS_STRUCT_DELETE) { + if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { temp.count -= 1; } @@ -963,7 +955,7 @@ static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, // synthetic move if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_STRUCT_DELETE, lfs->globals.move.id, 0)}); + lfs_mktag(LFS_TYPE_DELETE, lfs->globals.move.id, 0)}); if (err) { return err; } @@ -1102,7 +1094,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, // commit tail, which may be new after last size check // TODO le32 err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ - lfs_mktag(LFS_STRUCT_TAIL + dir->split*0x8, + lfs_mktag(LFS_TYPE_TAIL + dir->split, 0x3ff, sizeof(dir->tail)), .u.buffer=dir->tail}); if (err) { @@ -1290,7 +1282,7 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { lfs->globals = lfs_globals_xor(&lfs->globals, &dir->globals); int err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_TAIL + pdir.split*0x8, + {lfs_mktag(LFS_TYPE_TAIL + pdir.split, 0x3ff, sizeof(pdir.tail)), .u.buffer=pdir.tail}}); return err; @@ -1298,7 +1290,7 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { } int err = lfs_dir_commit(lfs, dir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_DELETE, id, 0)}}); + {lfs_mktag(LFS_TYPE_DELETE, id, 0)}}); if (err) { return err; } @@ -1339,7 +1331,7 @@ static int lfs_dir_getscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { if ((attr.tag & get->mask) == (get->tag & get->mask)) { *get->attr = attr; return true; - } else if (lfs_tag_type(attr.tag) == LFS_STRUCT_DELETE) { + } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { if (lfs_tag_id(attr.tag) <= lfs_tag_id(get->tag)) { get->tag += lfs_mktag(0, 1, 0); } @@ -1403,26 +1395,26 @@ static int lfs_dir_getentry(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { lfs_mattr_t attr = { - lfs_mktag(LFS_STRUCT_NAME, id, lfs->name_size+1), + lfs_mktag(LFS_TYPE_NAME, id, lfs->name_size+1), .u.buffer=info->name, }; - int err = lfs_dir_getbuffer(lfs, dir, 0x71fff000, &attr); + int err = lfs_dir_getbuffer(lfs, dir, 0x7c3ff000, &attr); if (err) { return err; } - info->type = 0x38 & lfs_tag_subtype(attr.tag); + info->type = lfs_tag_type(attr.tag); - err = lfs_dir_getentry(lfs, dir, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); + err = lfs_dir_getentry(lfs, dir, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 0), &attr); if (err) { return err; } - if (lfs_tag_struct(attr.tag) == LFS_STRUCT_CTZ) { + if (lfs_tag_type(attr.tag) == LFS_STRUCT_CTZ) { info->size = attr.u.ctz.size; - } else if (lfs_tag_struct(attr.tag) == LFS_STRUCT_INLINE) { + } else if (lfs_tag_type(attr.tag) == LFS_STRUCT_INLINE) { info->size = lfs_tag_size(attr.tag); } @@ -1441,7 +1433,7 @@ struct lfs_dir_find { static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_dir_find *find = p; - if (lfs_tag_struct(attr.tag) == LFS_STRUCT_NAME && + if (lfs_tag_subtype(attr.tag) == LFS_TYPE_NAME && lfs_tag_size(attr.tag) == find->len) { int res = lfs_bd_cmp(lfs, attr.u.d.block, attr.u.d.off, find->name, find->len); @@ -1452,9 +1444,9 @@ static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { if (res) { // found a match find->tempid = lfs_tag_id(attr.tag); - find->tempfindtype = 0x38 & lfs_tag_subtype(attr.tag); + find->tempfindtype = lfs_tag_type(attr.tag); } - } else if (lfs_tag_type(attr.tag) == LFS_STRUCT_DELETE) { + } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { if (lfs_tag_id(attr.tag) == find->tempid) { find->tempid = -1; } else if (lfs_tag_id(attr.tag) < find->tempid) { @@ -1569,8 +1561,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, // TODO optimize grab for inline files and like? // TODO would this mean more code? // grab the entry data - int err = lfs_dir_getentry(lfs, dir, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, find.id, 0), &attr); + int err = lfs_dir_getentry(lfs, dir, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, find.id, 0), &attr); if (err) { return err; } @@ -1626,7 +1618,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_NAME | LFS_TYPE_DIR, id, nlen), + {lfs_mktag(LFS_TYPE_DIR, id, nlen), .u.buffer=(void*)path}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_STRUCT_DIR, id, sizeof(dir.pair)), .u.buffer=dir.pair}, &(lfs_mattrlist_t){ @@ -1657,8 +1649,8 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { attr.u.pair[1] = lfs->root[1]; } else { // get dir pair from parent - err = lfs_dir_getentry(lfs, &dir->m, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); + err = lfs_dir_getentry(lfs, &dir->m, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 0), &attr); if (err) { return err; } @@ -2020,7 +2012,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_NAME | LFS_TYPE_REG, id, nlen), + {lfs_mktag(LFS_TYPE_REG, id, nlen), .u.buffer=(void*)path}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_STRUCT_INLINE, id, 0)}}}); if (err) { @@ -2043,8 +2035,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_EXIST; } - attr.tag = lfs_mktag(LFS_SCOPE_STRUCT, id, 0); - err = lfs_dir_get(lfs, &cwd, 0x703ff000, &attr); + attr.tag = lfs_mktag(LFS_TYPE_STRUCT, id, 0); + err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, &attr); if (err) { return err; } @@ -2074,7 +2066,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } - if (lfs_tag_struct(attr.tag) == LFS_STRUCT_INLINE) { + if (lfs_tag_type(attr.tag) == LFS_STRUCT_INLINE) { // load inline files file->head = 0xfffffffe; file->size = lfs_tag_size(attr.tag); @@ -2711,8 +2703,8 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (type == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t attr; - err = lfs_dir_getentry(lfs, &cwd, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); + err = lfs_dir_getentry(lfs, &cwd, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 0), &attr); if (err) { return err; } @@ -2791,8 +2783,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (prevtype == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t prevattr; - err = lfs_dir_getentry(lfs, &newcwd, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, newid, 0), &prevattr); + err = lfs_dir_getentry(lfs, &newcwd, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, newid, 0), &prevattr); if (err) { return err; } @@ -2828,7 +2820,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // move over all attributes err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_NAME | oldtype, newid, strlen(newpath)), + {lfs_mktag(oldtype, newid, strlen(newpath)), .u.buffer=(void*)newpath}, &(lfs_mattrlist_t){ {lfs_mktag(LFS_FROM_MOVE, newid, oldid), .u.dir=&oldcwd}}}); @@ -3216,8 +3208,8 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { lfs_mattr_t attr; - int err = lfs_dir_getentry(lfs, &dir, 0x703ff000, - lfs_mktag(LFS_SCOPE_STRUCT, id, 0), &attr); + int err = lfs_dir_getentry(lfs, &dir, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 0), &attr); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -3225,7 +3217,7 @@ int lfs_fs_traverse(lfs_t *lfs, return err; } - if (lfs_tag_struct(attr.tag) == LFS_STRUCT_CTZ) { + if (lfs_tag_type(attr.tag) == LFS_STRUCT_CTZ) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, attr.u.ctz.head, attr.u.ctz.size, cb, data); if (err) { @@ -3384,7 +3376,7 @@ struct lfs_dir_parentscan { static int lfs_parentscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_dir_parentscan *parentscan = p; - if (lfs_tag_struct(attr.tag) == LFS_STRUCT_DIR) { + if (lfs_tag_type(attr.tag) == LFS_STRUCT_DIR) { int err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, &attr.u, sizeof(attr.u)); if (err) { @@ -3395,7 +3387,7 @@ static int lfs_parentscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { // found a match parentscan->tempid = lfs_tag_id(attr.tag); } - } else if (lfs_tag_struct(attr.tag) == LFS_STRUCT_DELETE) { + } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { if (lfs_tag_id(attr.tag) == parentscan->tempid) { parentscan->tempid = -1; } else if (lfs_tag_id(attr.tag) < parentscan->tempid) { @@ -3428,7 +3420,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], } if (parentscan.id != -1) { - int err = lfs_dir_getentry(lfs, parent, 0x71fff000, + int err = lfs_dir_getentry(lfs, parent, 0x7ffff000, lfs_mktag(LFS_STRUCT_DIR, parentscan.id, 0), attr); if (err) { return err; @@ -3553,7 +3545,7 @@ static int lfs_relocate(lfs_t *lfs, parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; int err = lfs_dir_commit(lfs, &parent, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_TAIL + parent.split*0x8, // TODO hm + {lfs_mktag(LFS_TYPE_TAIL + parent.split, // TODO hm 0x3ff, sizeof(lfs_block_t[2])), .u.pair[0]=newpair[0], .u.pair[1]=newpair[1]}}); if (err) { diff --git a/lfs.h b/lfs.h index 55d72cfa..cef224bc 100644 --- a/lfs.h +++ b/lfs.h @@ -96,39 +96,29 @@ enum lfs_error { // File types enum lfs_type { // file types - LFS_TYPE_REG = 0x008, - LFS_TYPE_DIR = 0x010, - LFS_TYPE_SUPERBLOCK = 0x042, - - // internally used entry structures - LFS_STRUCT_CTZ = 0x001, - LFS_STRUCT_DIR = 0x002, - LFS_STRUCT_INLINE = 0x003, - - LFS_STRUCT_NAME = 0x041, - LFS_STRUCT_DELETE = 0x047, - - LFS_STRUCT_TAIL = 0x081, - LFS_STRUCT_CRC = 0x087, - LFS_STRUCT_IDELETE = 0x0c1, + LFS_TYPE_REG = 0x001, + LFS_TYPE_DIR = 0x002, // internally used types - LFS_TYPE_SOFTTAIL = 0x081, - LFS_TYPE_HARDTAIL = 0x089, - LFS_TYPE_CRC = 0x087, - LFS_TYPE_IDELETE = 0x0c1, - - // internally used scopes - LFS_SCOPE_STRUCT = 0x000, - LFS_SCOPE_ENTRY = 0x040, - LFS_SCOPE_DIR = 0x080, - LFS_SCOPE_FS = 0x0c0, - LFS_SCOPE_USER = 0x100, - - // internal sources + LFS_TYPE_USER = 0x100, + LFS_TYPE_SUPERBLOCK = 0x010, + LFS_TYPE_NAME = 0x000, + LFS_TYPE_DELETE = 0x03f, + LFS_TYPE_STRUCT = 0x040, + LFS_TYPE_GLOBALS = 0x080, + LFS_TYPE_TAIL = 0x0c0, + LFS_TYPE_SOFTTAIL = 0x0c0, + LFS_TYPE_HARDTAIL = 0x0c1, + LFS_TYPE_CRC = 0x0ff, + + LFS_STRUCT_INLINE = 0x040, + LFS_STRUCT_CTZ = 0x041, + LFS_STRUCT_DIR = 0x042, + + // internal chip sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x03f, + LFS_FROM_MOVE = 0x030, }; // File open flags From fd121dc2e212c075bdc47f1c64374315bf4f8e1a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 9 Jul 2018 14:51:57 -0500 Subject: [PATCH 052/139] Dropped "has id" bit encoding in favor of invalid id I've been trying to keep tag types organized with an encoding that hints if a tag uses its id field for file ids. However this seem to have been a mistake. Using a null id of 0x3ff greatly simplified quite a bit of the logic around managing file related tags. The downside is one less id we can use, but if we look at the encoding cost, donating one full bit costs us 2^9 id permutations vs 1 id permutation. So even if we had a perfect encoding it's in our favor to use a null id. The cost of null ids is code size, but with the complexity around figuring out if a type used it's id or not it just works out better to use a null id. --- lfs.c | 25 +++++++++---------------- lfs.h | 2 +- tests/test_move.sh | 12 ++++++------ 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/lfs.c b/lfs.c index 708ae36a..8912407b 100644 --- a/lfs.c +++ b/lfs.c @@ -439,10 +439,6 @@ static inline bool lfs_tag_isuser(lfs_tag_t tag) { return (tag & 0x40000000); } -static inline bool lfs_tag_hasid(lfs_tag_t tag) { - return (tag & 0x60000000) != 0x20000000; -} - static inline uint16_t lfs_tag_type(lfs_tag_t tag) { return (tag & 0x7fc00000) >> 22; } @@ -498,14 +494,14 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, static int lfs_commit_commit(lfs_t *lfs, struct lfs_commit *commit, lfs_mattr_t attr) { // filter out ids - if (lfs_tag_hasid(attr.tag) && ( + if (lfs_tag_id(attr.tag) < 0x3ff && ( lfs_tag_id(attr.tag) < commit->filter.begin || lfs_tag_id(attr.tag) >= commit->filter.end)) { return 0; } // special cases - if (lfs_tag_type(attr.tag) == LFS_FROM_MOVE) { + if (lfs_tag_type(attr.tag) == LFS_FROM_DIR) { return lfs_commit_move(lfs, commit, lfs_tag_size(attr.tag), lfs_tag_id(attr.tag), attr.u.dir, NULL); @@ -657,8 +653,7 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { return 0; } - // TODO need this if 0x3ff is required? - if (!lfs_tag_hasid(attr.tag) || lfs_tag_id(attr.tag) != move->id.from) { + if (lfs_tag_id(attr.tag) != move->id.from) { // ignore non-matching ids return 0; } @@ -721,9 +716,8 @@ static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, // TODO check performance/complexity of different strategies here lfs_globals_t res = lfs_globals_xor(source, diff); int err = lfs_commit_commit(lfs, commit, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_GLOBALS, - res.move.id, sizeof(res.move.pair)), - .u.buffer=res.move.pair}); + lfs_mktag(LFS_TYPE_GLOBALS, 0x3ff, sizeof(res)), + .u.buffer=&res}); if (err) { return err; } @@ -899,15 +893,14 @@ static int lfs_dir_fetchwith(lfs_t *lfs, return err; } } else if (lfs_tag_type(tag) == LFS_TYPE_GLOBALS) { - temp.globals.move.id = lfs_tag_id(tag); err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), - &temp.globals.move.pair, - sizeof(temp.globals.move.pair)); + &temp.globals, sizeof(temp.globals)); if (err) { return err; } } else { - if (lfs_tag_hasid(tag) && lfs_tag_id(tag) >= temp.count) { + if (lfs_tag_id(tag) < 0x3ff && + lfs_tag_id(tag) >= temp.count) { temp.count = lfs_tag_id(tag)+1; } @@ -2822,7 +2815,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ {lfs_mktag(oldtype, newid, strlen(newpath)), .u.buffer=(void*)newpath}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_FROM_MOVE, newid, oldid), + {lfs_mktag(LFS_FROM_DIR, newid, oldid), .u.dir=&oldcwd}}}); if (err) { return err; diff --git a/lfs.h b/lfs.h index cef224bc..5c00a124 100644 --- a/lfs.h +++ b/lfs.h @@ -118,7 +118,7 @@ enum lfs_type { // internal chip sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x030, + LFS_FROM_DIR = 0x030, }; // File open flags diff --git a/tests/test_move.sh b/tests/test_move.sh index 5b851c5d..790706fd 100755 --- a/tests/test_move.sh +++ b/tests/test_move.sh @@ -59,7 +59,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-11 blocks/6 +truncate -s-7 blocks/6 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -86,8 +86,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hello", "d/hello") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-11 blocks/8 -truncate -s-11 blocks/a +truncate -s-7 blocks/8 +truncate -s-7 blocks/a tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; @@ -166,7 +166,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-11 blocks/7 +truncate -s-7 blocks/7 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -193,8 +193,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hi", "d/hi") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-11 blocks/9 -truncate -s-11 blocks/b +truncate -s-7 blocks/9 +truncate -s-7 blocks/b tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; From fe31f79b5fad04c0bf69f2592cbf48399bfba1f5 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 9 Jul 2018 17:29:40 -0500 Subject: [PATCH 053/139] Consolidated find/parent scanning functions An interesting observation about the find and parent scanning functions is that at their core, they're both actually doing the same operation. They search a metadata-pair during fetch for an entry uses the entry data instead of the entry tag. This means we can combine these functions and get a decent code savings. It's a little bit trickier because pair ordering isn't guaranteed. But to work around that we can simply search for both pair orderings. It's a bit more expensive but may be worth the code savings. A fancier implementation in the future can avoid the 2x lfs_parent scans. --- lfs.c | 294 +++++++++++++++++++--------------------------------------- 1 file changed, 94 insertions(+), 200 deletions(-) diff --git a/lfs.c b/lfs.c index 8912407b..2bc6fc61 100644 --- a/lfs.c +++ b/lfs.c @@ -1415,64 +1415,85 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, } struct lfs_dir_find { - const char *name; - uint16_t len; - int16_t id; - int16_t tempid; - uint8_t findtype; - uint8_t tempfindtype; + uint32_t mask; + lfs_tag_t tag; + const void *buffer; + lfs_tag_t foundtag; + lfs_tag_t temptag; }; static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_dir_find *find = p; - if (lfs_tag_subtype(attr.tag) == LFS_TYPE_NAME && - lfs_tag_size(attr.tag) == find->len) { + if ((attr.tag & find->mask) == (find->tag & find->mask)) { int res = lfs_bd_cmp(lfs, attr.u.d.block, attr.u.d.off, - find->name, find->len); + find->buffer, lfs_tag_size(attr.tag)); if (res < 0) { return res; } if (res) { // found a match - find->tempid = lfs_tag_id(attr.tag); - find->tempfindtype = lfs_tag_type(attr.tag); + find->temptag = attr.tag; } } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(attr.tag) == find->tempid) { - find->tempid = -1; - } else if (lfs_tag_id(attr.tag) < find->tempid) { - find->tempid -= 1; + if (lfs_tag_id(attr.tag) == lfs_tag_id(find->temptag)) { + find->temptag = 0xffffffff; + } else if (lfs_tag_id(find->temptag) < 0x3ff && + lfs_tag_id(attr.tag) < lfs_tag_id(find->temptag)) { + find->temptag -= lfs_mktag(0, 1, 0); } } else if (lfs_tag_type(attr.tag) == LFS_TYPE_CRC) { - find->id = find->tempid; - find->findtype = find->tempfindtype; + find->foundtag = find->temptag; } return 0; } +static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t *pair, + uint32_t mask, lfs_tag_t tag, + const void *buffer, lfs_tag_t *foundtag) { + struct lfs_dir_find find = { + .mask = mask, + .tag = tag, + .buffer = buffer, + .foundtag = 0xffffffff, + .temptag = 0xffffffff, + }; + + int err = lfs_dir_fetchwith(lfs, dir, pair, lfs_dir_findscan, &find); + if (err) { + return err; + } + + if (find.foundtag == 0xffffffff) { + return LFS_ERR_NOENT; + } + + *foundtag = find.foundtag; + return 0; +} + + // TODO drop others, make this only return id, also make get take in only entry to populate (with embedded tag) -static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, +static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path, uint16_t *id, uint8_t *type) { lfs_mattr_t attr = { .u.pair[0] = lfs->root[0], .u.pair[1] = lfs->root[1], }; - struct lfs_dir_find find = { - .name = *path, - }; + const char *name = *path; + lfs_size_t namelen; while (true) { nextname: // skip slashes - find.name += strspn(find.name, "/"); - find.len = strcspn(find.name, "/"); + name += strspn(name, "/"); + namelen = strcspn(name, "/"); // special case for root dir - if (find.name[0] == '\0') { + if (name[0] == '\0') { // Return ISDIR when we hit root // TODO change this to -1 or 0x3ff? *type = LFS_TYPE_DIR; @@ -1480,14 +1501,14 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, } // skip '.' and root '..' - if ((find.len == 1 && memcmp(find.name, ".", 1) == 0) || - (find.len == 2 && memcmp(find.name, "..", 2) == 0)) { - find.name += find.len; + if ((namelen == 1 && memcmp(name, ".", 1) == 0) || + (namelen == 2 && memcmp(name, "..", 2) == 0)) { + name += namelen; goto nextname; } // skip if matched by '..' in name - const char *suffix = find.name + find.len; + const char *suffix = name + namelen; lfs_size_t sufflen; int depth = 1; while (true) { @@ -1500,7 +1521,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { depth -= 1; if (depth == 0) { - find.name = suffix + sufflen; + name = suffix + sufflen; goto nextname; } } else { @@ -1511,21 +1532,22 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, } // update what we've found - *path = find.name; + *path = name; // find path while (true) { - //printf("checking %d %d for %s\n", attr.u.pair[0], attr.u.pair[1], *path); - find.id = -1; - find.tempid = -1; - int err = lfs_dir_fetchwith(lfs, dir, attr.u.pair, - lfs_dir_findscan, &find); - if (err) { + lfs_tag_t foundtag = -1; + int err = lfs_dir_find(lfs, dir, attr.u.pair, + 0x7c000fff, lfs_mktag(LFS_TYPE_NAME, 0, namelen), + name, &foundtag); + if (err && err != LFS_ERR_NOENT) { return err; } - if (find.id >= 0) { + if (err != LFS_ERR_NOENT) { // found it + *id = lfs_tag_id(foundtag); + *type = lfs_tag_type(foundtag); break; } @@ -1537,17 +1559,15 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, attr.u.pair[1] = dir->tail[1]; } - *id = find.id; - *type = find.findtype; - find.name += find.len; - find.name += strspn(find.name, "/"); - if (find.name[0] == '\0') { + name += namelen; + name += strspn(name, "/"); + if (name[0] == '\0') { return 0; } // don't continue on if we didn't hit a directory // TODO update with what's on master? - if (find.findtype != LFS_TYPE_DIR) { + if (*type != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } @@ -1555,7 +1575,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, // TODO would this mean more code? // grab the entry data int err = lfs_dir_getentry(lfs, dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, find.id, 0), &attr); + lfs_mktag(LFS_TYPE_STRUCT, *id, 0), &attr); if (err) { return err; } @@ -1573,7 +1593,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } lfs_mdir_t cwd; - int err = lfs_dir_find(lfs, &cwd, &path, &(uint16_t){0}, &(uint8_t){0}); + int err = lfs_dir_lookup(lfs, &cwd, &path, &(uint16_t){0}, &(uint8_t){0}); if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { if (!err || err == LFS_ERR_ISDIR) { return LFS_ERR_EXIST; @@ -1626,7 +1646,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { uint16_t id; uint8_t type; - int err = lfs_dir_find(lfs, &dir->m, &path, &id, &type); + int err = lfs_dir_lookup(lfs, &dir->m, &path, &id, &type); if (err && err != LFS_ERR_ISDIR) { return err; } @@ -1979,7 +1999,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, lfs_mdir_t cwd; uint16_t id; uint8_t type; - int err = lfs_dir_find(lfs, &cwd, &path, &id, &type); + int err = lfs_dir_lookup(lfs, &cwd, &path, &id, &type); if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) && err != LFS_ERR_ISDIR) { return err; @@ -2655,7 +2675,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { lfs_mdir_t cwd; uint16_t id; // TODO pass to getinfo? - int err = lfs_dir_find(lfs, &cwd, &path, &id, &(uint8_t){0}); + int err = lfs_dir_lookup(lfs, &cwd, &path, &id, &(uint8_t){0}); if (err && err != LFS_ERR_ISDIR) { return err; } @@ -2687,7 +2707,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { uint16_t id; uint8_t type; - err = lfs_dir_find(lfs, &cwd, &path, &id, &type); + err = lfs_dir_lookup(lfs, &cwd, &path, &id, &type); if (err) { return err; } @@ -2749,7 +2769,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_mdir_t oldcwd; uint16_t oldid; uint8_t oldtype; - int err = lfs_dir_find(lfs, &oldcwd, &oldpath, &oldid, &oldtype); + int err = lfs_dir_lookup(lfs, &oldcwd, &oldpath, &oldid, &oldtype); if (err) { return err; } @@ -2758,7 +2778,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_mdir_t newcwd; uint16_t newid; uint8_t prevtype; - err = lfs_dir_find(lfs, &newcwd, &newpath, &newid, &prevtype); + err = lfs_dir_lookup(lfs, &newcwd, &newpath, &newid, &prevtype); if (err && err != LFS_ERR_NOENT) { return err; } @@ -2883,7 +2903,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // } // // lfs_mattr_t entry; -// err = lfs_dir_find(lfs, &cwd, &entry, &path); +// err = lfs_dir_lookup(lfs, &cwd, &entry, &path); // if (err) { // return err; // } @@ -2900,7 +2920,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // } // // lfs_mattr_t entry; -// err = lfs_dir_find(lfs, &cwd, &entry, &path); +// err = lfs_dir_lookup(lfs, &cwd, &entry, &path); // if (err) { // return err; // } @@ -3331,168 +3351,42 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { return false; } -/* -static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir) { - if (lfs_pairisnull(lfs->root)) { - return 0; - } - - // iterate directories - int err = lfs_dir_fetch(lfs, pdir, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - while (!lfs_pairisnull(pdir->d.tail)) { - if (lfs_paircmp(pdir->d.tail, dir) == 0) { - return true; - } - - err = lfs_dir_fetch(lfs, pdir, pdir->d.tail); - if (err) { - return err; - } - } - - return false; -} -*/ - - -// TODO combine parentscan and findscan? -struct lfs_dir_parentscan { - lfs_block_t pair[2]; - int16_t id; - int16_t tempid; -}; - -static int lfs_parentscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { - struct lfs_dir_parentscan *parentscan = p; - - if (lfs_tag_type(attr.tag) == LFS_STRUCT_DIR) { - int err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, - &attr.u, sizeof(attr.u)); - if (err) { - return err; - } - - if (lfs_paircmp(attr.u.pair, parentscan->pair) == 0) { - // found a match - parentscan->tempid = lfs_tag_id(attr.tag); - } - } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(attr.tag) == parentscan->tempid) { - parentscan->tempid = -1; - } else if (lfs_tag_id(attr.tag) < parentscan->tempid) { - parentscan->tempid -= 1; - } - } else if (lfs_tag_type(attr.tag) == LFS_TYPE_CRC) { - parentscan->id = parentscan->tempid; - } - - return 0; -} static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *parent, lfs_mattr_t *attr) { - // iterate over all directory directory entries - parent->tail[0] = 0; - parent->tail[1] = 1; - while (!lfs_pairisnull(parent->tail)) { - struct lfs_dir_parentscan parentscan = { - .pair[0] = pair[0], - .pair[1] = pair[1], - .id = -1, - .tempid = -1, - }; - - int err = lfs_dir_fetchwith(lfs, parent, parent->tail, - lfs_parentscan, &parentscan); - if (err) { - return err; - } + // search for both orderings so we can reuse the find function + lfs_block_t child[2] = {pair[0], pair[1]}; - if (parentscan.id != -1) { - int err = lfs_dir_getentry(lfs, parent, 0x7ffff000, - lfs_mktag(LFS_STRUCT_DIR, parentscan.id, 0), attr); - if (err) { - return err; - } - - return true; - } - } - - return false; -} -// -//static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], -// lfs_mdir_t *parent, lfs_mattr_t *attr) { -// // iterate over all directory directory entries -// parent->tail[0] = 0; -// parent->tail[1] = 1; -// while (!lfs_pairisnull(parent->tail)) { -// int err = lfs_dir_fetch(lfs, parent, parent->tail); -// if (err) { -// return err; -// } -// -// // TODO make this O(n) by using fetchwith to match the pointers -// for (uint16_t id = 0; id < parent->count; id++) { -// int err = lfs_dir_getentry(lfs, parent, 0x43dff000, -// lfs_mktag(LFS_STRUCT_DIR, id, 0), attr); -// if (err) { -// if (err == LFS_ERR_NOENT) { -// continue; -// } -// return err; -// } -// -// if (lfs_paircmp(attr->u.pair, pair) == 0) { -// return true; -// } -// } -// } -// -// return false; -//} -/* -static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], - lfs_mdir_t *parent, lfs_mattr_t *attr) { - if (lfs_pairisnull(lfs->root)) { - return 0; - } - - parent->d.tail[0] = 0; - parent->d.tail[1] = 1; - - // iterate over all directory directory entries - while (!lfs_pairisnull(parent->d.tail)) { - int err = lfs_dir_fetch(lfs, parent, parent->d.tail); - if (err) { - return err; - } - - while (true) { - err = lfs_dir_next(lfs, parent, attr); + for (int i = 0; i < 2; i++) { + // iterate over all directory directory entries + parent->tail[0] = 0; + parent->tail[1] = 1; + while (!lfs_pairisnull(parent->tail)) { + lfs_tag_t foundtag = -1; + int err = lfs_dir_find(lfs, parent, parent->tail, + 0x7fc00fff, lfs_mktag(LFS_STRUCT_DIR, 0, sizeof(child)), + child, &foundtag); if (err && err != LFS_ERR_NOENT) { return err; } - if (err == LFS_ERR_NOENT) { - break; - } + if (err != LFS_ERR_NOENT) { + // found our parent + int err = lfs_dir_getentry(lfs, parent, + 0x7ffff000, foundtag, attr); + if (err) { + return err; + } - if (((0x70 & attr->d.type) == LFS_STRUCT_DIR) && - lfs_paircmp(attr->d.u.dir, dir) == 0) { return true; } } + + lfs_pairswap(child); } return false; } -*/ // TODO rename to lfs_dir_relocate? static int lfs_relocate(lfs_t *lfs, @@ -3725,7 +3619,7 @@ int lfs_deorphan(lfs_t *lfs) { // } // // lfs_mattr_t entry; -// err = lfs_dir_find(lfs, &cwd, &entry, &path); +// err = lfs_dir_lookup(lfs, &cwd, &entry, &path); // if (err) { // return err; // } From 7ad9700d9e10bdda2d7c8066ce724920fde67fd2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 10 Jul 2018 13:07:47 -0500 Subject: [PATCH 054/139] Integrated findscan into fetch as a built in side effect Now that littlefs's fetchwith operations have stabilized a bit, there's actually only a single fetchwith operation, the findscan function. Given that there's no need for the internal functions to be a forward compatible API, we can integrate the findscan behaviour directly into fetchwith and avoid the (annoyingly) costly generalization overhead. As an added benefit, we can easily add additional tag modifications during fetch, such as the synthetic moves needed to resolve in-flight move operations without disk modifications. --- lfs.c | 205 +++++++++++++++++++++------------------------------------- 1 file changed, 74 insertions(+), 131 deletions(-) diff --git a/lfs.c b/lfs.c index 2bc6fc61..65953b66 100644 --- a/lfs.c +++ b/lfs.c @@ -507,8 +507,11 @@ static int lfs_commit_commit(lfs_t *lfs, attr.u.dir, NULL); } - uint16_t id = lfs_tag_id(attr.tag) - commit->filter.begin; - attr.tag = lfs_mktag(0, id, 0) | (attr.tag & 0xffc00fff); + if (lfs_tag_id(attr.tag) != 0x3ff) { + // TODO eh? + uint16_t id = lfs_tag_id(attr.tag) - commit->filter.begin; + attr.tag = lfs_mktag(0, id, 0) | (attr.tag & 0xffc00fff); + } // check if we fit lfs_size_t size = lfs_tag_size(attr.tag); @@ -757,12 +760,14 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, return 0; } -static int lfs_dir_fetchwith(lfs_t *lfs, +static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2], - int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data) { + uint32_t findmask, lfs_tag_t findtag, + const void *findbuffer, lfs_tag_t *foundtag) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; dir->stop_at_commit = false; + *foundtag = 0xffffffff; // find the block with the most recent revision uint32_t rev[2]; @@ -794,12 +799,14 @@ static int lfs_dir_fetchwith(lfs_t *lfs, lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); dir->rev = lfs_fromle32(dir->rev); - lfs_mdir_t temp = *dir; + lfs_mdir_t tempdir = *dir; + lfs_tag_t tempfoundtag = *foundtag; while (true) { // extract next tag lfs_tag_t tag; - int err = lfs_bd_read(lfs, temp.pair[0], off, &tag, sizeof(tag)); + int err = lfs_bd_read(lfs, tempdir.pair[0], + off, &tag, sizeof(tag)); if (err) { return err; } @@ -809,19 +816,8 @@ static int lfs_dir_fetchwith(lfs_t *lfs, // next commit not yet programmed if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_isvalid(tag)) { - // synthetic move - if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 - && cb) { - int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_DELETE, - lfs->globals.move.id, 0)}); - if (err) { - return err; - } - } - dir->erased = true; - return 0; + goto done; } // check we're in valid range @@ -829,93 +825,75 @@ static int lfs_dir_fetchwith(lfs_t *lfs, break; } - //printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, temp.pair[0], off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); if (lfs_tag_type(tag) == LFS_TYPE_CRC) { // check the crc attr uint32_t dcrc; - int err = lfs_bd_read(lfs, temp.pair[0], + int err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { return err; } if (crc != lfs_fromle32(dcrc)) { - if (off == sizeof(temp.rev)) { + if (off == sizeof(tempdir.rev)) { // try other block break; } else { - // snythetic move - // TODO combine with above? - if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 - && cb) { - int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_DELETE, - lfs->globals.move.id, 0)}); - if (err) { - return err; - } - } - // consider what we have good enough dir->erased = false; - return 0; + goto done; } } - temp.off = off + sizeof(tag)+lfs_tag_size(tag); - temp.etag = tag; + tempdir.off = off + sizeof(tag)+lfs_tag_size(tag); + tempdir.etag = tag; crc = 0xffffffff; - *dir = temp; - - // TODO simplify this? - if (cb) { - err = cb(lfs, data, (lfs_mattr_t){ - (tag | 0x80000000), - .u.d.block=temp.pair[0], - .u.d.off=off+sizeof(tag)}); - if (err) { - return err; - } - } + *dir = tempdir; + *foundtag = tempfoundtag; } else { - // TODO crc before callback??? - err = lfs_bd_crc(lfs, temp.pair[0], + err = lfs_bd_crc(lfs, tempdir.pair[0], off+sizeof(tag), lfs_tag_size(tag), &crc); if (err) { return err; } + if (lfs_tag_id(tag) < 0x3ff && + lfs_tag_id(tag) >= tempdir.count) { + tempdir.count = lfs_tag_id(tag)+1; + } + if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { - temp.split = (lfs_tag_type(tag) & 1); - err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), - temp.tail, sizeof(temp.tail)); + tempdir.split = (lfs_tag_type(tag) & 1); + err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), + tempdir.tail, sizeof(tempdir.tail)); if (err) { return err; } } else if (lfs_tag_type(tag) == LFS_TYPE_GLOBALS) { - err = lfs_bd_read(lfs, temp.pair[0], off+sizeof(tag), - &temp.globals, sizeof(temp.globals)); + err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), + &tempdir.globals, sizeof(tempdir.globals)); if (err) { return err; } - } else { - if (lfs_tag_id(tag) < 0x3ff && - lfs_tag_id(tag) >= temp.count) { - temp.count = lfs_tag_id(tag)+1; + } else if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { + tempdir.count -= 1; + + if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { + tempfoundtag = 0xffffffff; + } else if (lfs_tag_id(tempfoundtag) < 0x3ff && + lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { + tempfoundtag -= lfs_mktag(0, 1, 0); } - - if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { - temp.count -= 1; + } else if ((tag & findmask) == (findtag & findmask)) { + int res = lfs_bd_cmp(lfs, tempdir.pair[0], off+sizeof(tag), + findbuffer, lfs_tag_size(tag)); + if (res < 0) { + return res; } - if (cb) { - err = cb(lfs, data, (lfs_mattr_t){ - (tag | 0x80000000), - .u.d.block=temp.pair[0], - .u.d.off=off+sizeof(tag)}); - if (err) { - return err; - } + if (res) { + // found a match + tempfoundtag = tag; } } } @@ -931,11 +909,34 @@ static int lfs_dir_fetchwith(lfs_t *lfs, LFS_ERROR("Corrupted dir pair at %d %d", dir->pair[0], dir->pair[1]); return LFS_ERR_CORRUPT; + +done: + // synthetic move + if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { + if (lfs->globals.move.id == lfs_tag_id(*foundtag)) { + *foundtag = 0xffffffff; + } else if (lfs_tag_id(*foundtag) < 0x3ff && + lfs->globals.move.id < lfs_tag_id(*foundtag)) { + *foundtag -= lfs_mktag(0, 1, 0); + } + } + + if (*foundtag == 0xffffffff) { + return LFS_ERR_NOENT; + } + + return 0; } static int lfs_dir_fetch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2]) { - return lfs_dir_fetchwith(lfs, dir, pair, NULL, NULL); + int err = lfs_dir_find(lfs, dir, pair, + 0xffffffff, 0xffffffff, NULL, &(lfs_tag_t){0}); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + return 0; } static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, @@ -1115,6 +1116,8 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, break; split: + // TODO update dirs that get split here? + // commit no longer fits, need to split dir, // drop caches and create tail lfs->pcache.block = 0xffffffff; @@ -1414,66 +1417,6 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, return 0; } -struct lfs_dir_find { - uint32_t mask; - lfs_tag_t tag; - const void *buffer; - lfs_tag_t foundtag; - lfs_tag_t temptag; -}; - -static int lfs_dir_findscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { - struct lfs_dir_find *find = p; - - if ((attr.tag & find->mask) == (find->tag & find->mask)) { - int res = lfs_bd_cmp(lfs, attr.u.d.block, attr.u.d.off, - find->buffer, lfs_tag_size(attr.tag)); - if (res < 0) { - return res; - } - - if (res) { - // found a match - find->temptag = attr.tag; - } - } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(attr.tag) == lfs_tag_id(find->temptag)) { - find->temptag = 0xffffffff; - } else if (lfs_tag_id(find->temptag) < 0x3ff && - lfs_tag_id(attr.tag) < lfs_tag_id(find->temptag)) { - find->temptag -= lfs_mktag(0, 1, 0); - } - } else if (lfs_tag_type(attr.tag) == LFS_TYPE_CRC) { - find->foundtag = find->temptag; - } - - return 0; -} - -static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t *pair, - uint32_t mask, lfs_tag_t tag, - const void *buffer, lfs_tag_t *foundtag) { - struct lfs_dir_find find = { - .mask = mask, - .tag = tag, - .buffer = buffer, - .foundtag = 0xffffffff, - .temptag = 0xffffffff, - }; - - int err = lfs_dir_fetchwith(lfs, dir, pair, lfs_dir_findscan, &find); - if (err) { - return err; - } - - if (find.foundtag == 0xffffffff) { - return LFS_ERR_NOENT; - } - - *foundtag = find.foundtag; - return 0; -} - // TODO drop others, make this only return id, also make get take in only entry to populate (with embedded tag) static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, From 67d9f88b0ec7c6b3844e7e9e7dc45f87e67de94a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 11 Jul 2018 05:52:50 -0500 Subject: [PATCH 055/139] Combined get functions into one Unfortunately, the three different sets of get functions were not contributing very much, and having three different get functions means we may be wasting code on redundant code paths. By dropping the user of the lfs_mattr_t struct in favor of a buffer, we can combine the three code paths with a bit of tweaking. --- lfs.c | 133 +++++++++++++++++++++++++++------------------------------- 1 file changed, 62 insertions(+), 71 deletions(-) diff --git a/lfs.c b/lfs.c index 65953b66..84c3220c 100644 --- a/lfs.c +++ b/lfs.c @@ -644,7 +644,8 @@ static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data); static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t mask, lfs_mattr_t *attr); + uint32_t getmask, lfs_tag_t gettag, + lfs_tag_t *gottag, void *getbuffer); static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { struct lfs_commit_move *move = p; @@ -669,9 +670,9 @@ static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { .etag=move->commit->ptag, .stop_at_commit=true}, lfs_tag_isuser(attr.tag) ? 0x7ffff000 : 0x7c3ff000, - &(lfs_mattr_t){ - lfs_mktag(lfs_tag_type(attr.tag), - move->id.to - move->commit->filter.begin, 0)}); // TODO can all these filter adjustments be consolidated? + lfs_mktag(lfs_tag_type(attr.tag), + move->id.to - move->commit->filter.begin, 0), // TODO can all these filter adjustments be consolidated? + NULL, NULL); if (err && err != LFS_ERR_NOENT) { return err; } @@ -1337,10 +1338,13 @@ static int lfs_dir_getscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { } static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t mask, lfs_mattr_t *attr) { - uint16_t id = lfs_tag_id(attr->tag); + uint32_t getmask, lfs_tag_t gettag, + lfs_tag_t *foundtag, void *buffer) { + uint16_t id = lfs_tag_id(gettag); + lfs_size_t size = lfs_tag_size(gettag); + lfs_mattr_t attr; int res = lfs_dir_traverse(lfs, dir, lfs_dir_getscan, - &(struct lfs_dir_get){mask, attr->tag, attr}); + &(struct lfs_dir_get){getmask, gettag, &attr}); if (res < 0) { return res; } @@ -1349,40 +1353,18 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, return LFS_ERR_NOENT; } - attr->tag = lfs_mktag(0, id, 0) | (attr->tag & 0xffc00fff); - return 0; -} - -static int lfs_dir_getbuffer(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t mask, lfs_mattr_t *attr) { - void *buffer = attr->u.buffer; - lfs_size_t size = lfs_tag_size(attr->tag); - int err = lfs_dir_get(lfs, dir, mask, attr); - if (err) { - return err; + if (foundtag) { + *foundtag = lfs_mktag(0, id, 0) | (attr.tag & 0xffc00fff); } - lfs_size_t diff = lfs_min(size, lfs_tag_size(attr->tag)); - memset((uint8_t*)buffer + diff, 0, size - diff); - err = lfs_bd_read(lfs, attr->u.d.block, attr->u.d.off, buffer, diff); - if (err) { - return err; - } - - if (lfs_tag_size(attr->tag) > size) { - return LFS_ERR_RANGE; - } - - return 0; -} - -static int lfs_dir_getentry(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t mask, lfs_tag_t tag, lfs_mattr_t *attr) { - attr->tag = tag | sizeof(attr->u); - attr->u.buffer = &attr->u; - int err = lfs_dir_getbuffer(lfs, dir, mask, attr); - if (err && err != LFS_ERR_RANGE) { - return err; + if (buffer) { + lfs_size_t diff = lfs_min(size, lfs_tag_size(attr.tag)); + memset((uint8_t*)buffer + diff, 0, size - diff); + int err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, + buffer, diff); + if (err) { + return err; + } } return 0; @@ -1390,20 +1372,22 @@ static int lfs_dir_getentry(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { - lfs_mattr_t attr = { - lfs_mktag(LFS_TYPE_NAME, id, lfs->name_size+1), - .u.buffer=info->name, - }; - - int err = lfs_dir_getbuffer(lfs, dir, 0x7c3ff000, &attr); + lfs_mattr_t attr; + int err = lfs_dir_get(lfs, dir, 0x7c3ff000, + lfs_mktag(LFS_TYPE_NAME, id, lfs->name_size+1), + &attr.tag, info->name); if (err) { return err; } info->type = lfs_tag_type(attr.tag); + if (lfs_tag_size(attr.tag) > lfs->name_size) { + return LFS_ERR_RANGE; + } - err = lfs_dir_getentry(lfs, dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 0), &attr); + err = lfs_dir_get(lfs, dir, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 8), + &attr.tag, &attr.u); if (err) { return err; } @@ -1517,8 +1501,9 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // TODO optimize grab for inline files and like? // TODO would this mean more code? // grab the entry data - int err = lfs_dir_getentry(lfs, dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, *id, 0), &attr); + int err = lfs_dir_get(lfs, dir, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, *id, 8), + &attr.tag, &attr.u); if (err) { return err; } @@ -1605,8 +1590,9 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { attr.u.pair[1] = lfs->root[1]; } else { // get dir pair from parent - err = lfs_dir_getentry(lfs, &dir->m, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 0), &attr); + err = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 8), + &attr.tag, &attr.u); if (err) { return err; } @@ -1991,8 +1977,11 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_EXIST; } - attr.tag = lfs_mktag(LFS_TYPE_STRUCT, id, 0); - err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, &attr); + // TODO allow no entry? + // TODO move this into one load? If cache >= 8 would work + err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 8), + &attr.tag, &file->head); if (err) { return err; } @@ -2023,6 +2012,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } if (lfs_tag_type(attr.tag) == LFS_STRUCT_INLINE) { + // TODO make inline the better path? + // TODO can inline and trunc be combined? // load inline files file->head = 0xfffffffe; file->size = lfs_tag_size(attr.tag); @@ -2038,10 +2029,6 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return err; } } - } else { - // use ctz list from entry - err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, - &file->head, 2*sizeof(uint32_t)); } // truncate if requested @@ -2053,7 +2040,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, file->head = 0xfffffffe; file->size = 0; file->flags |= LFS_F_INLINE; - file->cache.block = file->head; + file->cache.block = 0xfffffffe; file->cache.off = 0; } @@ -2659,8 +2646,9 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (type == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t attr; - err = lfs_dir_getentry(lfs, &cwd, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 0), &attr); + err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 8), + &attr.tag, &attr.u); if (err) { return err; } @@ -2739,8 +2727,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (prevtype == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t prevattr; - err = lfs_dir_getentry(lfs, &newcwd, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, newid, 0), &prevattr); + err = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, newid, 8), + &prevattr.tag, &prevattr.u); if (err) { return err; } @@ -3069,10 +3058,10 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_superblock_t superblock; - err = lfs_dir_getbuffer(lfs, &dir, 0x7ffff000, &(lfs_mattr_t){ + err = lfs_dir_get(lfs, &dir, 0x7ffff000, lfs_mktag(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), - .u.buffer=&superblock}); - if (err && err != LFS_ERR_RANGE) { + NULL, &superblock); + if (err) { return err; } @@ -3089,9 +3078,9 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return LFS_ERR_INVAL; } - err = lfs_dir_getbuffer(lfs, &dir, 0x7ffff000, &(lfs_mattr_t){ + err = lfs_dir_get(lfs, &dir, 0x7ffff000, lfs_mktag(LFS_STRUCT_DIR, 0, sizeof(lfs->root)), - .u.buffer=lfs->root}); + NULL, &lfs->root); if (err) { return err; } @@ -3164,8 +3153,9 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { lfs_mattr_t attr; - int err = lfs_dir_getentry(lfs, &dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 0), &attr); + int err = lfs_dir_get(lfs, &dir, 0x7c3ff000, + lfs_mktag(LFS_TYPE_STRUCT, id, 8), + &attr.tag, &attr.u); if (err) { if (err == LFS_ERR_NOENT) { continue; @@ -3315,8 +3305,9 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], if (err != LFS_ERR_NOENT) { // found our parent - int err = lfs_dir_getentry(lfs, parent, - 0x7ffff000, foundtag, attr); + int err = lfs_dir_get(lfs, parent, + 0x7ffff000, foundtag, + &attr->tag, &attr->u); if (err) { return err; } From e3b867897ab1f6eff39121591bd2af8ada4810a3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 11 Jul 2018 07:18:30 -0500 Subject: [PATCH 056/139] Modified results from find-like functions to use tags Tags offer all of the necessary info from the find functions (which makes sense, this is the structure that stores the info on disk). Passing around a single tag instead of separate id and type fields simplifies the internal functions while leverages the tag's compactness. --- lfs.c | 158 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 78 insertions(+), 80 deletions(-) diff --git a/lfs.c b/lfs.c index 84c3220c..058739c1 100644 --- a/lfs.c +++ b/lfs.c @@ -263,7 +263,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(lfs_t*, void*, lfs_block_t), void *data); static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], - lfs_mdir_t *parent, lfs_mattr_t *attr); + lfs_mdir_t *parent, lfs_tag_t *foundtag); static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); int lfs_scan(lfs_t *lfs); @@ -768,7 +768,7 @@ static int lfs_dir_find(lfs_t *lfs, dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; dir->stop_at_commit = false; - *foundtag = 0xffffffff; + lfs_tag_t localfoundtag = 0xffffffff; // find the block with the most recent revision uint32_t rev[2]; @@ -801,7 +801,7 @@ static int lfs_dir_find(lfs_t *lfs, dir->rev = lfs_fromle32(dir->rev); lfs_mdir_t tempdir = *dir; - lfs_tag_t tempfoundtag = *foundtag; + lfs_tag_t tempfoundtag = localfoundtag; while (true) { // extract next tag @@ -850,7 +850,7 @@ static int lfs_dir_find(lfs_t *lfs, tempdir.etag = tag; crc = 0xffffffff; *dir = tempdir; - *foundtag = tempfoundtag; + localfoundtag = tempfoundtag; } else { err = lfs_bd_crc(lfs, tempdir.pair[0], off+sizeof(tag), lfs_tag_size(tag), &crc); @@ -914,25 +914,29 @@ static int lfs_dir_find(lfs_t *lfs, done: // synthetic move if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { - if (lfs->globals.move.id == lfs_tag_id(*foundtag)) { - *foundtag = 0xffffffff; - } else if (lfs_tag_id(*foundtag) < 0x3ff && - lfs->globals.move.id < lfs_tag_id(*foundtag)) { - *foundtag -= lfs_mktag(0, 1, 0); + if (lfs->globals.move.id == lfs_tag_id(localfoundtag)) { + localfoundtag = 0xffffffff; + } else if (lfs_tag_id(localfoundtag) < 0x3ff && + lfs->globals.move.id < lfs_tag_id(localfoundtag)) { + localfoundtag -= lfs_mktag(0, 1, 0); } } - if (*foundtag == 0xffffffff) { + if (localfoundtag == 0xffffffff) { return LFS_ERR_NOENT; } + if (foundtag) { + *foundtag = localfoundtag; + } + return 0; } static int lfs_dir_fetch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2]) { int err = lfs_dir_find(lfs, dir, pair, - 0xffffffff, 0xffffffff, NULL, &(lfs_tag_t){0}); + 0xffffffff, 0xffffffff, NULL, NULL); if (err && err != LFS_ERR_NOENT) { return err; } @@ -1402,9 +1406,8 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, } -// TODO drop others, make this only return id, also make get take in only entry to populate (with embedded tag) static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, - const char **path, uint16_t *id, uint8_t *type) { + const char **path, lfs_tag_t *foundtag) { lfs_mattr_t attr = { .u.pair[0] = lfs->root[0], .u.pair[1] = lfs->root[1], @@ -1412,6 +1415,7 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char *name = *path; lfs_size_t namelen; + lfs_tag_t localfoundtag; while (true) { nextname: @@ -1423,7 +1427,9 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, if (name[0] == '\0') { // Return ISDIR when we hit root // TODO change this to -1 or 0x3ff? - *type = LFS_TYPE_DIR; + if (foundtag) { + *foundtag = lfs_mktag(LFS_TYPE_DIR, 0, 0); + } return LFS_ERR_ISDIR; } @@ -1463,18 +1469,15 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // find path while (true) { - lfs_tag_t foundtag = -1; int err = lfs_dir_find(lfs, dir, attr.u.pair, 0x7c000fff, lfs_mktag(LFS_TYPE_NAME, 0, namelen), - name, &foundtag); + name, &localfoundtag); if (err && err != LFS_ERR_NOENT) { return err; } if (err != LFS_ERR_NOENT) { // found it - *id = lfs_tag_id(foundtag); - *type = lfs_tag_type(foundtag); break; } @@ -1486,6 +1489,10 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, attr.u.pair[1] = dir->tail[1]; } + if (foundtag) { + *foundtag = localfoundtag; + } + name += namelen; name += strspn(name, "/"); if (name[0] == '\0') { @@ -1494,7 +1501,7 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // don't continue on if we didn't hit a directory // TODO update with what's on master? - if (*type != LFS_TYPE_DIR) { + if (lfs_tag_type(localfoundtag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } @@ -1502,7 +1509,7 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // TODO would this mean more code? // grab the entry data int err = lfs_dir_get(lfs, dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, *id, 8), + lfs_mktag(LFS_TYPE_STRUCT, lfs_tag_id(localfoundtag), 8), &attr.tag, &attr.u); if (err) { return err; @@ -1521,7 +1528,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } lfs_mdir_t cwd; - int err = lfs_dir_lookup(lfs, &cwd, &path, &(uint16_t){0}, &(uint8_t){0}); + int err = lfs_dir_lookup(lfs, &cwd, &path, NULL); if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { if (!err || err == LFS_ERR_ISDIR) { return LFS_ERR_EXIST; @@ -1572,14 +1579,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { - uint16_t id; - uint8_t type; - int err = lfs_dir_lookup(lfs, &dir->m, &path, &id, &type); + lfs_tag_t tag; + int err = lfs_dir_lookup(lfs, &dir->m, &path, &tag); if (err && err != LFS_ERR_ISDIR) { return err; } - if (type != LFS_TYPE_DIR) { + if (lfs_tag_type(tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } @@ -1591,7 +1597,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { } else { // get dir pair from parent err = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 8), + lfs_mktag(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), &attr.tag, &attr.u); if (err) { return err; @@ -1926,15 +1932,16 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // allocate entry for file if it doesn't exist lfs_mdir_t cwd; - uint16_t id; - uint8_t type; - int err = lfs_dir_lookup(lfs, &cwd, &path, &id, &type); + lfs_tag_t tag; + int err = lfs_dir_lookup(lfs, &cwd, &path, &tag); if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) && err != LFS_ERR_ISDIR) { return err; } + uint16_t id = lfs_tag_id(tag); + uint8_t type = lfs_tag_type(tag); - lfs_mattr_t attr; + lfs_mattr_t attr; // TODO stop copying things (attr, id, type, tag) if (err == LFS_ERR_NOENT) { if (!(flags & LFS_O_CREAT)) { return LFS_ERR_NOENT; @@ -2603,9 +2610,9 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { lfs_mdir_t cwd; - uint16_t id; + lfs_tag_t tag; // TODO pass to getinfo? - int err = lfs_dir_lookup(lfs, &cwd, &path, &id, &(uint8_t){0}); + int err = lfs_dir_lookup(lfs, &cwd, &path, &tag); if (err && err != LFS_ERR_ISDIR) { return err; } @@ -2617,7 +2624,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { return 0; } - return lfs_dir_getinfo(lfs, &cwd, id, info); + return lfs_dir_getinfo(lfs, &cwd, lfs_tag_id(tag), info); } int lfs_remove(lfs_t *lfs, const char *path) { @@ -2635,19 +2642,18 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } - uint16_t id; - uint8_t type; - err = lfs_dir_lookup(lfs, &cwd, &path, &id, &type); + lfs_tag_t tag; + err = lfs_dir_lookup(lfs, &cwd, &path, &tag); if (err) { return err; } lfs_mdir_t dir; - if (type == LFS_TYPE_DIR) { + if (lfs_tag_type(tag) == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t attr; err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 8), + lfs_mktag(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), &attr.tag, &attr.u); if (err) { return err; @@ -2665,12 +2671,12 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // delete the entry - err = lfs_dir_delete(lfs, &cwd, id); + err = lfs_dir_delete(lfs, &cwd, lfs_tag_id(tag)); if (err) { return err; } - if (type == LFS_TYPE_DIR) { + if (lfs_tag_type(tag) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, dir.pair, &cwd); if (res < 0) { return res; @@ -2698,33 +2704,32 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find old entry lfs_mdir_t oldcwd; - uint16_t oldid; - uint8_t oldtype; - int err = lfs_dir_lookup(lfs, &oldcwd, &oldpath, &oldid, &oldtype); + lfs_tag_t oldtag; + int err = lfs_dir_lookup(lfs, &oldcwd, &oldpath, &oldtag); if (err) { return err; } // find new entry lfs_mdir_t newcwd; - uint16_t newid; - uint8_t prevtype; - err = lfs_dir_lookup(lfs, &newcwd, &newpath, &newid, &prevtype); + lfs_tag_t prevtag; + err = lfs_dir_lookup(lfs, &newcwd, &newpath, &prevtag); if (err && err != LFS_ERR_NOENT) { return err; } + uint16_t newid = lfs_tag_id(prevtag); bool prevexists = (err != LFS_ERR_NOENT); //bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); lfs_mdir_t prevdir; if (prevexists) { // check that we have same type - if (prevtype != oldtype) { + if (lfs_tag_type(prevtag) != lfs_tag_type(oldtag)) { return LFS_ERR_ISDIR; } - if (prevtype == LFS_TYPE_DIR) { + if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t prevattr; err = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, @@ -2759,15 +2764,15 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // create move to fix later - lfs->diff.move.pair[0] = oldcwd.pair[0] ^ lfs->globals.move.pair[0]; - lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; - lfs->diff.move.id = oldid ^ lfs->globals.move.id; + lfs->diff.move.pair[0] = oldcwd.pair[0] ^ lfs->globals.move.pair[0]; + lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; + lfs->diff.move.id = lfs_tag_id(oldtag) ^ lfs->globals.move.id; // move over all attributes err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ - {lfs_mktag(oldtype, newid, strlen(newpath)), + {lfs_mktag(lfs_tag_type(oldtag), newid, strlen(newpath)), .u.buffer=(void*)newpath}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_FROM_DIR, newid, oldid), + {lfs_mktag(LFS_FROM_DIR, newid, lfs_tag_id(oldtag)), .u.dir=&oldcwd}}}); if (err) { return err; @@ -2779,7 +2784,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - if (prevexists && prevtype == LFS_TYPE_DIR) { + if (prevexists && lfs_tag_type(prevtag) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, prevdir.pair, &newcwd); if (res < 0) { return res; @@ -3286,7 +3291,7 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { } static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], - lfs_mdir_t *parent, lfs_mattr_t *attr) { + lfs_mdir_t *parent, lfs_tag_t *foundtag) { // search for both orderings so we can reuse the find function lfs_block_t child[2] = {pair[0], pair[1]}; @@ -3295,31 +3300,18 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], parent->tail[0] = 0; parent->tail[1] = 1; while (!lfs_pairisnull(parent->tail)) { - lfs_tag_t foundtag = -1; int err = lfs_dir_find(lfs, parent, parent->tail, 0x7fc00fff, lfs_mktag(LFS_STRUCT_DIR, 0, sizeof(child)), - child, &foundtag); - if (err && err != LFS_ERR_NOENT) { - return err; - } - + child, foundtag); if (err != LFS_ERR_NOENT) { - // found our parent - int err = lfs_dir_get(lfs, parent, - 0x7ffff000, foundtag, - &attr->tag, &attr->u); - if (err) { - return err; - } - - return true; + return err; } } lfs_pairswap(child); } - return false; + return LFS_ERR_NOENT; } // TODO rename to lfs_dir_relocate? @@ -3328,12 +3320,12 @@ static int lfs_relocate(lfs_t *lfs, // find parent lfs_mdir_t parent; lfs_mattr_t attr; - int res = lfs_parent(lfs, oldpair, &parent, &attr); - if (res < 0) { - return res; + int err = lfs_parent(lfs, oldpair, &parent, &attr.tag); + if (err && err != LFS_ERR_NOENT) { + return err; } - if (res) { + if (err != LFS_ERR_NOENT) { // update disk, this creates a desync attr.u.pair[0] = newpair[0]; attr.u.pair[1] = newpair[1]; @@ -3356,7 +3348,7 @@ static int lfs_relocate(lfs_t *lfs, } // find pred - res = lfs_pred(lfs, oldpair, &parent); + int res = lfs_pred(lfs, oldpair, &parent); if (res < 0) { return res; } @@ -3476,12 +3468,12 @@ int lfs_deorphan(lfs_t *lfs) { // check if we have a parent lfs_mdir_t parent; lfs_mattr_t attr; - int res = lfs_parent(lfs, pdir.tail, &parent, &attr); - if (res < 0) { - return res; + int err = lfs_parent(lfs, pdir.tail, &parent, &attr.tag); + if (err && err != LFS_ERR_NOENT) { + return err; } - if (!res) { + if (err == LFS_ERR_NOENT) { // we are an orphan LFS_DEBUG("Found orphan %d %d", pdir.tail[0], pdir.tail[1]); @@ -3499,6 +3491,12 @@ int lfs_deorphan(lfs_t *lfs) { break; } + err = lfs_dir_get(lfs, &parent, + 0x7ffff000, attr.tag, NULL, &attr.u); + if (err) { + return err; + } + if (!lfs_pairsync(attr.u.pair, pdir.tail)) { // we have desynced LFS_DEBUG("Found half-orphan %d %d", From 7c88bc96b682e73059866d20f865e275713a63ad Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 12 Jul 2018 18:11:18 -0500 Subject: [PATCH 057/139] Restructured get/traverse functions While it makes sense to reuse as many code paths as possible, it turns out that the logic behind the traversal of littlefs's metadata-pairs is so simple that it's actually cheaper to duplicate the traversal code where needed. This means instead of the code path move -> traverse -> movescan -> get -> traverse -> getscan, we can use the relatively flatter code path of move -> get. --- lfs.c | 284 ++++++++++++++++++++++------------------------------------ 1 file changed, 107 insertions(+), 177 deletions(-) diff --git a/lfs.c b/lfs.c index 058739c1..7e9ecde1 100644 --- a/lfs.c +++ b/lfs.c @@ -613,101 +613,75 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return 0; } -static int lfs_commit_list(lfs_t *lfs, struct lfs_commit *commit, - lfs_mattrlist_t *list) { - for (; list; list = list->next) { - int err = lfs_commit_commit(lfs, commit, list->e); - if (err) { - return err; - } - } - - return 0; -} - - -// committer for moves -// TODO rename? -struct lfs_commit_move { - lfs_mdir_t *dir; // TODO need dir? - struct { - uint16_t from; - uint16_t to; - } id; - - struct lfs_commit *commit; -}; - - -// TODO redeclare -static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, - int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), - void *data); +// TODO predeclare static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, lfs_tag_t gettag, - lfs_tag_t *gottag, void *getbuffer); + lfs_tag_t *foundtag, void *buffer); -static int lfs_commit_movescan(lfs_t *lfs, void *p, lfs_mattr_t attr) { - struct lfs_commit_move *move = p; - - if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE && - lfs_tag_id(attr.tag) <= move->id.from) { - // something was deleted, we need to move around it - move->id.from += 1; - return 0; - } +static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, + uint16_t fromid, uint16_t toid, + lfs_mdir_t *dir, lfs_mattrlist_t *list) { + // Iterate through list and commits, only committing unique entries + lfs_off_t off = dir->off + sizeof(uint32_t); + lfs_tag_t ntag = dir->etag; - if (lfs_tag_id(attr.tag) != move->id.from) { - // ignore non-matching ids - return 0; - } + while (list || off > 2*sizeof(uint32_t)) { + lfs_mattr_t attr; + if (list) { + attr = list->e; + list = list->next; + } else { + LFS_ASSERT(off > 2*sizeof(ntag)+lfs_tag_size(ntag)); + off -= sizeof(ntag)+lfs_tag_size(ntag); - // check if type has already been committed - int err = lfs_dir_get(lfs, - &(lfs_mdir_t){ - .pair[0]=move->commit->block, - .off=move->commit->off, - .etag=move->commit->ptag, - .stop_at_commit=true}, - lfs_tag_isuser(attr.tag) ? 0x7ffff000 : 0x7c3ff000, - lfs_mktag(lfs_tag_type(attr.tag), - move->id.to - move->commit->filter.begin, 0), // TODO can all these filter adjustments be consolidated? - NULL, NULL); - if (err && err != LFS_ERR_NOENT) { - return err; - } + attr.tag = ntag; + attr.u.d.block = dir->pair[0]; + attr.u.d.off = off; - if (err != LFS_ERR_NOENT) { - // already committed - return 0; - } + int err = lfs_bd_read(lfs, dir->pair[0], + off-sizeof(ntag), &ntag, sizeof(ntag)); + if (err) { + return err; + } - // update id and commit, as we are currently unique - attr.tag = lfs_mktag(0, move->id.to, 0) | (attr.tag & 0xffc00fff); - return lfs_commit_commit(lfs, move->commit, attr); -} + ntag = lfs_fromle32(ntag); + ntag ^= attr.tag; + attr.tag |= 0x80000000; + } -static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, - uint16_t fromid, uint16_t toid, - lfs_mdir_t *dir, lfs_mattrlist_t *list) { - struct lfs_commit_move move = { - .id.from = fromid, - .id.to = toid, - .commit = commit, - }; + if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE && + lfs_tag_id(attr.tag) <= fromid) { + // something was deleted, we need to move around it + fromid += 1; + } else if (lfs_tag_id(attr.tag) != fromid) { + // ignore non-matching ids + } else { + // check if type has already been committed + int err = lfs_dir_get(lfs, + &(lfs_mdir_t){ + .pair[0]=commit->block, + .off=commit->off, + .etag=commit->ptag, + .stop_at_commit=true}, + lfs_tag_isuser(attr.tag) ? 0x7ffff000 : 0x7c3ff000, + lfs_mktag(lfs_tag_type(attr.tag), + toid - commit->filter.begin, 0), // TODO can all these filter adjustments be consolidated? + NULL, NULL); + if (err && err != LFS_ERR_NOENT) { + return err; + } - for (; list; list = list->next) { - int err = lfs_commit_movescan(lfs, &move, list->e); - if (err) { - return err; + if (err == LFS_ERR_NOENT) { + // update id and commit, as we are currently unique + attr.tag = lfs_mktag(0, toid, 0) | (attr.tag & 0xffc00fff); + int err = lfs_commit_commit(lfs, commit, attr); + if (err) { + return err; + } + } } } - int err = lfs_dir_traverse(lfs, dir, lfs_commit_movescan, &move); - if (err) { - return err; - } - return 0; } @@ -944,56 +918,6 @@ static int lfs_dir_fetch(lfs_t *lfs, return 0; } -static int lfs_dir_traverse(lfs_t *lfs, lfs_mdir_t *dir, - int (*cb)(lfs_t *lfs, void *data, lfs_mattr_t attr), void *data) { - // iterate over dir block backwards (for faster lookups) - lfs_block_t block = dir->pair[0]; - lfs_off_t off = dir->off; - lfs_tag_t tag = dir->etag; - - // synthetic move - if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { - int err = cb(lfs, data, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_DELETE, lfs->globals.move.id, 0)}); - if (err) { - return err; - } - } - - while (off != sizeof(uint32_t)) { - // TODO rm me - //printf("tag r %#010x (%x:%x %03x %03x %03x)\n", tag, block, off-lfs_tag_size(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); - - // TODO hmm - if (lfs_tag_type(tag) == LFS_TYPE_CRC) { - if (dir->stop_at_commit) { - break; - } - } else { - int err = cb(lfs, data, (lfs_mattr_t){ - (0x80000000 | tag), - .u.d.block=block, - .u.d.off=off-lfs_tag_size(tag)}); - if (err) { - return err; - } - } - - LFS_ASSERT(off > sizeof(tag)+lfs_tag_size(tag)); - off -= sizeof(tag)+lfs_tag_size(tag); - - lfs_tag_t ntag; - int err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); - if (err) { - return err; - } - - tag ^= lfs_fromle32(ntag); - } - - return 0; -} - static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad @@ -1203,15 +1127,17 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { .filter.end = 0x3ff, }; - int err = lfs_commit_list(lfs, &commit, list); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - goto compact; + for (lfs_mattrlist_t *a = list; a; a = a->next) { + int err = lfs_commit_commit(lfs, &commit, a->e); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; } - return err; } - err = lfs_commit_globals(lfs, &commit, &dir->globals, &lfs->diff); + int err = lfs_commit_globals(lfs, &commit, &dir->globals, &lfs->diff); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1320,58 +1246,62 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { return 0; } -struct lfs_dir_get { - uint32_t mask; - lfs_tag_t tag; - lfs_mattr_t *attr; -}; - -static int lfs_dir_getscan(lfs_t *lfs, void *p, lfs_mattr_t attr) { - struct lfs_dir_get *get = p; - - if ((attr.tag & get->mask) == (get->tag & get->mask)) { - *get->attr = attr; - return true; - } else if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(attr.tag) <= lfs_tag_id(get->tag)) { - get->tag += lfs_mktag(0, 1, 0); - } - } - - return false; -} - static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, lfs_tag_t gettag, lfs_tag_t *foundtag, void *buffer) { uint16_t id = lfs_tag_id(gettag); lfs_size_t size = lfs_tag_size(gettag); - lfs_mattr_t attr; - int res = lfs_dir_traverse(lfs, dir, lfs_dir_getscan, - &(struct lfs_dir_get){getmask, gettag, &attr}); - if (res < 0) { - return res; - } - if (!res) { - return LFS_ERR_NOENT; + // synthetic moves + if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 + && lfs_tag_id(gettag) <= lfs->globals.move.id) { + gettag += lfs_mktag(0, 1, 0); } - if (foundtag) { - *foundtag = lfs_mktag(0, id, 0) | (attr.tag & 0xffc00fff); - } + // iterate over dir block backwards (for faster lookups) + lfs_off_t off = dir->off + sizeof(uint32_t); + lfs_tag_t tag = dir->etag; + + while (off > 2*sizeof(tag)) { + LFS_ASSERT(off > 2*sizeof(tag)+lfs_tag_size(tag)); + off -= sizeof(tag)+lfs_tag_size(tag); + + if (lfs_tag_type(tag) == LFS_TYPE_CRC) { + if (dir->stop_at_commit) { + break; + } + } else if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { + if (lfs_tag_id(tag) <= lfs_tag_id(gettag)) { + gettag += lfs_mktag(0, 1, 0); + } + } else if ((tag & getmask) == (gettag & getmask)) { + if (foundtag) { + *foundtag = lfs_mktag(0, id, 0) | (tag & 0xffc00fff); + } - if (buffer) { - lfs_size_t diff = lfs_min(size, lfs_tag_size(attr.tag)); - memset((uint8_t*)buffer + diff, 0, size - diff); - int err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, - buffer, diff); + if (buffer) { + lfs_size_t diff = lfs_min(size, lfs_tag_size(tag)); + int err = lfs_bd_read(lfs, dir->pair[0], off, buffer, diff); + if (err) { + return err; + } + + memset((uint8_t*)buffer + diff, 0, size - diff); + } + + return 0; + } + + lfs_tag_t ntag; + int err = lfs_bd_read(lfs, dir->pair[0], + off-sizeof(ntag), &ntag, sizeof(ntag)); if (err) { return err; } + tag ^= lfs_fromle32(ntag); } - return 0; + return LFS_ERR_NOENT; } static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, From 2b35c36b6734736b2549c234d50c267f197c3cae Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 12 Jul 2018 19:07:56 -0500 Subject: [PATCH 058/139] Renamed tag functions and macros - lfs_tagverb -> lfs_tag_verb - lfs_mktag -> LFS_MKTAG (it's a macro now) - LFS_STRUCT_THING -> LFS_THINGSTRUCT --- lfs.c | 319 +++++++++++++++++++++++++++++----------------------------- lfs.h | 38 +++---- 2 files changed, 178 insertions(+), 179 deletions(-) diff --git a/lfs.c b/lfs.c index 7e9ecde1..37e06eff 100644 --- a/lfs.c +++ b/lfs.c @@ -426,32 +426,34 @@ static inline bool lfs_pairsync( } /// Entry tag operations /// -static inline lfs_tag_t lfs_mktag( - uint16_t type, uint16_t id, lfs_size_t size) { - return (type << 22) | (id << 12) | size; -} +#define LFS_MKTAG(type, id, size) \ + (((uint32_t)(type) << 22) | ((uint32_t)(id) << 12) | (uint32_t)(size)) + +#define LFS_MKATTR(type, id, buffer_, size, next) \ + &(lfs_mattrlist_t){ \ + {LFS_MKTAG(type, id, size), .u.buffer=(void*)(buffer_)}, (next)} -static inline bool lfs_tag_isvalid(lfs_tag_t tag) { +static inline bool lfs_tagisvalid(lfs_tag_t tag) { return !(tag & 0x80000000); } -static inline bool lfs_tag_isuser(lfs_tag_t tag) { +static inline bool lfs_tagisuser(lfs_tag_t tag) { return (tag & 0x40000000); } -static inline uint16_t lfs_tag_type(lfs_tag_t tag) { +static inline uint16_t lfs_tagtype(lfs_tag_t tag) { return (tag & 0x7fc00000) >> 22; } -static inline uint16_t lfs_tag_subtype(lfs_tag_t tag) { +static inline uint16_t lfs_tagsubtype(lfs_tag_t tag) { return (tag & 0x7c000000) >> 22; } -static inline uint16_t lfs_tag_id(lfs_tag_t tag) { +static inline uint16_t lfs_tagid(lfs_tag_t tag) { return (tag & 0x003ff000) >> 12; } -static inline lfs_size_t lfs_tag_size(lfs_tag_t tag) { +static inline lfs_size_t lfs_tagsize(lfs_tag_t tag) { return tag & 0x00000fff; } @@ -494,34 +496,34 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, static int lfs_commit_commit(lfs_t *lfs, struct lfs_commit *commit, lfs_mattr_t attr) { // filter out ids - if (lfs_tag_id(attr.tag) < 0x3ff && ( - lfs_tag_id(attr.tag) < commit->filter.begin || - lfs_tag_id(attr.tag) >= commit->filter.end)) { + if (lfs_tagid(attr.tag) < 0x3ff && ( + lfs_tagid(attr.tag) < commit->filter.begin || + lfs_tagid(attr.tag) >= commit->filter.end)) { return 0; } // special cases - if (lfs_tag_type(attr.tag) == LFS_FROM_DIR) { + if (lfs_tagtype(attr.tag) == LFS_FROM_DIR) { return lfs_commit_move(lfs, commit, - lfs_tag_size(attr.tag), lfs_tag_id(attr.tag), + lfs_tagsize(attr.tag), lfs_tagid(attr.tag), attr.u.dir, NULL); } - if (lfs_tag_id(attr.tag) != 0x3ff) { + if (lfs_tagid(attr.tag) != 0x3ff) { // TODO eh? - uint16_t id = lfs_tag_id(attr.tag) - commit->filter.begin; - attr.tag = lfs_mktag(0, id, 0) | (attr.tag & 0xffc00fff); + uint16_t id = lfs_tagid(attr.tag) - commit->filter.begin; + attr.tag = LFS_MKTAG(0, id, 0) | (attr.tag & 0xffc00fff); } // check if we fit - lfs_size_t size = lfs_tag_size(attr.tag); + lfs_size_t size = lfs_tagsize(attr.tag); if (commit->off + sizeof(lfs_tag_t)+size > commit->end) { return LFS_ERR_NOSPC; } // write out tag // TODO rm me - //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", attr.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tag_type(attr.tag), lfs_tag_id(attr.tag), lfs_tag_size(attr.tag)); + //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", attr.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tagtype(attr.tag), lfs_tagid(attr.tag), lfs_tagsize(attr.tag)); lfs_tag_t tag = lfs_tole32((attr.tag & 0x7fffffff) ^ commit->ptag); lfs_crc(&commit->crc, &tag, sizeof(tag)); int err = lfs_bd_prog(lfs, commit->block, commit->off, &tag, sizeof(tag)); @@ -575,11 +577,11 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // build crc tag tag = (0x80000000 & ~lfs_fromle32(tag)) | - lfs_mktag(LFS_TYPE_CRC, 0x3ff, + LFS_MKTAG(LFS_TYPE_CRC, 0x3ff, noff - (commit->off+sizeof(uint32_t))); // write out crc - //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", tag, commit->block, commit->off+sizeof(tag), lfs_tag_type(tag), lfs_tag_id(tag), lfs_tag_size(tag)); + //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", tag, commit->block, commit->off+sizeof(tag), lfs_tagtype(tag), lfs_tagid(tag), lfs_tagsize(tag)); uint32_t footer[2]; footer[0] = lfs_tole32(tag ^ commit->ptag); lfs_crc(&commit->crc, &footer[0], sizeof(footer[0])); @@ -589,7 +591,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { if (err) { return err; } - commit->off += sizeof(tag)+lfs_tag_size(tag); + commit->off += sizeof(tag)+lfs_tagsize(tag); commit->ptag = tag; // flush buffers @@ -601,7 +603,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // successful commit, check checksum to make sure uint32_t crc = 0xffffffff; err = lfs_bd_crc(lfs, commit->block, commit->begin, - commit->off-lfs_tag_size(tag) - commit->begin, &crc); + commit->off-lfs_tagsize(tag) - commit->begin, &crc); if (err) { return err; } @@ -631,8 +633,8 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, attr = list->e; list = list->next; } else { - LFS_ASSERT(off > 2*sizeof(ntag)+lfs_tag_size(ntag)); - off -= sizeof(ntag)+lfs_tag_size(ntag); + LFS_ASSERT(off > 2*sizeof(ntag)+lfs_tagsize(ntag)); + off -= sizeof(ntag)+lfs_tagsize(ntag); attr.tag = ntag; attr.u.d.block = dir->pair[0]; @@ -649,11 +651,11 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, attr.tag |= 0x80000000; } - if (lfs_tag_type(attr.tag) == LFS_TYPE_DELETE && - lfs_tag_id(attr.tag) <= fromid) { + if (lfs_tagtype(attr.tag) == LFS_TYPE_DELETE && + lfs_tagid(attr.tag) <= fromid) { // something was deleted, we need to move around it fromid += 1; - } else if (lfs_tag_id(attr.tag) != fromid) { + } else if (lfs_tagid(attr.tag) != fromid) { // ignore non-matching ids } else { // check if type has already been committed @@ -663,8 +665,8 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, .off=commit->off, .etag=commit->ptag, .stop_at_commit=true}, - lfs_tag_isuser(attr.tag) ? 0x7ffff000 : 0x7c3ff000, - lfs_mktag(lfs_tag_type(attr.tag), + lfs_tagisuser(attr.tag) ? 0x7ffff000 : 0x7c3ff000, + LFS_MKTAG(lfs_tagtype(attr.tag), toid - commit->filter.begin, 0), // TODO can all these filter adjustments be consolidated? NULL, NULL); if (err && err != LFS_ERR_NOENT) { @@ -673,7 +675,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, if (err == LFS_ERR_NOENT) { // update id and commit, as we are currently unique - attr.tag = lfs_mktag(0, toid, 0) | (attr.tag & 0xffc00fff); + attr.tag = LFS_MKTAG(0, toid, 0) | (attr.tag & 0xffc00fff); int err = lfs_commit_commit(lfs, commit, attr); if (err) { return err; @@ -694,7 +696,7 @@ static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, // TODO check performance/complexity of different strategies here lfs_globals_t res = lfs_globals_xor(source, diff); int err = lfs_commit_commit(lfs, commit, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_GLOBALS, 0x3ff, sizeof(res)), + LFS_MKTAG(LFS_TYPE_GLOBALS, 0x3ff, sizeof(res)), .u.buffer=&res}); if (err) { return err; @@ -790,17 +792,17 @@ static int lfs_dir_find(lfs_t *lfs, tag = lfs_fromle32(tag) ^ ptag; // next commit not yet programmed - if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_isvalid(tag)) { + if (lfs_tagtype(ptag) == LFS_TYPE_CRC && !lfs_tagisvalid(tag)) { dir->erased = true; goto done; } // check we're in valid range - if (off + sizeof(tag)+lfs_tag_size(tag) > lfs->cfg->block_size) { + if (off + sizeof(tag)+lfs_tagsize(tag) > lfs->cfg->block_size) { break; } - if (lfs_tag_type(tag) == LFS_TYPE_CRC) { + if (lfs_tagtype(tag) == LFS_TYPE_CRC) { // check the crc attr uint32_t dcrc; int err = lfs_bd_read(lfs, tempdir.pair[0], @@ -820,48 +822,48 @@ static int lfs_dir_find(lfs_t *lfs, } } - tempdir.off = off + sizeof(tag)+lfs_tag_size(tag); + tempdir.off = off + sizeof(tag)+lfs_tagsize(tag); tempdir.etag = tag; crc = 0xffffffff; *dir = tempdir; localfoundtag = tempfoundtag; } else { err = lfs_bd_crc(lfs, tempdir.pair[0], - off+sizeof(tag), lfs_tag_size(tag), &crc); + off+sizeof(tag), lfs_tagsize(tag), &crc); if (err) { return err; } - if (lfs_tag_id(tag) < 0x3ff && - lfs_tag_id(tag) >= tempdir.count) { - tempdir.count = lfs_tag_id(tag)+1; + if (lfs_tagid(tag) < 0x3ff && + lfs_tagid(tag) >= tempdir.count) { + tempdir.count = lfs_tagid(tag)+1; } - if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { - tempdir.split = (lfs_tag_type(tag) & 1); + if (lfs_tagsubtype(tag) == LFS_TYPE_TAIL) { + tempdir.split = (lfs_tagtype(tag) & 1); err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), tempdir.tail, sizeof(tempdir.tail)); if (err) { return err; } - } else if (lfs_tag_type(tag) == LFS_TYPE_GLOBALS) { + } else if (lfs_tagtype(tag) == LFS_TYPE_GLOBALS) { err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), &tempdir.globals, sizeof(tempdir.globals)); if (err) { return err; } - } else if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { + } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { tempdir.count -= 1; - if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { + if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { tempfoundtag = 0xffffffff; - } else if (lfs_tag_id(tempfoundtag) < 0x3ff && - lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { - tempfoundtag -= lfs_mktag(0, 1, 0); + } else if (lfs_tagid(tempfoundtag) < 0x3ff && + lfs_tagid(tag) < lfs_tagid(tempfoundtag)) { + tempfoundtag -= LFS_MKTAG(0, 1, 0); } } else if ((tag & findmask) == (findtag & findmask)) { int res = lfs_bd_cmp(lfs, tempdir.pair[0], off+sizeof(tag), - findbuffer, lfs_tag_size(tag)); + findbuffer, lfs_tagsize(tag)); if (res < 0) { return res; } @@ -874,7 +876,7 @@ static int lfs_dir_find(lfs_t *lfs, } ptag = tag; - off += sizeof(tag)+lfs_tag_size(tag); + off += sizeof(tag)+lfs_tagsize(tag); } // failed, try the other crc? @@ -888,11 +890,11 @@ static int lfs_dir_find(lfs_t *lfs, done: // synthetic move if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { - if (lfs->globals.move.id == lfs_tag_id(localfoundtag)) { + if (lfs->globals.move.id == lfs_tagid(localfoundtag)) { localfoundtag = 0xffffffff; - } else if (lfs_tag_id(localfoundtag) < 0x3ff && - lfs->globals.move.id < lfs_tag_id(localfoundtag)) { - localfoundtag -= lfs_mktag(0, 1, 0); + } else if (lfs_tagid(localfoundtag) < 0x3ff && + lfs->globals.move.id < lfs_tagid(localfoundtag)) { + localfoundtag -= LFS_MKTAG(0, 1, 0); } } @@ -1017,7 +1019,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, // commit tail, which may be new after last size check // TODO le32 err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ - lfs_mktag(LFS_TYPE_TAIL + dir->split, + LFS_MKTAG(LFS_TYPE_TAIL + dir->split, 0x3ff, sizeof(dir->tail)), .u.buffer=dir->tail}); if (err) { @@ -1208,16 +1210,14 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { lfs->diff = dir->globals; lfs->globals = lfs_globals_xor(&lfs->globals, &dir->globals); - int err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_TAIL + pdir.split, - 0x3ff, sizeof(pdir.tail)), - .u.buffer=pdir.tail}}); - return err; + return lfs_dir_commit(lfs, &pdir, + LFS_MKATTR(LFS_TYPE_TAIL + pdir.split, 0x3ff, + pdir.tail, sizeof(pdir.tail), NULL)); } } - int err = lfs_dir_commit(lfs, dir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_DELETE, id, 0)}}); + int err = lfs_dir_commit(lfs, dir, + LFS_MKATTR(LFS_TYPE_DELETE, id, NULL, 0, NULL)); if (err) { return err; } @@ -1249,13 +1249,13 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, lfs_tag_t gettag, lfs_tag_t *foundtag, void *buffer) { - uint16_t id = lfs_tag_id(gettag); - lfs_size_t size = lfs_tag_size(gettag); + uint16_t id = lfs_tagid(gettag); + lfs_size_t size = lfs_tagsize(gettag); // synthetic moves if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 - && lfs_tag_id(gettag) <= lfs->globals.move.id) { - gettag += lfs_mktag(0, 1, 0); + && lfs_tagid(gettag) <= lfs->globals.move.id) { + gettag += LFS_MKTAG(0, 1, 0); } // iterate over dir block backwards (for faster lookups) @@ -1263,24 +1263,24 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, lfs_tag_t tag = dir->etag; while (off > 2*sizeof(tag)) { - LFS_ASSERT(off > 2*sizeof(tag)+lfs_tag_size(tag)); - off -= sizeof(tag)+lfs_tag_size(tag); + LFS_ASSERT(off > 2*sizeof(tag)+lfs_tagsize(tag)); + off -= sizeof(tag)+lfs_tagsize(tag); - if (lfs_tag_type(tag) == LFS_TYPE_CRC) { + if (lfs_tagtype(tag) == LFS_TYPE_CRC) { if (dir->stop_at_commit) { break; } - } else if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(tag) <= lfs_tag_id(gettag)) { - gettag += lfs_mktag(0, 1, 0); + } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { + if (lfs_tagid(tag) <= lfs_tagid(gettag)) { + gettag += LFS_MKTAG(0, 1, 0); } } else if ((tag & getmask) == (gettag & getmask)) { if (foundtag) { - *foundtag = lfs_mktag(0, id, 0) | (tag & 0xffc00fff); + *foundtag = LFS_MKTAG(0, id, 0) | (tag & 0xffc00fff); } if (buffer) { - lfs_size_t diff = lfs_min(size, lfs_tag_size(tag)); + lfs_size_t diff = lfs_min(size, lfs_tagsize(tag)); int err = lfs_bd_read(lfs, dir->pair[0], off, buffer, diff); if (err) { return err; @@ -1308,28 +1308,28 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { lfs_mattr_t attr; int err = lfs_dir_get(lfs, dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_NAME, id, lfs->name_size+1), + LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_size+1), &attr.tag, info->name); if (err) { return err; } - info->type = lfs_tag_type(attr.tag); - if (lfs_tag_size(attr.tag) > lfs->name_size) { + info->type = lfs_tagtype(attr.tag); + if (lfs_tagsize(attr.tag) > lfs->name_size) { return LFS_ERR_RANGE; } err = lfs_dir_get(lfs, dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 8), + LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &attr.tag, &attr.u); if (err) { return err; } - if (lfs_tag_type(attr.tag) == LFS_STRUCT_CTZ) { + if (lfs_tagtype(attr.tag) == LFS_TYPE_CTZSTRUCT) { info->size = attr.u.ctz.size; - } else if (lfs_tag_type(attr.tag) == LFS_STRUCT_INLINE) { - info->size = lfs_tag_size(attr.tag); + } else if (lfs_tagtype(attr.tag) == LFS_TYPE_INLINESTRUCT) { + info->size = lfs_tagsize(attr.tag); } return 0; @@ -1358,7 +1358,7 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // Return ISDIR when we hit root // TODO change this to -1 or 0x3ff? if (foundtag) { - *foundtag = lfs_mktag(LFS_TYPE_DIR, 0, 0); + *foundtag = LFS_MKTAG(LFS_TYPE_DIR, 0, 0); } return LFS_ERR_ISDIR; } @@ -1400,7 +1400,7 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // find path while (true) { int err = lfs_dir_find(lfs, dir, attr.u.pair, - 0x7c000fff, lfs_mktag(LFS_TYPE_NAME, 0, namelen), + 0x7c000fff, LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), name, &localfoundtag); if (err && err != LFS_ERR_NOENT) { return err; @@ -1431,7 +1431,7 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // don't continue on if we didn't hit a directory // TODO update with what's on master? - if (lfs_tag_type(localfoundtag) != LFS_TYPE_DIR) { + if (lfs_tagtype(localfoundtag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } @@ -1439,7 +1439,7 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // TODO would this mean more code? // grab the entry data int err = lfs_dir_get(lfs, dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, lfs_tag_id(localfoundtag), 8), + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(localfoundtag), 8), &attr.tag, &attr.u); if (err) { return err; @@ -1495,13 +1495,14 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; - err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_DIR, id, nlen), - .u.buffer=(void*)path}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_DIR, id, sizeof(dir.pair)), - .u.buffer=dir.pair}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(cwd.tail)), - .u.buffer=cwd.tail}}}}); + err = lfs_dir_commit(lfs, &cwd, + LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, + LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), + NULL)))); + if (err) { + return err; + } // TODO need ack here? lfs_alloc_ack(lfs); @@ -1515,7 +1516,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { return err; } - if (lfs_tag_type(tag) != LFS_TYPE_DIR) { + if (lfs_tagtype(tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } @@ -1527,7 +1528,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { } else { // get dir pair from parent err = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &attr.tag, &attr.u); if (err) { return err; @@ -1868,8 +1869,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, err != LFS_ERR_ISDIR) { return err; } - uint16_t id = lfs_tag_id(tag); - uint8_t type = lfs_tag_type(tag); + uint16_t id = lfs_tagid(tag); + uint8_t type = lfs_tagtype(tag); lfs_mattr_t attr; // TODO stop copying things (attr, id, type, tag) if (err == LFS_ERR_NOENT) { @@ -1890,10 +1891,9 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change - err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_REG, id, nlen), - .u.buffer=(void*)path}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_INLINE, id, 0)}}}); + err = lfs_dir_commit(lfs, &cwd, + LFS_MKATTR(LFS_TYPE_REG, id, path, nlen, + LFS_MKATTR(LFS_TYPE_INLINESTRUCT, id, NULL, 0, NULL))); if (err) { return err; } @@ -1906,7 +1906,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, cwd.pair[1] = cwd.tail[1]; } - attr.tag = lfs_mktag(LFS_STRUCT_INLINE, id, 0); + attr.tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, id, 0); } else { if (type != LFS_TYPE_REG) { return LFS_ERR_ISDIR; @@ -1917,7 +1917,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // TODO allow no entry? // TODO move this into one load? If cache >= 8 would work err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 8), + LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &attr.tag, &file->head); if (err) { return err; @@ -1948,12 +1948,12 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } - if (lfs_tag_type(attr.tag) == LFS_STRUCT_INLINE) { + if (lfs_tagtype(attr.tag) == LFS_TYPE_INLINESTRUCT) { // TODO make inline the better path? // TODO can inline and trunc be combined? // load inline files file->head = 0xfffffffe; - file->size = lfs_tag_size(attr.tag); + file->size = lfs_tagsize(attr.tag); file->flags |= LFS_F_INLINE; file->cache.block = file->head; file->cache.off = 0; @@ -2147,18 +2147,16 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { - int err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_CTZ, - file->id, 2*sizeof(uint32_t)), .u.buffer=&file->head}, - file->attrs}); + int err = lfs_dir_commit(lfs, &cwd, + LFS_MKATTR(LFS_TYPE_CTZSTRUCT, file->id, + &file->head, 2*sizeof(uint32_t), file->attrs)); if (err) { return err; } } else { - int err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_INLINE, - file->id, file->size), .u.buffer=file->cache.buffer}, - file->attrs}); + int err = lfs_dir_commit(lfs, &cwd, + LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, + file->cache.buffer, file->size, file->attrs)); if (err) { return err; } @@ -2554,7 +2552,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { return 0; } - return lfs_dir_getinfo(lfs, &cwd, lfs_tag_id(tag), info); + return lfs_dir_getinfo(lfs, &cwd, lfs_tagid(tag), info); } int lfs_remove(lfs_t *lfs, const char *path) { @@ -2579,11 +2577,11 @@ int lfs_remove(lfs_t *lfs, const char *path) { } lfs_mdir_t dir; - if (lfs_tag_type(tag) == LFS_TYPE_DIR) { + if (lfs_tagtype(tag) == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t attr; err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &attr.tag, &attr.u); if (err) { return err; @@ -2601,12 +2599,12 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // delete the entry - err = lfs_dir_delete(lfs, &cwd, lfs_tag_id(tag)); + err = lfs_dir_delete(lfs, &cwd, lfs_tagid(tag)); if (err) { return err; } - if (lfs_tag_type(tag) == LFS_TYPE_DIR) { + if (lfs_tagtype(tag) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, dir.pair, &cwd); if (res < 0) { return res; @@ -2615,9 +2613,12 @@ int lfs_remove(lfs_t *lfs, const char *path) { LFS_ASSERT(res); // must have pred cwd.tail[0] = dir.tail[0]; cwd.tail[1] = dir.tail[1]; - err = lfs_dir_commit(lfs, &cwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(cwd.tail)), - .u.buffer=cwd.tail}}); + err = lfs_dir_commit(lfs, &cwd, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, + cwd.tail, sizeof(cwd.tail), NULL)); + if (err) { + return err; + } } return 0; @@ -2648,22 +2649,22 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - uint16_t newid = lfs_tag_id(prevtag); + uint16_t newid = lfs_tagid(prevtag); bool prevexists = (err != LFS_ERR_NOENT); //bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); lfs_mdir_t prevdir; if (prevexists) { // check that we have same type - if (lfs_tag_type(prevtag) != lfs_tag_type(oldtag)) { + if (lfs_tagtype(prevtag) != lfs_tagtype(oldtag)) { return LFS_ERR_ISDIR; } - if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { + if (lfs_tagtype(prevtag) == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t prevattr; err = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, newid, 8), + LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), &prevattr.tag, &prevattr.u); if (err) { return err; @@ -2696,14 +2697,13 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // create move to fix later lfs->diff.move.pair[0] = oldcwd.pair[0] ^ lfs->globals.move.pair[0]; lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; - lfs->diff.move.id = lfs_tag_id(oldtag) ^ lfs->globals.move.id; + lfs->diff.move.id = lfs_tagid(oldtag) ^ lfs->globals.move.id; // move over all attributes - err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ - {lfs_mktag(lfs_tag_type(oldtag), newid, strlen(newpath)), - .u.buffer=(void*)newpath}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_FROM_DIR, newid, lfs_tag_id(oldtag)), - .u.dir=&oldcwd}}}); + err = lfs_dir_commit(lfs, &newcwd, + LFS_MKATTR(lfs_tagtype(oldtag), newid, newpath, strlen(newpath), + LFS_MKATTR(LFS_FROM_DIR, newid, &oldcwd, lfs_tagid(oldtag), + NULL))); if (err) { return err; } @@ -2714,7 +2714,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - if (prevexists && lfs_tag_type(prevtag) == LFS_TYPE_DIR) { + if (prevexists && lfs_tagtype(prevtag) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, prevdir.pair, &newcwd); if (res < 0) { return res; @@ -2727,9 +2727,12 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { LFS_ASSERT(res); // must have pred newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; - err = lfs_dir_commit(lfs, &newcwd, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL, 0x3ff, sizeof(newcwd.tail)), - .u.buffer=newcwd.tail}}); + err = lfs_dir_commit(lfs, &newcwd, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, + newcwd.tail, sizeof(newcwd.tail), NULL)); + if (err) { + return err; + } } return 0; @@ -2751,7 +2754,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // // // if we were a directory, find pred, replace tail // // TODO can this just deorphan? -// if (prevexists && lfs_tag_subtype(prevattr.tag) == LFS_TYPE_DIR) { +// if (prevexists && lfs_tagsubtype(prevattr.tag) == LFS_TYPE_DIR) { // err = lfs_deorphan(lfs); // if (err) { // return err; @@ -2952,11 +2955,10 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; dir.count += 1; - err = lfs_dir_commit(lfs, &dir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), - .u.buffer=&superblock}, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_STRUCT_DIR, 0, sizeof(lfs->root)), - .u.buffer=lfs->root}}}); + err = lfs_dir_commit(lfs, &dir, + LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), + LFS_MKATTR(LFS_TYPE_DIRSTRUCT, 0, lfs->root, sizeof(lfs->root), + NULL))); if (err) { return err; } @@ -2994,7 +2996,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_t superblock; err = lfs_dir_get(lfs, &dir, 0x7ffff000, - lfs_mktag(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), + LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), NULL, &superblock); if (err) { return err; @@ -3014,7 +3016,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } err = lfs_dir_get(lfs, &dir, 0x7ffff000, - lfs_mktag(LFS_STRUCT_DIR, 0, sizeof(lfs->root)), + LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(lfs->root)), NULL, &lfs->root); if (err) { return err; @@ -3089,7 +3091,7 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { lfs_mattr_t attr; int err = lfs_dir_get(lfs, &dir, 0x7c3ff000, - lfs_mktag(LFS_TYPE_STRUCT, id, 8), + LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &attr.tag, &attr.u); if (err) { if (err == LFS_ERR_NOENT) { @@ -3098,7 +3100,7 @@ int lfs_fs_traverse(lfs_t *lfs, return err; } - if (lfs_tag_type(attr.tag) == LFS_STRUCT_CTZ) { + if (lfs_tagtype(attr.tag) == LFS_TYPE_CTZSTRUCT) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, attr.u.ctz.head, attr.u.ctz.size, cb, data); if (err) { @@ -3163,7 +3165,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } dir.off += lfs_entry_size(&entry); - if ((0x70 & entry.d.type) == LFS_STRUCT_CTZ) { + if ((0x70 & entry.d.type) == LFS_TYPE_CTZSTRUCT) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, entry.d.u.file.head, entry.d.u.file.size, cb, data); if (err) { @@ -3230,8 +3232,8 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], parent->tail[0] = 0; parent->tail[1] = 1; while (!lfs_pairisnull(parent->tail)) { - int err = lfs_dir_find(lfs, parent, parent->tail, - 0x7fc00fff, lfs_mktag(LFS_STRUCT_DIR, 0, sizeof(child)), + int err = lfs_dir_find(lfs, parent, parent->tail, 0x7fc00fff, + LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(child)), child, foundtag); if (err != LFS_ERR_NOENT) { return err; @@ -3287,10 +3289,9 @@ static int lfs_relocate(lfs_t *lfs, // just replace bad pair, no desync can occur parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; - int err = lfs_dir_commit(lfs, &parent, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_TAIL + parent.split, // TODO hm - 0x3ff, sizeof(lfs_block_t[2])), - .u.pair[0]=newpair[0], .u.pair[1]=newpair[1]}}); + int err = lfs_dir_commit(lfs, &parent, + LFS_MKATTR(LFS_TYPE_TAIL + parent.split, 0x3ff, + newpair, sizeof(lfs_block_t[2]), NULL)); if (err) { return err; } @@ -3410,10 +3411,9 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL, - 0x3ff, sizeof(pdir.tail)), - .u.buffer=pdir.tail}}); + err = lfs_dir_commit(lfs, &pdir, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, + pdir.tail, sizeof(pdir.tail), NULL)); if (err) { return err; } @@ -3434,10 +3434,9 @@ int lfs_deorphan(lfs_t *lfs) { pdir.tail[0] = attr.u.pair[0]; pdir.tail[1] = attr.u.pair[1]; - err = lfs_dir_commit(lfs, &pdir, &(lfs_mattrlist_t){ - {lfs_mktag(LFS_TYPE_SOFTTAIL, - 0x3ff, sizeof(pdir.tail)), - .u.buffer=pdir.tail}}); + err = lfs_dir_commit(lfs, &pdir, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, + pdir.tail, sizeof(pdir.tail), NULL)); if (err) { return err; } diff --git a/lfs.h b/lfs.h index 5c00a124..85d5d40d 100644 --- a/lfs.h +++ b/lfs.h @@ -96,29 +96,29 @@ enum lfs_error { // File types enum lfs_type { // file types - LFS_TYPE_REG = 0x001, - LFS_TYPE_DIR = 0x002, + LFS_TYPE_REG = 0x001, + LFS_TYPE_DIR = 0x002, // internally used types - LFS_TYPE_USER = 0x100, - LFS_TYPE_SUPERBLOCK = 0x010, - LFS_TYPE_NAME = 0x000, - LFS_TYPE_DELETE = 0x03f, - LFS_TYPE_STRUCT = 0x040, - LFS_TYPE_GLOBALS = 0x080, - LFS_TYPE_TAIL = 0x0c0, - LFS_TYPE_SOFTTAIL = 0x0c0, - LFS_TYPE_HARDTAIL = 0x0c1, - LFS_TYPE_CRC = 0x0ff, - - LFS_STRUCT_INLINE = 0x040, - LFS_STRUCT_CTZ = 0x041, - LFS_STRUCT_DIR = 0x042, + LFS_TYPE_USER = 0x100, + LFS_TYPE_SUPERBLOCK = 0x010, + LFS_TYPE_NAME = 0x000, + LFS_TYPE_DELETE = 0x03f, + LFS_TYPE_STRUCT = 0x040, + LFS_TYPE_GLOBALS = 0x080, + LFS_TYPE_TAIL = 0x0c0, + LFS_TYPE_SOFTTAIL = 0x0c0, + LFS_TYPE_HARDTAIL = 0x0c1, + LFS_TYPE_CRC = 0x0ff, + + LFS_TYPE_INLINESTRUCT = 0x040, + LFS_TYPE_CTZSTRUCT = 0x041, + LFS_TYPE_DIRSTRUCT = 0x042, // internal chip sources - LFS_FROM_REGION = 0x000, - LFS_FROM_DISK = 0x200, - LFS_FROM_DIR = 0x030, + LFS_FROM_REGION = 0x000, + LFS_FROM_DISK = 0x200, + LFS_FROM_DIR = 0x030, }; // File open flags From 5fc53bd7264da52661231cf89eb195bf634f8a5b Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 12 Jul 2018 20:22:06 -0500 Subject: [PATCH 059/139] Changed internal functions to return tags over pointers One neat (if gimmicky) trick, is that each tag has a valid bit in the highest bit position of the 32-bit word. This is used to determine when to stop a fetch operation, but after fetch, the bit is free to use in the driver. This means we can create a typed-union of sorts with error codes and tags, returning both as the return value from a function. Say what you will about this trick, it does have a significant impact on code size. I suspect this is primarily due to the compiler having a hard time optimizing around pointer access. --- lfs.c | 347 ++++++++++++++++++++++++++-------------------------------- lfs.h | 3 +- 2 files changed, 154 insertions(+), 196 deletions(-) diff --git a/lfs.c b/lfs.c index 37e06eff..fde8ed27 100644 --- a/lfs.c +++ b/lfs.c @@ -262,8 +262,8 @@ static int lfs_bd_sync(lfs_t *lfs) { int lfs_fs_traverse(lfs_t *lfs, int (*cb)(lfs_t*, void*, lfs_block_t), void *data); static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); -static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], - lfs_mdir_t *parent, lfs_tag_t *foundtag); +static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], + lfs_mdir_t *parent); static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); int lfs_scan(lfs_t *lfs); @@ -433,27 +433,27 @@ static inline bool lfs_pairsync( &(lfs_mattrlist_t){ \ {LFS_MKTAG(type, id, size), .u.buffer=(void*)(buffer_)}, (next)} -static inline bool lfs_tagisvalid(lfs_tag_t tag) { +static inline bool lfs_tagisvalid(uint32_t tag) { return !(tag & 0x80000000); } -static inline bool lfs_tagisuser(lfs_tag_t tag) { +static inline bool lfs_tagisuser(uint32_t tag) { return (tag & 0x40000000); } -static inline uint16_t lfs_tagtype(lfs_tag_t tag) { +static inline uint16_t lfs_tagtype(uint32_t tag) { return (tag & 0x7fc00000) >> 22; } -static inline uint16_t lfs_tagsubtype(lfs_tag_t tag) { +static inline uint16_t lfs_tagsubtype(uint32_t tag) { return (tag & 0x7c000000) >> 22; } -static inline uint16_t lfs_tagid(lfs_tag_t tag) { +static inline uint16_t lfs_tagid(uint32_t tag) { return (tag & 0x003ff000) >> 12; } -static inline lfs_size_t lfs_tagsize(lfs_tag_t tag) { +static inline lfs_size_t lfs_tagsize(uint32_t tag) { return tag & 0x00000fff; } @@ -479,7 +479,7 @@ struct lfs_commit { lfs_off_t begin; lfs_off_t end; - lfs_tag_t ptag; + uint32_t ptag; uint32_t crc; struct { @@ -517,14 +517,14 @@ static int lfs_commit_commit(lfs_t *lfs, // check if we fit lfs_size_t size = lfs_tagsize(attr.tag); - if (commit->off + sizeof(lfs_tag_t)+size > commit->end) { + if (commit->off + sizeof(uint32_t)+size > commit->end) { return LFS_ERR_NOSPC; } // write out tag // TODO rm me //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", attr.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tagtype(attr.tag), lfs_tagid(attr.tag), lfs_tagsize(attr.tag)); - lfs_tag_t tag = lfs_tole32((attr.tag & 0x7fffffff) ^ commit->ptag); + uint32_t tag = lfs_tole32((attr.tag & 0x7fffffff) ^ commit->ptag); lfs_crc(&commit->crc, &tag, sizeof(tag)); int err = lfs_bd_prog(lfs, commit->block, commit->off, &tag, sizeof(tag)); if (err) { @@ -569,7 +569,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { commit->off + 2*sizeof(uint32_t), lfs->cfg->prog_size); // read erased state from next program unit - lfs_tag_t tag; + uint32_t tag; int err = lfs_bd_read(lfs, commit->block, noff, &tag, sizeof(tag)); if (err) { return err; @@ -616,16 +616,15 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { } // TODO predeclare -static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t getmask, lfs_tag_t gettag, - lfs_tag_t *foundtag, void *buffer); +static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, + uint32_t getmask, uint32_t gettag, void *buffer); static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, lfs_mdir_t *dir, lfs_mattrlist_t *list) { // Iterate through list and commits, only committing unique entries lfs_off_t off = dir->off + sizeof(uint32_t); - lfs_tag_t ntag = dir->etag; + uint32_t ntag = dir->etag; while (list || off > 2*sizeof(uint32_t)) { lfs_mattr_t attr; @@ -659,7 +658,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, // ignore non-matching ids } else { // check if type has already been committed - int err = lfs_dir_get(lfs, + int32_t res = lfs_dir_get(lfs, &(lfs_mdir_t){ .pair[0]=commit->block, .off=commit->off, @@ -668,12 +667,12 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, lfs_tagisuser(attr.tag) ? 0x7ffff000 : 0x7c3ff000, LFS_MKTAG(lfs_tagtype(attr.tag), toid - commit->filter.begin, 0), // TODO can all these filter adjustments be consolidated? - NULL, NULL); - if (err && err != LFS_ERR_NOENT) { - return err; + NULL); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; } - if (err == LFS_ERR_NOENT) { + if (res == LFS_ERR_NOENT) { // update id and commit, as we are currently unique attr.tag = LFS_MKTAG(0, toid, 0) | (attr.tag & 0xffc00fff); int err = lfs_commit_commit(lfs, commit, attr); @@ -737,14 +736,14 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, return 0; } -static int lfs_dir_find(lfs_t *lfs, +static int32_t lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2], - uint32_t findmask, lfs_tag_t findtag, - const void *findbuffer, lfs_tag_t *foundtag) { + uint32_t findmask, uint32_t findtag, + const void *findbuffer) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; dir->stop_at_commit = false; - lfs_tag_t localfoundtag = 0xffffffff; + int32_t foundtag = LFS_ERR_NOENT; // find the block with the most recent revision uint32_t rev[2]; @@ -764,7 +763,7 @@ static int lfs_dir_find(lfs_t *lfs, // load blocks and check crc for (int i = 0; i < 2; i++) { lfs_off_t off = sizeof(dir->rev); - lfs_tag_t ptag = 0; + uint32_t ptag = 0; uint32_t crc = 0xffffffff; dir->tail[0] = 0xffffffff; dir->tail[1] = 0xffffffff; @@ -777,11 +776,11 @@ static int lfs_dir_find(lfs_t *lfs, dir->rev = lfs_fromle32(dir->rev); lfs_mdir_t tempdir = *dir; - lfs_tag_t tempfoundtag = localfoundtag; + uint32_t tempfoundtag = foundtag; while (true) { // extract next tag - lfs_tag_t tag; + uint32_t tag; int err = lfs_bd_read(lfs, tempdir.pair[0], off, &tag, sizeof(tag)); if (err) { @@ -826,7 +825,7 @@ static int lfs_dir_find(lfs_t *lfs, tempdir.etag = tag; crc = 0xffffffff; *dir = tempdir; - localfoundtag = tempfoundtag; + foundtag = tempfoundtag; } else { err = lfs_bd_crc(lfs, tempdir.pair[0], off+sizeof(tag), lfs_tagsize(tag), &crc); @@ -856,8 +855,8 @@ static int lfs_dir_find(lfs_t *lfs, tempdir.count -= 1; if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { - tempfoundtag = 0xffffffff; - } else if (lfs_tagid(tempfoundtag) < 0x3ff && + tempfoundtag = LFS_ERR_NOENT; + } else if (lfs_tagisvalid(tempfoundtag) && lfs_tagid(tag) < lfs_tagid(tempfoundtag)) { tempfoundtag -= LFS_MKTAG(0, 1, 0); } @@ -890,31 +889,22 @@ static int lfs_dir_find(lfs_t *lfs, done: // synthetic move if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { - if (lfs->globals.move.id == lfs_tagid(localfoundtag)) { - localfoundtag = 0xffffffff; - } else if (lfs_tagid(localfoundtag) < 0x3ff && - lfs->globals.move.id < lfs_tagid(localfoundtag)) { - localfoundtag -= LFS_MKTAG(0, 1, 0); + if (lfs->globals.move.id == lfs_tagid(foundtag)) { + foundtag = LFS_ERR_NOENT; + } else if (lfs_tagisvalid(foundtag) && + lfs->globals.move.id < lfs_tagid(foundtag)) { + foundtag -= LFS_MKTAG(0, 1, 0); } } - if (localfoundtag == 0xffffffff) { - return LFS_ERR_NOENT; - } - - if (foundtag) { - *foundtag = localfoundtag; - } - - return 0; + return foundtag; } static int lfs_dir_fetch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2]) { - int err = lfs_dir_find(lfs, dir, pair, - 0xffffffff, 0xffffffff, NULL, NULL); - if (err && err != LFS_ERR_NOENT) { - return err; + int32_t res = lfs_dir_find(lfs, dir, pair, 0xffffffff, 0xffffffff, NULL); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; } return 0; @@ -1246,9 +1236,8 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { return 0; } -static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t getmask, lfs_tag_t gettag, - lfs_tag_t *foundtag, void *buffer) { +static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, + uint32_t getmask, uint32_t gettag, void *buffer) { uint16_t id = lfs_tagid(gettag); lfs_size_t size = lfs_tagsize(gettag); @@ -1260,7 +1249,7 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, // iterate over dir block backwards (for faster lookups) lfs_off_t off = dir->off + sizeof(uint32_t); - lfs_tag_t tag = dir->etag; + uint32_t tag = dir->etag; while (off > 2*sizeof(tag)) { LFS_ASSERT(off > 2*sizeof(tag)+lfs_tagsize(tag)); @@ -1275,10 +1264,6 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, gettag += LFS_MKTAG(0, 1, 0); } } else if ((tag & getmask) == (gettag & getmask)) { - if (foundtag) { - *foundtag = LFS_MKTAG(0, id, 0) | (tag & 0xffc00fff); - } - if (buffer) { lfs_size_t diff = lfs_min(size, lfs_tagsize(tag)); int err = lfs_bd_read(lfs, dir->pair[0], off, buffer, diff); @@ -1289,10 +1274,10 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, memset((uint8_t*)buffer + diff, 0, size - diff); } - return 0; + return LFS_MKTAG(0, id, 0) | (tag & 0xffc00fff); } - lfs_tag_t ntag; + uint32_t ntag; int err = lfs_bd_read(lfs, dir->pair[0], off-sizeof(ntag), &ntag, sizeof(ntag)); if (err) { @@ -1307,11 +1292,10 @@ static int lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { lfs_mattr_t attr; - int err = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_size+1), - &attr.tag, info->name); - if (err) { - return err; + attr.tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_size+1), info->name); + if (attr.tag < 0) { + return attr.tag; } info->type = lfs_tagtype(attr.tag); @@ -1319,11 +1303,10 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, return LFS_ERR_RANGE; } - err = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), - &attr.tag, &attr.u); - if (err) { - return err; + attr.tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &attr.u); + if (attr.tag < 0) { + return attr.tag; } if (lfs_tagtype(attr.tag) == LFS_TYPE_CTZSTRUCT) { @@ -1336,8 +1319,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, } -static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, - const char **path, lfs_tag_t *foundtag) { +static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { lfs_mattr_t attr = { .u.pair[0] = lfs->root[0], .u.pair[1] = lfs->root[1], @@ -1345,7 +1327,7 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char *name = *path; lfs_size_t namelen; - lfs_tag_t localfoundtag; + int32_t tag; while (true) { nextname: @@ -1353,14 +1335,9 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, name += strspn(name, "/"); namelen = strcspn(name, "/"); - // special case for root dir if (name[0] == '\0') { - // Return ISDIR when we hit root - // TODO change this to -1 or 0x3ff? - if (foundtag) { - *foundtag = LFS_MKTAG(LFS_TYPE_DIR, 0, 0); - } - return LFS_ERR_ISDIR; + // special case for root dir + return LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); } // skip '.' and root '..' @@ -1399,14 +1376,13 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, // find path while (true) { - int err = lfs_dir_find(lfs, dir, attr.u.pair, - 0x7c000fff, LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), - name, &localfoundtag); - if (err && err != LFS_ERR_NOENT) { - return err; + tag = lfs_dir_find(lfs, dir, attr.u.pair, + 0x7c000fff, LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), name); + if (tag < 0 && tag != LFS_ERR_NOENT) { + return tag; } - if (err != LFS_ERR_NOENT) { + if (tag != LFS_ERR_NOENT) { // found it break; } @@ -1419,30 +1395,25 @@ static int lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, attr.u.pair[1] = dir->tail[1]; } - if (foundtag) { - *foundtag = localfoundtag; - } - name += namelen; name += strspn(name, "/"); if (name[0] == '\0') { - return 0; + return tag; } // don't continue on if we didn't hit a directory // TODO update with what's on master? - if (lfs_tagtype(localfoundtag) != LFS_TYPE_DIR) { + if (lfs_tagtype(tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } // TODO optimize grab for inline files and like? // TODO would this mean more code? // grab the entry data - int err = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(localfoundtag), 8), - &attr.tag, &attr.u); - if (err) { - return err; + attr.tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &attr.u); + if (attr.tag < 0) { + return attr.tag; } } } @@ -1458,12 +1429,12 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } lfs_mdir_t cwd; - int err = lfs_dir_lookup(lfs, &cwd, &path, NULL); - if (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) { - if (!err || err == LFS_ERR_ISDIR) { + int32_t res = lfs_dir_lookup(lfs, &cwd, &path); + if (res != LFS_ERR_NOENT || strchr(path, '/') != NULL) { + if (res >= 0) { return LFS_ERR_EXIST; } - return err; + return res; } // check that name fits @@ -1476,7 +1447,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_alloc_ack(lfs); lfs_mdir_t dir; - err = lfs_dir_alloc(lfs, &dir, false, cwd.tail); + int err = lfs_dir_alloc(lfs, &dir, false, cwd.tail); if (err) { return err; } @@ -1510,10 +1481,9 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { - lfs_tag_t tag; - int err = lfs_dir_lookup(lfs, &dir->m, &path, &tag); - if (err && err != LFS_ERR_ISDIR) { - return err; + int32_t tag = lfs_dir_lookup(lfs, &dir->m, &path); + if (tag < 0) { + return tag; } if (lfs_tagtype(tag) != LFS_TYPE_DIR) { @@ -1521,22 +1491,21 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { } lfs_mattr_t attr; - if (err == LFS_ERR_ISDIR) { + if (lfs_tagid(tag) == 0x3ff) { // handle root dir separately attr.u.pair[0] = lfs->root[0]; attr.u.pair[1] = lfs->root[1]; } else { // get dir pair from parent - err = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), - &attr.tag, &attr.u); - if (err) { - return err; + attr.tag = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &attr.u); + if (attr.tag < 0) { + return attr.tag; } } // fetch first pair - err = lfs_dir_fetch(lfs, &dir->m, attr.u.pair); + int err = lfs_dir_fetch(lfs, &dir->m, attr.u.pair); if (err) { return err; } @@ -1863,17 +1832,15 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // allocate entry for file if it doesn't exist lfs_mdir_t cwd; - lfs_tag_t tag; - int err = lfs_dir_lookup(lfs, &cwd, &path, &tag); - if (err && (err != LFS_ERR_NOENT || strchr(path, '/') != NULL) && - err != LFS_ERR_ISDIR) { - return err; + int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); + if (tag < 0 && (tag != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { + return tag; } uint16_t id = lfs_tagid(tag); uint8_t type = lfs_tagtype(tag); lfs_mattr_t attr; // TODO stop copying things (attr, id, type, tag) - if (err == LFS_ERR_NOENT) { + if (tag == LFS_ERR_NOENT) { if (!(flags & LFS_O_CREAT)) { return LFS_ERR_NOENT; } @@ -1885,7 +1852,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // get next slot and create entry to remember name - err = lfs_dir_append(lfs, &cwd, &id); + int err = lfs_dir_append(lfs, &cwd, &id); if (err) { return err; } @@ -1916,11 +1883,10 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // TODO allow no entry? // TODO move this into one load? If cache >= 8 would work - err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), - &attr.tag, &file->head); - if (err) { - return err; + attr.tag = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &file->head); + if (attr.tag < 0) { + return attr.tag; } } @@ -1959,7 +1925,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, file->cache.off = 0; // don't always read (may be new file) if (file->size > 0) { - err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, + int err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, file->cache.buffer, file->size); if (err) { lfs_free(file->cache.buffer); @@ -2538,14 +2504,13 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { lfs_mdir_t cwd; - lfs_tag_t tag; // TODO pass to getinfo? - int err = lfs_dir_lookup(lfs, &cwd, &path, &tag); - if (err && err != LFS_ERR_ISDIR) { - return err; + int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); + if (tag < 0) { + return tag; } - if (err == LFS_ERR_ISDIR) { + if (lfs_tagid(tag) == 0x3ff) { // special case for root strcpy(info->name, "/"); info->type = LFS_TYPE_DIR; @@ -2570,24 +2535,22 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } - lfs_tag_t tag; - err = lfs_dir_lookup(lfs, &cwd, &path, &tag); - if (err) { - return err; + int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); + if (tag < 0) { + return tag; } lfs_mdir_t dir; if (lfs_tagtype(tag) == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t attr; - err = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), - &attr.tag, &attr.u); - if (err) { - return err; + attr.tag = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &attr.u); + if (attr.tag < 0) { + return attr.tag; } - err = lfs_dir_fetch(lfs, &dir, attr.u.pair); + int err = lfs_dir_fetch(lfs, &dir, attr.u.pair); if (err) { return err; } @@ -2635,26 +2598,24 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find old entry lfs_mdir_t oldcwd; - lfs_tag_t oldtag; - int err = lfs_dir_lookup(lfs, &oldcwd, &oldpath, &oldtag); - if (err) { - return err; + int32_t oldtag = lfs_dir_lookup(lfs, &oldcwd, &oldpath); + if (oldtag < 0) { + return oldtag; } // find new entry lfs_mdir_t newcwd; - lfs_tag_t prevtag; - err = lfs_dir_lookup(lfs, &newcwd, &newpath, &prevtag); - if (err && err != LFS_ERR_NOENT) { - return err; + int32_t prevtag = lfs_dir_lookup(lfs, &newcwd, &newpath); + if (prevtag < 0 && prevtag != LFS_ERR_NOENT) { + return prevtag; } uint16_t newid = lfs_tagid(prevtag); - bool prevexists = (err != LFS_ERR_NOENT); + //bool prevexists = (prevtag != LFS_ERR_NOENT); //bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); lfs_mdir_t prevdir; - if (prevexists) { + if (prevtag != LFS_ERR_NOENT) { // check that we have same type if (lfs_tagtype(prevtag) != lfs_tagtype(oldtag)) { return LFS_ERR_ISDIR; @@ -2663,15 +2624,14 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (lfs_tagtype(prevtag) == LFS_TYPE_DIR) { // must be empty before removal lfs_mattr_t prevattr; - err = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), - &prevattr.tag, &prevattr.u); - if (err) { - return err; + prevattr.tag = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), &prevattr.u); + if (prevattr.tag < 0) { + return prevattr.tag; } // must be empty before removal - err = lfs_dir_fetch(lfs, &prevdir, prevattr.u.pair); + int err = lfs_dir_fetch(lfs, &prevdir, prevattr.u.pair); if (err) { return err; } @@ -2688,19 +2648,19 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // get next id - err = lfs_dir_append(lfs, &newcwd, &newid); + int err = lfs_dir_append(lfs, &newcwd, &newid); if (err) { return err; } } // create move to fix later - lfs->diff.move.pair[0] = oldcwd.pair[0] ^ lfs->globals.move.pair[0]; - lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; + lfs->diff.move.pair[0] = oldcwd.pair[0] ^ lfs->globals.move.pair[0]; + lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; lfs->diff.move.id = lfs_tagid(oldtag) ^ lfs->globals.move.id; // move over all attributes - err = lfs_dir_commit(lfs, &newcwd, + int err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTR(lfs_tagtype(oldtag), newid, newpath, strlen(newpath), LFS_MKATTR(LFS_FROM_DIR, newid, &oldcwd, lfs_tagid(oldtag), NULL))); @@ -2714,7 +2674,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - if (prevexists && lfs_tagtype(prevtag) == LFS_TYPE_DIR) { + if (prevtag != LFS_ERR_NOENT && lfs_tagtype(prevtag) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, prevdir.pair, &newcwd); if (res < 0) { return res; @@ -2995,11 +2955,11 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_superblock_t superblock; - err = lfs_dir_get(lfs, &dir, 0x7ffff000, + int32_t res = lfs_dir_get(lfs, &dir, 0x7ffff000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), - NULL, &superblock); - if (err) { - return err; + &superblock); + if (res < 0) { + return res; } if (memcmp(superblock.magic, "littlefs", 8) != 0) { @@ -3015,11 +2975,11 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return LFS_ERR_INVAL; } - err = lfs_dir_get(lfs, &dir, 0x7ffff000, + res = lfs_dir_get(lfs, &dir, 0x7ffff000, LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(lfs->root)), - NULL, &lfs->root); - if (err) { - return err; + &lfs->root); + if (res < 0) { + return res; } if (superblock.inline_size) { @@ -3090,18 +3050,17 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { lfs_mattr_t attr; - int err = lfs_dir_get(lfs, &dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), - &attr.tag, &attr.u); - if (err) { - if (err == LFS_ERR_NOENT) { + attr.tag = lfs_dir_get(lfs, &dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &attr.u); + if (attr.tag < 0) { + if (attr.tag == LFS_ERR_NOENT) { continue; } - return err; + return attr.tag; } if (lfs_tagtype(attr.tag) == LFS_TYPE_CTZSTRUCT) { - err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, + int err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, attr.u.ctz.head, attr.u.ctz.size, cb, data); if (err) { return err; @@ -3222,8 +3181,8 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { return false; } -static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], - lfs_mdir_t *parent, lfs_tag_t *foundtag) { +static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], + lfs_mdir_t *parent) { // search for both orderings so we can reuse the find function lfs_block_t child[2] = {pair[0], pair[1]}; @@ -3232,11 +3191,11 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], parent->tail[0] = 0; parent->tail[1] = 1; while (!lfs_pairisnull(parent->tail)) { - int err = lfs_dir_find(lfs, parent, parent->tail, 0x7fc00fff, + int32_t tag = lfs_dir_find(lfs, parent, parent->tail, 0x7fc00fff, LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(child)), - child, foundtag); - if (err != LFS_ERR_NOENT) { - return err; + child); + if (tag != LFS_ERR_NOENT) { + return tag; } } @@ -3252,12 +3211,12 @@ static int lfs_relocate(lfs_t *lfs, // find parent lfs_mdir_t parent; lfs_mattr_t attr; - int err = lfs_parent(lfs, oldpair, &parent, &attr.tag); - if (err && err != LFS_ERR_NOENT) { - return err; + attr.tag = lfs_parent(lfs, oldpair, &parent); + if (attr.tag < 0 && attr.tag != LFS_ERR_NOENT) { + return attr.tag; } - if (err != LFS_ERR_NOENT) { + if (attr.tag != LFS_ERR_NOENT) { // update disk, this creates a desync attr.u.pair[0] = newpair[0]; attr.u.pair[1] = newpair[1]; @@ -3399,12 +3358,12 @@ int lfs_deorphan(lfs_t *lfs) { // check if we have a parent lfs_mdir_t parent; lfs_mattr_t attr; - int err = lfs_parent(lfs, pdir.tail, &parent, &attr.tag); - if (err && err != LFS_ERR_NOENT) { - return err; + attr.tag = lfs_parent(lfs, pdir.tail, &parent); + if (attr.tag < 0&& attr.tag != LFS_ERR_NOENT) { + return attr.tag; } - if (err == LFS_ERR_NOENT) { + if (attr.tag == LFS_ERR_NOENT) { // we are an orphan LFS_DEBUG("Found orphan %d %d", pdir.tail[0], pdir.tail[1]); @@ -3421,10 +3380,10 @@ int lfs_deorphan(lfs_t *lfs) { break; } - err = lfs_dir_get(lfs, &parent, - 0x7ffff000, attr.tag, NULL, &attr.u); - if (err) { - return err; + int32_t res = lfs_dir_get(lfs, &parent, + 0x7ffff000, attr.tag, &attr.u); + if (res < 0) { + return res; } if (!lfs_pairsync(attr.u.pair, pdir.tail)) { diff --git a/lfs.h b/lfs.h index 85d5d40d..b74802f4 100644 --- a/lfs.h +++ b/lfs.h @@ -262,9 +262,8 @@ struct lfs_attr { /// littlefs data structures /// -typedef uint32_t lfs_tag_t; typedef struct lfs_mattr { - lfs_tag_t tag; + int32_t tag; union { void *buffer; lfs_block_t pair[2]; From d3f37115609930b3cb242cbc8ac65e8669a25c7f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 12 Jul 2018 20:43:55 -0500 Subject: [PATCH 060/139] Cleaned up attributes and related logic The biggest change here is to make littlefs less obsessed with the lfs_mattr_t struct. It was limiting our flexibility and can be entirely replaced by passing the tag + data explicitly. The remaining use of lfs_mattr_t is specific to the commit logic, where it replaces the lfs_mattrlist_t struct. Other changes: - Added global lfs_diskoff struct for embedding disk references inside the lfs_mattr_t. - Reordered lfs_mattrlist_t to squeeze out some code savings - Added commit_get for explicit access to entries from unfinished metadata-pairs - Parameterized the "stop_at_commit" flag instead of hackily storing it in the lfs_mdir_t temporarily - Changed return value of lfs_pred to error-only with ENOENT representing a missing predecessor - Adopted const where possible --- lfs.c | 576 +++++++++++++++++++++++++++++----------------------------- lfs.h | 36 +--- 2 files changed, 293 insertions(+), 319 deletions(-) diff --git a/lfs.c b/lfs.c index fde8ed27..01faa004 100644 --- a/lfs.c +++ b/lfs.c @@ -429,9 +429,8 @@ static inline bool lfs_pairsync( #define LFS_MKTAG(type, id, size) \ (((uint32_t)(type) << 22) | ((uint32_t)(id) << 12) | (uint32_t)(size)) -#define LFS_MKATTR(type, id, buffer_, size, next) \ - &(lfs_mattrlist_t){ \ - {LFS_MKTAG(type, id, size), .u.buffer=(void*)(buffer_)}, (next)} +#define LFS_MKATTR(type, id, buffer, size, next) \ + &(lfs_mattr_t){(next), LFS_MKTAG(type, id, size), (buffer)} static inline bool lfs_tagisvalid(uint32_t tag) { return !(tag & 0x80000000); @@ -488,64 +487,69 @@ struct lfs_commit { } filter; }; +struct lfs_diskoff { + lfs_block_t block; + lfs_off_t off; +}; + // TODO predelcare static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, - lfs_mdir_t *dir, lfs_mattrlist_t *list); + const lfs_mdir_t *dir, const lfs_mattr_t *attrs); -static int lfs_commit_commit(lfs_t *lfs, - struct lfs_commit *commit, lfs_mattr_t attr) { +static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, + uint32_t tag, const void *buffer) { // filter out ids - if (lfs_tagid(attr.tag) < 0x3ff && ( - lfs_tagid(attr.tag) < commit->filter.begin || - lfs_tagid(attr.tag) >= commit->filter.end)) { + if (lfs_tagid(tag) < 0x3ff && ( + lfs_tagid(tag) < commit->filter.begin || + lfs_tagid(tag) >= commit->filter.end)) { return 0; } // special cases - if (lfs_tagtype(attr.tag) == LFS_FROM_DIR) { + if (lfs_tagtype(tag) == LFS_FROM_MOVE) { return lfs_commit_move(lfs, commit, - lfs_tagsize(attr.tag), lfs_tagid(attr.tag), - attr.u.dir, NULL); + lfs_tagsize(tag), lfs_tagid(tag), + buffer, NULL); } - if (lfs_tagid(attr.tag) != 0x3ff) { + if (lfs_tagid(tag) != 0x3ff) { // TODO eh? - uint16_t id = lfs_tagid(attr.tag) - commit->filter.begin; - attr.tag = LFS_MKTAG(0, id, 0) | (attr.tag & 0xffc00fff); + uint16_t id = lfs_tagid(tag) - commit->filter.begin; + tag = LFS_MKTAG(0, id, 0) | (tag & 0xffc00fff); } // check if we fit - lfs_size_t size = lfs_tagsize(attr.tag); + lfs_size_t size = lfs_tagsize(tag); if (commit->off + sizeof(uint32_t)+size > commit->end) { return LFS_ERR_NOSPC; } // write out tag // TODO rm me - //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", attr.tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tagtype(attr.tag), lfs_tagid(attr.tag), lfs_tagsize(attr.tag)); - uint32_t tag = lfs_tole32((attr.tag & 0x7fffffff) ^ commit->ptag); - lfs_crc(&commit->crc, &tag, sizeof(tag)); - int err = lfs_bd_prog(lfs, commit->block, commit->off, &tag, sizeof(tag)); + //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tagtype(tag), lfs_tagid(tag), lfs_tagsize(tag)); + uint32_t ntag = lfs_tole32((tag & 0x7fffffff) ^ commit->ptag); + lfs_crc(&commit->crc, &ntag, sizeof(ntag)); + int err = lfs_bd_prog(lfs, commit->block, commit->off, + &ntag, sizeof(ntag)); if (err) { return err; } - commit->off += sizeof(tag); + commit->off += sizeof(ntag); - if (!(attr.tag & 0x80000000)) { + if (!(tag & 0x80000000)) { // from memory - lfs_crc(&commit->crc, attr.u.buffer, size); - err = lfs_bd_prog(lfs, commit->block, commit->off, - attr.u.buffer, size); + lfs_crc(&commit->crc, buffer, size); + err = lfs_bd_prog(lfs, commit->block, commit->off, buffer, size); if (err) { return err; } } else { // from disk + const struct lfs_diskoff *disk = buffer; for (lfs_off_t i = 0; i < size; i++) { uint8_t dat; - int err = lfs_bd_read(lfs, - attr.u.d.block, attr.u.d.off+i, &dat, 1); + int err = lfs_bd_read(lfs, disk->block, disk->off+i, &dat, 1); if (err) { return err; } @@ -558,7 +562,7 @@ static int lfs_commit_commit(lfs_t *lfs, } } commit->off += size; - commit->ptag = attr.tag & 0x7fffffff; // TODO do this once + commit->ptag = tag & 0x7fffffff; // TODO do this once return 0; } @@ -616,28 +620,85 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { } // TODO predeclare +static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, + uint32_t etag, uint32_t getmask, uint32_t gettag, void *buffer, + bool stopatcommit) { + uint16_t id = lfs_tagid(gettag); + lfs_size_t size = lfs_tagsize(gettag); + + // synthetic moves + if ((block == lfs->globals.move.pair[0] || + block == lfs->globals.move.pair[1]) + && lfs_tagid(gettag) <= lfs->globals.move.id) { + gettag += LFS_MKTAG(0, 1, 0); + } + + // iterate over dir block backwards (for faster lookups) + off += sizeof(uint32_t); // TODO nah + uint32_t tag = etag; + + while (off > 2*sizeof(tag)) { + LFS_ASSERT(off > 2*sizeof(tag)+lfs_tagsize(tag)); + off -= sizeof(tag)+lfs_tagsize(tag); + + if (lfs_tagtype(tag) == LFS_TYPE_CRC && stopatcommit) { + break; + } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { + if (lfs_tagid(tag) <= lfs_tagid(gettag)) { + gettag += LFS_MKTAG(0, 1, 0); + } + } else if ((tag & getmask) == (gettag & getmask)) { + if (buffer) { + lfs_size_t diff = lfs_min(size, lfs_tagsize(tag)); + int err = lfs_bd_read(lfs, block, off, buffer, diff); + if (err) { + return err; + } + + memset((uint8_t*)buffer + diff, 0, size - diff); + } + + return LFS_MKTAG(0, id, 0) | (tag & 0xffc00fff); + } + + uint32_t ntag; + int err = lfs_bd_read(lfs, block, + off-sizeof(ntag), &ntag, sizeof(ntag)); + if (err) { + return err; + } + tag ^= lfs_fromle32(ntag); + } + + return LFS_ERR_NOENT; +} + static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, uint32_t gettag, void *buffer); static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, - lfs_mdir_t *dir, lfs_mattrlist_t *list) { + const lfs_mdir_t *dir, const lfs_mattr_t *attrs) { // Iterate through list and commits, only committing unique entries lfs_off_t off = dir->off + sizeof(uint32_t); uint32_t ntag = dir->etag; - while (list || off > 2*sizeof(uint32_t)) { - lfs_mattr_t attr; - if (list) { - attr = list->e; - list = list->next; + while (attrs || off > 2*sizeof(uint32_t)) { + struct lfs_diskoff disk; + uint32_t tag; + const void *buffer; + if (attrs) { + tag = attrs->tag; + buffer = attrs->buffer; + attrs = attrs->next; } else { LFS_ASSERT(off > 2*sizeof(ntag)+lfs_tagsize(ntag)); off -= sizeof(ntag)+lfs_tagsize(ntag); - attr.tag = ntag; - attr.u.d.block = dir->pair[0]; - attr.u.d.off = off; + tag = ntag; + disk.block = dir->pair[0]; + disk.off = off; + buffer = &disk; int err = lfs_bd_read(lfs, dir->pair[0], off-sizeof(ntag), &ntag, sizeof(ntag)); @@ -646,36 +707,32 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, } ntag = lfs_fromle32(ntag); - ntag ^= attr.tag; - attr.tag |= 0x80000000; + ntag ^= tag; + tag |= 0x80000000; } - if (lfs_tagtype(attr.tag) == LFS_TYPE_DELETE && - lfs_tagid(attr.tag) <= fromid) { + if (lfs_tagtype(tag) == LFS_TYPE_DELETE && lfs_tagid(tag) <= fromid) { // something was deleted, we need to move around it fromid += 1; - } else if (lfs_tagid(attr.tag) != fromid) { + } else if (lfs_tagid(tag) != fromid) { // ignore non-matching ids } else { // check if type has already been committed - int32_t res = lfs_dir_get(lfs, - &(lfs_mdir_t){ - .pair[0]=commit->block, - .off=commit->off, - .etag=commit->ptag, - .stop_at_commit=true}, - lfs_tagisuser(attr.tag) ? 0x7ffff000 : 0x7c3ff000, - LFS_MKTAG(lfs_tagtype(attr.tag), - toid - commit->filter.begin, 0), // TODO can all these filter adjustments be consolidated? - NULL); + int32_t res = lfs_commit_get(lfs, commit->block, + commit->off, commit->ptag, + lfs_tagisuser(tag) ? 0x7ffff000 : 0x7c3ff000, + LFS_MKTAG(lfs_tagtype(tag), toid-commit->filter.begin, 0), + // TODO can all these filter adjustments be consolidated? + NULL, true); if (res < 0 && res != LFS_ERR_NOENT) { return res; } if (res == LFS_ERR_NOENT) { // update id and commit, as we are currently unique - attr.tag = LFS_MKTAG(0, toid, 0) | (attr.tag & 0xffc00fff); - int err = lfs_commit_commit(lfs, commit, attr); + int err = lfs_commit_attr(lfs, commit, + LFS_MKTAG(0, toid, 0) | (tag & 0xffc00fff), + buffer); if (err) { return err; } @@ -694,9 +751,8 @@ static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, // TODO check performance/complexity of different strategies here lfs_globals_t res = lfs_globals_xor(source, diff); - int err = lfs_commit_commit(lfs, commit, (lfs_mattr_t){ - LFS_MKTAG(LFS_TYPE_GLOBALS, 0x3ff, sizeof(res)), - .u.buffer=&res}); + int err = lfs_commit_attr(lfs, commit, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0x3ff, sizeof(res)), &res); if (err) { return err; } @@ -742,7 +798,6 @@ static int32_t lfs_dir_find(lfs_t *lfs, const void *findbuffer) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; - dir->stop_at_commit = false; int32_t foundtag = LFS_ERR_NOENT; // find the block with the most recent revision @@ -910,7 +965,7 @@ static int lfs_dir_fetch(lfs_t *lfs, return 0; } -static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, +static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; @@ -989,7 +1044,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, // commit with a move for (uint16_t id = begin; id < end; id++) { - err = lfs_commit_move(lfs, &commit, id, id, source, list); + err = lfs_commit_move(lfs, &commit, id, id, source, attrs); if (err) { if (err == LFS_ERR_NOSPC) { goto split; @@ -1008,10 +1063,9 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, if (!lfs_pairisnull(dir->tail)) { // commit tail, which may be new after last size check // TODO le32 - err = lfs_commit_commit(lfs, &commit, (lfs_mattr_t){ + err = lfs_commit_attr(lfs, &commit, LFS_MKTAG(LFS_TYPE_TAIL + dir->split, - 0x3ff, sizeof(dir->tail)), - .u.buffer=dir->tail}); + 0x3ff, sizeof(dir->tail)), dir->tail); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1049,7 +1103,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, return err; } - err = lfs_dir_compact(lfs, &tail, list, dir, ack+1, end); + err = lfs_dir_compact(lfs, &tail, attrs, dir, ack+1, end); if (err) { return err; } @@ -1101,7 +1155,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list, return 0; } -static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { +static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs) { while (true) { if (!dir->erased) { // not erased, must compact @@ -1119,8 +1173,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { .filter.end = 0x3ff, }; - for (lfs_mattrlist_t *a = list; a; a = a->next) { - int err = lfs_commit_commit(lfs, &commit, a->e); + for (lfs_mattr_t *a = attrs; a; a = a->next) { + int err = lfs_commit_attr(lfs, &commit, a->tag, a->buffer); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1156,7 +1210,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattrlist_t *list) { compact: lfs->pcache.block = 0xffffffff; - err = lfs_dir_compact(lfs, dir, list, dir, 0, dir->count); + err = lfs_dir_compact(lfs, dir, attrs, dir, 0, dir->count); if (err) { return err; } @@ -1187,12 +1241,12 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { // check if we should drop the directory block if (dir->count == 0) { lfs_mdir_t pdir; - int res = lfs_pred(lfs, dir->pair, &pdir); - if (res < 0) { - return res; + int err = lfs_pred(lfs, dir->pair, &pdir); + if (err && err != LFS_ERR_NOENT) { + return err; } - if (res && pdir.split) { + if (err != LFS_ERR_NOENT && pdir.split) { // steal tail, and global state pdir.split = dir->split; pdir.tail[0] = dir->tail[0]; @@ -1202,12 +1256,14 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { return lfs_dir_commit(lfs, &pdir, LFS_MKATTR(LFS_TYPE_TAIL + pdir.split, 0x3ff, - pdir.tail, sizeof(pdir.tail), NULL)); + pdir.tail, sizeof(pdir.tail), + NULL)); } } int err = lfs_dir_commit(lfs, dir, - LFS_MKATTR(LFS_TYPE_DELETE, id, NULL, 0, NULL)); + LFS_MKATTR(LFS_TYPE_DELETE, id, NULL, 0, + NULL)); if (err) { return err; } @@ -1238,81 +1294,34 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, uint32_t gettag, void *buffer) { - uint16_t id = lfs_tagid(gettag); - lfs_size_t size = lfs_tagsize(gettag); - - // synthetic moves - if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 - && lfs_tagid(gettag) <= lfs->globals.move.id) { - gettag += LFS_MKTAG(0, 1, 0); - } - - // iterate over dir block backwards (for faster lookups) - lfs_off_t off = dir->off + sizeof(uint32_t); - uint32_t tag = dir->etag; - - while (off > 2*sizeof(tag)) { - LFS_ASSERT(off > 2*sizeof(tag)+lfs_tagsize(tag)); - off -= sizeof(tag)+lfs_tagsize(tag); - - if (lfs_tagtype(tag) == LFS_TYPE_CRC) { - if (dir->stop_at_commit) { - break; - } - } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { - if (lfs_tagid(tag) <= lfs_tagid(gettag)) { - gettag += LFS_MKTAG(0, 1, 0); - } - } else if ((tag & getmask) == (gettag & getmask)) { - if (buffer) { - lfs_size_t diff = lfs_min(size, lfs_tagsize(tag)); - int err = lfs_bd_read(lfs, dir->pair[0], off, buffer, diff); - if (err) { - return err; - } - - memset((uint8_t*)buffer + diff, 0, size - diff); - } - - return LFS_MKTAG(0, id, 0) | (tag & 0xffc00fff); - } - - uint32_t ntag; - int err = lfs_bd_read(lfs, dir->pair[0], - off-sizeof(ntag), &ntag, sizeof(ntag)); - if (err) { - return err; - } - tag ^= lfs_fromle32(ntag); - } - - return LFS_ERR_NOENT; + return lfs_commit_get(lfs, dir->pair[0], dir->off, dir->etag, + getmask, gettag, buffer, false); } static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, int16_t id, struct lfs_info *info) { - lfs_mattr_t attr; - attr.tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + int32_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_size+1), info->name); - if (attr.tag < 0) { - return attr.tag; + if (tag < 0) { + return tag; } - info->type = lfs_tagtype(attr.tag); - if (lfs_tagsize(attr.tag) > lfs->name_size) { + info->type = lfs_tagtype(tag); + if (lfs_tagsize(tag) > lfs->name_size) { return LFS_ERR_RANGE; } - attr.tag = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &attr.u); - if (attr.tag < 0) { - return attr.tag; + struct lfs_ctz ctz; + tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); + if (tag < 0) { + return tag; } - if (lfs_tagtype(attr.tag) == LFS_TYPE_CTZSTRUCT) { - info->size = attr.u.ctz.size; - } else if (lfs_tagtype(attr.tag) == LFS_TYPE_INLINESTRUCT) { - info->size = lfs_tagsize(attr.tag); + if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { + info->size = ctz.size; + } else if (lfs_tagtype(tag) == LFS_TYPE_INLINESTRUCT) { + info->size = lfs_tagsize(tag); } return 0; @@ -1320,11 +1329,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { - lfs_mattr_t attr = { - .u.pair[0] = lfs->root[0], - .u.pair[1] = lfs->root[1], - }; - + lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; const char *name = *path; lfs_size_t namelen; int32_t tag; @@ -1376,8 +1381,8 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { // find path while (true) { - tag = lfs_dir_find(lfs, dir, attr.u.pair, - 0x7c000fff, LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), name); + tag = lfs_dir_find(lfs, dir, pair, 0x7c000fff, + LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), name); if (tag < 0 && tag != LFS_ERR_NOENT) { return tag; } @@ -1391,8 +1396,8 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { return LFS_ERR_NOENT; } - attr.u.pair[0] = dir->tail[0]; - attr.u.pair[1] = dir->tail[1]; + pair[0] = dir->tail[0]; + pair[1] = dir->tail[1]; } name += namelen; @@ -1410,10 +1415,10 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { // TODO optimize grab for inline files and like? // TODO would this mean more code? // grab the entry data - attr.tag = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &attr.u); - if (attr.tag < 0) { - return attr.tag; + int32_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), pair); + if (res < 0) { + return res; } } } @@ -1470,7 +1475,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), - NULL)))); + NULL)))); if (err) { return err; } @@ -1490,22 +1495,22 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { return LFS_ERR_NOTDIR; } - lfs_mattr_t attr; + lfs_block_t pair[2]; if (lfs_tagid(tag) == 0x3ff) { // handle root dir separately - attr.u.pair[0] = lfs->root[0]; - attr.u.pair[1] = lfs->root[1]; + pair[0] = lfs->root[0]; + pair[1] = lfs->root[1]; } else { // get dir pair from parent - attr.tag = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &attr.u); - if (attr.tag < 0) { - return attr.tag; + int32_t res = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), pair); + if (res < 0) { + return res; } } // fetch first pair - int err = lfs_dir_fetch(lfs, &dir->m, attr.u.pair); + int err = lfs_dir_fetch(lfs, &dir->m, pair); if (err) { return err; } @@ -1836,10 +1841,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, if (tag < 0 && (tag != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { return tag; } - uint16_t id = lfs_tagid(tag); - uint8_t type = lfs_tagtype(tag); - lfs_mattr_t attr; // TODO stop copying things (attr, id, type, tag) if (tag == LFS_ERR_NOENT) { if (!(flags & LFS_O_CREAT)) { return LFS_ERR_NOENT; @@ -1852,20 +1854,23 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // get next slot and create entry to remember name + uint16_t id; int err = lfs_dir_append(lfs, &cwd, &id); if (err) { return err; } // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change + // TODO don't use inline struct? just leave it out? err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_REG, id, path, nlen, - LFS_MKATTR(LFS_TYPE_INLINESTRUCT, id, NULL, 0, NULL))); + LFS_MKATTR(LFS_TYPE_INLINESTRUCT, id, NULL, 0, + NULL))); if (err) { return err; } - // TODO eh + // TODO eh AHHHHHHHHHHHHHH if (id >= cwd.count) { // catch updates from a compact in the above commit id -= cwd.count; @@ -1873,27 +1878,28 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, cwd.pair[1] = cwd.tail[1]; } - attr.tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, id, 0); + tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, id, 0); + } else if (flags & LFS_O_EXCL) { + return LFS_ERR_EXIST; + } else if (lfs_tagtype(tag) != LFS_TYPE_REG) { + return LFS_ERR_ISDIR; + } else if (flags & LFS_O_TRUNC) { + // truncate if requested + tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, lfs_tagid(tag), 0); + flags |= LFS_F_DIRTY; } else { - if (type != LFS_TYPE_REG) { - return LFS_ERR_ISDIR; - } else if (flags & LFS_O_EXCL) { - return LFS_ERR_EXIST; - } - - // TODO allow no entry? - // TODO move this into one load? If cache >= 8 would work - attr.tag = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &file->head); - if (attr.tag < 0) { - return attr.tag; + // try to load what's on disk, if it's inlined we'll fix it later + tag = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &file->ctz); + if (tag < 0) { + return tag; } } // setup file struct file->pair[0] = cwd.pair[0]; file->pair[1] = cwd.pair[1]; - file->id = id; + file->id = lfs_tagid(tag); file->flags = flags; file->pos = 0; file->attrs = NULL; @@ -1914,39 +1920,26 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } - if (lfs_tagtype(attr.tag) == LFS_TYPE_INLINESTRUCT) { - // TODO make inline the better path? - // TODO can inline and trunc be combined? + if (lfs_tagtype(tag) == LFS_TYPE_INLINESTRUCT) { // load inline files - file->head = 0xfffffffe; - file->size = lfs_tagsize(attr.tag); + file->ctz.head = 0xfffffffe; + file->ctz.size = lfs_tagsize(tag); file->flags |= LFS_F_INLINE; - file->cache.block = file->head; + file->cache.block = file->ctz.head; file->cache.off = 0; - // don't always read (may be new file) - if (file->size > 0) { - int err = lfs_bd_read(lfs, attr.u.d.block, attr.u.d.off, - file->cache.buffer, file->size); - if (err) { + + // don't always read (may be new/trunc file) + if (file->ctz.size > 0) { + int32_t res = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), file->ctz.size), + file->cache.buffer); + if (res < 0) { lfs_free(file->cache.buffer); - return err; + return res; } } } - // truncate if requested - if (flags & LFS_O_TRUNC) { - if (file->size != 0) { - file->flags |= LFS_F_DIRTY; - } - - file->head = 0xfffffffe; - file->size = 0; - file->flags |= LFS_F_INLINE; - file->cache.block = 0xfffffffe; - file->cache.off = 0; - } - // add to list of files file->next = lfs->files; lfs->files = file; @@ -2030,15 +2023,15 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { if (!(file->flags & LFS_F_INLINE)) { // copy over anything after current branch lfs_file_t orig = { - .head = file->head, - .size = file->size, + .ctz.head = file->ctz.head, + .ctz.size = file->ctz.size, .flags = LFS_O_RDONLY, .pos = file->pos, .cache = lfs->rcache, }; lfs->rcache.block = 0xffffffff; - while (file->pos < file->size) { + while (file->pos < file->ctz.size) { // copy over a byte at a time, leave it up to caching // to make this efficient uint8_t data; @@ -2078,12 +2071,12 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { } } } else { - file->size = lfs_max(file->pos, file->size); + file->ctz.size = lfs_max(file->pos, file->ctz.size); } // actual file updates - file->head = file->block; - file->size = file->pos; + file->ctz.head = file->block; + file->ctz.size = file->pos; file->flags &= ~LFS_F_WRITING; file->flags |= LFS_F_DIRTY; @@ -2115,14 +2108,16 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { if (!(file->flags & LFS_F_INLINE)) { int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_CTZSTRUCT, file->id, - &file->head, 2*sizeof(uint32_t), file->attrs)); + &file->ctz.head, 2*sizeof(uint32_t), + file->attrs)); if (err) { return err; } } else { int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, - file->cache.buffer, file->size, file->attrs)); + file->cache.buffer, file->ctz.size, + file->attrs)); if (err) { return err; } @@ -2151,12 +2146,12 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, } } - if (file->pos >= file->size) { + if (file->pos >= file->ctz.size) { // eof if past end return 0; } - size = lfs_min(size, file->size - file->pos); + size = lfs_min(size, file->ctz.size - file->pos); nsize = size; while (nsize > 0) { @@ -2165,7 +2160,7 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, file->off == lfs->cfg->block_size) { if (!(file->flags & LFS_F_INLINE)) { int err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, + file->ctz.head, file->ctz.size, file->pos, &file->block, &file->off); if (err) { return err; @@ -2212,14 +2207,14 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } } - if ((file->flags & LFS_O_APPEND) && file->pos < file->size) { - file->pos = file->size; + if ((file->flags & LFS_O_APPEND) && file->pos < file->ctz.size) { + file->pos = file->ctz.size; } - if (!(file->flags & LFS_F_WRITING) && file->pos > file->size) { + if (!(file->flags & LFS_F_WRITING) && file->pos > file->ctz.size) { // fill with zeros lfs_off_t pos = file->pos; - file->pos = file->size; + file->pos = file->ctz.size; while (file->pos < pos) { lfs_ssize_t res = lfs_file_write(lfs, file, &(uint8_t){0}, 1); @@ -2254,7 +2249,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { // find out which block we're extending from int err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, + file->ctz.head, file->ctz.size, file->pos-1, &file->block, &file->off); if (err) { file->flags |= LFS_F_ERRED; @@ -2334,11 +2329,11 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, file->pos = file->pos + off; } else if (whence == LFS_SEEK_END) { - if (off < 0 && (lfs_off_t)-off > file->size) { + if (off < 0 && (lfs_off_t)-off > file->ctz.size) { return LFS_ERR_INVAL; } - file->pos = file->size + off; + file->pos = file->ctz.size + off; } return file->pos; @@ -2359,13 +2354,13 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { // lookup new head in ctz skip list err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, - size, &file->head, &(lfs_off_t){0}); + file->ctz.head, file->ctz.size, + size, &file->ctz.head, &(lfs_off_t){0}); if (err) { return err; } - file->size = size; + file->ctz.size = size; file->flags |= LFS_F_DIRTY; } else if (size > oldsize) { lfs_off_t pos = file->pos; @@ -2413,9 +2408,9 @@ int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file) { lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { (void)lfs; if (file->flags & LFS_F_WRITING) { - return lfs_max(file->pos, file->size); + return lfs_max(file->pos, file->ctz.size); } else { - return file->size; + return file->ctz.size; } } @@ -2543,14 +2538,14 @@ int lfs_remove(lfs_t *lfs, const char *path) { lfs_mdir_t dir; if (lfs_tagtype(tag) == LFS_TYPE_DIR) { // must be empty before removal - lfs_mattr_t attr; - attr.tag = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &attr.u); - if (attr.tag < 0) { - return attr.tag; + lfs_block_t pair[2]; + int32_t res = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), pair); + if (res < 0) { + return res; } - int err = lfs_dir_fetch(lfs, &dir, attr.u.pair); + int err = lfs_dir_fetch(lfs, &dir, pair); if (err) { return err; } @@ -2568,17 +2563,17 @@ int lfs_remove(lfs_t *lfs, const char *path) { } if (lfs_tagtype(tag) == LFS_TYPE_DIR) { - int res = lfs_pred(lfs, dir.pair, &cwd); - if (res < 0) { - return res; + int err = lfs_pred(lfs, dir.pair, &cwd); + if (err) { + return err; } - LFS_ASSERT(res); // must have pred cwd.tail[0] = dir.tail[0]; cwd.tail[1] = dir.tail[1]; err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - cwd.tail, sizeof(cwd.tail), NULL)); + cwd.tail, sizeof(cwd.tail), + NULL)); if (err) { return err; } @@ -2623,15 +2618,15 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (lfs_tagtype(prevtag) == LFS_TYPE_DIR) { // must be empty before removal - lfs_mattr_t prevattr; - prevattr.tag = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), &prevattr.u); - if (prevattr.tag < 0) { - return prevattr.tag; + lfs_block_t prevpair[2]; + int32_t res = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair); + if (res < 0) { + return res; } // must be empty before removal - int err = lfs_dir_fetch(lfs, &prevdir, prevattr.u.pair); + int err = lfs_dir_fetch(lfs, &prevdir, prevpair); if (err) { return err; } @@ -2662,8 +2657,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // move over all attributes int err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTR(lfs_tagtype(oldtag), newid, newpath, strlen(newpath), - LFS_MKATTR(LFS_FROM_DIR, newid, &oldcwd, lfs_tagid(oldtag), - NULL))); + LFS_MKATTR(LFS_FROM_MOVE, newid, &oldcwd, lfs_tagid(oldtag), + NULL))); if (err) { return err; } @@ -2675,21 +2670,21 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } if (prevtag != LFS_ERR_NOENT && lfs_tagtype(prevtag) == LFS_TYPE_DIR) { - int res = lfs_pred(lfs, prevdir.pair, &newcwd); - if (res < 0) { - return res; + int err = lfs_pred(lfs, prevdir.pair, &newcwd); + if (err) { + return err; } // TODO test for global state stealing? // steal global state lfs->globals = lfs_globals_xor(&lfs->globals, &prevdir.globals); - LFS_ASSERT(res); // must have pred newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - newcwd.tail, sizeof(newcwd.tail), NULL)); + newcwd.tail, sizeof(newcwd.tail), + NULL)); if (err) { return err; } @@ -2918,7 +2913,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { err = lfs_dir_commit(lfs, &dir, LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), LFS_MKATTR(LFS_TYPE_DIRSTRUCT, 0, lfs->root, sizeof(lfs->root), - NULL))); + NULL))); if (err) { return err; } @@ -3049,19 +3044,19 @@ int lfs_fs_traverse(lfs_t *lfs, } for (uint16_t id = 0; id < dir.count; id++) { - lfs_mattr_t attr; - attr.tag = lfs_dir_get(lfs, &dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, id, 8), &attr.u); - if (attr.tag < 0) { - if (attr.tag == LFS_ERR_NOENT) { + struct lfs_ctz ctz; + int32_t tag = lfs_dir_get(lfs, &dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); + if (tag < 0) { + if (tag == LFS_ERR_NOENT) { continue; } - return attr.tag; + return tag; } - if (lfs_tagtype(attr.tag) == LFS_TYPE_CTZSTRUCT) { + if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { int err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, - attr.u.ctz.head, attr.u.ctz.size, cb, data); + ctz.head, ctz.size, cb, data); if (err) { return err; } @@ -3073,7 +3068,7 @@ int lfs_fs_traverse(lfs_t *lfs, for (lfs_file_t *f = lfs->files; f; f = f->next) { if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, - f->head, f->size, cb, data); + f->ctz.head, f->ctz.size, cb, data); if (err) { return err; } @@ -3169,7 +3164,8 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { pdir->tail[1] = 1; while (!lfs_pairisnull(pdir->tail)) { if (lfs_paircmp(pdir->tail, pair) == 0) { - return true; // TODO should we return true only if pred is part of dir? + //return true; // TODO should we return true only if pred is part of dir? + return 0; } int err = lfs_dir_fetch(lfs, pdir, pdir->tail); @@ -3178,7 +3174,7 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { } } - return false; + return LFS_ERR_NOENT; } static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], @@ -3210,17 +3206,15 @@ static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]) { // find parent lfs_mdir_t parent; - lfs_mattr_t attr; - attr.tag = lfs_parent(lfs, oldpair, &parent); - if (attr.tag < 0 && attr.tag != LFS_ERR_NOENT) { - return attr.tag; + int32_t tag = lfs_parent(lfs, oldpair, &parent); + if (tag < 0 && tag != LFS_ERR_NOENT) { + return tag; } - if (attr.tag != LFS_ERR_NOENT) { + if (tag != LFS_ERR_NOENT) { // update disk, this creates a desync - attr.u.pair[0] = newpair[0]; - attr.u.pair[1] = newpair[1]; - int err = lfs_dir_commit(lfs, &parent, &(lfs_mattrlist_t){attr}); + int err = lfs_dir_commit(lfs, &parent, + &(lfs_mattr_t){.tag=tag, .buffer=newpair}); if (err) { return err; } @@ -3239,18 +3233,19 @@ static int lfs_relocate(lfs_t *lfs, } // find pred - int res = lfs_pred(lfs, oldpair, &parent); - if (res < 0) { - return res; + int err = lfs_pred(lfs, oldpair, &parent); + if (err && err != LFS_ERR_NOENT) { + return err; } - if (res) { + if (err != LFS_ERR_NOENT) { // just replace bad pair, no desync can occur parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; int err = lfs_dir_commit(lfs, &parent, LFS_MKATTR(LFS_TYPE_TAIL + parent.split, 0x3ff, - newpair, sizeof(lfs_block_t[2]), NULL)); + newpair, sizeof(lfs_block_t[2]), + NULL)); if (err) { return err; } @@ -3357,22 +3352,21 @@ int lfs_deorphan(lfs_t *lfs) { if (!pdir.split) { // check if we have a parent lfs_mdir_t parent; - lfs_mattr_t attr; - attr.tag = lfs_parent(lfs, pdir.tail, &parent); - if (attr.tag < 0&& attr.tag != LFS_ERR_NOENT) { - return attr.tag; + int32_t tag = lfs_parent(lfs, pdir.tail, &parent); + if (tag < 0 && tag != LFS_ERR_NOENT) { + return tag; } - if (attr.tag == LFS_ERR_NOENT) { + if (tag == LFS_ERR_NOENT) { // we are an orphan - LFS_DEBUG("Found orphan %d %d", - pdir.tail[0], pdir.tail[1]); + LFS_DEBUG("Found orphan %d %d", pdir.tail[0], pdir.tail[1]); pdir.tail[0] = dir.tail[0]; pdir.tail[1] = dir.tail[1]; err = lfs_dir_commit(lfs, &pdir, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pdir.tail, sizeof(pdir.tail), NULL)); + pdir.tail, sizeof(pdir.tail), + NULL)); if (err) { return err; } @@ -3380,22 +3374,22 @@ int lfs_deorphan(lfs_t *lfs) { break; } - int32_t res = lfs_dir_get(lfs, &parent, - 0x7ffff000, attr.tag, &attr.u); + lfs_block_t pair[2]; + int32_t res = lfs_dir_get(lfs, &parent, 0x7ffff000, tag, pair); if (res < 0) { return res; } - if (!lfs_pairsync(attr.u.pair, pdir.tail)) { + if (!lfs_pairsync(pair, pdir.tail)) { // we have desynced - LFS_DEBUG("Found half-orphan %d %d", - attr.u.pair[0], attr.u.pair[1]); + LFS_DEBUG("Found half-orphan %d %d", pair[0], pair[1]); - pdir.tail[0] = attr.u.pair[0]; - pdir.tail[1] = attr.u.pair[1]; + pdir.tail[0] = pair[0]; + pdir.tail[1] = pair[1]; err = lfs_dir_commit(lfs, &pdir, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pdir.tail, sizeof(pdir.tail), NULL)); + pdir.tail, sizeof(pdir.tail), + NULL)); if (err) { return err; } diff --git a/lfs.h b/lfs.h index b74802f4..9aecf89e 100644 --- a/lfs.h +++ b/lfs.h @@ -118,7 +118,7 @@ enum lfs_type { // internal chip sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_DIR = 0x030, + LFS_FROM_MOVE = 0x030, }; // File open flags @@ -263,32 +263,11 @@ struct lfs_attr { /// littlefs data structures /// typedef struct lfs_mattr { + struct lfs_mattr *next; int32_t tag; - union { - void *buffer; - lfs_block_t pair[2]; - struct { - lfs_block_t head; - lfs_size_t size; - } ctz; - struct { - lfs_block_t block; - lfs_off_t off; - } d; - struct lfs_mdir *dir; - } u; + const void *buffer; } lfs_mattr_t; -typedef struct lfs_mattrlist { - lfs_mattr_t e; - struct lfs_mattrlist *next; -} lfs_mattrlist_t; - -//typedef struct lfs_entry { -// lfs_block_t pair[2]; -// uint16_t id; -//} lfs_entry_t; - typedef struct lfs_globals { struct lfs_move { lfs_block_t pair[2]; @@ -305,7 +284,6 @@ typedef struct lfs_mdir { uint16_t count; bool erased; bool split; - bool stop_at_commit; // TODO hmmm lfs_globals_t globals; } lfs_mdir_t; @@ -319,8 +297,10 @@ typedef struct lfs_file { struct lfs_file *next; lfs_block_t pair[2]; uint16_t id; - lfs_block_t head; - lfs_size_t size; + struct lfs_ctz { + lfs_block_t head; + lfs_size_t size; + } ctz; uint32_t flags; lfs_off_t pos; @@ -328,7 +308,7 @@ typedef struct lfs_file { lfs_off_t off; lfs_cache_t cache; - lfs_mattrlist_t *attrs; + lfs_mattr_t *attrs; } lfs_file_t; typedef struct lfs_dir { From 5d24e656f1ae4dbe30711fec59ff31c9924ade5e Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 13 Jul 2018 15:04:31 -0500 Subject: [PATCH 061/139] Cleaned up commit logic and function organization Restrctured function organization to make a bit more sense, and made some small refactoring tweaks, specifically around the commit logic and global related functions. --- lfs.c | 877 ++++++++++++++++++++++++++++------------------------------ lfs.h | 5 +- 2 files changed, 419 insertions(+), 463 deletions(-) diff --git a/lfs.c b/lfs.c index 01faa004..0409d29a 100644 --- a/lfs.c +++ b/lfs.c @@ -457,16 +457,13 @@ static inline lfs_size_t lfs_tagsize(uint32_t tag) { } // operations on globals -static lfs_globals_t lfs_globals_xor( - const lfs_globals_t *a, const lfs_globals_t *b) { - lfs_globals_t res; - res.move.pair[0] = a->move.pair[0] ^ b->move.pair[0]; - res.move.pair[1] = a->move.pair[1] ^ b->move.pair[1]; - res.move.id = a->move.id ^ b->move.id; - return res; +static void lfs_globalsxor(lfs_globals_t *a, const lfs_globals_t *b) { + a->move.pair[0] ^= b->move.pair[0]; + a->move.pair[1] ^= b->move.pair[1]; + a->move.id ^= b->move.id; } -static bool lfs_globals_iszero(const lfs_globals_t *a) { +static bool lfs_globalsiszero(const lfs_globals_t *a) { return (a->move.pair[0] == 0 && a->move.pair[1] == 0 && a->move.id == 0); } @@ -475,16 +472,11 @@ static bool lfs_globals_iszero(const lfs_globals_t *a) { struct lfs_commit { lfs_block_t block; lfs_off_t off; - lfs_off_t begin; - lfs_off_t end; - uint32_t ptag; uint32_t crc; - struct { - uint16_t begin; - uint16_t end; - } filter; + lfs_off_t begin; + lfs_off_t end; }; struct lfs_diskoff { @@ -492,42 +484,68 @@ struct lfs_diskoff { lfs_off_t off; }; -// TODO predelcare -static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, +static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, + uint32_t tag, uint32_t getmask, uint32_t gettag, int32_t difftag, + void *buffer, bool stopatcommit) { + // iterate over dir block backwards (for faster lookups) + while (off > sizeof(tag)) { + LFS_ASSERT(off > sizeof(tag)+lfs_tagsize(tag)); + off -= sizeof(tag)+lfs_tagsize(tag); + + if (lfs_tagtype(tag) == LFS_TYPE_CRC && stopatcommit) { + break; + } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { + if (lfs_tagid(tag) <= lfs_tagid(gettag + difftag)) { + difftag += LFS_MKTAG(0, 1, 0); + } + } else if ((tag & getmask) == ((gettag + difftag) & getmask)) { + if (buffer) { + lfs_size_t diff = lfs_min( + lfs_tagsize(gettag), lfs_tagsize(tag)); + int err = lfs_bd_read(lfs, block, + off+sizeof(tag), buffer, diff); + if (err) { + return err; + } + + memset((uint8_t*)buffer + diff, 0, + lfs_tagsize(gettag) - diff); + } + + return tag - difftag; + } + + uint32_t ntag; + int err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); + if (err) { + return err; + } + tag ^= lfs_fromle32(ntag); + } + + return LFS_ERR_NOENT; +} + +static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, const lfs_mdir_t *dir, const lfs_mattr_t *attrs); -static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commitattr(lfs_t *lfs, struct lfs_commit *commit, uint32_t tag, const void *buffer) { - // filter out ids - if (lfs_tagid(tag) < 0x3ff && ( - lfs_tagid(tag) < commit->filter.begin || - lfs_tagid(tag) >= commit->filter.end)) { - return 0; - } - - // special cases if (lfs_tagtype(tag) == LFS_FROM_MOVE) { - return lfs_commit_move(lfs, commit, + // special case for moves + return lfs_commitmove(lfs, commit, lfs_tagsize(tag), lfs_tagid(tag), buffer, NULL); } - if (lfs_tagid(tag) != 0x3ff) { - // TODO eh? - uint16_t id = lfs_tagid(tag) - commit->filter.begin; - tag = LFS_MKTAG(0, id, 0) | (tag & 0xffc00fff); - } - // check if we fit lfs_size_t size = lfs_tagsize(tag); - if (commit->off + sizeof(uint32_t)+size > commit->end) { + if (commit->off + sizeof(tag)+size > commit->end) { return LFS_ERR_NOSPC; } // write out tag - // TODO rm me - //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", tag, commit->block, commit->off+sizeof(lfs_tag_t), lfs_tagtype(tag), lfs_tagid(tag), lfs_tagsize(tag)); uint32_t ntag = lfs_tole32((tag & 0x7fffffff) ^ commit->ptag); lfs_crc(&commit->crc, &ntag, sizeof(ntag)); int err = lfs_bd_prog(lfs, commit->block, commit->off, @@ -548,6 +566,7 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, // from disk const struct lfs_diskoff *disk = buffer; for (lfs_off_t i = 0; i < size; i++) { + // rely on caching to make this efficient uint8_t dat; int err = lfs_bd_read(lfs, disk->block, disk->off+i, &dat, 1); if (err) { @@ -561,129 +580,19 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, } } } - commit->off += size; - commit->ptag = tag & 0x7fffffff; // TODO do this once - - return 0; -} - -static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { - // align to program units - lfs_off_t noff = lfs_alignup( - commit->off + 2*sizeof(uint32_t), lfs->cfg->prog_size); - - // read erased state from next program unit - uint32_t tag; - int err = lfs_bd_read(lfs, commit->block, noff, &tag, sizeof(tag)); - if (err) { - return err; - } - - // build crc tag - tag = (0x80000000 & ~lfs_fromle32(tag)) | - LFS_MKTAG(LFS_TYPE_CRC, 0x3ff, - noff - (commit->off+sizeof(uint32_t))); - - // write out crc - //printf("tag w %#010x (%x:%x %03x %03x %03x)\n", tag, commit->block, commit->off+sizeof(tag), lfs_tagtype(tag), lfs_tagid(tag), lfs_tagsize(tag)); - uint32_t footer[2]; - footer[0] = lfs_tole32(tag ^ commit->ptag); - lfs_crc(&commit->crc, &footer[0], sizeof(footer[0])); - footer[1] = lfs_tole32(commit->crc); - err = lfs_bd_prog(lfs, commit->block, commit->off, - footer, sizeof(footer)); - if (err) { - return err; - } - commit->off += sizeof(tag)+lfs_tagsize(tag); - commit->ptag = tag; - - // flush buffers - err = lfs_bd_sync(lfs); - if (err) { - return err; - } - - // successful commit, check checksum to make sure - uint32_t crc = 0xffffffff; - err = lfs_bd_crc(lfs, commit->block, commit->begin, - commit->off-lfs_tagsize(tag) - commit->begin, &crc); - if (err) { - return err; - } - - if (crc != commit->crc) { - return LFS_ERR_CORRUPT; - } + commit->off += size; + commit->ptag = tag & 0x7fffffff; return 0; } -// TODO predeclare -static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, - uint32_t etag, uint32_t getmask, uint32_t gettag, void *buffer, - bool stopatcommit) { - uint16_t id = lfs_tagid(gettag); - lfs_size_t size = lfs_tagsize(gettag); - - // synthetic moves - if ((block == lfs->globals.move.pair[0] || - block == lfs->globals.move.pair[1]) - && lfs_tagid(gettag) <= lfs->globals.move.id) { - gettag += LFS_MKTAG(0, 1, 0); - } - - // iterate over dir block backwards (for faster lookups) - off += sizeof(uint32_t); // TODO nah - uint32_t tag = etag; - - while (off > 2*sizeof(tag)) { - LFS_ASSERT(off > 2*sizeof(tag)+lfs_tagsize(tag)); - off -= sizeof(tag)+lfs_tagsize(tag); - - if (lfs_tagtype(tag) == LFS_TYPE_CRC && stopatcommit) { - break; - } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { - if (lfs_tagid(tag) <= lfs_tagid(gettag)) { - gettag += LFS_MKTAG(0, 1, 0); - } - } else if ((tag & getmask) == (gettag & getmask)) { - if (buffer) { - lfs_size_t diff = lfs_min(size, lfs_tagsize(tag)); - int err = lfs_bd_read(lfs, block, off, buffer, diff); - if (err) { - return err; - } - - memset((uint8_t*)buffer + diff, 0, size - diff); - } - - return LFS_MKTAG(0, id, 0) | (tag & 0xffc00fff); - } - - uint32_t ntag; - int err = lfs_bd_read(lfs, block, - off-sizeof(ntag), &ntag, sizeof(ntag)); - if (err) { - return err; - } - tag ^= lfs_fromle32(ntag); - } - - return LFS_ERR_NOENT; -} - -static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t getmask, uint32_t gettag, void *buffer); - -static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, const lfs_mdir_t *dir, const lfs_mattr_t *attrs) { - // Iterate through list and commits, only committing unique entries - lfs_off_t off = dir->off + sizeof(uint32_t); + // iterate through list and commits, only committing unique entries + lfs_off_t off = dir->off; uint32_t ntag = dir->etag; - - while (attrs || off > 2*sizeof(uint32_t)) { + while (attrs || off > sizeof(uint32_t)) { struct lfs_diskoff disk; uint32_t tag; const void *buffer; @@ -692,16 +601,15 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, buffer = attrs->buffer; attrs = attrs->next; } else { - LFS_ASSERT(off > 2*sizeof(ntag)+lfs_tagsize(ntag)); + LFS_ASSERT(off > sizeof(ntag)+lfs_tagsize(ntag)); off -= sizeof(ntag)+lfs_tagsize(ntag); tag = ntag; - disk.block = dir->pair[0]; - disk.off = off; buffer = &disk; + disk.block = dir->pair[0]; + disk.off = off + sizeof(tag); - int err = lfs_bd_read(lfs, dir->pair[0], - off-sizeof(ntag), &ntag, sizeof(ntag)); + int err = lfs_bd_read(lfs, dir->pair[0], off, &ntag, sizeof(ntag)); if (err) { return err; } @@ -718,20 +626,19 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, // ignore non-matching ids } else { // check if type has already been committed - int32_t res = lfs_commit_get(lfs, commit->block, + int32_t res = lfs_commitget(lfs, commit->block, commit->off, commit->ptag, lfs_tagisuser(tag) ? 0x7ffff000 : 0x7c3ff000, - LFS_MKTAG(lfs_tagtype(tag), toid-commit->filter.begin, 0), - // TODO can all these filter adjustments be consolidated? - NULL, true); + LFS_MKTAG(lfs_tagtype(tag), toid, 0), + 0, NULL, true); if (res < 0 && res != LFS_ERR_NOENT) { return res; } if (res == LFS_ERR_NOENT) { // update id and commit, as we are currently unique - int err = lfs_commit_attr(lfs, commit, - LFS_MKTAG(0, toid, 0) | (tag & 0xffc00fff), + int err = lfs_commitattr(lfs, commit, + (tag & 0xffc00fff) | LFS_MKTAG(0, toid, 0), buffer); if (err) { return err; @@ -743,23 +650,70 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, - const lfs_globals_t *source, const lfs_globals_t *diff) { - if (lfs_globals_iszero(diff)) { +static int lfs_commitglobals(lfs_t *lfs, struct lfs_commit *commit, + lfs_globals_t *locals) { + if (lfs_globalsiszero(&lfs->diff)) { return 0; } - // TODO check performance/complexity of different strategies here - lfs_globals_t res = lfs_globals_xor(source, diff); - int err = lfs_commit_attr(lfs, commit, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0x3ff, sizeof(res)), &res); + lfs_globalsxor(locals, &lfs->diff); + int err = lfs_commitattr(lfs, commit, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0x3ff, sizeof(*locals)), locals); + lfs_globalsxor(locals, &lfs->diff); + return err; +} + +static int lfs_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { + // align to program units + lfs_off_t off = lfs_alignup(commit->off + 2*sizeof(uint32_t), + lfs->cfg->prog_size); + + // read erased state from next program unit + uint32_t tag; + int err = lfs_bd_read(lfs, commit->block, off, &tag, sizeof(tag)); if (err) { return err; } + // build crc tag + tag = (0x80000000 & ~lfs_fromle32(tag)) | + LFS_MKTAG(LFS_TYPE_CRC, 0x3ff, + off - (commit->off+sizeof(uint32_t))); + + // write out crc + uint32_t footer[2]; + footer[0] = lfs_tole32(tag ^ commit->ptag); + lfs_crc(&commit->crc, &footer[0], sizeof(footer[0])); + footer[1] = lfs_tole32(commit->crc); + err = lfs_bd_prog(lfs, commit->block, commit->off, footer, sizeof(footer)); + if (err) { + return err; + } + commit->off += sizeof(tag)+lfs_tagsize(tag); + commit->ptag = tag; + + // flush buffers + err = lfs_bd_sync(lfs); + if (err) { + return err; + } + + // successful commit, check checksum to make sure + uint32_t crc = 0xffffffff; + err = lfs_bd_crc(lfs, commit->block, commit->begin, + commit->off-lfs_tagsize(tag)-commit->begin, &crc); + if (err) { + return err; + } + + if (crc != commit->crc) { + return LFS_ERR_CORRUPT; + } + return 0; } +// internal dir operations static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, bool split, const lfs_block_t tail[2]) { // allocate pair of dir blocks (backwards, so we write to block 1 first) @@ -786,186 +740,14 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, dir->tail[1] = tail[1]; dir->erased = false; dir->split = split; - dir->globals = (lfs_globals_t){0}; + dir->locals = (lfs_globals_t){0}; // don't write out yet, let caller take care of that return 0; } -static int32_t lfs_dir_find(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2], - uint32_t findmask, uint32_t findtag, - const void *findbuffer) { - dir->pair[0] = pair[0]; - dir->pair[1] = pair[1]; - int32_t foundtag = LFS_ERR_NOENT; - - // find the block with the most recent revision - uint32_t rev[2]; - for (int i = 0; i < 2; i++) { - int err = lfs_bd_read(lfs, dir->pair[i], 0, &rev[i], sizeof(rev[i])); - rev[i] = lfs_fromle32(rev[i]); - if (err) { - return err; - } - } - - if (lfs_scmp(rev[1], rev[0]) > 0) { - lfs_pairswap(dir->pair); - lfs_pairswap(rev); - } - - // load blocks and check crc - for (int i = 0; i < 2; i++) { - lfs_off_t off = sizeof(dir->rev); - uint32_t ptag = 0; - uint32_t crc = 0xffffffff; - dir->tail[0] = 0xffffffff; - dir->tail[1] = 0xffffffff; - dir->count = 0; - dir->split = false; - dir->globals = (lfs_globals_t){0}; - - dir->rev = lfs_tole32(rev[0]); - lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); - dir->rev = lfs_fromle32(dir->rev); - - lfs_mdir_t tempdir = *dir; - uint32_t tempfoundtag = foundtag; - - while (true) { - // extract next tag - uint32_t tag; - int err = lfs_bd_read(lfs, tempdir.pair[0], - off, &tag, sizeof(tag)); - if (err) { - return err; - } - - lfs_crc(&crc, &tag, sizeof(tag)); - tag = lfs_fromle32(tag) ^ ptag; - - // next commit not yet programmed - if (lfs_tagtype(ptag) == LFS_TYPE_CRC && !lfs_tagisvalid(tag)) { - dir->erased = true; - goto done; - } - - // check we're in valid range - if (off + sizeof(tag)+lfs_tagsize(tag) > lfs->cfg->block_size) { - break; - } - - if (lfs_tagtype(tag) == LFS_TYPE_CRC) { - // check the crc attr - uint32_t dcrc; - int err = lfs_bd_read(lfs, tempdir.pair[0], - off+sizeof(tag), &dcrc, sizeof(dcrc)); - if (err) { - return err; - } - - if (crc != lfs_fromle32(dcrc)) { - if (off == sizeof(tempdir.rev)) { - // try other block - break; - } else { - // consider what we have good enough - dir->erased = false; - goto done; - } - } - - tempdir.off = off + sizeof(tag)+lfs_tagsize(tag); - tempdir.etag = tag; - crc = 0xffffffff; - *dir = tempdir; - foundtag = tempfoundtag; - } else { - err = lfs_bd_crc(lfs, tempdir.pair[0], - off+sizeof(tag), lfs_tagsize(tag), &crc); - if (err) { - return err; - } - - if (lfs_tagid(tag) < 0x3ff && - lfs_tagid(tag) >= tempdir.count) { - tempdir.count = lfs_tagid(tag)+1; - } - - if (lfs_tagsubtype(tag) == LFS_TYPE_TAIL) { - tempdir.split = (lfs_tagtype(tag) & 1); - err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), - tempdir.tail, sizeof(tempdir.tail)); - if (err) { - return err; - } - } else if (lfs_tagtype(tag) == LFS_TYPE_GLOBALS) { - err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), - &tempdir.globals, sizeof(tempdir.globals)); - if (err) { - return err; - } - } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { - tempdir.count -= 1; - - if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { - tempfoundtag = LFS_ERR_NOENT; - } else if (lfs_tagisvalid(tempfoundtag) && - lfs_tagid(tag) < lfs_tagid(tempfoundtag)) { - tempfoundtag -= LFS_MKTAG(0, 1, 0); - } - } else if ((tag & findmask) == (findtag & findmask)) { - int res = lfs_bd_cmp(lfs, tempdir.pair[0], off+sizeof(tag), - findbuffer, lfs_tagsize(tag)); - if (res < 0) { - return res; - } - - if (res) { - // found a match - tempfoundtag = tag; - } - } - } - - ptag = tag; - off += sizeof(tag)+lfs_tagsize(tag); - } - - // failed, try the other crc? - lfs_pairswap(dir->pair); - lfs_pairswap(rev); - } - - LFS_ERROR("Corrupted dir pair at %d %d", dir->pair[0], dir->pair[1]); - return LFS_ERR_CORRUPT; - -done: - // synthetic move - if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { - if (lfs->globals.move.id == lfs_tagid(foundtag)) { - foundtag = LFS_ERR_NOENT; - } else if (lfs_tagisvalid(foundtag) && - lfs->globals.move.id < lfs_tagid(foundtag)) { - foundtag -= LFS_MKTAG(0, 1, 0); - } - } - - return foundtag; -} - -static int lfs_dir_fetch(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2]) { - int32_t res = lfs_dir_find(lfs, dir, pair, 0xffffffff, 0xffffffff, NULL); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - - return 0; -} - -static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs, +static int lfs_dir_compact(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_mattr_t *attrs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; @@ -973,11 +755,8 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs, // There's nothing special about our global delta, so feed it back // into the global global delta - // TODO IMMENSE HMM globals get bleed into from above, need to be fixed after commits due to potential moves - lfs_globals_t gtemp = dir->globals; // TODO hmm, why did we have different variables then? - - lfs->diff = lfs_globals_xor(&lfs->diff, &dir->globals); - dir->globals = (lfs_globals_t){0}; + lfs_globalsxor(&lfs->diff, &dir->locals); + dir->locals = (lfs_globals_t){0}; // increment revision count dir->rev += 1; @@ -1013,25 +792,21 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs, struct lfs_commit commit = { .block = dir->pair[1], .off = sizeof(dir->rev), - - // space is complicated, we need room for tail, crc, idelete, - // and we keep cap at around half a block - .begin = 0, - .end = lfs_min( - lfs_alignup(lfs->cfg->block_size / 2, - lfs->cfg->prog_size), - lfs->cfg->block_size - 5*sizeof(uint32_t)), .crc = crc, .ptag = 0, - // filter out ids - .filter.begin = begin, - .filter.end = end, + // space is complicated, we need room for tail, crc, globals, + // and we cap at half a block to give room for metadata updates + .begin = 0, + .end = lfs_min( + lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size), + lfs->cfg->block_size - 34), }; - if (!relocated) { - err = lfs_commit_globals(lfs, &commit, - &dir->globals, &lfs->diff); + // commit with a move + for (uint16_t id = begin; id < end; id++) { + err = lfs_commitmove(lfs, &commit, + id, id - begin, source, attrs); if (err) { if (err == LFS_ERR_NOSPC) { goto split; @@ -1040,30 +815,27 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs, } return err; } + + ack = id; } - // commit with a move - for (uint16_t id = begin; id < end; id++) { - err = lfs_commit_move(lfs, &commit, id, id, source, attrs); + // reopen reserved space at the end + commit.end = lfs->cfg->block_size - 8; + + if (!relocated) { + err = lfs_commitglobals(lfs, &commit, &dir->locals); if (err) { - if (err == LFS_ERR_NOSPC) { - goto split; - } else if (err == LFS_ERR_CORRUPT) { + if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } - - ack = id; } - // reopen reserved space at the end - commit.end = lfs->cfg->block_size - 2*sizeof(uint32_t); - if (!lfs_pairisnull(dir->tail)) { // commit tail, which may be new after last size check // TODO le32 - err = lfs_commit_attr(lfs, &commit, + err = lfs_commitattr(lfs, &commit, LFS_MKTAG(LFS_TYPE_TAIL + dir->split, 0x3ff, sizeof(dir->tail)), dir->tail); if (err) { @@ -1074,7 +846,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs, } } - err = lfs_commit_crc(lfs, &commit); + err = lfs_commitcrc(lfs, &commit); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1137,7 +909,11 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs, continue; } - if (relocated) { + if (!relocated) { + // successful commit, update globals + lfs_globalsxor(&dir->locals, &lfs->diff); + lfs->diff = (lfs_globals_t){0}; + } else { // update references if we relocated LFS_DEBUG("Relocating %d %d to %d %d", oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); @@ -1145,17 +921,13 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs, if (err) { return err; } - } else { - lfs->globals = lfs_globals_xor(&lfs->globals, &lfs->diff); - lfs->diff = (lfs_globals_t){0}; } - lfs->globals = lfs_globals_xor(&lfs->globals, >emp); // TODO hmm, why did we have different variables then? - return 0; } -static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs) { +static int lfs_dir_commit(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_mattr_t *attrs) { while (true) { if (!dir->erased) { // not erased, must compact @@ -1164,17 +936,16 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs) { struct lfs_commit commit = { .block = dir->pair[0], - .begin = dir->off, .off = dir->off, - .end = lfs->cfg->block_size - 2*sizeof(uint32_t), .crc = 0xffffffff, .ptag = dir->etag, - .filter.begin = 0, - .filter.end = 0x3ff, + + .begin = dir->off, + .end = lfs->cfg->block_size - 8, }; - for (lfs_mattr_t *a = attrs; a; a = a->next) { - int err = lfs_commit_attr(lfs, &commit, a->tag, a->buffer); + for (const lfs_mattr_t *a = attrs; a; a = a->next) { + int err = lfs_commitattr(lfs, &commit, a->tag, a->buffer); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1183,7 +954,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs) { } } - int err = lfs_commit_globals(lfs, &commit, &dir->globals, &lfs->diff); + int err = lfs_commitglobals(lfs, &commit, &dir->locals); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1191,7 +962,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs) { return err; } - err = lfs_commit_crc(lfs, &commit); + err = lfs_commitcrc(lfs, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1199,12 +970,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs) { return err; } - // successful commit, lets update dir + // successful commit, update dir dir->off = commit.off; dir->etag = commit.ptag; -// // TODO hm -// dir->globals = lfs_globals_xor(&dir->globals, &lfs->diff); - lfs->globals = lfs_globals_xor(&lfs->globals, &lfs->diff); + // successful commit, update globals + lfs_globalsxor(&dir->locals, &lfs->diff); lfs->diff = (lfs_globals_t){0}; break; @@ -1229,6 +999,192 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t *attrs) { return 0; } +static int32_t lfs_dir_find(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2], + uint32_t findmask, uint32_t findtag, + const void *findbuffer) { + dir->pair[0] = pair[0]; + dir->pair[1] = pair[1]; + int32_t foundtag = LFS_ERR_NOENT; + + // find the block with the most recent revision + uint32_t rev[2]; + for (int i = 0; i < 2; i++) { + int err = lfs_bd_read(lfs, dir->pair[i], 0, &rev[i], sizeof(rev[i])); + rev[i] = lfs_fromle32(rev[i]); + if (err) { + return err; + } + } + + if (lfs_scmp(rev[1], rev[0]) > 0) { + lfs_pairswap(dir->pair); + lfs_pairswap(rev); + } + + // load blocks and check crc + for (int i = 0; i < 2; i++) { + lfs_off_t off = sizeof(dir->rev); + uint32_t ptag = 0; + uint32_t crc = 0xffffffff; + dir->tail[0] = 0xffffffff; + dir->tail[1] = 0xffffffff; + dir->count = 0; + dir->split = false; + dir->locals = (lfs_globals_t){0}; + + dir->rev = lfs_tole32(rev[0]); + lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); + dir->rev = lfs_fromle32(dir->rev); + + lfs_mdir_t tempdir = *dir; + uint32_t tempfoundtag = foundtag; + + while (true) { + // extract next tag + uint32_t tag; + int err = lfs_bd_read(lfs, tempdir.pair[0], + off, &tag, sizeof(tag)); + if (err) { + return err; + } + + lfs_crc(&crc, &tag, sizeof(tag)); + tag = lfs_fromle32(tag) ^ ptag; + + // next commit not yet programmed + if (lfs_tagtype(ptag) == LFS_TYPE_CRC && !lfs_tagisvalid(tag)) { + dir->erased = true; + goto done; + } + + // check we're in valid range + if (off + sizeof(tag)+lfs_tagsize(tag) > lfs->cfg->block_size) { + break; + } + + if (lfs_tagtype(tag) == LFS_TYPE_CRC) { + // check the crc attr + uint32_t dcrc; + int err = lfs_bd_read(lfs, tempdir.pair[0], + off+sizeof(tag), &dcrc, sizeof(dcrc)); + if (err) { + return err; + } + + if (crc != lfs_fromle32(dcrc)) { + if (off == sizeof(tempdir.rev)) { + // try other block + break; + } else { + // consider what we have good enough + dir->erased = false; + goto done; + } + } + + tempdir.off = off + sizeof(tag)+lfs_tagsize(tag); + tempdir.etag = tag; + crc = 0xffffffff; + *dir = tempdir; + foundtag = tempfoundtag; + } else { + err = lfs_bd_crc(lfs, tempdir.pair[0], + off+sizeof(tag), lfs_tagsize(tag), &crc); + if (err) { + return err; + } + + if (lfs_tagid(tag) < 0x3ff && + lfs_tagid(tag) >= tempdir.count) { + tempdir.count = lfs_tagid(tag)+1; + } + + if (lfs_tagsubtype(tag) == LFS_TYPE_TAIL) { + tempdir.split = (lfs_tagtype(tag) & 1); + err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), + tempdir.tail, sizeof(tempdir.tail)); + if (err) { + return err; + } + } else if (lfs_tagtype(tag) == LFS_TYPE_GLOBALS) { + err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), + &tempdir.locals, sizeof(tempdir.locals)); + if (err) { + return err; + } + } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { + tempdir.count -= 1; + + if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { + tempfoundtag = LFS_ERR_NOENT; + } else if (lfs_tagisvalid(tempfoundtag) && + lfs_tagid(tag) < lfs_tagid(tempfoundtag)) { + tempfoundtag -= LFS_MKTAG(0, 1, 0); + } + } else if ((tag & findmask) == (findtag & findmask)) { + int res = lfs_bd_cmp(lfs, tempdir.pair[0], off+sizeof(tag), + findbuffer, lfs_tagsize(tag)); + if (res < 0) { + return res; + } + + if (res) { + // found a match + tempfoundtag = tag; + } + } + } + + ptag = tag; + off += sizeof(tag)+lfs_tagsize(tag); + } + + // failed, try the other crc? + lfs_pairswap(dir->pair); + lfs_pairswap(rev); + } + + LFS_ERROR("Corrupted dir pair at %d %d", dir->pair[0], dir->pair[1]); + return LFS_ERR_CORRUPT; + +done: + // synthetic move + if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { + if (lfs->globals.move.id == lfs_tagid(foundtag)) { + foundtag = LFS_ERR_NOENT; + } else if (lfs_tagisvalid(foundtag) && + lfs->globals.move.id < lfs_tagid(foundtag)) { + foundtag -= LFS_MKTAG(0, 1, 0); + } + } + + return foundtag; +} + +static int lfs_dir_fetch(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2]) { + int32_t res = lfs_dir_find(lfs, dir, pair, 0xffffffff, 0xffffffff, NULL); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + + return 0; +} + +static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, + uint32_t getmask, uint32_t gettag, void *buffer) { + int32_t difftag = 0; + if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 && + lfs_tagid(gettag) <= lfs->globals.move.id) { + // synthetic moves + difftag = LFS_MKTAG(0, 1, 0); + } + + return lfs_commitget(lfs, dir->pair[0], dir->off, dir->etag, + getmask, gettag, difftag, buffer, false); +} + static int lfs_dir_append(lfs_t *lfs, lfs_mdir_t *dir, uint16_t *id) { *id = dir->count; dir->count += 1; @@ -1247,12 +1203,11 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { } if (err != LFS_ERR_NOENT && pdir.split) { - // steal tail, and global state + // steal tail and global state pdir.split = dir->split; pdir.tail[0] = dir->tail[0]; pdir.tail[1] = dir->tail[1]; - lfs->diff = dir->globals; - lfs->globals = lfs_globals_xor(&lfs->globals, &dir->globals); + lfs_globalsxor(&lfs->diff, &dir->locals); return lfs_dir_commit(lfs, &pdir, LFS_MKATTR(LFS_TYPE_TAIL + pdir.split, 0x3ff, @@ -1292,42 +1247,6 @@ static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { return 0; } -static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t getmask, uint32_t gettag, void *buffer) { - return lfs_commit_get(lfs, dir->pair[0], dir->off, dir->etag, - getmask, gettag, buffer, false); -} - -static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, - int16_t id, struct lfs_info *info) { - int32_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_size+1), info->name); - if (tag < 0) { - return tag; - } - - info->type = lfs_tagtype(tag); - if (lfs_tagsize(tag) > lfs->name_size) { - return LFS_ERR_RANGE; - } - - struct lfs_ctz ctz; - tag = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); - if (tag < 0) { - return tag; - } - - if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { - info->size = ctz.size; - } else if (lfs_tagtype(tag) == LFS_TYPE_INLINESTRUCT) { - info->size = lfs_tagsize(tag); - } - - return 0; -} - - static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; const char *name = *path; @@ -1423,6 +1342,35 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { } } +static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, + int16_t id, struct lfs_info *info) { + int32_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_size+1), info->name); + if (tag < 0) { + return tag; + } + + info->type = lfs_tagtype(tag); + if (lfs_tagsize(tag) > lfs->name_size) { + return LFS_ERR_RANGE; + } + + struct lfs_ctz ctz; + tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); + if (tag < 0) { + return tag; + } + + if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { + info->size = ctz.size; + } else if (lfs_tagtype(tag) == LFS_TYPE_INLINESTRUCT) { + info->size = lfs_tagsize(tag); + } + + return 0; +} + /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron @@ -1638,7 +1586,7 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { /// File index list operations /// -static int lfs_ctz_index(lfs_t *lfs, lfs_off_t *off) { +static int lfs_ctzindex(lfs_t *lfs, lfs_off_t *off) { lfs_off_t size = *off; lfs_off_t b = lfs->cfg->block_size - 2*4; lfs_off_t i = size / b; @@ -1651,7 +1599,7 @@ static int lfs_ctz_index(lfs_t *lfs, lfs_off_t *off) { return i; } -static int lfs_ctz_find(lfs_t *lfs, +static int lfs_ctzfind(lfs_t *lfs, lfs_cache_t *rcache, const lfs_cache_t *pcache, lfs_block_t head, lfs_size_t size, lfs_size_t pos, lfs_block_t *block, lfs_off_t *off) { @@ -1661,8 +1609,8 @@ static int lfs_ctz_find(lfs_t *lfs, return 0; } - lfs_off_t current = lfs_ctz_index(lfs, &(lfs_off_t){size-1}); - lfs_off_t target = lfs_ctz_index(lfs, &pos); + lfs_off_t current = lfs_ctzindex(lfs, &(lfs_off_t){size-1}); + lfs_off_t target = lfs_ctzindex(lfs, &pos); while (current > target) { lfs_size_t skip = lfs_min( @@ -1684,7 +1632,7 @@ static int lfs_ctz_find(lfs_t *lfs, return 0; } -static int lfs_ctz_extend(lfs_t *lfs, +static int lfs_ctzextend(lfs_t *lfs, lfs_cache_t *rcache, lfs_cache_t *pcache, lfs_block_t head, lfs_size_t size, lfs_block_t *block, lfs_off_t *off) { @@ -1713,7 +1661,7 @@ static int lfs_ctz_extend(lfs_t *lfs, } size -= 1; - lfs_off_t index = lfs_ctz_index(lfs, &size); + lfs_off_t index = lfs_ctzindex(lfs, &size); size += 1; // just copy out the last block if it is incomplete @@ -1782,7 +1730,7 @@ static int lfs_ctz_extend(lfs_t *lfs, } } -static int lfs_ctz_traverse(lfs_t *lfs, +static int lfs_ctztraverse(lfs_t *lfs, lfs_cache_t *rcache, const lfs_cache_t *pcache, lfs_block_t head, lfs_size_t size, int (*cb)(lfs_t*, void*, lfs_block_t), void *data) { @@ -1790,7 +1738,7 @@ static int lfs_ctz_traverse(lfs_t *lfs, return 0; } - lfs_off_t index = lfs_ctz_index(lfs, &(lfs_off_t){size-1}); + lfs_off_t index = lfs_ctzindex(lfs, &(lfs_off_t){size-1}); while (true) { int err = cb(lfs, data, head); @@ -2159,7 +2107,7 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { if (!(file->flags & LFS_F_INLINE)) { - int err = lfs_ctz_find(lfs, &file->cache, NULL, + int err = lfs_ctzfind(lfs, &file->cache, NULL, file->ctz.head, file->ctz.size, file->pos, &file->block, &file->off); if (err) { @@ -2248,7 +2196,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, if (!(file->flags & LFS_F_INLINE)) { if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { // find out which block we're extending from - int err = lfs_ctz_find(lfs, &file->cache, NULL, + int err = lfs_ctzfind(lfs, &file->cache, NULL, file->ctz.head, file->ctz.size, file->pos-1, &file->block, &file->off); if (err) { @@ -2262,7 +2210,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // extend file with new blocks lfs_alloc_ack(lfs); - int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache, + int err = lfs_ctzextend(lfs, &lfs->rcache, &file->cache, file->block, file->pos, &file->block, &file->off); if (err) { @@ -2353,7 +2301,7 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { } // lookup new head in ctz skip list - err = lfs_ctz_find(lfs, &file->cache, NULL, + err = lfs_ctzfind(lfs, &file->cache, NULL, file->ctz.head, file->ctz.size, size, &file->ctz.head, &(lfs_off_t){0}); if (err) { @@ -2653,6 +2601,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs->diff.move.pair[0] = oldcwd.pair[0] ^ lfs->globals.move.pair[0]; lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; lfs->diff.move.id = lfs_tagid(oldtag) ^ lfs->globals.move.id; + lfs->globals.move.pair[0] = oldcwd.pair[0]; + lfs->globals.move.pair[1] = oldcwd.pair[1]; + lfs->globals.move.id = lfs_tagid(oldtag); // move over all attributes int err = lfs_dir_commit(lfs, &newcwd, @@ -2675,9 +2626,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - // TODO test for global state stealing? // steal global state - lfs->globals = lfs_globals_xor(&lfs->globals, &prevdir.globals); + // TODO test for global state stealing? + lfs_globalsxor(&lfs->diff, &prevdir.locals); newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; @@ -2828,8 +2779,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->deorphaned = false; lfs->globals.move.pair[0] = 0xffffffff; lfs->globals.move.pair[1] = 0xffffffff; - lfs->globals.move.id = 0x3ff; - lfs->diff = (lfs_globals_t){0}; + lfs->globals.move.id = 0x3ff; // scan for any global updates // TODO rm me? need to grab any inits @@ -3055,7 +3005,7 @@ int lfs_fs_traverse(lfs_t *lfs, } if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { - int err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, + int err = lfs_ctztraverse(lfs, &lfs->rcache, NULL, ctz.head, ctz.size, cb, data); if (err) { return err; @@ -3067,7 +3017,7 @@ int lfs_fs_traverse(lfs_t *lfs, // iterate over any open files for (lfs_file_t *f = lfs->files; f; f = f->next) { if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { - int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, + int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, f->ctz.head, f->ctz.size, cb, data); if (err) { return err; @@ -3075,7 +3025,7 @@ int lfs_fs_traverse(lfs_t *lfs, } if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { - int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, + int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, f->block, f->pos, cb, data); if (err) { return err; @@ -3120,7 +3070,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { dir.off += lfs_entry_size(&entry); if ((0x70 & entry.d.type) == LFS_TYPE_CTZSTRUCT) { - err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, + err = lfs_ctztraverse(lfs, &lfs->rcache, NULL, entry.d.u.file.head, entry.d.u.file.size, cb, data); if (err) { return err; @@ -3139,7 +3089,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { // iterate over any open files for (lfs_file_t *f = lfs->files; f; f = f->next) { if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { - int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, + int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, f->head, f->size, cb, data); if (err) { return err; @@ -3147,7 +3097,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { - int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, + int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, f->block, f->pos, cb, data); if (err) { return err; @@ -3238,6 +3188,7 @@ static int lfs_relocate(lfs_t *lfs, return err; } + // if we can't find dir, it must be new if (err != LFS_ERR_NOENT) { // just replace bad pair, no desync can occur parent.tail[0] = newpair[0]; @@ -3262,7 +3213,6 @@ static int lfs_relocate(lfs_t *lfs, } } - // couldn't find dir, must be new return 0; } @@ -3272,7 +3222,7 @@ int lfs_scan(lfs_t *lfs) { } lfs_mdir_t dir = {.tail = {0, 1}}; - lfs_globals_t globals = {{{0xffffffff, 0xffffffff}, 0x3ff}}; + lfs->diff = (lfs_globals_t){0}; // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { @@ -3282,11 +3232,13 @@ int lfs_scan(lfs_t *lfs) { } // xor together indirect deletes - globals = lfs_globals_xor(&globals, &dir.globals); + lfs_globalsxor(&lfs->diff, &dir.locals); } // update littlefs with globals - lfs->globals = globals; + // TODO does this only run once? + // TODO Should we inline this into init?? + lfs_globalsxor(&lfs->globals, &lfs->diff); lfs->diff = (lfs_globals_t){0}; if (!lfs_pairisnull(lfs->globals.move.pair)) { LFS_DEBUG("Found move %d %d %d", @@ -3321,6 +3273,9 @@ int lfs_fixmove(lfs_t *lfs) { return err; } + lfs->globals.move.pair[0] = 0xffffffff; + lfs->globals.move.pair[1] = 0xffffffff; + lfs->globals.move.id = 0x3ff; return 0; } diff --git a/lfs.h b/lfs.h index 9aecf89e..121b3590 100644 --- a/lfs.h +++ b/lfs.h @@ -263,7 +263,7 @@ struct lfs_attr { /// littlefs data structures /// typedef struct lfs_mattr { - struct lfs_mattr *next; + const struct lfs_mattr *next; int32_t tag; const void *buffer; } lfs_mattr_t; @@ -284,7 +284,7 @@ typedef struct lfs_mdir { uint16_t count; bool erased; bool split; - lfs_globals_t globals; + lfs_globals_t locals; } lfs_mdir_t; typedef struct lfs_cache { @@ -353,6 +353,7 @@ typedef struct lfs { lfs_free_t free; bool deorphaned; + lfs_globals_t globals2; lfs_globals_t globals; lfs_globals_t diff; From d9a24d0a2b7814ad8d7dabe9382301cd5c2c9ac8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 17 Jul 2018 18:31:30 -0500 Subject: [PATCH 062/139] Fixed move handling when caught in a relocate This was a surprisingly tricky issue. One of the subtle requirements for the new move handling to work is that the block containing the move does not change until the move is resolved. Initially, this seemed easy to implement, given that a move is always immediately followed by its resolution. However, the extra metadata-pair operations needed to maintain integrity present a challenge. At any commit, a directory block may end up moved as a side effect of relocation due to a bad block. The fix here is to move the move resolution directly into the commit logic. This means that any commit to a block containing a move will be implicitly resolved, leaving the later attempt at move resolution as a noop. This fix required quite a bit of restructuring, but as a nice side-effect some of the complexity around moves actually went away. Additionally, the new move handling is surprisingly powerful at combining moves with nearby commits. And we now get same-metadata-pair renames for free! A win for procrasination on that minor feature. --- lfs.c | 282 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 138 insertions(+), 144 deletions(-) diff --git a/lfs.c b/lfs.c index 0409d29a..bbd9f8b6 100644 --- a/lfs.c +++ b/lfs.c @@ -430,7 +430,7 @@ static inline bool lfs_pairsync( (((uint32_t)(type) << 22) | ((uint32_t)(id) << 12) | (uint32_t)(size)) #define LFS_MKATTR(type, id, buffer, size, next) \ - &(lfs_mattr_t){(next), LFS_MKTAG(type, id, size), (buffer)} + &(const lfs_mattr_t){(next), LFS_MKTAG(type, id, size), (buffer)} static inline bool lfs_tagisvalid(uint32_t tag) { return !(tag & 0x80000000); @@ -485,7 +485,7 @@ struct lfs_diskoff { }; static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, - uint32_t tag, uint32_t getmask, uint32_t gettag, int32_t difftag, + uint32_t tag, uint32_t getmask, uint32_t gettag, int32_t getdiff, void *buffer, bool stopatcommit) { // iterate over dir block backwards (for faster lookups) while (off > sizeof(tag)) { @@ -495,10 +495,10 @@ static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, if (lfs_tagtype(tag) == LFS_TYPE_CRC && stopatcommit) { break; } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { - if (lfs_tagid(tag) <= lfs_tagid(gettag + difftag)) { - difftag += LFS_MKTAG(0, 1, 0); + if (lfs_tagid(tag) <= lfs_tagid(gettag + getdiff)) { + getdiff += LFS_MKTAG(0, 1, 0); } - } else if ((tag & getmask) == ((gettag + difftag) & getmask)) { + } else if ((tag & getmask) == ((gettag + getdiff) & getmask)) { if (buffer) { lfs_size_t diff = lfs_min( lfs_tagsize(gettag), lfs_tagsize(tag)); @@ -512,7 +512,7 @@ static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, lfs_tagsize(gettag) - diff); } - return tag - difftag; + return tag - getdiff; } uint32_t ntag; @@ -926,14 +926,67 @@ static int lfs_dir_compact(lfs_t *lfs, return 0; } -static int lfs_dir_commit(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_mattr_t *attrs) { - while (true) { - if (!dir->erased) { - // not erased, must compact - goto compact; +static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, + const lfs_mattr_t *attrs) { + bool canceling = (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0); + lfs_mattr_t cancel; + if (canceling) { + // Wait, we have the move? Just cancel this out here + // We need to, or else the move can become outdated + lfs->diff.move.pair[0] ^= 0xffffffff ^ lfs->globals.move.pair[0]; + lfs->diff.move.pair[1] ^= 0xffffffff ^ lfs->globals.move.pair[1]; + lfs->diff.move.id ^= 0x3ff ^ lfs->globals.move.id; + + cancel.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.move.id, 0); + cancel.next = attrs; + attrs = &cancel; + } + + // calculate new directory size + uint32_t deletetag = 0xffffffff; + for (const lfs_mattr_t *a = attrs; a; a = a->next) { + if (lfs_tagid(a->tag) < 0x3ff && lfs_tagid(a->tag) >= dir->count) { + dir->count = lfs_tagid(a->tag)+1; } + if (lfs_tagtype(a->tag) == LFS_TYPE_DELETE) { + LFS_ASSERT(dir->count > 0); + dir->count -= 1; + deletetag = a->tag; + + if (dir->count == 0) { + // should we actually drop the directory block? + lfs_mdir_t pdir; + int err = lfs_pred(lfs, dir->pair, &pdir); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + if (err != LFS_ERR_NOENT && pdir.split) { + // steal tail and global state + pdir.split = dir->split; + pdir.tail[0] = dir->tail[0]; + pdir.tail[1] = dir->tail[1]; + lfs_globalsxor(&lfs->diff, &dir->locals); + return lfs_dir_commit(lfs, &pdir, + LFS_MKATTR(LFS_TYPE_TAIL + pdir.split, 0x3ff, + pdir.tail, sizeof(pdir.tail), + NULL)); + } + } + } + } + + if (!dir->erased) { +compact: + // fall back to compaction + lfs->pcache.block = 0xffffffff; + int err = lfs_dir_compact(lfs, dir, attrs, dir, 0, dir->count); + if (err) { + return err; + } + } else { + // try to commit struct lfs_commit commit = { .block = dir->pair[0], .off = dir->off, @@ -945,7 +998,20 @@ static int lfs_dir_commit(lfs_t *lfs, }; for (const lfs_mattr_t *a = attrs; a; a = a->next) { - int err = lfs_commitattr(lfs, &commit, a->tag, a->buffer); + if (lfs_tagtype(a->tag) != LFS_TYPE_DELETE) { + int err = lfs_commitattr(lfs, &commit, a->tag, a->buffer); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; + } + } + } + + if (lfs_tagisvalid(deletetag)) { + // special case for deletes, since order matters + int err = lfs_commitattr(lfs, &commit, deletetag, NULL); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -976,15 +1042,13 @@ static int lfs_dir_commit(lfs_t *lfs, // successful commit, update globals lfs_globalsxor(&dir->locals, &lfs->diff); lfs->diff = (lfs_globals_t){0}; - break; + } -compact: - lfs->pcache.block = 0xffffffff; - err = lfs_dir_compact(lfs, dir, attrs, dir, 0, dir->count); - if (err) { - return err; - } - break; + // update globals that are affected + if (canceling) { + lfs->globals.move.pair[0] = 0xffffffff; + lfs->globals.move.pair[1] = 0xffffffff; + lfs->globals.move.id = 0x3ff; } // update any directories that are affected @@ -992,10 +1056,24 @@ static int lfs_dir_commit(lfs_t *lfs, for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->m.pair, dir->pair) == 0) { d->m = *dir; + if (d->id > lfs_tagid(deletetag)) { + d->id -= 1; + d->pos -= 1; + } + } + } + + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0) { + if (f->id == lfs_tagid(deletetag)) { + f->pair[0] = 0xffffffff; + f->pair[1] = 0xffffffff; + } else if (f->id > lfs_tagid(deletetag)) { + f->id -= 1; + } } } - // TODO what if we relocated the block containing the move? return 0; } @@ -1114,6 +1192,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, return err; } } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { + LFS_ASSERT(tempdir.count > 0); tempdir.count -= 1; if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { @@ -1174,77 +1253,15 @@ static int lfs_dir_fetch(lfs_t *lfs, static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, uint32_t gettag, void *buffer) { - int32_t difftag = 0; + int32_t getdiff = 0; if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 && lfs_tagid(gettag) <= lfs->globals.move.id) { // synthetic moves - difftag = LFS_MKTAG(0, 1, 0); + getdiff = LFS_MKTAG(0, 1, 0); } return lfs_commitget(lfs, dir->pair[0], dir->off, dir->etag, - getmask, gettag, difftag, buffer, false); -} - -static int lfs_dir_append(lfs_t *lfs, lfs_mdir_t *dir, uint16_t *id) { - *id = dir->count; - dir->count += 1; - return 0; -} - -static int lfs_dir_delete(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id) { - dir->count -= 1; - - // check if we should drop the directory block - if (dir->count == 0) { - lfs_mdir_t pdir; - int err = lfs_pred(lfs, dir->pair, &pdir); - if (err && err != LFS_ERR_NOENT) { - return err; - } - - if (err != LFS_ERR_NOENT && pdir.split) { - // steal tail and global state - pdir.split = dir->split; - pdir.tail[0] = dir->tail[0]; - pdir.tail[1] = dir->tail[1]; - lfs_globalsxor(&lfs->diff, &dir->locals); - - return lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_TAIL + pdir.split, 0x3ff, - pdir.tail, sizeof(pdir.tail), - NULL)); - } - } - - int err = lfs_dir_commit(lfs, dir, - LFS_MKATTR(LFS_TYPE_DELETE, id, NULL, 0, - NULL)); - if (err) { - return err; - } - - // shift over any dirs/files that are affected - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->m.pair, dir->pair) == 0) { - if (d->id > id) { - d->id -= 1; - d->pos -= 1; - } - } - } - - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->id == id) { - f->pair[0] = 0xffffffff; - f->pair[1] = 0xffffffff; - } else if (f->id > id) { - f->id -= 1; - } - } - } - - return 0; + getmask, gettag, getdiff, buffer, false); } static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { @@ -1411,12 +1428,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } // get next slot and commit - uint16_t id; - err = lfs_dir_append(lfs, &cwd, &id); - if (err) { - return err; - } - + uint16_t id = cwd.count; cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; err = lfs_dir_commit(lfs, &cwd, @@ -1802,15 +1814,10 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // get next slot and create entry to remember name - uint16_t id; - int err = lfs_dir_append(lfs, &cwd, &id); - if (err) { - return err; - } - // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change // TODO don't use inline struct? just leave it out? - err = lfs_dir_commit(lfs, &cwd, + uint16_t id = cwd.count; + int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_REG, id, path, nlen, LFS_MKATTR(LFS_TYPE_INLINESTRUCT, id, NULL, 0, NULL))); @@ -2056,7 +2063,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { if (!(file->flags & LFS_F_INLINE)) { int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_CTZSTRUCT, file->id, - &file->ctz.head, 2*sizeof(uint32_t), + &file->ctz.head, sizeof(file->ctz), file->attrs)); if (err) { return err; @@ -2505,7 +2512,9 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // delete the entry - err = lfs_dir_delete(lfs, &cwd, lfs_tagid(tag)); + err = lfs_dir_commit(lfs, &cwd, + LFS_MKATTR(LFS_TYPE_DELETE, lfs_tagid(tag), NULL, 0, + NULL)); if (err) { return err; } @@ -2516,8 +2525,11 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } + // steal state + // TODO test for global state stealing? cwd.tail[0] = dir.tail[0]; cwd.tail[1] = dir.tail[1]; + lfs_globalsxor(&lfs->diff, &dir.locals); err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), @@ -2591,10 +2603,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // get next id - int err = lfs_dir_append(lfs, &newcwd, &newid); - if (err) { - return err; - } + newid = newcwd.count; } // create move to fix later @@ -2614,10 +2623,13 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - // clean up after ourselves - err = lfs_fixmove(lfs); - if (err) { - return err; + // let commit clean up after move (if we're different! otherwise move + // logic already fixed it for us) + if (lfs_paircmp(oldcwd.pair, newcwd.pair) != 0) { + err = lfs_dir_commit(lfs, &oldcwd, NULL); + if (err) { + return err; + } } if (prevtag != LFS_ERR_NOENT && lfs_tagtype(prevtag) == LFS_TYPE_DIR) { @@ -2626,12 +2638,11 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - // steal global state + // steal state // TODO test for global state stealing? - lfs_globalsxor(&lfs->diff, &prevdir.locals); - newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; + lfs_globalsxor(&lfs->diff, &prevdir.locals); err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, newcwd.tail, sizeof(newcwd.tail), @@ -2859,7 +2870,6 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .name_size = lfs->name_size, }; - dir.count += 1; err = lfs_dir_commit(lfs, &dir, LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), LFS_MKATTR(LFS_TYPE_DIRSTRUCT, 0, lfs->root, sizeof(lfs->root), @@ -3250,35 +3260,6 @@ int lfs_scan(lfs_t *lfs) { return 0; } -int lfs_fixmove(lfs_t *lfs) { - LFS_DEBUG("Fixing move %d %d %d", // TODO move to just deorphan - lfs->globals.move.pair[0], - lfs->globals.move.pair[1], - lfs->globals.move.id); - - // mark global state to clear move entry - lfs->diff.move.pair[0] = 0xffffffff ^ lfs->globals.move.pair[0]; - lfs->diff.move.pair[1] = 0xffffffff ^ lfs->globals.move.pair[1]; - lfs->diff.move.id = 0x3ff ^ lfs->globals.move.id; - - // fetch and delete the moved entry - lfs_mdir_t movedir; - int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.move.pair); - if (err) { - return err; - } - - err = lfs_dir_delete(lfs, &movedir, lfs->globals.move.id); - if (err) { - return err; - } - - lfs->globals.move.pair[0] = 0xffffffff; - lfs->globals.move.pair[1] = 0xffffffff; - lfs->globals.move.id = 0x3ff; - return 0; -} - int lfs_deorphan(lfs_t *lfs) { lfs->deorphaned = true; if (lfs_pairisnull(lfs->root)) { // TODO rm me? @@ -3287,7 +3268,20 @@ int lfs_deorphan(lfs_t *lfs) { // Fix bad moves if (!lfs_pairisnull(lfs->globals.move.pair)) { - int err = lfs_fixmove(lfs); + LFS_DEBUG("Fixing move %d %d %d", // TODO move to just deorphan? + lfs->globals.move.pair[0], + lfs->globals.move.pair[1], + lfs->globals.move.id); + + // fetch and delete the moved entry + lfs_mdir_t movedir; + int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.move.pair); + if (err) { + return err; + } + + // rely on cancel logic inside commit + err = lfs_dir_commit(lfs, &movedir, NULL); if (err) { return err; } From 392b2ac79f428affebf19a05234b01e461292fef Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 28 Jul 2018 09:47:57 -0500 Subject: [PATCH 063/139] Refactored the updates of in-flight files/dirs Updated to account for changes as a result of commits/compacts. And changed instances of iteration over both files and dirs to use a single nested loop. This does rely implicitly on the structure layout of dirs/files and their location in lfs_t, which isn't great. But it gets the job done with less code duplication. --- lfs.c | 33 ++++++++++++++------------------- lfs.h | 4 ++-- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/lfs.c b/lfs.c index bbd9f8b6..6ff4c808 100644 --- a/lfs.c +++ b/lfs.c @@ -863,8 +863,6 @@ static int lfs_dir_compact(lfs_t *lfs, break; split: - // TODO update dirs that get split here? - // commit no longer fits, need to split dir, // drop caches and create tail lfs->pcache.block = 0xffffffff; @@ -923,6 +921,18 @@ static int lfs_dir_compact(lfs_t *lfs, } } + // update any dirs/files that are affected + for (int i = 0; i < 2; i++) { + for (lfs_file_t *f = ((lfs_file_t**)&lfs->files)[i]; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0 && + f->id >= begin && f->id < end) { + f->pair[0] = dir->pair[0]; + f->pair[1] = dir->pair[1]; + f->id -= begin; + } + } + } + return 0; } @@ -1052,19 +1062,17 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } // update any directories that are affected - // TODO what about pairs? what if we're splitting?? for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->m.pair, dir->pair) == 0) { d->m = *dir; if (d->id > lfs_tagid(deletetag)) { - d->id -= 1; d->pos -= 1; } } } - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { + for (int i = 0; i < 2; i++) { + for (lfs_file_t *f = ((lfs_file_t**)&lfs->files)[i]; f; f = f->next) { if (f->id == lfs_tagid(deletetag)) { f->pair[0] = 0xffffffff; f->pair[1] = 0xffffffff; @@ -3186,8 +3194,6 @@ static int lfs_relocate(lfs_t *lfs, lfs->root[1] = newpair[1]; } - // TODO update dir list!!? - // clean up bad block, which should now be a desync return lfs_deorphan(lfs); } @@ -3212,17 +3218,6 @@ static int lfs_relocate(lfs_t *lfs, } } - // shift over any dirs/files that are affected - for (int i = 0; i < 2; i++) { - for (lfs_dir_t *d = ((void*[2]){lfs->dirs, lfs->files})[i]; - d; d = d->next) { - if (lfs_paircmp(d->m.pair, oldpair) == 0) { - d->m.pair[0] = newpair[0]; - d->m.pair[1] = newpair[1]; - } - } - } - return 0; } diff --git a/lfs.h b/lfs.h index 121b3590..bd79e9b6 100644 --- a/lfs.h +++ b/lfs.h @@ -295,8 +295,8 @@ typedef struct lfs_cache { typedef struct lfs_file { struct lfs_file *next; - lfs_block_t pair[2]; uint16_t id; + lfs_block_t pair[2]; struct lfs_ctz { lfs_block_t head; lfs_size_t size; @@ -313,10 +313,10 @@ typedef struct lfs_file { typedef struct lfs_dir { struct lfs_dir *next; + uint16_t id; struct lfs_mdir m; lfs_block_t head[2]; - uint16_t id; lfs_off_t pos; } lfs_dir_t; From 3914cdf39febd4132a0926af105f85abb816dbbd Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 28 Jul 2018 12:07:18 -0500 Subject: [PATCH 064/139] Pulled in fixes for additional path corner cases Pulled in 015b86b. Merging this now avoids duplicate effort restructuring the path lookup logic. --- lfs.c | 75 ++++++++++++++++++++++----------------------- tests/test_paths.sh | 16 ++++++++++ 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/lfs.c b/lfs.c index 6ff4c808..bccb27ee 100644 --- a/lfs.c +++ b/lfs.c @@ -1273,21 +1273,19 @@ static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, } static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { - lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; + // we reduce path to a single name if we can find it const char *name = *path; - lfs_size_t namelen; - int32_t tag; + *path = NULL; + + // default to root dir + int32_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); + lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; while (true) { - nextname: +nextname: // skip slashes name += strspn(name, "/"); - namelen = strcspn(name, "/"); - - if (name[0] == '\0') { - // special case for root dir - return LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); - } + lfs_size_t namelen = strcspn(name, "/"); // skip '.' and root '..' if ((namelen == 1 && memcmp(name, ".", 1) == 0) || @@ -1320,10 +1318,31 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { suffix += sufflen; } - // update what we've found - *path = name; + // found path + if (name[0] == '\0') { + return tag; + } - // find path + // update what we've found if path is only a name + if (strchr(name, '/') == NULL) { + *path = name; + } + + // only continue if we hit a directory + if (lfs_tagtype(tag) != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + + // grab the entry data + if (lfs_tagid(tag) != 0x3ff) { + int32_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), pair); + if (res < 0) { + return res; + } + } + + // find entry matching name while (true) { tag = lfs_dir_find(lfs, dir, pair, 0x7c000fff, LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), name); @@ -1337,6 +1356,7 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { } if (!dir->split) { + // couldn't find it return LFS_ERR_NOENT; } @@ -1344,26 +1364,8 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { pair[1] = dir->tail[1]; } + // to next name name += namelen; - name += strspn(name, "/"); - if (name[0] == '\0') { - return tag; - } - - // don't continue on if we didn't hit a directory - // TODO update with what's on master? - if (lfs_tagtype(tag) != LFS_TYPE_DIR) { - return LFS_ERR_NOTDIR; - } - - // TODO optimize grab for inline files and like? - // TODO would this mean more code? - // grab the entry data - int32_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), pair); - if (res < 0) { - return res; - } } } @@ -1408,11 +1410,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_mdir_t cwd; int32_t res = lfs_dir_lookup(lfs, &cwd, &path); - if (res != LFS_ERR_NOENT || strchr(path, '/') != NULL) { - if (res >= 0) { - return LFS_ERR_EXIST; - } - return res; + if (res != LFS_ERR_NOENT || !path) { + return (res < 0) ? res : LFS_ERR_EXIST; } // check that name fits @@ -1806,7 +1805,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // allocate entry for file if it doesn't exist lfs_mdir_t cwd; int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); - if (tag < 0 && (tag != LFS_ERR_NOENT || strchr(path, '/') != NULL)) { + if (tag < 0 && (tag != LFS_ERR_NOENT || !path)) { return tag; } diff --git a/tests/test_paths.sh b/tests/test_paths.sh index 33843296..ea5eb216 100755 --- a/tests/test_paths.sh +++ b/tests/test_paths.sh @@ -90,6 +90,22 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Trailing dot path tests ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "tea/hottea/", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "tea/hottea/.", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "tea/hottea/./.", &info) => 0; + strcmp(info.name, "hottea") => 0; + lfs_stat(&lfs, "tea/hottea/..", &info) => 0; + strcmp(info.name, "tea") => 0; + lfs_stat(&lfs, "tea/hottea/../.", &info) => 0; + strcmp(info.name, "tea") => 0; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Root dot dot path tests ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; From 15d156082cf59a845582ec32e541c64edf58d47c Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 29 Jul 2018 15:03:23 -0500 Subject: [PATCH 065/139] Added support for custom attributes leveraging the new metadata logging Now that littlefs has been rebuilt almost from the ground up with the intention to support custom attributes, adding in custom attribute support is relatively easy. The highest bit in the 9-bit type structure indicates that an attribute is a user-specified custom attribute. The user then has a full 8-bits to specify the attribute type. Other than that, custom attributes are treated the same as system-level attributes. Also made some tweaks to custom attributes: - Adopted the opencfg for file-level attributes provided by dpgeorge - Changed setattrs/getattrs to the simpler setattr/getattr functions users will probably be more familiar with. Note that multiple attributes can still be committed atomically with files, though not with directories. - Changed LFS_ATTRS_MAX -> LFS_ATTR_MAX since there's no longer a global limit on the sum of attribute sizes, which was rather confusing. Though they are still limited by what can fit in a metadata-pair. --- Makefile | 6 +- lfs.c | 181 +++++++++++++++++++++++++----- lfs.h | 113 ++++++++++--------- tests/test_attrs.sh | 264 ++++++++++++++++++++------------------------ 4 files changed, 337 insertions(+), 227 deletions(-) diff --git a/Makefile b/Makefile index 55c95dbd..21cc1c96 100644 --- a/Makefile +++ b/Makefile @@ -33,9 +33,9 @@ size: $(OBJ) $(SIZE) -t $^ .SUFFIXES: -test: test_format test_dirs test_files test_seek test_truncate test_entries \ - test_interspersed test_alloc test_paths test_attrs \ - test_orphan test_move test_corrupt +test: test_format test_dirs test_files test_seek test_truncate \ + test_entries test_interspersed test_alloc test_paths test_attrs \ + test_move test_orphan test_corrupt test_%: tests/test_%.sh ifdef QUIET @./$< | sed -n '/^[-=]/p' diff --git a/lfs.c b/lfs.c index bccb27ee..ae8b4c50 100644 --- a/lfs.c +++ b/lfs.c @@ -364,7 +364,7 @@ static void lfs_alloc_ack(lfs_t *lfs) { // d->block_count = lfs_fromle32(d->block_count); // d->version = lfs_fromle32(d->version); // d->inline_size = lfs_fromle32(d->inline_size); -// d->attrs_size = lfs_fromle32(d->attrs_size); +// d->attr_size = lfs_fromle32(d->attr_size); // d->name_size = lfs_fromle32(d->name_size); //} // @@ -375,7 +375,7 @@ static void lfs_alloc_ack(lfs_t *lfs) { // d->block_count = lfs_tole32(d->block_count); // d->version = lfs_tole32(d->version); // d->inline_size = lfs_tole32(d->inline_size); -// d->attrs_size = lfs_tole32(d->attrs_size); +// d->attr_size = lfs_tole32(d->attr_size); // d->name_size = lfs_tole32(d->name_size); //} @@ -430,7 +430,7 @@ static inline bool lfs_pairsync( (((uint32_t)(type) << 22) | ((uint32_t)(id) << 12) | (uint32_t)(size)) #define LFS_MKATTR(type, id, buffer, size, next) \ - &(const lfs_mattr_t){(next), LFS_MKTAG(type, id, size), (buffer)} + &(const lfs_mattr_t){LFS_MKTAG(type, id, size), (buffer), (next)} static inline bool lfs_tagisvalid(uint32_t tag) { return !(tag & 0x80000000); @@ -526,13 +526,20 @@ static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, return LFS_ERR_NOENT; } +static int lfs_commitattrs(lfs_t *lfs, struct lfs_commit *commit, + uint16_t id, const struct lfs_attr *attrs); + static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, const lfs_mdir_t *dir, const lfs_mattr_t *attrs); static int lfs_commitattr(lfs_t *lfs, struct lfs_commit *commit, uint32_t tag, const void *buffer) { - if (lfs_tagtype(tag) == LFS_FROM_MOVE) { + if (lfs_tagtype(tag) == LFS_FROM_ATTRS) { + // special case for custom attributes + return lfs_commitattrs(lfs, commit, + lfs_tagid(tag), buffer); + } else if (lfs_tagtype(tag) == LFS_FROM_MOVE) { // special case for moves return lfs_commitmove(lfs, commit, lfs_tagsize(tag), lfs_tagid(tag), @@ -586,6 +593,19 @@ static int lfs_commitattr(lfs_t *lfs, struct lfs_commit *commit, return 0; } +static int lfs_commitattrs(lfs_t *lfs, struct lfs_commit *commit, + uint16_t id, const struct lfs_attr *attrs) { + for (const struct lfs_attr *a = attrs; a; a = a->next) { + int err = lfs_commitattr(lfs, commit, + LFS_MKTAG(0x100 | a->type, id, a->size), a->buffer); + if (err) { + return err; + } + } + + return 0; +} + static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, const lfs_mdir_t *dir, const lfs_mattr_t *attrs) { @@ -1378,9 +1398,6 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, } info->type = lfs_tagtype(tag); - if (lfs_tagsize(tag) > lfs->name_size) { - return LFS_ERR_RANGE; - } struct lfs_ctz ctz; tag = lfs_dir_get(lfs, dir, 0x7c3ff000, @@ -1410,7 +1427,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_mdir_t cwd; int32_t res = lfs_dir_lookup(lfs, &cwd, &path); - if (res != LFS_ERR_NOENT || !path) { + if (!(res == LFS_ERR_NOENT && path)) { return (res < 0) ? res : LFS_ERR_EXIST; } @@ -1792,8 +1809,9 @@ static int lfs_ctztraverse(lfs_t *lfs, /// Top level file operations /// -int lfs_file_open(lfs_t *lfs, lfs_file_t *file, - const char *path, int flags) { +int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags, + const struct lfs_file_config *cfg) { // deorphan if we haven't yet, needed at most once after poweron if ((flags & 3) != LFS_O_RDONLY && !lfs->deorphaned) { int err = lfs_deorphan(lfs); @@ -1805,7 +1823,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // allocate entry for file if it doesn't exist lfs_mdir_t cwd; int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); - if (tag < 0 && (tag != LFS_ERR_NOENT || !path)) { + if (tag < 0 && !(tag == LFS_ERR_NOENT && path)) { return tag; } @@ -1859,6 +1877,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // setup file struct + file->cfg = cfg; file->pair[0] = cwd.pair[0]; file->pair[1] = cwd.pair[1]; file->id = lfs_tagid(tag); @@ -1866,9 +1885,34 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, file->pos = 0; file->attrs = NULL; + if (cfg && cfg->attrs) { + // fetch attrs + for (const struct lfs_attr *a = cfg->attrs; a; a = a->next) { + if ((file->flags & 3) != LFS_O_WRONLY) { + int32_t res = lfs_dir_get(lfs, &cwd, 0x7ffff000, + LFS_MKTAG(0x100 | a->type, file->id, a->size), a->buffer); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + } + + if ((file->flags & 3) != LFS_O_RDONLY) { + if (a->size > lfs->attr_size) { + return LFS_ERR_NOSPC; + } + + file->flags |= LFS_F_DIRTY; + } + } + + file->attrs = cfg->attrs; + } + // allocate buffer if needed file->cache.block = 0xffffffff; - if (lfs->cfg->file_buffer) { + if (file->cfg && file->cfg->buffer) { + file->cache.buffer = file->cfg->buffer; + } else if (lfs->cfg->file_buffer) { file->cache.buffer = lfs->cfg->file_buffer; } else if ((file->flags & 3) == LFS_O_RDONLY) { file->cache.buffer = lfs_malloc(lfs->cfg->read_size); @@ -1909,6 +1953,11 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return 0; } +int lfs_file_open(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags) { + return lfs_file_opencfg(lfs, file, path, flags, NULL); +} + int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { int err = lfs_file_sync(lfs, file); @@ -1921,7 +1970,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { } // clean up memory - if (!lfs->cfg->file_buffer) { + if (!(file->cfg && file->cfg->buffer) && !lfs->cfg->file_buffer) { lfs_free(file->cache.buffer); } @@ -2071,7 +2120,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_CTZSTRUCT, file->id, &file->ctz.head, sizeof(file->ctz), - file->attrs)); + LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->attrs, 0, + NULL))); if (err) { return err; } @@ -2079,7 +2129,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, file->cache.buffer, file->ctz.size, - file->attrs)); + LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->attrs, 0, + NULL))); if (err) { return err; } @@ -2688,7 +2739,85 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return 0; } -//int lfs_getattrs(lfs_t *lfs, const char *path, +lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size) { + lfs_mdir_t cwd; + int32_t res = lfs_dir_lookup(lfs, &cwd, &path); + if (res < 0) { + return res; + } + + res = lfs_dir_get(lfs, &cwd, 0x7ffff000, + LFS_MKTAG(0x100 | type, lfs_tagid(res), + lfs_min(size, lfs->attr_size)), buffer); + if (res < 0) { + if (res == LFS_ERR_NOENT) { + return LFS_ERR_NOATTR; + } + return res; + } + + return lfs_tagsize(res); +} + +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size) { + if (size > lfs->attr_size) { + return LFS_ERR_NOSPC; + } + + lfs_mdir_t cwd; + int32_t res = lfs_dir_lookup(lfs, &cwd, &path); + if (res < 0) { + return res; + } + + return lfs_dir_commit(lfs, &cwd, + LFS_MKATTR(0x100 | type, lfs_tagid(res), buffer, size, + NULL)); +} + +lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, + uint8_t type, void *buffer, lfs_size_t size) { + lfs_mdir_t superdir; + int err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + int32_t res = lfs_dir_get(lfs, &superdir, 0x7ffff000, + LFS_MKTAG(0x100 | type, 0, + lfs_min(size, lfs->attr_size)), buffer); + if (res < 0) { + if (res == LFS_ERR_NOENT) { + return LFS_ERR_NOATTR; + } + return res; + } + + return lfs_tagsize(res); +} + +int lfs_fs_setattr(lfs_t *lfs, + uint8_t type, const void *buffer, lfs_size_t size) { + if (size > lfs->attr_size) { + return LFS_ERR_NOSPC; + } + + lfs_mdir_t superdir; + int err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + return lfs_dir_commit(lfs, &superdir, + LFS_MKATTR(0x100 | type, 0, buffer, size, + NULL)); +} + +// +// +// // const struct lfs_attr *attrs, int count) { // lfs_mdir_t cwd; // int err = lfs_dir_fetch(lfs, &cwd, lfs->root); @@ -2777,10 +2906,10 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->inline_size = lfs_min(LFS_INLINE_MAX, lfs->cfg->read_size); } - LFS_ASSERT(lfs->cfg->attrs_size <= LFS_ATTRS_MAX); - lfs->attrs_size = lfs->cfg->attrs_size; - if (!lfs->attrs_size) { - lfs->attrs_size = LFS_ATTRS_MAX; + LFS_ASSERT(lfs->cfg->attr_size <= LFS_ATTR_MAX); + lfs->attr_size = lfs->cfg->attr_size; + if (!lfs->attr_size) { + lfs->attr_size = LFS_ATTR_MAX; } LFS_ASSERT(lfs->cfg->name_size <= LFS_NAME_MAX); @@ -2873,7 +3002,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .block_size = lfs->cfg->block_size, .block_count = lfs->cfg->block_count, .inline_size = lfs->inline_size, - .attrs_size = lfs->attrs_size, + .attr_size = lfs->attr_size, .name_size = lfs->name_size, }; @@ -2954,14 +3083,14 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->inline_size = superblock.inline_size; } - if (superblock.attrs_size) { - if (superblock.attrs_size > lfs->attrs_size) { - LFS_ERROR("Unsupported attrs size (%d > %d)", - superblock.attrs_size, lfs->attrs_size); + if (superblock.attr_size) { + if (superblock.attr_size > lfs->attr_size) { + LFS_ERROR("Unsupported attr size (%d > %d)", + superblock.attr_size, lfs->attr_size); return LFS_ERR_INVAL; } - lfs->attrs_size = superblock.attrs_size; + lfs->attr_size = superblock.attr_size; } if (superblock.name_size) { diff --git a/lfs.h b/lfs.h index bd79e9b6..b83a25c0 100644 --- a/lfs.h +++ b/lfs.h @@ -52,18 +52,18 @@ typedef uint32_t lfs_block_t; // Maximum inline file size in bytes. Large inline files require a larger // read and prog cache, but if a file can be inline it does not need its own -// data block. LFS_ATTRS_MAX + LFS_INLINE_MAX must be <= 0xffff. Stored in +// data block. LFS_ATTR_MAX + LFS_INLINE_MAX must be <= 0xffff. Stored in // superblock and must be respected by other littlefs drivers. #ifndef LFS_INLINE_MAX #define LFS_INLINE_MAX 0x3ff #endif // Maximum size of all attributes per file in bytes, may be redefined but a -// a smaller LFS_ATTRS_MAX has no benefit. LFS_ATTRS_MAX + LFS_INLINE_MAX +// a smaller LFS_ATTR_MAX has no benefit. LFS_ATTR_MAX + LFS_INLINE_MAX // must be <= 0xffff. Stored in superblock and must be respected by other // littlefs drivers. -#ifndef LFS_ATTRS_MAX -#define LFS_ATTRS_MAX 0x3f +#ifndef LFS_ATTR_MAX +#define LFS_ATTR_MAX 0x3f #endif // Max name size in bytes, may be redefined to reduce the size of the @@ -89,8 +89,7 @@ enum lfs_error { LFS_ERR_NOSPC = -28, // No space left on device LFS_ERR_NOMEM = -12, // No more memory available LFS_ERR_NAMETOOLONG = -36, // File name too long - LFS_ERR_NODATA = -61, // No data/attr available - LFS_ERR_RANGE = -34, // Result not representable + LFS_ERR_NOATTR = -61, // No data/attr available }; // File types @@ -119,6 +118,7 @@ enum lfs_type { LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, LFS_FROM_MOVE = 0x030, + LFS_FROM_ATTRS = 0x020, }; // File open flags @@ -223,10 +223,10 @@ struct lfs_config { lfs_size_t inline_size; // Optional upper limit on attributes per file in bytes. No downside for - // larger attributes size but must be less than LFS_ATTRS_MAX. Defaults to - // LFS_ATTRS_MAX when zero.Stored in superblock and must be respected by + // larger attributes size but must be less than LFS_ATTR_MAX. Defaults to + // LFS_ATTR_MAX when zero.Stored in superblock and must be respected by // other littlefs drivers. - lfs_size_t attrs_size; + lfs_size_t attr_size; // Optional upper limit on length of file names in bytes. No downside for // larger names except the size of the info struct which is controlled by @@ -235,7 +235,6 @@ struct lfs_config { lfs_size_t name_size; }; - // File info structure struct lfs_info { // Type of the file, either LFS_TYPE_REG or LFS_TYPE_DIR @@ -256,16 +255,30 @@ struct lfs_attr { // Pointer to buffer containing the attribute void *buffer; - // Size of attribute in bytes, limited to LFS_ATTRS_MAX + // Size of attribute in bytes, limited to LFS_ATTR_MAX lfs_size_t size; + + // Pointer to next attribute in linked list + const struct lfs_attr *next; +}; + +// Optional configuration provided during lfs_file_opencfg +struct lfs_file_config { + // Optional, statically allocated buffer for files. Must be program sized. + // If NULL, malloc will be used by default. + void *buffer; + + // Optional, custom attributes + // TODO document more + const struct lfs_attr *attrs; }; /// littlefs data structures /// typedef struct lfs_mattr { - const struct lfs_mattr *next; int32_t tag; const void *buffer; + const struct lfs_mattr *next; } lfs_mattr_t; typedef struct lfs_globals { @@ -302,13 +315,13 @@ typedef struct lfs_file { lfs_size_t size; } ctz; + const struct lfs_file_config *cfg; + const struct lfs_attr *attrs; uint32_t flags; lfs_off_t pos; lfs_block_t block; lfs_off_t off; lfs_cache_t cache; - - lfs_mattr_t *attrs; } lfs_file_t; typedef struct lfs_dir { @@ -328,7 +341,7 @@ typedef struct lfs_superblock { lfs_size_t block_count; lfs_size_t inline_size; - lfs_size_t attrs_size; + lfs_size_t attr_size; lfs_size_t name_size; } lfs_superblock_t; @@ -358,7 +371,7 @@ typedef struct lfs { lfs_globals_t diff; lfs_size_t inline_size; - lfs_size_t attrs_size; + lfs_size_t attr_size; lfs_size_t name_size; } lfs_t; @@ -368,7 +381,8 @@ typedef struct lfs { // Format a block device with the littlefs // // Requires a littlefs object and config struct. This clobbers the littlefs -// object, and does not leave the filesystem mounted. +// object, and does not leave the filesystem mounted. The config struct must +// be zeroed for defaults and backwards compatibility. // // Returns a negative error code on failure. int lfs_format(lfs_t *lfs, const struct lfs_config *config); @@ -377,7 +391,8 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *config); // // Requires a littlefs object and config struct. Multiple filesystems // may be mounted simultaneously with multiple littlefs objects. Both -// lfs and config must be allocated while mounted. +// lfs and config must be allocated while mounted. The config struct must +// be zeroed for defaults and backwards compatibility. // // Returns a negative error code on failure. int lfs_mount(lfs_t *lfs, const struct lfs_config *config); @@ -416,9 +431,10 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); // smaller than the buffer, it is padded with zeros. It the stored attribute // is larger than the buffer, LFS_ERR_RANGE is returned. // +// TODO doc // Returns a negative error code on failure. -int lfs_getattrs(lfs_t *lfs, const char *path, - const struct lfs_attr *attrs, int count); +lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size); // Set custom attributes // @@ -426,23 +442,38 @@ int lfs_getattrs(lfs_t *lfs, const char *path, // disk based on their type id. Unspecified attributes are left unmodified. // Specifying an attribute with zero size deletes the attribute. // +// TODO doc // Returns a negative error code on failure. -int lfs_setattrs(lfs_t *lfs, const char *path, - const struct lfs_attr *attrs, int count); +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size); /// File operations /// // Open a file // -// The mode that the file is opened in is determined -// by the flags, which are values from the enum lfs_open_flags -// that are bitwise-ored together. +// The mode that the file is opened in is determined by the flags, which +// are values from the enum lfs_open_flags that are bitwise-ored together. // // Returns a negative error code on failure. int lfs_file_open(lfs_t *lfs, lfs_file_t *file, const char *path, int flags); + +// Open a file with extra configuration +// +// The mode that the file is opened in is determined by the flags, which +// are values from the enum lfs_open_flags that are bitwise-ored together. +// +// The config struct provides additional config options per file as described +// above. The config struct must be allocated while the file is open, and the +// config struct must be zeroed for defaults and backwards compatibility. +// +// Returns a negative error code on failure. +int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, + const char *path, int flags, + const struct lfs_file_config *config); + // Close a file // // Any pending writes are written out to storage as though @@ -503,30 +534,6 @@ int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file); // Returns the size of the file, or a negative error code on failure. lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file); -// Get custom attributes attached to a file -// -// Attributes are looked up based on the type id. If the stored attribute is -// smaller than the buffer, it is padded with zeros. It the stored attribute -// is larger than the buffer, LFS_ERR_RANGE is returned. -// -// Returns a negative error code on failure. -int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, - const struct lfs_attr *attrs, int count); - -// Set custom attributes on a file -// -// The array of attributes will be used to update the attributes stored on -// disk based on their type id. Unspecified attributes are left unmodified. -// Specifying an attribute with zero size deletes the attribute. -// -// Note: Attributes are not written out until a call to lfs_file_sync -// or lfs_file_close and must be allocated until the file is closed or -// lfs_file_setattrs is called with a count of zero. -// -// Returns a negative error code on failure. -int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, - const struct lfs_attr *attrs, int count); - /// Directory operations /// @@ -583,8 +590,10 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); // smaller than the buffer, it is padded with zeros. It the stored attribute // is larger than the buffer, LFS_ERR_RANGE is returned. // +// TODO doc // Returns a negative error code on failure. -int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); +lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, + uint8_t type, void *buffer, lfs_size_t size); // Set custom attributes on the filesystem // @@ -594,8 +603,10 @@ int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); // // Note: Filesystem level attributes are not available for wear-leveling // +// TODO doc // Returns a negative error code on failure. -int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); +int lfs_fs_setattr(lfs_t *lfs, + uint8_t type, const void *buffer, lfs_size_t size); // Finds the current size of the filesystem // diff --git a/tests/test_attrs.sh b/tests/test_attrs.sh index 286b909d..2fde13ed 100755 --- a/tests/test_attrs.sh +++ b/tests/test_attrs.sh @@ -19,65 +19,53 @@ TEST echo "--- Set/get attribute ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ - {'A', "aaaa", 4}, - {'B', "bbbbbb", 6}, - {'C', "ccccc", 5}}, 3) => 0; - lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_setattr(&lfs, "hello", 'A', "aaaa", 4) => 0; + lfs_setattr(&lfs, "hello", 'B', "bbbbbb", 6) => 0; + lfs_setattr(&lfs, "hello", 'C', "ccccc", 5) => 0; + lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "hello", 'B', buffer+4, 6) => 6; + lfs_getattr(&lfs, "hello", 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "bbbbbb", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ - {'B', "", 0}}, 1) => 0; - lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_setattr(&lfs, "hello", 'B', "", 0) => 0; + lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "hello", 'B', buffer+4, 6) => 0; + lfs_getattr(&lfs, "hello", 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ - {'B', "dddddd", 6}}, 1) => 0; - lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_setattr(&lfs, "hello", 'B', "dddddd", 6) => 0; + lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "hello", 'B', buffer+4, 6) => 6; + lfs_getattr(&lfs, "hello", 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "dddddd", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ - {'B', "eee", 3}}, 1) => 0; - lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_setattr(&lfs, "hello", 'B', "eee", 3) => 0; + lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "hello", 'B', buffer+4, 6) => 3; + lfs_getattr(&lfs, "hello", 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "eee\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ - {'A', buffer, LFS_ATTRS_MAX+1}}, 1) => LFS_ERR_NOSPC; - lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ - {'B', "fffffffff", 9}}, 1) => 0; - lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + lfs_setattr(&lfs, "hello", 'A', buffer, LFS_ATTR_MAX+1) => LFS_ERR_NOSPC; + lfs_setattr(&lfs, "hello", 'B', "fffffffff", 9) => 0; + lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "hello", 'B', buffer+4, 6) => 9; + lfs_getattr(&lfs, "hello", 'C', buffer+10, 5) => 5; lfs_unmount(&lfs) => 0; TEST tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 9}, - {'C', buffer+13, 5}}, 3) => 0; + lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "hello", 'B', buffer+4, 9) => 9; + lfs_getattr(&lfs, "hello", 'C', buffer+13, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "fffffffff", 9) => 0; memcmp(buffer+13, "ccccc", 5) => 0; @@ -92,64 +80,52 @@ TEST echo "--- Set/get fs attribute ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ - {'A', "aaaa", 4}, - {'B', "bbbbbb", 6}, - {'C', "ccccc", 5}}, 3) => 0; - lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_fs_setattr(&lfs, 'A', "aaaa", 4) => 0; + lfs_fs_setattr(&lfs, 'B', "bbbbbb", 6) => 0; + lfs_fs_setattr(&lfs, 'C', "ccccc", 5) => 0; + lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; + lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 6; + lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "bbbbbb", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ - {'B', "", 0}}, 1) => 0; - lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_fs_setattr(&lfs, 'B', "", 0) => 0; + lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; + lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 0; + lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ - {'B', "dddddd", 6}}, 1) => 0; - lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_fs_setattr(&lfs, 'B', "dddddd", 6) => 0; + lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; + lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 6; + lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "dddddd", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ - {'B', "eee", 3}}, 1) => 0; - lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_fs_setattr(&lfs, 'B', "eee", 3) => 0; + lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; + lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 3; + lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "eee\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ - {'A', buffer, LFS_ATTRS_MAX+1}}, 1) => LFS_ERR_NOSPC; - lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ - {'B', "fffffffff", 9}}, 1) => 0; - lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + lfs_fs_setattr(&lfs, 'A', buffer, LFS_ATTR_MAX+1) => LFS_ERR_NOSPC; + lfs_fs_setattr(&lfs, 'B', "fffffffff", 9) => 0; + lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; + lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 9; + lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; lfs_unmount(&lfs) => 0; TEST tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 9}, - {'C', buffer+13, 5}}, 3) => 0; + lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; + lfs_fs_getattr(&lfs, 'B', buffer+4, 9) => 9; + lfs_fs_getattr(&lfs, 'C', buffer+13, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "fffffffff", 9) => 0; memcmp(buffer+13, "ccccc", 5) => 0; @@ -164,78 +140,84 @@ TEST echo "--- Set/get file attribute ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_WRONLY) => 0; + struct lfs_attr a1 = {'A', buffer, 4}; + struct lfs_attr b1 = {'B', buffer+4, 6, &a1}; + struct lfs_attr c1 = {'C', buffer+10, 5, &b1}; + struct lfs_file_config cfg1 = {.attrs = &c1}; - struct lfs_attr attr[3]; - attr[0] = (struct lfs_attr){'A', "aaaa", 4}; - attr[1] = (struct lfs_attr){'B', "bbbbbb", 6}; - attr[2] = (struct lfs_attr){'C', "ccccc", 5}; - lfs_file_setattrs(&lfs, &file[0], attr, 3) => 0; - lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; + memcpy(buffer, "aaaa", 4); + memcpy(buffer+4, "bbbbbb", 6); + memcpy(buffer+10, "ccccc", 5); + lfs_file_close(&lfs, &file[0]) => 0; + memset(buffer, 0, 15); + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; + lfs_file_close(&lfs, &file[0]) => 0; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "bbbbbb", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_file_sync(&lfs, &file[0]) => 0; - attr[0] = (struct lfs_attr){'B', "", 0}; - lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; - lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + b1.size = 0; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + memset(buffer, 0, 15); + b1.size = 6; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; + lfs_file_close(&lfs, &file[0]) => 0; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_file_sync(&lfs, &file[0]) => 0; - attr[0] = (struct lfs_attr){'B', "dddddd", 6}; - lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; - lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + b1.size = 6; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; + memcpy(buffer+4, "dddddd", 6); + lfs_file_close(&lfs, &file[0]) => 0; + memset(buffer, 0, 15); + b1.size = 6; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; + lfs_file_close(&lfs, &file[0]) => 0; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "dddddd", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_file_sync(&lfs, &file[0]) => 0; - attr[0] = (struct lfs_attr){'B', "eee", 3}; - lfs_file_setattrs(&lfs, &file[0], attr, 1); - lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => 0; + b1.size = 3; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; + memcpy(buffer+4, "eee", 3); + lfs_file_close(&lfs, &file[0]) => 0; + memset(buffer, 0, 15); + b1.size = 6; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; + lfs_file_close(&lfs, &file[0]) => 0; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "eee\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_file_sync(&lfs, &file[0]) => 0; - attr[0] = (struct lfs_attr){'A', buffer, LFS_ATTRS_MAX+1}; - lfs_file_setattrs(&lfs, &file[0], attr, 1) => LFS_ERR_NOSPC; - attr[0] = (struct lfs_attr){'B', "fffffffff", 9}; - lfs_file_open(&lfs, &file[1], "hello/hello", LFS_O_RDONLY) => 0; - lfs_file_setattrs(&lfs, &file[1], attr, 1) => LFS_ERR_BADF; - lfs_file_close(&lfs, &file[1]) => 0; - lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; - lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 6}, - {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + a1.size = LFS_ATTR_MAX+1; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) + => LFS_ERR_NOSPC; + struct lfs_attr a2 = {'A', buffer, 4}; + struct lfs_attr b2 = {'B', buffer+4, 9, &a2}; + struct lfs_attr c2 = {'C', buffer+13, 5, &b2}; + struct lfs_file_config cfg2 = {.attrs = &c2}; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDWR, &cfg2) => 0; + memcpy(buffer+4, "fffffffff", 9); + lfs_file_close(&lfs, &file[0]) => 0; + a1.size = 4; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; TEST tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + struct lfs_attr a2 = {'A', buffer, 4}; + struct lfs_attr b2 = {'B', buffer+4, 9, &a2}; + struct lfs_attr c2 = {'C', buffer+13, 5, &b2}; + struct lfs_file_config cfg2 = {.attrs = &c2}; - lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ - {'A', buffer, 4}, - {'B', buffer+4, 9}, - {'C', buffer+13, 5}}, 3) => 0; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg2) => 0; + lfs_file_close(&lfs, &file[0]) => 0; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "fffffffff", 9) => 0; memcmp(buffer+13, "ccccc", 5) => 0; @@ -250,36 +232,24 @@ TEST echo "--- Deferred file attributes ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDWR) => 0; - - struct lfs_attr attr[] = { - {'B', "gggg", 4}, - {'C', "", 0}, - {'D', "hhhh", 4}, - }; + struct lfs_attr a1 = {'B', "gggg", 4}; + struct lfs_attr b1 = {'C', "", 0, &a1}; + struct lfs_attr c1 = {'D', "hhhh", 4, &b1}; + struct lfs_file_config cfg1 = {.attrs = &c1}; - lfs_file_setattrs(&lfs, &file[0], attr, 3) => 0; - lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ - {'B', buffer, 9}, - {'C', buffer+9, 9}, - {'D', buffer+18, 9}}, 3) => 0; - memcmp(buffer, "gggg\0\0\0\0\0", 9) => 0; - memcmp(buffer+9, "\0\0\0\0\0\0\0\0\0", 9) => 0; - memcmp(buffer+18, "hhhh\0\0\0\0\0", 9) => 0; + lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; - lfs_getattrs(&lfs, "hello/hello", (struct lfs_attr[]){ - {'B', buffer, 9}, - {'C', buffer+9, 9}, - {'D', buffer+18, 9}}, 3) => 0; + lfs_getattr(&lfs, "hello/hello", 'B', buffer, 9) => 9; + lfs_getattr(&lfs, "hello/hello", 'C', buffer+9, 9) => 5; + lfs_getattr(&lfs, "hello/hello", 'D', buffer+18, 9) => LFS_ERR_NOATTR; memcmp(buffer, "fffffffff", 9) => 0; memcmp(buffer+9, "ccccc\0\0\0\0", 9) => 0; memcmp(buffer+18, "\0\0\0\0\0\0\0\0\0", 9) => 0; lfs_file_sync(&lfs, &file[0]) => 0; - lfs_getattrs(&lfs, "hello/hello", (struct lfs_attr[]){ - {'B', buffer, 9}, - {'C', buffer+9, 9}, - {'D', buffer+18, 9}}, 3) => 0; + lfs_getattr(&lfs, "hello/hello", 'B', buffer, 9) => 4; + lfs_getattr(&lfs, "hello/hello", 'C', buffer+9, 9) => 0; + lfs_getattr(&lfs, "hello/hello", 'D', buffer+18, 9) => 4; memcmp(buffer, "gggg\0\0\0\0\0", 9) => 0; memcmp(buffer+9, "\0\0\0\0\0\0\0\0\0", 9) => 0; memcmp(buffer+18, "hhhh\0\0\0\0\0", 9) => 0; From 3e246da52c206c1989ae598b444693941f89dc95 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 30 Jul 2018 09:10:04 -0500 Subject: [PATCH 066/139] Fixed the orphan test to handle logging metadata-pairs The main issue here was that the old orphan test relied on deleting the block that contained the most recent update. In the new design this doesn't really work since updates get appended to metadata-pairs incrementally. This is fixed by instead using the truncate command on the appropriate block. We're now passing orphan tests. --- lfs.c | 42 +++++++++++++++++++++++++++--------------- lfs.h | 27 +++++++++------------------ tests/test_corrupt.sh | 3 +++ tests/test_orphan.sh | 28 ++++++++++++++++------------ 4 files changed, 55 insertions(+), 45 deletions(-) diff --git a/lfs.c b/lfs.c index ae8b4c50..03c4b812 100644 --- a/lfs.c +++ b/lfs.c @@ -3255,6 +3255,12 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } */ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { + if (lfs_pairisnull(lfs->root)) { + // TODO best place for this? + // TODO needed for relocate + return LFS_ERR_NOENT; + } + // iterate over all directory directory entries pdir->tail[0] = 0; pdir->tail[1] = 1; @@ -3275,6 +3281,11 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *parent) { + if (lfs_pairisnull(lfs->root)) { + // TODO best place for this? + return LFS_ERR_NOENT; + } + // search for both orderings so we can reuse the find function lfs_block_t child[2] = {pair[0], pair[1]}; @@ -3531,18 +3542,19 @@ int lfs_deorphan(lfs_t *lfs) { // return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); //} -//static int lfs_fs_size_count(void *p, lfs_block_t block) { -// lfs_size_t *size = p; -// *size += 1; -// return 0; -//} -// -//lfs_ssize_t lfs_fs_size(lfs_t *lfs) { -// lfs_size_t size = 0; -// int err = lfs_fs_traverse(lfs, lfs_fs_size_count, &size); -// if (err) { -// return err; -// } -// -// return size; -//} +// TODO need lfs? +static int lfs_fs_size_count(lfs_t *lfs, void *p, lfs_block_t block) { + lfs_size_t *size = p; + *size += 1; + return 0; +} + +lfs_ssize_t lfs_fs_size(lfs_t *lfs) { + lfs_size_t size = 0; + int err = lfs_fs_traverse(lfs, lfs_fs_size_count, &size); + if (err) { + return err; + } + + return size; +} diff --git a/lfs.h b/lfs.h index b83a25c0..b384ed33 100644 --- a/lfs.h +++ b/lfs.h @@ -249,7 +249,8 @@ struct lfs_info { // Custom attribute structure struct lfs_attr { - // Type of attribute, provided by user and used to identify the attribute + // 8-bit Type of attribute, provided by user and used to + // identify the attribute uint8_t type; // Pointer to buffer containing the attribute @@ -259,7 +260,7 @@ struct lfs_attr { lfs_size_t size; // Pointer to next attribute in linked list - const struct lfs_attr *next; + struct lfs_attr *next; }; // Optional configuration provided during lfs_file_opencfg @@ -268,9 +269,9 @@ struct lfs_file_config { // If NULL, malloc will be used by default. void *buffer; - // Optional, custom attributes + // Optional, linked list of custom attributes. // TODO document more - const struct lfs_attr *attrs; + struct lfs_attr *attrs; }; @@ -582,7 +583,8 @@ lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir); int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); -/// Filesystem filesystem operations /// +/// Filesystem filesystem operations /// TODO choose one +/// Miscellaneous littlefs specific operations /// TODO choose one // Get custom attributes on the filesystem // @@ -616,9 +618,6 @@ int lfs_fs_setattr(lfs_t *lfs, // Returns the number of allocated blocks, or a negative error code on failure. lfs_ssize_t lfs_fs_size(lfs_t *lfs); - -/// Miscellaneous littlefs specific operations /// - // Traverse through all blocks in use by the filesystem // // The provided callback will be called with each block address that is @@ -626,16 +625,8 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // blocks are in use or how much of the storage is available. // // Returns a negative error code on failure. -int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); - -// Prunes any recoverable errors that may have occured in the filesystem -// -// Not needed to be called by user unless an operation is interrupted -// but the filesystem is still mounted. This is already called on first -// allocation. -// -// Returns a negative error code on failure. -int lfs_deorphan(lfs_t *lfs); +// TODO don't pass lfs_t? +int lfs_fs_traverse(lfs_t *lfs, int (*cb)(lfs_t*, void*, lfs_block_t), void *data); #endif diff --git a/tests/test_corrupt.sh b/tests/test_corrupt.sh index 1491dac3..faa6f7ec 100755 --- a/tests/test_corrupt.sh +++ b/tests/test_corrupt.sh @@ -78,8 +78,11 @@ do rm -rf blocks mkdir blocks ln -s /dev/zero blocks/$(printf '%x' $i) + echo $i 1i lfs_mktree + echo $i 2i lfs_chktree + echo $i 3i done echo "--- Block persistance ---" diff --git a/tests/test_orphan.sh b/tests/test_orphan.sh index 71d6d4fc..a76088b2 100755 --- a/tests/test_orphan.sh +++ b/tests/test_orphan.sh @@ -15,25 +15,29 @@ tests/test.py << TEST lfs_mkdir(&lfs, "parent/child") => 0; lfs_remove(&lfs, "parent/orphan") => 0; TEST -# remove most recent file, this should be the update to the previous +# corrupt most recent commit, this should be the update to the previous # linked-list entry and should orphan the child -rm -v blocks/8 -tests/test.py << TEST +truncate -s-14 blocks/8 +tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; + + lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; + lfs_ssize_t before = lfs_fs_size(&lfs); + before => 10; + + lfs_unmount(&lfs) => 0; + lfs_mount(&lfs, &cfg) => 0; + lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; - unsigned before = 0; - lfs_traverse(&lfs, test_count, &before) => 0; - test_log("before", before); + lfs_ssize_t orphaned = lfs_fs_size(&lfs); + orphaned => 10; - lfs_deorphan(&lfs) => 0; + lfs_mkdir(&lfs, "parent/otherchild") => 0; lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; - unsigned after = 0; - lfs_traverse(&lfs, test_count, &after) => 0; - test_log("after", after); + lfs_ssize_t deorphaned = lfs_fs_size(&lfs); + deorphaned => 10; - int diff = before - after; - diff => 2; lfs_unmount(&lfs) => 0; TEST From 225706044e767b8d2348a2fe8b59642736f0a288 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 30 Jul 2018 14:23:51 -0500 Subject: [PATCH 067/139] Fixed test bugs around handling corruption The main thing to consider was how lfs_dir_fetchwith reacts to corruption it finds and to make sure falling back to old values works correctly. Some of the tricky bits involved making sure we could fall back to both old commits and old metadata blocks while still handling things like synthetic moves correctly. --- lfs.c | 95 +++++++++++++++++++++---------------------- tests/test_corrupt.sh | 3 -- 2 files changed, 47 insertions(+), 51 deletions(-) diff --git a/lfs.c b/lfs.c index 03c4b812..b512b36b 100644 --- a/lfs.c +++ b/lfs.c @@ -488,8 +488,7 @@ static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, uint32_t tag, uint32_t getmask, uint32_t gettag, int32_t getdiff, void *buffer, bool stopatcommit) { // iterate over dir block backwards (for faster lookups) - while (off > sizeof(tag)) { - LFS_ASSERT(off > sizeof(tag)+lfs_tagsize(tag)); + while (off >= 2*sizeof(tag)+lfs_tagsize(tag)) { off -= sizeof(tag)+lfs_tagsize(tag); if (lfs_tagtype(tag) == LFS_TYPE_CRC && stopatcommit) { @@ -1133,23 +1132,22 @@ static int32_t lfs_dir_find(lfs_t *lfs, lfs_off_t off = sizeof(dir->rev); uint32_t ptag = 0; uint32_t crc = 0xffffffff; - dir->tail[0] = 0xffffffff; - dir->tail[1] = 0xffffffff; - dir->count = 0; - dir->split = false; - dir->locals = (lfs_globals_t){0}; dir->rev = lfs_tole32(rev[0]); lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); dir->rev = lfs_fromle32(dir->rev); + dir->off = 0; - lfs_mdir_t tempdir = *dir; uint32_t tempfoundtag = foundtag; + uint16_t tempcount = 0; + lfs_block_t temptail[2] = {0xffffffff, 0xffffffff}; + bool tempsplit = false; + lfs_globals_t templocals = (lfs_globals_t){0}; while (true) { // extract next tag uint32_t tag; - int err = lfs_bd_read(lfs, tempdir.pair[0], + int err = lfs_bd_read(lfs, dir->pair[0], off, &tag, sizeof(tag)); if (err) { return err; @@ -1161,67 +1159,66 @@ static int32_t lfs_dir_find(lfs_t *lfs, // next commit not yet programmed if (lfs_tagtype(ptag) == LFS_TYPE_CRC && !lfs_tagisvalid(tag)) { dir->erased = true; - goto done; + break; } // check we're in valid range if (off + sizeof(tag)+lfs_tagsize(tag) > lfs->cfg->block_size) { + dir->erased = false; break; } if (lfs_tagtype(tag) == LFS_TYPE_CRC) { // check the crc attr uint32_t dcrc; - int err = lfs_bd_read(lfs, tempdir.pair[0], + int err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { return err; } if (crc != lfs_fromle32(dcrc)) { - if (off == sizeof(tempdir.rev)) { - // try other block - break; - } else { - // consider what we have good enough - dir->erased = false; - goto done; - } + dir->erased = false; + break; } - tempdir.off = off + sizeof(tag)+lfs_tagsize(tag); - tempdir.etag = tag; - crc = 0xffffffff; - *dir = tempdir; foundtag = tempfoundtag; + dir->off = off + sizeof(tag)+lfs_tagsize(tag); + dir->etag = tag; + dir->count = tempcount; + dir->tail[0] = temptail[0]; + dir->tail[1] = temptail[1]; + dir->split = tempsplit; + dir->locals = templocals; + crc = 0xffffffff; } else { - err = lfs_bd_crc(lfs, tempdir.pair[0], + err = lfs_bd_crc(lfs, dir->pair[0], off+sizeof(tag), lfs_tagsize(tag), &crc); if (err) { return err; } - if (lfs_tagid(tag) < 0x3ff && - lfs_tagid(tag) >= tempdir.count) { - tempdir.count = lfs_tagid(tag)+1; + if (lfs_tagid(tag) < 0x3ff && lfs_tagid(tag) >= tempcount) { + tempcount = lfs_tagid(tag)+1; } + // TODO use subtype accross all of these? if (lfs_tagsubtype(tag) == LFS_TYPE_TAIL) { - tempdir.split = (lfs_tagtype(tag) & 1); - err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), - tempdir.tail, sizeof(tempdir.tail)); + tempsplit = (lfs_tagtype(tag) & 1); + err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), + temptail, sizeof(temptail)); if (err) { return err; } } else if (lfs_tagtype(tag) == LFS_TYPE_GLOBALS) { - err = lfs_bd_read(lfs, tempdir.pair[0], off+sizeof(tag), - &tempdir.locals, sizeof(tempdir.locals)); + err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), + &templocals, sizeof(templocals)); if (err) { return err; } } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { - LFS_ASSERT(tempdir.count > 0); - tempdir.count -= 1; + LFS_ASSERT(tempcount > 0); + tempcount -= 1; if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { tempfoundtag = LFS_ERR_NOENT; @@ -1230,7 +1227,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, tempfoundtag -= LFS_MKTAG(0, 1, 0); } } else if ((tag & findmask) == (findtag & findmask)) { - int res = lfs_bd_cmp(lfs, tempdir.pair[0], off+sizeof(tag), + int res = lfs_bd_cmp(lfs, dir->pair[0], off+sizeof(tag), findbuffer, lfs_tagsize(tag)); if (res < 0) { return res; @@ -1247,6 +1244,21 @@ static int32_t lfs_dir_find(lfs_t *lfs, off += sizeof(tag)+lfs_tagsize(tag); } + // consider what we have good enough + if (dir->off > 0) { + // synthetic move + if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { + if (lfs->globals.move.id == lfs_tagid(foundtag)) { + foundtag = LFS_ERR_NOENT; + } else if (lfs_tagisvalid(foundtag) && + lfs->globals.move.id < lfs_tagid(foundtag)) { + foundtag -= LFS_MKTAG(0, 1, 0); + } + } + + return foundtag; + } + // failed, try the other crc? lfs_pairswap(dir->pair); lfs_pairswap(rev); @@ -1254,19 +1266,6 @@ static int32_t lfs_dir_find(lfs_t *lfs, LFS_ERROR("Corrupted dir pair at %d %d", dir->pair[0], dir->pair[1]); return LFS_ERR_CORRUPT; - -done: - // synthetic move - if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { - if (lfs->globals.move.id == lfs_tagid(foundtag)) { - foundtag = LFS_ERR_NOENT; - } else if (lfs_tagisvalid(foundtag) && - lfs->globals.move.id < lfs_tagid(foundtag)) { - foundtag -= LFS_MKTAG(0, 1, 0); - } - } - - return foundtag; } static int lfs_dir_fetch(lfs_t *lfs, diff --git a/tests/test_corrupt.sh b/tests/test_corrupt.sh index faa6f7ec..1491dac3 100755 --- a/tests/test_corrupt.sh +++ b/tests/test_corrupt.sh @@ -78,11 +78,8 @@ do rm -rf blocks mkdir blocks ln -s /dev/zero blocks/$(printf '%x' $i) - echo $i 1i lfs_mktree - echo $i 2i lfs_chktree - echo $i 3i done echo "--- Block persistance ---" From df1b6073516ce5be6e97cda27412d4dde26243f8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 30 Jul 2018 14:40:27 -0500 Subject: [PATCH 068/139] Removed the implicit lfs_t parameter to lfs_traverse This is a very minor thing but it has been bugging me. On one hand, all a callback ever needs is a single pointer for context. On the other hand, you could make the argument that in the context of littlefs, the lfs_t struct represents global state and should always be available to callbacks passed to littlefs. In the end I'm sticking with only a single context pointer, since this is satisfies the minimum requirements and has the highest chance of function reuse. If a user needs access to the lfs_t struct, it can be passed by reference in the context provided to the callback. This also matches callbacks used in other languages with more emphasis on objects and classes. Usually the callback doesn't get a reference to the caller. --- lfs.c | 20 ++++++++++---------- lfs.h | 3 +-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/lfs.c b/lfs.c index b512b36b..21603baa 100644 --- a/lfs.c +++ b/lfs.c @@ -259,8 +259,7 @@ static int lfs_bd_sync(lfs_t *lfs) { /// Internal operations predeclared here /// -int lfs_fs_traverse(lfs_t *lfs, - int (*cb)(lfs_t*, void*, lfs_block_t), void *data); +int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent); @@ -272,7 +271,8 @@ int lfs_deorphan(lfs_t *lfs); /// Block allocator /// -static int lfs_alloc_lookahead(lfs_t *lfs, void *p, lfs_block_t block) { +static int lfs_alloc_lookahead(void *p, lfs_block_t block) { + lfs_t *lfs = (lfs_t*)p; lfs_block_t off = ((block - lfs->free.off) + lfs->cfg->block_count) % lfs->cfg->block_count; @@ -320,7 +320,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { // find mask of free blocks from tree memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); - int err = lfs_fs_traverse(lfs, lfs_alloc_lookahead, NULL); + int err = lfs_fs_traverse(lfs, lfs_alloc_lookahead, lfs); if (err) { return err; } @@ -1768,7 +1768,7 @@ static int lfs_ctzextend(lfs_t *lfs, static int lfs_ctztraverse(lfs_t *lfs, lfs_cache_t *rcache, const lfs_cache_t *pcache, lfs_block_t head, lfs_size_t size, - int (*cb)(lfs_t*, void*, lfs_block_t), void *data) { + int (*cb)(void*, lfs_block_t), void *data) { if (size == 0) { return 0; } @@ -1776,7 +1776,7 @@ static int lfs_ctztraverse(lfs_t *lfs, lfs_off_t index = lfs_ctzindex(lfs, &(lfs_off_t){size-1}); while (true) { - int err = cb(lfs, data, head); + int err = cb(data, head); if (err) { return err; } @@ -1795,7 +1795,7 @@ static int lfs_ctztraverse(lfs_t *lfs, } for (int i = 0; i < count-1; i++) { - err = cb(lfs, data, heads[i]); + err = cb(data, heads[i]); if (err) { return err; } @@ -3117,7 +3117,7 @@ int lfs_unmount(lfs_t *lfs) { /// Internal filesystem filesystem operations /// int lfs_fs_traverse(lfs_t *lfs, - int (*cb)(lfs_t *lfs, void *data, lfs_block_t block), void *data) { + int (*cb)(void *data, lfs_block_t block), void *data) { if (lfs_pairisnull(lfs->root)) { return 0; } @@ -3126,7 +3126,7 @@ int lfs_fs_traverse(lfs_t *lfs, lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pairisnull(dir.tail)) { for (int i = 0; i < 2; i++) { - int err = cb(lfs, data, dir.tail[i]); + int err = cb(data, dir.tail[i]); if (err) { return err; } @@ -3542,7 +3542,7 @@ int lfs_deorphan(lfs_t *lfs) { //} // TODO need lfs? -static int lfs_fs_size_count(lfs_t *lfs, void *p, lfs_block_t block) { +static int lfs_fs_size_count(void *p, lfs_block_t block) { lfs_size_t *size = p; *size += 1; return 0; diff --git a/lfs.h b/lfs.h index b384ed33..647d6c8e 100644 --- a/lfs.h +++ b/lfs.h @@ -625,8 +625,7 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // blocks are in use or how much of the storage is available. // // Returns a negative error code on failure. -// TODO don't pass lfs_t? -int lfs_fs_traverse(lfs_t *lfs, int (*cb)(lfs_t*, void*, lfs_block_t), void *data); +int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); #endif From 105907ba6631654e16605a7818eac34b7a9419f9 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 30 Jul 2018 15:15:05 -0500 Subject: [PATCH 069/139] Cleaned up config usage in file logic The main change here was to drop the in-place twiddling of custom attributes to match the internal attribute structures. The original thought was that this could allow the compiler to garbage collect more of the custom attribute logic when not used, but since this occurs in the common lfs_file_opencfg function, gc can't really happen. Not twiddling the user's structure is the polite thing to do, opens up the ability to store the lfs_attr structure in ROM, and avoids surprising the user if they attempt to use the structure for their own purposes. This means we can make the lfs_attr structure const and rely on the list in the lfs_file_config structure, similar to how we rely on the global lfs_config structure. Some other tweaks: - Dropped the global file_buffer, replaced entirely by per-file buffers. - Updated LFS_INLINE_MAX and LFS_ATTR_MAX to correct values - Added workaround for compiler bug related to zero initializer: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119 --- lfs.c | 62 +++++++++++++++++++++++++++-------------------------------- lfs.h | 11 ++++------- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/lfs.c b/lfs.c index 21603baa..5c46c4c3 100644 --- a/lfs.c +++ b/lfs.c @@ -759,7 +759,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, dir->tail[1] = tail[1]; dir->erased = false; dir->split = split; - dir->locals = (lfs_globals_t){0}; + dir->locals = (lfs_globals_t){{{0}}}; // don't write out yet, let caller take care of that return 0; @@ -775,7 +775,7 @@ static int lfs_dir_compact(lfs_t *lfs, // There's nothing special about our global delta, so feed it back // into the global global delta lfs_globalsxor(&lfs->diff, &dir->locals); - dir->locals = (lfs_globals_t){0}; + dir->locals = (lfs_globals_t){{{0}}}; // increment revision count dir->rev += 1; @@ -929,7 +929,7 @@ static int lfs_dir_compact(lfs_t *lfs, if (!relocated) { // successful commit, update globals lfs_globalsxor(&dir->locals, &lfs->diff); - lfs->diff = (lfs_globals_t){0}; + lfs->diff = (lfs_globals_t){{{0}}}; } else { // update references if we relocated LFS_DEBUG("Relocating %d %d to %d %d", @@ -1070,7 +1070,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, dir->etag = commit.ptag; // successful commit, update globals lfs_globalsxor(&dir->locals, &lfs->diff); - lfs->diff = (lfs_globals_t){0}; + lfs->diff = (lfs_globals_t){{{0}}}; } // update globals that are affected @@ -1142,7 +1142,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, uint16_t tempcount = 0; lfs_block_t temptail[2] = {0xffffffff, 0xffffffff}; bool tempsplit = false; - lfs_globals_t templocals = (lfs_globals_t){0}; + lfs_globals_t templocals = (lfs_globals_t){{{0}}}; while (true) { // extract next tag @@ -1882,37 +1882,30 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, file->id = lfs_tagid(tag); file->flags = flags; file->pos = 0; - file->attrs = NULL; - - if (cfg && cfg->attrs) { - // fetch attrs - for (const struct lfs_attr *a = cfg->attrs; a; a = a->next) { - if ((file->flags & 3) != LFS_O_WRONLY) { - int32_t res = lfs_dir_get(lfs, &cwd, 0x7ffff000, - LFS_MKTAG(0x100 | a->type, file->id, a->size), a->buffer); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - } - - if ((file->flags & 3) != LFS_O_RDONLY) { - if (a->size > lfs->attr_size) { - return LFS_ERR_NOSPC; - } - file->flags |= LFS_F_DIRTY; + // fetch attrs + for (const struct lfs_attr *a = file->cfg->attrs; a; a = a->next) { + if ((file->flags & 3) != LFS_O_WRONLY) { + int32_t res = lfs_dir_get(lfs, &cwd, 0x7ffff000, + LFS_MKTAG(0x100 | a->type, file->id, a->size), a->buffer); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; } } - file->attrs = cfg->attrs; + if ((file->flags & 3) != LFS_O_RDONLY) { + if (a->size > lfs->attr_size) { + return LFS_ERR_NOSPC; + } + + file->flags |= LFS_F_DIRTY; + } } // allocate buffer if needed file->cache.block = 0xffffffff; - if (file->cfg && file->cfg->buffer) { + if (file->cfg->buffer) { file->cache.buffer = file->cfg->buffer; - } else if (lfs->cfg->file_buffer) { - file->cache.buffer = lfs->cfg->file_buffer; } else if ((file->flags & 3) == LFS_O_RDONLY) { file->cache.buffer = lfs_malloc(lfs->cfg->read_size); if (!file->cache.buffer) { @@ -1954,7 +1947,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, int lfs_file_open(lfs_t *lfs, lfs_file_t *file, const char *path, int flags) { - return lfs_file_opencfg(lfs, file, path, flags, NULL); + static const struct lfs_file_config defaults = {0}; + return lfs_file_opencfg(lfs, file, path, flags, &defaults); } int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { @@ -1969,7 +1963,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { } // clean up memory - if (!(file->cfg && file->cfg->buffer) && !lfs->cfg->file_buffer) { + if (file->cfg->buffer) { lfs_free(file->cache.buffer); } @@ -2119,7 +2113,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_CTZSTRUCT, file->id, &file->ctz.head, sizeof(file->ctz), - LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->attrs, 0, + LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, NULL))); if (err) { return err; @@ -2128,7 +2122,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, file->cache.buffer, file->ctz.size, - LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->attrs, 0, + LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, NULL))); if (err) { return err; @@ -2237,7 +2231,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } if ((file->flags & LFS_F_INLINE) && - file->pos + nsize >= lfs->cfg->inline_size) { + file->pos + nsize >= lfs->inline_size) { // inline file doesn't fit anymore file->block = 0xfffffffe; file->off = file->pos; @@ -3365,7 +3359,7 @@ int lfs_scan(lfs_t *lfs) { } lfs_mdir_t dir = {.tail = {0, 1}}; - lfs->diff = (lfs_globals_t){0}; + lfs->diff = (lfs_globals_t){{{0}}}; // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { @@ -3382,7 +3376,7 @@ int lfs_scan(lfs_t *lfs) { // TODO does this only run once? // TODO Should we inline this into init?? lfs_globalsxor(&lfs->globals, &lfs->diff); - lfs->diff = (lfs_globals_t){0}; + lfs->diff = (lfs_globals_t){{{0}}}; if (!lfs_pairisnull(lfs->globals.move.pair)) { LFS_DEBUG("Found move %d %d %d", lfs->globals.move.pair[0], diff --git a/lfs.h b/lfs.h index 647d6c8e..62d170d7 100644 --- a/lfs.h +++ b/lfs.h @@ -54,16 +54,18 @@ typedef uint32_t lfs_block_t; // read and prog cache, but if a file can be inline it does not need its own // data block. LFS_ATTR_MAX + LFS_INLINE_MAX must be <= 0xffff. Stored in // superblock and must be respected by other littlefs drivers. +// TODO doc #ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 0x3ff +#define LFS_INLINE_MAX 0xfff #endif // Maximum size of all attributes per file in bytes, may be redefined but a // a smaller LFS_ATTR_MAX has no benefit. LFS_ATTR_MAX + LFS_INLINE_MAX // must be <= 0xffff. Stored in superblock and must be respected by other // littlefs drivers. +// TODO doc #ifndef LFS_ATTR_MAX -#define LFS_ATTR_MAX 0x3f +#define LFS_ATTR_MAX 0xfff #endif // Max name size in bytes, may be redefined to reduce the size of the @@ -211,10 +213,6 @@ struct lfs_config { // lookahead block. void *lookahead_buffer; - // Optional, statically allocated buffer for files. Must be program sized. - // If enabled, only one file may be opened at a time. - void *file_buffer; - // Optional upper limit on inlined files in bytes. Large inline files // require a larger read and prog cache, but if a file can be inlined it // does not need its own data block. Must be smaller than the read size @@ -317,7 +315,6 @@ typedef struct lfs_file { } ctz; const struct lfs_file_config *cfg; - const struct lfs_attr *attrs; uint32_t flags; lfs_off_t pos; lfs_block_t block; From 1a58ba799ce0a73f78166b434bc979b52bb83128 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 30 Jul 2018 21:12:00 -0500 Subject: [PATCH 070/139] Fixed ENOSPC issues with zero-granularity blocks Result of testing on zero-granularity blocks, where the prog size and read size equals the block size. This represents SD cards and other traditional forms of block storage where we don't really get a benefit from the metadata logging. Unfortunately, since updates in both are tested by the same script, we can't really use simple bash commands. Added a more complex script to simulate corruption. Fortunately this should be more robust than the previous solutions. The main fixes were around corner cases where the commit logic fell apart when it didn't have room to complete commits, but these were fixable in the current design. --- lfs.c | 137 +++++++++++++++++++++++++------------------ tests/corrupt.py | 39 ++++++++++++ tests/test_move.sh | 12 ++-- tests/test_orphan.sh | 2 +- 4 files changed, 125 insertions(+), 65 deletions(-) create mode 100755 tests/corrupt.py diff --git a/lfs.c b/lfs.c index 5c46c4c3..cbafda08 100644 --- a/lfs.c +++ b/lfs.c @@ -886,6 +886,11 @@ static int lfs_dir_compact(lfs_t *lfs, // drop caches and create tail lfs->pcache.block = 0xffffffff; + if (ack == -1) { + // If we can't fit in this block, we won't fit in next block + return LFS_ERR_NOSPC; + } + lfs_mdir_t tail; int err = lfs_dir_alloc(lfs, &tail, dir->split, dir->tail); if (err) { @@ -1971,49 +1976,53 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { -relocate:; - // just relocate what exists into new block - lfs_block_t nblock; - int err = lfs_alloc(lfs, &nblock); - if (err) { - return err; - } - - err = lfs_bd_erase(lfs, nblock); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - // either read from dirty cache or disk - for (lfs_off_t i = 0; i < file->off; i++) { - uint8_t data; - err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, - file->block, i, &data, 1); + while (true) { + // just relocate what exists into new block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); if (err) { return err; } - err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, - nblock, i, &data, 1); + err = lfs_bd_erase(lfs, nblock); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } - } - // copy over new state of file - memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); - file->cache.block = lfs->pcache.block; - file->cache.off = lfs->pcache.off; - lfs->pcache.block = 0xffffffff; + // either read from dirty cache or disk + for (lfs_off_t i = 0; i < file->off; i++) { + uint8_t data; + err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, + file->block, i, &data, 1); + if (err) { + return err; + } - file->block = nblock; - return 0; + err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, + nblock, i, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + + // copy over new state of file + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); + file->cache.block = lfs->pcache.block; + file->cache.off = lfs->pcache.off; + lfs->pcache.block = 0xffffffff; + + file->block = nblock; + return 0; + +relocate: + continue; + } } static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { @@ -2067,6 +2076,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { } break; + relocate: LFS_DEBUG("Bad block at %d", file->block); err = lfs_file_relocate(lfs, file); @@ -2091,48 +2101,58 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { } int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { - int err = lfs_file_flush(lfs, file); - if (err) { - return err; - } - - if ((file->flags & LFS_F_DIRTY) && - !(file->flags & LFS_F_ERRED) && - !lfs_pairisnull(file->pair)) { - // update dir entry - // TODO keep list of dirs including these guys for no - // need of another reload? - lfs_mdir_t cwd; - err = lfs_dir_fetch(lfs, &cwd, file->pair); + while (true) { + int err = lfs_file_flush(lfs, file); if (err) { return err; } - // either update the references or inline the whole file - if (!(file->flags & LFS_F_INLINE)) { - int err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_CTZSTRUCT, file->id, - &file->ctz.head, sizeof(file->ctz), - LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, - NULL))); + if ((file->flags & LFS_F_DIRTY) && + !(file->flags & LFS_F_ERRED) && + !lfs_pairisnull(file->pair)) { + // update dir entry + // TODO keep list of dirs including these guys for no + // need of another reload? + lfs_mdir_t cwd; + err = lfs_dir_fetch(lfs, &cwd, file->pair); if (err) { return err; } - } else { + + // either update the references or inline the whole file int err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, - file->cache.buffer, file->ctz.size, LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, - NULL))); + (file->flags & LFS_F_INLINE) ? + LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, + file->cache.buffer, file->ctz.size, NULL) : + LFS_MKATTR(LFS_TYPE_CTZSTRUCT, file->id, + &file->ctz.head, sizeof(file->ctz), NULL))); if (err) { + if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) { + goto relocate; + } return err; } + + file->flags &= ~LFS_F_DIRTY; } - file->flags &= ~LFS_F_DIRTY; - } + return 0; - return 0; +relocate: + // inline file doesn't fit anymore + file->block = 0xfffffffe; + file->off = file->pos; + + lfs_alloc_ack(lfs); + err = lfs_file_relocate(lfs, file); + if (err) { + return err; + } + + file->flags &= ~LFS_F_INLINE; + file->flags |= LFS_F_WRITING; + } } lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, @@ -3304,6 +3324,7 @@ static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], // TODO rename to lfs_dir_relocate? static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]) { + // TODO name lfs_dir_relocate? // find parent lfs_mdir_t parent; int32_t tag = lfs_parent(lfs, oldpair, &parent); diff --git a/tests/corrupt.py b/tests/corrupt.py new file mode 100755 index 00000000..76f07ced --- /dev/null +++ b/tests/corrupt.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import struct +import sys +import os + +def main(*paths): + # find most recent block + file = None + rev = None + for path in paths: + try: + nfile = open(path, 'r+b') + nrev, = struct.unpack(' 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-7 blocks/6 +tests/corrupt.py blocks/{6,7} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -86,8 +86,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hello", "d/hello") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-7 blocks/8 -truncate -s-7 blocks/a +tests/corrupt.py blocks/{8,9} +tests/corrupt.py blocks/{a,b} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; @@ -166,7 +166,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-7 blocks/7 +tests/corrupt.py blocks/{6,7} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -193,8 +193,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hi", "d/hi") => 0; lfs_unmount(&lfs) => 0; TEST -truncate -s-7 blocks/9 -truncate -s-7 blocks/b +tests/corrupt.py blocks/{8,9} +tests/corrupt.py blocks/{a,b} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; diff --git a/tests/test_orphan.sh b/tests/test_orphan.sh index a76088b2..6aa36d9a 100755 --- a/tests/test_orphan.sh +++ b/tests/test_orphan.sh @@ -17,7 +17,7 @@ tests/test.py << TEST TEST # corrupt most recent commit, this should be the update to the previous # linked-list entry and should orphan the child -truncate -s-14 blocks/8 +tests/corrupt.py blocks/{8,9} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; From 64df0a5e2092a86745def995b44298881a864b93 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 31 Jul 2018 08:07:36 -0500 Subject: [PATCH 071/139] Added orphan bit to xored-globals Unfortunately for us, even with the new ability to store global state, orphans can not be handled as gracefully as moves. This is due to the fact that directory operations can create an unbounded number of orphans. It's usually small, the fact that it's unbounded means we can't store the orphan info in xored-globals. However, one thing we can do to leverage the xored-global state is store a bit indicating if _any_ orphans are present. This means in the common case we can completely avoid the deorphan step, while only using a single bit of the global state, which is effectively free since we can store it in the globals tag itself. If a littlefs drive does not want to consider the orphan bit, it's free to use the previous behaviour of always checking for orphans on first write. --- lfs.c | 363 +++++++++++++++++++++++++++++++++------------------------- lfs.h | 29 ++--- 2 files changed, 220 insertions(+), 172 deletions(-) diff --git a/lfs.c b/lfs.c index cbafda08..25c9a570 100644 --- a/lfs.c +++ b/lfs.c @@ -267,7 +267,7 @@ static int lfs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); int lfs_scan(lfs_t *lfs); int lfs_fixmove(lfs_t *lfs); -int lfs_deorphan(lfs_t *lfs); +int lfs_forceconsistency(lfs_t *lfs); /// Block allocator /// @@ -456,15 +456,68 @@ static inline lfs_size_t lfs_tagsize(uint32_t tag) { return tag & 0x00000fff; } -// operations on globals -static void lfs_globalsxor(lfs_globals_t *a, const lfs_globals_t *b) { - a->move.pair[0] ^= b->move.pair[0]; - a->move.pair[1] ^= b->move.pair[1]; - a->move.id ^= b->move.id; +// operations on set of globals +static inline void lfs_globalxor(lfs_global_t *a, const lfs_global_t *b) { + for (int i = 0; i < sizeof(lfs_global_t)/2; i++) { + a->u16[i] ^= b->u16[i]; + } +} + +static inline bool lfs_globaliszero(const lfs_global_t *a) { + for (int i = 0; i < sizeof(lfs_global_t)/2; i++) { + if (a->u16[i] != 0) { + return false; + } + } + return true; } -static bool lfs_globalsiszero(const lfs_globals_t *a) { - return (a->move.pair[0] == 0 && a->move.pair[1] == 0 && a->move.id == 0); +static inline void lfs_globalzero(lfs_global_t *a) { + memset(a->u16, 0x00, sizeof(lfs_global_t)); +} + +static inline void lfs_globalones(lfs_global_t *a) { + memset(a->u16, 0xff, sizeof(lfs_global_t)); +} + +static inline void lfs_globalxormove(lfs_global_t *a, + const lfs_block_t pair[2], uint16_t id) { + a->u16[0] ^= id; + for (int i = 0; i < sizeof(lfs_block_t[2])/2; i++) { + a->u16[1+i] ^= ((uint16_t*)pair)[i]; + } +} + +static inline void lfs_globalxordeorphaned(lfs_global_t *a, bool deorphaned) { + a->u16[0] ^= deorphaned << 15; +} + +static inline const lfs_block_t *lfs_globalmovepair(const lfs_t *lfs) { + return (const lfs_block_t*)&lfs->globals.u16[1]; +} + +static inline uint16_t lfs_globalmoveid(const lfs_t *lfs) { + return 0x3ff & lfs->globals.u16[0]; +} + +static inline bool lfs_globalisdeorphaned(const lfs_t *lfs) { + return 0x8000 & lfs->globals.u16[0]; +} + +static inline void lfs_globalmove(lfs_t *lfs, + const lfs_block_t pair[2], uint16_t id) { + lfs_global_t diff; + lfs_globalzero(&diff); + lfs_globalxormove(&diff, lfs_globalmovepair(lfs), lfs_globalmoveid(lfs)); + lfs_globalxormove(&diff, pair, id); + lfs_globalxor(&lfs->locals, &diff); + lfs_globalxor(&lfs->globals, &diff); +} + +static inline void lfs_globaldeorphaned(lfs_t *lfs, bool deorphaned) { + deorphaned ^= lfs_globalisdeorphaned(lfs); + lfs_globalxordeorphaned(&lfs->locals, deorphaned); + lfs_globalxordeorphaned(&lfs->globals, deorphaned); } @@ -670,15 +723,15 @@ static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, } static int lfs_commitglobals(lfs_t *lfs, struct lfs_commit *commit, - lfs_globals_t *locals) { - if (lfs_globalsiszero(&lfs->diff)) { + lfs_global_t *locals) { + if (lfs_globaliszero(&lfs->locals)) { return 0; } - lfs_globalsxor(locals, &lfs->diff); + lfs_globalxor(locals, &lfs->locals); int err = lfs_commitattr(lfs, commit, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0x3ff, sizeof(*locals)), locals); - lfs_globalsxor(locals, &lfs->diff); + LFS_MKTAG(LFS_TYPE_GLOBALS, 0x3ff, sizeof(lfs_global_t)), locals); + lfs_globalxor(locals, &lfs->locals); return err; } @@ -759,7 +812,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, dir->tail[1] = tail[1]; dir->erased = false; dir->split = split; - dir->locals = (lfs_globals_t){{{0}}}; + lfs_globalzero(&dir->locals); // don't write out yet, let caller take care of that return 0; @@ -774,8 +827,8 @@ static int lfs_dir_compact(lfs_t *lfs, // There's nothing special about our global delta, so feed it back // into the global global delta - lfs_globalsxor(&lfs->diff, &dir->locals); - dir->locals = (lfs_globals_t){{{0}}}; + lfs_globalxor(&lfs->locals, &dir->locals); + lfs_globalzero(&dir->locals); // increment revision count dir->rev += 1; @@ -933,8 +986,8 @@ static int lfs_dir_compact(lfs_t *lfs, if (!relocated) { // successful commit, update globals - lfs_globalsxor(&dir->locals, &lfs->diff); - lfs->diff = (lfs_globals_t){{{0}}}; + lfs_globalxor(&dir->locals, &lfs->locals); + lfs_globalzero(&lfs->locals); } else { // update references if we relocated LFS_DEBUG("Relocating %d %d to %d %d", @@ -962,18 +1015,21 @@ static int lfs_dir_compact(lfs_t *lfs, static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mattr_t *attrs) { - bool canceling = (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0); - lfs_mattr_t cancel; - if (canceling) { + lfs_mattr_t cancelattr; + lfs_global_t canceldiff; + lfs_globalzero(&canceldiff); + if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0) { // Wait, we have the move? Just cancel this out here // We need to, or else the move can become outdated - lfs->diff.move.pair[0] ^= 0xffffffff ^ lfs->globals.move.pair[0]; - lfs->diff.move.pair[1] ^= 0xffffffff ^ lfs->globals.move.pair[1]; - lfs->diff.move.id ^= 0x3ff ^ lfs->globals.move.id; + lfs_globalxormove(&canceldiff, + lfs_globalmovepair(lfs), lfs_globalmoveid(lfs)); + lfs_globalxormove(&canceldiff, + (lfs_block_t[2]){0xffffffff, 0xffffffff}, 0x3ff); + lfs_globalxor(&lfs->locals, &canceldiff); - cancel.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.move.id, 0); - cancel.next = attrs; - attrs = &cancel; + cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs_globalmoveid(lfs), 0); + cancelattr.next = attrs; + attrs = &cancelattr; } // calculate new directory size @@ -1001,7 +1057,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, pdir.split = dir->split; pdir.tail[0] = dir->tail[0]; pdir.tail[1] = dir->tail[1]; - lfs_globalsxor(&lfs->diff, &dir->locals); + lfs_globalxor(&lfs->locals, &dir->locals); return lfs_dir_commit(lfs, &pdir, LFS_MKATTR(LFS_TYPE_TAIL + pdir.split, 0x3ff, pdir.tail, sizeof(pdir.tail), @@ -1074,16 +1130,12 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, dir->off = commit.off; dir->etag = commit.ptag; // successful commit, update globals - lfs_globalsxor(&dir->locals, &lfs->diff); - lfs->diff = (lfs_globals_t){{{0}}}; + lfs_globalxor(&dir->locals, &lfs->locals); + lfs_globalzero(&lfs->locals); } // update globals that are affected - if (canceling) { - lfs->globals.move.pair[0] = 0xffffffff; - lfs->globals.move.pair[1] = 0xffffffff; - lfs->globals.move.id = 0x3ff; - } + lfs_globalxor(&lfs->globals, &canceldiff); // update any directories that are affected for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { @@ -1147,7 +1199,8 @@ static int32_t lfs_dir_find(lfs_t *lfs, uint16_t tempcount = 0; lfs_block_t temptail[2] = {0xffffffff, 0xffffffff}; bool tempsplit = false; - lfs_globals_t templocals = (lfs_globals_t){{{0}}}; + lfs_global_t templocals; + lfs_globalzero(&templocals); while (true) { // extract next tag @@ -1252,11 +1305,11 @@ static int32_t lfs_dir_find(lfs_t *lfs, // consider what we have good enough if (dir->off > 0) { // synthetic move - if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0) { - if (lfs->globals.move.id == lfs_tagid(foundtag)) { + if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0) { + if (lfs_globalmoveid(lfs) == lfs_tagid(foundtag)) { foundtag = LFS_ERR_NOENT; } else if (lfs_tagisvalid(foundtag) && - lfs->globals.move.id < lfs_tagid(foundtag)) { + lfs_globalmoveid(lfs) < lfs_tagid(foundtag)) { foundtag -= LFS_MKTAG(0, 1, 0); } } @@ -1286,8 +1339,8 @@ static int lfs_dir_fetch(lfs_t *lfs, static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, uint32_t gettag, void *buffer) { int32_t getdiff = 0; - if (lfs_paircmp(dir->pair, lfs->globals.move.pair) == 0 && - lfs_tagid(gettag) <= lfs->globals.move.id) { + if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0 && + lfs_tagid(gettag) <= lfs_globalmoveid(lfs)) { // synthetic moves getdiff = LFS_MKTAG(0, 1, 0); } @@ -1422,11 +1475,9 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron - if (!lfs->deorphaned) { - int err = lfs_deorphan(lfs); - if (err) { - return err; - } + int err = lfs_forceconsistency(lfs); + if (err) { + return err; } lfs_mdir_t cwd; @@ -1445,7 +1496,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_alloc_ack(lfs); lfs_mdir_t dir; - int err = lfs_dir_alloc(lfs, &dir, false, cwd.tail); + err = lfs_dir_alloc(lfs, &dir, false, cwd.tail); if (err) { return err; } @@ -1817,8 +1868,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, const char *path, int flags, const struct lfs_file_config *cfg) { // deorphan if we haven't yet, needed at most once after poweron - if ((flags & 3) != LFS_O_RDONLY && !lfs->deorphaned) { - int err = lfs_deorphan(lfs); + if ((flags & 3) != LFS_O_RDONLY) { + int err = lfs_forceconsistency(lfs); if (err) { return err; } @@ -2543,15 +2594,13 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { int lfs_remove(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron - if (!lfs->deorphaned) { - int err = lfs_deorphan(lfs); - if (err) { - return err; - } + int err = lfs_forceconsistency(lfs); + if (err) { + return err; } lfs_mdir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + err = lfs_dir_fetch(lfs, &cwd, lfs->root); if (err) { return err; } @@ -2580,6 +2629,9 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (dir.count > 0 || dir.split) { return LFS_ERR_NOTEMPTY; } + + // mark fs as orphaned + lfs_globaldeorphaned(lfs, false); } // delete the entry @@ -2596,11 +2648,14 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } + // fix orphan + lfs_globaldeorphaned(lfs, true); + // steal state // TODO test for global state stealing? cwd.tail[0] = dir.tail[0]; cwd.tail[1] = dir.tail[1]; - lfs_globalsxor(&lfs->diff, &dir.locals); + lfs_globalxor(&lfs->locals, &dir.locals); err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), @@ -2615,11 +2670,9 @@ int lfs_remove(lfs_t *lfs, const char *path) { int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // deorphan if we haven't yet, needed at most once after poweron - if (!lfs->deorphaned) { - int err = lfs_deorphan(lfs); - if (err) { - return err; - } + int err = lfs_forceconsistency(lfs); + if (err) { + return err; } // find old entry @@ -2665,6 +2718,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (prevdir.count > 0 || prevdir.split) { return LFS_ERR_NOTEMPTY; } + + // mark fs as orphaned + lfs_globaldeorphaned(lfs, false); } } else { // check that name fits @@ -2678,15 +2734,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // create move to fix later - lfs->diff.move.pair[0] = oldcwd.pair[0] ^ lfs->globals.move.pair[0]; - lfs->diff.move.pair[1] = oldcwd.pair[1] ^ lfs->globals.move.pair[1]; - lfs->diff.move.id = lfs_tagid(oldtag) ^ lfs->globals.move.id; - lfs->globals.move.pair[0] = oldcwd.pair[0]; - lfs->globals.move.pair[1] = oldcwd.pair[1]; - lfs->globals.move.id = lfs_tagid(oldtag); + lfs_globalmove(lfs, oldcwd.pair, lfs_tagid(oldtag)); // move over all attributes - int err = lfs_dir_commit(lfs, &newcwd, + err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTR(lfs_tagtype(oldtag), newid, newpath, strlen(newpath), LFS_MKATTR(LFS_FROM_MOVE, newid, &oldcwd, lfs_tagid(oldtag), NULL))); @@ -2709,11 +2760,14 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } + // fix orphan + lfs_globaldeorphaned(lfs, true); + // steal state // TODO test for global state stealing? newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; - lfs_globalsxor(&lfs->diff, &prevdir.locals); + lfs_globalxor(&lfs->locals, &prevdir.locals); err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, newcwd.tail, sizeof(newcwd.tail), @@ -2743,7 +2797,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // // if we were a directory, find pred, replace tail // // TODO can this just deorphan? // if (prevexists && lfs_tagsubtype(prevattr.tag) == LFS_TYPE_DIR) { -// err = lfs_deorphan(lfs); +// err = lfs_forceconsistency(lfs); // if (err) { // return err; // } @@ -2936,10 +2990,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[1] = 0xffffffff; lfs->files = NULL; lfs->dirs = NULL; - lfs->deorphaned = false; - lfs->globals.move.pair[0] = 0xffffffff; - lfs->globals.move.pair[1] = 0xffffffff; - lfs->globals.move.id = 0x3ff; + lfs_globalones(&lfs->globals); // scan for any global updates // TODO rm me? need to grab any inits @@ -3348,7 +3399,7 @@ static int lfs_relocate(lfs_t *lfs, } // clean up bad block, which should now be a desync - return lfs_deorphan(lfs); + return lfs_forceconsistency(lfs); } // find pred @@ -3380,7 +3431,7 @@ int lfs_scan(lfs_t *lfs) { } lfs_mdir_t dir = {.tail = {0, 1}}; - lfs->diff = (lfs_globals_t){{{0}}}; + lfs_globalzero(&lfs->locals); // iterate over all directory directory entries while (!lfs_pairisnull(dir.tail)) { @@ -3390,112 +3441,114 @@ int lfs_scan(lfs_t *lfs) { } // xor together indirect deletes - lfs_globalsxor(&lfs->diff, &dir.locals); + lfs_globalxor(&lfs->locals, &dir.locals); } // update littlefs with globals // TODO does this only run once? // TODO Should we inline this into init?? - lfs_globalsxor(&lfs->globals, &lfs->diff); - lfs->diff = (lfs_globals_t){{{0}}}; - if (!lfs_pairisnull(lfs->globals.move.pair)) { + lfs_globalxor(&lfs->globals, &lfs->locals); + lfs_globalzero(&lfs->locals); + if (!lfs_pairisnull(lfs_globalmovepair(lfs))) { LFS_DEBUG("Found move %d %d %d", - lfs->globals.move.pair[0], - lfs->globals.move.pair[1], - lfs->globals.move.id); + lfs_globalmovepair(lfs)[0], + lfs_globalmovepair(lfs)[1], + lfs_globalmoveid(lfs)); } return 0; } -int lfs_deorphan(lfs_t *lfs) { - lfs->deorphaned = true; - if (lfs_pairisnull(lfs->root)) { // TODO rm me? - return 0; - } +int lfs_forceconsistency(lfs_t *lfs) { + if (!lfs_globalisdeorphaned(lfs)) { + // Fix any orphans + lfs_mdir_t pdir = {.split = true}; + lfs_mdir_t dir = {.tail = {0, 1}}; - // Fix bad moves - if (!lfs_pairisnull(lfs->globals.move.pair)) { - LFS_DEBUG("Fixing move %d %d %d", // TODO move to just deorphan? - lfs->globals.move.pair[0], - lfs->globals.move.pair[1], - lfs->globals.move.id); + // iterate over all directory directory entries + while (!lfs_pairisnull(dir.tail)) { + int err = lfs_dir_fetch(lfs, &dir, dir.tail); + if (err) { + return err; + } - // fetch and delete the moved entry - lfs_mdir_t movedir; - int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.move.pair); - if (err) { - return err; - } + // check head blocks for orphans + if (!pdir.split) { + // check if we have a parent + lfs_mdir_t parent; + int32_t tag = lfs_parent(lfs, pdir.tail, &parent); + if (tag < 0 && tag != LFS_ERR_NOENT) { + return tag; + } - // rely on cancel logic inside commit - err = lfs_dir_commit(lfs, &movedir, NULL); - if (err) { - return err; - } - } + if (tag == LFS_ERR_NOENT) { + // we are an orphan + LFS_DEBUG("Found orphan %d %d", + pdir.tail[0], pdir.tail[1]); + + pdir.tail[0] = dir.tail[0]; + pdir.tail[1] = dir.tail[1]; + err = lfs_dir_commit(lfs, &pdir, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, + pdir.tail, sizeof(pdir.tail), + NULL)); + if (err) { + return err; + } - lfs_mdir_t pdir = {.split = true}; - lfs_mdir_t dir = {.tail = {0, 1}}; + break; + } - // iterate over all directory directory entries - while (!lfs_pairisnull(dir.tail)) { - int err = lfs_dir_fetch(lfs, &dir, dir.tail); - if (err) { - return err; - } + lfs_block_t pair[2]; + int32_t res = lfs_dir_get(lfs, &parent, 0x7ffff000, tag, pair); + if (res < 0) { + return res; + } - // check head blocks for orphans - if (!pdir.split) { - // check if we have a parent - lfs_mdir_t parent; - int32_t tag = lfs_parent(lfs, pdir.tail, &parent); - if (tag < 0 && tag != LFS_ERR_NOENT) { - return tag; - } + if (!lfs_pairsync(pair, pdir.tail)) { + // we have desynced + LFS_DEBUG("Found half-orphan %d %d", pair[0], pair[1]); - if (tag == LFS_ERR_NOENT) { - // we are an orphan - LFS_DEBUG("Found orphan %d %d", pdir.tail[0], pdir.tail[1]); + pdir.tail[0] = pair[0]; + pdir.tail[1] = pair[1]; + err = lfs_dir_commit(lfs, &pdir, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, + pdir.tail, sizeof(pdir.tail), + NULL)); + if (err) { + return err; + } - pdir.tail[0] = dir.tail[0]; - pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pdir.tail, sizeof(pdir.tail), - NULL)); - if (err) { - return err; + break; } - - break; } - lfs_block_t pair[2]; - int32_t res = lfs_dir_get(lfs, &parent, 0x7ffff000, tag, pair); - if (res < 0) { - return res; - } + memcpy(&pdir, &dir, sizeof(pdir)); + } - if (!lfs_pairsync(pair, pdir.tail)) { - // we have desynced - LFS_DEBUG("Found half-orphan %d %d", pair[0], pair[1]); + // mark orphan as fixed + lfs_globaldeorphaned(lfs, false); + } - pdir.tail[0] = pair[0]; - pdir.tail[1] = pair[1]; - err = lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pdir.tail, sizeof(pdir.tail), - NULL)); - if (err) { - return err; - } + if (lfs_globalmoveid(lfs) != 0x3ff) { + // Fix bad moves + LFS_DEBUG("Fixing move %d %d %d", // TODO move to just deorphan? + lfs_globalmovepair(lfs)[0], + lfs_globalmovepair(lfs)[1], + lfs_globalmoveid(lfs)); - break; - } + // fetch and delete the moved entry + lfs_mdir_t movedir; + int err = lfs_dir_fetch(lfs, &movedir, lfs_globalmovepair(lfs)); + if (err) { + return err; } - memcpy(&pdir, &dir, sizeof(pdir)); + // rely on cancel logic inside commit + err = lfs_dir_commit(lfs, &movedir, NULL); + if (err) { + return err; + } } return 0; diff --git a/lfs.h b/lfs.h index 62d170d7..092161e8 100644 --- a/lfs.h +++ b/lfs.h @@ -110,7 +110,7 @@ enum lfs_type { LFS_TYPE_TAIL = 0x0c0, LFS_TYPE_SOFTTAIL = 0x0c0, LFS_TYPE_HARDTAIL = 0x0c1, - LFS_TYPE_CRC = 0x0ff, + LFS_TYPE_CRC = 0x0ff, // TODO are trailing ones useful? LFS_TYPE_INLINESTRUCT = 0x040, LFS_TYPE_CTZSTRUCT = 0x041, @@ -280,12 +280,9 @@ typedef struct lfs_mattr { const struct lfs_mattr *next; } lfs_mattr_t; -typedef struct lfs_globals { - struct lfs_move { - lfs_block_t pair[2]; - uint16_t id; - } move; -} lfs_globals_t; +typedef union lfs_global { + uint16_t u16[5]; +} lfs_global_t; typedef struct lfs_mdir { lfs_block_t pair[2]; @@ -296,7 +293,7 @@ typedef struct lfs_mdir { uint16_t count; bool erased; bool split; - lfs_globals_t locals; + lfs_global_t locals; } lfs_mdir_t; typedef struct lfs_cache { @@ -314,12 +311,13 @@ typedef struct lfs_file { lfs_size_t size; } ctz; - const struct lfs_file_config *cfg; uint32_t flags; lfs_off_t pos; lfs_block_t block; lfs_off_t off; lfs_cache_t cache; + + const struct lfs_file_config *cfg; } lfs_file_t; typedef struct lfs_dir { @@ -353,21 +351,18 @@ typedef struct lfs_free { // The littlefs type typedef struct lfs { - const struct lfs_config *cfg; + lfs_cache_t rcache; + lfs_cache_t pcache; lfs_block_t root[2]; lfs_file_t *files; lfs_dir_t *dirs; - lfs_cache_t rcache; - lfs_cache_t pcache; - + lfs_global_t globals; + lfs_global_t locals; lfs_free_t free; - bool deorphaned; - lfs_globals_t globals2; - lfs_globals_t globals; - lfs_globals_t diff; + const struct lfs_config *cfg; lfs_size_t inline_size; lfs_size_t attr_size; lfs_size_t name_size; From 112fefc068c0f2d2d16967490283dd3db7d30529 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 1 Aug 2018 18:10:24 -0500 Subject: [PATCH 072/139] Added back big-endian support again on the new metadata structures The only interesting thing to note is that we now have to also support le16 due to storing the id outside of tags in the globals structure. --- lfs.c | 189 ++++++++++++++++++++++++++++++----------------------- lfs_util.h | 25 ++++++- 2 files changed, 131 insertions(+), 83 deletions(-) diff --git a/lfs.c b/lfs.c index 25c9a570..439fbc1c 100644 --- a/lfs.c +++ b/lfs.c @@ -264,7 +264,7 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent); static int lfs_relocate(lfs_t *lfs, - const lfs_block_t oldpair[2], const lfs_block_t newpair[2]); + const lfs_block_t oldpair[2], lfs_block_t newpair[2]); int lfs_scan(lfs_t *lfs); int lfs_fixmove(lfs_t *lfs); int lfs_forceconsistency(lfs_t *lfs); @@ -332,74 +332,6 @@ static void lfs_alloc_ack(lfs_t *lfs) { } -/// Endian swapping functions /// -//static void lfs_dir_fromle32(struct lfs_disk_dir *d) { -// d->rev = lfs_fromle32(d->rev); -// d->size = lfs_fromle32(d->size); -// d->tail[0] = lfs_fromle32(d->tail[0]); -// d->tail[1] = lfs_fromle32(d->tail[1]); -//} -// -//static void lfs_mdir_tole32(struct lfs_disk_dir *d) { -// d->rev = lfs_tole32(d->rev); -// d->size = lfs_tole32(d->size); -// d->tail[0] = lfs_tole32(d->tail[0]); -// d->tail[1] = lfs_tole32(d->tail[1]); -//} -// -//static void lfs_entry_fromle32(struct lfs_disk_entry *d) { -// d->u.dir[0] = lfs_fromle32(d->u.dir[0]); -// d->u.dir[1] = lfs_fromle32(d->u.dir[1]); -//} -// -//static void lfs_entry_tole32(struct lfs_disk_entry *d) { -// d->u.dir[0] = lfs_tole32(d->u.dir[0]); -// d->u.dir[1] = lfs_tole32(d->u.dir[1]); -//} - -///*static*/ void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { -// d->root[0] = lfs_fromle32(d->root[0]); -// d->root[1] = lfs_fromle32(d->root[1]); -// d->block_size = lfs_fromle32(d->block_size); -// d->block_count = lfs_fromle32(d->block_count); -// d->version = lfs_fromle32(d->version); -// d->inline_size = lfs_fromle32(d->inline_size); -// d->attr_size = lfs_fromle32(d->attr_size); -// d->name_size = lfs_fromle32(d->name_size); -//} -// -///*static*/ void lfs_superblock_tole32(struct lfs_disk_superblock *d) { -// d->root[0] = lfs_tole32(d->root[0]); -// d->root[1] = lfs_tole32(d->root[1]); -// d->block_size = lfs_tole32(d->block_size); -// d->block_count = lfs_tole32(d->block_count); -// d->version = lfs_tole32(d->version); -// d->inline_size = lfs_tole32(d->inline_size); -// d->attr_size = lfs_tole32(d->attr_size); -// d->name_size = lfs_tole32(d->name_size); -//} - -/// Other struct functions /// -//static inline lfs_size_t lfs_entry_elen(const lfs_mattr_t *attr) { -// return (lfs_size_t)(attr->d.elen) | -// ((lfs_size_t)(attr->d.alen & 0xc0) << 2); -//} -// -//static inline lfs_size_t lfs_entry_alen(const lfs_mattr_t *attr) { -// return attr->d.alen & 0x3f; -//} -// -//static inline lfs_size_t lfs_entry_nlen(const lfs_mattr_t *attr) { -// return attr->d.nlen; -//} -// -//static inline lfs_size_t lfs_entry_size(const lfs_mattr_t *attr) { -// return 4 + lfs_entry_elen(attr) + -// lfs_entry_alen(attr) + -// lfs_entry_nlen(attr); -//} - - /// Metadata pair and directory operations /// static inline void lfs_pairswap(lfs_block_t pair[2]) { lfs_block_t t = pair[0]; @@ -425,6 +357,27 @@ static inline bool lfs_pairsync( (paira[0] == pairb[1] && paira[1] == pairb[0]); } +static inline void lfs_pairfromle32(lfs_block_t *pair) { + pair[0] = lfs_fromle32(pair[0]); + pair[1] = lfs_fromle32(pair[1]); +} + +static inline void lfs_pairtole32(lfs_block_t *pair) { + pair[0] = lfs_tole32(pair[0]); + pair[1] = lfs_tole32(pair[1]); +} + +static void lfs_ctzfromle32(struct lfs_ctz *ctz) { + ctz->head = lfs_fromle32(ctz->head); + ctz->size = lfs_fromle32(ctz->size); +} + +static void lfs_ctztole32(struct lfs_ctz *ctz) { + ctz->head = lfs_tole32(ctz->head); + ctz->size = lfs_tole32(ctz->size); +} + + /// Entry tag operations /// #define LFS_MKTAG(type, id, size) \ (((uint32_t)(type) << 22) | ((uint32_t)(id) << 12) | (uint32_t)(size)) @@ -492,6 +445,16 @@ static inline void lfs_globalxordeorphaned(lfs_global_t *a, bool deorphaned) { a->u16[0] ^= deorphaned << 15; } +static inline void lfs_globalfromle32(lfs_global_t *a) { + a->u16[0] = lfs_fromle16(a->u16[0]); + lfs_pairfromle32((lfs_block_t*)&a->u16[1]); +} + +static inline void lfs_globaltole32(lfs_global_t *a) { + a->u16[0] = lfs_tole16(a->u16[0]); + lfs_pairtole32((lfs_block_t*)&a->u16[1]); +} + static inline const lfs_block_t *lfs_globalmovepair(const lfs_t *lfs) { return (const lfs_block_t*)&lfs->globals.u16[1]; } @@ -510,13 +473,17 @@ static inline void lfs_globalmove(lfs_t *lfs, lfs_globalzero(&diff); lfs_globalxormove(&diff, lfs_globalmovepair(lfs), lfs_globalmoveid(lfs)); lfs_globalxormove(&diff, pair, id); + lfs_globalfromle32(&lfs->locals); lfs_globalxor(&lfs->locals, &diff); + lfs_globaltole32(&lfs->locals); lfs_globalxor(&lfs->globals, &diff); } static inline void lfs_globaldeorphaned(lfs_t *lfs, bool deorphaned) { deorphaned ^= lfs_globalisdeorphaned(lfs); + lfs_globalfromle32(&lfs->locals); lfs_globalxordeorphaned(&lfs->locals, deorphaned); + lfs_globaltole32(&lfs->locals); lfs_globalxordeorphaned(&lfs->globals, deorphaned); } @@ -748,7 +715,8 @@ static int lfs_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { } // build crc tag - tag = (0x80000000 & ~lfs_fromle32(tag)) | + tag = lfs_fromle32(tag); + tag = (0x80000000 & ~tag) | LFS_MKTAG(LFS_TYPE_CRC, 0x3ff, off - (commit->off+sizeof(uint32_t))); @@ -907,9 +875,11 @@ static int lfs_dir_compact(lfs_t *lfs, if (!lfs_pairisnull(dir->tail)) { // commit tail, which may be new after last size check // TODO le32 + lfs_pairtole32(dir->tail); err = lfs_commitattr(lfs, &commit, LFS_MKTAG(LFS_TYPE_TAIL + dir->split, 0x3ff, sizeof(dir->tail)), dir->tail); + lfs_pairfromle32(dir->tail); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1025,7 +995,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_globalmovepair(lfs), lfs_globalmoveid(lfs)); lfs_globalxormove(&canceldiff, (lfs_block_t[2]){0xffffffff, 0xffffffff}, 0x3ff); + lfs_globalfromle32(&lfs->locals); lfs_globalxor(&lfs->locals, &canceldiff); + lfs_globaltole32(&lfs->locals); cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs_globalmoveid(lfs), 0); cancelattr.next = attrs; @@ -1089,7 +1061,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, for (const lfs_mattr_t *a = attrs; a; a = a->next) { if (lfs_tagtype(a->tag) != LFS_TYPE_DELETE) { + lfs_pairtole32(dir->tail); int err = lfs_commitattr(lfs, &commit, a->tag, a->buffer); + lfs_pairfromle32(dir->tail); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1173,10 +1147,10 @@ static int32_t lfs_dir_find(lfs_t *lfs, uint32_t rev[2]; for (int i = 0; i < 2; i++) { int err = lfs_bd_read(lfs, dir->pair[i], 0, &rev[i], sizeof(rev[i])); - rev[i] = lfs_fromle32(rev[i]); if (err) { return err; } + rev[i] = lfs_fromle32(rev[i]); } if (lfs_scmp(rev[1], rev[0]) > 0) { @@ -1234,8 +1208,9 @@ static int32_t lfs_dir_find(lfs_t *lfs, if (err) { return err; } + dcrc = lfs_fromle32(dcrc); - if (crc != lfs_fromle32(dcrc)) { + if (crc != dcrc) { dir->erased = false; break; } @@ -1268,6 +1243,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, if (err) { return err; } + lfs_pairfromle32(temptail); } else if (lfs_tagtype(tag) == LFS_TYPE_GLOBALS) { err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), &templocals, sizeof(templocals)); @@ -1417,6 +1393,7 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { if (res < 0) { return res; } + lfs_pairfromle32(pair); } // find entry matching name @@ -1462,6 +1439,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, if (tag < 0) { return tag; } + lfs_ctzfromle32(&ctz); if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { info->size = ctz.size; @@ -1510,11 +1488,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { uint16_t id = cwd.count; cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; + lfs_pairtole32(dir.pair); err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), NULL)))); + lfs_pairfromle32(dir.pair); if (err) { return err; } @@ -1546,6 +1526,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { if (res < 0) { return res; } + lfs_pairfromle32(pair); } // fetch first pair @@ -1929,6 +1910,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, if (tag < 0) { return tag; } + lfs_ctzfromle32(&file->ctz); } // setup file struct @@ -2170,14 +2152,28 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - // either update the references or inline the whole file + uint16_t type; + const void *buffer; + lfs_size_t size; + if (file->flags & LFS_F_INLINE) { + // inline the whole file + type = LFS_TYPE_INLINESTRUCT; + buffer = file->cache.buffer; + size = file->ctz.size; + } else { + // update the ctz reference + type = LFS_TYPE_CTZSTRUCT; + buffer = &file->ctz; + size = sizeof(file->ctz); + } + + // commit file data and attributes + lfs_ctztole32(&file->ctz); int err = lfs_dir_commit(lfs, &cwd, + LFS_MKATTR(type, file->id, buffer, size, LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, - (file->flags & LFS_F_INLINE) ? - LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, - file->cache.buffer, file->ctz.size, NULL) : - LFS_MKATTR(LFS_TYPE_CTZSTRUCT, file->id, - &file->ctz.head, sizeof(file->ctz), NULL))); + NULL))); + lfs_ctzfromle32(&file->ctz); if (err) { if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) { goto relocate; @@ -2619,6 +2615,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (res < 0) { return res; } + lfs_pairfromle32(pair); int err = lfs_dir_fetch(lfs, &dir, pair); if (err) { @@ -2708,6 +2705,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (res < 0) { return res; } + lfs_pairfromle32(prevpair); // must be empty before removal int err = lfs_dir_fetch(lfs, &prevdir, prevpair); @@ -2920,6 +2918,24 @@ int lfs_fs_setattr(lfs_t *lfs, /// Filesystem operations /// +static inline void lfs_superblockfromle32(lfs_superblock_t *superblock) { + superblock->version = lfs_fromle32(superblock->version); + superblock->block_size = lfs_fromle32(superblock->block_size); + superblock->block_count = lfs_fromle32(superblock->block_count); + superblock->inline_size = lfs_fromle32(superblock->inline_size); + superblock->attr_size = lfs_fromle32(superblock->attr_size); + superblock->name_size = lfs_fromle32(superblock->name_size); +} + +static inline void lfs_superblocktole32(lfs_superblock_t *superblock) { + superblock->version = lfs_tole32(superblock->version); + superblock->block_size = lfs_tole32(superblock->block_size); + superblock->block_count = lfs_tole32(superblock->block_count); + superblock->inline_size = lfs_tole32(superblock->inline_size); + superblock->attr_size = lfs_tole32(superblock->attr_size); + superblock->name_size = lfs_tole32(superblock->name_size); +} + static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->cfg = cfg; @@ -3070,10 +3086,14 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .name_size = lfs->name_size, }; + lfs_superblocktole32(&superblock); + lfs_pairtole32(lfs->root); err = lfs_dir_commit(lfs, &dir, LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), LFS_MKATTR(LFS_TYPE_DIRSTRUCT, 0, lfs->root, sizeof(lfs->root), NULL))); + lfs_pairfromle32(lfs->root); + lfs_superblockfromle32(&superblock); if (err) { return err; } @@ -3116,6 +3136,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { if (res < 0) { return res; } + lfs_superblockfromle32(&superblock); if (memcmp(superblock.magic, "littlefs", 8) != 0) { LFS_ERROR("Invalid superblock at %d %d", 0, 1); @@ -3136,6 +3157,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { if (res < 0) { return res; } + lfs_pairfromle32(lfs->root); if (superblock.inline_size) { if (superblock.inline_size > lfs->inline_size) { @@ -3213,6 +3235,7 @@ int lfs_fs_traverse(lfs_t *lfs, } return tag; } + lfs_ctzfromle32(&ctz); if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { int err = lfs_ctztraverse(lfs, &lfs->rcache, NULL, @@ -3352,7 +3375,7 @@ static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], // search for both orderings so we can reuse the find function lfs_block_t child[2] = {pair[0], pair[1]}; - + lfs_pairtole32(child); for (int i = 0; i < 2; i++) { // iterate over all directory directory entries parent->tail[0] = 0; @@ -3374,7 +3397,7 @@ static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], // TODO rename to lfs_dir_relocate? static int lfs_relocate(lfs_t *lfs, - const lfs_block_t oldpair[2], const lfs_block_t newpair[2]) { + const lfs_block_t oldpair[2], lfs_block_t newpair[2]) { // TODO name lfs_dir_relocate? // find parent lfs_mdir_t parent; @@ -3385,8 +3408,10 @@ static int lfs_relocate(lfs_t *lfs, if (tag != LFS_ERR_NOENT) { // update disk, this creates a desync + lfs_pairtole32(newpair); int err = lfs_dir_commit(lfs, &parent, &(lfs_mattr_t){.tag=tag, .buffer=newpair}); + lfs_pairfromle32(newpair); if (err) { return err; } @@ -3415,7 +3440,7 @@ static int lfs_relocate(lfs_t *lfs, parent.tail[1] = newpair[1]; int err = lfs_dir_commit(lfs, &parent, LFS_MKATTR(LFS_TYPE_TAIL + parent.split, 0x3ff, - newpair, sizeof(lfs_block_t[2]), + parent.tail, sizeof(parent.tail), NULL)); if (err) { return err; @@ -3447,6 +3472,7 @@ int lfs_scan(lfs_t *lfs) { // update littlefs with globals // TODO does this only run once? // TODO Should we inline this into init?? + lfs_globalfromle32(&lfs->locals); lfs_globalxor(&lfs->globals, &lfs->locals); lfs_globalzero(&lfs->locals); if (!lfs_pairisnull(lfs_globalmovepair(lfs))) { @@ -3504,6 +3530,7 @@ int lfs_forceconsistency(lfs_t *lfs) { if (res < 0) { return res; } + lfs_pairfromle32(pair); if (!lfs_pairsync(pair, pdir.tail)) { // we have desynced diff --git a/lfs_util.h b/lfs_util.h index ecd5dd30..6a577eb8 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -136,7 +136,7 @@ static inline int lfs_scmp(uint32_t a, uint32_t b) { return (int)(unsigned)(a - b); } -// Convert from 32-bit little-endian to native order +// Convert between 32-bit little-endian and native order static inline uint32_t lfs_fromle32(uint32_t a) { #if !defined(LFS_NO_INTRINSICS) && ( \ (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ @@ -156,11 +156,32 @@ static inline uint32_t lfs_fromle32(uint32_t a) { #endif } -// Convert to 32-bit little-endian from native order static inline uint32_t lfs_tole32(uint32_t a) { return lfs_fromle32(a); } +// Convert between 16-bit little-endian and native order +static inline uint16_t lfs_fromle16(uint16_t a) { +#if !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + return a; +#elif !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + return __builtin_bswap16(a); +#else + return (((uint8_t*)&a)[0] << 0) | + (((uint8_t*)&a)[1] << 8); +#endif +} + +static inline uint16_t lfs_tole16(uint16_t a) { + return lfs_fromle16(a); +} + // Align to nearest multiple of a size static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) { return (a + alignment-1) - ((a + alignment-1) % alignment); From 01d837e08d6cf9160a90aca54aa456157bcf6037 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 1 Aug 2018 01:05:04 -0500 Subject: [PATCH 073/139] Removed redundant lfs_scan in lfs_init Interestingly enough, lfs_scan is only needed during mount, as the state of the filesystem is well know during format. --- lfs.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/lfs.c b/lfs.c index 439fbc1c..094182d4 100644 --- a/lfs.c +++ b/lfs.c @@ -3007,13 +3007,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->files = NULL; lfs->dirs = NULL; lfs_globalones(&lfs->globals); - - // scan for any global updates - // TODO rm me? need to grab any inits - int err = lfs_scan(lfs); - if (err) { - return err; - } + lfs_globalzero(&lfs->locals); return 0; } @@ -3189,6 +3183,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->name_size = superblock.name_size; } + // scan for any global updates err = lfs_scan(lfs); if (err) { return err; @@ -3343,8 +3338,6 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { */ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { if (lfs_pairisnull(lfs->root)) { - // TODO best place for this? - // TODO needed for relocate return LFS_ERR_NOENT; } @@ -3369,7 +3362,6 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *parent) { if (lfs_pairisnull(lfs->root)) { - // TODO best place for this? return LFS_ERR_NOENT; } @@ -3451,14 +3443,12 @@ static int lfs_relocate(lfs_t *lfs, } int lfs_scan(lfs_t *lfs) { - if (lfs_pairisnull(lfs->root)) { // TODO rm me + if (lfs_pairisnull(lfs->root)) { return 0; } - lfs_mdir_t dir = {.tail = {0, 1}}; - lfs_globalzero(&lfs->locals); - // iterate over all directory directory entries + lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pairisnull(dir.tail)) { int err = lfs_dir_fetch(lfs, &dir, dir.tail); if (err) { From bd1e0c405950db089b419f13d4cbc79d076efa68 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 1 Aug 2018 05:52:48 -0500 Subject: [PATCH 074/139] Cleaned up several TODOs Other than removed outdated TODOs, there are several tweaks: - Standardized naming of fs-level functions (mostly internal names) - Tweaked low-level use of subtype to hopefully take advantage of redundant code removal - Moved root-handling into lfs_dir_getinfo - Updated DEBUG statements around move/orphan fixes - Removed trailing 1s in type fields - Removed unused code --- lfs.c | 444 +++++++++++----------------------------------------------- lfs.h | 8 +- 2 files changed, 86 insertions(+), 366 deletions(-) diff --git a/lfs.c b/lfs.c index 094182d4..0193bb80 100644 --- a/lfs.c +++ b/lfs.c @@ -259,15 +259,14 @@ static int lfs_bd_sync(lfs_t *lfs) { /// Internal operations predeclared here /// -int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); -static int lfs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); -static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], +static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], + lfs_mdir_t *pdir); +static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent); -static int lfs_relocate(lfs_t *lfs, +static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]); -int lfs_scan(lfs_t *lfs); -int lfs_fixmove(lfs_t *lfs); -int lfs_forceconsistency(lfs_t *lfs); +static int lfs_fs_scan(lfs_t *lfs); +static int lfs_fs_forceconsistency(lfs_t *lfs); /// Block allocator /// @@ -874,7 +873,6 @@ static int lfs_dir_compact(lfs_t *lfs, if (!lfs_pairisnull(dir->tail)) { // commit tail, which may be new after last size check - // TODO le32 lfs_pairtole32(dir->tail); err = lfs_commitattr(lfs, &commit, LFS_MKTAG(LFS_TYPE_TAIL + dir->split, @@ -962,7 +960,7 @@ static int lfs_dir_compact(lfs_t *lfs, // update references if we relocated LFS_DEBUG("Relocating %d %d to %d %d", oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); - int err = lfs_relocate(lfs, oldpair, dir->pair); + int err = lfs_fs_relocate(lfs, oldpair, dir->pair); if (err) { return err; } @@ -1019,7 +1017,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, if (dir->count == 0) { // should we actually drop the directory block? lfs_mdir_t pdir; - int err = lfs_pred(lfs, dir->pair, &pdir); + int err = lfs_fs_pred(lfs, dir->pair, &pdir); if (err && err != LFS_ERR_NOENT) { return err; } @@ -1235,7 +1233,6 @@ static int32_t lfs_dir_find(lfs_t *lfs, tempcount = lfs_tagid(tag)+1; } - // TODO use subtype accross all of these? if (lfs_tagsubtype(tag) == LFS_TYPE_TAIL) { tempsplit = (lfs_tagtype(tag) & 1); err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), @@ -1244,13 +1241,13 @@ static int32_t lfs_dir_find(lfs_t *lfs, return err; } lfs_pairfromle32(temptail); - } else if (lfs_tagtype(tag) == LFS_TYPE_GLOBALS) { + } else if (lfs_tagsubtype(tag) == LFS_TYPE_GLOBALS) { err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), &templocals, sizeof(templocals)); if (err) { return err; } - } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { + } else if (lfs_tagsubtype(tag) == LFS_TYPE_DELETE) { LFS_ASSERT(tempcount > 0); tempcount -= 1; @@ -1424,7 +1421,14 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { } static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, - int16_t id, struct lfs_info *info) { + uint16_t id, struct lfs_info *info) { + if (id == 0x3ff) { + // special case for root + strcpy(info->name, "/"); + info->type = LFS_TYPE_DIR; + return 0; + } + int32_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_size+1), info->name); if (tag < 0) { @@ -1453,7 +1457,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron - int err = lfs_forceconsistency(lfs); + int err = lfs_fs_forceconsistency(lfs); if (err) { return err; } @@ -1499,8 +1503,6 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } - // TODO need ack here? - lfs_alloc_ack(lfs); return 0; } @@ -1605,7 +1607,6 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { return true; } -// TODO does this work? int lfs_dir_seek(lfs_t *lfs, lfs_dir_t *dir, lfs_off_t off) { // simply walk from head dir int err = lfs_dir_rewind(lfs, dir); @@ -1850,7 +1851,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, const struct lfs_file_config *cfg) { // deorphan if we haven't yet, needed at most once after poweron if ((flags & 3) != LFS_O_RDONLY) { - int err = lfs_forceconsistency(lfs); + int err = lfs_fs_forceconsistency(lfs); if (err) { return err; } @@ -1876,7 +1877,6 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // get next slot and create entry to remember name // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change - // TODO don't use inline struct? just leave it out? uint16_t id = cwd.count; int err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_REG, id, path, nlen, @@ -1886,7 +1886,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, return err; } - // TODO eh AHHHHHHHHHHHHHH + // TODO should we be catching this here? if (id >= cwd.count) { // catch updates from a compact in the above commit id -= cwd.count; @@ -1924,6 +1924,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // fetch attrs for (const struct lfs_attr *a = file->cfg->attrs; a; a = a->next) { if ((file->flags & 3) != LFS_O_WRONLY) { + // TODO what if cwd is invalid from above compact? int32_t res = lfs_dir_get(lfs, &cwd, 0x7ffff000, LFS_MKTAG(0x100 | a->type, file->id, a->size), a->buffer); if (res < 0 && res != LFS_ERR_NOENT) { @@ -2487,110 +2488,21 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { } } -//int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, -// const struct lfs_attr *attrs, int count) { -// // set to null in case we can't find the attrs (missing file?) -// for (int j = 0; j < count; j++) { -// memset(attrs[j].buffer, 0, attrs[j].size); -// } -// -// // load from disk if we haven't already been deleted -// if (!lfs_pairisnull(file->pair)) { -// lfs_mdir_t cwd; -// int err = lfs_dir_fetch(lfs, &cwd, file->pair); -// if (err) { -// return err; -// } -// -// lfs_mattr_t entry = {.off = file->pairoff}; -// err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); -// if (err) { -// return err; -// } -// entry.size = lfs_entry_size(&entry); -// -// err = lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); -// if (err) { -// return err; -// } -// } -// -// // override an attrs we have stored locally -// for (int i = 0; i < file->attrcount; i++) { -// for (int j = 0; j < count; j++) { -// if (attrs[j].type == file->attrs[i].type) { -// if (attrs[j].size < file->attrs[i].size) { -// return LFS_ERR_RANGE; -// } -// -// memset(attrs[j].buffer, 0, attrs[j].size); -// memcpy(attrs[j].buffer, -// file->attrs[i].buffer, file->attrs[i].size); -// } -// } -// } -// -// return 0; -//} - -//int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, -// const struct lfs_attr *attrs, int count) { -// if ((file->flags & 3) == LFS_O_RDONLY) { -// return LFS_ERR_BADF; -// } -// -// // at least make sure attributes fit -// if (!lfs_pairisnull(file->pair)) { -// lfs_mdir_t cwd; -// int err = lfs_dir_fetch(lfs, &cwd, file->pair); -// if (err) { -// return err; -// } -// -// lfs_mattr_t entry = {.off = file->pairoff}; -// err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); -// if (err) { -// return err; -// } -// entry.size = lfs_entry_size(&entry); -// -// lfs_ssize_t res = lfs_dir_checkattrs(lfs, &cwd, &entry, attrs, count); -// if (res < 0) { -// return res; -// } -// } -// -// // just tack to the file, will be written at sync time -// file->attrs = attrs; -// file->attrcount = count; -// file->flags |= LFS_F_DIRTY; -// -// return 0; -//} - /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { lfs_mdir_t cwd; - // TODO pass to getinfo? int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); if (tag < 0) { return tag; } - if (lfs_tagid(tag) == 0x3ff) { - // special case for root - strcpy(info->name, "/"); - info->type = LFS_TYPE_DIR; - return 0; - } - return lfs_dir_getinfo(lfs, &cwd, lfs_tagid(tag), info); } int lfs_remove(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron - int err = lfs_forceconsistency(lfs); + int err = lfs_fs_forceconsistency(lfs); if (err) { return err; } @@ -2622,7 +2534,6 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } - // TODO lfs_dir_empty? if (dir.count > 0 || dir.split) { return LFS_ERR_NOTEMPTY; } @@ -2640,7 +2551,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } if (lfs_tagtype(tag) == LFS_TYPE_DIR) { - int err = lfs_pred(lfs, dir.pair, &cwd); + int err = lfs_fs_pred(lfs, dir.pair, &cwd); if (err) { return err; } @@ -2667,7 +2578,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // deorphan if we haven't yet, needed at most once after poweron - int err = lfs_forceconsistency(lfs); + int err = lfs_fs_forceconsistency(lfs); if (err) { return err; } @@ -2753,7 +2664,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } if (prevtag != LFS_ERR_NOENT && lfs_tagtype(prevtag) == LFS_TYPE_DIR) { - int err = lfs_pred(lfs, prevdir.pair, &newcwd); + int err = lfs_fs_pred(lfs, prevdir.pair, &newcwd); if (err) { return err; } @@ -2776,32 +2687,6 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } return 0; - - -// if (samepair) { -// // update pair if newcwd == oldcwd -// oldcwd = newcwd; -// } -// -// err = fix -// -// // remove old entry -// //printf("RENAME DELETE %d %d %d\n", oldcwd.pair[0], oldcwd.pair[1], oldid); -// err = lfs_dir_delete(lfs, &oldcwd, oldid); -// if (err) { -// return err; -// } -// -// // if we were a directory, find pred, replace tail -// // TODO can this just deorphan? -// if (prevexists && lfs_tagsubtype(prevattr.tag) == LFS_TYPE_DIR) { -// err = lfs_forceconsistency(lfs); -// if (err) { -// return err; -// } -// } -// - return 0; } lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, @@ -2842,80 +2727,6 @@ int lfs_setattr(lfs_t *lfs, const char *path, NULL)); } -lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, - uint8_t type, void *buffer, lfs_size_t size) { - lfs_mdir_t superdir; - int err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - int32_t res = lfs_dir_get(lfs, &superdir, 0x7ffff000, - LFS_MKTAG(0x100 | type, 0, - lfs_min(size, lfs->attr_size)), buffer); - if (res < 0) { - if (res == LFS_ERR_NOENT) { - return LFS_ERR_NOATTR; - } - return res; - } - - return lfs_tagsize(res); -} - -int lfs_fs_setattr(lfs_t *lfs, - uint8_t type, const void *buffer, lfs_size_t size) { - if (size > lfs->attr_size) { - return LFS_ERR_NOSPC; - } - - lfs_mdir_t superdir; - int err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - return lfs_dir_commit(lfs, &superdir, - LFS_MKATTR(0x100 | type, 0, buffer, size, - NULL)); -} - -// -// -// -// const struct lfs_attr *attrs, int count) { -// lfs_mdir_t cwd; -// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); -// if (err) { -// return err; -// } -// -// lfs_mattr_t entry; -// err = lfs_dir_lookup(lfs, &cwd, &entry, &path); -// if (err) { -// return err; -// } -// -// return lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); -//} -// -//int lfs_setattrs(lfs_t *lfs, const char *path, -// const struct lfs_attr *attrs, int count) { -// lfs_mdir_t cwd; -// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); -// if (err) { -// return err; -// } -// -// lfs_mattr_t entry; -// err = lfs_dir_lookup(lfs, &cwd, &entry, &path); -// if (err) { -// return err; -// } -// -// return lfs_dir_setattrs(lfs, &cwd, &entry, attrs, count); -//} - /// Filesystem operations /// static inline void lfs_superblockfromle32(lfs_superblock_t *superblock) { @@ -3184,7 +2995,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } // scan for any global updates - err = lfs_scan(lfs); + err = lfs_fs_scan(lfs); if (err) { return err; } @@ -3197,7 +3008,7 @@ int lfs_unmount(lfs_t *lfs) { } -/// Internal filesystem filesystem operations /// +/// Filesystem filesystem operations /// int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *data, lfs_block_t block), void *data) { if (lfs_pairisnull(lfs->root)) { @@ -3263,80 +3074,9 @@ int lfs_fs_traverse(lfs_t *lfs, return 0; } -/* -int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { - if (lfs_pairisnull(lfs->root)) { - return 0; - } - - // iterate over metadata pairs - lfs_block_t cwd[2] = {0, 1}; - - while (true) { - for (int i = 0; i < 2; i++) { - int err = cb(data, cwd[i]); - if (err) { - return err; - } - } - - lfs_mdir_t dir; - int err = lfs_dir_fetch(lfs, &dir, cwd); - if (err) { - return err; - } - - // iterate over contents - lfs_mattr_t entry; - while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { - err = lfs_dir_get(lfs, &dir, - dir.off, &entry.d, sizeof(entry.d)); - lfs_entry_fromle32(&entry.d); - if (err) { - return err; - } - - dir.off += lfs_entry_size(&entry); - if ((0x70 & entry.d.type) == LFS_TYPE_CTZSTRUCT) { - err = lfs_ctztraverse(lfs, &lfs->rcache, NULL, - entry.d.u.file.head, entry.d.u.file.size, cb, data); - if (err) { - return err; - } - } - } - - cwd[0] = dir.d.tail[0]; - cwd[1] = dir.d.tail[1]; - if (lfs_pairisnull(cwd)) { - break; - } - } - - // iterate over any open files - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { - int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, - f->head, f->size, cb, data); - if (err) { - return err; - } - } - - if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { - int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, - f->block, f->pos, cb, data); - if (err) { - return err; - } - } - } - - return 0; -} -*/ -static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { +static int lfs_fs_pred(lfs_t *lfs, + const lfs_block_t pair[2], lfs_mdir_t *pdir) { if (lfs_pairisnull(lfs->root)) { return LFS_ERR_NOENT; } @@ -3346,7 +3086,6 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { pdir->tail[1] = 1; while (!lfs_pairisnull(pdir->tail)) { if (lfs_paircmp(pdir->tail, pair) == 0) { - //return true; // TODO should we return true only if pred is part of dir? return 0; } @@ -3359,7 +3098,7 @@ static int lfs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { return LFS_ERR_NOENT; } -static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], +static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *parent) { if (lfs_pairisnull(lfs->root)) { return LFS_ERR_NOENT; @@ -3387,13 +3126,11 @@ static int32_t lfs_parent(lfs_t *lfs, const lfs_block_t pair[2], return LFS_ERR_NOENT; } -// TODO rename to lfs_dir_relocate? -static int lfs_relocate(lfs_t *lfs, +static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]) { - // TODO name lfs_dir_relocate? // find parent lfs_mdir_t parent; - int32_t tag = lfs_parent(lfs, oldpair, &parent); + int32_t tag = lfs_fs_parent(lfs, oldpair, &parent); if (tag < 0 && tag != LFS_ERR_NOENT) { return tag; } @@ -3416,11 +3153,11 @@ static int lfs_relocate(lfs_t *lfs, } // clean up bad block, which should now be a desync - return lfs_forceconsistency(lfs); + return lfs_fs_forceconsistency(lfs); } // find pred - int err = lfs_pred(lfs, oldpair, &parent); + int err = lfs_fs_pred(lfs, oldpair, &parent); if (err && err != LFS_ERR_NOENT) { return err; } @@ -3442,7 +3179,7 @@ static int lfs_relocate(lfs_t *lfs, return 0; } -int lfs_scan(lfs_t *lfs) { +static int lfs_fs_scan(lfs_t *lfs) { if (lfs_pairisnull(lfs->root)) { return 0; } @@ -3460,8 +3197,6 @@ int lfs_scan(lfs_t *lfs) { } // update littlefs with globals - // TODO does this only run once? - // TODO Should we inline this into init?? lfs_globalfromle32(&lfs->locals); lfs_globalxor(&lfs->globals, &lfs->locals); lfs_globalzero(&lfs->locals); @@ -3475,8 +3210,11 @@ int lfs_scan(lfs_t *lfs) { return 0; } -int lfs_forceconsistency(lfs_t *lfs) { +static int lfs_fs_forceconsistency(lfs_t *lfs) { if (!lfs_globalisdeorphaned(lfs)) { + LFS_DEBUG("Found orphans %d", + lfs_globalisdeorphaned(lfs)); + // Fix any orphans lfs_mdir_t pdir = {.split = true}; lfs_mdir_t dir = {.tail = {0, 1}}; @@ -3492,14 +3230,14 @@ int lfs_forceconsistency(lfs_t *lfs) { if (!pdir.split) { // check if we have a parent lfs_mdir_t parent; - int32_t tag = lfs_parent(lfs, pdir.tail, &parent); + int32_t tag = lfs_fs_parent(lfs, pdir.tail, &parent); if (tag < 0 && tag != LFS_ERR_NOENT) { return tag; } if (tag == LFS_ERR_NOENT) { // we are an orphan - LFS_DEBUG("Found orphan %d %d", + LFS_DEBUG("Fixing orphan %d %d", pdir.tail[0], pdir.tail[1]); pdir.tail[0] = dir.tail[0]; @@ -3524,7 +3262,7 @@ int lfs_forceconsistency(lfs_t *lfs) { if (!lfs_pairsync(pair, pdir.tail)) { // we have desynced - LFS_DEBUG("Found half-orphan %d %d", pair[0], pair[1]); + LFS_DEBUG("Fixing half-orphan %d %d", pair[0], pair[1]); pdir.tail[0] = pair[0]; pdir.tail[1] = pair[1]; @@ -3549,7 +3287,7 @@ int lfs_forceconsistency(lfs_t *lfs) { if (lfs_globalmoveid(lfs) != 0x3ff) { // Fix bad moves - LFS_DEBUG("Fixing move %d %d %d", // TODO move to just deorphan? + LFS_DEBUG("Fixing move %d %d %d", lfs_globalmovepair(lfs)[0], lfs_globalmovepair(lfs)[1], lfs_globalmoveid(lfs)); @@ -3571,62 +3309,44 @@ int lfs_forceconsistency(lfs_t *lfs) { return 0; } -/// External filesystem filesystem operations /// -//int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { -// lfs_mdir_t dir; -// int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); -// if (err) { -// return err; -// } -// -// lfs_mattr_t entry = {.off = sizeof(dir.d)}; -// err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); -// if (err) { -// return err; -// } -// entry.size = lfs_entry_size(&entry); -// -// if (err != LFS_ERR_NOENT) { -// if (!err) { -// break; -// } -// return err; -// } -// -// lfs_mdir_t cwd; -// int err = lfs_dir_fetch(lfs, &cwd, lfs->root); -// if (err) { -// return err; -// } -// -// lfs_mattr_t entry; -// err = lfs_dir_lookup(lfs, &cwd, &entry, &path); -// if (err) { -// return err; -// } -// -// return lfs_dir_getinfo(lfs, &cwd, &entry, info); -// return lfs_dir_getattrs(lfs, &dir, &entry, attrs, count); -//} -// -//int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { -// lfs_mdir_t dir; -// int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); -// if (err) { -// return err; -// } -// -// lfs_mattr_t entry = {.off = sizeof(dir.d)}; -// err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); -// if (err) { -// return err; -// } -// entry.size = lfs_entry_size(&entry); -// -// return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); -//} - -// TODO need lfs? +lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, + uint8_t type, void *buffer, lfs_size_t size) { + lfs_mdir_t superdir; + int err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + int32_t res = lfs_dir_get(lfs, &superdir, 0x7ffff000, + LFS_MKTAG(0x100 | type, 0, + lfs_min(size, lfs->attr_size)), buffer); + if (res < 0) { + if (res == LFS_ERR_NOENT) { + return LFS_ERR_NOATTR; + } + return res; + } + + return lfs_tagsize(res); +} + +int lfs_fs_setattr(lfs_t *lfs, + uint8_t type, const void *buffer, lfs_size_t size) { + if (size > lfs->attr_size) { + return LFS_ERR_NOSPC; + } + + lfs_mdir_t superdir; + int err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + return lfs_dir_commit(lfs, &superdir, + LFS_MKATTR(0x100 | type, 0, buffer, size, + NULL)); +} + static int lfs_fs_size_count(void *p, lfs_block_t block) { lfs_size_t *size = p; *size += 1; diff --git a/lfs.h b/lfs.h index 092161e8..3fc80dc7 100644 --- a/lfs.h +++ b/lfs.h @@ -104,13 +104,13 @@ enum lfs_type { LFS_TYPE_USER = 0x100, LFS_TYPE_SUPERBLOCK = 0x010, LFS_TYPE_NAME = 0x000, - LFS_TYPE_DELETE = 0x03f, + LFS_TYPE_DELETE = 0x030, LFS_TYPE_STRUCT = 0x040, LFS_TYPE_GLOBALS = 0x080, LFS_TYPE_TAIL = 0x0c0, LFS_TYPE_SOFTTAIL = 0x0c0, LFS_TYPE_HARDTAIL = 0x0c1, - LFS_TYPE_CRC = 0x0ff, // TODO are trailing ones useful? + LFS_TYPE_CRC = 0x0f0, // TODO are trailing ones useful? LFS_TYPE_INLINESTRUCT = 0x040, LFS_TYPE_CTZSTRUCT = 0x041, @@ -119,8 +119,8 @@ enum lfs_type { // internal chip sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x030, - LFS_FROM_ATTRS = 0x020, + LFS_FROM_MOVE = 0x021, + LFS_FROM_ATTRS = 0x022, }; // File open flags From 35f68d28cc00b3fb766e322bdf88e08103a91843 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 1 Aug 2018 10:24:59 -0500 Subject: [PATCH 075/139] Squished in-flight files/dirs into single list This is an effort to try to consolidate the handling of in-flight files and dirs opened by the user (and possibly opened internally). Both files and dirs have metadata state that need to be kept in sync by the commit logic. This metadata state is mostly contained in the lfs_mdir_t type, which is present in both the lfs_file_t and lfs_dir_t. Unfortunately both of these structs have some relatively unrelated metadata that needs to be kept in sync: - Files store an id representing the open file - Dirs store an id during iteration While these take up the same space, they unfortunately need to be managed differently by the commit logic. The best solution I can come up with is to simple store a general purpose list and tag both structures with LFS_TYPE_REG and LFS_TYPE_DIR respectively. This is kinda funky, but wins out over duplicated the commit logic. --- lfs.c | 609 +++++++++++++++++++++++++++++----------------------------- lfs.h | 45 +++-- 2 files changed, 335 insertions(+), 319 deletions(-) diff --git a/lfs.c b/lfs.c index 0193bb80..fc099842 100644 --- a/lfs.c +++ b/lfs.c @@ -753,8 +753,7 @@ static int lfs_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { } // internal dir operations -static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, - bool split, const lfs_block_t tail[2]) { +static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { // allocate pair of dir blocks (backwards, so we write to block 1 first) for (int i = 0; i < 2; i++) { int err = lfs_alloc(lfs, &dir->pair[(i+1)%2]); @@ -775,16 +774,205 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir, dir->off = sizeof(dir->rev); dir->etag = 0; dir->count = 0; - dir->tail[0] = tail[0]; - dir->tail[1] = tail[1]; + dir->tail[0] = 0xffffffff; + dir->tail[1] = 0xffffffff; dir->erased = false; - dir->split = split; + dir->split = false; lfs_globalzero(&dir->locals); // don't write out yet, let caller take care of that return 0; } +static int32_t lfs_dir_find(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2], + uint32_t findmask, uint32_t findtag, + const void *findbuffer) { + dir->pair[0] = pair[0]; + dir->pair[1] = pair[1]; + int32_t foundtag = LFS_ERR_NOENT; + + // find the block with the most recent revision + uint32_t rev[2]; + for (int i = 0; i < 2; i++) { + int err = lfs_bd_read(lfs, dir->pair[i], 0, &rev[i], sizeof(rev[i])); + if (err) { + return err; + } + rev[i] = lfs_fromle32(rev[i]); + } + + if (lfs_scmp(rev[1], rev[0]) > 0) { + lfs_pairswap(dir->pair); + lfs_pairswap(rev); + } + + // load blocks and check crc + for (int i = 0; i < 2; i++) { + lfs_off_t off = sizeof(dir->rev); + uint32_t ptag = 0; + uint32_t crc = 0xffffffff; + + dir->rev = lfs_tole32(rev[0]); + lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); + dir->rev = lfs_fromle32(dir->rev); + dir->off = 0; + + uint32_t tempfoundtag = foundtag; + uint16_t tempcount = 0; + lfs_block_t temptail[2] = {0xffffffff, 0xffffffff}; + bool tempsplit = false; + lfs_global_t templocals; + lfs_globalzero(&templocals); + + while (true) { + // extract next tag + uint32_t tag; + int err = lfs_bd_read(lfs, dir->pair[0], + off, &tag, sizeof(tag)); + if (err) { + return err; + } + + lfs_crc(&crc, &tag, sizeof(tag)); + tag = lfs_fromle32(tag) ^ ptag; + + // next commit not yet programmed + if (lfs_tagtype(ptag) == LFS_TYPE_CRC && !lfs_tagisvalid(tag)) { + dir->erased = true; + break; + } + + // check we're in valid range + if (off + sizeof(tag)+lfs_tagsize(tag) > lfs->cfg->block_size) { + dir->erased = false; + break; + } + + if (lfs_tagtype(tag) == LFS_TYPE_CRC) { + // check the crc attr + uint32_t dcrc; + int err = lfs_bd_read(lfs, dir->pair[0], + off+sizeof(tag), &dcrc, sizeof(dcrc)); + if (err) { + return err; + } + dcrc = lfs_fromle32(dcrc); + + if (crc != dcrc) { + dir->erased = false; + break; + } + + foundtag = tempfoundtag; + dir->off = off + sizeof(tag)+lfs_tagsize(tag); + dir->etag = tag; + dir->count = tempcount; + dir->tail[0] = temptail[0]; + dir->tail[1] = temptail[1]; + dir->split = tempsplit; + dir->locals = templocals; + crc = 0xffffffff; + } else { + err = lfs_bd_crc(lfs, dir->pair[0], + off+sizeof(tag), lfs_tagsize(tag), &crc); + if (err) { + return err; + } + + if (lfs_tagid(tag) < 0x3ff && lfs_tagid(tag) >= tempcount) { + tempcount = lfs_tagid(tag)+1; + } + + if (lfs_tagsubtype(tag) == LFS_TYPE_TAIL) { + tempsplit = (lfs_tagtype(tag) & 1); + err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), + temptail, sizeof(temptail)); + if (err) { + return err; + } + lfs_pairfromle32(temptail); + } else if (lfs_tagsubtype(tag) == LFS_TYPE_GLOBALS) { + err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), + &templocals, sizeof(templocals)); + if (err) { + return err; + } + } else if (lfs_tagsubtype(tag) == LFS_TYPE_DELETE) { + LFS_ASSERT(tempcount > 0); + tempcount -= 1; + + if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { + tempfoundtag = LFS_ERR_NOENT; + } else if (lfs_tagisvalid(tempfoundtag) && + lfs_tagid(tag) < lfs_tagid(tempfoundtag)) { + tempfoundtag -= LFS_MKTAG(0, 1, 0); + } + } else if ((tag & findmask) == (findtag & findmask)) { + int res = lfs_bd_cmp(lfs, dir->pair[0], off+sizeof(tag), + findbuffer, lfs_tagsize(tag)); + if (res < 0) { + return res; + } + + if (res) { + // found a match + tempfoundtag = tag; + } + } + } + + ptag = tag; + off += sizeof(tag)+lfs_tagsize(tag); + } + + // consider what we have good enough + if (dir->off > 0) { + // synthetic move + if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0) { + if (lfs_globalmoveid(lfs) == lfs_tagid(foundtag)) { + foundtag = LFS_ERR_NOENT; + } else if (lfs_tagisvalid(foundtag) && + lfs_globalmoveid(lfs) < lfs_tagid(foundtag)) { + foundtag -= LFS_MKTAG(0, 1, 0); + } + } + + return foundtag; + } + + // failed, try the other crc? + lfs_pairswap(dir->pair); + lfs_pairswap(rev); + } + + LFS_ERROR("Corrupted dir pair at %d %d", dir->pair[0], dir->pair[1]); + return LFS_ERR_CORRUPT; +} + +static int lfs_dir_fetch(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2]) { + int32_t res = lfs_dir_find(lfs, dir, pair, 0xffffffff, 0xffffffff, NULL); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + + return 0; +} + +static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, + uint32_t getmask, uint32_t gettag, void *buffer) { + int32_t getdiff = 0; + if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0 && + lfs_tagid(gettag) <= lfs_globalmoveid(lfs)) { + // synthetic moves + getdiff = LFS_MKTAG(0, 1, 0); + } + + return lfs_commitget(lfs, dir->pair[0], dir->off, dir->etag, + getmask, gettag, getdiff, buffer, false); +} + static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mattr_t *attrs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { @@ -913,11 +1101,15 @@ static int lfs_dir_compact(lfs_t *lfs, } lfs_mdir_t tail; - int err = lfs_dir_alloc(lfs, &tail, dir->split, dir->tail); + int err = lfs_dir_alloc(lfs, &tail); if (err) { return err; } + tail.split = dir->split; + tail.tail[0] = dir->tail[0]; + tail.tail[1] = dir->tail[1]; + err = lfs_dir_compact(lfs, &tail, attrs, dir, ack+1, end); if (err) { return err; @@ -966,18 +1158,6 @@ static int lfs_dir_compact(lfs_t *lfs, } } - // update any dirs/files that are affected - for (int i = 0; i < 2; i++) { - for (lfs_file_t *f = ((lfs_file_t**)&lfs->files)[i]; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0 && - f->id >= begin && f->id < end) { - f->pair[0] = dir->pair[0]; - f->pair[1] = dir->pair[1]; - f->id -= begin; - } - } - } - return 0; } @@ -1110,218 +1290,33 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_globalxor(&lfs->globals, &canceldiff); // update any directories that are affected - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + for (lfs_mlist_t *d = lfs->mlist; d; d = d->next) { if (lfs_paircmp(d->m.pair, dir->pair) == 0) { d->m = *dir; - if (d->id > lfs_tagid(deletetag)) { - d->pos -= 1; - } - } - } - - for (int i = 0; i < 2; i++) { - for (lfs_file_t *f = ((lfs_file_t**)&lfs->files)[i]; f; f = f->next) { - if (f->id == lfs_tagid(deletetag)) { - f->pair[0] = 0xffffffff; - f->pair[1] = 0xffffffff; - } else if (f->id > lfs_tagid(deletetag)) { - f->id -= 1; - } - } - } - - return 0; -} - -static int32_t lfs_dir_find(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2], - uint32_t findmask, uint32_t findtag, - const void *findbuffer) { - dir->pair[0] = pair[0]; - dir->pair[1] = pair[1]; - int32_t foundtag = LFS_ERR_NOENT; - - // find the block with the most recent revision - uint32_t rev[2]; - for (int i = 0; i < 2; i++) { - int err = lfs_bd_read(lfs, dir->pair[i], 0, &rev[i], sizeof(rev[i])); - if (err) { - return err; - } - rev[i] = lfs_fromle32(rev[i]); - } - - if (lfs_scmp(rev[1], rev[0]) > 0) { - lfs_pairswap(dir->pair); - lfs_pairswap(rev); - } - - // load blocks and check crc - for (int i = 0; i < 2; i++) { - lfs_off_t off = sizeof(dir->rev); - uint32_t ptag = 0; - uint32_t crc = 0xffffffff; - - dir->rev = lfs_tole32(rev[0]); - lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); - dir->rev = lfs_fromle32(dir->rev); - dir->off = 0; - - uint32_t tempfoundtag = foundtag; - uint16_t tempcount = 0; - lfs_block_t temptail[2] = {0xffffffff, 0xffffffff}; - bool tempsplit = false; - lfs_global_t templocals; - lfs_globalzero(&templocals); - - while (true) { - // extract next tag - uint32_t tag; - int err = lfs_bd_read(lfs, dir->pair[0], - off, &tag, sizeof(tag)); - if (err) { - return err; - } - - lfs_crc(&crc, &tag, sizeof(tag)); - tag = lfs_fromle32(tag) ^ ptag; - - // next commit not yet programmed - if (lfs_tagtype(ptag) == LFS_TYPE_CRC && !lfs_tagisvalid(tag)) { - dir->erased = true; - break; - } - - // check we're in valid range - if (off + sizeof(tag)+lfs_tagsize(tag) > lfs->cfg->block_size) { - dir->erased = false; - break; + if (d->id == lfs_tagid(deletetag)) { + d->m.pair[0] = 0xffffffff; + d->m.pair[1] = 0xffffffff; + } else if (d->id > lfs_tagid(deletetag)) { + d->id -= 1; + if (d->type == LFS_TYPE_DIR) { + ((lfs_dir_t*)d)->pos -= 1; + } } - if (lfs_tagtype(tag) == LFS_TYPE_CRC) { - // check the crc attr - uint32_t dcrc; - int err = lfs_bd_read(lfs, dir->pair[0], - off+sizeof(tag), &dcrc, sizeof(dcrc)); + while (d->id >= d->m.count && d->m.split) { + // we split and id is on tail now + d->id -= d->m.count; + int err = lfs_dir_fetch(lfs, &d->m, d->m.tail); if (err) { return err; } - dcrc = lfs_fromle32(dcrc); - - if (crc != dcrc) { - dir->erased = false; - break; - } - - foundtag = tempfoundtag; - dir->off = off + sizeof(tag)+lfs_tagsize(tag); - dir->etag = tag; - dir->count = tempcount; - dir->tail[0] = temptail[0]; - dir->tail[1] = temptail[1]; - dir->split = tempsplit; - dir->locals = templocals; - crc = 0xffffffff; - } else { - err = lfs_bd_crc(lfs, dir->pair[0], - off+sizeof(tag), lfs_tagsize(tag), &crc); - if (err) { - return err; - } - - if (lfs_tagid(tag) < 0x3ff && lfs_tagid(tag) >= tempcount) { - tempcount = lfs_tagid(tag)+1; - } - - if (lfs_tagsubtype(tag) == LFS_TYPE_TAIL) { - tempsplit = (lfs_tagtype(tag) & 1); - err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), - temptail, sizeof(temptail)); - if (err) { - return err; - } - lfs_pairfromle32(temptail); - } else if (lfs_tagsubtype(tag) == LFS_TYPE_GLOBALS) { - err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), - &templocals, sizeof(templocals)); - if (err) { - return err; - } - } else if (lfs_tagsubtype(tag) == LFS_TYPE_DELETE) { - LFS_ASSERT(tempcount > 0); - tempcount -= 1; - - if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { - tempfoundtag = LFS_ERR_NOENT; - } else if (lfs_tagisvalid(tempfoundtag) && - lfs_tagid(tag) < lfs_tagid(tempfoundtag)) { - tempfoundtag -= LFS_MKTAG(0, 1, 0); - } - } else if ((tag & findmask) == (findtag & findmask)) { - int res = lfs_bd_cmp(lfs, dir->pair[0], off+sizeof(tag), - findbuffer, lfs_tagsize(tag)); - if (res < 0) { - return res; - } - - if (res) { - // found a match - tempfoundtag = tag; - } - } } - - ptag = tag; - off += sizeof(tag)+lfs_tagsize(tag); } - - // consider what we have good enough - if (dir->off > 0) { - // synthetic move - if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0) { - if (lfs_globalmoveid(lfs) == lfs_tagid(foundtag)) { - foundtag = LFS_ERR_NOENT; - } else if (lfs_tagisvalid(foundtag) && - lfs_globalmoveid(lfs) < lfs_tagid(foundtag)) { - foundtag -= LFS_MKTAG(0, 1, 0); - } - } - - return foundtag; - } - - // failed, try the other crc? - lfs_pairswap(dir->pair); - lfs_pairswap(rev); - } - - LFS_ERROR("Corrupted dir pair at %d %d", dir->pair[0], dir->pair[1]); - return LFS_ERR_CORRUPT; -} - -static int lfs_dir_fetch(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2]) { - int32_t res = lfs_dir_find(lfs, dir, pair, 0xffffffff, 0xffffffff, NULL); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; } return 0; } -static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t getmask, uint32_t gettag, void *buffer) { - int32_t getdiff = 0; - if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0 && - lfs_tagid(gettag) <= lfs_globalmoveid(lfs)) { - // synthetic moves - getdiff = LFS_MKTAG(0, 1, 0); - } - - return lfs_commitget(lfs, dir->pair[0], dir->off, dir->etag, - getmask, gettag, getdiff, buffer, false); -} - static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { // we reduce path to a single name if we can find it const char *name = *path; @@ -1478,11 +1473,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_alloc_ack(lfs); lfs_mdir_t dir; - err = lfs_dir_alloc(lfs, &dir, false, cwd.tail); + err = lfs_dir_alloc(lfs, &dir); if (err) { return err; } + dir.tail[0] = cwd.tail[0]; + dir.tail[1] = cwd.tail[1]; err = lfs_dir_commit(lfs, &dir, NULL); if (err) { return err; @@ -1543,18 +1540,19 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { dir->id = 0; dir->pos = 0; - // add to list of directories - dir->next = lfs->dirs; - lfs->dirs = dir; + // add to list of mdirs + dir->type = LFS_TYPE_DIR; + dir->next = (lfs_dir_t*)lfs->mlist; + lfs->mlist = (lfs_mlist_t*)dir; return 0; } int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir) { - // remove from list of directories - for (lfs_dir_t **p = &lfs->dirs; *p; p = &(*p)->next) { - if (*p == dir) { - *p = dir->next; + // remove from list of mdirs + for (lfs_mlist_t **p = &lfs->mlist; *p; p = &(*p)->next) { + if (*p == (lfs_mlist_t*)dir) { + *p = (*p)->next; break; } } @@ -1857,84 +1855,87 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } } + // setup simple file details + int err = 0; + file->cfg = cfg; + file->flags = flags; + file->pos = 0; + file->cache.buffer = NULL; + // allocate entry for file if it doesn't exist - lfs_mdir_t cwd; - int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); + int32_t tag = lfs_dir_lookup(lfs, &file->m, &path); if (tag < 0 && !(tag == LFS_ERR_NOENT && path)) { - return tag; + err = tag; + goto cleanup; } + // get id, add to list of mdirs to catch update changes + file->id = lfs_tagid(tag); + file->type = LFS_TYPE_REG; + file->next = (lfs_file_t*)lfs->mlist; + lfs->mlist = (lfs_mlist_t*)file; + if (tag == LFS_ERR_NOENT) { if (!(flags & LFS_O_CREAT)) { - return LFS_ERR_NOENT; + err = LFS_ERR_NOENT; + goto cleanup; } // check that name fits lfs_size_t nlen = strlen(path); if (nlen > lfs->name_size) { - return LFS_ERR_NAMETOOLONG; + err = LFS_ERR_NAMETOOLONG; + goto cleanup; } // get next slot and create entry to remember name - // TODO do we need to make file registered to list to catch updates from this commit? ie if id/cwd change - uint16_t id = cwd.count; - int err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_REG, id, path, nlen, - LFS_MKATTR(LFS_TYPE_INLINESTRUCT, id, NULL, 0, + file->id = file->m.count; + int err = lfs_dir_commit(lfs, &file->m, + LFS_MKATTR(LFS_TYPE_REG, file->id, path, nlen, + LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, NULL, 0, NULL))); if (err) { - return err; - } - - // TODO should we be catching this here? - if (id >= cwd.count) { - // catch updates from a compact in the above commit - id -= cwd.count; - cwd.pair[0] = cwd.tail[0]; - cwd.pair[1] = cwd.tail[1]; + err = LFS_ERR_NAMETOOLONG; + goto cleanup; } - tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, id, 0); + tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, 0); } else if (flags & LFS_O_EXCL) { - return LFS_ERR_EXIST; + err = LFS_ERR_EXIST; + goto cleanup; } else if (lfs_tagtype(tag) != LFS_TYPE_REG) { - return LFS_ERR_ISDIR; + err = LFS_ERR_ISDIR; + goto cleanup; } else if (flags & LFS_O_TRUNC) { // truncate if requested - tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, lfs_tagid(tag), 0); - flags |= LFS_F_DIRTY; + tag = LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0); + file->flags |= LFS_F_DIRTY; } else { // try to load what's on disk, if it's inlined we'll fix it later - tag = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), &file->ctz); + tag = lfs_dir_get(lfs, &file->m, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, file->id, 8), &file->ctz); if (tag < 0) { - return tag; + err = tag; + goto cleanup; } lfs_ctzfromle32(&file->ctz); } - // setup file struct - file->cfg = cfg; - file->pair[0] = cwd.pair[0]; - file->pair[1] = cwd.pair[1]; - file->id = lfs_tagid(tag); - file->flags = flags; - file->pos = 0; - // fetch attrs for (const struct lfs_attr *a = file->cfg->attrs; a; a = a->next) { if ((file->flags & 3) != LFS_O_WRONLY) { - // TODO what if cwd is invalid from above compact? - int32_t res = lfs_dir_get(lfs, &cwd, 0x7ffff000, + int32_t res = lfs_dir_get(lfs, &file->m, 0x7ffff000, LFS_MKTAG(0x100 | a->type, file->id, a->size), a->buffer); if (res < 0 && res != LFS_ERR_NOENT) { - return res; + err = res; + goto cleanup; } } if ((file->flags & 3) != LFS_O_RDONLY) { if (a->size > lfs->attr_size) { - return LFS_ERR_NOSPC; + err = LFS_ERR_NOSPC; + goto cleanup; } file->flags |= LFS_F_DIRTY; @@ -1946,14 +1947,17 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, if (file->cfg->buffer) { file->cache.buffer = file->cfg->buffer; } else if ((file->flags & 3) == LFS_O_RDONLY) { + // TODO cache_size file->cache.buffer = lfs_malloc(lfs->cfg->read_size); if (!file->cache.buffer) { - return LFS_ERR_NOMEM; + err = LFS_ERR_NOMEM; + goto cleanup; } } else { file->cache.buffer = lfs_malloc(lfs->cfg->prog_size); if (!file->cache.buffer) { - return LFS_ERR_NOMEM; + err = LFS_ERR_NOMEM; + goto cleanup; } } @@ -1967,21 +1971,23 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // don't always read (may be new/trunc file) if (file->ctz.size > 0) { - int32_t res = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), file->ctz.size), + int32_t res = lfs_dir_get(lfs, &file->m, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->ctz.size), file->cache.buffer); if (res < 0) { - lfs_free(file->cache.buffer); - return res; + err = res; + goto cleanup; } } } - // add to list of files - file->next = lfs->files; - lfs->files = file; - return 0; + +cleanup: + // clean up lingering resources + file->flags |= LFS_F_ERRED; + lfs_file_close(lfs, file); + return err; } int lfs_file_open(lfs_t *lfs, lfs_file_t *file, @@ -1993,10 +1999,10 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { int err = lfs_file_sync(lfs, file); - // remove from list of files - for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { - if (*p == file) { - *p = file->next; + // remove from list of mdirs + for (lfs_mlist_t **p = &lfs->mlist; *p; p = &(*p)->next) { + if (*p == (lfs_mlist_t*)file) { + *p = (*p)->next; break; } } @@ -2143,16 +2149,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { if ((file->flags & LFS_F_DIRTY) && !(file->flags & LFS_F_ERRED) && - !lfs_pairisnull(file->pair)) { + !lfs_pairisnull(file->m.pair)) { // update dir entry - // TODO keep list of dirs including these guys for no - // need of another reload? - lfs_mdir_t cwd; - err = lfs_dir_fetch(lfs, &cwd, file->pair); - if (err) { - return err; - } - uint16_t type; const void *buffer; lfs_size_t size; @@ -2170,7 +2168,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // commit file data and attributes lfs_ctztole32(&file->ctz); - int err = lfs_dir_commit(lfs, &cwd, + int err = lfs_dir_commit(lfs, &file->m, LFS_MKATTR(type, file->id, buffer, size, LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, NULL))); @@ -2815,8 +2813,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { // setup default state lfs->root[0] = 0xffffffff; lfs->root[1] = 0xffffffff; - lfs->files = NULL; - lfs->dirs = NULL; + lfs->mlist = NULL; lfs_globalones(&lfs->globals); lfs_globalzero(&lfs->locals); @@ -2855,16 +2852,14 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // create superblock dir lfs_mdir_t dir; - err = lfs_dir_alloc(lfs, &dir, false, - (const lfs_block_t[2]){0xffffffff, 0xffffffff}); + err = lfs_dir_alloc(lfs, &dir); if (err) { return err; } // write root directory lfs_mdir_t root; - err = lfs_dir_alloc(lfs, &root, false, - (const lfs_block_t[2]){0xffffffff, 0xffffffff}); + err = lfs_dir_alloc(lfs, &root); if (err) { return err; } @@ -3054,7 +3049,11 @@ int lfs_fs_traverse(lfs_t *lfs, } // iterate over any open files - for (lfs_file_t *f = lfs->files; f; f = f->next) { + for (lfs_file_t *f = (lfs_file_t*)lfs->mlist; f; f = f->next) { + if (f->type != LFS_TYPE_REG) { + continue; + } + if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, f->ctz.head, f->ctz.size, cb, data); @@ -3128,6 +3127,21 @@ static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]) { + // update internal root + if (lfs_paircmp(oldpair, lfs->root) == 0) { + LFS_DEBUG("Relocating root %d %d", newpair[0], newpair[1]); + lfs->root[0] = newpair[0]; + lfs->root[1] = newpair[1]; + } + + // update internally tracked dirs + for (lfs_mlist_t *d = lfs->mlist; d; d = d->next) { + if (lfs_paircmp(oldpair, d->m.pair) == 0) { + d->m.pair[0] = newpair[0]; + d->m.pair[1] = newpair[1]; + } + } + // find parent lfs_mdir_t parent; int32_t tag = lfs_fs_parent(lfs, oldpair, &parent); @@ -3145,13 +3159,6 @@ static int lfs_fs_relocate(lfs_t *lfs, return err; } - // update internal root - if (lfs_paircmp(oldpair, lfs->root) == 0) { - LFS_DEBUG("Relocating root %d %d", newpair[0], newpair[1]); - lfs->root[0] = newpair[0]; - lfs->root[1] = newpair[1]; - } - // clean up bad block, which should now be a desync return lfs_fs_forceconsistency(lfs); } diff --git a/lfs.h b/lfs.h index 3fc80dc7..720a36fb 100644 --- a/lfs.h +++ b/lfs.h @@ -280,6 +280,12 @@ typedef struct lfs_mattr { const struct lfs_mattr *next; } lfs_mattr_t; +typedef struct lfs_cache { + lfs_block_t block; + lfs_off_t off; + uint8_t *buffer; +} lfs_cache_t; + typedef union lfs_global { uint16_t u16[5]; } lfs_global_t; @@ -288,24 +294,37 @@ typedef struct lfs_mdir { lfs_block_t pair[2]; lfs_block_t tail[2]; uint32_t rev; - lfs_off_t off; uint32_t etag; + lfs_off_t off; uint16_t count; bool erased; bool split; lfs_global_t locals; } lfs_mdir_t; -typedef struct lfs_cache { - lfs_block_t block; - lfs_off_t off; - uint8_t *buffer; -} lfs_cache_t; +typedef struct lfs_mlist { + struct lfs_mlist *next; + uint16_t id; + uint8_t type; + lfs_mdir_t m; +} lfs_mlist_t; + +typedef struct lfs_dir { + struct lfs_dir *next; + uint16_t id; + uint8_t type; + lfs_mdir_t m; + + lfs_off_t pos; + lfs_block_t head[2]; +} lfs_dir_t; typedef struct lfs_file { struct lfs_file *next; uint16_t id; - lfs_block_t pair[2]; + uint8_t type; + lfs_mdir_t m; + struct lfs_ctz { lfs_block_t head; lfs_size_t size; @@ -320,15 +339,6 @@ typedef struct lfs_file { const struct lfs_file_config *cfg; } lfs_file_t; -typedef struct lfs_dir { - struct lfs_dir *next; - uint16_t id; - struct lfs_mdir m; - - lfs_block_t head[2]; - lfs_off_t pos; -} lfs_dir_t; - typedef struct lfs_superblock { char magic[8]; uint32_t version; @@ -355,8 +365,7 @@ typedef struct lfs { lfs_cache_t pcache; lfs_block_t root[2]; - lfs_file_t *files; - lfs_dir_t *dirs; + lfs_mlist_t *mlist; lfs_global_t globals; lfs_global_t locals; From 97f35c3e05f3465c76216a918cd4792d875d6827 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 3 Aug 2018 19:01:27 -0500 Subject: [PATCH 076/139] Simplified the internal xored-globals implementation There wasn't much use (and inconsistent compiler support) for storing small values next to the unaligned lfs_global_t struct. So instead, I've rounded the struct up to the nearest word to try to take advantage of the alignment in xor and memset operations. I've also moved the global fetching into lfs_mount, since that was the only use of the operation. This allows for some variable reuse in the mount function. --- lfs.c | 165 ++++++++++++++++++++++------------------------------------ lfs.h | 9 +++- 2 files changed, 69 insertions(+), 105 deletions(-) diff --git a/lfs.c b/lfs.c index fc099842..9b90e99f 100644 --- a/lfs.c +++ b/lfs.c @@ -265,7 +265,6 @@ static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent); static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]); -static int lfs_fs_scan(lfs_t *lfs); static int lfs_fs_forceconsistency(lfs_t *lfs); @@ -410,14 +409,14 @@ static inline lfs_size_t lfs_tagsize(uint32_t tag) { // operations on set of globals static inline void lfs_globalxor(lfs_global_t *a, const lfs_global_t *b) { - for (int i = 0; i < sizeof(lfs_global_t)/2; i++) { - a->u16[i] ^= b->u16[i]; + for (int i = 0; i < sizeof(lfs_global_t)/4; i++) { + a->u32[i] ^= b->u32[i]; } } static inline bool lfs_globaliszero(const lfs_global_t *a) { - for (int i = 0; i < sizeof(lfs_global_t)/2; i++) { - if (a->u16[i] != 0) { + for (int i = 0; i < sizeof(lfs_global_t)/4; i++) { + if (a->u32[i] != 0) { return false; } } @@ -425,53 +424,26 @@ static inline bool lfs_globaliszero(const lfs_global_t *a) { } static inline void lfs_globalzero(lfs_global_t *a) { - memset(a->u16, 0x00, sizeof(lfs_global_t)); -} - -static inline void lfs_globalones(lfs_global_t *a) { - memset(a->u16, 0xff, sizeof(lfs_global_t)); -} - -static inline void lfs_globalxormove(lfs_global_t *a, - const lfs_block_t pair[2], uint16_t id) { - a->u16[0] ^= id; - for (int i = 0; i < sizeof(lfs_block_t[2])/2; i++) { - a->u16[1+i] ^= ((uint16_t*)pair)[i]; - } -} - -static inline void lfs_globalxordeorphaned(lfs_global_t *a, bool deorphaned) { - a->u16[0] ^= deorphaned << 15; + lfs_globalxor(a, a); } static inline void lfs_globalfromle32(lfs_global_t *a) { - a->u16[0] = lfs_fromle16(a->u16[0]); - lfs_pairfromle32((lfs_block_t*)&a->u16[1]); + lfs_pairfromle32(a->s.movepair); + a->s.moveid = lfs_fromle16(a->s.moveid); } static inline void lfs_globaltole32(lfs_global_t *a) { - a->u16[0] = lfs_tole16(a->u16[0]); - lfs_pairtole32((lfs_block_t*)&a->u16[1]); -} - -static inline const lfs_block_t *lfs_globalmovepair(const lfs_t *lfs) { - return (const lfs_block_t*)&lfs->globals.u16[1]; -} - -static inline uint16_t lfs_globalmoveid(const lfs_t *lfs) { - return 0x3ff & lfs->globals.u16[0]; -} - -static inline bool lfs_globalisdeorphaned(const lfs_t *lfs) { - return 0x8000 & lfs->globals.u16[0]; + lfs_pairtole32(a->s.movepair); + a->s.moveid = lfs_tole16(a->s.moveid); } static inline void lfs_globalmove(lfs_t *lfs, const lfs_block_t pair[2], uint16_t id) { lfs_global_t diff; lfs_globalzero(&diff); - lfs_globalxormove(&diff, lfs_globalmovepair(lfs), lfs_globalmoveid(lfs)); - lfs_globalxormove(&diff, pair, id); + diff.s.movepair[0] ^= lfs->globals.s.movepair[0] ^ pair[0]; + diff.s.movepair[1] ^= lfs->globals.s.movepair[1] ^ pair[1]; + diff.s.moveid ^= lfs->globals.s.moveid ^ id; lfs_globalfromle32(&lfs->locals); lfs_globalxor(&lfs->locals, &diff); lfs_globaltole32(&lfs->locals); @@ -479,11 +451,8 @@ static inline void lfs_globalmove(lfs_t *lfs, } static inline void lfs_globaldeorphaned(lfs_t *lfs, bool deorphaned) { - deorphaned ^= lfs_globalisdeorphaned(lfs); - lfs_globalfromle32(&lfs->locals); - lfs_globalxordeorphaned(&lfs->locals, deorphaned); - lfs_globaltole32(&lfs->locals); - lfs_globalxordeorphaned(&lfs->globals, deorphaned); + lfs->locals.s.deorphaned ^= lfs->globals.s.deorphaned ^ deorphaned; + lfs->globals.s.deorphaned ^= lfs->globals.s.deorphaned ^ deorphaned; } @@ -696,7 +665,8 @@ static int lfs_commitglobals(lfs_t *lfs, struct lfs_commit *commit, lfs_globalxor(locals, &lfs->locals); int err = lfs_commitattr(lfs, commit, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0x3ff, sizeof(lfs_global_t)), locals); + LFS_MKTAG(LFS_TYPE_GLOBALS + locals->s.deorphaned, + 0x3ff, sizeof(lfs_global_t)), locals); lfs_globalxor(locals, &lfs->locals); return err; } @@ -893,6 +863,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, } lfs_pairfromle32(temptail); } else if (lfs_tagsubtype(tag) == LFS_TYPE_GLOBALS) { + templocals.s.deorphaned = (lfs_tagtype(tag) & 1); err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), &templocals, sizeof(templocals)); if (err) { @@ -929,11 +900,11 @@ static int32_t lfs_dir_find(lfs_t *lfs, // consider what we have good enough if (dir->off > 0) { // synthetic move - if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0) { - if (lfs_globalmoveid(lfs) == lfs_tagid(foundtag)) { + if (lfs_paircmp(dir->pair, lfs->globals.s.movepair) == 0) { + if (lfs->globals.s.moveid == lfs_tagid(foundtag)) { foundtag = LFS_ERR_NOENT; } else if (lfs_tagisvalid(foundtag) && - lfs_globalmoveid(lfs) < lfs_tagid(foundtag)) { + lfs->globals.s.moveid < lfs_tagid(foundtag)) { foundtag -= LFS_MKTAG(0, 1, 0); } } @@ -963,8 +934,8 @@ static int lfs_dir_fetch(lfs_t *lfs, static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, uint32_t gettag, void *buffer) { int32_t getdiff = 0; - if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0 && - lfs_tagid(gettag) <= lfs_globalmoveid(lfs)) { + if (lfs_paircmp(dir->pair, lfs->globals.s.movepair) == 0 && + lfs_tagid(gettag) <= lfs->globals.s.moveid) { // synthetic moves getdiff = LFS_MKTAG(0, 1, 0); } @@ -1166,18 +1137,17 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t cancelattr; lfs_global_t canceldiff; lfs_globalzero(&canceldiff); - if (lfs_paircmp(dir->pair, lfs_globalmovepair(lfs)) == 0) { + if (lfs_paircmp(dir->pair, lfs->globals.s.movepair) == 0) { // Wait, we have the move? Just cancel this out here // We need to, or else the move can become outdated - lfs_globalxormove(&canceldiff, - lfs_globalmovepair(lfs), lfs_globalmoveid(lfs)); - lfs_globalxormove(&canceldiff, - (lfs_block_t[2]){0xffffffff, 0xffffffff}, 0x3ff); + canceldiff.s.movepair[0] ^= lfs->globals.s.movepair[0] ^ 0xffffffff; + canceldiff.s.movepair[1] ^= lfs->globals.s.movepair[1] ^ 0xffffffff; + canceldiff.s.moveid ^= lfs->globals.s.moveid ^ 0x3ff; lfs_globalfromle32(&lfs->locals); lfs_globalxor(&lfs->locals, &canceldiff); lfs_globaltole32(&lfs->locals); - cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs_globalmoveid(lfs), 0); + cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.s.moveid, 0); cancelattr.next = attrs; attrs = &cancelattr; } @@ -2814,7 +2784,10 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[0] = 0xffffffff; lfs->root[1] = 0xffffffff; lfs->mlist = NULL; - lfs_globalones(&lfs->globals); + lfs->globals.s.movepair[0] = 0xffffffff; + lfs->globals.s.movepair[1] = 0xffffffff; + lfs->globals.s.moveid = 0x3ff; + lfs->globals.s.deorphaned = true; lfs_globalzero(&lfs->locals); return 0; @@ -2920,8 +2893,8 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_alloc_ack(lfs); // load superblock - lfs_mdir_t dir; - err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + lfs_mdir_t superdir; + err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); if (err) { if (err == LFS_ERR_CORRUPT) { LFS_ERROR("Invalid superblock at %d %d", 0, 1); @@ -2930,7 +2903,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_superblock_t superblock; - int32_t res = lfs_dir_get(lfs, &dir, 0x7ffff000, + int32_t res = lfs_dir_get(lfs, &superdir, 0x7ffff000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), &superblock); if (res < 0) { @@ -2951,7 +2924,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return LFS_ERR_INVAL; } - res = lfs_dir_get(lfs, &dir, 0x7ffff000, + res = lfs_dir_get(lfs, &superdir, 0x7ffff000, LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(lfs->root)), &lfs->root); if (res < 0) { @@ -2990,9 +2963,26 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } // scan for any global updates - err = lfs_fs_scan(lfs); - if (err) { - return err; + lfs_mdir_t dir = {.tail = {0, 1}}; + while (!lfs_pairisnull(dir.tail)) { + int err = lfs_dir_fetch(lfs, &dir, dir.tail); + if (err) { + return err; + } + + // xor together indirect deletes + lfs_globalxor(&lfs->locals, &dir.locals); + } + + // update littlefs with globals + lfs_globalfromle32(&lfs->locals); + lfs_globalxor(&lfs->globals, &lfs->locals); + lfs_globalzero(&lfs->locals); + if (!lfs_pairisnull(lfs->globals.s.movepair)) { + LFS_DEBUG("Found move %d %d %d", + lfs->globals.s.movepair[0], + lfs->globals.s.movepair[1], + lfs->globals.s.moveid); } return 0; @@ -3186,41 +3176,10 @@ static int lfs_fs_relocate(lfs_t *lfs, return 0; } -static int lfs_fs_scan(lfs_t *lfs) { - if (lfs_pairisnull(lfs->root)) { - return 0; - } - - // iterate over all directory directory entries - lfs_mdir_t dir = {.tail = {0, 1}}; - while (!lfs_pairisnull(dir.tail)) { - int err = lfs_dir_fetch(lfs, &dir, dir.tail); - if (err) { - return err; - } - - // xor together indirect deletes - lfs_globalxor(&lfs->locals, &dir.locals); - } - - // update littlefs with globals - lfs_globalfromle32(&lfs->locals); - lfs_globalxor(&lfs->globals, &lfs->locals); - lfs_globalzero(&lfs->locals); - if (!lfs_pairisnull(lfs_globalmovepair(lfs))) { - LFS_DEBUG("Found move %d %d %d", - lfs_globalmovepair(lfs)[0], - lfs_globalmovepair(lfs)[1], - lfs_globalmoveid(lfs)); - } - - return 0; -} - static int lfs_fs_forceconsistency(lfs_t *lfs) { - if (!lfs_globalisdeorphaned(lfs)) { + if (!lfs->globals.s.deorphaned) { LFS_DEBUG("Found orphans %d", - lfs_globalisdeorphaned(lfs)); + lfs->globals.s.deorphaned); // Fix any orphans lfs_mdir_t pdir = {.split = true}; @@ -3292,16 +3251,16 @@ static int lfs_fs_forceconsistency(lfs_t *lfs) { lfs_globaldeorphaned(lfs, false); } - if (lfs_globalmoveid(lfs) != 0x3ff) { + if (lfs->globals.s.moveid != 0x3ff) { // Fix bad moves LFS_DEBUG("Fixing move %d %d %d", - lfs_globalmovepair(lfs)[0], - lfs_globalmovepair(lfs)[1], - lfs_globalmoveid(lfs)); + lfs->globals.s.movepair[0], + lfs->globals.s.movepair[1], + lfs->globals.s.moveid); // fetch and delete the moved entry lfs_mdir_t movedir; - int err = lfs_dir_fetch(lfs, &movedir, lfs_globalmovepair(lfs)); + int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.s.movepair); if (err) { return err; } diff --git a/lfs.h b/lfs.h index 720a36fb..e648c006 100644 --- a/lfs.h +++ b/lfs.h @@ -287,18 +287,23 @@ typedef struct lfs_cache { } lfs_cache_t; typedef union lfs_global { - uint16_t u16[5]; + uint32_t u32[3]; + struct { + lfs_block_t movepair[2]; + uint16_t moveid; + bool deorphaned; + } s; } lfs_global_t; typedef struct lfs_mdir { lfs_block_t pair[2]; - lfs_block_t tail[2]; uint32_t rev; uint32_t etag; lfs_off_t off; uint16_t count; bool erased; bool split; + lfs_block_t tail[2]; lfs_global_t locals; } lfs_mdir_t; From 3cfa08602a7087f84672f7cf53a2e59b22076b18 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 4 Aug 2018 14:48:27 -0500 Subject: [PATCH 077/139] Introduced cache_size as alternative to hardware read/write sizes The introduction of an explicit cache_size configuration allows customization of the cache buffers independently from the hardware read/write sizes. This has been one of littlefs's main handicaps. Without a distinction between cache units and hardware limitations, littlefs isn't able to read or program _less_ than the cache size. This leads to the counter-intuitive case where larger cache sizes can actually be harmful, since larger read/prog sizes require sending more data over the bus if we're only accessing a small set of data (for example the CTZ skip-list traversal). This is compounded with metadata logging, since a large program size limits the number of commits we can write out in a single metadata block. It really doesn't make sense to link program size + cache size here. With a separate cache_size configuration, we can be much smarter about what we actually read/write from disk. This also simplifies cache handling a bit. Before there were two possible cache sizes, but these were rarely used. Note that the cache_size is NOT written to the superblock and can be freely changed without breaking backwards compatibility. --- .travis.yml | 4 +- lfs.c | 179 +++++++++++++++++++++------------------------ lfs.h | 20 +++-- lfs_util.h | 6 +- tests/template.fmt | 7 +- 5 files changed, 108 insertions(+), 108 deletions(-) diff --git a/.travis.yml b/.travis.yml index 54a5771e..a3d18aa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,8 +18,8 @@ script: - make test QUIET=1 # run tests with a few different configurations - - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_PROG_SIZE=1" - - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_PROG_SIZE=512" + - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_CACHE_SIZE=4" + - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_CACHE_SIZE=512" - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048" - make clean test QUIET=1 CFLAGS+="-DLFS_INLINE_MAX=0" diff --git a/lfs.c b/lfs.c index 9b90e99f..baf2c1ae 100644 --- a/lfs.c +++ b/lfs.c @@ -20,18 +20,19 @@ /// Caching block device operations /// -static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, - const lfs_cache_t *pcache, lfs_block_t block, - lfs_off_t off, void *buffer, lfs_size_t size) { +static int lfs_cache_read(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, bool store, + lfs_block_t block, lfs_off_t off, + void *buffer, lfs_size_t size) { uint8_t *data = buffer; LFS_ASSERT(block != 0xffffffff); while (size > 0) { - if (pcache && block == pcache->block && off >= pcache->off && - off < pcache->off + lfs->cfg->prog_size) { + if (pcache && block == pcache->block && + off >= pcache->off && + off < pcache->off + pcache->size) { // is already in pcache? - lfs_size_t diff = lfs_min(size, - lfs->cfg->prog_size - (off-pcache->off)); + lfs_size_t diff = lfs_min(size, pcache->size - (off-pcache->off)); memcpy(data, &pcache->buffer[off-pcache->off], diff); data += diff; @@ -40,11 +41,14 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, continue; } - if (block == rcache->block && off >= rcache->off && - off < rcache->off + lfs->cfg->read_size) { + if (block == rcache->block && + off >= rcache->off && + off < rcache->off + rcache->size) { // is already in rcache? - lfs_size_t diff = lfs_min(size, - lfs->cfg->read_size - (off-rcache->off)); + lfs_size_t diff = lfs_min(size, rcache->size - (off-rcache->off)); + if (pcache && block == pcache->block) { + diff = lfs_min(diff, pcache->off - off); + } memcpy(data, &rcache->buffer[off-rcache->off], diff); data += diff; @@ -53,7 +57,8 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, continue; } - if (off % lfs->cfg->read_size == 0 && size >= lfs->cfg->read_size) { + if (!store && off % lfs->cfg->read_size == 0 && + size >= lfs->cfg->read_size) { // bypass cache? lfs_size_t diff = size - (size % lfs->cfg->read_size); int err = lfs->cfg->read(lfs->cfg, block, off, data, diff); @@ -69,10 +74,12 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, // load to cache, first condition can no longer fail LFS_ASSERT(block < lfs->cfg->block_count); + lfs_size_t size = store ? lfs->cfg->cache_size : lfs->cfg->prog_size; rcache->block = block; - rcache->off = off - (off % lfs->cfg->read_size); + rcache->off = lfs_aligndown(off, size); + rcache->size = size; int err = lfs->cfg->read(lfs->cfg, rcache->block, - rcache->off, rcache->buffer, lfs->cfg->read_size); + rcache->off, rcache->buffer, size); if (err) { return err; } @@ -81,14 +88,15 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, return 0; } -static int lfs_cache_cmp(lfs_t *lfs, lfs_cache_t *rcache, - const lfs_cache_t *pcache, lfs_block_t block, - lfs_off_t off, const void *buffer, lfs_size_t size) { +static int lfs_cache_cmp(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, + lfs_block_t block, lfs_off_t off, + const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; for (lfs_off_t i = 0; i < size; i++) { uint8_t c; - int err = lfs_cache_read(lfs, rcache, pcache, + int err = lfs_cache_read(lfs, pcache, rcache, true, block, off+i, &c, 1); if (err) { return err; @@ -102,12 +110,12 @@ static int lfs_cache_cmp(lfs_t *lfs, lfs_cache_t *rcache, return true; } -static int lfs_cache_crc(lfs_t *lfs, lfs_cache_t *rcache, - const lfs_cache_t *pcache, lfs_block_t block, - lfs_off_t off, lfs_size_t size, uint32_t *crc) { +static int lfs_cache_crc(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, + lfs_block_t block, lfs_off_t off, lfs_size_t size, uint32_t *crc) { for (lfs_off_t i = 0; i < size; i++) { uint8_t c; - int err = lfs_cache_read(lfs, rcache, pcache, + int err = lfs_cache_read(lfs, pcache, rcache, true, block, off+i, &c, 1); if (err) { return err; @@ -120,18 +128,21 @@ static int lfs_cache_crc(lfs_t *lfs, lfs_cache_t *rcache, } static int lfs_cache_flush(lfs_t *lfs, - lfs_cache_t *pcache, lfs_cache_t *rcache) { + lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { if (pcache->block != 0xffffffff) { LFS_ASSERT(pcache->block < lfs->cfg->block_count); + lfs_size_t diff = lfs_alignup(pcache->size, lfs->cfg->prog_size); int err = lfs->cfg->prog(lfs->cfg, pcache->block, - pcache->off, pcache->buffer, lfs->cfg->prog_size); + pcache->off, pcache->buffer, diff); if (err) { return err; } - if (rcache) { - int res = lfs_cache_cmp(lfs, rcache, NULL, pcache->block, - pcache->off, pcache->buffer, lfs->cfg->prog_size); + if (validate) { + // check data on disk + rcache->block = 0xffffffff; + int res = lfs_cache_cmp(lfs, NULL, rcache, pcache->block, + pcache->off, pcache->buffer, diff); if (res < 0) { return res; } @@ -147,28 +158,31 @@ static int lfs_cache_flush(lfs_t *lfs, return 0; } -static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, - lfs_cache_t *rcache, lfs_block_t block, - lfs_off_t off, const void *buffer, lfs_size_t size) { +static int lfs_cache_prog(lfs_t *lfs, + lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate, + lfs_block_t block, lfs_off_t off, + const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; LFS_ASSERT(block != 0xffffffff); LFS_ASSERT(off + size <= lfs->cfg->block_size); while (size > 0) { - if (block == pcache->block && off >= pcache->off && - off < pcache->off + lfs->cfg->prog_size) { - // is already in pcache? + if (block == pcache->block && + off >= pcache->off && + off < pcache->off + lfs->cfg->cache_size) { + // already fits in pcache? lfs_size_t diff = lfs_min(size, - lfs->cfg->prog_size - (off-pcache->off)); + lfs->cfg->cache_size - (off-pcache->off)); memcpy(&pcache->buffer[off-pcache->off], data, diff); data += diff; off += diff; size -= diff; - if (off % lfs->cfg->prog_size == 0) { + pcache->size = off - pcache->off; + if (pcache->size == lfs->cfg->cache_size) { // eagerly flush out pcache if we fill up - int err = lfs_cache_flush(lfs, pcache, rcache); + int err = lfs_cache_flush(lfs, pcache, rcache, validate); if (err) { return err; } @@ -181,37 +195,10 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, // entire block or manually flushing the pcache LFS_ASSERT(pcache->block == 0xffffffff); - if (off % lfs->cfg->prog_size == 0 && - size >= lfs->cfg->prog_size) { - // bypass pcache? - LFS_ASSERT(block < lfs->cfg->block_count); - lfs_size_t diff = size - (size % lfs->cfg->prog_size); - int err = lfs->cfg->prog(lfs->cfg, block, off, data, diff); - if (err) { - return err; - } - - if (rcache) { - int res = lfs_cache_cmp(lfs, rcache, NULL, - block, off, data, diff); - if (res < 0) { - return res; - } - - if (!res) { - return LFS_ERR_CORRUPT; - } - } - - data += diff; - off += diff; - size -= diff; - continue; - } - // prepare pcache, first condition can no longer fail pcache->block = block; - pcache->off = off - (off % lfs->cfg->prog_size); + pcache->off = lfs_aligndown(off, lfs->cfg->prog_size); + pcache->size = 0; } return 0; @@ -221,24 +208,24 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, /// General lfs block device operations /// static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { - return lfs_cache_read(lfs, &lfs->rcache, &lfs->pcache, + return lfs_cache_read(lfs, &lfs->pcache, &lfs->rcache, true, block, off, buffer, size); } static int lfs_bd_prog(lfs_t *lfs, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { - return lfs_cache_prog(lfs, &lfs->pcache, NULL, + return lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, false, block, off, buffer, size); } static int lfs_bd_cmp(lfs_t *lfs, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { - return lfs_cache_cmp(lfs, &lfs->rcache, NULL, block, off, buffer, size); + return lfs_cache_cmp(lfs, NULL, &lfs->rcache, block, off, buffer, size); } static int lfs_bd_crc(lfs_t *lfs, lfs_block_t block, lfs_off_t off, lfs_size_t size, uint32_t *crc) { - return lfs_cache_crc(lfs, &lfs->rcache, NULL, block, off, size, crc); + return lfs_cache_crc(lfs, NULL, &lfs->rcache, block, off, size, crc); } static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { @@ -249,7 +236,7 @@ static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { static int lfs_bd_sync(lfs_t *lfs) { lfs->rcache.block = 0xffffffff; - int err = lfs_cache_flush(lfs, &lfs->pcache, NULL); + int err = lfs_cache_flush(lfs, &lfs->pcache, &lfs->rcache, false); if (err) { return err; } @@ -1658,7 +1645,8 @@ static int lfs_ctzfind(lfs_t *lfs, lfs_npw2(current-target+1) - 1, lfs_ctz(current)); - int err = lfs_cache_read(lfs, rcache, pcache, head, 4*skip, &head, 4); + int err = lfs_cache_read(lfs, pcache, rcache, false, + head, 4*skip, &head, 4); head = lfs_fromle32(head); if (err) { return err; @@ -1709,13 +1697,13 @@ static int lfs_ctzextend(lfs_t *lfs, if (size != lfs->cfg->block_size) { for (lfs_off_t i = 0; i < size; i++) { uint8_t data; - err = lfs_cache_read(lfs, rcache, NULL, + err = lfs_cache_read(lfs, NULL, rcache, true, head, i, &data, 1); if (err) { return err; } - err = lfs_cache_prog(lfs, pcache, rcache, + err = lfs_cache_prog(lfs, pcache, rcache, true, nblock, i, &data, 1); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1736,7 +1724,7 @@ static int lfs_ctzextend(lfs_t *lfs, for (lfs_off_t i = 0; i < skips; i++) { head = lfs_tole32(head); - err = lfs_cache_prog(lfs, pcache, rcache, + err = lfs_cache_prog(lfs, pcache, rcache, true, nblock, 4*i, &head, 4); head = lfs_fromle32(head); if (err) { @@ -1747,7 +1735,7 @@ static int lfs_ctzextend(lfs_t *lfs, } if (i != skips-1) { - err = lfs_cache_read(lfs, rcache, NULL, + err = lfs_cache_read(lfs, NULL, rcache, false, head, 4*i, &head, 4); head = lfs_fromle32(head); if (err) { @@ -1793,7 +1781,8 @@ static int lfs_ctztraverse(lfs_t *lfs, lfs_block_t heads[2]; int count = 2 - (index & 1); - err = lfs_cache_read(lfs, rcache, pcache, head, 0, &heads, count*4); + err = lfs_cache_read(lfs, pcache, rcache, false, + head, 0, &heads, count*4); heads[0] = lfs_fromle32(heads[0]); heads[1] = lfs_fromle32(heads[1]); if (err) { @@ -1916,15 +1905,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, file->cache.block = 0xffffffff; if (file->cfg->buffer) { file->cache.buffer = file->cfg->buffer; - } else if ((file->flags & 3) == LFS_O_RDONLY) { - // TODO cache_size - file->cache.buffer = lfs_malloc(lfs->cfg->read_size); - if (!file->cache.buffer) { - err = LFS_ERR_NOMEM; - goto cleanup; - } } else { - file->cache.buffer = lfs_malloc(lfs->cfg->prog_size); + file->cache.buffer = lfs_malloc(lfs->cfg->cache_size); if (!file->cache.buffer) { err = LFS_ERR_NOMEM; goto cleanup; @@ -1938,6 +1920,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, file->flags |= LFS_F_INLINE; file->cache.block = file->ctz.head; file->cache.off = 0; + file->cache.size = lfs->cfg->cache_size; // don't always read (may be new/trunc file) if (file->ctz.size > 0) { @@ -2005,13 +1988,13 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { // either read from dirty cache or disk for (lfs_off_t i = 0; i < file->off; i++) { uint8_t data; - err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, + err = lfs_cache_read(lfs, &file->cache, &lfs->rcache, true, file->block, i, &data, 1); if (err) { return err; } - err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, + err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, true, nblock, i, &data, 1); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -2022,9 +2005,10 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { } // copy over new state of file - memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->cache_size); file->cache.block = lfs->pcache.block; file->cache.off = lfs->pcache.off; + file->cache.size = lfs->pcache.size; lfs->pcache.block = 0xffffffff; file->block = nblock; @@ -2077,7 +2061,8 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { // write out what we have while (true) { - int err = lfs_cache_flush(lfs, &file->cache, &lfs->rcache); + int err = lfs_cache_flush(lfs, + &file->cache, &lfs->rcache, true); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -2217,7 +2202,7 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, // read as much as we can in current block lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); - int err = lfs_cache_read(lfs, &file->cache, NULL, + int err = lfs_cache_read(lfs, NULL, &file->cache, true, file->block, file->off, data, diff); if (err) { return err; @@ -2322,7 +2307,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // program as much as we can in current block lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); while (true) { - int err = lfs_cache_prog(lfs, &file->cache, &lfs->rcache, + int err = lfs_cache_prog(lfs, &file->cache, &lfs->rcache, true, file->block, file->off, data, diff); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -2723,7 +2708,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { if (lfs->cfg->read_buffer) { lfs->rcache.buffer = lfs->cfg->read_buffer; } else { - lfs->rcache.buffer = lfs_malloc(lfs->cfg->read_size); + lfs->rcache.buffer = lfs_malloc(lfs->cfg->cache_size); if (!lfs->rcache.buffer) { return LFS_ERR_NOMEM; } @@ -2734,7 +2719,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { if (lfs->cfg->prog_buffer) { lfs->pcache.buffer = lfs->cfg->prog_buffer; } else { - lfs->pcache.buffer = lfs_malloc(lfs->cfg->prog_size); + lfs->pcache.buffer = lfs_malloc(lfs->cfg->cache_size); if (!lfs->pcache.buffer) { return LFS_ERR_NOMEM; } @@ -2752,9 +2737,11 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { } } - // check that program and read sizes are multiples of the block size - LFS_ASSERT(lfs->cfg->prog_size % lfs->cfg->read_size == 0); - LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->prog_size == 0); + // check that block size is a multiple of cache size is a multiple + // of prog and read sizes + LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->read_size == 0); + LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->prog_size == 0); + LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->cache_size == 0); // check that the block size is large enough to fit ctz pointers LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) @@ -2762,7 +2749,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { // check that the size limits are sane LFS_ASSERT(lfs->cfg->inline_size <= LFS_INLINE_MAX); - LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->read_size); + LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->read_size); // TODO lfs->inline_size = lfs->cfg->inline_size; if (!lfs->inline_size) { lfs->inline_size = lfs_min(LFS_INLINE_MAX, lfs->cfg->read_size); diff --git a/lfs.h b/lfs.h index e648c006..d4d0639f 100644 --- a/lfs.h +++ b/lfs.h @@ -177,21 +177,24 @@ struct lfs_config { // are propogated to the user. int (*sync)(const struct lfs_config *c); - // Minimum size of a block read. This determines the size of read buffers. - // This may be larger than the physical read size to improve performance - // by caching more of the block device. + // Minimum size of a block read. All read operations will be a + // multiple of this value. lfs_size_t read_size; - // Minimum size of a block program. This determines the size of program - // buffers. This may be larger than the physical program size to improve - // performance by caching more of the block device. - // Must be a multiple of the read size. + // Minimum size of a block program. All program operations will be a + // multiple of this value. lfs_size_t prog_size; + // Size of block caches. Each cache buffers a portion of a block in RAM. + // This determines the size of the read cache, the program cache, and a + // cache per file. Larger caches can improve performance by storing more + // data. Must be a multiple of the read and program sizes. + lfs_size_t cache_size; + // Size of an erasable block. This does not impact ram consumption and // may be larger than the physical erase size. However, this should be // kept small as each file currently takes up an entire block. - // Must be a multiple of the program size. + // Must be a multiple of the read, program, and cache sizes. lfs_size_t block_size; // Number of erasable blocks on the device. @@ -283,6 +286,7 @@ typedef struct lfs_mattr { typedef struct lfs_cache { lfs_block_t block; lfs_off_t off; + lfs_size_t size; uint8_t *buffer; } lfs_cache_t; diff --git a/lfs_util.h b/lfs_util.h index 6a577eb8..6dd28da8 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -183,8 +183,12 @@ static inline uint16_t lfs_tole16(uint16_t a) { } // Align to nearest multiple of a size +static inline uint32_t lfs_aligndown(uint32_t a, uint32_t alignment) { + return a - (a % alignment); +} + static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) { - return (a + alignment-1) - ((a + alignment-1) % alignment); + return lfs_aligndown(a + alignment-1, alignment); } // Calculate CRC-32 with polynomial = 0x04c11db7 diff --git a/tests/template.fmt b/tests/template.fmt index 8f1e8efa..7f136c8c 100644 --- a/tests/template.fmt +++ b/tests/template.fmt @@ -66,7 +66,11 @@ uintmax_t test; #endif #ifndef LFS_PROG_SIZE -#define LFS_PROG_SIZE 16 +#define LFS_PROG_SIZE LFS_READ_SIZE +#endif + +#ifndef LFS_CACHE_SIZE +#define LFS_CACHE_SIZE 64 #endif #ifndef LFS_BLOCK_SIZE @@ -92,6 +96,7 @@ const struct lfs_config cfg = {{ .read_size = LFS_READ_SIZE, .prog_size = LFS_PROG_SIZE, + .cache_size = LFS_CACHE_SIZE, .block_size = LFS_BLOCK_SIZE, .block_count = LFS_BLOCK_COUNT, .lookahead = LFS_LOOKAHEAD, From 1941bbda768bade0cc0a1031fddce0f9749fe13e Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 4 Aug 2018 16:04:24 -0500 Subject: [PATCH 078/139] Cleaned up config options - Updated documentation where needed - Added asserts which take into account relationships with the new cache_size configuration - Restructured ordering to be consistent for the three main configurables: LFS_ATTR_MAX, LFS_NAME_MAX, and LFS_INLINE_MAX --- lfs.c | 56 +++++++++++++++++++++++++++----------------------------- lfs.h | 54 +++++++++++++++++++++++++++--------------------------- 2 files changed, 54 insertions(+), 56 deletions(-) diff --git a/lfs.c b/lfs.c index baf2c1ae..dd479165 100644 --- a/lfs.c +++ b/lfs.c @@ -2703,6 +2703,16 @@ static inline void lfs_superblocktole32(lfs_superblock_t *superblock) { static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->cfg = cfg; + // check that block size is a multiple of cache size is a multiple + // of prog and read sizes + LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->read_size == 0); + LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->prog_size == 0); + LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->cache_size == 0); + + // check that the block size is large enough to fit ctz pointers + LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) + <= lfs->cfg->block_size); + // setup read cache lfs->rcache.block = 0xffffffff; if (lfs->cfg->read_buffer) { @@ -2725,7 +2735,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { } } - // setup lookahead, round down to nearest 32-bits + // setup lookahead, must be multiple of 32-bits LFS_ASSERT(lfs->cfg->lookahead % 32 == 0); LFS_ASSERT(lfs->cfg->lookahead > 0); if (lfs->cfg->lookahead_buffer) { @@ -2737,22 +2747,12 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { } } - // check that block size is a multiple of cache size is a multiple - // of prog and read sizes - LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->read_size == 0); - LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->prog_size == 0); - LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->cache_size == 0); - - // check that the block size is large enough to fit ctz pointers - LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) - <= lfs->cfg->block_size); - // check that the size limits are sane LFS_ASSERT(lfs->cfg->inline_size <= LFS_INLINE_MAX); - LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->read_size); // TODO + LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->cache_size); lfs->inline_size = lfs->cfg->inline_size; if (!lfs->inline_size) { - lfs->inline_size = lfs_min(LFS_INLINE_MAX, lfs->cfg->read_size); + lfs->inline_size = lfs_min(LFS_INLINE_MAX, lfs->cfg->cache_size); } LFS_ASSERT(lfs->cfg->attr_size <= LFS_ATTR_MAX); @@ -2841,9 +2841,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .block_size = lfs->cfg->block_size, .block_count = lfs->cfg->block_count, - .inline_size = lfs->inline_size, .attr_size = lfs->attr_size, .name_size = lfs->name_size, + .inline_size = lfs->inline_size, }; lfs_superblocktole32(&superblock); @@ -2883,9 +2883,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_mdir_t superdir; err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); if (err) { - if (err == LFS_ERR_CORRUPT) { - LFS_ERROR("Invalid superblock at %d %d", 0, 1); - } return err; } @@ -2899,8 +2896,8 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblockfromle32(&superblock); if (memcmp(superblock.magic, "littlefs", 8) != 0) { - LFS_ERROR("Invalid superblock at %d %d", 0, 1); - return LFS_ERR_CORRUPT; + LFS_ERROR("Invalid superblock \"%.8s\"", superblock.magic); + return LFS_ERR_INVAL; } uint16_t major_version = (0xffff & (superblock.version >> 16)); @@ -2919,16 +2916,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_pairfromle32(lfs->root); - if (superblock.inline_size) { - if (superblock.inline_size > lfs->inline_size) { - LFS_ERROR("Unsupported inline size (%d > %d)", - superblock.inline_size, lfs->inline_size); - return LFS_ERR_INVAL; - } - - lfs->inline_size = superblock.inline_size; - } - + // check superblock configuration if (superblock.attr_size) { if (superblock.attr_size > lfs->attr_size) { LFS_ERROR("Unsupported attr size (%d > %d)", @@ -2949,6 +2937,16 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->name_size = superblock.name_size; } + if (superblock.inline_size) { + if (superblock.inline_size > lfs->inline_size) { + LFS_ERROR("Unsupported inline size (%d > %d)", + superblock.inline_size, lfs->inline_size); + return LFS_ERR_INVAL; + } + + lfs->inline_size = superblock.inline_size; + } + // scan for any global updates lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pairisnull(dir.tail)) { diff --git a/lfs.h b/lfs.h index d4d0639f..088b27f7 100644 --- a/lfs.h +++ b/lfs.h @@ -50,31 +50,29 @@ typedef int32_t lfs_soff_t; typedef uint32_t lfs_block_t; -// Maximum inline file size in bytes. Large inline files require a larger -// read and prog cache, but if a file can be inline it does not need its own -// data block. LFS_ATTR_MAX + LFS_INLINE_MAX must be <= 0xffff. Stored in -// superblock and must be respected by other littlefs drivers. -// TODO doc -#ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 0xfff -#endif - // Maximum size of all attributes per file in bytes, may be redefined but a -// a smaller LFS_ATTR_MAX has no benefit. LFS_ATTR_MAX + LFS_INLINE_MAX -// must be <= 0xffff. Stored in superblock and must be respected by other +// a smaller LFS_ATTR_MAX has no benefit. Stored in 12-bits and limited +// to <= 0xfff. Stored in superblock and must be respected by other // littlefs drivers. -// TODO doc #ifndef LFS_ATTR_MAX #define LFS_ATTR_MAX 0xfff #endif -// Max name size in bytes, may be redefined to reduce the size of the -// info struct. Stored in superblock and must be respected by other -// littlefs drivers. +// Maximum name size in bytes, may be redefined to reduce the size of the +// info struct. Limited to <= LFS_ATTR_MAX. Stored in superblock and must +// be respected by other littlefs drivers. #ifndef LFS_NAME_MAX #define LFS_NAME_MAX 0xff #endif +// Maximum inline file size in bytes. Large inline files require a larger +// cache size, but if a file can be inline it does not need its own data +// block. Limited to <= LFS_ATTR_MAX and <= cache_size. Stored in superblock +// and must be respected by other littlefs drivers. +#ifndef LFS_INLINE_MAX +#define LFS_INLINE_MAX 0xfff +#endif + // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { @@ -110,7 +108,7 @@ enum lfs_type { LFS_TYPE_TAIL = 0x0c0, LFS_TYPE_SOFTTAIL = 0x0c0, LFS_TYPE_HARDTAIL = 0x0c1, - LFS_TYPE_CRC = 0x0f0, // TODO are trailing ones useful? + LFS_TYPE_CRC = 0x0f0, LFS_TYPE_INLINESTRUCT = 0x040, LFS_TYPE_CTZSTRUCT = 0x041, @@ -216,15 +214,8 @@ struct lfs_config { // lookahead block. void *lookahead_buffer; - // Optional upper limit on inlined files in bytes. Large inline files - // require a larger read and prog cache, but if a file can be inlined it - // does not need its own data block. Must be smaller than the read size - // and prog size. Defaults to min(LFS_INLINE_MAX, read_size) when zero. - // Stored in superblock and must be respected by other littlefs drivers. - lfs_size_t inline_size; - - // Optional upper limit on attributes per file in bytes. No downside for - // larger attributes size but must be less than LFS_ATTR_MAX. Defaults to + // Optional upper limit on file attributes in bytes. No downside for larger + // attributes size but must be less than LFS_ATTR_MAX. Defaults to // LFS_ATTR_MAX when zero.Stored in superblock and must be respected by // other littlefs drivers. lfs_size_t attr_size; @@ -234,6 +225,13 @@ struct lfs_config { // the LFS_NAME_MAX define. Defaults to LFS_NAME_MAX when zero. Stored in // superblock and must be respected by other littlefs drivers. lfs_size_t name_size; + + // Optional upper limit on inlined files in bytes. Large inline files + // require a larger cache size, but if a file can be inlined it does not + // need its own data block. Must be smaller than cache_size and less than + // LFS_INLINE_MAX. Defaults to min(LFS_INLINE_MAX, read_size) when zero. + // Stored in superblock and must be respected by other littlefs drivers. + lfs_size_t inline_size; }; // File info structure @@ -355,9 +353,9 @@ typedef struct lfs_superblock { lfs_size_t block_size; lfs_size_t block_count; - lfs_size_t inline_size; lfs_size_t attr_size; lfs_size_t name_size; + lfs_size_t inline_size; } lfs_superblock_t; typedef struct lfs_free { @@ -381,9 +379,11 @@ typedef struct lfs { lfs_free_t free; const struct lfs_config *cfg; - lfs_size_t inline_size; + lfs_size_t block_size; + lfs_size_t block_count; lfs_size_t attr_size; lfs_size_t name_size; + lfs_size_t inline_size; } lfs_t; From f369f80540193f28103a73b30acbc2063768c98a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 4 Aug 2018 18:55:04 -0500 Subject: [PATCH 079/139] Added tests for global state stealing State stealing is a tricky part of managing the xored-globals. When removing a metadata-pair from the metadata chain, whichever metadata-pair does the removing is also responsible for stealing the removed metadata-pair's global delta and incorporating it into it's own global delta. Otherwise the global state would become corrupted. --- lfs.c | 2 -- tests/test_move.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lfs.c b/lfs.c index dd479165..46c769a9 100644 --- a/lfs.c +++ b/lfs.c @@ -2513,7 +2513,6 @@ int lfs_remove(lfs_t *lfs, const char *path) { lfs_globaldeorphaned(lfs, true); // steal state - // TODO test for global state stealing? cwd.tail[0] = dir.tail[0]; cwd.tail[1] = dir.tail[1]; lfs_globalxor(&lfs->locals, &dir.locals); @@ -2626,7 +2625,6 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_globaldeorphaned(lfs, true); // steal state - // TODO test for global state stealing? newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; lfs_globalxor(&lfs->locals, &prevdir.locals); diff --git a/tests/test_move.sh b/tests/test_move.sh index bea618c8..a36d0581 100755 --- a/tests/test_move.sh +++ b/tests/test_move.sh @@ -283,6 +283,52 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Move state stealing ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + + lfs_remove(&lfs, "b") => 0; + lfs_remove(&lfs, "c") => 0; + + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + + lfs_dir_open(&lfs, &dir[0], "a/hi") => LFS_ERR_NOENT; + lfs_dir_open(&lfs, &dir[0], "b") => LFS_ERR_NOENT; + lfs_dir_open(&lfs, &dir[0], "c") => LFS_ERR_NOENT; + + lfs_dir_open(&lfs, &dir[0], "d/hi") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "hola") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "bonjour") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "ohayo") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_dir_close(&lfs, &dir[0]) => 0; + + lfs_dir_open(&lfs, &dir[0], "a/hello") => LFS_ERR_NOENT; + lfs_dir_open(&lfs, &dir[0], "b") => LFS_ERR_NOENT; + lfs_dir_open(&lfs, &dir[0], "c") => LFS_ERR_NOENT; + + lfs_file_open(&lfs, &file[0], "d/hello", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], buffer, 5) => 5; + memcmp(buffer, "hola\n", 5) => 0; + lfs_file_read(&lfs, &file[0], buffer, 8) => 8; + memcmp(buffer, "bonjour\n", 8) => 0; + lfs_file_read(&lfs, &file[0], buffer, 6) => 6; + memcmp(buffer, "ohayo\n", 6) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + + lfs_unmount(&lfs) => 0; +TEST + echo "--- Results ---" tests/stats.py From a88230ae6a597c270020d716670d3c0738fd00c1 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 4 Aug 2018 19:23:49 -0500 Subject: [PATCH 080/139] Updated custom attribute documentation and tweaked nonexistant attributes Because of limitations in how littlefs manages attributes on disk, littlefs views zero-length attributes and missing attributes as the same thing. The simpliest implementation of attributes mirrors this behaviour transparently for the user. --- lfs.c | 12 ++---- lfs.h | 95 ++++++++++++++++++++++++++------------------- tests/test_attrs.sh | 2 +- 3 files changed, 59 insertions(+), 50 deletions(-) diff --git a/lfs.c b/lfs.c index 46c769a9..0cb5788b 100644 --- a/lfs.c +++ b/lfs.c @@ -2651,14 +2651,11 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, res = lfs_dir_get(lfs, &cwd, 0x7ffff000, LFS_MKTAG(0x100 | type, lfs_tagid(res), lfs_min(size, lfs->attr_size)), buffer); - if (res < 0) { - if (res == LFS_ERR_NOENT) { - return LFS_ERR_NOATTR; - } + if (res < 0 && res != LFS_ERR_NOENT) { return res; } - return lfs_tagsize(res); + return (res == LFS_ERR_NOENT) ? 0 : lfs_tagsize(res); } int lfs_setattr(lfs_t *lfs, const char *path, @@ -3270,13 +3267,10 @@ lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, LFS_MKTAG(0x100 | type, 0, lfs_min(size, lfs->attr_size)), buffer); if (res < 0) { - if (res == LFS_ERR_NOENT) { - return LFS_ERR_NOATTR; - } return res; } - return lfs_tagsize(res); + return (res == LFS_ERR_NOENT) ? 0 : lfs_tagsize(res); } int lfs_fs_setattr(lfs_t *lfs, diff --git a/lfs.h b/lfs.h index 088b27f7..d63ea784 100644 --- a/lfs.h +++ b/lfs.h @@ -89,7 +89,6 @@ enum lfs_error { LFS_ERR_NOSPC = -28, // No space left on device LFS_ERR_NOMEM = -12, // No more memory available LFS_ERR_NAMETOOLONG = -36, // File name too long - LFS_ERR_NOATTR = -61, // No data/attr available }; // File types @@ -248,7 +247,7 @@ struct lfs_info { // Custom attribute structure struct lfs_attr { - // 8-bit Type of attribute, provided by user and used to + // 8-bit type of attribute, provided by user and used to // identify the attribute uint8_t type; @@ -268,8 +267,17 @@ struct lfs_file_config { // If NULL, malloc will be used by default. void *buffer; - // Optional, linked list of custom attributes. - // TODO document more + // Optional, linked list of custom attributes related to the file. If the + // file is opened with read access, the attributes will be read from + // during the open call. If the file is opened with write access, the + // attributes will be written to disk every file sync or close. This + // write occurs atomically with update to the file's contents. + // + // Custom attributes are uniquely identified by an 8-bit type and limited + // to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller + // than the buffer, it will be padded with zeros. If the stored attribute + // is larger, then it will be silently truncated. If the attribute is not + // found, it will be created implicitly. struct lfs_attr *attrs; }; @@ -436,24 +444,27 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath); // Returns a negative error code on failure. int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); -// Get custom attributes +// Get a custom attribute // -// Attributes are looked up based on the type id. If the stored attribute is -// smaller than the buffer, it is padded with zeros. It the stored attribute -// is larger than the buffer, LFS_ERR_RANGE is returned. +// Custom attributes are uniquely identified by an 8-bit type and limited +// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller than +// the buffer, it will be padded with zeros. If the stored attribute is larger, +// then it will be silently truncated. // -// TODO doc -// Returns a negative error code on failure. +// Returns the size of the attribute, or a negative error code on failure. +// Note, the returned size is the size of the attribute on disk, irrespective +// of the size of the buffer. This can be used to dynamically allocate a buffer +// or check for existance. lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, uint8_t type, void *buffer, lfs_size_t size); // Set custom attributes // -// The array of attributes will be used to update the attributes stored on -// disk based on their type id. Unspecified attributes are left unmodified. -// Specifying an attribute with zero size deletes the attribute. +// Custom attributes are uniquely identified by an 8-bit type and limited +// to LFS_ATTR_MAX bytes. If an attribute is not found, it will be +// implicitly created, and setting the size of an attribute to zero deletes +// the attribute. // -// TODO doc // Returns a negative error code on failure. int lfs_setattr(lfs_t *lfs, const char *path, uint8_t type, const void *buffer, lfs_size_t size); @@ -593,32 +604,7 @@ lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir); int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); -/// Filesystem filesystem operations /// TODO choose one -/// Miscellaneous littlefs specific operations /// TODO choose one - -// Get custom attributes on the filesystem -// -// Attributes are looked up based on the type id. If the stored attribute is -// smaller than the buffer, it is padded with zeros. It the stored attribute -// is larger than the buffer, LFS_ERR_RANGE is returned. -// -// TODO doc -// Returns a negative error code on failure. -lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, - uint8_t type, void *buffer, lfs_size_t size); - -// Set custom attributes on the filesystem -// -// The array of attributes will be used to update the attributes stored on -// disk based on their type id. Unspecified attributes are left unmodified. -// Specifying an attribute with zero size deletes the attribute. -// -// Note: Filesystem level attributes are not available for wear-leveling -// -// TODO doc -// Returns a negative error code on failure. -int lfs_fs_setattr(lfs_t *lfs, - uint8_t type, const void *buffer, lfs_size_t size); +/// Filesystem-level filesystem operations // Finds the current size of the filesystem // @@ -637,5 +623,34 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // Returns a negative error code on failure. int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); +// Get custom attributes on the filesystem +// +// Custom attributes are uniquely identified by an 8-bit type and limited +// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller than +// the buffer, it will be padded with zeros. If the stored attribute is larger, +// then it will be silently truncated. +// +// Note, filesystem-level attributes are not available for wear-leveling +// +// Returns the size of the attribute, or a negative error code on failure. +// Note, the returned size is the size of the attribute on disk, irrespective +// of the size of the buffer. This can be used to dynamically allocate a buffer +// or check for existance. +lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, + uint8_t type, void *buffer, lfs_size_t size); + +// Set custom attributes on the filesystem +// +// Custom attributes are uniquely identified by an 8-bit type and limited +// to LFS_ATTR_MAX bytes. If an attribute is not found, it will be +// implicitly created, and setting the size of an attribute to zero deletes +// the attribute. +// +// Note, filesystem-level attributes are not available for wear-leveling +// +// Returns a negative error code on failure. +int lfs_fs_setattr(lfs_t *lfs, + uint8_t type, const void *buffer, lfs_size_t size); + #endif diff --git a/tests/test_attrs.sh b/tests/test_attrs.sh index 2fde13ed..cb4f4dc5 100755 --- a/tests/test_attrs.sh +++ b/tests/test_attrs.sh @@ -241,7 +241,7 @@ tests/test.py << TEST lfs_getattr(&lfs, "hello/hello", 'B', buffer, 9) => 9; lfs_getattr(&lfs, "hello/hello", 'C', buffer+9, 9) => 5; - lfs_getattr(&lfs, "hello/hello", 'D', buffer+18, 9) => LFS_ERR_NOATTR; + lfs_getattr(&lfs, "hello/hello", 'D', buffer+18, 9) => 0; memcmp(buffer, "fffffffff", 9) => 0; memcmp(buffer+9, "ccccc\0\0\0\0", 9) => 0; memcmp(buffer+18, "\0\0\0\0\0\0\0\0\0", 9) => 0; From 213530c3767da4acf6965b9f3c2f8a0f21f3b3b1 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 4 Aug 2018 19:26:08 -0500 Subject: [PATCH 081/139] Changed LFS_ERR_CORRUPT to match EILSEQ instead of EBADE LFS_ERR_CORRUPT is unfortunately not a well defined error code. It's very important in the context of littlefs, but missing from the standard error codes defined in Linux. After some discussions with other developers, it was encouraged to use the encoding for EILSEQ over EBADE for representing on disk corrupt, as EILSEQ implies that there is something wrong with the data. I've changed this now to take advantage of the breaking changes in v2 to avoid a risky change to a return value. --- lfs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lfs.h b/lfs.h index d63ea784..7635f3b0 100644 --- a/lfs.h +++ b/lfs.h @@ -78,7 +78,7 @@ typedef uint32_t lfs_block_t; enum lfs_error { LFS_ERR_OK = 0, // No error LFS_ERR_IO = -5, // Error during device operation - LFS_ERR_CORRUPT = -52, // Corrupted + LFS_ERR_CORRUPT = -84, // Corrupted LFS_ERR_NOENT = -2, // No directory entry LFS_ERR_EXIST = -17, // Entry already exists LFS_ERR_NOTDIR = -20, // Entry is not a dir From dbcbe4e088a916231611beff34b0e71726bd2d5d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 4 Aug 2018 20:10:08 -0500 Subject: [PATCH 082/139] Changed name of upper-limits from blah_size to blah_max This standardizes the naming between the LFS_BLAH_MAX macros and the blah_max configuration in the lfs_config structure. --- lfs.c | 94 +++++++++++++++++++++++----------------------- lfs.h | 18 ++++----- tests/template.fmt | 2 - 3 files changed, 56 insertions(+), 58 deletions(-) diff --git a/lfs.c b/lfs.c index 0cb5788b..64cb5840 100644 --- a/lfs.c +++ b/lfs.c @@ -1382,7 +1382,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, } int32_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_size+1), info->name); + LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name); if (tag < 0) { return tag; } @@ -1422,7 +1422,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // check that name fits lfs_size_t nlen = strlen(path); - if (nlen > lfs->name_size) { + if (nlen > lfs->name_max) { return LFS_ERR_NAMETOOLONG; } @@ -1842,7 +1842,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // check that name fits lfs_size_t nlen = strlen(path); - if (nlen > lfs->name_size) { + if (nlen > lfs->name_max) { err = LFS_ERR_NAMETOOLONG; goto cleanup; } @@ -1892,7 +1892,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } if ((file->flags & 3) != LFS_O_RDONLY) { - if (a->size > lfs->attr_size) { + if (a->size > lfs->attr_max) { err = LFS_ERR_NOSPC; goto cleanup; } @@ -2252,7 +2252,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } if ((file->flags & LFS_F_INLINE) && - file->pos + nsize >= lfs->inline_size) { + file->pos + nsize >= lfs->inline_max) { // inline file doesn't fit anymore file->block = 0xfffffffe; file->off = file->pos; @@ -2586,7 +2586,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } else { // check that name fits lfs_size_t nlen = strlen(newpath); - if (nlen > lfs->name_size) { + if (nlen > lfs->name_max) { return LFS_ERR_NAMETOOLONG; } @@ -2650,7 +2650,7 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, res = lfs_dir_get(lfs, &cwd, 0x7ffff000, LFS_MKTAG(0x100 | type, lfs_tagid(res), - lfs_min(size, lfs->attr_size)), buffer); + lfs_min(size, lfs->attr_max)), buffer); if (res < 0 && res != LFS_ERR_NOENT) { return res; } @@ -2660,7 +2660,7 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, int lfs_setattr(lfs_t *lfs, const char *path, uint8_t type, const void *buffer, lfs_size_t size) { - if (size > lfs->attr_size) { + if (size > lfs->attr_max) { return LFS_ERR_NOSPC; } @@ -2681,18 +2681,18 @@ static inline void lfs_superblockfromle32(lfs_superblock_t *superblock) { superblock->version = lfs_fromle32(superblock->version); superblock->block_size = lfs_fromle32(superblock->block_size); superblock->block_count = lfs_fromle32(superblock->block_count); - superblock->inline_size = lfs_fromle32(superblock->inline_size); - superblock->attr_size = lfs_fromle32(superblock->attr_size); - superblock->name_size = lfs_fromle32(superblock->name_size); + superblock->inline_max = lfs_fromle32(superblock->inline_max); + superblock->attr_max = lfs_fromle32(superblock->attr_max); + superblock->name_max = lfs_fromle32(superblock->name_max); } static inline void lfs_superblocktole32(lfs_superblock_t *superblock) { superblock->version = lfs_tole32(superblock->version); superblock->block_size = lfs_tole32(superblock->block_size); superblock->block_count = lfs_tole32(superblock->block_count); - superblock->inline_size = lfs_tole32(superblock->inline_size); - superblock->attr_size = lfs_tole32(superblock->attr_size); - superblock->name_size = lfs_tole32(superblock->name_size); + superblock->inline_max = lfs_tole32(superblock->inline_max); + superblock->attr_max = lfs_tole32(superblock->attr_max); + superblock->name_max = lfs_tole32(superblock->name_max); } static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { @@ -2743,23 +2743,23 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { } // check that the size limits are sane - LFS_ASSERT(lfs->cfg->inline_size <= LFS_INLINE_MAX); - LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->cache_size); - lfs->inline_size = lfs->cfg->inline_size; - if (!lfs->inline_size) { - lfs->inline_size = lfs_min(LFS_INLINE_MAX, lfs->cfg->cache_size); + LFS_ASSERT(lfs->cfg->inline_max <= LFS_INLINE_MAX); + LFS_ASSERT(lfs->cfg->inline_max <= lfs->cfg->cache_size); + lfs->inline_max = lfs->cfg->inline_max; + if (!lfs->inline_max) { + lfs->inline_max = lfs_min(LFS_INLINE_MAX, lfs->cfg->cache_size); } - LFS_ASSERT(lfs->cfg->attr_size <= LFS_ATTR_MAX); - lfs->attr_size = lfs->cfg->attr_size; - if (!lfs->attr_size) { - lfs->attr_size = LFS_ATTR_MAX; + LFS_ASSERT(lfs->cfg->attr_max <= LFS_ATTR_MAX); + lfs->attr_max = lfs->cfg->attr_max; + if (!lfs->attr_max) { + lfs->attr_max = LFS_ATTR_MAX; } - LFS_ASSERT(lfs->cfg->name_size <= LFS_NAME_MAX); - lfs->name_size = lfs->cfg->name_size; - if (!lfs->name_size) { - lfs->name_size = LFS_NAME_MAX; + LFS_ASSERT(lfs->cfg->name_max <= LFS_NAME_MAX); + lfs->name_max = lfs->cfg->name_max; + if (!lfs->name_max) { + lfs->name_max = LFS_NAME_MAX; } // setup default state @@ -2836,9 +2836,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .block_size = lfs->cfg->block_size, .block_count = lfs->cfg->block_count, - .attr_size = lfs->attr_size, - .name_size = lfs->name_size, - .inline_size = lfs->inline_size, + .attr_max = lfs->attr_max, + .name_max = lfs->name_max, + .inline_max = lfs->inline_max, }; lfs_superblocktole32(&superblock); @@ -2912,34 +2912,34 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_pairfromle32(lfs->root); // check superblock configuration - if (superblock.attr_size) { - if (superblock.attr_size > lfs->attr_size) { - LFS_ERROR("Unsupported attr size (%d > %d)", - superblock.attr_size, lfs->attr_size); + if (superblock.attr_max) { + if (superblock.attr_max > lfs->attr_max) { + LFS_ERROR("Unsupported attr_max (%d > %d)", + superblock.attr_max, lfs->attr_max); return LFS_ERR_INVAL; } - lfs->attr_size = superblock.attr_size; + lfs->attr_max = superblock.attr_max; } - if (superblock.name_size) { - if (superblock.name_size > lfs->name_size) { - LFS_ERROR("Unsupported name size (%d > %d)", - superblock.name_size, lfs->name_size); + if (superblock.name_max) { + if (superblock.name_max > lfs->name_max) { + LFS_ERROR("Unsupported name_max (%d > %d)", + superblock.name_max, lfs->name_max); return LFS_ERR_INVAL; } - lfs->name_size = superblock.name_size; + lfs->name_max = superblock.name_max; } - if (superblock.inline_size) { - if (superblock.inline_size > lfs->inline_size) { - LFS_ERROR("Unsupported inline size (%d > %d)", - superblock.inline_size, lfs->inline_size); + if (superblock.inline_max) { + if (superblock.inline_max > lfs->inline_max) { + LFS_ERROR("Unsupported inline_max (%d > %d)", + superblock.inline_max, lfs->inline_max); return LFS_ERR_INVAL; } - lfs->inline_size = superblock.inline_size; + lfs->inline_max = superblock.inline_max; } // scan for any global updates @@ -3265,7 +3265,7 @@ lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, int32_t res = lfs_dir_get(lfs, &superdir, 0x7ffff000, LFS_MKTAG(0x100 | type, 0, - lfs_min(size, lfs->attr_size)), buffer); + lfs_min(size, lfs->attr_max)), buffer); if (res < 0) { return res; } @@ -3275,7 +3275,7 @@ lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, int lfs_fs_setattr(lfs_t *lfs, uint8_t type, const void *buffer, lfs_size_t size) { - if (size > lfs->attr_size) { + if (size > lfs->attr_max) { return LFS_ERR_NOSPC; } diff --git a/lfs.h b/lfs.h index 7635f3b0..e7889648 100644 --- a/lfs.h +++ b/lfs.h @@ -217,20 +217,20 @@ struct lfs_config { // attributes size but must be less than LFS_ATTR_MAX. Defaults to // LFS_ATTR_MAX when zero.Stored in superblock and must be respected by // other littlefs drivers. - lfs_size_t attr_size; + lfs_size_t attr_max; // Optional upper limit on length of file names in bytes. No downside for // larger names except the size of the info struct which is controlled by // the LFS_NAME_MAX define. Defaults to LFS_NAME_MAX when zero. Stored in // superblock and must be respected by other littlefs drivers. - lfs_size_t name_size; + lfs_size_t name_max; // Optional upper limit on inlined files in bytes. Large inline files // require a larger cache size, but if a file can be inlined it does not // need its own data block. Must be smaller than cache_size and less than // LFS_INLINE_MAX. Defaults to min(LFS_INLINE_MAX, read_size) when zero. // Stored in superblock and must be respected by other littlefs drivers. - lfs_size_t inline_size; + lfs_size_t inline_max; }; // File info structure @@ -361,9 +361,9 @@ typedef struct lfs_superblock { lfs_size_t block_size; lfs_size_t block_count; - lfs_size_t attr_size; - lfs_size_t name_size; - lfs_size_t inline_size; + lfs_size_t attr_max; + lfs_size_t name_max; + lfs_size_t inline_max; } lfs_superblock_t; typedef struct lfs_free { @@ -389,9 +389,9 @@ typedef struct lfs { const struct lfs_config *cfg; lfs_size_t block_size; lfs_size_t block_count; - lfs_size_t attr_size; - lfs_size_t name_size; - lfs_size_t inline_size; + lfs_size_t attr_max; + lfs_size_t name_max; + lfs_size_t inline_max; } lfs_t; diff --git a/tests/template.fmt b/tests/template.fmt index 7f136c8c..c745d24c 100644 --- a/tests/template.fmt +++ b/tests/template.fmt @@ -92,8 +92,6 @@ const struct lfs_config cfg = {{ .erase = &lfs_emubd_erase, .sync = &lfs_emubd_sync, - .name_size = 255, - .read_size = LFS_READ_SIZE, .prog_size = LFS_PROG_SIZE, .cache_size = LFS_CACHE_SIZE, From 3186e89b14acdbd162911bc6af89ab94489a2a0a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 4 Aug 2018 20:14:34 -0500 Subject: [PATCH 083/139] Changed littlefs-fuse target for testing purposes This is a downside caused by relying on and external repo for testing, but also storing the CI configuration inside this repo. Fortunately we can use a temporary v2-alpha branch in the FUSE repo mirroring the v2-alpha branch for testing. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a3d18aa7..b2caee86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,7 +103,7 @@ jobs: - NAME=littlefs-fuse install: - sudo apt-get install libfuse-dev - - git clone --depth 1 https://github.com/geky/littlefs-fuse + - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha - fusermount -V - gcc --version before_script: From c3e36bd2a797ce2789ae9e84b8bd2bc7d008b39c Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 4 Aug 2018 23:57:43 -0500 Subject: [PATCH 084/139] Standardized naming for internal functions - lfs_pairblah -> lfs_pair_blah - lfs_ctzblah -> lfs_ctz_blah - lfs_tagblah -> lfs_tag_blah - lfs_globalblah -> lfs_global_blah - lfs_commitblah -> lfs_commit_blah --- lfs.c | 458 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 228 insertions(+), 230 deletions(-) diff --git a/lfs.c b/lfs.c index 4eb4821b..c7bcc04a 100644 --- a/lfs.c +++ b/lfs.c @@ -333,46 +333,46 @@ static void lfs_alloc_ack(lfs_t *lfs) { /// Metadata pair and directory operations /// -static inline void lfs_pairswap(lfs_block_t pair[2]) { +static inline void lfs_pair_swap(lfs_block_t pair[2]) { lfs_block_t t = pair[0]; pair[0] = pair[1]; pair[1] = t; } -static inline bool lfs_pairisnull(const lfs_block_t pair[2]) { +static inline bool lfs_pair_isnull(const lfs_block_t pair[2]) { return pair[0] == 0xffffffff || pair[1] == 0xffffffff; } -static inline int lfs_paircmp( +static inline int lfs_pair_cmp( const lfs_block_t paira[2], const lfs_block_t pairb[2]) { return !(paira[0] == pairb[0] || paira[1] == pairb[1] || paira[0] == pairb[1] || paira[1] == pairb[0]); } -static inline bool lfs_pairsync( +static inline bool lfs_pair_sync( const lfs_block_t paira[2], const lfs_block_t pairb[2]) { return (paira[0] == pairb[0] && paira[1] == pairb[1]) || (paira[0] == pairb[1] && paira[1] == pairb[0]); } -static inline void lfs_pairfromle32(lfs_block_t *pair) { +static inline void lfs_pair_fromle32(lfs_block_t pair[2]) { pair[0] = lfs_fromle32(pair[0]); pair[1] = lfs_fromle32(pair[1]); } -static inline void lfs_pairtole32(lfs_block_t *pair) { +static inline void lfs_pair_tole32(lfs_block_t pair[2]) { pair[0] = lfs_tole32(pair[0]); pair[1] = lfs_tole32(pair[1]); } -static void lfs_ctzfromle32(struct lfs_ctz *ctz) { +static void lfs_ctz_fromle32(struct lfs_ctz *ctz) { ctz->head = lfs_fromle32(ctz->head); ctz->size = lfs_fromle32(ctz->size); } -static void lfs_ctztole32(struct lfs_ctz *ctz) { +static void lfs_ctz_tole32(struct lfs_ctz *ctz) { ctz->head = lfs_tole32(ctz->head); ctz->size = lfs_tole32(ctz->size); } @@ -385,38 +385,38 @@ static void lfs_ctztole32(struct lfs_ctz *ctz) { #define LFS_MKATTR(type, id, buffer, size, next) \ &(const lfs_mattr_t){LFS_MKTAG(type, id, size), (buffer), (next)} -static inline bool lfs_tagisvalid(uint32_t tag) { +static inline bool lfs_tag_isvalid(uint32_t tag) { return !(tag & 0x80000000); } -static inline bool lfs_tagisuser(uint32_t tag) { +static inline bool lfs_tag_isuser(uint32_t tag) { return (tag & 0x40000000); } -static inline uint16_t lfs_tagtype(uint32_t tag) { +static inline uint16_t lfs_tag_type(uint32_t tag) { return (tag & 0x7fc00000) >> 22; } -static inline uint16_t lfs_tagsubtype(uint32_t tag) { +static inline uint16_t lfs_tag_subtype(uint32_t tag) { return (tag & 0x7c000000) >> 22; } -static inline uint16_t lfs_tagid(uint32_t tag) { +static inline uint16_t lfs_tag_id(uint32_t tag) { return (tag & 0x003ff000) >> 12; } -static inline lfs_size_t lfs_tagsize(uint32_t tag) { +static inline lfs_size_t lfs_tag_size(uint32_t tag) { return tag & 0x00000fff; } // operations on set of globals -static inline void lfs_globalxor(lfs_global_t *a, const lfs_global_t *b) { +static inline void lfs_global_xor(lfs_global_t *a, const lfs_global_t *b) { for (int i = 0; i < sizeof(lfs_global_t)/4; i++) { a->u32[i] ^= b->u32[i]; } } -static inline bool lfs_globaliszero(const lfs_global_t *a) { +static inline bool lfs_global_iszero(const lfs_global_t *a) { for (int i = 0; i < sizeof(lfs_global_t)/4; i++) { if (a->u32[i] != 0) { return false; @@ -425,34 +425,34 @@ static inline bool lfs_globaliszero(const lfs_global_t *a) { return true; } -static inline void lfs_globalzero(lfs_global_t *a) { - lfs_globalxor(a, a); +static inline void lfs_global_zero(lfs_global_t *a) { + lfs_global_xor(a, a); } -static inline void lfs_globalfromle32(lfs_global_t *a) { - lfs_pairfromle32(a->s.movepair); +static inline void lfs_global_fromle32(lfs_global_t *a) { + lfs_pair_fromle32(a->s.movepair); a->s.moveid = lfs_fromle16(a->s.moveid); } -static inline void lfs_globaltole32(lfs_global_t *a) { - lfs_pairtole32(a->s.movepair); +static inline void lfs_global_tole32(lfs_global_t *a) { + lfs_pair_tole32(a->s.movepair); a->s.moveid = lfs_tole16(a->s.moveid); } -static inline void lfs_globalmove(lfs_t *lfs, +static inline void lfs_global_move(lfs_t *lfs, const lfs_block_t pair[2], uint16_t id) { lfs_global_t diff; - lfs_globalzero(&diff); + lfs_global_zero(&diff); diff.s.movepair[0] ^= lfs->globals.s.movepair[0] ^ pair[0]; diff.s.movepair[1] ^= lfs->globals.s.movepair[1] ^ pair[1]; diff.s.moveid ^= lfs->globals.s.moveid ^ id; - lfs_globalfromle32(&lfs->locals); - lfs_globalxor(&lfs->locals, &diff); - lfs_globaltole32(&lfs->locals); - lfs_globalxor(&lfs->globals, &diff); + lfs_global_fromle32(&lfs->locals); + lfs_global_xor(&lfs->locals, &diff); + lfs_global_tole32(&lfs->locals); + lfs_global_xor(&lfs->globals, &diff); } -static inline void lfs_globaldeorphaned(lfs_t *lfs, bool deorphaned) { +static inline void lfs_global_deorphaned(lfs_t *lfs, bool deorphaned) { lfs->locals.s.deorphaned ^= lfs->globals.s.deorphaned ^ deorphaned; lfs->globals.s.deorphaned ^= lfs->globals.s.deorphaned ^ deorphaned; } @@ -474,23 +474,23 @@ struct lfs_diskoff { lfs_off_t off; }; -static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, +static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, uint32_t tag, uint32_t getmask, uint32_t gettag, int32_t getdiff, void *buffer, bool stopatcommit) { // iterate over dir block backwards (for faster lookups) - while (off >= 2*sizeof(tag)+lfs_tagsize(tag)) { - off -= sizeof(tag)+lfs_tagsize(tag); + while (off >= 2*sizeof(tag)+lfs_tag_size(tag)) { + off -= sizeof(tag)+lfs_tag_size(tag); - if (lfs_tagtype(tag) == LFS_TYPE_CRC && stopatcommit) { + if (lfs_tag_type(tag) == LFS_TYPE_CRC && stopatcommit) { break; - } else if (lfs_tagtype(tag) == LFS_TYPE_DELETE) { - if (lfs_tagid(tag) <= lfs_tagid(gettag + getdiff)) { + } else if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { + if (lfs_tag_id(tag) <= lfs_tag_id(gettag + getdiff)) { getdiff += LFS_MKTAG(0, 1, 0); } } else if ((tag & getmask) == ((gettag + getdiff) & getmask)) { if (buffer) { lfs_size_t diff = lfs_min( - lfs_tagsize(gettag), lfs_tagsize(tag)); + lfs_tag_size(gettag), lfs_tag_size(tag)); int err = lfs_bd_read(lfs, block, off+sizeof(tag), buffer, diff); if (err) { @@ -498,7 +498,7 @@ static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, } memset((uint8_t*)buffer + diff, 0, - lfs_tagsize(gettag) - diff); + lfs_tag_size(gettag) - diff); } return tag - getdiff; @@ -515,28 +515,28 @@ static int32_t lfs_commitget(lfs_t *lfs, lfs_block_t block, lfs_off_t off, return LFS_ERR_NOENT; } -static int lfs_commitattrs(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, uint16_t id, const struct lfs_attr *attrs); -static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, const lfs_mdir_t *dir, const lfs_mattr_t *attrs); -static int lfs_commitattr(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, uint32_t tag, const void *buffer) { - if (lfs_tagtype(tag) == LFS_FROM_ATTRS) { + if (lfs_tag_type(tag) == LFS_FROM_ATTRS) { // special case for custom attributes - return lfs_commitattrs(lfs, commit, - lfs_tagid(tag), buffer); - } else if (lfs_tagtype(tag) == LFS_FROM_MOVE) { + return lfs_commit_attrs(lfs, commit, + lfs_tag_id(tag), buffer); + } else if (lfs_tag_type(tag) == LFS_FROM_MOVE) { // special case for moves - return lfs_commitmove(lfs, commit, - lfs_tagsize(tag), lfs_tagid(tag), + return lfs_commit_move(lfs, commit, + lfs_tag_size(tag), lfs_tag_id(tag), buffer, NULL); } // check if we fit - lfs_size_t size = lfs_tagsize(tag); + lfs_size_t size = lfs_tag_size(tag); if (commit->off + sizeof(tag)+size > commit->end) { return LFS_ERR_NOSPC; } @@ -582,10 +582,10 @@ static int lfs_commitattr(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commitattrs(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, uint16_t id, const struct lfs_attr *attrs) { for (const struct lfs_attr *a = attrs; a; a = a->next) { - int err = lfs_commitattr(lfs, commit, + int err = lfs_commit_attr(lfs, commit, LFS_MKTAG(0x100 | a->type, id, a->size), a->buffer); if (err) { return err; @@ -595,7 +595,7 @@ static int lfs_commitattrs(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, uint16_t fromid, uint16_t toid, const lfs_mdir_t *dir, const lfs_mattr_t *attrs) { // iterate through list and commits, only committing unique entries @@ -610,8 +610,8 @@ static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, buffer = attrs->buffer; attrs = attrs->next; } else { - LFS_ASSERT(off > sizeof(ntag)+lfs_tagsize(ntag)); - off -= sizeof(ntag)+lfs_tagsize(ntag); + LFS_ASSERT(off > sizeof(ntag)+lfs_tag_size(ntag)); + off -= sizeof(ntag)+lfs_tag_size(ntag); tag = ntag; buffer = &disk; @@ -628,17 +628,17 @@ static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, tag |= 0x80000000; } - if (lfs_tagtype(tag) == LFS_TYPE_DELETE && lfs_tagid(tag) <= fromid) { + if (lfs_tag_type(tag) == LFS_TYPE_DELETE && lfs_tag_id(tag) <= fromid) { // something was deleted, we need to move around it fromid += 1; - } else if (lfs_tagid(tag) != fromid) { + } else if (lfs_tag_id(tag) != fromid) { // ignore non-matching ids } else { // check if type has already been committed - int32_t res = lfs_commitget(lfs, commit->block, + int32_t res = lfs_commit_get(lfs, commit->block, commit->off, commit->ptag, - lfs_tagisuser(tag) ? 0x7ffff000 : 0x7c3ff000, - LFS_MKTAG(lfs_tagtype(tag), toid, 0), + lfs_tag_isuser(tag) ? 0x7ffff000 : 0x7c3ff000, + LFS_MKTAG(lfs_tag_type(tag), toid, 0), 0, NULL, true); if (res < 0 && res != LFS_ERR_NOENT) { return res; @@ -646,7 +646,7 @@ static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, if (res == LFS_ERR_NOENT) { // update id and commit, as we are currently unique - int err = lfs_commitattr(lfs, commit, + int err = lfs_commit_attr(lfs, commit, (tag & 0xffc00fff) | LFS_MKTAG(0, toid, 0), buffer); if (err) { @@ -659,21 +659,21 @@ static int lfs_commitmove(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commitglobals(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, lfs_global_t *locals) { - if (lfs_globaliszero(&lfs->locals)) { + if (lfs_global_iszero(&lfs->locals)) { return 0; } - lfs_globalxor(locals, &lfs->locals); - int err = lfs_commitattr(lfs, commit, + lfs_global_xor(locals, &lfs->locals); + int err = lfs_commit_attr(lfs, commit, LFS_MKTAG(LFS_TYPE_GLOBALS + locals->s.deorphaned, 0x3ff, sizeof(lfs_global_t)), locals); - lfs_globalxor(locals, &lfs->locals); + lfs_global_xor(locals, &lfs->locals); return err; } -static int lfs_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { +static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // align to program units lfs_off_t off = lfs_alignup(commit->off + 2*sizeof(uint32_t), lfs->cfg->prog_size); @@ -700,7 +700,7 @@ static int lfs_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { if (err) { return err; } - commit->off += sizeof(tag)+lfs_tagsize(tag); + commit->off += sizeof(tag)+lfs_tag_size(tag); commit->ptag = tag; // flush buffers @@ -712,7 +712,7 @@ static int lfs_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { // successful commit, check checksum to make sure uint32_t crc = 0xffffffff; err = lfs_bd_crc(lfs, commit->block, commit->begin, - commit->off-lfs_tagsize(tag)-commit->begin, &crc); + commit->off-lfs_tag_size(tag)-commit->begin, &crc); if (err) { return err; } @@ -750,7 +750,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { dir->tail[1] = 0xffffffff; dir->erased = false; dir->split = false; - lfs_globalzero(&dir->locals); + lfs_global_zero(&dir->locals); // don't write out yet, let caller take care of that return 0; @@ -780,8 +780,8 @@ static int32_t lfs_dir_find(lfs_t *lfs, } if (lfs_scmp(rev[1], rev[0]) > 0) { - lfs_pairswap(dir->pair); - lfs_pairswap(rev); + lfs_pair_swap(dir->pair); + lfs_pair_swap(rev); } // load blocks and check crc @@ -800,7 +800,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, lfs_block_t temptail[2] = {0xffffffff, 0xffffffff}; bool tempsplit = false; lfs_global_t templocals; - lfs_globalzero(&templocals); + lfs_global_zero(&templocals); while (true) { // extract next tag @@ -820,18 +820,18 @@ static int32_t lfs_dir_find(lfs_t *lfs, tag = lfs_fromle32(tag) ^ ptag; // next commit not yet programmed - if (lfs_tagtype(ptag) == LFS_TYPE_CRC && !lfs_tagisvalid(tag)) { + if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_isvalid(tag)) { dir->erased = true; break; } // check we're in valid range - if (off + sizeof(tag)+lfs_tagsize(tag) > lfs->cfg->block_size) { + if (off + sizeof(tag)+lfs_tag_size(tag) > lfs->cfg->block_size) { dir->erased = false; break; } - if (lfs_tagtype(tag) == LFS_TYPE_CRC) { + if (lfs_tag_type(tag) == LFS_TYPE_CRC) { // check the crc attr uint32_t dcrc; err = lfs_bd_read(lfs, dir->pair[0], @@ -851,7 +851,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, } foundtag = tempfoundtag; - dir->off = off + sizeof(tag)+lfs_tagsize(tag); + dir->off = off + sizeof(tag)+lfs_tag_size(tag); dir->etag = tag; dir->count = tempcount; dir->tail[0] = temptail[0]; @@ -861,7 +861,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, crc = 0xffffffff; } else { err = lfs_bd_crc(lfs, dir->pair[0], - off+sizeof(tag), lfs_tagsize(tag), &crc); + off+sizeof(tag), lfs_tag_size(tag), &crc); if (err) { if (err == LFS_ERR_CORRUPT) { dir->erased = false; @@ -869,12 +869,12 @@ static int32_t lfs_dir_find(lfs_t *lfs, } } - if (lfs_tagid(tag) < 0x3ff && lfs_tagid(tag) >= tempcount) { - tempcount = lfs_tagid(tag)+1; + if (lfs_tag_id(tag) < 0x3ff && lfs_tag_id(tag) >= tempcount) { + tempcount = lfs_tag_id(tag)+1; } - if (lfs_tagsubtype(tag) == LFS_TYPE_TAIL) { - tempsplit = (lfs_tagtype(tag) & 1); + if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { + tempsplit = (lfs_tag_type(tag) & 1); err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), temptail, sizeof(temptail)); if (err) { @@ -883,9 +883,9 @@ static int32_t lfs_dir_find(lfs_t *lfs, break; } } - lfs_pairfromle32(temptail); - } else if (lfs_tagsubtype(tag) == LFS_TYPE_GLOBALS) { - templocals.s.deorphaned = (lfs_tagtype(tag) & 1); + lfs_pair_fromle32(temptail); + } else if (lfs_tag_subtype(tag) == LFS_TYPE_GLOBALS) { + templocals.s.deorphaned = (lfs_tag_type(tag) & 1); err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), &templocals, sizeof(templocals)); if (err) { @@ -894,19 +894,19 @@ static int32_t lfs_dir_find(lfs_t *lfs, break; } } - } else if (lfs_tagsubtype(tag) == LFS_TYPE_DELETE) { + } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { LFS_ASSERT(tempcount > 0); tempcount -= 1; - if (lfs_tagid(tag) == lfs_tagid(tempfoundtag)) { + if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { tempfoundtag = LFS_ERR_NOENT; - } else if (lfs_tagisvalid(tempfoundtag) && - lfs_tagid(tag) < lfs_tagid(tempfoundtag)) { + } else if (lfs_tag_isvalid(tempfoundtag) && + lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { tempfoundtag -= LFS_MKTAG(0, 1, 0); } } else if ((tag & findmask) == (findtag & findmask)) { int res = lfs_bd_cmp(lfs, dir->pair[0], off+sizeof(tag), - findbuffer, lfs_tagsize(tag)); + findbuffer, lfs_tag_size(tag)); if (res < 0) { if (res == LFS_ERR_CORRUPT) { dir->erased = false; @@ -923,17 +923,17 @@ static int32_t lfs_dir_find(lfs_t *lfs, } ptag = tag; - off += sizeof(tag)+lfs_tagsize(tag); + off += sizeof(tag)+lfs_tag_size(tag); } // consider what we have good enough if (dir->off > 0) { // synthetic move - if (lfs_paircmp(dir->pair, lfs->globals.s.movepair) == 0) { - if (lfs->globals.s.moveid == lfs_tagid(foundtag)) { + if (lfs_pair_cmp(dir->pair, lfs->globals.s.movepair) == 0) { + if (lfs->globals.s.moveid == lfs_tag_id(foundtag)) { foundtag = LFS_ERR_NOENT; - } else if (lfs_tagisvalid(foundtag) && - lfs->globals.s.moveid < lfs_tagid(foundtag)) { + } else if (lfs_tag_isvalid(foundtag) && + lfs->globals.s.moveid < lfs_tag_id(foundtag)) { foundtag -= LFS_MKTAG(0, 1, 0); } } @@ -942,8 +942,8 @@ static int32_t lfs_dir_find(lfs_t *lfs, } // failed, try the other crc? - lfs_pairswap(dir->pair); - lfs_pairswap(rev); + lfs_pair_swap(dir->pair); + lfs_pair_swap(rev); } LFS_ERROR("Corrupted dir pair at %"PRIu32" %"PRIu32, @@ -964,13 +964,13 @@ static int lfs_dir_fetch(lfs_t *lfs, static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, uint32_t gettag, void *buffer) { int32_t getdiff = 0; - if (lfs_paircmp(dir->pair, lfs->globals.s.movepair) == 0 && - lfs_tagid(gettag) <= lfs->globals.s.moveid) { + if (lfs_pair_cmp(dir->pair, lfs->globals.s.movepair) == 0 && + lfs_tag_id(gettag) <= lfs->globals.s.moveid) { // synthetic moves getdiff = LFS_MKTAG(0, 1, 0); } - return lfs_commitget(lfs, dir->pair[0], dir->off, dir->etag, + return lfs_commit_get(lfs, dir->pair[0], dir->off, dir->etag, getmask, gettag, getdiff, buffer, false); } @@ -983,8 +983,8 @@ static int lfs_dir_compact(lfs_t *lfs, // There's nothing special about our global delta, so feed it back // into the global global delta - lfs_globalxor(&lfs->locals, &dir->locals); - lfs_globalzero(&dir->locals); + lfs_global_xor(&lfs->locals, &dir->locals); + lfs_global_zero(&dir->locals); // increment revision count dir->rev += 1; @@ -1033,7 +1033,7 @@ static int lfs_dir_compact(lfs_t *lfs, // commit with a move for (uint16_t id = begin; id < end; id++) { - err = lfs_commitmove(lfs, &commit, + err = lfs_commit_move(lfs, &commit, id, id - begin, source, attrs); if (err) { if (err == LFS_ERR_NOSPC) { @@ -1051,7 +1051,7 @@ static int lfs_dir_compact(lfs_t *lfs, commit.end = lfs->cfg->block_size - 8; if (!relocated) { - err = lfs_commitglobals(lfs, &commit, &dir->locals); + err = lfs_commit_globals(lfs, &commit, &dir->locals); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1060,13 +1060,13 @@ static int lfs_dir_compact(lfs_t *lfs, } } - if (!lfs_pairisnull(dir->tail)) { + if (!lfs_pair_isnull(dir->tail)) { // commit tail, which may be new after last size check - lfs_pairtole32(dir->tail); - err = lfs_commitattr(lfs, &commit, + lfs_pair_tole32(dir->tail); + err = lfs_commit_attr(lfs, &commit, LFS_MKTAG(LFS_TYPE_TAIL + dir->split, 0x3ff, sizeof(dir->tail)), dir->tail); - lfs_pairfromle32(dir->tail); + lfs_pair_fromle32(dir->tail); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1075,7 +1075,7 @@ static int lfs_dir_compact(lfs_t *lfs, } } - err = lfs_commitcrc(lfs, &commit); + err = lfs_commit_crc(lfs, &commit); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1084,7 +1084,7 @@ static int lfs_dir_compact(lfs_t *lfs, } // successful compaction, swap dir pair to indicate most recent - lfs_pairswap(dir->pair); + lfs_pair_swap(dir->pair); dir->off = commit.off; dir->etag = commit.ptag; dir->erased = true; @@ -1132,7 +1132,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_cache_drop(lfs, &lfs->pcache); // can't relocate superblock, filesystem is now frozen - if (lfs_paircmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { + if (lfs_pair_cmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { LFS_WARN("Superblock %"PRIu32" has become unwritable", oldpair[1]); return LFS_ERR_CORRUPT; } @@ -1148,8 +1148,8 @@ static int lfs_dir_compact(lfs_t *lfs, if (!relocated) { // successful commit, update globals - lfs_globalxor(&dir->locals, &lfs->locals); - lfs_globalzero(&lfs->locals); + lfs_global_xor(&dir->locals, &lfs->locals); + lfs_global_zero(&lfs->locals); } else { // update references if we relocated LFS_DEBUG("Relocating %"PRIu32" %"PRIu32" to %"PRIu32" %"PRIu32, @@ -1167,16 +1167,16 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mattr_t *attrs) { lfs_mattr_t cancelattr; lfs_global_t canceldiff; - lfs_globalzero(&canceldiff); - if (lfs_paircmp(dir->pair, lfs->globals.s.movepair) == 0) { + lfs_global_zero(&canceldiff); + if (lfs_pair_cmp(dir->pair, lfs->globals.s.movepair) == 0) { // Wait, we have the move? Just cancel this out here // We need to, or else the move can become outdated canceldiff.s.movepair[0] ^= lfs->globals.s.movepair[0] ^ 0xffffffff; canceldiff.s.movepair[1] ^= lfs->globals.s.movepair[1] ^ 0xffffffff; canceldiff.s.moveid ^= lfs->globals.s.moveid ^ 0x3ff; - lfs_globalfromle32(&lfs->locals); - lfs_globalxor(&lfs->locals, &canceldiff); - lfs_globaltole32(&lfs->locals); + lfs_global_fromle32(&lfs->locals); + lfs_global_xor(&lfs->locals, &canceldiff); + lfs_global_tole32(&lfs->locals); cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.s.moveid, 0); cancelattr.next = attrs; @@ -1186,11 +1186,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // calculate new directory size uint32_t deletetag = 0xffffffff; for (const lfs_mattr_t *a = attrs; a; a = a->next) { - if (lfs_tagid(a->tag) < 0x3ff && lfs_tagid(a->tag) >= dir->count) { - dir->count = lfs_tagid(a->tag)+1; + if (lfs_tag_id(a->tag) < 0x3ff && lfs_tag_id(a->tag) >= dir->count) { + dir->count = lfs_tag_id(a->tag)+1; } - if (lfs_tagtype(a->tag) == LFS_TYPE_DELETE) { + if (lfs_tag_type(a->tag) == LFS_TYPE_DELETE) { LFS_ASSERT(dir->count > 0); dir->count -= 1; deletetag = a->tag; @@ -1208,7 +1208,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, pdir.split = dir->split; pdir.tail[0] = dir->tail[0]; pdir.tail[1] = dir->tail[1]; - lfs_globalxor(&lfs->locals, &dir->locals); + lfs_global_xor(&lfs->locals, &dir->locals); return lfs_dir_commit(lfs, &pdir, LFS_MKATTR(LFS_TYPE_TAIL + pdir.split, 0x3ff, pdir.tail, sizeof(pdir.tail), @@ -1231,10 +1231,10 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, }; for (const lfs_mattr_t *a = attrs; a; a = a->next) { - if (lfs_tagtype(a->tag) != LFS_TYPE_DELETE) { - lfs_pairtole32(dir->tail); - int err = lfs_commitattr(lfs, &commit, a->tag, a->buffer); - lfs_pairfromle32(dir->tail); + if (lfs_tag_type(a->tag) != LFS_TYPE_DELETE) { + lfs_pair_tole32(dir->tail); + int err = lfs_commit_attr(lfs, &commit, a->tag, a->buffer); + lfs_pair_fromle32(dir->tail); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1244,9 +1244,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } } - if (lfs_tagisvalid(deletetag)) { + if (lfs_tag_isvalid(deletetag)) { // special case for deletes, since order matters - int err = lfs_commitattr(lfs, &commit, deletetag, NULL); + int err = lfs_commit_attr(lfs, &commit, deletetag, NULL); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1255,7 +1255,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } } - int err = lfs_commitglobals(lfs, &commit, &dir->locals); + int err = lfs_commit_globals(lfs, &commit, &dir->locals); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1263,7 +1263,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, return err; } - err = lfs_commitcrc(lfs, &commit); + err = lfs_commit_crc(lfs, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1275,8 +1275,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, dir->off = commit.off; dir->etag = commit.ptag; // successful commit, update globals - lfs_globalxor(&dir->locals, &lfs->locals); - lfs_globalzero(&lfs->locals); + lfs_global_xor(&dir->locals, &lfs->locals); + lfs_global_zero(&lfs->locals); } else { compact: // fall back to compaction @@ -1288,16 +1288,16 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } // update globals that are affected - lfs_globalxor(&lfs->globals, &canceldiff); + lfs_global_xor(&lfs->globals, &canceldiff); // update any directories that are affected for (lfs_mlist_t *d = lfs->mlist; d; d = d->next) { - if (lfs_paircmp(d->m.pair, dir->pair) == 0) { + if (lfs_pair_cmp(d->m.pair, dir->pair) == 0) { d->m = *dir; - if (d->id == lfs_tagid(deletetag)) { + if (d->id == lfs_tag_id(deletetag)) { d->m.pair[0] = 0xffffffff; d->m.pair[1] = 0xffffffff; - } else if (d->id > lfs_tagid(deletetag)) { + } else if (d->id > lfs_tag_id(deletetag)) { d->id -= 1; if (d->type == LFS_TYPE_DIR) { ((lfs_dir_t*)d)->pos -= 1; @@ -1375,18 +1375,18 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { } // only continue if we hit a directory - if (lfs_tagtype(tag) != LFS_TYPE_DIR) { + if (lfs_tag_type(tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } // grab the entry data - if (lfs_tagid(tag) != 0x3ff) { + if (lfs_tag_id(tag) != 0x3ff) { int32_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), pair); + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; } - lfs_pairfromle32(pair); + lfs_pair_fromle32(pair); } // find entry matching name @@ -1431,7 +1431,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, return tag; } - info->type = lfs_tagtype(tag); + info->type = lfs_tag_type(tag); struct lfs_ctz ctz; tag = lfs_dir_get(lfs, dir, 0x7c3ff000, @@ -1439,12 +1439,12 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, if (tag < 0) { return tag; } - lfs_ctzfromle32(&ctz); + lfs_ctz_fromle32(&ctz); - if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { + if (lfs_tag_type(tag) == LFS_TYPE_CTZSTRUCT) { info->size = ctz.size; - } else if (lfs_tagtype(tag) == LFS_TYPE_INLINESTRUCT) { - info->size = lfs_tagsize(tag); + } else if (lfs_tag_type(tag) == LFS_TYPE_INLINESTRUCT) { + info->size = lfs_tag_size(tag); } return 0; @@ -1490,13 +1490,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { uint16_t id = cwd.count; cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; - lfs_pairtole32(dir.pair); + lfs_pair_tole32(dir.pair); err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), NULL)))); - lfs_pairfromle32(dir.pair); + lfs_pair_fromle32(dir.pair); if (err) { return err; } @@ -1510,23 +1510,23 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { return tag; } - if (lfs_tagtype(tag) != LFS_TYPE_DIR) { + if (lfs_tag_type(tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } lfs_block_t pair[2]; - if (lfs_tagid(tag) == 0x3ff) { + if (lfs_tag_id(tag) == 0x3ff) { // handle root dir separately pair[0] = lfs->root[0]; pair[1] = lfs->root[1]; } else { // get dir pair from parent int32_t res = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), pair); + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; } - lfs_pairfromle32(pair); + lfs_pair_fromle32(pair); } // fetch first pair @@ -1658,7 +1658,7 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { /// File index list operations /// -static int lfs_ctzindex(lfs_t *lfs, lfs_off_t *off) { +static int lfs_ctz_index(lfs_t *lfs, lfs_off_t *off) { lfs_off_t size = *off; lfs_off_t b = lfs->cfg->block_size - 2*4; lfs_off_t i = size / b; @@ -1671,8 +1671,8 @@ static int lfs_ctzindex(lfs_t *lfs, lfs_off_t *off) { return i; } -static int lfs_ctzfind(lfs_t *lfs, - lfs_cache_t *rcache, const lfs_cache_t *pcache, +static int lfs_ctz_find(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_block_t head, lfs_size_t size, lfs_size_t pos, lfs_block_t *block, lfs_off_t *off) { if (size == 0) { @@ -1681,8 +1681,8 @@ static int lfs_ctzfind(lfs_t *lfs, return 0; } - lfs_off_t current = lfs_ctzindex(lfs, &(lfs_off_t){size-1}); - lfs_off_t target = lfs_ctzindex(lfs, &pos); + lfs_off_t current = lfs_ctz_index(lfs, &(lfs_off_t){size-1}); + lfs_off_t target = lfs_ctz_index(lfs, &pos); while (current > target) { lfs_size_t skip = lfs_min( @@ -1705,8 +1705,8 @@ static int lfs_ctzfind(lfs_t *lfs, return 0; } -static int lfs_ctzextend(lfs_t *lfs, - lfs_cache_t *rcache, lfs_cache_t *pcache, +static int lfs_ctz_extend(lfs_t *lfs, + lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_block_t head, lfs_size_t size, lfs_block_t *block, lfs_off_t *off) { while (true) { @@ -1734,7 +1734,7 @@ static int lfs_ctzextend(lfs_t *lfs, } size -= 1; - lfs_off_t index = lfs_ctzindex(lfs, &size); + lfs_off_t index = lfs_ctz_index(lfs, &size); size += 1; // just copy out the last block if it is incomplete @@ -1803,15 +1803,15 @@ static int lfs_ctzextend(lfs_t *lfs, } } -static int lfs_ctztraverse(lfs_t *lfs, - lfs_cache_t *rcache, const lfs_cache_t *pcache, +static int lfs_ctz_traverse(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_block_t head, lfs_size_t size, int (*cb)(void*, lfs_block_t), void *data) { if (size == 0) { return 0; } - lfs_off_t index = lfs_ctzindex(lfs, &(lfs_off_t){size-1}); + lfs_off_t index = lfs_ctz_index(lfs, &(lfs_off_t){size-1}); while (true) { int err = cb(data, head); @@ -1873,7 +1873,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } // get id, add to list of mdirs to catch update changes - file->id = lfs_tagid(tag); + file->id = lfs_tag_id(tag); file->type = LFS_TYPE_REG; file->next = (lfs_file_t*)lfs->mlist; lfs->mlist = (lfs_mlist_t*)file; @@ -1906,7 +1906,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } else if (flags & LFS_O_EXCL) { err = LFS_ERR_EXIST; goto cleanup; - } else if (lfs_tagtype(tag) != LFS_TYPE_REG) { + } else if (lfs_tag_type(tag) != LFS_TYPE_REG) { err = LFS_ERR_ISDIR; goto cleanup; } else if (flags & LFS_O_TRUNC) { @@ -1921,7 +1921,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, err = tag; goto cleanup; } - lfs_ctzfromle32(&file->ctz); + lfs_ctz_fromle32(&file->ctz); } // fetch attrs @@ -1959,10 +1959,10 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // zero to avoid information leak lfs_cache_zero(lfs, &file->cache); - if (lfs_tagtype(tag) == LFS_TYPE_INLINESTRUCT) { + if (lfs_tag_type(tag) == LFS_TYPE_INLINESTRUCT) { // load inline files file->ctz.head = 0xfffffffe; - file->ctz.size = lfs_tagsize(tag); + file->ctz.size = lfs_tag_size(tag); file->flags |= LFS_F_INLINE; file->cache.block = file->ctz.head; file->cache.off = 0; @@ -2150,7 +2150,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { if ((file->flags & LFS_F_DIRTY) && !(file->flags & LFS_F_ERRED) && - !lfs_pairisnull(file->m.pair)) { + !lfs_pair_isnull(file->m.pair)) { // update dir entry uint16_t type; const void *buffer; @@ -2168,12 +2168,12 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } // commit file data and attributes - lfs_ctztole32(&file->ctz); + lfs_ctz_tole32(&file->ctz); err = lfs_dir_commit(lfs, &file->m, LFS_MKATTR(type, file->id, buffer, size, LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, NULL))); - lfs_ctzfromle32(&file->ctz); + lfs_ctz_fromle32(&file->ctz); if (err) { if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) { goto relocate; @@ -2232,7 +2232,7 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { if (!(file->flags & LFS_F_INLINE)) { - int err = lfs_ctzfind(lfs, &file->cache, NULL, + int err = lfs_ctz_find(lfs, NULL, &file->cache, file->ctz.head, file->ctz.size, file->pos, &file->block, &file->off); if (err) { @@ -2321,7 +2321,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, if (!(file->flags & LFS_F_INLINE)) { if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { // find out which block we're extending from - int err = lfs_ctzfind(lfs, &file->cache, NULL, + int err = lfs_ctz_find(lfs, NULL, &file->cache, file->ctz.head, file->ctz.size, file->pos-1, &file->block, &file->off); if (err) { @@ -2335,7 +2335,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // extend file with new blocks lfs_alloc_ack(lfs); - int err = lfs_ctzextend(lfs, &lfs->rcache, &file->cache, + int err = lfs_ctz_extend(lfs, &file->cache, &lfs->rcache, file->block, file->pos, &file->block, &file->off); if (err) { @@ -2426,7 +2426,7 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { } // lookup new head in ctz skip list - err = lfs_ctzfind(lfs, &file->cache, NULL, + err = lfs_ctz_find(lfs, NULL, &file->cache, file->ctz.head, file->ctz.size, size, &file->ctz.head, &(lfs_off_t){0}); if (err) { @@ -2496,7 +2496,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { return tag; } - return lfs_dir_getinfo(lfs, &cwd, lfs_tagid(tag), info); + return lfs_dir_getinfo(lfs, &cwd, lfs_tag_id(tag), info); } int lfs_remove(lfs_t *lfs, const char *path) { @@ -2518,15 +2518,15 @@ int lfs_remove(lfs_t *lfs, const char *path) { } lfs_mdir_t dir; - if (lfs_tagtype(tag) == LFS_TYPE_DIR) { + if (lfs_tag_type(tag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t pair[2]; int32_t res = lfs_dir_get(lfs, &cwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tagid(tag), 8), pair); + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; } - lfs_pairfromle32(pair); + lfs_pair_fromle32(pair); err = lfs_dir_fetch(lfs, &dir, pair); if (err) { @@ -2538,30 +2538,30 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // mark fs as orphaned - lfs_globaldeorphaned(lfs, false); + lfs_global_deorphaned(lfs, false); } // delete the entry err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_DELETE, lfs_tagid(tag), NULL, 0, + LFS_MKATTR(LFS_TYPE_DELETE, lfs_tag_id(tag), NULL, 0, NULL)); if (err) { return err; } - if (lfs_tagtype(tag) == LFS_TYPE_DIR) { + if (lfs_tag_type(tag) == LFS_TYPE_DIR) { err = lfs_fs_pred(lfs, dir.pair, &cwd); if (err) { return err; } // fix orphan - lfs_globaldeorphaned(lfs, true); + lfs_global_deorphaned(lfs, true); // steal state cwd.tail[0] = dir.tail[0]; cwd.tail[1] = dir.tail[1]; - lfs_globalxor(&lfs->locals, &dir.locals); + lfs_global_xor(&lfs->locals, &dir.locals); err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), @@ -2595,18 +2595,16 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return prevtag; } - uint16_t newid = lfs_tagid(prevtag); - //bool prevexists = (prevtag != LFS_ERR_NOENT); - //bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); + uint16_t newid = lfs_tag_id(prevtag); lfs_mdir_t prevdir; if (prevtag != LFS_ERR_NOENT) { // check that we have same type - if (lfs_tagtype(prevtag) != lfs_tagtype(oldtag)) { + if (lfs_tag_type(prevtag) != lfs_tag_type(oldtag)) { return LFS_ERR_ISDIR; } - if (lfs_tagtype(prevtag) == LFS_TYPE_DIR) { + if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t prevpair[2]; int32_t res = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, @@ -2614,7 +2612,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (res < 0) { return res; } - lfs_pairfromle32(prevpair); + lfs_pair_fromle32(prevpair); // must be empty before removal err = lfs_dir_fetch(lfs, &prevdir, prevpair); @@ -2627,7 +2625,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // mark fs as orphaned - lfs_globaldeorphaned(lfs, false); + lfs_global_deorphaned(lfs, false); } } else { // check that name fits @@ -2641,12 +2639,12 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // create move to fix later - lfs_globalmove(lfs, oldcwd.pair, lfs_tagid(oldtag)); + lfs_global_move(lfs, oldcwd.pair, lfs_tag_id(oldtag)); // move over all attributes err = lfs_dir_commit(lfs, &newcwd, - LFS_MKATTR(lfs_tagtype(oldtag), newid, newpath, strlen(newpath), - LFS_MKATTR(LFS_FROM_MOVE, newid, &oldcwd, lfs_tagid(oldtag), + LFS_MKATTR(lfs_tag_type(oldtag), newid, newpath, strlen(newpath), + LFS_MKATTR(LFS_FROM_MOVE, newid, &oldcwd, lfs_tag_id(oldtag), NULL))); if (err) { return err; @@ -2654,26 +2652,26 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // let commit clean up after move (if we're different! otherwise move // logic already fixed it for us) - if (lfs_paircmp(oldcwd.pair, newcwd.pair) != 0) { + if (lfs_pair_cmp(oldcwd.pair, newcwd.pair) != 0) { err = lfs_dir_commit(lfs, &oldcwd, NULL); if (err) { return err; } } - if (prevtag != LFS_ERR_NOENT && lfs_tagtype(prevtag) == LFS_TYPE_DIR) { + if (prevtag != LFS_ERR_NOENT && lfs_tag_type(prevtag) == LFS_TYPE_DIR) { err = lfs_fs_pred(lfs, prevdir.pair, &newcwd); if (err) { return err; } // fix orphan - lfs_globaldeorphaned(lfs, true); + lfs_global_deorphaned(lfs, true); // steal state newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; - lfs_globalxor(&lfs->locals, &prevdir.locals); + lfs_global_xor(&lfs->locals, &prevdir.locals); err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, newcwd.tail, sizeof(newcwd.tail), @@ -2695,13 +2693,13 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, } res = lfs_dir_get(lfs, &cwd, 0x7ffff000, - LFS_MKTAG(0x100 | type, lfs_tagid(res), + LFS_MKTAG(0x100 | type, lfs_tag_id(res), lfs_min(size, lfs->attr_max)), buffer); if (res < 0 && res != LFS_ERR_NOENT) { return res; } - return (res == LFS_ERR_NOENT) ? 0 : lfs_tagsize(res); + return (res == LFS_ERR_NOENT) ? 0 : lfs_tag_size(res); } int lfs_setattr(lfs_t *lfs, const char *path, @@ -2717,7 +2715,7 @@ int lfs_setattr(lfs_t *lfs, const char *path, } return lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(0x100 | type, lfs_tagid(res), buffer, size, + LFS_MKATTR(0x100 | type, lfs_tag_id(res), buffer, size, NULL)); } @@ -2822,7 +2820,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->globals.s.movepair[1] = 0xffffffff; lfs->globals.s.moveid = 0x3ff; lfs->globals.s.deorphaned = true; - lfs_globalzero(&lfs->locals); + lfs_global_zero(&lfs->locals); return 0; @@ -2898,12 +2896,12 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; lfs_superblocktole32(&superblock); - lfs_pairtole32(lfs->root); + lfs_pair_tole32(lfs->root); err = lfs_dir_commit(lfs, &dir, LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), LFS_MKATTR(LFS_TYPE_DIRSTRUCT, 0, lfs->root, sizeof(lfs->root), NULL))); - lfs_pairfromle32(lfs->root); + lfs_pair_fromle32(lfs->root); lfs_superblockfromle32(&superblock); if (err) { goto cleanup; @@ -2971,7 +2969,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { err = res; goto cleanup; } - lfs_pairfromle32(lfs->root); + lfs_pair_fromle32(lfs->root); // check superblock configuration if (superblock.attr_max) { @@ -3009,7 +3007,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // scan for any global updates lfs_mdir_t dir = {.tail = {0, 1}}; - while (!lfs_pairisnull(dir.tail)) { + while (!lfs_pair_isnull(dir.tail)) { err = lfs_dir_fetch(lfs, &dir, dir.tail); if (err) { err = LFS_ERR_INVAL; @@ -3017,14 +3015,14 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } // xor together indirect deletes - lfs_globalxor(&lfs->locals, &dir.locals); + lfs_global_xor(&lfs->locals, &dir.locals); } // update littlefs with globals - lfs_globalfromle32(&lfs->locals); - lfs_globalxor(&lfs->globals, &lfs->locals); - lfs_globalzero(&lfs->locals); - if (!lfs_pairisnull(lfs->globals.s.movepair)) { + lfs_global_fromle32(&lfs->locals); + lfs_global_xor(&lfs->globals, &lfs->locals); + lfs_global_zero(&lfs->locals); + if (!lfs_pair_isnull(lfs->globals.s.movepair)) { LFS_DEBUG("Found move %"PRIu32" %"PRIu32" %"PRIu32, lfs->globals.s.movepair[0], lfs->globals.s.movepair[1], @@ -3046,13 +3044,13 @@ int lfs_unmount(lfs_t *lfs) { /// Filesystem filesystem operations /// int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *data, lfs_block_t block), void *data) { - if (lfs_pairisnull(lfs->root)) { + if (lfs_pair_isnull(lfs->root)) { return 0; } // iterate over metadata pairs lfs_mdir_t dir = {.tail = {0, 1}}; - while (!lfs_pairisnull(dir.tail)) { + while (!lfs_pair_isnull(dir.tail)) { for (int i = 0; i < 2; i++) { int err = cb(data, dir.tail[i]); if (err) { @@ -3076,10 +3074,10 @@ int lfs_fs_traverse(lfs_t *lfs, } return tag; } - lfs_ctzfromle32(&ctz); + lfs_ctz_fromle32(&ctz); - if (lfs_tagtype(tag) == LFS_TYPE_CTZSTRUCT) { - err = lfs_ctztraverse(lfs, &lfs->rcache, NULL, + if (lfs_tag_type(tag) == LFS_TYPE_CTZSTRUCT) { + err = lfs_ctz_traverse(lfs, NULL, &lfs->rcache, ctz.head, ctz.size, cb, data); if (err) { return err; @@ -3095,7 +3093,7 @@ int lfs_fs_traverse(lfs_t *lfs, } if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { - int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, + int err = lfs_ctz_traverse(lfs, &f->cache, &lfs->rcache, f->ctz.head, f->ctz.size, cb, data); if (err) { return err; @@ -3103,7 +3101,7 @@ int lfs_fs_traverse(lfs_t *lfs, } if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { - int err = lfs_ctztraverse(lfs, &lfs->rcache, &f->cache, + int err = lfs_ctz_traverse(lfs, &f->cache, &lfs->rcache, f->block, f->pos, cb, data); if (err) { return err; @@ -3116,15 +3114,15 @@ int lfs_fs_traverse(lfs_t *lfs, static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { - if (lfs_pairisnull(lfs->root)) { + if (lfs_pair_isnull(lfs->root)) { return LFS_ERR_NOENT; } // iterate over all directory directory entries pdir->tail[0] = 0; pdir->tail[1] = 1; - while (!lfs_pairisnull(pdir->tail)) { - if (lfs_paircmp(pdir->tail, pair) == 0) { + while (!lfs_pair_isnull(pdir->tail)) { + if (lfs_pair_cmp(pdir->tail, pair) == 0) { return 0; } @@ -3139,18 +3137,18 @@ static int lfs_fs_pred(lfs_t *lfs, static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *parent) { - if (lfs_pairisnull(lfs->root)) { + if (lfs_pair_isnull(lfs->root)) { return LFS_ERR_NOENT; } // search for both orderings so we can reuse the find function lfs_block_t child[2] = {pair[0], pair[1]}; - lfs_pairtole32(child); + lfs_pair_tole32(child); for (int i = 0; i < 2; i++) { // iterate over all directory directory entries parent->tail[0] = 0; parent->tail[1] = 1; - while (!lfs_pairisnull(parent->tail)) { + while (!lfs_pair_isnull(parent->tail)) { int32_t tag = lfs_dir_find(lfs, parent, parent->tail, 0x7fc00fff, LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(child)), child); @@ -3159,7 +3157,7 @@ static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], } } - lfs_pairswap(child); + lfs_pair_swap(child); } return LFS_ERR_NOENT; @@ -3168,7 +3166,7 @@ static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]) { // update internal root - if (lfs_paircmp(oldpair, lfs->root) == 0) { + if (lfs_pair_cmp(oldpair, lfs->root) == 0) { LFS_DEBUG("Relocating root %"PRIu32" %"PRIu32, newpair[0], newpair[1]); lfs->root[0] = newpair[0]; @@ -3177,7 +3175,7 @@ static int lfs_fs_relocate(lfs_t *lfs, // update internally tracked dirs for (lfs_mlist_t *d = lfs->mlist; d; d = d->next) { - if (lfs_paircmp(oldpair, d->m.pair) == 0) { + if (lfs_pair_cmp(oldpair, d->m.pair) == 0) { d->m.pair[0] = newpair[0]; d->m.pair[1] = newpair[1]; } @@ -3192,10 +3190,10 @@ static int lfs_fs_relocate(lfs_t *lfs, if (tag != LFS_ERR_NOENT) { // update disk, this creates a desync - lfs_pairtole32(newpair); + lfs_pair_tole32(newpair); int err = lfs_dir_commit(lfs, &parent, &(lfs_mattr_t){.tag=tag, .buffer=newpair}); - lfs_pairfromle32(newpair); + lfs_pair_fromle32(newpair); if (err) { return err; } @@ -3237,7 +3235,7 @@ static int lfs_fs_forceconsistency(lfs_t *lfs) { lfs_mdir_t dir = {.tail = {0, 1}}; // iterate over all directory directory entries - while (!lfs_pairisnull(dir.tail)) { + while (!lfs_pair_isnull(dir.tail)) { int err = lfs_dir_fetch(lfs, &dir, dir.tail); if (err) { return err; @@ -3275,9 +3273,9 @@ static int lfs_fs_forceconsistency(lfs_t *lfs) { if (res < 0) { return res; } - lfs_pairfromle32(pair); + lfs_pair_fromle32(pair); - if (!lfs_pairsync(pair, pdir.tail)) { + if (!lfs_pair_sync(pair, pdir.tail)) { // we have desynced LFS_DEBUG("Fixing half-orphan %"PRIu32" %"PRIu32, pair[0], pair[1]); @@ -3300,7 +3298,7 @@ static int lfs_fs_forceconsistency(lfs_t *lfs) { } // mark orphan as fixed - lfs_globaldeorphaned(lfs, false); + lfs_global_deorphaned(lfs, false); } if (lfs->globals.s.moveid != 0x3ff) { @@ -3342,7 +3340,7 @@ lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, return res; } - return (res == LFS_ERR_NOENT) ? 0 : lfs_tagsize(res); + return (res == LFS_ERR_NOENT) ? 0 : lfs_tag_size(res); } int lfs_fs_setattr(lfs_t *lfs, From 7c70068b89a50e76f1329107a9b89d5ccfaf5ad2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 6 Aug 2018 13:30:51 -0500 Subject: [PATCH 085/139] Added root entry and expanding superblocks Expanding superblocks has been on my wishlist for a while. The basic idea is that instead of maintaining a fixed offset blocks {0, 1} to the the root directory (1 pointer), we maintain a dynamically sized linked-list of superblocks that point to the actual root. If the number of writes to the root exceeds some value, we increase the size of the superblock linked-list. This can leverage existing metadata-pair operations. The revision count for metadata-pairs provides some knowledge on how much wear we've put on the superblock, and the threaded linked-list can also be reused for this purpose. This means superblock expansion is both optional and cheap to implement. Expanding superblocks helps both extremely small and extremely large filesystem (extreme being relative of course). On the small end, we can actually collapse the superblock into the root directory and drop the hard requirement of 4-blocks for the superblock. On the large end, our superblock will now last longer than the rest of the filesystem. Each time we expand, the number of cycles until the superblock dies is increased by a power. Before we were stuck with this layout: level cycles limit layout 1 E^2 390 MiB s0 -> root Now we expand every time a fixed offset is exceeded: level cycles limit layout 0 E 4 KiB s0+root 1 E^2 390 MiB s0 -> root 2 E^3 37 TiB s0 -> s1 -> root 3 E^4 3.6 EiB s0 -> s1 -> s2 -> root ... Where the cycles are the number of cycles before death, and the limit is the worst-case size a filesystem where early superblock death becomes a concern (all writes to root using this formula: E^|s| = E*B, E = erase cycles = 100000, B = block count, assuming 4096 byte blocks). Note we can also store copies of the superblock entry on the expanded superblocks. This may help filesystem recover tools in the future. --- lfs.c | 208 +++++++++++++++++++++---------------------- lfs.h | 8 +- tests/test_alloc.sh | 18 ++-- tests/test_format.sh | 30 +++++-- tests/test_move.sh | 8 +- tests/test_orphan.sh | 10 +-- 6 files changed, 146 insertions(+), 136 deletions(-) diff --git a/lfs.c b/lfs.c index c7bcc04a..ee8a7a8f 100644 --- a/lfs.c +++ b/lfs.c @@ -377,6 +377,25 @@ static void lfs_ctz_tole32(struct lfs_ctz *ctz) { ctz->size = lfs_tole32(ctz->size); } +static inline void lfs_superblock_fromle32(lfs_superblock_t *superblock) { + superblock->version = lfs_fromle32(superblock->version); + superblock->block_size = lfs_fromle32(superblock->block_size); + superblock->block_count = lfs_fromle32(superblock->block_count); + superblock->inline_max = lfs_fromle32(superblock->inline_max); + superblock->attr_max = lfs_fromle32(superblock->attr_max); + superblock->name_max = lfs_fromle32(superblock->name_max); +} + +static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { + superblock->version = lfs_tole32(superblock->version); + superblock->block_size = lfs_tole32(superblock->block_size); + superblock->block_count = lfs_tole32(superblock->block_count); + superblock->inline_max = lfs_tole32(superblock->inline_max); + superblock->attr_max = lfs_tole32(superblock->attr_max); + superblock->name_max = lfs_tole32(superblock->name_max); +} + + /// Entry tag operations /// #define LFS_MKTAG(type, id, size) \ @@ -524,15 +543,15 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, uint32_t tag, const void *buffer) { - if (lfs_tag_type(tag) == LFS_FROM_ATTRS) { - // special case for custom attributes - return lfs_commit_attrs(lfs, commit, - lfs_tag_id(tag), buffer); - } else if (lfs_tag_type(tag) == LFS_FROM_MOVE) { + if (lfs_tag_subtype(tag) == LFS_FROM_MOVE) { // special case for moves return lfs_commit_move(lfs, commit, lfs_tag_size(tag), lfs_tag_id(tag), buffer, NULL); + } else if (lfs_tag_subtype(tag) == LFS_FROM_ATTRS) { + // special case for custom attributes + return lfs_commit_attrs(lfs, commit, + lfs_tag_id(tag), buffer); } // check if we fit @@ -628,7 +647,8 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, tag |= 0x80000000; } - if (lfs_tag_type(tag) == LFS_TYPE_DELETE && lfs_tag_id(tag) <= fromid) { + if (lfs_tag_type(tag) == LFS_TYPE_DELETE && + lfs_tag_id(tag) <= fromid) { // something was deleted, we need to move around it fromid += 1; } else if (lfs_tag_id(tag) != fromid) { @@ -667,8 +687,8 @@ static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, lfs_global_xor(locals, &lfs->locals); int err = lfs_commit_attr(lfs, commit, - LFS_MKTAG(LFS_TYPE_GLOBALS + locals->s.deorphaned, - 0x3ff, sizeof(lfs_global_t)), locals); + LFS_MKTAG(LFS_TYPE_GLOBALS + locals->s.deorphaned, 0x3ff, 10), + locals); lfs_global_xor(locals, &lfs->locals); return err; } @@ -726,7 +746,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // internal dir operations static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { - // allocate pair of dir blocks (backwards, so we write to block 1 first) + // allocate pair of dir blocks (backwards, so we write block 1 first) for (int i = 0; i < 2; i++) { int err = lfs_alloc(lfs, &dir->pair[(i+1)%2]); if (err) { @@ -756,10 +776,9 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { return 0; } -static int32_t lfs_dir_find(lfs_t *lfs, +static int32_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2], - uint32_t findmask, uint32_t findtag, - const void *findbuffer) { + uint32_t findmask, uint32_t findtag, const void *findbuffer) { dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; int32_t foundtag = LFS_ERR_NOENT; @@ -887,7 +906,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, } else if (lfs_tag_subtype(tag) == LFS_TYPE_GLOBALS) { templocals.s.deorphaned = (lfs_tag_type(tag) & 1); err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), - &templocals, sizeof(templocals)); + &templocals, 10); if (err) { if (err == LFS_ERR_CORRUPT) { dir->erased = false; @@ -906,7 +925,7 @@ static int32_t lfs_dir_find(lfs_t *lfs, } } else if ((tag & findmask) == (findtag & findmask)) { int res = lfs_bd_cmp(lfs, dir->pair[0], off+sizeof(tag), - findbuffer, lfs_tag_size(tag)); + findbuffer, lfs_tag_size(findtag)); if (res < 0) { if (res == LFS_ERR_CORRUPT) { dir->erased = false; @@ -953,7 +972,8 @@ static int32_t lfs_dir_find(lfs_t *lfs, static int lfs_dir_fetch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2]) { - int32_t res = lfs_dir_find(lfs, dir, pair, 0xffffffff, 0xffffffff, NULL); + int32_t res = lfs_dir_fetchmatch(lfs, dir, pair, + 0xffffffff, 0xffffffff, NULL); if (res < 0 && res != LFS_ERR_NOENT) { return res; } @@ -961,6 +981,23 @@ static int lfs_dir_fetch(lfs_t *lfs, return 0; } +static int32_t lfs_dir_find(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2], bool fs, + uint32_t findmask, uint32_t findtag, const void *findbuffer) { + dir->split = true; + dir->tail[0] = pair[0]; + dir->tail[1] = pair[1]; + while ((dir->split || fs) && !lfs_pair_isnull(dir->tail)) { + int32_t tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, + findmask, findtag, findbuffer); + if (tag != LFS_ERR_NOENT) { + return tag; + } + } + + return LFS_ERR_NOENT; +} + static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, uint32_t gettag, void *buffer) { int32_t getdiff = 0; @@ -979,6 +1016,8 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; + int16_t ack; + bool expanding = false; bool relocated = false; // There's nothing special about our global delta, so feed it back @@ -989,9 +1028,25 @@ static int lfs_dir_compact(lfs_t *lfs, // increment revision count dir->rev += 1; + if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0 && + dir->rev % 16 == 0) { + // we're writing too much to the superblock, should we expand? + lfs_ssize_t res = lfs_fs_size(lfs); + if (res < 0) { + return res; + } + + // do we have enough space to expand? + if (res < lfs->cfg->block_count/2) { + expanding = (lfs_pair_cmp(dir->pair, lfs->root) != 0); + ack = 0; + goto split; + } + } + while (true) { // last complete id - int16_t ack = -1; + ack = -1; dir->count = end - begin; if (true) { @@ -1111,7 +1166,7 @@ static int lfs_dir_compact(lfs_t *lfs, tail.tail[0] = dir->tail[0]; tail.tail[1] = dir->tail[1]; - err = lfs_dir_compact(lfs, &tail, attrs, dir, ack+1, end); + err = lfs_dir_compact(lfs, &tail, attrs, dir, ack+1-expanding, end); if (err) { return err; } @@ -1390,25 +1445,10 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { } // find entry matching name - while (true) { - tag = lfs_dir_find(lfs, dir, pair, 0x7c000fff, - LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), name); - if (tag < 0 && tag != LFS_ERR_NOENT) { - return tag; - } - - if (tag != LFS_ERR_NOENT) { - // found it - break; - } - - if (!dir->split) { - // couldn't find it - return LFS_ERR_NOENT; - } - - pair[0] = dir->tail[0]; - pair[1] = dir->tail[1]; + tag = lfs_dir_find(lfs, dir, pair, false, 0x7c000fff, + LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), name); + if (tag < 0) { + return tag; } // to next name @@ -2721,24 +2761,6 @@ int lfs_setattr(lfs_t *lfs, const char *path, /// Filesystem operations /// -static inline void lfs_superblockfromle32(lfs_superblock_t *superblock) { - superblock->version = lfs_fromle32(superblock->version); - superblock->block_size = lfs_fromle32(superblock->block_size); - superblock->block_count = lfs_fromle32(superblock->block_count); - superblock->inline_max = lfs_fromle32(superblock->inline_max); - superblock->attr_max = lfs_fromle32(superblock->attr_max); - superblock->name_max = lfs_fromle32(superblock->name_max); -} - -static inline void lfs_superblocktole32(lfs_superblock_t *superblock) { - superblock->version = lfs_tole32(superblock->version); - superblock->block_size = lfs_tole32(superblock->block_size); - superblock->block_count = lfs_tole32(superblock->block_count); - superblock->inline_max = lfs_tole32(superblock->inline_max); - superblock->attr_max = lfs_tole32(superblock->attr_max); - superblock->name_max = lfs_tole32(superblock->name_max); -} - static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->cfg = cfg; int err = 0; @@ -2859,30 +2881,13 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs->free.i = 0; lfs_alloc_ack(lfs); - // create superblock dir - lfs_mdir_t dir; - err = lfs_dir_alloc(lfs, &dir); - if (err) { - goto cleanup; - } - - // write root directory + // create root dir lfs_mdir_t root; err = lfs_dir_alloc(lfs, &root); if (err) { goto cleanup; } - err = lfs_dir_commit(lfs, &root, NULL); - if (err) { - goto cleanup; - } - - lfs->root[0] = root.pair[0]; - lfs->root[1] = root.pair[1]; - dir.tail[0] = lfs->root[0]; - dir.tail[1] = lfs->root[1]; - // write one superblock lfs_superblock_t superblock = { .magic = {"littlefs"}, @@ -2895,20 +2900,17 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .inline_max = lfs->inline_max, }; - lfs_superblocktole32(&superblock); - lfs_pair_tole32(lfs->root); - err = lfs_dir_commit(lfs, &dir, + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &root, LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), - LFS_MKATTR(LFS_TYPE_DIRSTRUCT, 0, lfs->root, sizeof(lfs->root), + LFS_MKATTR(LFS_TYPE_ROOT, 1, NULL, 0, NULL))); - lfs_pair_fromle32(lfs->root); - lfs_superblockfromle32(&superblock); if (err) { goto cleanup; } // sanity check that fetch works - err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + err = lfs_dir_fetch(lfs, &root, (const lfs_block_t[2]){0, 1}); if (err) { goto cleanup; } @@ -2931,27 +2933,34 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_alloc_ack(lfs); // load superblock - lfs_mdir_t superdir; - err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); + lfs_mdir_t root; + err = lfs_dir_fetch(lfs, &root, (const lfs_block_t[2]){0, 1}); if (err) { - goto cleanup; + return err; } lfs_superblock_t superblock; - int32_t res = lfs_dir_get(lfs, &superdir, 0x7ffff000, + int32_t res = lfs_dir_get(lfs, &root, 0x7fc00000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), &superblock); if (res < 0) { err = res; goto cleanup; } - lfs_superblockfromle32(&superblock); + lfs_superblock_fromle32(&superblock); - if (memcmp(superblock.magic, "littlefs", 8) != 0) { - LFS_ERROR("Invalid superblock \"%.8s\"", superblock.magic); - return LFS_ERR_INVAL; + // find root + int32_t tag = lfs_dir_find(lfs, + &root, (const lfs_block_t[2]){0, 1}, false, 0x7fc00000, + LFS_MKTAG(LFS_TYPE_ROOT, 0, 0), NULL); + if (tag < 0) { + return tag; } + lfs->root[0] = root.pair[0]; + lfs->root[1] = root.pair[1]; + + // check version uint16_t major_version = (0xffff & (superblock.version >> 16)); uint16_t minor_version = (0xffff & (superblock.version >> 0)); if ((major_version != LFS_DISK_VERSION_MAJOR || @@ -2962,15 +2971,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { goto cleanup; } - res = lfs_dir_get(lfs, &superdir, 0x7ffff000, - LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(lfs->root)), - &lfs->root); - if (res < 0) { - err = res; - goto cleanup; - } - lfs_pair_fromle32(lfs->root); - // check superblock configuration if (superblock.attr_max) { if (superblock.attr_max > lfs->attr_max) { @@ -3145,16 +3145,11 @@ static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_block_t child[2] = {pair[0], pair[1]}; lfs_pair_tole32(child); for (int i = 0; i < 2; i++) { - // iterate over all directory directory entries - parent->tail[0] = 0; - parent->tail[1] = 1; - while (!lfs_pair_isnull(parent->tail)) { - int32_t tag = lfs_dir_find(lfs, parent, parent->tail, 0x7fc00fff, - LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(child)), - child); - if (tag != LFS_ERR_NOENT) { - return tag; - } + int32_t tag = lfs_dir_find(lfs, parent, + (const lfs_block_t[2]){0, 1}, true, 0x7fc00fff, + LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(child)), child); + if (tag != LFS_ERR_NOENT) { + return tag; } lfs_pair_swap(child); @@ -3190,6 +3185,7 @@ static int lfs_fs_relocate(lfs_t *lfs, if (tag != LFS_ERR_NOENT) { // update disk, this creates a desync + lfs_global_deorphaned(lfs, false); lfs_pair_tole32(newpair); int err = lfs_dir_commit(lfs, &parent, &(lfs_mattr_t){.tag=tag, .buffer=newpair}); diff --git a/lfs.h b/lfs.h index 4ced50ae..df0927ed 100644 --- a/lfs.h +++ b/lfs.h @@ -93,7 +93,8 @@ enum lfs_type { // internally used types LFS_TYPE_USER = 0x100, - LFS_TYPE_SUPERBLOCK = 0x010, + LFS_TYPE_SUPERBLOCK = 0x011, + LFS_TYPE_ROOT = 0x012, LFS_TYPE_NAME = 0x000, LFS_TYPE_DELETE = 0x030, LFS_TYPE_STRUCT = 0x040, @@ -110,8 +111,9 @@ enum lfs_type { // internal chip sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x021, - LFS_FROM_ATTRS = 0x022, + LFS_FROM_MOVE = 0x050, + LFS_FROM_ATTRS = 0x060, + LFS_FROM_SUPERBLOCK = 0x070, }; // File open flags diff --git a/tests/test_alloc.sh b/tests/test_alloc.sh index d3c8da40..2be8e17d 100755 --- a/tests/test_alloc.sh +++ b/tests/test_alloc.sh @@ -206,7 +206,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < (cfg.block_count-6)*(cfg.block_size-8); + i < (cfg.block_count-4)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } @@ -286,7 +286,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < (cfg.block_count-6)*(cfg.block_size-8); + i < (cfg.block_count-4)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } @@ -320,7 +320,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-4)/2)*(cfg.block_size-8); + i < ((cfg.block_count-2)/2)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } @@ -331,7 +331,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-4+1)/2)*(cfg.block_size-8); + i < ((cfg.block_count-2+1)/2)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } @@ -349,7 +349,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-4)/2)*(cfg.block_size-8); + i < ((cfg.block_count-2)/2)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } @@ -363,7 +363,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-4+1)/2)*(cfg.block_size-8); + i < ((cfg.block_count-2+1)/2)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } @@ -383,7 +383,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-4)/2)*(cfg.block_size-8); + i < ((cfg.block_count-2)/2)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } @@ -394,7 +394,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-4+1)/2)*(cfg.block_size-8); + i < ((cfg.block_count-2+1)/2)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } @@ -412,7 +412,7 @@ tests/test.py << TEST size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); for (lfs_size_t i = 0; - i < ((cfg.block_count-4)/2 - 1)*(cfg.block_size-8); + i < ((cfg.block_count-2)/2 - 1)*(cfg.block_size-8); i += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } diff --git a/tests/test_format.sh b/tests/test_format.sh index cff0baca..06438bcc 100755 --- a/tests/test_format.sh +++ b/tests/test_format.sh @@ -9,6 +9,15 @@ tests/test.py << TEST lfs_format(&lfs, &cfg) => 0; TEST +echo "--- Basic mounting ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Invalid superblocks ---" ln -f -s /dev/zero blocks/0 ln -f -s /dev/zero blocks/1 @@ -17,22 +26,25 @@ tests/test.py << TEST TEST rm blocks/0 blocks/1 -echo "--- Basic mounting ---" -tests/test.py << TEST - lfs_format(&lfs, &cfg) => 0; -TEST +echo "--- Invalid mount ---" tests/test.py << TEST - lfs_mount(&lfs, &cfg) => 0; - lfs_unmount(&lfs) => 0; + lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; TEST -echo "--- Invalid mount ---" +echo "--- Expanding superblock ---" tests/test.py << TEST lfs_format(&lfs, &cfg) => 0; + lfs_mount(&lfs, &cfg) => 0; + for (int i = 0; i < 100; i++) { + lfs_mkdir(&lfs, "dummy") => 0; + lfs_remove(&lfs, "dummy") => 0; + } + lfs_unmount(&lfs) => 0; TEST -rm -f blocks/0 blocks/1 tests/test.py << TEST - lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "dummy") => 0; + lfs_unmount(&lfs) => 0; TEST echo "--- Results ---" diff --git a/tests/test_move.sh b/tests/test_move.sh index a36d0581..824d4d30 100755 --- a/tests/test_move.sh +++ b/tests/test_move.sh @@ -59,7 +59,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hello", "c/hello") => 0; lfs_unmount(&lfs) => 0; TEST -tests/corrupt.py blocks/{6,7} +tests/corrupt.py blocks/{4,5} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -86,8 +86,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hello", "d/hello") => 0; lfs_unmount(&lfs) => 0; TEST +tests/corrupt.py blocks/{6,7} tests/corrupt.py blocks/{8,9} -tests/corrupt.py blocks/{a,b} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; @@ -166,7 +166,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; TEST -tests/corrupt.py blocks/{6,7} +tests/corrupt.py blocks/{4,5} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -193,8 +193,8 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hi", "d/hi") => 0; lfs_unmount(&lfs) => 0; TEST +tests/corrupt.py blocks/{6,7} tests/corrupt.py blocks/{8,9} -tests/corrupt.py blocks/{a,b} tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; diff --git a/tests/test_orphan.sh b/tests/test_orphan.sh index 6aa36d9a..e66e5927 100755 --- a/tests/test_orphan.sh +++ b/tests/test_orphan.sh @@ -17,26 +17,26 @@ tests/test.py << TEST TEST # corrupt most recent commit, this should be the update to the previous # linked-list entry and should orphan the child -tests/corrupt.py blocks/{8,9} -tests/test.py << TEST +tests/corrupt.py blocks/{6,7} +tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; lfs_ssize_t before = lfs_fs_size(&lfs); - before => 10; + before => 8; lfs_unmount(&lfs) => 0; lfs_mount(&lfs, &cfg) => 0; lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; lfs_ssize_t orphaned = lfs_fs_size(&lfs); - orphaned => 10; + orphaned => 8; lfs_mkdir(&lfs, "parent/otherchild") => 0; lfs_stat(&lfs, "parent/orphan", &info) => LFS_ERR_NOENT; lfs_ssize_t deorphaned = lfs_fs_size(&lfs); - deorphaned => 10; + deorphaned => 8; lfs_unmount(&lfs) => 0; TEST From d8f930eeabe7c494d5f81a41a52da519617f66d8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 7 Aug 2018 09:57:54 -0500 Subject: [PATCH 086/139] Modified CTZ struct type to make space for erased files in the future In v1, littlefs didn't trust blocks that were been previously erased and conservatively erased any blocks before writing to them. This was a part of the design since the beginning because of the complexity of managing erased blocks when we can lose power at any time. However, we theoretically could keep track of files that have been properly erased by marking them with an "erased bit". A file marked this way could be opened and appended to without needing to COW the last block. The requirement would be that the "erased bit" is cleared during a write, since a power-loss would require that littlefs no longer trust the erased state of the file. This commit just shuffles the struct types around to make space for an "erased bit" in the struct type field to be added in the future. This ordering also makes more sense, since there will likely be more file representations than directory representations on disk. --- lfs.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lfs.h b/lfs.h index df0927ed..aa962620 100644 --- a/lfs.h +++ b/lfs.h @@ -104,9 +104,9 @@ enum lfs_type { LFS_TYPE_HARDTAIL = 0x0c1, LFS_TYPE_CRC = 0x0f0, - LFS_TYPE_INLINESTRUCT = 0x040, - LFS_TYPE_CTZSTRUCT = 0x041, - LFS_TYPE_DIRSTRUCT = 0x042, + LFS_TYPE_DIRSTRUCT = 0x040, + LFS_TYPE_INLINESTRUCT = 0x041, + LFS_TYPE_CTZSTRUCT = 0x042, // internal chip sources LFS_FROM_REGION = 0x000, From 3b3981eb7468714d1c938b52a13a20f003d8ae1d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 7 Aug 2018 14:10:03 -0500 Subject: [PATCH 087/139] Fixed testing issues introduced by expanding superblocks This was mostly tweaking test cases to be accommodating for variable sized superblock-lists. Though there were a few bugs that needed fixing: - Changed compact to use source dir for move since the original dir could have changed as a result of an expand. - Created copy of current directory so we don't overwrite ourselves during an internal commit update. Also made sure all of the test suites provide reproducable results when ran independently (the entry tests were behaving differently based on which tests were ran before). (Some where legitimate test failures) --- lfs.c | 10 ++-- tests/test_alloc.sh | 103 +++++++++++++++++++++++++++++++----------- tests/test_entries.sh | 3 +- 3 files changed, 85 insertions(+), 31 deletions(-) diff --git a/lfs.c b/lfs.c index ee8a7a8f..2d174a2c 100644 --- a/lfs.c +++ b/lfs.c @@ -658,7 +658,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int32_t res = lfs_commit_get(lfs, commit->block, commit->off, commit->ptag, lfs_tag_isuser(tag) ? 0x7ffff000 : 0x7c3ff000, - LFS_MKTAG(lfs_tag_type(tag), toid, 0), + (tag & 0x7fc00000) | LFS_MKTAG(0, toid, 0), 0, NULL, true); if (res < 0 && res != LFS_ERR_NOENT) { return res; @@ -1166,7 +1166,7 @@ static int lfs_dir_compact(lfs_t *lfs, tail.tail[0] = dir->tail[0]; tail.tail[1] = dir->tail[1]; - err = lfs_dir_compact(lfs, &tail, attrs, dir, ack+1-expanding, end); + err = lfs_dir_compact(lfs, &tail, attrs, source, ack+1-expanding, end); if (err) { return err; } @@ -1346,8 +1346,12 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_global_xor(&lfs->globals, &canceldiff); // update any directories that are affected + lfs_mdir_t copy = *dir; + + // two passes, once for things that aren't us, and one + // for things that are for (lfs_mlist_t *d = lfs->mlist; d; d = d->next) { - if (lfs_pair_cmp(d->m.pair, dir->pair) == 0) { + if (lfs_pair_cmp(d->m.pair, copy.pair) == 0) { d->m = *dir; if (d->id == lfs_tag_id(deletetag)) { d->m.pair[0] = 0xffffffff; diff --git a/tests/test_alloc.sh b/tests/test_alloc.sh index 2be8e17d..55d1043c 100755 --- a/tests/test_alloc.sh +++ b/tests/test_alloc.sh @@ -194,56 +194,99 @@ tests/test.py << TEST lfs_file_read(&lfs, &file[0], buffer, size) => size; memcmp(buffer, "exhaustion", size) => 0; lfs_file_close(&lfs, &file[0]) => 0; + lfs_remove(&lfs, "exhaustion") => 0; lfs_unmount(&lfs) => 0; TEST echo "--- Dir exhaustion test ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_remove(&lfs, "exhaustion") => 0; - lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + // find out max file size + lfs_mkdir(&lfs, "exhaustiondir") => 0; size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); - for (lfs_size_t i = 0; - i < (cfg.block_count-4)*(cfg.block_size-8); - i += size) { + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + int count = 0; + int err; + while (true) { + err = lfs_file_write(&lfs, &file[0], buffer, size); + if (err < 0) { + break; + } + + count += 1; + } + err => LFS_ERR_NOSPC; + lfs_file_close(&lfs, &file[0]) => 0; + + lfs_remove(&lfs, "exhaustion") => 0; + lfs_remove(&lfs, "exhaustiondir") => 0; + + // see if dir fits with max file size + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + for (int i = 0; i < count; i++) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } lfs_file_close(&lfs, &file[0]) => 0; lfs_mkdir(&lfs, "exhaustiondir") => 0; lfs_remove(&lfs, "exhaustiondir") => 0; + lfs_remove(&lfs, "exhaustion") => 0; - lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_APPEND); - size = strlen("blahblahblahblah"); - memcpy(buffer, "blahblahblahblah", size); - for (lfs_size_t i = 0; - i < (cfg.block_size-8); - i += size) { + // see if dir fits with > max file size + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + for (int i = 0; i < count+1; i++) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } lfs_file_close(&lfs, &file[0]) => 0; lfs_mkdir(&lfs, "exhaustiondir") => LFS_ERR_NOSPC; + + lfs_remove(&lfs, "exhaustion") => 0; lfs_unmount(&lfs) => 0; TEST echo "--- Chained dir exhaustion test ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_remove(&lfs, "exhaustion") => 0; - lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + // find out max file size + lfs_mkdir(&lfs, "exhaustiondir") => 0; + for (int i = 0; i < 9; i++) { + sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); + lfs_mkdir(&lfs, (char*)buffer) => 0; + } size = strlen("blahblahblahblah"); memcpy(buffer, "blahblahblahblah", size); - for (lfs_size_t i = 0; - i < (cfg.block_count-24)*(cfg.block_size-8); - i += size) { - lfs_file_write(&lfs, &file[0], buffer, size) => size; + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + int count = 0; + int err; + while (true) { + err = lfs_file_write(&lfs, &file[0], buffer, size); + if (err < 0) { + break; + } + + count += 1; } + err => LFS_ERR_NOSPC; lfs_file_close(&lfs, &file[0]) => 0; + lfs_remove(&lfs, "exhaustion") => 0; + lfs_remove(&lfs, "exhaustiondir") => 0; + for (int i = 0; i < 9; i++) { + sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); + lfs_remove(&lfs, (char*)buffer) => 0; + } + + // see that chained dir fails + lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); + for (int i = 0; i < count+1; i++) { + lfs_file_write(&lfs, &file[0], buffer, size) => size; + } + lfs_file_sync(&lfs, &file[0]) => 0; + for (int i = 0; i < 9; i++) { sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); lfs_mkdir(&lfs, (char*)buffer) => 0; @@ -251,19 +294,25 @@ tests/test.py << TEST lfs_mkdir(&lfs, "exhaustiondir") => LFS_ERR_NOSPC; - lfs_remove(&lfs, "exhaustion") => 0; - lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); - size = strlen("blahblahblahblah"); - memcpy(buffer, "blahblahblahblah", size); - for (lfs_size_t i = 0; - i < (cfg.block_count-26)*(cfg.block_size-8); - i += size) { - lfs_file_write(&lfs, &file[0], buffer, size) => size; + // shorten file to try a second chained dir + while (true) { + err = lfs_mkdir(&lfs, "exhaustiondir"); + if (err != LFS_ERR_NOSPC) { + break; + } + + lfs_ssize_t filesize = lfs_file_size(&lfs, &file[0]); + filesize > 0 => true; + + lfs_file_truncate(&lfs, &file[0], filesize - size) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; } - lfs_file_close(&lfs, &file[0]) => 0; + err => 0; - lfs_mkdir(&lfs, "exhaustiondir") => 0; lfs_mkdir(&lfs, "exhaustiondir2") => LFS_ERR_NOSPC; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; TEST echo "--- Split dir test ---" diff --git a/tests/test_entries.sh b/tests/test_entries.sh index 447d9bca..4728b7f3 100755 --- a/tests/test_entries.sh +++ b/tests/test_entries.sh @@ -4,7 +4,8 @@ set -eu # Note: These tests are intended for 512 byte inline size at different # inline sizes they should still pass, but won't be testing anything -echo "=== Directory tests ===" +echo "=== Entry tests ===" +rm -rf blocks function read_file { cat << TEST From 10f45ac02f246c44b066d85dacd7cb43c0b7a521 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 7 Aug 2018 14:59:44 -0500 Subject: [PATCH 088/139] Changed lfs_crc to match more common API In looking at the common CRC APIs out there, this seemed the most common. At least more common than the current modified-in-place pointer API. It also seems to have a slightly better code footprint. I'm blaming pointer optimization issues. One downside is that lfs_crc can't report errors, however it was already assumed that lfs_crc can not error. --- lfs.c | 22 +++++++++++----------- lfs_util.c | 8 +++++--- lfs_util.h | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lfs.c b/lfs.c index 2d174a2c..a950c4a2 100644 --- a/lfs.c +++ b/lfs.c @@ -121,7 +121,7 @@ static int lfs_cache_crc(lfs_t *lfs, return err; } - lfs_crc(crc, &c, 1); + *crc = lfs_crc(*crc, &c, 1); } return 0; @@ -236,7 +236,7 @@ static int lfs_bd_cmp(lfs_t *lfs, lfs_block_t block, return lfs_cache_cmp(lfs, NULL, &lfs->rcache, block, off, buffer, size); } -static int lfs_bd_crc(lfs_t *lfs, lfs_block_t block, +static int lfs_bd_crc32(lfs_t *lfs, lfs_block_t block, lfs_off_t off, lfs_size_t size, uint32_t *crc) { return lfs_cache_crc(lfs, NULL, &lfs->rcache, block, off, size, crc); } @@ -562,7 +562,7 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, // write out tag uint32_t ntag = lfs_tole32((tag & 0x7fffffff) ^ commit->ptag); - lfs_crc(&commit->crc, &ntag, sizeof(ntag)); + commit->crc = lfs_crc(commit->crc, &ntag, sizeof(ntag)); int err = lfs_bd_prog(lfs, commit->block, commit->off, &ntag, sizeof(ntag)); if (err) { @@ -572,7 +572,7 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, if (!(tag & 0x80000000)) { // from memory - lfs_crc(&commit->crc, buffer, size); + commit->crc = lfs_crc(commit->crc, buffer, size); err = lfs_bd_prog(lfs, commit->block, commit->off, buffer, size); if (err) { return err; @@ -588,7 +588,7 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, return err; } - lfs_crc(&commit->crc, &dat, 1); + commit->crc = lfs_crc(commit->crc, &dat, 1); err = lfs_bd_prog(lfs, commit->block, commit->off+i, &dat, 1); if (err) { return err; @@ -714,7 +714,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // write out crc uint32_t footer[2]; footer[0] = lfs_tole32(tag ^ commit->ptag); - lfs_crc(&commit->crc, &footer[0], sizeof(footer[0])); + commit->crc = lfs_crc(commit->crc, &footer[0], sizeof(footer[0])); footer[1] = lfs_tole32(commit->crc); err = lfs_bd_prog(lfs, commit->block, commit->off, footer, sizeof(footer)); if (err) { @@ -731,7 +731,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // successful commit, check checksum to make sure uint32_t crc = 0xffffffff; - err = lfs_bd_crc(lfs, commit->block, commit->begin, + err = lfs_bd_crc32(lfs, commit->block, commit->begin, commit->off-lfs_tag_size(tag)-commit->begin, &crc); if (err) { return err; @@ -810,7 +810,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, uint32_t crc = 0xffffffff; dir->rev = lfs_tole32(rev[0]); - lfs_crc(&crc, &dir->rev, sizeof(dir->rev)); + crc = lfs_crc(crc, &dir->rev, sizeof(dir->rev)); dir->rev = lfs_fromle32(dir->rev); dir->off = 0; @@ -835,7 +835,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, return err; } - lfs_crc(&crc, &tag, sizeof(tag)); + crc = lfs_crc(crc, &tag, sizeof(tag)); tag = lfs_fromle32(tag) ^ ptag; // next commit not yet programmed @@ -879,7 +879,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, dir->locals = templocals; crc = 0xffffffff; } else { - err = lfs_bd_crc(lfs, dir->pair[0], + err = lfs_bd_crc32(lfs, dir->pair[0], off+sizeof(tag), lfs_tag_size(tag), &crc); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1062,7 +1062,7 @@ static int lfs_dir_compact(lfs_t *lfs, // write out header uint32_t crc = 0xffffffff; uint32_t rev = lfs_tole32(dir->rev); - lfs_crc(&crc, &rev, sizeof(rev)); + crc = lfs_crc(crc, &rev, sizeof(rev)); err = lfs_bd_prog(lfs, dir->pair[1], 0, &rev, sizeof(rev)); if (err) { if (err == LFS_ERR_CORRUPT) { diff --git a/lfs_util.c b/lfs_util.c index 9ca0756d..0b60e3b4 100644 --- a/lfs_util.c +++ b/lfs_util.c @@ -11,7 +11,7 @@ // Software CRC implementation with small lookup table -void lfs_crc(uint32_t *restrict crc, const void *buffer, size_t size) { +uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size) { static const uint32_t rtable[16] = { 0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c, @@ -22,9 +22,11 @@ void lfs_crc(uint32_t *restrict crc, const void *buffer, size_t size) { const uint8_t *data = buffer; for (size_t i = 0; i < size; i++) { - *crc = (*crc >> 4) ^ rtable[(*crc ^ (data[i] >> 0)) & 0xf]; - *crc = (*crc >> 4) ^ rtable[(*crc ^ (data[i] >> 4)) & 0xf]; + crc = (crc >> 4) ^ rtable[(crc ^ (data[i] >> 0)) & 0xf]; + crc = (crc >> 4) ^ rtable[(crc ^ (data[i] >> 4)) & 0xf]; } + + return crc; } diff --git a/lfs_util.h b/lfs_util.h index ed55a1ce..80b4002b 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -187,7 +187,7 @@ static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) { } // Calculate CRC-32 with polynomial = 0x04c11db7 -void lfs_crc(uint32_t *crc, const void *buffer, size_t size); +uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size); // Allocate memory, only used if buffers are not provided to littlefs static inline void *lfs_malloc(size_t size) { From 20b669a23d50bf191dc7225527487f9148122b8c Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 8 Aug 2018 09:37:43 -0500 Subject: [PATCH 089/139] Fixed issue with big-endian CTZ lists intertwined in commit logic Found while testing big-endian support. Basically, if littlefs is really really unlucky, the block allocator could kick in while committing a file's CTZ reference. If this happens, the block allocator will need to traverse all CTZ skip-lists in memory, including the skip-list we're committing. This means we can't convert the CTZ's endianness in place, and need to make a copy on big-endian systems. We rely on dead-code elimination from the compiler to make the conditional behaviour for big-endian vs little-endian system a noop determined by the lfs_tole32 intrinsic. --- lfs.c | 12 +++++++----- lfs_util.h | 22 +++++++++++----------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lfs.c b/lfs.c index a950c4a2..faa5bbe6 100644 --- a/lfs.c +++ b/lfs.c @@ -2199,25 +2199,27 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { uint16_t type; const void *buffer; lfs_size_t size; + struct lfs_ctz ctz; if (file->flags & LFS_F_INLINE) { // inline the whole file type = LFS_TYPE_INLINESTRUCT; buffer = file->cache.buffer; size = file->ctz.size; - } else { + } else if (lfs_tole32(0x11223344) == 0x11223344) { // update the ctz reference type = LFS_TYPE_CTZSTRUCT; - buffer = &file->ctz; - size = sizeof(file->ctz); + // copy ctz so alloc will work during a relocate + ctz = file->ctz; + lfs_ctz_tole32(&ctz); + buffer = &ctz; + size = sizeof(ctz); } // commit file data and attributes - lfs_ctz_tole32(&file->ctz); err = lfs_dir_commit(lfs, &file->m, LFS_MKATTR(type, file->id, buffer, size, LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, NULL))); - lfs_ctz_fromle32(&file->ctz); if (err) { if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) { goto relocate; diff --git a/lfs_util.h b/lfs_util.h index 80b4002b..8a65694c 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -11,8 +11,8 @@ // LFS_CONFIG as a header file to include (-DLFS_CONFIG=lfs_config.h). // // If LFS_CONFIG is used, none of the default utils will be emitted and must be -// provided by the config file. To start I would suggest copying lfs_util.h and -// modifying as needed. +// provided by the config file. To start, I would suggest copying lfs_util.h +// and modifying as needed. #ifdef LFS_CONFIG #define LFS_STRINGIZE(x) LFS_STRINGIZE2(x) #define LFS_STRINGIZE2(x) #x @@ -88,6 +88,15 @@ static inline uint32_t lfs_min(uint32_t a, uint32_t b) { return (a < b) ? a : b; } +// Align to nearest multiple of a size +static inline uint32_t lfs_aligndown(uint32_t a, uint32_t alignment) { + return a - (a % alignment); +} + +static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) { + return lfs_aligndown(a + alignment-1, alignment); +} + // Find the next smallest power of 2 less than or equal to a static inline uint32_t lfs_npw2(uint32_t a) { #if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM)) @@ -177,15 +186,6 @@ static inline uint16_t lfs_tole16(uint16_t a) { return lfs_fromle16(a); } -// Align to nearest multiple of a size -static inline uint32_t lfs_aligndown(uint32_t a, uint32_t alignment) { - return a - (a % alignment); -} - -static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) { - return lfs_aligndown(a + alignment-1, alignment); -} - // Calculate CRC-32 with polynomial = 0x04c11db7 uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size); From e4a0d586d5b38f092bd0c44ccece8659c6f4a025 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 8 Aug 2018 16:34:56 -0500 Subject: [PATCH 090/139] Added building blocks for dynamic wear-leveling Initially, littlefs relied entirely on bad-block detection for wear-leveling. Conceptually, at the end of a devices lifespan, all blocks would be worn evenly, even if they weren't worn out at the same time. However, this doesn't work for all devices, rather than causing corruption during writes, wear reduces a devices "sticking power", causing bits to flip over time. This means for many devices, true wear-leveling (dynamic or static) is required. Fortunately, way back at the beginning, littlefs was designed to do full dynamic wear-leveling, only dropping it when making the retrospectively short-sighted realization that bad-block detection is theoretically sufficient. We can enable dynamic wear-leveling with only a few tweaks to littlefs. These can be implemented without breaking backwards compatibility. 1. Evict metadata-pairs after a certain number of writes. Eviction in this case is identical to a relocation to recover from a bad block. We move our data and stick the old block back into our pool of blocks. For knowing when to evict, we already have a revision count for each metadata-pair which gives us enough information. We add the configuration option block_cycles and evict when our revision count is a multiple of this value. 2. Now all blocks participate in COW behaviour. However we don't store the state of our allocator, so every boot cycle we reuse the first blocks on storage. This is very bad on a microcontroller, where we may reboot often. We need a way to spread our usage across the disk. To pull this off, we can simply randomize which block we start our allocator at. But we need a random number generator that is different on each boot. Fortunately we have a great source of entropy, our filesystem. So we seed our block allocator with a simple hash of the CRCs on our metadata-pairs. This can be done for free since we already need to scan the metadata-pairs during mount. What we end up with is a uniform distribution of wear on storage. The wear is not perfect, if a block is used for metadata it gets more wear, and the randomization may not be exact. But we can never actually get perfect wear-leveling, since we're already resigned to dynamic wear-leveling at the file level. With the addition of metadata logging, we end up with a really interesting two-stage wear-leveling algorithm. At the low-level, metadata is statically wear-leveled. At the high-level, blocks are dynamically wear-leveled. --- This specific commit implements the first step, eviction of metadata pairs. Entertwining this into the already complicated compact logic was a bit annoying, however we can combine the logic for superblock expansion with the logic for metadata-pair eviction. --- .travis.yml | 2 +- lfs.c | 540 ++++++++++++++++++++++++--------------------- lfs.h | 17 +- tests/template.fmt | 25 ++- 4 files changed, 317 insertions(+), 267 deletions(-) diff --git a/.travis.yml b/.travis.yml index 32c21d86..0a47a56b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ script: # run tests with a few different configurations - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_CACHE_SIZE=4" - - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_CACHE_SIZE=512" + - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_CACHE_SIZE=512 -DLFS_BLOCK_CYCLES=16" - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048" - make clean test QUIET=1 CFLAGS+="-DLFS_INLINE_MAX=0" diff --git a/lfs.c b/lfs.c index faa5bbe6..3f8c33c6 100644 --- a/lfs.c +++ b/lfs.c @@ -265,6 +265,8 @@ static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent); static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]); +static int lfs_fs_deorphan(lfs_t *lfs); +static int lfs_fs_demove(lfs_t *lfs); static int lfs_fs_forceconsistency(lfs_t *lfs); static int lfs_deinit(lfs_t *lfs); @@ -924,18 +926,40 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, tempfoundtag -= LFS_MKTAG(0, 1, 0); } } else if ((tag & findmask) == (findtag & findmask)) { - int res = lfs_bd_cmp(lfs, dir->pair[0], off+sizeof(tag), - findbuffer, lfs_tag_size(findtag)); - if (res < 0) { - if (res == LFS_ERR_CORRUPT) { - dir->erased = false; - break; + // found a match? + if (lfs_tag_type(findtag) == LFS_TYPE_DIRSTRUCT) { + lfs_block_t child[2]; + err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), + child, sizeof(child)); + if (err < 0) { + if (err == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + return err; } - return res; - } - if (res) { - // found a match + lfs_pair_fromle32(child); + if (lfs_pair_cmp(child, + (const lfs_block_t *)findbuffer) == 0) { + tempfoundtag = tag; + } + } else if (lfs_tag_type(findtag) == LFS_TYPE_NAME) { + int res = lfs_bd_cmp(lfs, + dir->pair[0], off+sizeof(tag), + findbuffer, lfs_tag_size(findtag)); + if (res < 0) { + if (res == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + return res; + } + + if (res) { + tempfoundtag = tag; + } + } else { tempfoundtag = tag; } } @@ -1016,8 +1040,6 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; - int16_t ack; - bool expanding = false; bool relocated = false; // There's nothing special about our global delta, so feed it back @@ -1025,125 +1047,129 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_global_xor(&lfs->locals, &dir->locals); lfs_global_zero(&dir->locals); - // increment revision count - dir->rev += 1; - - if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0 && - dir->rev % 16 == 0) { - // we're writing too much to the superblock, should we expand? - lfs_ssize_t res = lfs_fs_size(lfs); - if (res < 0) { - return res; - } - - // do we have enough space to expand? - if (res < lfs->cfg->block_count/2) { - expanding = (lfs_pair_cmp(dir->pair, lfs->root) != 0); - ack = 0; - goto split; - } - } - while (true) { // last complete id - ack = -1; dir->count = end - begin; + int16_t ack = -1; + bool exhausted = false; + + // increment revision count + dir->rev += 1; + if (lfs->cfg->block_cycles && dir->rev % lfs->cfg->block_cycles == 0) { + if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { + // we're writing too much to the superblock, should we expand? + lfs_ssize_t res = lfs_fs_size(lfs); + if (res < 0) { + return res; + } - if (true) { - // erase block to write to - int err = lfs_bd_erase(lfs, dir->pair[1]); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + // do we have enough space to expand? + if (res < lfs->cfg->block_count/2) { + LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); + ack = 0; + exhausted = (lfs_pair_cmp(dir->pair, lfs->root) != 0); + goto split; } - return err; + } else { + // we're writing too much, time to relocate + exhausted = true; + goto relocate; } + } - // write out header - uint32_t crc = 0xffffffff; - uint32_t rev = lfs_tole32(dir->rev); - crc = lfs_crc(crc, &rev, sizeof(rev)); - err = lfs_bd_prog(lfs, dir->pair[1], 0, &rev, sizeof(rev)); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; + // erase block to write to + int err = lfs_bd_erase(lfs, dir->pair[1]); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; } + return err; + } - // setup compaction - struct lfs_commit commit = { - .block = dir->pair[1], - .off = sizeof(dir->rev), - .crc = crc, - .ptag = 0, + // write out header + uint32_t crc = 0xffffffff; + uint32_t rev = lfs_tole32(dir->rev); + crc = lfs_crc(crc, &rev, sizeof(rev)); + err = lfs_bd_prog(lfs, dir->pair[1], 0, &rev, sizeof(rev)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } - // space is complicated, we need room for tail, crc, globals, - // and we cap at half a block to give room for metadata updates - .begin = 0, - .end = lfs_min( - lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size), - lfs->cfg->block_size - 34), - }; + // setup compaction + struct lfs_commit commit = { + .block = dir->pair[1], + .off = sizeof(dir->rev), + .crc = crc, + .ptag = 0, + + // space is complicated, we need room for tail, crc, globals, + // and we cap at half a block to give room for metadata updates + .begin = 0, + .end = lfs_min( + lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size), + lfs->cfg->block_size - 34), + }; - // commit with a move - for (uint16_t id = begin; id < end; id++) { - err = lfs_commit_move(lfs, &commit, - id, id - begin, source, attrs); - if (err) { - if (err == LFS_ERR_NOSPC) { - goto split; - } else if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; + // commit with a move + for (uint16_t id = begin; id < end; id++) { + err = lfs_commit_move(lfs, &commit, + id, id - begin, source, attrs); + if (err) { + if (err == LFS_ERR_NOSPC) { + goto split; + } else if (err == LFS_ERR_CORRUPT) { + goto relocate; } - - ack = id; + return err; } - // reopen reserved space at the end - commit.end = lfs->cfg->block_size - 8; + ack = id; + } - if (!relocated) { - err = lfs_commit_globals(lfs, &commit, &dir->locals); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - } + // reopen reserved space at the end + commit.end = lfs->cfg->block_size - 8; - if (!lfs_pair_isnull(dir->tail)) { - // commit tail, which may be new after last size check - lfs_pair_tole32(dir->tail); - err = lfs_commit_attr(lfs, &commit, - LFS_MKTAG(LFS_TYPE_TAIL + dir->split, - 0x3ff, sizeof(dir->tail)), dir->tail); - lfs_pair_fromle32(dir->tail); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; + if (!relocated) { + err = lfs_commit_globals(lfs, &commit, &dir->locals); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; } + return err; } + } - err = lfs_commit_crc(lfs, &commit); + if (!lfs_pair_isnull(dir->tail)) { + // commit tail, which may be new after last size check + lfs_pair_tole32(dir->tail); + err = lfs_commit_attr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_TAIL + dir->split, + 0x3ff, sizeof(dir->tail)), dir->tail); + lfs_pair_fromle32(dir->tail); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } + } - // successful compaction, swap dir pair to indicate most recent - lfs_pair_swap(dir->pair); - dir->off = commit.off; - dir->etag = commit.ptag; - dir->erased = true; + err = lfs_commit_crc(lfs, &commit); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; } + + // successful compaction, swap dir pair to indicate most recent + lfs_pair_swap(dir->pair); + dir->off = commit.off; + dir->etag = commit.ptag; + dir->erased = true; break; split: @@ -1157,7 +1183,7 @@ static int lfs_dir_compact(lfs_t *lfs, } lfs_mdir_t tail; - int err = lfs_dir_alloc(lfs, &tail); + err = lfs_dir_alloc(lfs, &tail); if (err) { return err; } @@ -1166,7 +1192,7 @@ static int lfs_dir_compact(lfs_t *lfs, tail.tail[0] = dir->tail[0]; tail.tail[1] = dir->tail[1]; - err = lfs_dir_compact(lfs, &tail, attrs, source, ack+1-expanding, end); + err = lfs_dir_compact(lfs, &tail, attrs, source, ack+1-exhausted, end); if (err) { return err; } @@ -1178,13 +1204,12 @@ static int lfs_dir_compact(lfs_t *lfs, continue; relocate: - //commit was corrupted - LFS_DEBUG("Bad block at %"PRIu32, - dir->pair[1]); - - // drop caches and prepare to relocate block + // commit was corrupted, drop caches and prepare to relocate block relocated = true; lfs_cache_drop(lfs, &lfs->pcache); + if (!exhausted) { + LFS_DEBUG("Bad block at %"PRIu32, dir->pair[1]); + } // can't relocate superblock, filesystem is now frozen if (lfs_pair_cmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { @@ -1194,7 +1219,7 @@ static int lfs_dir_compact(lfs_t *lfs, // relocate half of pair err = lfs_alloc(lfs, &dir->pair[1]); - if (err) { + if (err && (err != LFS_ERR_NOSPC && !exhausted)) { return err; } @@ -1273,7 +1298,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } } - if (dir->erased) { + while (true) { + if (!dir->erased) { + goto compact; + } + // try to commit struct lfs_commit commit = { .block = dir->pair[0], @@ -1332,14 +1361,17 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // successful commit, update globals lfs_global_xor(&dir->locals, &lfs->locals); lfs_global_zero(&lfs->locals); - } else { + break; + compact: // fall back to compaction lfs_cache_drop(lfs, &lfs->pcache); - int err = lfs_dir_compact(lfs, dir, attrs, dir, 0, dir->count); + err = lfs_dir_compact(lfs, dir, attrs, dir, 0, dir->count); if (err) { return err; } + + break; } // update globals that are affected @@ -1762,83 +1794,81 @@ static int lfs_ctz_extend(lfs_t *lfs, } LFS_ASSERT(nblock >= 2 && nblock <= lfs->cfg->block_count); - if (true) { - err = lfs_bd_erase(lfs, nblock); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - if (size == 0) { - *block = nblock; - *off = 0; - return 0; + err = lfs_bd_erase(lfs, nblock); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; } + return err; + } - size -= 1; - lfs_off_t index = lfs_ctz_index(lfs, &size); - size += 1; + if (size == 0) { + *block = nblock; + *off = 0; + return 0; + } - // just copy out the last block if it is incomplete - if (size != lfs->cfg->block_size) { - for (lfs_off_t i = 0; i < size; i++) { - uint8_t data; - err = lfs_cache_read(lfs, NULL, rcache, true, - head, i, &data, 1); - if (err) { - return err; - } + size -= 1; + lfs_off_t index = lfs_ctz_index(lfs, &size); + size += 1; - err = lfs_cache_prog(lfs, pcache, rcache, true, - nblock, i, &data, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } + // just copy out the last block if it is incomplete + if (size != lfs->cfg->block_size) { + for (lfs_off_t i = 0; i < size; i++) { + uint8_t data; + err = lfs_cache_read(lfs, NULL, rcache, true, + head, i, &data, 1); + if (err) { + return err; } - *block = nblock; - *off = size; - return 0; - } - - // append block - index += 1; - lfs_size_t skips = lfs_ctz(index) + 1; - - for (lfs_off_t i = 0; i < skips; i++) { - head = lfs_tole32(head); err = lfs_cache_prog(lfs, pcache, rcache, true, - nblock, 4*i, &head, 4); - head = lfs_fromle32(head); + nblock, i, &data, 1); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } + } - if (i != skips-1) { - err = lfs_cache_read(lfs, NULL, rcache, false, - head, 4*i, &head, 4); - head = lfs_fromle32(head); - if (err) { - return err; - } + *block = nblock; + *off = size; + return 0; + } + + // append block + index += 1; + lfs_size_t skips = lfs_ctz(index) + 1; + + for (lfs_off_t i = 0; i < skips; i++) { + head = lfs_tole32(head); + err = lfs_cache_prog(lfs, pcache, rcache, true, + nblock, 4*i, &head, 4); + head = lfs_fromle32(head); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; } + return err; + } - LFS_ASSERT(head >= 2 && head <= lfs->cfg->block_count); + if (i != skips-1) { + err = lfs_cache_read(lfs, NULL, rcache, false, + head, 4*i, &head, 4); + head = lfs_fromle32(head); + if (err) { + return err; + } } - *block = nblock; - *off = 4*skips; - return 0; + LFS_ASSERT(head >= 2 && head <= lfs->cfg->block_count); } + *block = nblock; + *off = 4*skips; + return 0; + relocate: LFS_DEBUG("Bad block at %"PRIu32, nblock); @@ -2205,7 +2235,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { type = LFS_TYPE_INLINESTRUCT; buffer = file->cache.buffer; size = file->ctz.size; - } else if (lfs_tole32(0x11223344) == 0x11223344) { + } else { // update the ctz reference type = LFS_TYPE_CTZSTRUCT; // copy ctz so alloc will work during a relocate @@ -3148,17 +3178,13 @@ static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], } // search for both orderings so we can reuse the find function - lfs_block_t child[2] = {pair[0], pair[1]}; - lfs_pair_tole32(child); for (int i = 0; i < 2; i++) { int32_t tag = lfs_dir_find(lfs, parent, (const lfs_block_t[2]){0, 1}, true, 0x7fc00fff, - LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, sizeof(child)), child); + LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), pair); if (tag != LFS_ERR_NOENT) { return tag; } - - lfs_pair_swap(child); } return LFS_ERR_NOENT; @@ -3201,7 +3227,7 @@ static int lfs_fs_relocate(lfs_t *lfs, } // clean up bad block, which should now be a desync - return lfs_fs_forceconsistency(lfs); + return lfs_fs_deorphan(lfs); } // find pred @@ -3227,98 +3253,112 @@ static int lfs_fs_relocate(lfs_t *lfs, return 0; } -static int lfs_fs_forceconsistency(lfs_t *lfs) { - if (!lfs->globals.s.deorphaned) { - LFS_DEBUG("Found orphans %"PRIu32, - lfs->globals.s.deorphaned); +static int lfs_fs_deorphan(lfs_t *lfs) { + // Fix any orphans + lfs_mdir_t pdir = {.split = true}; + lfs_mdir_t dir = {.tail = {0, 1}}; - // Fix any orphans - lfs_mdir_t pdir = {.split = true}; - lfs_mdir_t dir = {.tail = {0, 1}}; + // iterate over all directory directory entries + while (!lfs_pair_isnull(dir.tail)) { + int err = lfs_dir_fetch(lfs, &dir, dir.tail); + if (err) { + return err; + } - // iterate over all directory directory entries - while (!lfs_pair_isnull(dir.tail)) { - int err = lfs_dir_fetch(lfs, &dir, dir.tail); - if (err) { - return err; + // check head blocks for orphans + if (!pdir.split) { + // check if we have a parent + lfs_mdir_t parent; + int32_t tag = lfs_fs_parent(lfs, pdir.tail, &parent); + if (tag < 0 && tag != LFS_ERR_NOENT) { + return tag; } - // check head blocks for orphans - if (!pdir.split) { - // check if we have a parent - lfs_mdir_t parent; - int32_t tag = lfs_fs_parent(lfs, pdir.tail, &parent); - if (tag < 0 && tag != LFS_ERR_NOENT) { - return tag; + if (tag == LFS_ERR_NOENT) { + // we are an orphan + LFS_DEBUG("Fixing orphan %"PRIu32" %"PRIu32, + pdir.tail[0], pdir.tail[1]); + + pdir.tail[0] = dir.tail[0]; + pdir.tail[1] = dir.tail[1]; + err = lfs_dir_commit(lfs, &pdir, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, + pdir.tail, sizeof(pdir.tail), + NULL)); + if (err) { + return err; } - if (tag == LFS_ERR_NOENT) { - // we are an orphan - LFS_DEBUG("Fixing orphan %"PRIu32" %"PRIu32, - pdir.tail[0], pdir.tail[1]); - - pdir.tail[0] = dir.tail[0]; - pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pdir.tail, sizeof(pdir.tail), - NULL)); - if (err) { - return err; - } + break; + } - break; - } + lfs_block_t pair[2]; + int32_t res = lfs_dir_get(lfs, &parent, 0x7ffff000, tag, pair); + if (res < 0) { + return res; + } + lfs_pair_fromle32(pair); - lfs_block_t pair[2]; - int32_t res = lfs_dir_get(lfs, &parent, 0x7ffff000, tag, pair); - if (res < 0) { - return res; + if (!lfs_pair_sync(pair, pdir.tail)) { + // we have desynced + LFS_DEBUG("Fixing half-orphan %"PRIu32" %"PRIu32, + pair[0], pair[1]); + + pdir.tail[0] = pair[0]; + pdir.tail[1] = pair[1]; + err = lfs_dir_commit(lfs, &pdir, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, + pdir.tail, sizeof(pdir.tail), + NULL)); + if (err) { + return err; } - lfs_pair_fromle32(pair); - if (!lfs_pair_sync(pair, pdir.tail)) { - // we have desynced - LFS_DEBUG("Fixing half-orphan %"PRIu32" %"PRIu32, - pair[0], pair[1]); + break; + } + } - pdir.tail[0] = pair[0]; - pdir.tail[1] = pair[1]; - err = lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pdir.tail, sizeof(pdir.tail), - NULL)); - if (err) { - return err; - } + memcpy(&pdir, &dir, sizeof(pdir)); + } - break; - } - } + // mark orphans as fixed + lfs_global_deorphaned(lfs, true); + return 0; +} - memcpy(&pdir, &dir, sizeof(pdir)); - } +static int lfs_fs_demove(lfs_t *lfs) { + // Fix bad moves + LFS_DEBUG("Fixing move %"PRIu32" %"PRIu32" %"PRIu32, + lfs->globals.s.movepair[0], + lfs->globals.s.movepair[1], + lfs->globals.s.moveid); - // mark orphan as fixed - lfs_global_deorphaned(lfs, false); + // fetch and delete the moved entry + lfs_mdir_t movedir; + int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.s.movepair); + if (err) { + return err; } - if (lfs->globals.s.moveid != 0x3ff) { - // Fix bad moves - LFS_DEBUG("Fixing move %"PRIu32" %"PRIu32" %"PRIu32, - lfs->globals.s.movepair[0], - lfs->globals.s.movepair[1], - lfs->globals.s.moveid); + // rely on cancel logic inside commit + err = lfs_dir_commit(lfs, &movedir, NULL); + if (err) { + return err; + } - // fetch and delete the moved entry - lfs_mdir_t movedir; - int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.s.movepair); + return 0; +} + +static int lfs_fs_forceconsistency(lfs_t *lfs) { + if (!lfs->globals.s.deorphaned) { + int err = lfs_fs_deorphan(lfs); if (err) { return err; } + } - // rely on cancel logic inside commit - err = lfs_dir_commit(lfs, &movedir, NULL); + if (lfs->globals.s.moveid != 0x3ff) { + int err = lfs_fs_demove(lfs); if (err) { return err; } diff --git a/lfs.h b/lfs.h index aa962620..dfafb5bd 100644 --- a/lfs.h +++ b/lfs.h @@ -178,12 +178,6 @@ struct lfs_config { // multiple of this value. lfs_size_t prog_size; - // Size of block caches. Each cache buffers a portion of a block in RAM. - // This determines the size of the read cache, the program cache, and a - // cache per file. Larger caches can improve performance by storing more - // data. Must be a multiple of the read and program sizes. - lfs_size_t cache_size; - // Size of an erasable block. This does not impact ram consumption and // may be larger than the physical erase size. However, this should be // kept small as each file currently takes up an entire block. @@ -193,6 +187,17 @@ struct lfs_config { // Number of erasable blocks on the device. lfs_size_t block_count; + // Number of erase cycles before we should move data to another block. + // May be zero to never move data, in which case no block-level + // wear-leveling is performed. + uint32_t block_cycles; + + // Size of block caches. Each cache buffers a portion of a block in RAM. + // This determines the size of the read cache, the program cache, and a + // cache per file. Larger caches can improve performance by storing more + // data. Must be a multiple of the read and program sizes. + lfs_size_t cache_size; + // Number of blocks to lookahead during block allocation. A larger // lookahead reduces the number of passes required to allocate a block. // The lookahead buffer requires only 1 bit per block so it can be quite diff --git a/tests/template.fmt b/tests/template.fmt index c745d24c..85759db6 100644 --- a/tests/template.fmt +++ b/tests/template.fmt @@ -69,10 +69,6 @@ uintmax_t test; #define LFS_PROG_SIZE LFS_READ_SIZE #endif -#ifndef LFS_CACHE_SIZE -#define LFS_CACHE_SIZE 64 -#endif - #ifndef LFS_BLOCK_SIZE #define LFS_BLOCK_SIZE 512 #endif @@ -81,6 +77,14 @@ uintmax_t test; #define LFS_BLOCK_COUNT 1024 #endif +#ifndef LFS_BLOCK_CYCLES +#define LFS_BLOCK_CYCLES 1024 +#endif + +#ifndef LFS_CACHE_SIZE +#define LFS_CACHE_SIZE 64 +#endif + #ifndef LFS_LOOKAHEAD #define LFS_LOOKAHEAD 128 #endif @@ -92,12 +96,13 @@ const struct lfs_config cfg = {{ .erase = &lfs_emubd_erase, .sync = &lfs_emubd_sync, - .read_size = LFS_READ_SIZE, - .prog_size = LFS_PROG_SIZE, - .cache_size = LFS_CACHE_SIZE, - .block_size = LFS_BLOCK_SIZE, - .block_count = LFS_BLOCK_COUNT, - .lookahead = LFS_LOOKAHEAD, + .read_size = LFS_READ_SIZE, + .prog_size = LFS_PROG_SIZE, + .block_size = LFS_BLOCK_SIZE, + .block_count = LFS_BLOCK_COUNT, + .block_cycles = LFS_BLOCK_CYCLES, + .cache_size = LFS_CACHE_SIZE, + .lookahead = LFS_LOOKAHEAD, }}; From 126ef8b07fc5f111fa2e3b4076441471dcd2e40e Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 9 Aug 2018 09:06:17 -0500 Subject: [PATCH 091/139] Added allocation randomization for dynamic wear-leveling This implements the second step of full dynamic wear-leveling, block allocation randomization. This is the key part the uniformly distributes wear across the filesystem, even through reboots. The entropy actually comes from the filesystem itself, by xoring together all of the CRCs in the metadata-pairs on the filesystem. While this sounds like a ridiculous operation, it's easy to do when we already scan the metadata-pairs at mount time. This gives us a random number we can use for block allocation. Unfortunately it's not a great general purpose random generator as the output only changes every filesystem write. Fortunately that's exactly when we need our allocator. --- Additionally, the randomization created a mess for the testing framework. Fortunately, this method of randomization is deterministic. A very useful property for reproducing bugs. --- emubd/lfs_emubd.c | 104 +++++++++++++++++++++++++++++++++++++----- emubd/lfs_emubd.h | 4 ++ lfs.c | 15 +++--- lfs.h | 1 + tests/corrupt.py | 66 ++++++++++++++------------- tests/debug.py | 98 +++++++++++++++++++++++++++++++++++++++ tests/stats.py | 4 +- tests/test_corrupt.sh | 13 +++--- tests/test_move.sh | 10 ++-- tests/test_orphan.sh | 2 +- 10 files changed, 254 insertions(+), 63 deletions(-) create mode 100755 tests/debug.py diff --git a/emubd/lfs_emubd.c b/emubd/lfs_emubd.c index 682ad925..de63057c 100644 --- a/emubd/lfs_emubd.c +++ b/emubd/lfs_emubd.c @@ -19,6 +19,40 @@ #include +// Emulated block device utils +static inline void lfs_emubd_tole32(lfs_emubd_t *emu) { + emu->cfg.read_size = lfs_tole32(emu->cfg.read_size); + emu->cfg.prog_size = lfs_tole32(emu->cfg.prog_size); + emu->cfg.block_size = lfs_tole32(emu->cfg.block_size); + emu->cfg.block_count = lfs_tole32(emu->cfg.block_count); + + emu->stats.read_count = lfs_tole32(emu->stats.read_count); + emu->stats.prog_count = lfs_tole32(emu->stats.prog_count); + emu->stats.erase_count = lfs_tole32(emu->stats.erase_count); + + for (int i = 0; i < sizeof(emu->history.blocks) / + sizeof(emu->history.blocks[0]); i++) { + emu->history.blocks[i] = lfs_tole32(emu->history.blocks[i]); + } +} + +static inline void lfs_emubd_fromle32(lfs_emubd_t *emu) { + emu->cfg.read_size = lfs_fromle32(emu->cfg.read_size); + emu->cfg.prog_size = lfs_fromle32(emu->cfg.prog_size); + emu->cfg.block_size = lfs_fromle32(emu->cfg.block_size); + emu->cfg.block_count = lfs_fromle32(emu->cfg.block_count); + + emu->stats.read_count = lfs_fromle32(emu->stats.read_count); + emu->stats.prog_count = lfs_fromle32(emu->stats.prog_count); + emu->stats.erase_count = lfs_fromle32(emu->stats.erase_count); + + for (int i = 0; i < sizeof(emu->history.blocks) / + sizeof(emu->history.blocks[0]); i++) { + emu->history.blocks[i] = lfs_fromle32(emu->history.blocks[i]); + } +} + + // Block device emulated on existing filesystem int lfs_emubd_create(const struct lfs_config *cfg, const char *path) { lfs_emubd_t *emu = cfg->context; @@ -46,20 +80,39 @@ int lfs_emubd_create(const struct lfs_config *cfg, const char *path) { } // Load stats to continue incrementing - snprintf(emu->child, LFS_NAME_MAX, "stats"); + snprintf(emu->child, LFS_NAME_MAX, ".stats"); FILE *f = fopen(emu->path, "r"); if (!f) { - return -errno; - } + memset(&emu->stats, 0, sizeof(emu->stats)); + } else { + size_t res = fread(&emu->stats, sizeof(emu->stats), 1, f); + lfs_emubd_fromle32(emu); + if (res < 1) { + return -errno; + } - size_t res = fread(&emu->stats, sizeof(emu->stats), 1, f); - if (res < 1) { - return -errno; + err = fclose(f); + if (err) { + return -errno; + } } - err = fclose(f); - if (err) { - return -errno; + // Load history + snprintf(emu->child, LFS_NAME_MAX, ".history"); + f = fopen(emu->path, "r"); + if (!f) { + memset(&emu->history, 0, sizeof(emu->history)); + } else { + size_t res = fread(&emu->history, sizeof(emu->history), 1, f); + lfs_emubd_fromle32(emu); + if (res < 1) { + return -errno; + } + + err = fclose(f); + if (err) { + return -errno; + } } return 0; @@ -161,6 +214,13 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, return -errno; } + // update history and stats + if (block != emu->history.blocks[0]) { + memcpy(&emu->history.blocks[1], &emu->history.blocks[0], + sizeof(emu->history) - sizeof(emu->history.blocks[0])); + emu->history.blocks[0] = block; + } + emu->stats.prog_count += 1; return 0; } @@ -206,13 +266,15 @@ int lfs_emubd_sync(const struct lfs_config *cfg) { lfs_emubd_t *emu = cfg->context; // Just write out info/stats for later lookup - snprintf(emu->child, LFS_NAME_MAX, "config"); + snprintf(emu->child, LFS_NAME_MAX, ".config"); FILE *f = fopen(emu->path, "w"); if (!f) { return -errno; } + lfs_emubd_tole32(emu); size_t res = fwrite(&emu->cfg, sizeof(emu->cfg), 1, f); + lfs_emubd_fromle32(emu); if (res < 1) { return -errno; } @@ -222,13 +284,33 @@ int lfs_emubd_sync(const struct lfs_config *cfg) { return -errno; } - snprintf(emu->child, LFS_NAME_MAX, "stats"); + snprintf(emu->child, LFS_NAME_MAX, ".stats"); f = fopen(emu->path, "w"); if (!f) { return -errno; } + lfs_emubd_tole32(emu); res = fwrite(&emu->stats, sizeof(emu->stats), 1, f); + lfs_emubd_fromle32(emu); + if (res < 1) { + return -errno; + } + + err = fclose(f); + if (err) { + return -errno; + } + + snprintf(emu->child, LFS_NAME_MAX, ".history"); + f = fopen(emu->path, "w"); + if (!f) { + return -errno; + } + + lfs_emubd_tole32(emu); + res = fwrite(&emu->history, sizeof(emu->history), 1, f); + lfs_emubd_fromle32(emu); if (res < 1) { return -errno; } diff --git a/emubd/lfs_emubd.h b/emubd/lfs_emubd.h index 0fd43875..64afa3ee 100644 --- a/emubd/lfs_emubd.h +++ b/emubd/lfs_emubd.h @@ -45,6 +45,10 @@ typedef struct lfs_emubd { uint64_t erase_count; } stats; + struct { + lfs_block_t blocks[4]; + } history; + struct { uint32_t read_size; uint32_t prog_size; diff --git a/lfs.c b/lfs.c index 3f8c33c6..24900c03 100644 --- a/lfs.c +++ b/lfs.c @@ -879,6 +879,8 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, dir->tail[1] = temptail[1]; dir->split = tempsplit; dir->locals = templocals; + + lfs->seed ^= crc; crc = 0xffffffff; } else { err = lfs_bd_crc32(lfs, dir->pair[0], @@ -2874,6 +2876,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[0] = 0xffffffff; lfs->root[1] = 0xffffffff; lfs->mlist = NULL; + lfs->seed = 0; lfs->globals.s.movepair[0] = 0xffffffff; lfs->globals.s.movepair[1] = 0xffffffff; lfs->globals.s.moveid = 0x3ff; @@ -2962,12 +2965,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - // setup free lookahead - lfs->free.off = 0; - lfs->free.size = 0; - lfs->free.i = 0; - lfs_alloc_ack(lfs); - // load superblock lfs_mdir_t root; err = lfs_dir_fetch(lfs, &root, (const lfs_block_t[2]){0, 1}); @@ -3065,6 +3062,12 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->globals.s.moveid); } + // setup free lookahead + lfs->free.off = lfs->seed % lfs->cfg->block_size; + lfs->free.size = 0; + lfs->free.i = 0; + lfs_alloc_ack(lfs); + return 0; cleanup: diff --git a/lfs.h b/lfs.h index dfafb5bd..f2672ac8 100644 --- a/lfs.h +++ b/lfs.h @@ -382,6 +382,7 @@ typedef struct lfs { lfs_block_t root[2]; lfs_mlist_t *mlist; + uint32_t seed; lfs_global_t globals; lfs_global_t locals; diff --git a/tests/corrupt.py b/tests/corrupt.py index 76f07ced..5e2a9a3a 100755 --- a/tests/corrupt.py +++ b/tests/corrupt.py @@ -3,37 +3,41 @@ import struct import sys import os +import argparse -def main(*paths): - # find most recent block - file = None - rev = None - for path in paths: - try: - nfile = open(path, 'r+b') - nrev, = struct.unpack('> 22 + id = (tag & 0x003ff000) >> 12 + size = (tag & 0x00000fff) >> 0 + + data = file.read(size) + if type == 0x0f0: + crc = binascii.crc32(data[:4], crc) + else: + crc = binascii.crc32(data, crc) + + print '%04x: %08x %-14s %3s %3d %-23s %-8s' % ( + off, tag, + typeof(type) + (' bad!' if type == 0x0f0 and ~crc else ''), + id if id != 0x3ff else '.', size, + ' '.join('%02x' % ord(c) for c in data[:8]), + ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8])) + + off += tag & 0xfff + if type == 0x0f0: + crc = 0 + +if __name__ == "__main__": + import sys + main(*sys.argv[1:]) diff --git a/tests/stats.py b/tests/stats.py index 2ba1fb65..ab21b59a 100755 --- a/tests/stats.py +++ b/tests/stats.py @@ -7,7 +7,7 @@ import re def main(): - with open('blocks/config') as file: + with open('blocks/.config') as file: s = struct.unpack(' 0; lfs_unmount(&lfs) => 0; TEST -tests/corrupt.py blocks/{4,5} +tests/corrupt.py -n 1 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -86,8 +86,7 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hello", "d/hello") => 0; lfs_unmount(&lfs) => 0; TEST -tests/corrupt.py blocks/{6,7} -tests/corrupt.py blocks/{8,9} +tests/corrupt.py -n 2 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; @@ -166,7 +165,7 @@ tests/test.py << TEST lfs_rename(&lfs, "b/hi", "c/hi") => 0; lfs_unmount(&lfs) => 0; TEST -tests/corrupt.py blocks/{4,5} +tests/corrupt.py -n 1 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "b") => 0; @@ -193,8 +192,7 @@ tests/test.py << TEST lfs_rename(&lfs, "c/hi", "d/hi") => 0; lfs_unmount(&lfs) => 0; TEST -tests/corrupt.py blocks/{6,7} -tests/corrupt.py blocks/{8,9} +tests/corrupt.py -n 2 tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_dir_open(&lfs, &dir[0], "c") => 0; diff --git a/tests/test_orphan.sh b/tests/test_orphan.sh index e66e5927..9c2cb7b6 100755 --- a/tests/test_orphan.sh +++ b/tests/test_orphan.sh @@ -17,7 +17,7 @@ tests/test.py << TEST TEST # corrupt most recent commit, this should be the update to the previous # linked-list entry and should orphan the child -tests/corrupt.py blocks/{6,7} +tests/corrupt.py tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; From 38011f4cd0e31295f1b25abf72d61e7c7f3827ec Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 11 Aug 2018 13:45:13 -0500 Subject: [PATCH 092/139] Fixed minor memory leak - Fixed memory leak - Change lfs_globals_zero to use memset as this made leak checking more effective - Checked for leaks with valgrind --- lfs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lfs.c b/lfs.c index 24900c03..ec0792d0 100644 --- a/lfs.c +++ b/lfs.c @@ -447,7 +447,7 @@ static inline bool lfs_global_iszero(const lfs_global_t *a) { } static inline void lfs_global_zero(lfs_global_t *a) { - lfs_global_xor(a, a); + memset(a, 0, sizeof(lfs_global_t)); } static inline void lfs_global_fromle32(lfs_global_t *a) { @@ -2083,7 +2083,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { } // clean up memory - if (file->cfg->buffer) { + if (!file->cfg->buffer) { lfs_free(file->cache.buffer); } From 21217d75ada4d9c2e91b39c93c89cdd2568d8bbf Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 11 Aug 2018 23:05:52 -0500 Subject: [PATCH 093/139] Dropped lfs_fs_getattr for the more implicit lfs_getattr("/") This was a pretty simple oversight on my part. Conceptually, there's no difference between lfs_fs_getattr and lfs_getattr("/"). Any operations on directories can be applied "globally" by referring to the root directory. Implementation wise, this actually fixes the "corner case" of storing attributes on the root directory, which is broken since the root directory doesn't have a related entry. Instead we need to use the root superblock for this purpose. Fewer functions means less code to document and maintain, so this is a nice benefit. Now we just have a single lfs_getattr/setattr/removeattr set of functions along with the ability to access attributes atomically in lfs_file_opencfg. --- lfs.c | 141 ++++++++++++++++++++++---------------------- lfs.h | 31 +--------- tests/debug.py | 12 +++- tests/test_attrs.sh | 54 ++++++++--------- 4 files changed, 107 insertions(+), 131 deletions(-) diff --git a/lfs.c b/lfs.c index ec0792d0..e1c41de3 100644 --- a/lfs.c +++ b/lfs.c @@ -540,7 +540,7 @@ static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, uint16_t id, const struct lfs_attr *attrs); static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, - uint16_t fromid, uint16_t toid, + uint32_t frommask, uint32_t fromtag, uint32_t tomask, uint32_t totag, const lfs_mdir_t *dir, const lfs_mattr_t *attrs); static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, @@ -548,7 +548,8 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, if (lfs_tag_subtype(tag) == LFS_FROM_MOVE) { // special case for moves return lfs_commit_move(lfs, commit, - lfs_tag_size(tag), lfs_tag_id(tag), + 0x003ff000, LFS_MKTAG(0, lfs_tag_size(tag), 0), + 0x003ff000, LFS_MKTAG(0, lfs_tag_id(tag), 0), buffer, NULL); } else if (lfs_tag_subtype(tag) == LFS_FROM_ATTRS) { // special case for custom attributes @@ -617,7 +618,7 @@ static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, } static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, - uint16_t fromid, uint16_t toid, + uint32_t frommask, uint32_t fromtag, uint32_t tomask, uint32_t totag, const lfs_mdir_t *dir, const lfs_mattr_t *attrs) { // iterate through list and commits, only committing unique entries lfs_off_t off = dir->off; @@ -650,17 +651,15 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, } if (lfs_tag_type(tag) == LFS_TYPE_DELETE && - lfs_tag_id(tag) <= fromid) { + lfs_tag_id(tag) <= lfs_tag_id(fromtag)) { // something was deleted, we need to move around it - fromid += 1; - } else if (lfs_tag_id(tag) != fromid) { - // ignore non-matching ids - } else { + fromtag += LFS_MKTAG(0, 1, 0); + } else if ((tag & frommask) == (fromtag & frommask)) { // check if type has already been committed int32_t res = lfs_commit_get(lfs, commit->block, commit->off, commit->ptag, lfs_tag_isuser(tag) ? 0x7ffff000 : 0x7c3ff000, - (tag & 0x7fc00000) | LFS_MKTAG(0, toid, 0), + (tag & ~tomask) | totag, 0, NULL, true); if (res < 0 && res != LFS_ERR_NOENT) { return res; @@ -669,7 +668,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, if (res == LFS_ERR_NOENT) { // update id and commit, as we are currently unique int err = lfs_commit_attr(lfs, commit, - (tag & 0xffc00fff) | LFS_MKTAG(0, toid, 0), + (tag & ~tomask) | totag, buffer); if (err) { return err; @@ -1068,8 +1067,7 @@ static int lfs_dir_compact(lfs_t *lfs, // do we have enough space to expand? if (res < lfs->cfg->block_count/2) { LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); - ack = 0; - exhausted = (lfs_pair_cmp(dir->pair, lfs->root) != 0); + exhausted = true; goto split; } } else { @@ -1118,7 +1116,9 @@ static int lfs_dir_compact(lfs_t *lfs, // commit with a move for (uint16_t id = begin; id < end; id++) { err = lfs_commit_move(lfs, &commit, - id, id - begin, source, attrs); + 0x003ff000, LFS_MKTAG(0, id, 0), + 0x003ff000, LFS_MKTAG(0, id - begin, 0), + source, attrs); if (err) { if (err == LFS_ERR_NOSPC) { goto split; @@ -1134,7 +1134,23 @@ static int lfs_dir_compact(lfs_t *lfs, // reopen reserved space at the end commit.end = lfs->cfg->block_size - 8; + if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { + // move over (duplicate) superblock if we are root + err = lfs_commit_move(lfs, &commit, + 0x7c000000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 0), + 0x7ffff000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 0), + source, attrs); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + if (!relocated) { + // commit any globals, unless we're relocating, in which case our + // parent will steal our globals err = lfs_commit_globals(lfs, &commit, &dir->locals); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1178,8 +1194,7 @@ static int lfs_dir_compact(lfs_t *lfs, // commit no longer fits, need to split dir, // drop caches and create tail lfs_cache_drop(lfs, &lfs->pcache); - - if (ack == -1) { + if (!exhausted && ack < 0) { // If we can't fit in this block, we won't fit in next block return LFS_ERR_NOSPC; } @@ -1190,11 +1205,16 @@ static int lfs_dir_compact(lfs_t *lfs, return err; } + if (exhausted) { + lfs->root[0] = tail.pair[0]; + lfs->root[1] = tail.pair[1]; + } + tail.split = dir->split; tail.tail[0] = dir->tail[0]; tail.tail[1] = dir->tail[1]; - err = lfs_dir_compact(lfs, &tail, attrs, source, ack+1-exhausted, end); + err = lfs_dir_compact(lfs, &tail, attrs, source, ack+1, end); if (err) { return err; } @@ -2770,9 +2790,19 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, return res; } + uint16_t id = lfs_tag_id(res); + if (id == 0x3ff) { + // special case for root + id = 0; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + } + res = lfs_dir_get(lfs, &cwd, 0x7ffff000, - LFS_MKTAG(0x100 | type, lfs_tag_id(res), - lfs_min(size, lfs->attr_max)), buffer); + LFS_MKTAG(0x100 | type, id, lfs_min(size, lfs->attr_max)), + buffer); if (res < 0 && res != LFS_ERR_NOENT) { return res; } @@ -2792,8 +2822,18 @@ int lfs_setattr(lfs_t *lfs, const char *path, return res; } + uint16_t id = lfs_tag_id(res); + if (id == 0x3ff) { + // special case for root + id = 0; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + } + return lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(0x100 | type, lfs_tag_id(res), buffer, size, + LFS_MKATTR(0x100 | type, id, buffer, size, NULL)); } @@ -2941,9 +2981,8 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_tole32(&superblock); err = lfs_dir_commit(lfs, &root, - LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), - LFS_MKATTR(LFS_TYPE_ROOT, 1, NULL, 0, - NULL))); + LFS_MKATTR(LFS_TYPE_ROOT, 0, &superblock, sizeof(superblock), + NULL)); if (err) { goto cleanup; } @@ -2965,15 +3004,18 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - // load superblock + // find root/superblock lfs_mdir_t root; - err = lfs_dir_fetch(lfs, &root, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; + lfs_superblock_t superblock; + int32_t tag = lfs_dir_find(lfs, + &root, (const lfs_block_t[2]){0, 1}, false, 0x7fc00000, + LFS_MKTAG(LFS_TYPE_ROOT, 0, 8), "littlefs"); + if (tag < 0) { + err = tag; + goto cleanup; } - lfs_superblock_t superblock; - int32_t res = lfs_dir_get(lfs, &root, 0x7fc00000, + int32_t res = lfs_dir_get(lfs, &root, 0x7c000000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), &superblock); if (res < 0) { @@ -2982,14 +3024,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_superblock_fromle32(&superblock); - // find root - int32_t tag = lfs_dir_find(lfs, - &root, (const lfs_block_t[2]){0, 1}, false, 0x7fc00000, - LFS_MKTAG(LFS_TYPE_ROOT, 0, 0), NULL); - if (tag < 0) { - return tag; - } - lfs->root[0] = root.pair[0]; lfs->root[1] = root.pair[1]; @@ -3370,41 +3404,6 @@ static int lfs_fs_forceconsistency(lfs_t *lfs) { return 0; } -lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, - uint8_t type, void *buffer, lfs_size_t size) { - lfs_mdir_t superdir; - int err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - int32_t res = lfs_dir_get(lfs, &superdir, 0x7ffff000, - LFS_MKTAG(0x100 | type, 0, - lfs_min(size, lfs->attr_max)), buffer); - if (res < 0) { - return res; - } - - return (res == LFS_ERR_NOENT) ? 0 : lfs_tag_size(res); -} - -int lfs_fs_setattr(lfs_t *lfs, - uint8_t type, const void *buffer, lfs_size_t size) { - if (size > lfs->attr_max) { - return LFS_ERR_NOSPC; - } - - lfs_mdir_t superdir; - int err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); - if (err) { - return err; - } - - return lfs_dir_commit(lfs, &superdir, - LFS_MKATTR(0x100 | type, 0, buffer, size, - NULL)); -} - static int lfs_fs_size_count(void *p, lfs_block_t block) { (void)block; lfs_size_t *size = p; diff --git a/lfs.h b/lfs.h index f2672ac8..e8165151 100644 --- a/lfs.h +++ b/lfs.h @@ -94,7 +94,7 @@ enum lfs_type { // internally used types LFS_TYPE_USER = 0x100, LFS_TYPE_SUPERBLOCK = 0x011, - LFS_TYPE_ROOT = 0x012, + LFS_TYPE_ROOT = 0x010, LFS_TYPE_NAME = 0x000, LFS_TYPE_DELETE = 0x030, LFS_TYPE_STRUCT = 0x040, @@ -624,35 +624,6 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // Returns a negative error code on failure. int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); -// Get custom attributes on the filesystem -// -// Custom attributes are uniquely identified by an 8-bit type and limited -// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller than -// the buffer, it will be padded with zeros. If the stored attribute is larger, -// then it will be silently truncated. -// -// Note, filesystem-level attributes are not available for wear-leveling -// -// Returns the size of the attribute, or a negative error code on failure. -// Note, the returned size is the size of the attribute on disk, irrespective -// of the size of the buffer. This can be used to dynamically allocate a buffer -// or check for existance. -lfs_ssize_t lfs_fs_getattr(lfs_t *lfs, - uint8_t type, void *buffer, lfs_size_t size); - -// Set custom attributes on the filesystem -// -// Custom attributes are uniquely identified by an 8-bit type and limited -// to LFS_ATTR_MAX bytes. If an attribute is not found, it will be -// implicitly created, and setting the size of an attribute to zero deletes -// the attribute. -// -// Note, filesystem-level attributes are not available for wear-leveling -// -// Returns a negative error code on failure. -int lfs_fs_setattr(lfs_t *lfs, - uint8_t type, const void *buffer, lfs_size_t size); - #ifdef __cplusplus } /* extern "C" */ diff --git a/tests/debug.py b/tests/debug.py index 58fb81e9..65b0ad0c 100755 --- a/tests/debug.py +++ b/tests/debug.py @@ -7,7 +7,7 @@ (0x1ff, 0x001): 'reg', (0x1ff, 0x002): 'dir', (0x1ff, 0x011): 'superblock', - (0x1ff, 0x012): 'root', + (0x1ff, 0x010): 'root', (0x1ff, 0x030): 'delete', (0x1f0, 0x080): 'globals', (0x1ff, 0x0c0): 'tail soft', @@ -50,9 +50,13 @@ def main(*blocks): crc = ncrc versions.append((nrev, '%s (rev %d)' % (block, nrev))) - except IOError: + except (IOError, struct.error): pass + if not file: + print 'Bad metadata pair {%s}' % ', '.join(blocks) + return 1 + print "--- %s ---" % ', '.join(v for _,v in sorted(versions, reverse=True)) # go through each tag, print useful information @@ -93,6 +97,8 @@ def main(*blocks): if type == 0x0f0: crc = 0 + return 0 + if __name__ == "__main__": import sys - main(*sys.argv[1:]) + sys.exit(main(*sys.argv[1:])) diff --git a/tests/test_attrs.sh b/tests/test_attrs.sh index cb4f4dc5..bbe78cc4 100755 --- a/tests/test_attrs.sh +++ b/tests/test_attrs.sh @@ -77,55 +77,55 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST -echo "--- Set/get fs attribute ---" +echo "--- Set/get root attribute ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_fs_setattr(&lfs, 'A', "aaaa", 4) => 0; - lfs_fs_setattr(&lfs, 'B', "bbbbbb", 6) => 0; - lfs_fs_setattr(&lfs, 'C', "ccccc", 5) => 0; - lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; - lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 6; - lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; + lfs_setattr(&lfs, "/", 'A', "aaaa", 4) => 0; + lfs_setattr(&lfs, "/", 'B', "bbbbbb", 6) => 0; + lfs_setattr(&lfs, "/", 'C', "ccccc", 5) => 0; + lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "/", 'B', buffer+4, 6) => 6; + lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "bbbbbb", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_fs_setattr(&lfs, 'B', "", 0) => 0; - lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; - lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 0; - lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; + lfs_setattr(&lfs, "/", 'B', "", 0) => 0; + lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "/", 'B', buffer+4, 6) => 0; + lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_fs_setattr(&lfs, 'B', "dddddd", 6) => 0; - lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; - lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 6; - lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; + lfs_setattr(&lfs, "/", 'B', "dddddd", 6) => 0; + lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "/", 'B', buffer+4, 6) => 6; + lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "dddddd", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_fs_setattr(&lfs, 'B', "eee", 3) => 0; - lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; - lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 3; - lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; + lfs_setattr(&lfs, "/", 'B', "eee", 3) => 0; + lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "/", 'B', buffer+4, 6) => 3; + lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "eee\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - lfs_fs_setattr(&lfs, 'A', buffer, LFS_ATTR_MAX+1) => LFS_ERR_NOSPC; - lfs_fs_setattr(&lfs, 'B', "fffffffff", 9) => 0; - lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; - lfs_fs_getattr(&lfs, 'B', buffer+4, 6) => 9; - lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5; + lfs_setattr(&lfs, "/", 'A', buffer, LFS_ATTR_MAX+1) => LFS_ERR_NOSPC; + lfs_setattr(&lfs, "/", 'B', "fffffffff", 9) => 0; + lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "/", 'B', buffer+4, 6) => 9; + lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5; lfs_unmount(&lfs) => 0; TEST tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_fs_getattr(&lfs, 'A', buffer, 4) => 4; - lfs_fs_getattr(&lfs, 'B', buffer+4, 9) => 9; - lfs_fs_getattr(&lfs, 'C', buffer+13, 5) => 5; + lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "/", 'B', buffer+4, 9) => 9; + lfs_getattr(&lfs, "/", 'C', buffer+13, 5) => 5; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "fffffffff", 9) => 0; memcmp(buffer+13, "ccccc", 5) => 0; From d5e800575daaf871ed629d5f52af2c4fe1a4cc29 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 13 Aug 2018 09:03:13 -0500 Subject: [PATCH 094/139] Collapsed recursive deorphans into a single pass Because a block can go bad at any time, if we're unlucky, we may end up generating multiple orphans in a single metadata write. This is exacerbated by the early eviction in dynamic wear-leveling. We can't track _all_ orphans, because that would require unbounded storage and significantly complicate things, but there are a handful of intentional orphans we do track because they are easy to resolve without the O(n^2) deorphan scan. These are anytime we intentionally remove a metadata-pair. Initially we cleaned up orphans as they occur with whatever knowledge we do have, and just accepted the extra O(n^2) deorphan scans in the unlucky case. However we can do a bit better by being lazy and leaving deorphaning up to the next metadata write. This needs to work with the known orphans while still setting the orphan flag on disk correctly. To accomplish this we replace the internal flag with a small counter. Note, this means that our internal representation of orphans differs from what's on disk. This is annoying but not the end of the world. --- lfs.c | 153 ++++++++++++++++++++++++++++------------------------------ lfs.h | 9 +++- 2 files changed, 82 insertions(+), 80 deletions(-) diff --git a/lfs.c b/lfs.c index e1c41de3..40744c9e 100644 --- a/lfs.c +++ b/lfs.c @@ -451,31 +451,32 @@ static inline void lfs_global_zero(lfs_global_t *a) { } static inline void lfs_global_fromle32(lfs_global_t *a) { - lfs_pair_fromle32(a->s.movepair); - a->s.moveid = lfs_fromle16(a->s.moveid); + lfs_pair_fromle32(a->l.movepair); + a->l.moveid = lfs_fromle16(a->l.moveid); } static inline void lfs_global_tole32(lfs_global_t *a) { - lfs_pair_tole32(a->s.movepair); - a->s.moveid = lfs_tole16(a->s.moveid); + lfs_pair_tole32(a->l.movepair); + a->l.moveid = lfs_tole16(a->l.moveid); } static inline void lfs_global_move(lfs_t *lfs, const lfs_block_t pair[2], uint16_t id) { lfs_global_t diff; lfs_global_zero(&diff); - diff.s.movepair[0] ^= lfs->globals.s.movepair[0] ^ pair[0]; - diff.s.movepair[1] ^= lfs->globals.s.movepair[1] ^ pair[1]; - diff.s.moveid ^= lfs->globals.s.moveid ^ id; + diff.l.movepair[0] ^= lfs->globals.g.movepair[0] ^ pair[0]; + diff.l.movepair[1] ^= lfs->globals.g.movepair[1] ^ pair[1]; + diff.l.moveid ^= lfs->globals.g.moveid ^ id; lfs_global_fromle32(&lfs->locals); lfs_global_xor(&lfs->locals, &diff); lfs_global_tole32(&lfs->locals); lfs_global_xor(&lfs->globals, &diff); } -static inline void lfs_global_deorphaned(lfs_t *lfs, bool deorphaned) { - lfs->locals.s.deorphaned ^= lfs->globals.s.deorphaned ^ deorphaned; - lfs->globals.s.deorphaned ^= lfs->globals.s.deorphaned ^ deorphaned; +static inline void lfs_global_orphans(lfs_t *lfs, int8_t orphans) { + lfs->locals.l.deorphaned ^= (lfs->globals.g.orphans == 0); + lfs->locals.l.deorphaned ^= (lfs->globals.g.orphans + orphans == 0); + lfs->globals.g.orphans += orphans; } @@ -688,7 +689,7 @@ static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, lfs_global_xor(locals, &lfs->locals); int err = lfs_commit_attr(lfs, commit, - LFS_MKTAG(LFS_TYPE_GLOBALS + locals->s.deorphaned, 0x3ff, 10), + LFS_MKTAG(LFS_TYPE_GLOBALS + locals->l.deorphaned, 0x3ff, 10), locals); lfs_global_xor(locals, &lfs->locals); return err; @@ -907,7 +908,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, } lfs_pair_fromle32(temptail); } else if (lfs_tag_subtype(tag) == LFS_TYPE_GLOBALS) { - templocals.s.deorphaned = (lfs_tag_type(tag) & 1); + templocals.l.deorphaned = (lfs_tag_type(tag) & 1); err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), &templocals, 10); if (err) { @@ -973,11 +974,11 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, // consider what we have good enough if (dir->off > 0) { // synthetic move - if (lfs_pair_cmp(dir->pair, lfs->globals.s.movepair) == 0) { - if (lfs->globals.s.moveid == lfs_tag_id(foundtag)) { + if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0) { + if (lfs->globals.g.moveid == lfs_tag_id(foundtag)) { foundtag = LFS_ERR_NOENT; } else if (lfs_tag_isvalid(foundtag) && - lfs->globals.s.moveid < lfs_tag_id(foundtag)) { + lfs->globals.g.moveid < lfs_tag_id(foundtag)) { foundtag -= LFS_MKTAG(0, 1, 0); } } @@ -1026,8 +1027,8 @@ static int32_t lfs_dir_find(lfs_t *lfs, static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, uint32_t getmask, uint32_t gettag, void *buffer) { int32_t getdiff = 0; - if (lfs_pair_cmp(dir->pair, lfs->globals.s.movepair) == 0 && - lfs_tag_id(gettag) <= lfs->globals.s.moveid) { + if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0 && + lfs_tag_id(gettag) <= lfs->globals.g.moveid) { // synthetic moves getdiff = LFS_MKTAG(0, 1, 0); } @@ -1270,17 +1271,17 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_mattr_t cancelattr; lfs_global_t canceldiff; lfs_global_zero(&canceldiff); - if (lfs_pair_cmp(dir->pair, lfs->globals.s.movepair) == 0) { + if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0) { // Wait, we have the move? Just cancel this out here // We need to, or else the move can become outdated - canceldiff.s.movepair[0] ^= lfs->globals.s.movepair[0] ^ 0xffffffff; - canceldiff.s.movepair[1] ^= lfs->globals.s.movepair[1] ^ 0xffffffff; - canceldiff.s.moveid ^= lfs->globals.s.moveid ^ 0x3ff; + canceldiff.l.movepair[0] ^= lfs->globals.g.movepair[0] ^ 0xffffffff; + canceldiff.l.movepair[1] ^= lfs->globals.g.movepair[1] ^ 0xffffffff; + canceldiff.l.moveid ^= lfs->globals.g.moveid ^ 0x3ff; lfs_global_fromle32(&lfs->locals); lfs_global_xor(&lfs->locals, &canceldiff); lfs_global_tole32(&lfs->locals); - cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.s.moveid, 0); + cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.l.moveid, 0); cancelattr.next = attrs; attrs = &cancelattr; } @@ -2636,7 +2637,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // mark fs as orphaned - lfs_global_deorphaned(lfs, false); + lfs_global_orphans(lfs, +1); } // delete the entry @@ -2648,14 +2649,14 @@ int lfs_remove(lfs_t *lfs, const char *path) { } if (lfs_tag_type(tag) == LFS_TYPE_DIR) { + // fix orphan + lfs_global_orphans(lfs, -1); + err = lfs_fs_pred(lfs, dir.pair, &cwd); if (err) { return err; } - // fix orphan - lfs_global_deorphaned(lfs, true); - // steal state cwd.tail[0] = dir.tail[0]; cwd.tail[1] = dir.tail[1]; @@ -2696,36 +2697,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { uint16_t newid = lfs_tag_id(prevtag); lfs_mdir_t prevdir; - if (prevtag != LFS_ERR_NOENT) { - // check that we have same type - if (lfs_tag_type(prevtag) != lfs_tag_type(oldtag)) { - return LFS_ERR_ISDIR; - } - - if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { - // must be empty before removal - lfs_block_t prevpair[2]; - int32_t res = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair); - if (res < 0) { - return res; - } - lfs_pair_fromle32(prevpair); - - // must be empty before removal - err = lfs_dir_fetch(lfs, &prevdir, prevpair); - if (err) { - return err; - } - - if (prevdir.count > 0 || prevdir.split) { - return LFS_ERR_NOTEMPTY; - } - - // mark fs as orphaned - lfs_global_deorphaned(lfs, false); - } - } else { + if (prevtag == LFS_ERR_NOENT) { // check that name fits lfs_size_t nlen = strlen(newpath); if (nlen > lfs->name_max) { @@ -2734,6 +2706,30 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // get next id newid = newcwd.count; + } else if (lfs_tag_type(prevtag) != lfs_tag_type(oldtag)) { + return LFS_ERR_ISDIR; + } else if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { + // must be empty before removal + lfs_block_t prevpair[2]; + int32_t res = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair); + if (res < 0) { + return res; + } + lfs_pair_fromle32(prevpair); + + // must be empty before removal + err = lfs_dir_fetch(lfs, &prevdir, prevpair); + if (err) { + return err; + } + + if (prevdir.count > 0 || prevdir.split) { + return LFS_ERR_NOTEMPTY; + } + + // mark fs as orphaned + lfs_global_orphans(lfs, +1); } // create move to fix later @@ -2758,14 +2754,14 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } if (prevtag != LFS_ERR_NOENT && lfs_tag_type(prevtag) == LFS_TYPE_DIR) { + // fix orphan + lfs_global_orphans(lfs, -1); + err = lfs_fs_pred(lfs, prevdir.pair, &newcwd); if (err) { return err; } - // fix orphan - lfs_global_deorphaned(lfs, true); - // steal state newcwd.tail[0] = prevdir.tail[0]; newcwd.tail[1] = prevdir.tail[1]; @@ -2917,10 +2913,10 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[1] = 0xffffffff; lfs->mlist = NULL; lfs->seed = 0; - lfs->globals.s.movepair[0] = 0xffffffff; - lfs->globals.s.movepair[1] = 0xffffffff; - lfs->globals.s.moveid = 0x3ff; - lfs->globals.s.deorphaned = true; + lfs->globals.g.movepair[0] = 0xffffffff; + lfs->globals.g.movepair[1] = 0xffffffff; + lfs->globals.g.moveid = 0x3ff; + lfs->globals.g.orphans = 0; lfs_global_zero(&lfs->locals); return 0; @@ -3089,11 +3085,11 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_global_fromle32(&lfs->locals); lfs_global_xor(&lfs->globals, &lfs->locals); lfs_global_zero(&lfs->locals); - if (!lfs_pair_isnull(lfs->globals.s.movepair)) { + if (!lfs_pair_isnull(lfs->globals.g.movepair)) { LFS_DEBUG("Found move %"PRIu32" %"PRIu32" %"PRIu32, - lfs->globals.s.movepair[0], - lfs->globals.s.movepair[1], - lfs->globals.s.moveid); + lfs->globals.g.movepair[0], + lfs->globals.g.movepair[1], + lfs->globals.g.moveid); } // setup free lookahead @@ -3254,7 +3250,8 @@ static int lfs_fs_relocate(lfs_t *lfs, if (tag != LFS_ERR_NOENT) { // update disk, this creates a desync - lfs_global_deorphaned(lfs, false); + lfs_global_orphans(lfs, +1); + lfs_pair_tole32(newpair); int err = lfs_dir_commit(lfs, &parent, &(lfs_mattr_t){.tag=tag, .buffer=newpair}); @@ -3263,8 +3260,8 @@ static int lfs_fs_relocate(lfs_t *lfs, return err; } - // clean up bad block, which should now be a desync - return lfs_fs_deorphan(lfs); + // next step, clean up orphans + lfs_global_orphans(lfs, -1); } // find pred @@ -3275,7 +3272,7 @@ static int lfs_fs_relocate(lfs_t *lfs, // if we can't find dir, it must be new if (err != LFS_ERR_NOENT) { - // just replace bad pair, no desync can occur + // replace bad pair, either we clean up desync, or no desync occured parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; err = lfs_dir_commit(lfs, &parent, @@ -3359,20 +3356,20 @@ static int lfs_fs_deorphan(lfs_t *lfs) { } // mark orphans as fixed - lfs_global_deorphaned(lfs, true); + lfs_global_orphans(lfs, -lfs->globals.g.orphans); return 0; } static int lfs_fs_demove(lfs_t *lfs) { // Fix bad moves LFS_DEBUG("Fixing move %"PRIu32" %"PRIu32" %"PRIu32, - lfs->globals.s.movepair[0], - lfs->globals.s.movepair[1], - lfs->globals.s.moveid); + lfs->globals.g.movepair[0], + lfs->globals.g.movepair[1], + lfs->globals.g.moveid); // fetch and delete the moved entry lfs_mdir_t movedir; - int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.s.movepair); + int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.g.movepair); if (err) { return err; } @@ -3387,14 +3384,14 @@ static int lfs_fs_demove(lfs_t *lfs) { } static int lfs_fs_forceconsistency(lfs_t *lfs) { - if (!lfs->globals.s.deorphaned) { + if (lfs->globals.g.orphans) { int err = lfs_fs_deorphan(lfs); if (err) { return err; } } - if (lfs->globals.s.moveid != 0x3ff) { + if (lfs->globals.g.moveid != 0x3ff) { int err = lfs_fs_demove(lfs); if (err) { return err; diff --git a/lfs.h b/lfs.h index e8165151..839cf831 100644 --- a/lfs.h +++ b/lfs.h @@ -302,8 +302,13 @@ typedef union lfs_global { struct { lfs_block_t movepair[2]; uint16_t moveid; - bool deorphaned; - } s; + uint8_t deorphaned; + } l; + struct { + lfs_block_t movepair[2]; + uint16_t moveid; + uint8_t orphans; + } g; } lfs_global_t; typedef struct lfs_mdir { From a2532a34cdfc39657436dcbec86ba7350c515be7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 13 Aug 2018 14:08:30 -0500 Subject: [PATCH 095/139] Fixed inline files when inline_max == cache_size The initial implementation of inline files was thrown together fairly quicky, however it has worked well so far and there hasn't been much reason to change it. One shortcut was to trick file writes into thinking they are writing to imaginary blocks. This works well and reuses most of the file code paths, as long as we don't flush the imaginary block out to disk. Initially we did this by limiting inline_max to cache_max-1, ensuring that the cache never fills up and gets flushed. This was a rather dirty hack, the better solution, implemented here, is to handle the representation of an "imaginary" block correctly all the way down into the cache layer. So now for files specifically, the value -1 represents a null pointer, and the value -2 represents an "imaginary" block. This may become a problem if the number of blocks approaches the max, however this -2 value is never written to disk and can be changed in the future without breaking compatibility. --- lfs.c | 9 ++++++--- tests/test_alloc.sh | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lfs.c b/lfs.c index 40744c9e..81a19f67 100644 --- a/lfs.c +++ b/lfs.c @@ -142,7 +142,7 @@ static inline void lfs_cache_zero(lfs_t *lfs, lfs_cache_t *pcache) { static int lfs_cache_flush(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { - if (pcache->block != 0xffffffff) { + if (pcache->block != 0xffffffff && pcache->block != 0xfffffffe) { LFS_ASSERT(pcache->block < lfs->cfg->block_count); lfs_size_t diff = lfs_alignup(pcache->size, lfs->cfg->prog_size); int err = lfs->cfg->prog(lfs->cfg, pcache->block, @@ -2158,7 +2158,10 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { return 0; relocate: - continue; + LFS_DEBUG("Bad block at %"PRIu32, nblock); + + // just clear cache and try a new block + lfs_cache_drop(lfs, &lfs->pcache); } } @@ -2397,7 +2400,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } if ((file->flags & LFS_F_INLINE) && - file->pos + nsize >= lfs->inline_max) { + file->pos + nsize > lfs->inline_max) { // inline file doesn't fit anymore file->block = 0xfffffffe; file->off = file->pos; diff --git a/tests/test_alloc.sh b/tests/test_alloc.sh index 55d1043c..21fee962 100755 --- a/tests/test_alloc.sh +++ b/tests/test_alloc.sh @@ -341,6 +341,10 @@ tests/test.py << TEST } lfs_file_close(&lfs, &file[0]) => 0; + // remount to force reset of lookahead + lfs_unmount(&lfs) => 0; + lfs_mount(&lfs, &cfg) => 0; + // open hole lfs_remove(&lfs, "bump") => 0; @@ -350,7 +354,7 @@ tests/test.py << TEST for (lfs_size_t i = 0; i < cfg.block_size; i += 2) { memcpy(&buffer[i], "hi", 2); } - lfs_file_write(&lfs, &file[0], buffer, cfg.block_size) => LFS_ERR_NOSPC; + lfs_file_write(&lfs, &file[0], buffer, 2*cfg.block_size) => LFS_ERR_NOSPC; lfs_file_close(&lfs, &file[0]) => 0; lfs_unmount(&lfs) => 0; @@ -388,7 +392,6 @@ tests/test.py << TEST // remount to force reset of lookahead lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; // rewrite one file @@ -451,7 +454,6 @@ tests/test.py << TEST // remount to force reset of lookahead lfs_unmount(&lfs) => 0; - lfs_mount(&lfs, &cfg) => 0; // rewrite one file with a hole of one block From 4db96d4d44cf4d3eb6d0731a024e6620028c64ba Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 13 Aug 2018 14:20:40 -0500 Subject: [PATCH 096/139] Changed unwritable superblock to ENOSPC for consistency While ECORRUPT is not a wrong error code, it doesn't match other instances of hitting a corrupt block during write. During writes, if blocks are detected as corrupt their data is evicted and moved to a new clean block. This means that at the end of a disk's lifetime, exhaustion errors will be reported as ENOSPC when littlefs can't find any new block to store the data. This has the benefit of matching behaviour when a new file is written and no more blocks can be found, due to either a small disk or corrupted blocks on disk. To littlefs it's like the disk shrinks in size over time. --- lfs.c | 2 +- tests/test_format.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lfs.c b/lfs.c index 81a19f67..1649ae5c 100644 --- a/lfs.c +++ b/lfs.c @@ -1237,7 +1237,7 @@ static int lfs_dir_compact(lfs_t *lfs, // can't relocate superblock, filesystem is now frozen if (lfs_pair_cmp(oldpair, (const lfs_block_t[2]){0, 1}) == 0) { LFS_WARN("Superblock %"PRIu32" has become unwritable", oldpair[1]); - return LFS_ERR_CORRUPT; + return LFS_ERR_NOSPC; } // relocate half of pair diff --git a/tests/test_format.sh b/tests/test_format.sh index 06438bcc..d885f835 100755 --- a/tests/test_format.sh +++ b/tests/test_format.sh @@ -22,7 +22,7 @@ echo "--- Invalid superblocks ---" ln -f -s /dev/zero blocks/0 ln -f -s /dev/zero blocks/1 tests/test.py << TEST - lfs_format(&lfs, &cfg) => LFS_ERR_CORRUPT; + lfs_format(&lfs, &cfg) => LFS_ERR_NOSPC; TEST rm blocks/0 blocks/1 From 478dcdddefb62f70edb0b95dae72e7da6f62ec43 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 20 Aug 2018 14:47:52 -0500 Subject: [PATCH 097/139] Revisited caching rules to optimize bus transactions The littlefs driver has always had this really weird quirk: larger cache sizes can significantly harm performance. This has probably been one of the most surprising pieces of configuraing and optimizing littlefs. The reason is that littlefs's caches are kinda dumb (this is somewhat intentional, as dumb caches take up much less code space than smart caches). When littlefs needs to read data, it will load the entire cache line. This means that even when we only need a small 4 byte piece of data, we may need to read a full 512 byte cache. And since microcontrollers may be reading from storage over relatively slow bus protocols, the time to send data over the bus may dominate other operations. Now that we have separate configuration options for "cache_size" and "read_size", we can start making littlefs's caches a bit smarter. They aren't going to be perfect, because code size is still a priority, but there are some small improvements we can do: 1. Program caches write to prog_size aligned units, but eagerly cache as much as possible. There's no downside to using the full cache in program operations. 2. Add a hint parameter to cached reads. This internal API allows callers to tell the cache how much data they expect to need. This avoids excess bus traffic, and now we can even bypass the cache if the caller provides enough of a buffer. We can still fall back to reading full cache-lines in the cases where we don't know how much data we need by providing the block size as the hint. We do this for directory fetches and for file reads. This has immediate improvements for both metadata-log traversal and CTZ skip-list traversal, since these both only need to read 4-byte pointers and can always bypass the cache, allowing reuse elsewhere. --- lfs.c | 300 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 168 insertions(+), 132 deletions(-) diff --git a/lfs.c b/lfs.c index 1649ae5c..3b413a53 100644 --- a/lfs.c +++ b/lfs.c @@ -20,8 +20,21 @@ /// Caching block device operations /// -static int lfs_cache_read(lfs_t *lfs, - const lfs_cache_t *pcache, lfs_cache_t *rcache, bool store, +static inline void lfs_cache_drop(lfs_t *lfs, lfs_cache_t *rcache) { + // do not zero, cheaper if cache is readonly or only going to be + // written with identical data (during relocates) + (void)lfs; + rcache->block = 0xffffffff; +} + +static inline void lfs_cache_zero(lfs_t *lfs, lfs_cache_t *pcache) { + // zero to avoid information leak + memset(pcache->buffer, 0xff, lfs->cfg->prog_size); + pcache->block = 0xffffffff; +} + +static int lfs_bd_read(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; @@ -57,7 +70,7 @@ static int lfs_cache_read(lfs_t *lfs, continue; } - if (!store && off % lfs->cfg->read_size == 0 && + if (size >= hint && off % lfs->cfg->read_size == 0 && size >= lfs->cfg->read_size) { // bypass cache? lfs_size_t diff = size - (size % lfs->cfg->read_size); @@ -74,12 +87,13 @@ static int lfs_cache_read(lfs_t *lfs, // load to cache, first condition can no longer fail LFS_ASSERT(block < lfs->cfg->block_count); - lfs_size_t nsize = store ? lfs->cfg->cache_size : lfs->cfg->prog_size; rcache->block = block; - rcache->off = lfs_aligndown(off, nsize); - rcache->size = nsize; + rcache->off = lfs_aligndown(off, lfs->cfg->prog_size); + rcache->size = lfs_min(lfs_min( + lfs_alignup(off+hint, lfs->cfg->prog_size), + lfs->cfg->block_size) - rcache->off, lfs->cfg->cache_size); int err = lfs->cfg->read(lfs->cfg, rcache->block, - rcache->off, rcache->buffer, nsize); + rcache->off, rcache->buffer, rcache->size); if (err) { return err; } @@ -88,21 +102,22 @@ static int lfs_cache_read(lfs_t *lfs, return 0; } -static int lfs_cache_cmp(lfs_t *lfs, - const lfs_cache_t *pcache, lfs_cache_t *rcache, +static int lfs_bd_cmp(lfs_t *lfs, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; for (lfs_off_t i = 0; i < size; i++) { - uint8_t c; - int err = lfs_cache_read(lfs, pcache, rcache, true, - block, off+i, &c, 1); + uint8_t dat; + int err = lfs_bd_read(lfs, + pcache, rcache, hint-i, + block, off+i, &dat, 1); if (err) { return err; } - if (c != data[i]) { + if (dat != data[i]) { return false; } } @@ -110,37 +125,7 @@ static int lfs_cache_cmp(lfs_t *lfs, return true; } -static int lfs_cache_crc(lfs_t *lfs, - const lfs_cache_t *pcache, lfs_cache_t *rcache, - lfs_block_t block, lfs_off_t off, lfs_size_t size, uint32_t *crc) { - for (lfs_off_t i = 0; i < size; i++) { - uint8_t c; - int err = lfs_cache_read(lfs, pcache, rcache, true, - block, off+i, &c, 1); - if (err) { - return err; - } - - *crc = lfs_crc(*crc, &c, 1); - } - - return 0; -} - -static inline void lfs_cache_drop(lfs_t *lfs, lfs_cache_t *rcache) { - // do not zero, cheaper if cache is readonly or only going to be - // written with identical data (during relocates) - (void)lfs; - rcache->block = 0xffffffff; -} - -static inline void lfs_cache_zero(lfs_t *lfs, lfs_cache_t *pcache) { - // zero to avoid information leak - memset(pcache->buffer, 0xff, lfs->cfg->prog_size); - pcache->block = 0xffffffff; -} - -static int lfs_cache_flush(lfs_t *lfs, +static int lfs_bd_flush(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { if (pcache->block != 0xffffffff && pcache->block != 0xfffffffe) { LFS_ASSERT(pcache->block < lfs->cfg->block_count); @@ -154,8 +139,9 @@ static int lfs_cache_flush(lfs_t *lfs, if (validate) { // check data on disk lfs_cache_drop(lfs, rcache); - int res = lfs_cache_cmp(lfs, NULL, rcache, pcache->block, - pcache->off, pcache->buffer, diff); + int res = lfs_bd_cmp(lfs, + NULL, rcache, diff, + pcache->block, pcache->off, pcache->buffer, diff); if (res < 0) { return res; } @@ -171,7 +157,19 @@ static int lfs_cache_flush(lfs_t *lfs, return 0; } -static int lfs_cache_prog(lfs_t *lfs, +static int lfs_bd_sync(lfs_t *lfs, + lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { + lfs_cache_drop(lfs, rcache); + + int err = lfs_bd_flush(lfs, pcache, rcache, validate); + if (err) { + return err; + } + + return lfs->cfg->sync(lfs->cfg); +} + +static int lfs_bd_prog(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { @@ -195,7 +193,7 @@ static int lfs_cache_prog(lfs_t *lfs, pcache->size = off - pcache->off; if (pcache->size == lfs->cfg->cache_size) { // eagerly flush out pcache if we fill up - int err = lfs_cache_flush(lfs, pcache, rcache, validate); + int err = lfs_bd_flush(lfs, pcache, rcache, validate); if (err) { return err; } @@ -217,46 +215,11 @@ static int lfs_cache_prog(lfs_t *lfs, return 0; } - -/// General lfs block device operations /// -static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, - lfs_off_t off, void *buffer, lfs_size_t size) { - return lfs_cache_read(lfs, &lfs->pcache, &lfs->rcache, true, - block, off, buffer, size); -} - -static int lfs_bd_prog(lfs_t *lfs, lfs_block_t block, - lfs_off_t off, const void *buffer, lfs_size_t size) { - return lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, false, - block, off, buffer, size); -} - -static int lfs_bd_cmp(lfs_t *lfs, lfs_block_t block, - lfs_off_t off, const void *buffer, lfs_size_t size) { - return lfs_cache_cmp(lfs, NULL, &lfs->rcache, block, off, buffer, size); -} - -static int lfs_bd_crc32(lfs_t *lfs, lfs_block_t block, - lfs_off_t off, lfs_size_t size, uint32_t *crc) { - return lfs_cache_crc(lfs, NULL, &lfs->rcache, block, off, size, crc); -} - static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { LFS_ASSERT(block < lfs->cfg->block_count); return lfs->cfg->erase(lfs->cfg, block); } -static int lfs_bd_sync(lfs_t *lfs) { - lfs_cache_drop(lfs, &lfs->rcache); - - int err = lfs_cache_flush(lfs, &lfs->pcache, &lfs->rcache, false); - if (err) { - return err; - } - - return lfs->cfg->sync(lfs->cfg); -} - /// Internal operations predeclared here /// static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], @@ -513,8 +476,9 @@ static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, if (buffer) { lfs_size_t diff = lfs_min( lfs_tag_size(gettag), lfs_tag_size(tag)); - int err = lfs_bd_read(lfs, block, - off+sizeof(tag), buffer, diff); + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, diff, + block, off+sizeof(tag), buffer, diff); if (err) { return err; } @@ -527,7 +491,9 @@ static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, } uint32_t ntag; - int err = lfs_bd_read(lfs, block, off, &ntag, sizeof(ntag)); + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(ntag), + block, off, &ntag, sizeof(ntag)); if (err) { return err; } @@ -567,8 +533,9 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, // write out tag uint32_t ntag = lfs_tole32((tag & 0x7fffffff) ^ commit->ptag); commit->crc = lfs_crc(commit->crc, &ntag, sizeof(ntag)); - int err = lfs_bd_prog(lfs, commit->block, commit->off, - &ntag, sizeof(ntag)); + int err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + commit->block, commit->off, &ntag, sizeof(ntag)); if (err) { return err; } @@ -577,7 +544,9 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, if (!(tag & 0x80000000)) { // from memory commit->crc = lfs_crc(commit->crc, buffer, size); - err = lfs_bd_prog(lfs, commit->block, commit->off, buffer, size); + err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + commit->block, commit->off, buffer, size); if (err) { return err; } @@ -587,13 +556,17 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, for (lfs_off_t i = 0; i < size; i++) { // rely on caching to make this efficient uint8_t dat; - err = lfs_bd_read(lfs, disk->block, disk->off+i, &dat, 1); + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, size-i, + disk->block, disk->off+i, &dat, 1); if (err) { return err; } commit->crc = lfs_crc(commit->crc, &dat, 1); - err = lfs_bd_prog(lfs, commit->block, commit->off+i, &dat, 1); + err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + commit->block, commit->off+i, &dat, 1); if (err) { return err; } @@ -641,7 +614,9 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, disk.block = dir->pair[0]; disk.off = off + sizeof(tag); - int err = lfs_bd_read(lfs, dir->pair[0], off, &ntag, sizeof(ntag)); + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(ntag), + dir->pair[0], off, &ntag, sizeof(ntag)); if (err) { return err; } @@ -701,10 +676,14 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { lfs->cfg->prog_size); // read erased state from next program unit - uint32_t tag; - int err = lfs_bd_read(lfs, commit->block, off, &tag, sizeof(tag)); - if (err) { - return err; + uint32_t tag = 0; + if (off < lfs->cfg->block_size) { + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + commit->block, off, &tag, sizeof(tag)); + if (err) { + return err; + } } // build crc tag @@ -718,7 +697,9 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { footer[0] = lfs_tole32(tag ^ commit->ptag); commit->crc = lfs_crc(commit->crc, &footer[0], sizeof(footer[0])); footer[1] = lfs_tole32(commit->crc); - err = lfs_bd_prog(lfs, commit->block, commit->off, footer, sizeof(footer)); + int err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + commit->block, commit->off, &footer, sizeof(footer)); if (err) { return err; } @@ -726,15 +707,27 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { commit->ptag = tag; // flush buffers - err = lfs_bd_sync(lfs); + err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false); if (err) { return err; } // successful commit, check checksum to make sure uint32_t crc = 0xffffffff; - err = lfs_bd_crc32(lfs, commit->block, commit->begin, - commit->off-lfs_tag_size(tag)-commit->begin, &crc); + lfs_size_t size = commit->off - lfs_tag_size(tag) - commit->begin; + for (lfs_off_t i = 0; i < size; i++) { + // leave it up to caching to make this efficient + uint8_t dat; + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, size-i, + commit->block, commit->begin+i, &dat, 1); + if (err) { + return err; + } + + crc = lfs_crc(crc, &dat, 1); + } + if (err) { return err; } @@ -758,7 +751,13 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { // rather than clobbering one of the blocks we just pretend // the revision may be valid - int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->rev, 4); + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(dir->rev), + dir->pair[0], 0, &dir->rev, sizeof(dir->rev)); + if (err) { + return err; + } + dir->rev = lfs_fromle32(dir->rev); if (err && err != LFS_ERR_CORRUPT) { return err; @@ -788,7 +787,8 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, // find the block with the most recent revision uint32_t rev[2]; for (int i = 0; i < 2; i++) { - int err = lfs_cache_read(lfs, &lfs->pcache, &lfs->rcache, false, + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(rev[i]), dir->pair[i], 0, &rev[i], sizeof(rev[i])); rev[i] = lfs_fromle32(rev[i]); if (err && err != LFS_ERR_CORRUPT) { @@ -824,10 +824,17 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_global_zero(&templocals); while (true) { + // reached end of block + if (off+sizeof(uint32_t) >= lfs->cfg->block_size) { + dir->erased = false; + break; + } + // extract next tag uint32_t tag; - int err = lfs_bd_read(lfs, dir->pair[0], - off, &tag, sizeof(tag)); + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off, &tag, sizeof(tag)); if (err) { if (err == LFS_ERR_CORRUPT) { // can't continue? @@ -855,8 +862,9 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, if (lfs_tag_type(tag) == LFS_TYPE_CRC) { // check the crc attr uint32_t dcrc; - err = lfs_bd_read(lfs, dir->pair[0], - off+sizeof(tag), &dcrc, sizeof(dcrc)); + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { if (err == LFS_ERR_CORRUPT) { dir->erased = false; @@ -883,23 +891,35 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, lfs->seed ^= crc; crc = 0xffffffff; } else { - err = lfs_bd_crc32(lfs, dir->pair[0], - off+sizeof(tag), lfs_tag_size(tag), &crc); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; + // crc the entry first, leaving it in the cache + for (lfs_off_t j = 0; j < lfs_tag_size(tag); j++) { + uint8_t dat; + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag)+j, &dat, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + return err; } + + crc = lfs_crc(crc, &dat, 1); } + // keep track of id count if (lfs_tag_id(tag) < 0x3ff && lfs_tag_id(tag) >= tempcount) { tempcount = lfs_tag_id(tag)+1; } + // check for special tags if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { tempsplit = (lfs_tag_type(tag) & 1); - err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), - temptail, sizeof(temptail)); + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), + &temptail, sizeof(temptail)); if (err) { if (err == LFS_ERR_CORRUPT) { dir->erased = false; @@ -909,7 +929,9 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_pair_fromle32(temptail); } else if (lfs_tag_subtype(tag) == LFS_TYPE_GLOBALS) { templocals.l.deorphaned = (lfs_tag_type(tag) & 1); - err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), &templocals, 10); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -931,8 +953,11 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, // found a match? if (lfs_tag_type(findtag) == LFS_TYPE_DIRSTRUCT) { lfs_block_t child[2]; - err = lfs_bd_read(lfs, dir->pair[0], off+sizeof(tag), - child, sizeof(child)); + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, + lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), + &child, sizeof(child)); if (err < 0) { if (err == LFS_ERR_CORRUPT) { dir->erased = false; @@ -948,6 +973,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, } } else if (lfs_tag_type(findtag) == LFS_TYPE_NAME) { int res = lfs_bd_cmp(lfs, + NULL, &lfs->rcache, lfs_tag_size(findtag), dir->pair[0], off+sizeof(tag), findbuffer, lfs_tag_size(findtag)); if (res < 0) { @@ -1091,7 +1117,9 @@ static int lfs_dir_compact(lfs_t *lfs, uint32_t crc = 0xffffffff; uint32_t rev = lfs_tole32(dir->rev); crc = lfs_crc(crc, &rev, sizeof(rev)); - err = lfs_bd_prog(lfs, dir->pair[1], 0, &rev, sizeof(rev)); + err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + dir->pair[1], 0, &rev, sizeof(rev)); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1788,8 +1816,9 @@ static int lfs_ctz_find(lfs_t *lfs, lfs_npw2(current-target+1) - 1, lfs_ctz(current)); - int err = lfs_cache_read(lfs, pcache, rcache, false, - head, 4*skip, &head, 4); + int err = lfs_bd_read(lfs, + pcache, rcache, sizeof(head), + head, 4*skip, &head, sizeof(head)); head = lfs_fromle32(head); if (err) { return err; @@ -1839,13 +1868,15 @@ static int lfs_ctz_extend(lfs_t *lfs, if (size != lfs->cfg->block_size) { for (lfs_off_t i = 0; i < size; i++) { uint8_t data; - err = lfs_cache_read(lfs, NULL, rcache, true, + err = lfs_bd_read(lfs, + NULL, rcache, size-i, head, i, &data, 1); if (err) { return err; } - err = lfs_cache_prog(lfs, pcache, rcache, true, + err = lfs_bd_prog(lfs, + pcache, rcache, true, nblock, i, &data, 1); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1866,7 +1897,7 @@ static int lfs_ctz_extend(lfs_t *lfs, for (lfs_off_t i = 0; i < skips; i++) { head = lfs_tole32(head); - err = lfs_cache_prog(lfs, pcache, rcache, true, + err = lfs_bd_prog(lfs, pcache, rcache, true, nblock, 4*i, &head, 4); head = lfs_fromle32(head); if (err) { @@ -1877,8 +1908,9 @@ static int lfs_ctz_extend(lfs_t *lfs, } if (i != skips-1) { - err = lfs_cache_read(lfs, NULL, rcache, false, - head, 4*i, &head, 4); + err = lfs_bd_read(lfs, + NULL, rcache, sizeof(head), + head, 4*i, &head, sizeof(head)); head = lfs_fromle32(head); if (err) { return err; @@ -1922,8 +1954,9 @@ static int lfs_ctz_traverse(lfs_t *lfs, lfs_block_t heads[2]; int count = 2 - (index & 1); - err = lfs_cache_read(lfs, pcache, rcache, false, - head, 0, &heads, count*4); + err = lfs_bd_read(lfs, + pcache, rcache, count*sizeof(head), + head, 0, &heads, count*sizeof(head)); heads[0] = lfs_fromle32(heads[0]); heads[1] = lfs_fromle32(heads[1]); if (err) { @@ -2131,13 +2164,15 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { // either read from dirty cache or disk for (lfs_off_t i = 0; i < file->off; i++) { uint8_t data; - err = lfs_cache_read(lfs, &file->cache, &lfs->rcache, true, + err = lfs_bd_read(lfs, + &file->cache, &lfs->rcache, file->off-i, file->block, i, &data, 1); if (err) { return err; } - err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, true, + err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, true, nblock, i, &data, 1); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -2207,7 +2242,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { // write out what we have while (true) { - int err = lfs_cache_flush(lfs, + int err = lfs_bd_flush(lfs, &file->cache, &lfs->rcache, true); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -2350,7 +2385,8 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, // read as much as we can in current block lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); - int err = lfs_cache_read(lfs, NULL, &file->cache, true, + int err = lfs_bd_read(lfs, + NULL, &file->cache, lfs->cfg->block_size, file->block, file->off, data, diff); if (err) { return err; @@ -2455,7 +2491,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // program as much as we can in current block lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); while (true) { - int err = lfs_cache_prog(lfs, &file->cache, &lfs->rcache, true, + int err = lfs_bd_prog(lfs, &file->cache, &lfs->rcache, true, file->block, file->off, data, diff); if (err) { if (err == LFS_ERR_CORRUPT) { From a43f9b3cd5d94c20e64ebb106d93821175660f34 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 20 Aug 2018 21:45:11 -0500 Subject: [PATCH 098/139] Modified lfs_dir_compact to avoid redundant erases during split The commit machine in littlefs has three stages: commit, compact, and then split. First we try to append our commit to the metadata log, if that fails we try to compact the metadata log to remove duplicates and make room for the commit, if that still fails we split the metadata into two metadata-pairs and try again. Each stage is less efficient but also less frequent. However, in the case that we're filling up a directory with new files, such as the bootstrap process in setting up a new system, we must pass through all three stages rather quickly in order to get enough metadata-pairs to hold all of our files. This means we'll compact, split, and then need to compact again. This creates more erases than is needed in the optimal case, which can be a big cost on disks with an expensive erase operation. In theory, we can actually avoid this redundant erase by reusing the data we wrote out in the first attempt to compact. In practice, this trick is very complicated to pull off. 1. We may need to cache a half-completed program while we write out the new metadata-pair. We need to write out the second pair first in order to get our new tail before we complete our first metadata-pair. This requires two pcaches, which we don't have The solution here is to just drop our cache and reconstruct what if would have been. This needs to be perfect down to the byte level because we don't have knowledge of where our cache lines are. 2. We may have written out entries that are then moved to the new metadata-pair. The solution here isn't pretty but it works, we just add a delete tag for any entry that was moved over. In the end the solution ends up a bit hacky, with different layers poked through the commit logic in order to manage writes at the byte level from where we manage splits. But it works fairly well and saves erases. --- lfs.c | 212 +++++++++++++++++++++++++++++++++------------------------- 1 file changed, 122 insertions(+), 90 deletions(-) diff --git a/lfs.c b/lfs.c index 3b413a53..5910021f 100644 --- a/lfs.c +++ b/lfs.c @@ -39,6 +39,9 @@ static int lfs_bd_read(lfs_t *lfs, void *buffer, lfs_size_t size) { uint8_t *data = buffer; LFS_ASSERT(block != 0xffffffff); + if (off+size > lfs->cfg->block_size) { + return LFS_ERR_CORRUPT; + } while (size > 0) { if (pcache && block == pcache->block && @@ -452,6 +455,7 @@ struct lfs_commit { lfs_off_t begin; lfs_off_t end; + lfs_off_t ack; }; struct lfs_diskoff { @@ -503,6 +507,24 @@ static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, return LFS_ERR_NOENT; } +static int lfs_commit_prog(lfs_t *lfs, struct lfs_commit *commit, + const void *buffer, lfs_size_t size) { + lfs_off_t skip = lfs_min(lfs_max(commit->ack, commit->off) + - commit->off, size); + int err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + commit->block, commit->off + skip, + (const uint8_t*)buffer + skip, size - skip); + if (err) { + return err; + } + + commit->crc = lfs_crc(commit->crc, buffer, size); + commit->off += size; + commit->ack = lfs_max(commit->off, commit->ack); + return 0; +} + static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, uint16_t id, const struct lfs_attr *attrs); @@ -532,21 +554,14 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, // write out tag uint32_t ntag = lfs_tole32((tag & 0x7fffffff) ^ commit->ptag); - commit->crc = lfs_crc(commit->crc, &ntag, sizeof(ntag)); - int err = lfs_bd_prog(lfs, - &lfs->pcache, &lfs->rcache, false, - commit->block, commit->off, &ntag, sizeof(ntag)); + int err = lfs_commit_prog(lfs, commit, &ntag, sizeof(ntag)); if (err) { return err; } - commit->off += sizeof(ntag); if (!(tag & 0x80000000)) { // from memory - commit->crc = lfs_crc(commit->crc, buffer, size); - err = lfs_bd_prog(lfs, - &lfs->pcache, &lfs->rcache, false, - commit->block, commit->off, buffer, size); + err = lfs_commit_prog(lfs, commit, buffer, size); if (err) { return err; } @@ -563,17 +578,13 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, return err; } - commit->crc = lfs_crc(commit->crc, &dat, 1); - err = lfs_bd_prog(lfs, - &lfs->pcache, &lfs->rcache, false, - commit->block, commit->off+i, &dat, 1); + err = lfs_commit_prog(lfs, commit, &dat, 1); if (err) { return err; } } } - commit->off += size; commit->ptag = tag & 0x7fffffff; return 0; } @@ -677,13 +688,11 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // read erased state from next program unit uint32_t tag = 0; - if (off < lfs->cfg->block_size) { - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - commit->block, off, &tag, sizeof(tag)); - if (err) { - return err; - } + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(tag), + commit->block, off, &tag, sizeof(tag)); + if (err && err != LFS_ERR_CORRUPT) { + return err; } // build crc tag @@ -697,7 +706,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { footer[0] = lfs_tole32(tag ^ commit->ptag); commit->crc = lfs_crc(commit->crc, &footer[0], sizeof(footer[0])); footer[1] = lfs_tole32(commit->crc); - int err = lfs_bd_prog(lfs, + err = lfs_bd_prog(lfs, &lfs->pcache, &lfs->rcache, false, commit->block, commit->off, &footer, sizeof(footer)); if (err) { @@ -824,12 +833,6 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_global_zero(&templocals); while (true) { - // reached end of block - if (off+sizeof(uint32_t) >= lfs->cfg->block_size) { - dir->erased = false; - break; - } - // extract next tag uint32_t tag; int err = lfs_bd_read(lfs, @@ -1076,50 +1079,74 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_global_zero(&dir->locals); while (true) { - // last complete id - dir->count = end - begin; - int16_t ack = -1; + // setup compaction + bool splitted = false; bool exhausted = false; - // increment revision count - dir->rev += 1; - if (lfs->cfg->block_cycles && dir->rev % lfs->cfg->block_cycles == 0) { - if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { - // we're writing too much to the superblock, should we expand? - lfs_ssize_t res = lfs_fs_size(lfs); - if (res < 0) { - return res; - } + struct lfs_commit commit; + commit.block = dir->pair[1]; + commit.ack = 0; + +commit: + // setup erase state + exhausted = false; + dir->count = end - begin; + int16_t ackid = -1; + + // setup commit state + commit.off = 0; + commit.crc = 0xffffffff; + commit.ptag = 0; + + // space is complicated, we need room for tail, crc, globals, + // cleanup delete, and we cap at half a block to give room + // for metadata updates + commit.begin = 0; + commit.end = lfs_min( + lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size), + lfs->cfg->block_size - 38); + + if (!splitted) { + // increment revision count + dir->rev += 1; + if (lfs->cfg->block_cycles && + dir->rev % lfs->cfg->block_cycles == 0) { + if (lfs_pair_cmp(dir->pair, + (const lfs_block_t[2]){0, 1}) == 0) { + // we're writing too much to the superblock, + // should we expand? + lfs_ssize_t res = lfs_fs_size(lfs); + if (res < 0) { + return res; + } - // do we have enough space to expand? - if (res < lfs->cfg->block_count/2) { - LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); + // do we have enough space to expand? + if (res < lfs->cfg->block_count/2) { + LFS_DEBUG("Expanding superblock at rev %"PRIu32, + dir->rev); + exhausted = true; + goto split; + } + } else { + // we're writing too much, time to relocate exhausted = true; - goto split; + goto relocate; } - } else { - // we're writing too much, time to relocate - exhausted = true; - goto relocate; } - } - // erase block to write to - int err = lfs_bd_erase(lfs, dir->pair[1]); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + // erase block to write to + int err = lfs_bd_erase(lfs, dir->pair[1]); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; } - return err; } // write out header - uint32_t crc = 0xffffffff; uint32_t rev = lfs_tole32(dir->rev); - crc = lfs_crc(crc, &rev, sizeof(rev)); - err = lfs_bd_prog(lfs, - &lfs->pcache, &lfs->rcache, false, - dir->pair[1], 0, &rev, sizeof(rev)); + int err = lfs_commit_prog(lfs, &commit, &rev, sizeof(rev)); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1127,28 +1154,13 @@ static int lfs_dir_compact(lfs_t *lfs, return err; } - // setup compaction - struct lfs_commit commit = { - .block = dir->pair[1], - .off = sizeof(dir->rev), - .crc = crc, - .ptag = 0, - - // space is complicated, we need room for tail, crc, globals, - // and we cap at half a block to give room for metadata updates - .begin = 0, - .end = lfs_min( - lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size), - lfs->cfg->block_size - 34), - }; - // commit with a move - for (uint16_t id = begin; id < end; id++) { + for (uint16_t id = begin; id < end || commit.off < commit.ack; id++) { err = lfs_commit_move(lfs, &commit, 0x003ff000, LFS_MKTAG(0, id, 0), 0x003ff000, LFS_MKTAG(0, id - begin, 0), source, attrs); - if (err) { + if (err && !(splitted && err == LFS_ERR_NOSPC)) { if (err == LFS_ERR_NOSPC) { goto split; } else if (err == LFS_ERR_CORRUPT) { @@ -1157,12 +1169,25 @@ static int lfs_dir_compact(lfs_t *lfs, return err; } - ack = id; + ackid = id; } // reopen reserved space at the end commit.end = lfs->cfg->block_size - 8; + if (ackid >= end) { + // extra garbage attributes were written out during split, + // need to clean up + err = lfs_commit_attr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_DELETE, ackid, 0), NULL); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } + if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { // move over (duplicate) superblock if we are root err = lfs_commit_move(lfs, &commit, @@ -1178,8 +1203,8 @@ static int lfs_dir_compact(lfs_t *lfs, } if (!relocated) { - // commit any globals, unless we're relocating, in which case our - // parent will steal our globals + // commit any globals, unless we're relocating, + // in which case our parent will steal our globals err = lfs_commit_globals(lfs, &commit, &dir->locals); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1222,8 +1247,13 @@ static int lfs_dir_compact(lfs_t *lfs, split: // commit no longer fits, need to split dir, // drop caches and create tail - lfs_cache_drop(lfs, &lfs->pcache); - if (!exhausted && ack < 0) { + splitted = !exhausted; + if (lfs->pcache.block != 0xffffffff) { + commit.ack -= lfs->pcache.size; + lfs_cache_drop(lfs, &lfs->pcache); + } + + if (!exhausted && ackid < 0) { // If we can't fit in this block, we won't fit in next block return LFS_ERR_NOSPC; } @@ -1234,25 +1264,26 @@ static int lfs_dir_compact(lfs_t *lfs, return err; } - if (exhausted) { - lfs->root[0] = tail.pair[0]; - lfs->root[1] = tail.pair[1]; - } - tail.split = dir->split; tail.tail[0] = dir->tail[0]; tail.tail[1] = dir->tail[1]; - err = lfs_dir_compact(lfs, &tail, attrs, source, ack+1, end); + err = lfs_dir_compact(lfs, &tail, attrs, source, ackid+1, end); if (err) { return err; } - end = ack+1; + end = ackid+1; dir->tail[0] = tail.pair[0]; dir->tail[1] = tail.pair[1]; dir->split = true; - continue; + + if (exhausted) { + lfs->root[0] = tail.pair[0]; + lfs->root[1] = tail.pair[1]; + } + + goto commit; relocate: // commit was corrupted, drop caches and prepare to relocate block @@ -1363,6 +1394,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, .begin = dir->off, .end = lfs->cfg->block_size - 8, + .ack = 0, }; for (const lfs_mattr_t *a = attrs; a; a = a->next) { From 6db5202bdc7393158914e349fedb63c5e34628e5 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 24 Aug 2018 17:45:45 -0500 Subject: [PATCH 099/139] Modified valid bit to provide an early check on all tags The valid bit present in tags is a requirement to properly detect the end of commits in metadata logs. The way it works is that the CRC entry is allowed to specify what is needed from the next tag's valid bit. If it's incorrect, we've reached the end of the commit. We then set the valid bit to indicate when we tried to program a new commit. If we lose power, this commit will still be thrown out by a bad checksum. However, the valid bit is unused outside of the CRC entry. Here we turn on the valid bit for all tags, which means we have a decent chance of exiting early if we hit a half-written commit. We still need to guarantee detection of the valid bit on commits following the CRC entry, so we allow the CRC entry to flip the expected valid bit. The only tricky part is what valid bit we expect by default, since this is used on the first commit on a metadata log. Here we default to a 1, which gives us the fastest exit on blocks that erase to 0. This is because blocks that erase to 1s will implicitly flip the valid bit of the next tag, allowing us to exit on the next tag. If we defaulted to 0, we could exit faster on disks that erase to 1, but would need to scan the entire block on disks that erase to 0 before we realize a CRC commit is never coming. --- lfs.c | 35 +++++++++++++++++++---------------- tests/corrupt.py | 2 +- tests/debug.py | 12 +++++++----- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/lfs.c b/lfs.c index 5910021f..b98aa547 100644 --- a/lfs.c +++ b/lfs.c @@ -385,7 +385,7 @@ static inline uint16_t lfs_tag_type(uint32_t tag) { } static inline uint16_t lfs_tag_subtype(uint32_t tag) { - return (tag & 0x7c000000) >> 22; + return ((tag & 0x7c000000) >> 26) << 4; } static inline uint16_t lfs_tag_id(uint32_t tag) { @@ -470,7 +470,7 @@ static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, while (off >= 2*sizeof(tag)+lfs_tag_size(tag)) { off -= sizeof(tag)+lfs_tag_size(tag); - if (lfs_tag_type(tag) == LFS_TYPE_CRC && stopatcommit) { + if (lfs_tag_subtype(tag) == LFS_TYPE_CRC && stopatcommit) { break; } else if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { if (lfs_tag_id(tag) <= lfs_tag_id(gettag + getdiff)) { @@ -502,6 +502,7 @@ static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, return err; } tag ^= lfs_fromle32(ntag); + tag &= 0x7fffffff; } return LFS_ERR_NOENT; @@ -632,8 +633,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, return err; } - ntag = lfs_fromle32(ntag); - ntag ^= tag; + ntag = lfs_fromle32(ntag) ^ tag; tag |= 0x80000000; } @@ -687,7 +687,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { lfs->cfg->prog_size); // read erased state from next program unit - uint32_t tag = 0; + uint32_t tag; int err = lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, sizeof(tag), commit->block, off, &tag, sizeof(tag)); @@ -696,10 +696,9 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { } // build crc tag - tag = lfs_fromle32(tag); - tag = (0x80000000 & ~tag) | - LFS_MKTAG(LFS_TYPE_CRC, 0x3ff, - off - (commit->off+sizeof(uint32_t))); + bool reset = ~lfs_fromle32(tag) >> 31; + tag = LFS_MKTAG(LFS_TYPE_CRC + reset, + 0x3ff, off - (commit->off+sizeof(uint32_t))); // write out crc uint32_t footer[2]; @@ -713,7 +712,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { return err; } commit->off += sizeof(tag)+lfs_tag_size(tag); - commit->ptag = tag; + commit->ptag = tag ^ (reset << 31); // flush buffers err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false); @@ -774,7 +773,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { // set defaults dir->off = sizeof(dir->rev); - dir->etag = 0; + dir->etag = 0xffffffff; dir->count = 0; dir->tail[0] = 0xffffffff; dir->tail[1] = 0xffffffff; @@ -817,7 +816,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, // load blocks and check crc for (int i = 0; i < 2; i++) { lfs_off_t off = sizeof(dir->rev); - uint32_t ptag = 0; + uint32_t ptag = 0xffffffff; uint32_t crc = 0xffffffff; dir->rev = lfs_tole32(rev[0]); @@ -851,8 +850,8 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, tag = lfs_fromle32(tag) ^ ptag; // next commit not yet programmed - if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_isvalid(tag)) { - dir->erased = true; + if (!lfs_tag_isvalid(tag)) { + dir->erased = (lfs_tag_subtype(ptag) == LFS_TYPE_CRC); break; } @@ -862,7 +861,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, break; } - if (lfs_tag_type(tag) == LFS_TYPE_CRC) { + if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { // check the crc attr uint32_t dcrc; err = lfs_bd_read(lfs, @@ -882,6 +881,10 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, break; } + // reset the next bit if we need to + tag ^= (lfs_tag_type(tag) & 1) << 31; + + // update with what's found so far foundtag = tempfoundtag; dir->off = off + sizeof(tag)+lfs_tag_size(tag); dir->etag = tag; @@ -1096,7 +1099,7 @@ static int lfs_dir_compact(lfs_t *lfs, // setup commit state commit.off = 0; commit.crc = 0xffffffff; - commit.ptag = 0; + commit.ptag = 0xffffffff; // space is complicated, we need room for tail, crc, globals, // cleanup delete, and we cap at half a block to give room diff --git a/tests/corrupt.py b/tests/corrupt.py index 5e2a9a3a..9b5d4bd2 100755 --- a/tests/corrupt.py +++ b/tests/corrupt.py @@ -11,7 +11,7 @@ def corrupt(block): file.read(4) # go to last commit - tag = 0 + tag = 0xffffffff while True: try: ntag, = struct.unpack('> 22 id = (tag & 0x003ff000) >> 12 size = (tag & 0x00000fff) >> 0 + iscrc = (type & 0x1f0) == 0x0f0 data = file.read(size) - if type == 0x0f0: + if iscrc: crc = binascii.crc32(data[:4], crc) else: crc = binascii.crc32(data, crc) print '%04x: %08x %-14s %3s %3d %-23s %-8s' % ( off, tag, - typeof(type) + (' bad!' if type == 0x0f0 and ~crc else ''), + typeof(type) + (' bad!' if iscrc and ~crc else ''), id if id != 0x3ff else '.', size, ' '.join('%02x' % ord(c) for c in data[:8]), ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8])) off += tag & 0xfff - if type == 0x0f0: + if iscrc: crc = 0 + tag ^= (type & 1) << 31 return 0 From 6046d85e6ed53b6ddc1232cd56f788add8a1d44b Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 9 Sep 2018 09:01:06 -0500 Subject: [PATCH 100/139] Added support for entry insertion Currently unused, the insertion of new file entries in arbitrary locations in a metadata-pair is very easy to add into the existing metadata logging. The only tricky things: 1. Name tags must strictly precede any tags related to a file. We can pull this off during a compact, but must make two passes. One for the name tag, one for the file. Though a benefit of this is that now our scans during moves can exit early upon finding the name tag. 1. We need to handle name tags appearing out of order. This makes name tags symmetric to deletes, although it doesn't seem like we can leverage this fact very well. Note this also means we need to make the superblock tag a type of name tag. --- lfs.c | 229 +++++++++++++++++++++++++++---------------------- lfs.h | 26 +++--- tests/debug.py | 18 ++-- 3 files changed, 148 insertions(+), 125 deletions(-) diff --git a/lfs.c b/lfs.c index b98aa547..38460651 100644 --- a/lfs.c +++ b/lfs.c @@ -463,20 +463,25 @@ struct lfs_diskoff { lfs_off_t off; }; -static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, - uint32_t tag, uint32_t getmask, uint32_t gettag, int32_t getdiff, - void *buffer, bool stopatcommit) { +static int32_t lfs_commit_get(lfs_t *lfs, + lfs_block_t block, lfs_off_t off, uint32_t tag, bool stopatcommit, + uint32_t getmask, uint32_t gettag, int32_t getdiff, void *buffer) { + gettag += getdiff; + // iterate over dir block backwards (for faster lookups) - while (off >= 2*sizeof(tag)+lfs_tag_size(tag)) { + while (off >= 2*sizeof(uint32_t)+lfs_tag_size(tag)) { off -= sizeof(tag)+lfs_tag_size(tag); if (lfs_tag_subtype(tag) == LFS_TYPE_CRC && stopatcommit) { break; - } else if (lfs_tag_type(tag) == LFS_TYPE_DELETE) { - if (lfs_tag_id(tag) <= lfs_tag_id(gettag + getdiff)) { - getdiff += LFS_MKTAG(0, 1, 0); + } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { + // something was deleted, need to move around it + if (lfs_tag_id(tag) <= lfs_tag_id(gettag - getdiff)) { + getdiff -= LFS_MKTAG(0, 1, 0); } - } else if ((tag & getmask) == ((gettag + getdiff) & getmask)) { + } + + if ((tag & getmask) == ((gettag - getdiff) & getmask)) { if (buffer) { lfs_size_t diff = lfs_min( lfs_tag_size(gettag), lfs_tag_size(tag)); @@ -491,7 +496,16 @@ static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off, lfs_tag_size(gettag) - diff); } - return tag - getdiff; + return tag + getdiff; + } + + if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { + // found where something was created + if (lfs_tag_id(tag) == lfs_tag_id(gettag - getdiff)) { + break; + } else if (lfs_tag_id(tag) < lfs_tag_id(gettag - getdiff)) { + getdiff += LFS_MKTAG(0, 1, 0); + } } uint32_t ntag; @@ -529,19 +543,20 @@ static int lfs_commit_prog(lfs_t *lfs, struct lfs_commit *commit, static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, uint16_t id, const struct lfs_attr *attrs); -static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, - uint32_t frommask, uint32_t fromtag, uint32_t tomask, uint32_t totag, +static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, + uint32_t frommask, uint32_t fromtag, int32_t fromdiff, const lfs_mdir_t *dir, const lfs_mattr_t *attrs); static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, uint32_t tag, const void *buffer) { - if (lfs_tag_subtype(tag) == LFS_FROM_MOVE) { + if (lfs_tag_type(tag) == LFS_FROM_MOVE) { // special case for moves - return lfs_commit_move(lfs, commit, + return lfs_commit_move(lfs, commit, 1, 0x003ff000, LFS_MKTAG(0, lfs_tag_size(tag), 0), - 0x003ff000, LFS_MKTAG(0, lfs_tag_id(tag), 0), + LFS_MKTAG(0, lfs_tag_id(tag), 0) - + LFS_MKTAG(0, lfs_tag_size(tag), 0), buffer, NULL); - } else if (lfs_tag_subtype(tag) == LFS_FROM_ATTRS) { + } else if (lfs_tag_type(tag) == LFS_FROM_ATTRS) { // special case for custom attributes return lfs_commit_attrs(lfs, commit, lfs_tag_id(tag), buffer); @@ -603,13 +618,15 @@ static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, - uint32_t frommask, uint32_t fromtag, uint32_t tomask, uint32_t totag, +static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, + uint32_t frommask, uint32_t fromtag, int32_t fromdiff, const lfs_mdir_t *dir, const lfs_mattr_t *attrs) { + fromtag += fromdiff; + // iterate through list and commits, only committing unique entries lfs_off_t off = dir->off; uint32_t ntag = dir->etag; - while (attrs || off > sizeof(uint32_t)) { + while (attrs || off >= 2*sizeof(uint32_t)+lfs_tag_size(ntag)) { struct lfs_diskoff disk; uint32_t tag; const void *buffer; @@ -618,7 +635,6 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, buffer = attrs->buffer; attrs = attrs->next; } else { - LFS_ASSERT(off > sizeof(ntag)+lfs_tag_size(ntag)); off -= sizeof(ntag)+lfs_tag_size(ntag); tag = ntag; @@ -637,31 +653,48 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, tag |= 0x80000000; } - if (lfs_tag_type(tag) == LFS_TYPE_DELETE && - lfs_tag_id(tag) <= lfs_tag_id(fromtag)) { + if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE && + lfs_tag_id(tag) <= lfs_tag_id(fromtag - fromdiff)) { // something was deleted, we need to move around it - fromtag += LFS_MKTAG(0, 1, 0); - } else if ((tag & frommask) == (fromtag & frommask)) { - // check if type has already been committed - int32_t res = lfs_commit_get(lfs, commit->block, - commit->off, commit->ptag, - lfs_tag_isuser(tag) ? 0x7ffff000 : 0x7c3ff000, - (tag & ~tomask) | totag, - 0, NULL, true); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; + fromdiff -= LFS_MKTAG(0, 1, 0); + } + + if ((tag & frommask) == ((fromtag - fromdiff) & frommask)) { + bool duplicate; + if (pass == 0) { + duplicate = (lfs_tag_subtype(tag) != LFS_TYPE_NAME); + } else { + // check if type has already been committed + int32_t res = lfs_commit_get(lfs, + commit->block, commit->off, commit->ptag, true, + lfs_tag_isuser(tag) ? 0x7ffff000 : 0x7c3ff000, + tag + fromdiff, 0, NULL); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + + duplicate = (res != LFS_ERR_NOENT); } - if (res == LFS_ERR_NOENT) { + if (!duplicate) { // update id and commit, as we are currently unique int err = lfs_commit_attr(lfs, commit, - (tag & ~tomask) | totag, + tag + fromdiff, buffer); if (err) { return err; } } } + + if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { + // found where something was created + if (lfs_tag_id(tag) == lfs_tag_id(fromtag - fromdiff)) { + break; + } else if (lfs_tag_id(tag) < lfs_tag_id(fromtag - fromdiff)) { + fromdiff += LFS_MKTAG(0, 1, 0); + } + } } return 0; @@ -914,13 +947,25 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, crc = lfs_crc(crc, &dat, 1); } - // keep track of id count - if (lfs_tag_id(tag) < 0x3ff && lfs_tag_id(tag) >= tempcount) { - tempcount = lfs_tag_id(tag)+1; - } - // check for special tags - if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { + if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { + tempcount += 1; + + if (lfs_tag_isvalid(tempfoundtag) && + lfs_tag_id(tag) <= lfs_tag_id(tempfoundtag)) { + tempfoundtag += LFS_MKTAG(0, 1, 0); + } + } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { + LFS_ASSERT(tempcount > 0); + tempcount -= 1; + + if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { + tempfoundtag = LFS_ERR_NOENT; + } else if (lfs_tag_isvalid(tempfoundtag) && + lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { + tempfoundtag -= LFS_MKTAG(0, 1, 0); + } + } else if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { tempsplit = (lfs_tag_type(tag) & 1); err = lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, @@ -945,17 +990,9 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, break; } } - } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { - LFS_ASSERT(tempcount > 0); - tempcount -= 1; + } - if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { - tempfoundtag = LFS_ERR_NOENT; - } else if (lfs_tag_isvalid(tempfoundtag) && - lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { - tempfoundtag -= LFS_MKTAG(0, 1, 0); - } - } else if ((tag & findmask) == (findtag & findmask)) { + if ((tag & findmask) == (findtag & findmask)) { // found a match? if (lfs_tag_type(findtag) == LFS_TYPE_DIRSTRUCT) { lfs_block_t child[2]; @@ -1062,11 +1099,13 @@ static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0 && lfs_tag_id(gettag) <= lfs->globals.g.moveid) { // synthetic moves - getdiff = LFS_MKTAG(0, 1, 0); + gettag += LFS_MKTAG(0, 1, 0); + getdiff -= LFS_MKTAG(0, 1, 0); } - return lfs_commit_get(lfs, dir->pair[0], dir->off, dir->etag, - getmask, gettag, getdiff, buffer, false); + return lfs_commit_get(lfs, + dir->pair[0], dir->off, dir->etag, false, + getmask, gettag, getdiff, buffer); } static int lfs_dir_compact(lfs_t *lfs, @@ -1159,17 +1198,19 @@ static int lfs_dir_compact(lfs_t *lfs, // commit with a move for (uint16_t id = begin; id < end || commit.off < commit.ack; id++) { - err = lfs_commit_move(lfs, &commit, - 0x003ff000, LFS_MKTAG(0, id, 0), - 0x003ff000, LFS_MKTAG(0, id - begin, 0), - source, attrs); - if (err && !(splitted && err == LFS_ERR_NOSPC)) { - if (err == LFS_ERR_NOSPC) { - goto split; - } else if (err == LFS_ERR_CORRUPT) { - goto relocate; + for (int pass = 0; pass < 2; pass++) { + err = lfs_commit_move(lfs, &commit, pass, + 0x003ff000, LFS_MKTAG(0, id, 0), + -LFS_MKTAG(0, begin, 0), + source, attrs); + if (err && !(splitted && err == LFS_ERR_NOSPC)) { + if (err == LFS_ERR_NOSPC) { + goto split; + } else if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; } - return err; } ackid = id; @@ -1191,20 +1232,6 @@ static int lfs_dir_compact(lfs_t *lfs, } } - if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { - // move over (duplicate) superblock if we are root - err = lfs_commit_move(lfs, &commit, - 0x7c000000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 0), - 0x7ffff000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 0), - source, attrs); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - } - if (!relocated) { // commit any globals, unless we're relocating, // in which case our parent will steal our globals @@ -1350,12 +1377,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // calculate new directory size uint32_t deletetag = 0xffffffff; + int attrcount = 0; for (const lfs_mattr_t *a = attrs; a; a = a->next) { - if (lfs_tag_id(a->tag) < 0x3ff && lfs_tag_id(a->tag) >= dir->count) { - dir->count = lfs_tag_id(a->tag)+1; - } - - if (lfs_tag_type(a->tag) == LFS_TYPE_DELETE) { + if (lfs_tag_subtype(a->tag) == LFS_TYPE_NAME) { + dir->count += 1; + } else if (lfs_tag_subtype(a->tag) == LFS_TYPE_DELETE) { LFS_ASSERT(dir->count > 0); dir->count -= 1; deletetag = a->tag; @@ -1381,6 +1407,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } } } + + attrcount += 1; } while (true) { @@ -1400,23 +1428,16 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, .ack = 0, }; - for (const lfs_mattr_t *a = attrs; a; a = a->next) { - if (lfs_tag_type(a->tag) != LFS_TYPE_DELETE) { - lfs_pair_tole32(dir->tail); - int err = lfs_commit_attr(lfs, &commit, a->tag, a->buffer); - lfs_pair_fromle32(dir->tail); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - goto compact; - } - return err; - } + // iterate over commits backwards, this lets us "append" commits cheaply + for (int i = 0; i < attrcount; i++) { + const lfs_mattr_t *a = attrs; + for (int j = 0; j < attrcount-i-1; j++) { + a = a->next; } - } - if (lfs_tag_isvalid(deletetag)) { - // special case for deletes, since order matters - int err = lfs_commit_attr(lfs, &commit, deletetag, NULL); + lfs_pair_tole32(dir->tail); + int err = lfs_commit_attr(lfs, &commit, a->tag, a->buffer); + lfs_pair_fromle32(dir->tail); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1654,9 +1675,9 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.tail[1] = dir.pair[1]; lfs_pair_tole32(dir.pair); err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, - LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), + LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), + LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, NULL)))); lfs_pair_fromle32(dir.pair); if (err) { @@ -2059,8 +2080,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // get next slot and create entry to remember name file->id = file->m.count; err = lfs_dir_commit(lfs, &file->m, - LFS_MKATTR(LFS_TYPE_REG, file->id, path, nlen, LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, NULL, 0, + LFS_MKATTR(LFS_TYPE_REG, file->id, path, nlen, NULL))); if (err) { err = LFS_ERR_NAMETOOLONG; @@ -2343,8 +2364,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // commit file data and attributes err = lfs_dir_commit(lfs, &file->m, - LFS_MKATTR(type, file->id, buffer, size, LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, + LFS_MKATTR(type, file->id, buffer, size, NULL))); if (err) { if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) { @@ -2811,9 +2832,11 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // move over all attributes err = lfs_dir_commit(lfs, &newcwd, - LFS_MKATTR(lfs_tag_type(oldtag), newid, newpath, strlen(newpath), LFS_MKATTR(LFS_FROM_MOVE, newid, &oldcwd, lfs_tag_id(oldtag), - NULL))); + LFS_MKATTR(lfs_tag_type(oldtag), newid, newpath, strlen(newpath), + (prevtag != LFS_ERR_NOENT) + ? LFS_MKATTR(LFS_TYPE_DELETE, newid, NULL, 0, NULL) + : NULL))); if (err) { return err; } @@ -3085,7 +3108,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { goto cleanup; } - int32_t res = lfs_dir_get(lfs, &root, 0x7c000000, + int32_t res = lfs_dir_get(lfs, &root, 0x7f800000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), &superblock); if (res < 0) { diff --git a/lfs.h b/lfs.h index 839cf831..7b0e8a18 100644 --- a/lfs.h +++ b/lfs.h @@ -88,21 +88,21 @@ enum lfs_error { // File types enum lfs_type { // file types - LFS_TYPE_REG = 0x001, - LFS_TYPE_DIR = 0x002, + LFS_TYPE_REG = 0x002, + LFS_TYPE_DIR = 0x003, // internally used types LFS_TYPE_USER = 0x100, - LFS_TYPE_SUPERBLOCK = 0x011, - LFS_TYPE_ROOT = 0x010, + LFS_TYPE_SUPERBLOCK = 0x001, + LFS_TYPE_ROOT = 0x000, LFS_TYPE_NAME = 0x000, - LFS_TYPE_DELETE = 0x030, + LFS_TYPE_DELETE = 0x020, LFS_TYPE_STRUCT = 0x040, - LFS_TYPE_GLOBALS = 0x080, - LFS_TYPE_TAIL = 0x0c0, - LFS_TYPE_SOFTTAIL = 0x0c0, - LFS_TYPE_HARDTAIL = 0x0c1, - LFS_TYPE_CRC = 0x0f0, + LFS_TYPE_GLOBALS = 0x0e0, + LFS_TYPE_TAIL = 0x080, + LFS_TYPE_SOFTTAIL = 0x080, + LFS_TYPE_HARDTAIL = 0x081, + LFS_TYPE_CRC = 0x0a0, LFS_TYPE_DIRSTRUCT = 0x040, LFS_TYPE_INLINESTRUCT = 0x041, @@ -111,9 +111,9 @@ enum lfs_type { // internal chip sources LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x050, - LFS_FROM_ATTRS = 0x060, - LFS_FROM_SUPERBLOCK = 0x070, + LFS_FROM_MOVE = 0x0c1, + LFS_FROM_ATTRS = 0x0c2, + LFS_FROM_SUPERBLOCK = 0x0c3, }; // File open flags diff --git a/tests/debug.py b/tests/debug.py index 117416a4..0beb4c4a 100755 --- a/tests/debug.py +++ b/tests/debug.py @@ -4,15 +4,15 @@ import binascii TYPES = { - (0x1ff, 0x001): 'reg', - (0x1ff, 0x002): 'dir', - (0x1ff, 0x011): 'superblock', - (0x1ff, 0x010): 'root', - (0x1ff, 0x030): 'delete', - (0x1f0, 0x080): 'globals', - (0x1ff, 0x0c0): 'tail soft', - (0x1ff, 0x0c1): 'tail hard', - (0x1f0, 0x0f0): 'crc', + (0x1ff, 0x002): 'reg', + (0x1ff, 0x003): 'dir', + (0x1ff, 0x001): 'superblock', + (0x1ff, 0x000): 'root', + (0x1ff, 0x020): 'delete', + (0x1f0, 0x0e0): 'globals', + (0x1ff, 0x080): 'tail soft', + (0x1ff, 0x081): 'tail hard', + (0x1f0, 0x0a0): 'crc', (0x1ff, 0x040): 'struct dir', (0x1ff, 0x041): 'struct inline', (0x1ff, 0x042): 'struct ctz', From c67a41af7a68390b9898b9f596e29879fe846e7d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 9 Sep 2018 17:48:11 -0500 Subject: [PATCH 101/139] Added support for deleting attributes littlefs has a mechanism for deleting file entries, but it doesn't have a mechanism for deleting individual tags. This _is_ sufficient for a filesystem, but limits our flexibility. Deleting attributes would be useful in the custom attribute API and for future improvements (hint the child pointers in B-trees). However, deleteing attributes is tricky. We can't just omit the attribute, since we can only add new tags. Additionally, we need a way to track what attributes have been deleted during compaction, which currently relies on writing out attributes to disk. The solution here is pretty nifty. First we have to come up with a way to represent a "deleted" attribute. Rather than adding an additional bit to the already squished tag structure, we use a -1 length field, specifically 0xfff. Now we can commit a delete attribute, and this deleted tag acts as a place holder during compacts. However our delete tag will never leave our metadata log. We need some way to discard our delete tag if we know it's the only representation of that tag on the metadata log. Ah! We know it's the only tag if it's in the first commit on the metadata log. So we add an additional bit to the CRC entry to indicate if we're on the first commit, and use that to decide if we need to keep delete tags around. Now we have working tag deletion. Interestingly enough, tag deletion is actually indirectly more efficient than entry deletion, since compacting entries requires multiple passes, whereas tag deletion gets cleaned up lazily. However we can't adopt the same strategy in entry deletion because of the compact ordering of entries. Tag deletion works because tag types are unique and static. Managing entry deletion in this manner would require static id allocation, which would cause problems when creating files, running out of space, and disallow arbitrary insertions of files. --- lfs.c | 66 +++++++++++++++++++++++++++++++++++++---------------------- lfs.h | 4 ++-- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/lfs.c b/lfs.c index 38460651..3b507ca9 100644 --- a/lfs.c +++ b/lfs.c @@ -380,6 +380,10 @@ static inline bool lfs_tag_isuser(uint32_t tag) { return (tag & 0x40000000); } +static inline bool lfs_tag_isdelete(uint32_t tag) { + return (tag & 0x00000fff) == 0xfff; +} + static inline uint16_t lfs_tag_type(uint32_t tag) { return (tag & 0x7fc00000) >> 22; } @@ -396,6 +400,10 @@ static inline lfs_size_t lfs_tag_size(uint32_t tag) { return tag & 0x00000fff; } +static inline lfs_size_t lfs_tag_dsize(uint32_t tag) { + return sizeof(tag) + lfs_tag_size(tag) + lfs_tag_isdelete(tag); +} + // operations on set of globals static inline void lfs_global_xor(lfs_global_t *a, const lfs_global_t *b) { for (int i = 0; i < sizeof(lfs_global_t)/4; i++) { @@ -469,8 +477,8 @@ static int32_t lfs_commit_get(lfs_t *lfs, gettag += getdiff; // iterate over dir block backwards (for faster lookups) - while (off >= 2*sizeof(uint32_t)+lfs_tag_size(tag)) { - off -= sizeof(tag)+lfs_tag_size(tag); + while (off >= sizeof(uint32_t) + lfs_tag_dsize(tag)) { + off -= lfs_tag_dsize(tag); if (lfs_tag_subtype(tag) == LFS_TYPE_CRC && stopatcommit) { break; @@ -481,7 +489,8 @@ static int32_t lfs_commit_get(lfs_t *lfs, } } - if ((tag & getmask) == ((gettag - getdiff) & getmask)) { + if (!lfs_tag_isdelete(tag) && + (tag & getmask) == ((gettag - getdiff) & getmask)) { if (buffer) { lfs_size_t diff = lfs_min( lfs_tag_size(gettag), lfs_tag_size(tag)); @@ -563,8 +572,8 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, } // check if we fit - lfs_size_t size = lfs_tag_size(tag); - if (commit->off + sizeof(tag)+size > commit->end) { + lfs_size_t dsize = lfs_tag_dsize(tag); + if (commit->off + dsize > commit->end) { return LFS_ERR_NOSPC; } @@ -577,18 +586,18 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, if (!(tag & 0x80000000)) { // from memory - err = lfs_commit_prog(lfs, commit, buffer, size); + err = lfs_commit_prog(lfs, commit, buffer, dsize-sizeof(tag)); if (err) { return err; } } else { // from disk const struct lfs_diskoff *disk = buffer; - for (lfs_off_t i = 0; i < size; i++) { + for (lfs_off_t i = 0; i < dsize-sizeof(tag); i++) { // rely on caching to make this efficient uint8_t dat; err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, size-i, + &lfs->pcache, &lfs->rcache, dsize-sizeof(tag)-i, disk->block, disk->off+i, &dat, 1); if (err) { return err; @@ -626,7 +635,8 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, // iterate through list and commits, only committing unique entries lfs_off_t off = dir->off; uint32_t ntag = dir->etag; - while (attrs || off >= 2*sizeof(uint32_t)+lfs_tag_size(ntag)) { + bool end = false; + while (attrs || off >= sizeof(uint32_t) + lfs_tag_dsize(ntag)) { struct lfs_diskoff disk; uint32_t tag; const void *buffer; @@ -635,7 +645,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, buffer = attrs->buffer; attrs = attrs->next; } else { - off -= sizeof(ntag)+lfs_tag_size(ntag); + off -= lfs_tag_dsize(ntag); tag = ntag; buffer = &disk; @@ -653,13 +663,16 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, tag |= 0x80000000; } - if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE && + if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { + end = 2 & lfs_tag_type(tag); + } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE && lfs_tag_id(tag) <= lfs_tag_id(fromtag - fromdiff)) { // something was deleted, we need to move around it fromdiff -= LFS_MKTAG(0, 1, 0); } - if ((tag & frommask) == ((fromtag - fromdiff) & frommask)) { + if ((tag & frommask) == ((fromtag - fromdiff) & frommask) && + !(lfs_tag_isdelete(tag) && end)) { bool duplicate; if (pass == 0) { duplicate = (lfs_tag_subtype(tag) != LFS_TYPE_NAME); @@ -714,7 +727,8 @@ static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, return err; } -static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { +static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, + bool compacting) { // align to program units lfs_off_t off = lfs_alignup(commit->off + 2*sizeof(uint32_t), lfs->cfg->prog_size); @@ -730,7 +744,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) { // build crc tag bool reset = ~lfs_fromle32(tag) >> 31; - tag = LFS_MKTAG(LFS_TYPE_CRC + reset, + tag = LFS_MKTAG(LFS_TYPE_CRC + (compacting << 1) + reset, 0x3ff, off - (commit->off+sizeof(uint32_t))); // write out crc @@ -889,7 +903,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, } // check we're in valid range - if (off + sizeof(tag)+lfs_tag_size(tag) > lfs->cfg->block_size) { + if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { dir->erased = false; break; } @@ -919,7 +933,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, // update with what's found so far foundtag = tempfoundtag; - dir->off = off + sizeof(tag)+lfs_tag_size(tag); + dir->off = off + lfs_tag_dsize(tag); dir->etag = tag; dir->count = tempcount; dir->tail[0] = temptail[0]; @@ -931,11 +945,11 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, crc = 0xffffffff; } else { // crc the entry first, leaving it in the cache - for (lfs_off_t j = 0; j < lfs_tag_size(tag); j++) { + for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) { uint8_t dat; err = lfs_bd_read(lfs, NULL, &lfs->rcache, lfs->cfg->block_size, - dir->pair[0], off+sizeof(tag)+j, &dat, 1); + dir->pair[0], off+j, &dat, 1); if (err) { if (err == LFS_ERR_CORRUPT) { dir->erased = false; @@ -994,7 +1008,9 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, if ((tag & findmask) == (findtag & findmask)) { // found a match? - if (lfs_tag_type(findtag) == LFS_TYPE_DIRSTRUCT) { + if (lfs_tag_isdelete(findtag)) { + tempfoundtag = LFS_ERR_NOENT; + } else if (lfs_tag_type(findtag) == LFS_TYPE_DIRSTRUCT) { lfs_block_t child[2]; err = lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, @@ -1037,7 +1053,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs, } ptag = tag; - off += sizeof(tag)+lfs_tag_size(tag); + off += lfs_tag_dsize(tag); } // consider what we have good enough @@ -1259,7 +1275,7 @@ static int lfs_dir_compact(lfs_t *lfs, } } - err = lfs_commit_crc(lfs, &commit); + err = lfs_commit_crc(lfs, &commit, true); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1428,7 +1444,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, .ack = 0, }; - // iterate over commits backwards, this lets us "append" commits cheaply + // iterate over commits backwards, this lets us "append" commits for (int i = 0; i < attrcount; i++) { const lfs_mattr_t *a = attrs; for (int j = 0; j < attrcount-i-1; j++) { @@ -1454,7 +1470,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, return err; } - err = lfs_commit_crc(lfs, &commit); + err = lfs_commit_crc(lfs, &commit, false); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -2926,8 +2942,8 @@ int lfs_setattr(lfs_t *lfs, const char *path, } return lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(0x100 | type, id, buffer, size, - NULL)); + LFS_MKATTR(0x100 | type, id, buffer, size, + NULL)); } diff --git a/lfs.h b/lfs.h index 7b0e8a18..987b234d 100644 --- a/lfs.h +++ b/lfs.h @@ -49,7 +49,7 @@ typedef uint32_t lfs_block_t; // to <= 0xfff. Stored in superblock and must be respected by other // littlefs drivers. #ifndef LFS_ATTR_MAX -#define LFS_ATTR_MAX 0xfff +#define LFS_ATTR_MAX 0xffe #endif // Maximum name size in bytes, may be redefined to reduce the size of the @@ -64,7 +64,7 @@ typedef uint32_t lfs_block_t; // block. Limited to <= LFS_ATTR_MAX and <= cache_size. Stored in superblock // and must be respected by other littlefs drivers. #ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 0xfff +#define LFS_INLINE_MAX 0xffe #endif // Possible error codes, these are negative to allow From 617dd87621e7e5120427fd0887dbfcdc8af525ac Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 9 Sep 2018 18:48:18 -0500 Subject: [PATCH 102/139] Added deletion to custom attributes This follows from enabling tag deletion, however does require some consideration with the APIs. Now we can remove custom attributes, as well as determine if an attribute exists or not. --- lfs.c | 39 +++++++++++++++++++++++++++------------ lfs.h | 14 +++++++++++--- tests/corrupt.py | 5 +++-- tests/debug.py | 9 +++++---- tests/test_attrs.sh | 18 +++++++++++++++++- 5 files changed, 63 insertions(+), 22 deletions(-) diff --git a/lfs.c b/lfs.c index 3b507ca9..5b4f0fd3 100644 --- a/lfs.c +++ b/lfs.c @@ -401,7 +401,7 @@ static inline lfs_size_t lfs_tag_size(uint32_t tag) { } static inline lfs_size_t lfs_tag_dsize(uint32_t tag) { - return sizeof(tag) + lfs_tag_size(tag) + lfs_tag_isdelete(tag); + return sizeof(tag) + lfs_tag_size(tag + lfs_tag_isdelete(tag)); } // operations on set of globals @@ -472,7 +472,7 @@ struct lfs_diskoff { }; static int32_t lfs_commit_get(lfs_t *lfs, - lfs_block_t block, lfs_off_t off, uint32_t tag, bool stopatcommit, + lfs_block_t block, lfs_off_t off, uint32_t tag, bool compacting, uint32_t getmask, uint32_t gettag, int32_t getdiff, void *buffer) { gettag += getdiff; @@ -480,7 +480,7 @@ static int32_t lfs_commit_get(lfs_t *lfs, while (off >= sizeof(uint32_t) + lfs_tag_dsize(tag)) { off -= lfs_tag_dsize(tag); - if (lfs_tag_subtype(tag) == LFS_TYPE_CRC && stopatcommit) { + if (lfs_tag_subtype(tag) == LFS_TYPE_CRC && compacting) { break; } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { // something was deleted, need to move around it @@ -489,8 +489,11 @@ static int32_t lfs_commit_get(lfs_t *lfs, } } - if (!lfs_tag_isdelete(tag) && - (tag & getmask) == ((gettag - getdiff) & getmask)) { + if ((tag & getmask) == ((gettag - getdiff) & getmask)) { + if (lfs_tag_isdelete(tag) && !compacting) { + return LFS_ERR_NOENT; + } + if (buffer) { lfs_size_t diff = lfs_min( lfs_tag_size(gettag), lfs_tag_size(tag)); @@ -2912,19 +2915,18 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, res = lfs_dir_get(lfs, &cwd, 0x7ffff000, LFS_MKTAG(0x100 | type, id, lfs_min(size, lfs->attr_max)), buffer); - if (res < 0 && res != LFS_ERR_NOENT) { + if (res < 0) { + if (res == LFS_ERR_NOENT) { + return LFS_ERR_NOATTR; + } return res; } - return (res == LFS_ERR_NOENT) ? 0 : lfs_tag_size(res); + return lfs_tag_size(res); } -int lfs_setattr(lfs_t *lfs, const char *path, +static int lfs_commitattr(lfs_t *lfs, const char *path, uint8_t type, const void *buffer, lfs_size_t size) { - if (size > lfs->attr_max) { - return LFS_ERR_NOSPC; - } - lfs_mdir_t cwd; int32_t res = lfs_dir_lookup(lfs, &cwd, &path); if (res < 0) { @@ -2946,6 +2948,19 @@ int lfs_setattr(lfs_t *lfs, const char *path, NULL)); } +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size) { + if (size > lfs->attr_max) { + return LFS_ERR_NOSPC; + } + + return lfs_commitattr(lfs, path, type, buffer, size); +} + +int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { + return lfs_commitattr(lfs, path, type, NULL, 0xfff); +} + /// Filesystem operations /// static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { diff --git a/lfs.h b/lfs.h index 987b234d..120a9a38 100644 --- a/lfs.h +++ b/lfs.h @@ -82,6 +82,7 @@ enum lfs_error { LFS_ERR_INVAL = -22, // Invalid parameter LFS_ERR_NOSPC = -28, // No space left on device LFS_ERR_NOMEM = -12, // No more memory available + LFS_ERR_NOATTR = -61, // No data/attr available LFS_ERR_NAMETOOLONG = -36, // File name too long }; @@ -456,7 +457,8 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); // Custom attributes are uniquely identified by an 8-bit type and limited // to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller than // the buffer, it will be padded with zeros. If the stored attribute is larger, -// then it will be silently truncated. +// then it will be silently truncated. If no attribute is found, the error +// LFS_ERR_NOATTR is returned and the buffer is filled with zeros. // // Returns the size of the attribute, or a negative error code on failure. // Note, the returned size is the size of the attribute on disk, irrespective @@ -469,13 +471,19 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, // // Custom attributes are uniquely identified by an 8-bit type and limited // to LFS_ATTR_MAX bytes. If an attribute is not found, it will be -// implicitly created, and setting the size of an attribute to zero deletes -// the attribute. +// implicitly created. // // Returns a negative error code on failure. int lfs_setattr(lfs_t *lfs, const char *path, uint8_t type, const void *buffer, lfs_size_t size); +// Removes a custom attribute +// +// If an attribute is not found, nothing happens. +// +// Returns a negative error code on failure. +int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type); + /// File operations /// diff --git a/tests/corrupt.py b/tests/corrupt.py index 9b5d4bd2..4a49d428 100755 --- a/tests/corrupt.py +++ b/tests/corrupt.py @@ -19,10 +19,11 @@ def corrupt(block): break tag ^= ntag - file.seek(tag & 0xfff, os.SEEK_CUR) + size = (tag & 0xfff) if (tag & 0xfff) != 0xfff else 0 + file.seek(size, os.SEEK_CUR) # lob off last 3 bytes - file.seek(-((tag & 0xfff) + 3), os.SEEK_CUR) + file.seek(-(size + 3), os.SEEK_CUR) file.truncate() def main(args): diff --git a/tests/debug.py b/tests/debug.py index 0beb4c4a..3c975c84 100755 --- a/tests/debug.py +++ b/tests/debug.py @@ -81,20 +81,21 @@ def main(*blocks): size = (tag & 0x00000fff) >> 0 iscrc = (type & 0x1f0) == 0x0f0 - data = file.read(size) + data = file.read(size if size != 0xfff else 0) if iscrc: crc = binascii.crc32(data[:4], crc) else: crc = binascii.crc32(data, crc) - print '%04x: %08x %-14s %3s %3d %-23s %-8s' % ( + print '%04x: %08x %-14s %3s %3s %-23s %-8s' % ( off, tag, typeof(type) + (' bad!' if iscrc and ~crc else ''), - id if id != 0x3ff else '.', size, + id if id != 0x3ff else '.', + size if size != 0xfff else 'x', ' '.join('%02x' % ord(c) for c in data[:8]), ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8])) - off += tag & 0xfff + off += size if size != 0xfff else 0 if iscrc: crc = 0 tag ^= (type & 1) << 31 diff --git a/tests/test_attrs.sh b/tests/test_attrs.sh index bbe78cc4..e9b2227d 100755 --- a/tests/test_attrs.sh +++ b/tests/test_attrs.sh @@ -37,6 +37,14 @@ tests/test.py << TEST memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; + lfs_removeattr(&lfs, "hello", 'B') => 0; + lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "hello", 'B', buffer+4, 6) => LFS_ERR_NOATTR; + lfs_getattr(&lfs, "hello", 'C', buffer+10, 5) => 5; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_setattr(&lfs, "hello", 'B', "dddddd", 6) => 0; lfs_getattr(&lfs, "hello", 'A', buffer, 4) => 4; lfs_getattr(&lfs, "hello", 'B', buffer+4, 6) => 6; @@ -98,6 +106,14 @@ tests/test.py << TEST memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; + lfs_removeattr(&lfs, "/", 'B') => 0; + lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; + lfs_getattr(&lfs, "/", 'B', buffer+4, 6) => LFS_ERR_NOATTR; + lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_setattr(&lfs, "/", 'B', "dddddd", 6) => 0; lfs_getattr(&lfs, "/", 'A', buffer, 4) => 4; lfs_getattr(&lfs, "/", 'B', buffer+4, 6) => 6; @@ -241,7 +257,7 @@ tests/test.py << TEST lfs_getattr(&lfs, "hello/hello", 'B', buffer, 9) => 9; lfs_getattr(&lfs, "hello/hello", 'C', buffer+9, 9) => 5; - lfs_getattr(&lfs, "hello/hello", 'D', buffer+18, 9) => 0; + lfs_getattr(&lfs, "hello/hello", 'D', buffer+18, 9) => LFS_ERR_NOATTR; memcmp(buffer, "fffffffff", 9) => 0; memcmp(buffer+9, "ccccc\0\0\0\0", 9) => 0; memcmp(buffer+18, "\0\0\0\0\0\0\0\0\0", 9) => 0; From 5eeeb9d6ac0e71f182c107a540939a56b26bb957 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 10 Sep 2018 22:07:59 -0500 Subject: [PATCH 103/139] Revisited some generic concepts, callbacks, and some reorganization - Callbacks for get/match, this does have a code cost, but allows more code reuse, which almost balances out the code cost, but also reduces maintenance and increased flexibility. Also callbacks may be able to be gc-ed in some cases. - Consistent struct vs _t usage, _t for external-facing struct that shouldn't be messed with outside the library. structs for external and internal structs where anyone with access is allowed to modify. - Reorganized several high-level function groups - Inlined structures that didn't need separate definitions in header --- lfs.c | 1344 +++++++++++++++++++++++++++++---------------------------- lfs.h | 44 +- 2 files changed, 707 insertions(+), 681 deletions(-) diff --git a/lfs.c b/lfs.c index 5b4f0fd3..869205ff 100644 --- a/lfs.c +++ b/lfs.c @@ -224,83 +224,8 @@ static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { } -/// Internal operations predeclared here /// -static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], - lfs_mdir_t *pdir); -static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], - lfs_mdir_t *parent); -static int lfs_fs_relocate(lfs_t *lfs, - const lfs_block_t oldpair[2], lfs_block_t newpair[2]); -static int lfs_fs_deorphan(lfs_t *lfs); -static int lfs_fs_demove(lfs_t *lfs); -static int lfs_fs_forceconsistency(lfs_t *lfs); -static int lfs_deinit(lfs_t *lfs); - - -/// Block allocator /// -static int lfs_alloc_lookahead(void *p, lfs_block_t block) { - lfs_t *lfs = (lfs_t*)p; - lfs_block_t off = ((block - lfs->free.off) - + lfs->cfg->block_count) % lfs->cfg->block_count; - - if (off < lfs->free.size) { - lfs->free.buffer[off / 32] |= 1U << (off % 32); - } - - return 0; -} - -static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { - while (true) { - while (lfs->free.i != lfs->free.size) { - lfs_block_t off = lfs->free.i; - lfs->free.i += 1; - lfs->free.ack -= 1; - - if (!(lfs->free.buffer[off / 32] & (1U << (off % 32)))) { - // found a free block - *block = (lfs->free.off + off) % lfs->cfg->block_count; - - // eagerly find next off so an alloc ack can - // discredit old lookahead blocks - while (lfs->free.i != lfs->free.size && - (lfs->free.buffer[lfs->free.i / 32] - & (1U << (lfs->free.i % 32)))) { - lfs->free.i += 1; - lfs->free.ack -= 1; - } - - return 0; - } - } - - // check if we have looked at all blocks since last ack - if (lfs->free.ack == 0) { - LFS_WARN("No more free space %"PRIu32, - lfs->free.i + lfs->free.off); - return LFS_ERR_NOSPC; - } - - lfs->free.off = (lfs->free.off + lfs->free.size) - % lfs->cfg->block_count; - lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->free.ack); - lfs->free.i = 0; - - // find mask of free blocks from tree - memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); - int err = lfs_fs_traverse(lfs, lfs_alloc_lookahead, lfs); - if (err) { - return err; - } - } -} - -static void lfs_alloc_ack(lfs_t *lfs) { - lfs->free.ack = lfs->cfg->block_count; -} - - -/// Metadata pair and directory operations /// +/// Small type-level utilities /// +// operations on block pairs static inline void lfs_pair_swap(lfs_block_t pair[2]) { lfs_block_t t = pair[0]; pair[0] = pair[1]; @@ -335,75 +260,60 @@ static inline void lfs_pair_tole32(lfs_block_t pair[2]) { pair[1] = lfs_tole32(pair[1]); } -static void lfs_ctz_fromle32(struct lfs_ctz *ctz) { - ctz->head = lfs_fromle32(ctz->head); - ctz->size = lfs_fromle32(ctz->size); -} - -static void lfs_ctz_tole32(struct lfs_ctz *ctz) { - ctz->head = lfs_tole32(ctz->head); - ctz->size = lfs_tole32(ctz->size); -} - -static inline void lfs_superblock_fromle32(lfs_superblock_t *superblock) { - superblock->version = lfs_fromle32(superblock->version); - superblock->block_size = lfs_fromle32(superblock->block_size); - superblock->block_count = lfs_fromle32(superblock->block_count); - superblock->inline_max = lfs_fromle32(superblock->inline_max); - superblock->attr_max = lfs_fromle32(superblock->attr_max); - superblock->name_max = lfs_fromle32(superblock->name_max); -} - -static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { - superblock->version = lfs_tole32(superblock->version); - superblock->block_size = lfs_tole32(superblock->block_size); - superblock->block_count = lfs_tole32(superblock->block_count); - superblock->inline_max = lfs_tole32(superblock->inline_max); - superblock->attr_max = lfs_tole32(superblock->attr_max); - superblock->name_max = lfs_tole32(superblock->name_max); -} - +// operations on 32-bit entry tags +typedef uint32_t lfs_tag_t; +typedef int32_t lfs_stag_t; - -/// Entry tag operations /// #define LFS_MKTAG(type, id, size) \ - (((uint32_t)(type) << 22) | ((uint32_t)(id) << 12) | (uint32_t)(size)) - -#define LFS_MKATTR(type, id, buffer, size, next) \ - &(const lfs_mattr_t){LFS_MKTAG(type, id, size), (buffer), (next)} + (((lfs_tag_t)(type) << 22) | ((lfs_tag_t)(id) << 12) | (lfs_tag_t)(size)) -static inline bool lfs_tag_isvalid(uint32_t tag) { +static inline bool lfs_tag_isvalid(lfs_tag_t tag) { return !(tag & 0x80000000); } -static inline bool lfs_tag_isuser(uint32_t tag) { +static inline bool lfs_tag_isuser(lfs_tag_t tag) { return (tag & 0x40000000); } -static inline bool lfs_tag_isdelete(uint32_t tag) { +static inline bool lfs_tag_isdelete(lfs_tag_t tag) { return (tag & 0x00000fff) == 0xfff; } -static inline uint16_t lfs_tag_type(uint32_t tag) { +static inline uint16_t lfs_tag_type(lfs_tag_t tag) { return (tag & 0x7fc00000) >> 22; } -static inline uint16_t lfs_tag_subtype(uint32_t tag) { +static inline uint16_t lfs_tag_subtype(lfs_tag_t tag) { return ((tag & 0x7c000000) >> 26) << 4; } -static inline uint16_t lfs_tag_id(uint32_t tag) { +static inline uint16_t lfs_tag_id(lfs_tag_t tag) { return (tag & 0x003ff000) >> 12; } -static inline lfs_size_t lfs_tag_size(uint32_t tag) { +static inline lfs_size_t lfs_tag_size(lfs_tag_t tag) { return tag & 0x00000fff; } -static inline lfs_size_t lfs_tag_dsize(uint32_t tag) { +static inline lfs_size_t lfs_tag_dsize(lfs_tag_t tag) { return sizeof(tag) + lfs_tag_size(tag + lfs_tag_isdelete(tag)); } +// operations on attributes in attribute lists +struct lfs_mattr { + lfs_tag_t tag; + const void *buffer; + const struct lfs_mattr *next; +}; + +#define LFS_MKATTR(type, id, buffer, size, next) \ + &(const struct lfs_mattr){LFS_MKTAG(type, id, size), (buffer), (next)} + +struct lfs_diskoff { + lfs_block_t block; + lfs_off_t off; +}; + // operations on set of globals static inline void lfs_global_xor(lfs_global_t *a, const lfs_global_t *b) { for (int i = 0; i < sizeof(lfs_global_t)/4; i++) { @@ -453,171 +363,566 @@ static inline void lfs_global_orphans(lfs_t *lfs, int8_t orphans) { lfs->globals.g.orphans += orphans; } +// other endianness operations +static void lfs_ctz_fromle32(struct lfs_ctz *ctz) { + ctz->head = lfs_fromle32(ctz->head); + ctz->size = lfs_fromle32(ctz->size); +} -// commit logic -struct lfs_commit { - lfs_block_t block; - lfs_off_t off; - uint32_t ptag; - uint32_t crc; +static void lfs_ctz_tole32(struct lfs_ctz *ctz) { + ctz->head = lfs_tole32(ctz->head); + ctz->size = lfs_tole32(ctz->size); +} - lfs_off_t begin; - lfs_off_t end; - lfs_off_t ack; -}; +static inline void lfs_superblock_fromle32(lfs_superblock_t *superblock) { + superblock->version = lfs_fromle32(superblock->version); + superblock->block_size = lfs_fromle32(superblock->block_size); + superblock->block_count = lfs_fromle32(superblock->block_count); + superblock->inline_max = lfs_fromle32(superblock->inline_max); + superblock->attr_max = lfs_fromle32(superblock->attr_max); + superblock->name_max = lfs_fromle32(superblock->name_max); +} -struct lfs_diskoff { - lfs_block_t block; - lfs_off_t off; -}; +static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { + superblock->version = lfs_tole32(superblock->version); + superblock->block_size = lfs_tole32(superblock->block_size); + superblock->block_count = lfs_tole32(superblock->block_count); + superblock->inline_max = lfs_tole32(superblock->inline_max); + superblock->attr_max = lfs_tole32(superblock->attr_max); + superblock->name_max = lfs_tole32(superblock->name_max); +} -static int32_t lfs_commit_get(lfs_t *lfs, - lfs_block_t block, lfs_off_t off, uint32_t tag, bool compacting, - uint32_t getmask, uint32_t gettag, int32_t getdiff, void *buffer) { - gettag += getdiff; - // iterate over dir block backwards (for faster lookups) - while (off >= sizeof(uint32_t) + lfs_tag_dsize(tag)) { - off -= lfs_tag_dsize(tag); +/// Internal operations predeclared here /// +static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], + lfs_mdir_t *pdir); +static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], + lfs_mdir_t *parent); +static int lfs_fs_relocate(lfs_t *lfs, + const lfs_block_t oldpair[2], lfs_block_t newpair[2]); +static int lfs_fs_forceconsistency(lfs_t *lfs); +static int lfs_deinit(lfs_t *lfs); - if (lfs_tag_subtype(tag) == LFS_TYPE_CRC && compacting) { - break; - } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { - // something was deleted, need to move around it - if (lfs_tag_id(tag) <= lfs_tag_id(gettag - getdiff)) { - getdiff -= LFS_MKTAG(0, 1, 0); - } - } - if ((tag & getmask) == ((gettag - getdiff) & getmask)) { - if (lfs_tag_isdelete(tag) && !compacting) { - return LFS_ERR_NOENT; - } +/// Block allocator /// +static int lfs_alloc_lookahead(void *p, lfs_block_t block) { + lfs_t *lfs = (lfs_t*)p; + lfs_block_t off = ((block - lfs->free.off) + + lfs->cfg->block_count) % lfs->cfg->block_count; - if (buffer) { - lfs_size_t diff = lfs_min( - lfs_tag_size(gettag), lfs_tag_size(tag)); - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, diff, - block, off+sizeof(tag), buffer, diff); - if (err) { - return err; + if (off < lfs->free.size) { + lfs->free.buffer[off / 32] |= 1U << (off % 32); + } + + return 0; +} + +static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { + while (true) { + while (lfs->free.i != lfs->free.size) { + lfs_block_t off = lfs->free.i; + lfs->free.i += 1; + lfs->free.ack -= 1; + + if (!(lfs->free.buffer[off / 32] & (1U << (off % 32)))) { + // found a free block + *block = (lfs->free.off + off) % lfs->cfg->block_count; + + // eagerly find next off so an alloc ack can + // discredit old lookahead blocks + while (lfs->free.i != lfs->free.size && + (lfs->free.buffer[lfs->free.i / 32] + & (1U << (lfs->free.i % 32)))) { + lfs->free.i += 1; + lfs->free.ack -= 1; } - memset((uint8_t*)buffer + diff, 0, - lfs_tag_size(gettag) - diff); + return 0; } - - return tag + getdiff; } - if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { - // found where something was created - if (lfs_tag_id(tag) == lfs_tag_id(gettag - getdiff)) { - break; - } else if (lfs_tag_id(tag) < lfs_tag_id(gettag - getdiff)) { - getdiff += LFS_MKTAG(0, 1, 0); - } + // check if we have looked at all blocks since last ack + if (lfs->free.ack == 0) { + LFS_WARN("No more free space %"PRIu32, + lfs->free.i + lfs->free.off); + return LFS_ERR_NOSPC; } - uint32_t ntag; - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(ntag), - block, off, &ntag, sizeof(ntag)); + lfs->free.off = (lfs->free.off + lfs->free.size) + % lfs->cfg->block_count; + lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->free.ack); + lfs->free.i = 0; + + // find mask of free blocks from tree + memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); + int err = lfs_fs_traverse(lfs, lfs_alloc_lookahead, lfs); if (err) { return err; } - tag ^= lfs_fromle32(ntag); - tag &= 0x7fffffff; } +} - return LFS_ERR_NOENT; +static void lfs_alloc_ack(lfs_t *lfs) { + lfs->free.ack = lfs->cfg->block_count; } -static int lfs_commit_prog(lfs_t *lfs, struct lfs_commit *commit, - const void *buffer, lfs_size_t size) { - lfs_off_t skip = lfs_min(lfs_max(commit->ack, commit->off) - - commit->off, size); - int err = lfs_bd_prog(lfs, - &lfs->pcache, &lfs->rcache, false, - commit->block, commit->off + skip, - (const uint8_t*)buffer + skip, size - skip); - if (err) { - return err; - } - commit->crc = lfs_crc(commit->crc, buffer, size); - commit->off += size; - commit->ack = lfs_max(commit->off, commit->ack); - return 0; -} +/// Metadata pair and directory operations /// +static int lfs_dir_traverse(lfs_t *lfs, + const lfs_mdir_t *dir, const struct lfs_mattr *attrs, + lfs_tag_t matchmask, lfs_tag_t matchtag, lfs_stag_t matchdiff, + int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + lfs_block_t block = dir->pair[0]; + lfs_off_t off = dir->off; + lfs_tag_t ntag = dir->etag; + bool lastcommit = false; + matchtag += matchdiff; -static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, - uint16_t id, const struct lfs_attr *attrs); + // iterate over dir block backwards (for faster lookups) + while (attrs || off >= sizeof(lfs_tag_t) + lfs_tag_dsize(ntag)) { + lfs_tag_t tag; + const void *buffer; + struct lfs_diskoff disk; + if (attrs) { + tag = attrs->tag; + buffer = attrs->buffer; + attrs = attrs->next; + } else { + off -= lfs_tag_dsize(ntag); -static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, - uint32_t frommask, uint32_t fromtag, int32_t fromdiff, - const lfs_mdir_t *dir, const lfs_mattr_t *attrs); + tag = ntag; + buffer = &disk; + disk.block = block; + disk.off = off + sizeof(tag); -static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, - uint32_t tag, const void *buffer) { - if (lfs_tag_type(tag) == LFS_FROM_MOVE) { - // special case for moves - return lfs_commit_move(lfs, commit, 1, - 0x003ff000, LFS_MKTAG(0, lfs_tag_size(tag), 0), - LFS_MKTAG(0, lfs_tag_id(tag), 0) - - LFS_MKTAG(0, lfs_tag_size(tag), 0), - buffer, NULL); - } else if (lfs_tag_type(tag) == LFS_FROM_ATTRS) { - // special case for custom attributes - return lfs_commit_attrs(lfs, commit, - lfs_tag_id(tag), buffer); - } + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(ntag), + block, off, &ntag, sizeof(ntag)); + if (err) { + return err; + } - // check if we fit - lfs_size_t dsize = lfs_tag_dsize(tag); - if (commit->off + dsize > commit->end) { - return LFS_ERR_NOSPC; - } + ntag = lfs_fromle32(ntag) ^ tag; + tag |= 0x80000000; + } - // write out tag - uint32_t ntag = lfs_tole32((tag & 0x7fffffff) ^ commit->ptag); - int err = lfs_commit_prog(lfs, commit, &ntag, sizeof(ntag)); - if (err) { - return err; + if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { + lastcommit = 2 & lfs_tag_type(tag); + } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { + // something was deleted, need to move around it + if (lfs_tag_id(tag) <= lfs_tag_id(matchtag - matchdiff)) { + matchdiff -= LFS_MKTAG(0, 1, 0); + } + } + + if ((tag & matchmask) == ((matchtag - matchdiff) & matchmask) && + !(lfs_tag_isdelete(tag) && lastcommit)) { + int res = cb(data, tag + matchdiff, buffer); + if (res) { + return res; + } + } + + if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { + // found where something was created + if (lfs_tag_id(tag) == lfs_tag_id(matchtag - matchdiff)) { + break; + } else if (lfs_tag_id(tag) < lfs_tag_id(matchtag - matchdiff)) { + matchdiff += LFS_MKTAG(0, 1, 0); + } + } } - if (!(tag & 0x80000000)) { - // from memory - err = lfs_commit_prog(lfs, commit, buffer, dsize-sizeof(tag)); - if (err) { + return 0; +} + +static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2], + lfs_tag_t matchmask, lfs_tag_t matchtag, + int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + // find the block with the most recent revision + uint32_t revs[2]; + int r = 0; + for (int i = 0; i < 2; i++) { + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(revs[i]), + pair[i], 0, &revs[i], sizeof(revs[i])); + revs[i] = lfs_fromle32(revs[i]); + if (err && err != LFS_ERR_CORRUPT) { return err; } - } else { - // from disk - const struct lfs_diskoff *disk = buffer; - for (lfs_off_t i = 0; i < dsize-sizeof(tag); i++) { - // rely on caching to make this efficient - uint8_t dat; - err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, dsize-sizeof(tag)-i, - disk->block, disk->off+i, &dat, 1); + + if (lfs_scmp(revs[i], revs[(i+1)%2]) > 0 || err == LFS_ERR_CORRUPT) { + r = i; + } + } + + // now fetch the actual dir (and find match) + lfs_stag_t foundtag = LFS_ERR_NOENT; + dir->pair[0] = pair[0]; + dir->pair[1] = pair[1]; + dir->off = 0; + if (r != 0) { + lfs_pair_swap(dir->pair); + lfs_pair_swap(revs); + } + + // scan tags and check crcs + for (int i = 0; i < 2; i++) { + lfs_block_t block = dir->pair[0]; + lfs_off_t off = sizeof(uint32_t); + lfs_tag_t ptag = 0xffffffff; + + lfs_tag_t tempfoundtag = foundtag; + lfs_mdir_t temp = { + .pair = {dir->pair[0], dir->pair[1]}, + .rev = revs[0], + .tail = {0xffffffff, 0xffffffff}, + .split = false, + .count = 0, + }; + lfs_global_zero(&temp.locals); + + temp.rev = lfs_tole32(temp.rev); + uint32_t crc = lfs_crc(0xffffffff, &temp.rev, sizeof(temp.rev)); + temp.rev = lfs_fromle32(temp.rev); + + while (true) { + // extract next tag + lfs_tag_t tag; + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + block, off, &tag, sizeof(tag)); if (err) { + if (err == LFS_ERR_CORRUPT) { + // can't continue? + dir->erased = false; + break; + } return err; } - err = lfs_commit_prog(lfs, commit, &dat, 1); - if (err) { - return err; + crc = lfs_crc(crc, &tag, sizeof(tag)); + tag = lfs_fromle32(tag) ^ ptag; + + // next commit not yet programmed + if (!lfs_tag_isvalid(tag)) { + dir->erased = (lfs_tag_subtype(ptag) == LFS_TYPE_CRC); + break; + } + + // check we're in valid range + if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { + dir->erased = false; + break; + } + + if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { + // check the crc attr + uint32_t dcrc; + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + block, off+sizeof(tag), &dcrc, sizeof(dcrc)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + return err; + } + dcrc = lfs_fromle32(dcrc); + + if (crc != dcrc) { + dir->erased = false; + break; + } + + // reset the next bit if we need to + tag ^= (lfs_tag_type(tag) & 1) << 31; + lfs->seed ^= crc; + crc = 0xffffffff; + + // update with what's found so far + foundtag = tempfoundtag; + *dir = temp; + dir->off = off + lfs_tag_dsize(tag); + dir->etag = tag; + } else { + // crc the entry first, leaving it in the cache + for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) { + uint8_t dat; + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + block, off+j, &dat, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + return err; + } + + crc = lfs_crc(crc, &dat, 1); + } + + // check for special tags + if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { + temp.count += 1; + + if (lfs_tag_isvalid(tempfoundtag) && + lfs_tag_id(tag) <= lfs_tag_id(tempfoundtag)) { + tempfoundtag += LFS_MKTAG(0, 1, 0); + } + } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { + LFS_ASSERT(temp.count > 0); + temp.count -= 1; + + if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { + tempfoundtag = LFS_ERR_NOENT; + } else if (lfs_tag_isvalid(tempfoundtag) && + lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { + tempfoundtag -= LFS_MKTAG(0, 1, 0); + } + } else if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { + temp.split = (lfs_tag_type(tag) & 1); + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + block, off+sizeof(tag), + &temp.tail, sizeof(temp.tail)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + } + lfs_pair_fromle32(temp.tail); + } else if (lfs_tag_subtype(tag) == LFS_TYPE_GLOBALS) { + temp.locals.l.deorphaned = (lfs_tag_type(tag) & 1); + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + block, off+sizeof(tag), + &temp.locals, 10); + if (err) { + if (err == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + } + } + + if ((tag & matchmask) == (matchtag & matchmask)) { + // found a match? + if (lfs_tag_isdelete(tag)) { + tempfoundtag = LFS_ERR_NOENT; + } else if (cb) { + int res = cb(data, tag, &(struct lfs_diskoff){ + block, off+sizeof(tag)}); + if (res < 0) { + if (res == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + return res; + } + + if (res) { + tempfoundtag = tag; + } + } + } + } + + ptag = tag; + off += lfs_tag_dsize(tag); + } + + // consider what we have good enough + if (dir->off > 0) { + // synthetic move + if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0) { + if (lfs->globals.g.moveid == lfs_tag_id(foundtag)) { + foundtag = LFS_ERR_NOENT; + } else if (lfs_tag_isvalid(foundtag) && + lfs->globals.g.moveid < lfs_tag_id(foundtag)) { + foundtag -= LFS_MKTAG(0, 1, 0); + } } + + return foundtag; } + + // failed, try the other crc? + lfs_pair_swap(dir->pair); + lfs_pair_swap(revs); + } + + LFS_ERROR("Corrupted dir pair at %"PRIu32" %"PRIu32, + dir->pair[0], dir->pair[1]); + return LFS_ERR_CORRUPT; +} + +static int lfs_dir_fetch(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2]) { + lfs_stag_t res = lfs_dir_fetchmatch(lfs, dir, pair, 0, 0, NULL, NULL); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; } - commit->ptag = tag & 0x7fffffff; return 0; } -static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, +static lfs_stag_t lfs_dir_findmatch(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2], bool fs, + lfs_tag_t findmask, lfs_tag_t findtag, + int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + dir->split = true; + dir->tail[0] = pair[0]; + dir->tail[1] = pair[1]; + while ((dir->split || fs) && !lfs_pair_isnull(dir->tail)) { + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, + findmask, findtag, cb, data); + if (tag != LFS_ERR_NOENT) { + return tag; + } + } + + return LFS_ERR_NOENT; +} + +struct lfs_dir_get_match { + lfs_t *lfs; + void *buffer; + lfs_size_t size; + bool compacting; +}; + +static int lfs_dir_get_match(void *data, + lfs_tag_t tag, const void *buffer) { + struct lfs_dir_get_match *get = data; + lfs_t *lfs = get->lfs; + const struct lfs_diskoff *disk = buffer; + + if (lfs_tag_isdelete(tag) && !get->compacting) { + return LFS_ERR_NOENT; + } + + if (get->buffer) { + lfs_size_t diff = lfs_min(lfs_tag_size(tag), get->size); + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, diff, + disk->block, disk->off, get->buffer, diff); + if (err) { + return err; + } + + memset((uint8_t*)get->buffer + diff, 0, get->size - diff); + } + + return tag & 0x7fffffff; +} + +static lfs_stag_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, + lfs_tag_t getmask, lfs_tag_t gettag, void *buffer) { + lfs_stag_t getdiff = 0; + if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0 && + lfs_tag_id(gettag) <= lfs->globals.g.moveid) { + // synthetic moves + gettag += LFS_MKTAG(0, 1, 0); + getdiff -= LFS_MKTAG(0, 1, 0); + } + + lfs_stag_t res = lfs_dir_traverse(lfs, dir, NULL, + getmask, gettag, getdiff, + lfs_dir_get_match, &(struct lfs_dir_get_match){ + lfs, buffer, lfs_tag_size(gettag)}); + if (res < 0) { + return res; + } + + return res ? res : LFS_ERR_NOENT; +} + +// commit logic +struct lfs_commit { + lfs_block_t block; + lfs_off_t off; + lfs_tag_t ptag; + uint32_t crc; + + lfs_off_t begin; + lfs_off_t end; + lfs_off_t ack; +}; + +static int lfs_commit_prog(lfs_t *lfs, struct lfs_commit *commit, + const void *buffer, lfs_size_t size) { + lfs_off_t skip = lfs_min(lfs_max(commit->ack, commit->off) + - commit->off, size); + int err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, false, + commit->block, commit->off + skip, + (const uint8_t*)buffer + skip, size - skip); + if (err) { + return err; + } + + commit->crc = lfs_crc(commit->crc, buffer, size); + commit->off += size; + commit->ack = lfs_max(commit->off, commit->ack); + return 0; +} + +static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, + lfs_tag_t tag, const void *buffer); + +struct lfs_commit_move_match { + lfs_t *lfs; + struct lfs_commit *commit; + int pass; +}; + +static int lfs_commit_move_match(void *data, + lfs_tag_t tag, const void *buffer) { + struct lfs_commit_move_match *move = data; + lfs_t *lfs = move->lfs; + struct lfs_commit *commit = move->commit; + + if (move->pass == 0) { + if (lfs_tag_subtype(tag) != LFS_TYPE_NAME) { + return 0; + } + } else { + // check if type has already been committed + lfs_stag_t res = lfs_dir_traverse(lfs, &(const lfs_mdir_t){ + .pair[0] = commit->block, + .off = commit->off, + .etag = commit->ptag}, NULL, + lfs_tag_isuser(tag) ? 0x7ffff000 : 0x7c3ff000, tag, 0, + lfs_dir_get_match, &(struct lfs_dir_get_match){ + lfs, NULL, 0, true}); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + + if (res > 0) { + return 0; + } + } + + // update id and commit, as we are currently unique + return lfs_commit_attr(lfs, commit, tag, buffer); +} + +static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, + lfs_tag_t frommask, lfs_tag_t fromtag, lfs_stag_t fromdiff, + const lfs_mdir_t *dir, const struct lfs_mattr *attrs) { + return lfs_dir_traverse(lfs, dir, attrs, + frommask, fromtag, fromdiff, + lfs_commit_move_match, &(struct lfs_commit_move_match){ + lfs, commit, pass}); +} + +static int lfs_commit_userattrs(lfs_t *lfs, struct lfs_commit *commit, uint16_t id, const struct lfs_attr *attrs) { for (const struct lfs_attr *a = attrs; a; a = a->next) { int err = lfs_commit_attr(lfs, commit, @@ -630,89 +935,61 @@ static int lfs_commit_attrs(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, - uint32_t frommask, uint32_t fromtag, int32_t fromdiff, - const lfs_mdir_t *dir, const lfs_mattr_t *attrs) { - fromtag += fromdiff; +static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, + lfs_tag_t tag, const void *buffer) { + if (lfs_tag_type(tag) == LFS_FROM_MOVE) { + // special case for moves + return lfs_commit_move(lfs, commit, 1, + 0x003ff000, LFS_MKTAG(0, lfs_tag_size(tag), 0), + LFS_MKTAG(0, lfs_tag_id(tag), 0) - + LFS_MKTAG(0, lfs_tag_size(tag), 0), + buffer, NULL); + } else if (lfs_tag_type(tag) == LFS_FROM_USERATTRS) { + // special case for custom attributes + return lfs_commit_userattrs(lfs, commit, + lfs_tag_id(tag), buffer); + } - // iterate through list and commits, only committing unique entries - lfs_off_t off = dir->off; - uint32_t ntag = dir->etag; - bool end = false; - while (attrs || off >= sizeof(uint32_t) + lfs_tag_dsize(ntag)) { - struct lfs_diskoff disk; - uint32_t tag; - const void *buffer; - if (attrs) { - tag = attrs->tag; - buffer = attrs->buffer; - attrs = attrs->next; - } else { - off -= lfs_tag_dsize(ntag); + // check if we fit + lfs_size_t dsize = lfs_tag_dsize(tag); + if (commit->off + dsize > commit->end) { + return LFS_ERR_NOSPC; + } - tag = ntag; - buffer = &disk; - disk.block = dir->pair[0]; - disk.off = off + sizeof(tag); + // write out tag + lfs_tag_t ntag = lfs_tole32((tag & 0x7fffffff) ^ commit->ptag); + int err = lfs_commit_prog(lfs, commit, &ntag, sizeof(ntag)); + if (err) { + return err; + } - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(ntag), - dir->pair[0], off, &ntag, sizeof(ntag)); + if (!(tag & 0x80000000)) { + // from memory + err = lfs_commit_prog(lfs, commit, buffer, dsize-sizeof(tag)); + if (err) { + return err; + } + } else { + // from disk + const struct lfs_diskoff *disk = buffer; + for (lfs_off_t i = 0; i < dsize-sizeof(tag); i++) { + // rely on caching to make this efficient + uint8_t dat; + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, dsize-sizeof(tag)-i, + disk->block, disk->off+i, &dat, 1); if (err) { return err; } - ntag = lfs_fromle32(ntag) ^ tag; - tag |= 0x80000000; - } - - if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { - end = 2 & lfs_tag_type(tag); - } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE && - lfs_tag_id(tag) <= lfs_tag_id(fromtag - fromdiff)) { - // something was deleted, we need to move around it - fromdiff -= LFS_MKTAG(0, 1, 0); - } - - if ((tag & frommask) == ((fromtag - fromdiff) & frommask) && - !(lfs_tag_isdelete(tag) && end)) { - bool duplicate; - if (pass == 0) { - duplicate = (lfs_tag_subtype(tag) != LFS_TYPE_NAME); - } else { - // check if type has already been committed - int32_t res = lfs_commit_get(lfs, - commit->block, commit->off, commit->ptag, true, - lfs_tag_isuser(tag) ? 0x7ffff000 : 0x7c3ff000, - tag + fromdiff, 0, NULL); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - - duplicate = (res != LFS_ERR_NOENT); - } - - if (!duplicate) { - // update id and commit, as we are currently unique - int err = lfs_commit_attr(lfs, commit, - tag + fromdiff, - buffer); - if (err) { - return err; - } - } - } - - if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { - // found where something was created - if (lfs_tag_id(tag) == lfs_tag_id(fromtag - fromdiff)) { - break; - } else if (lfs_tag_id(tag) < lfs_tag_id(fromtag - fromdiff)) { - fromdiff += LFS_MKTAG(0, 1, 0); + err = lfs_commit_prog(lfs, commit, &dat, 1); + if (err) { + return err; } } } + commit->ptag = tag & 0x7fffffff; return 0; } @@ -737,7 +1014,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, lfs->cfg->prog_size); // read erased state from next program unit - uint32_t tag; + lfs_tag_t tag; int err = lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, sizeof(tag), commit->block, off, &tag, sizeof(tag)); @@ -748,7 +1025,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, // build crc tag bool reset = ~lfs_fromle32(tag) >> 31; tag = LFS_MKTAG(LFS_TYPE_CRC + (compacting << 1) + reset, - 0x3ff, off - (commit->off+sizeof(uint32_t))); + 0x3ff, off - (commit->off+sizeof(lfs_tag_t))); // write out crc uint32_t footer[2]; @@ -797,7 +1074,6 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, return 0; } -// internal dir operations static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { // allocate pair of dir blocks (backwards, so we write block 1 first) for (int i = 0; i < 2; i++) { @@ -835,300 +1111,8 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { return 0; } -static int32_t lfs_dir_fetchmatch(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2], - uint32_t findmask, uint32_t findtag, const void *findbuffer) { - dir->pair[0] = pair[0]; - dir->pair[1] = pair[1]; - int32_t foundtag = LFS_ERR_NOENT; - - // find the block with the most recent revision - uint32_t rev[2]; - for (int i = 0; i < 2; i++) { - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(rev[i]), - dir->pair[i], 0, &rev[i], sizeof(rev[i])); - rev[i] = lfs_fromle32(rev[i]); - if (err && err != LFS_ERR_CORRUPT) { - return err; - } - - if (err == LFS_ERR_CORRUPT) { - rev[i] = rev[(i+1)%2] - 1; - } - } - - if (lfs_scmp(rev[1], rev[0]) > 0) { - lfs_pair_swap(dir->pair); - lfs_pair_swap(rev); - } - - // load blocks and check crc - for (int i = 0; i < 2; i++) { - lfs_off_t off = sizeof(dir->rev); - uint32_t ptag = 0xffffffff; - uint32_t crc = 0xffffffff; - - dir->rev = lfs_tole32(rev[0]); - crc = lfs_crc(crc, &dir->rev, sizeof(dir->rev)); - dir->rev = lfs_fromle32(dir->rev); - dir->off = 0; - - uint32_t tempfoundtag = foundtag; - uint16_t tempcount = 0; - lfs_block_t temptail[2] = {0xffffffff, 0xffffffff}; - bool tempsplit = false; - lfs_global_t templocals; - lfs_global_zero(&templocals); - - while (true) { - // extract next tag - uint32_t tag; - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - dir->pair[0], off, &tag, sizeof(tag)); - if (err) { - if (err == LFS_ERR_CORRUPT) { - // can't continue? - dir->erased = false; - break; - } - return err; - } - - crc = lfs_crc(crc, &tag, sizeof(tag)); - tag = lfs_fromle32(tag) ^ ptag; - - // next commit not yet programmed - if (!lfs_tag_isvalid(tag)) { - dir->erased = (lfs_tag_subtype(ptag) == LFS_TYPE_CRC); - break; - } - - // check we're in valid range - if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { - dir->erased = false; - break; - } - - if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { - // check the crc attr - uint32_t dcrc; - err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - return err; - } - dcrc = lfs_fromle32(dcrc); - - if (crc != dcrc) { - dir->erased = false; - break; - } - - // reset the next bit if we need to - tag ^= (lfs_tag_type(tag) & 1) << 31; - - // update with what's found so far - foundtag = tempfoundtag; - dir->off = off + lfs_tag_dsize(tag); - dir->etag = tag; - dir->count = tempcount; - dir->tail[0] = temptail[0]; - dir->tail[1] = temptail[1]; - dir->split = tempsplit; - dir->locals = templocals; - - lfs->seed ^= crc; - crc = 0xffffffff; - } else { - // crc the entry first, leaving it in the cache - for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) { - uint8_t dat; - err = lfs_bd_read(lfs, - NULL, &lfs->rcache, lfs->cfg->block_size, - dir->pair[0], off+j, &dat, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - return err; - } - - crc = lfs_crc(crc, &dat, 1); - } - - // check for special tags - if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { - tempcount += 1; - - if (lfs_tag_isvalid(tempfoundtag) && - lfs_tag_id(tag) <= lfs_tag_id(tempfoundtag)) { - tempfoundtag += LFS_MKTAG(0, 1, 0); - } - } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { - LFS_ASSERT(tempcount > 0); - tempcount -= 1; - - if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { - tempfoundtag = LFS_ERR_NOENT; - } else if (lfs_tag_isvalid(tempfoundtag) && - lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { - tempfoundtag -= LFS_MKTAG(0, 1, 0); - } - } else if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { - tempsplit = (lfs_tag_type(tag) & 1); - err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - dir->pair[0], off+sizeof(tag), - &temptail, sizeof(temptail)); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - } - lfs_pair_fromle32(temptail); - } else if (lfs_tag_subtype(tag) == LFS_TYPE_GLOBALS) { - templocals.l.deorphaned = (lfs_tag_type(tag) & 1); - err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - dir->pair[0], off+sizeof(tag), - &templocals, 10); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - } - } - - if ((tag & findmask) == (findtag & findmask)) { - // found a match? - if (lfs_tag_isdelete(findtag)) { - tempfoundtag = LFS_ERR_NOENT; - } else if (lfs_tag_type(findtag) == LFS_TYPE_DIRSTRUCT) { - lfs_block_t child[2]; - err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, - lfs->cfg->block_size, - dir->pair[0], off+sizeof(tag), - &child, sizeof(child)); - if (err < 0) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - return err; - } - - lfs_pair_fromle32(child); - if (lfs_pair_cmp(child, - (const lfs_block_t *)findbuffer) == 0) { - tempfoundtag = tag; - } - } else if (lfs_tag_type(findtag) == LFS_TYPE_NAME) { - int res = lfs_bd_cmp(lfs, - NULL, &lfs->rcache, lfs_tag_size(findtag), - dir->pair[0], off+sizeof(tag), - findbuffer, lfs_tag_size(findtag)); - if (res < 0) { - if (res == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - return res; - } - - if (res) { - tempfoundtag = tag; - } - } else { - tempfoundtag = tag; - } - } - } - - ptag = tag; - off += lfs_tag_dsize(tag); - } - - // consider what we have good enough - if (dir->off > 0) { - // synthetic move - if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0) { - if (lfs->globals.g.moveid == lfs_tag_id(foundtag)) { - foundtag = LFS_ERR_NOENT; - } else if (lfs_tag_isvalid(foundtag) && - lfs->globals.g.moveid < lfs_tag_id(foundtag)) { - foundtag -= LFS_MKTAG(0, 1, 0); - } - } - - return foundtag; - } - - // failed, try the other crc? - lfs_pair_swap(dir->pair); - lfs_pair_swap(rev); - } - - LFS_ERROR("Corrupted dir pair at %"PRIu32" %"PRIu32, - dir->pair[0], dir->pair[1]); - return LFS_ERR_CORRUPT; -} - -static int lfs_dir_fetch(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2]) { - int32_t res = lfs_dir_fetchmatch(lfs, dir, pair, - 0xffffffff, 0xffffffff, NULL); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - - return 0; -} - -static int32_t lfs_dir_find(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2], bool fs, - uint32_t findmask, uint32_t findtag, const void *findbuffer) { - dir->split = true; - dir->tail[0] = pair[0]; - dir->tail[1] = pair[1]; - while ((dir->split || fs) && !lfs_pair_isnull(dir->tail)) { - int32_t tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, - findmask, findtag, findbuffer); - if (tag != LFS_ERR_NOENT) { - return tag; - } - } - - return LFS_ERR_NOENT; -} - -static int32_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, - uint32_t getmask, uint32_t gettag, void *buffer) { - int32_t getdiff = 0; - if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0 && - lfs_tag_id(gettag) <= lfs->globals.g.moveid) { - // synthetic moves - gettag += LFS_MKTAG(0, 1, 0); - getdiff -= LFS_MKTAG(0, 1, 0); - } - - return lfs_commit_get(lfs, - dir->pair[0], dir->off, dir->etag, false, - getmask, gettag, getdiff, buffer); -} - static int lfs_dir_compact(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_mattr_t *attrs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; @@ -1375,8 +1359,8 @@ static int lfs_dir_compact(lfs_t *lfs, } static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, - const lfs_mattr_t *attrs) { - lfs_mattr_t cancelattr; + const struct lfs_mattr *attrs) { + struct lfs_mattr cancelattr; lfs_global_t canceldiff; lfs_global_zero(&canceldiff); if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0) { @@ -1395,9 +1379,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } // calculate new directory size - uint32_t deletetag = 0xffffffff; + lfs_tag_t deletetag = 0xffffffff; int attrcount = 0; - for (const lfs_mattr_t *a = attrs; a; a = a->next) { + for (const struct lfs_mattr *a = attrs; a; a = a->next) { if (lfs_tag_subtype(a->tag) == LFS_TYPE_NAME) { dir->count += 1; } else if (lfs_tag_subtype(a->tag) == LFS_TYPE_DELETE) { @@ -1449,7 +1433,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // iterate over commits backwards, this lets us "append" commits for (int i = 0; i < attrcount; i++) { - const lfs_mattr_t *a = attrs; + const struct lfs_mattr *a = attrs; for (int j = 0; j < attrcount-i-1; j++) { a = a->next; } @@ -1508,7 +1492,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // two passes, once for things that aren't us, and one // for things that are - for (lfs_mlist_t *d = lfs->mlist; d; d = d->next) { + for (struct lfs_mlist *d = lfs->mlist; d; d = d->next) { if (lfs_pair_cmp(d->m.pair, copy.pair) == 0) { d->m = *dir; if (d->id == lfs_tag_id(deletetag)) { @@ -1535,13 +1519,32 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, return 0; } -static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { +struct lfs_dir_find_match { + lfs_t *lfs; + const void *name; + lfs_size_t size; +}; + +static int lfs_dir_find_match(void *data, + lfs_tag_t tag, const void *buffer) { + struct lfs_dir_find_match *name = data; + lfs_t *lfs = name->lfs; + const struct lfs_diskoff *disk = buffer; + (void)tag; + + return lfs_bd_cmp(lfs, + NULL, &lfs->rcache, name->size, + disk->block, disk->off, name->name, name->size); +} + +static lfs_stag_t lfs_dir_find(lfs_t *lfs, + lfs_mdir_t *dir, const char **path) { // we reduce path to a single name if we can find it const char *name = *path; *path = NULL; // default to root dir - int32_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); + lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; while (true) { @@ -1598,7 +1601,7 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { // grab the entry data if (lfs_tag_id(tag) != 0x3ff) { - int32_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -1607,8 +1610,10 @@ static int32_t lfs_dir_lookup(lfs_t *lfs, lfs_mdir_t *dir, const char **path) { } // find entry matching name - tag = lfs_dir_find(lfs, dir, pair, false, 0x7c000fff, - LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), name); + tag = lfs_dir_findmatch(lfs, dir, pair, false, 0x7c000fff, + LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, name, namelen}); if (tag < 0) { return tag; } @@ -1627,7 +1632,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, return 0; } - int32_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + lfs_stag_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name); if (tag < 0) { return tag; @@ -1661,7 +1666,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } lfs_mdir_t cwd; - int32_t res = lfs_dir_lookup(lfs, &cwd, &path); + lfs_stag_t res = lfs_dir_find(lfs, &cwd, &path); if (!(res == LFS_ERR_NOENT && path)) { return (res < 0) ? res : LFS_ERR_EXIST; } @@ -1707,7 +1712,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { - int32_t tag = lfs_dir_lookup(lfs, &dir->m, &path); + lfs_stag_t tag = lfs_dir_find(lfs, &dir->m, &path); if (tag < 0) { return tag; } @@ -1723,7 +1728,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { pair[1] = lfs->root[1]; } else { // get dir pair from parent - int32_t res = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -1746,15 +1751,15 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { // add to list of mdirs dir->type = LFS_TYPE_DIR; dir->next = (lfs_dir_t*)lfs->mlist; - lfs->mlist = (lfs_mlist_t*)dir; + lfs->mlist = (struct lfs_mlist*)dir; return 0; } int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir) { // remove from list of mdirs - for (lfs_mlist_t **p = &lfs->mlist; *p; p = &(*p)->next) { - if (*p == (lfs_mlist_t*)dir) { + for (struct lfs_mlist **p = &lfs->mlist; *p; p = &(*p)->next) { + if (*p == (struct lfs_mlist*)dir) { *p = (*p)->next; break; } @@ -2071,7 +2076,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, file->cache.buffer = NULL; // allocate entry for file if it doesn't exist - int32_t tag = lfs_dir_lookup(lfs, &file->m, &path); + lfs_stag_t tag = lfs_dir_find(lfs, &file->m, &path); if (tag < 0 && !(tag == LFS_ERR_NOENT && path)) { err = tag; goto cleanup; @@ -2081,7 +2086,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, file->id = lfs_tag_id(tag); file->type = LFS_TYPE_REG; file->next = (lfs_file_t*)lfs->mlist; - lfs->mlist = (lfs_mlist_t*)file; + lfs->mlist = (struct lfs_mlist*)file; if (tag == LFS_ERR_NOENT) { if (!(flags & LFS_O_CREAT)) { @@ -2132,7 +2137,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // fetch attrs for (const struct lfs_attr *a = file->cfg->attrs; a; a = a->next) { if ((file->flags & 3) != LFS_O_WRONLY) { - int32_t res = lfs_dir_get(lfs, &file->m, 0x7ffff000, + lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x7ffff000, LFS_MKTAG(0x100 | a->type, file->id, a->size), a->buffer); if (res < 0 && res != LFS_ERR_NOENT) { err = res; @@ -2175,7 +2180,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // don't always read (may be new/trunc file) if (file->ctz.size > 0) { - int32_t res = lfs_dir_get(lfs, &file->m, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->ctz.size), file->cache.buffer); if (res < 0) { @@ -2204,8 +2209,8 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { int err = lfs_file_sync(lfs, file); // remove from list of mdirs - for (lfs_mlist_t **p = &lfs->mlist; *p; p = &(*p)->next) { - if (*p == (lfs_mlist_t*)file) { + for (struct lfs_mlist **p = &lfs->mlist; *p; p = &(*p)->next) { + if (*p == (struct lfs_mlist*)file) { *p = (*p)->next; break; } @@ -2383,7 +2388,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // commit file data and attributes err = lfs_dir_commit(lfs, &file->m, - LFS_MKATTR(LFS_FROM_ATTRS, file->id, file->cfg->attrs, 0, + LFS_MKATTR(LFS_FROM_USERATTRS, + file->id, file->cfg->attrs, 0, LFS_MKATTR(type, file->id, buffer, size, NULL))); if (err) { @@ -2704,7 +2710,7 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { lfs_mdir_t cwd; - int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path); if (tag < 0) { return tag; } @@ -2725,7 +2731,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } - int32_t tag = lfs_dir_lookup(lfs, &cwd, &path); + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path); if (tag < 0) { return tag; } @@ -2734,7 +2740,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (lfs_tag_type(tag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t pair[2]; - int32_t res = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, &cwd, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -2796,14 +2802,14 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find old entry lfs_mdir_t oldcwd; - int32_t oldtag = lfs_dir_lookup(lfs, &oldcwd, &oldpath); + lfs_stag_t oldtag = lfs_dir_find(lfs, &oldcwd, &oldpath); if (oldtag < 0) { return oldtag; } // find new entry lfs_mdir_t newcwd; - int32_t prevtag = lfs_dir_lookup(lfs, &newcwd, &newpath); + lfs_stag_t prevtag = lfs_dir_find(lfs, &newcwd, &newpath); if (prevtag < 0 && prevtag != LFS_ERR_NOENT) { return prevtag; } @@ -2825,7 +2831,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } else if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t prevpair[2]; - int32_t res = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair); if (res < 0) { return res; @@ -2897,7 +2903,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, uint8_t type, void *buffer, lfs_size_t size) { lfs_mdir_t cwd; - int32_t res = lfs_dir_lookup(lfs, &cwd, &path); + lfs_stag_t res = lfs_dir_find(lfs, &cwd, &path); if (res < 0) { return res; } @@ -2928,7 +2934,7 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, static int lfs_commitattr(lfs_t *lfs, const char *path, uint8_t type, const void *buffer, lfs_size_t size) { lfs_mdir_t cwd; - int32_t res = lfs_dir_lookup(lfs, &cwd, &path); + lfs_stag_t res = lfs_dir_find(lfs, &cwd, &path); if (res < 0) { return res; } @@ -2958,7 +2964,7 @@ int lfs_setattr(lfs_t *lfs, const char *path, } int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { - return lfs_commitattr(lfs, path, type, NULL, 0xfff); + return lfs_commitattr(lfs, path, type, NULL, LFS_ATTR_MAX+1); } @@ -3131,15 +3137,17 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // find root/superblock lfs_mdir_t root; lfs_superblock_t superblock; - int32_t tag = lfs_dir_find(lfs, + lfs_stag_t tag = lfs_dir_findmatch(lfs, &root, (const lfs_block_t[2]){0, 1}, false, 0x7fc00000, - LFS_MKTAG(LFS_TYPE_ROOT, 0, 8), "littlefs"); + LFS_MKTAG(LFS_TYPE_ROOT, 0, 8), + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, "littlefs", 8}); if (tag < 0) { err = tag; goto cleanup; } - int32_t res = lfs_dir_get(lfs, &root, 0x7f800000, + lfs_stag_t res = lfs_dir_get(lfs, &root, 0x7f800000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), &superblock); if (res < 0) { @@ -3263,7 +3271,7 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { struct lfs_ctz ctz; - int32_t tag = lfs_dir_get(lfs, &dir, 0x7c3ff000, + lfs_stag_t tag = lfs_dir_get(lfs, &dir, 0x7c3ff000, LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); if (tag < 0) { if (tag == LFS_ERR_NOENT) { @@ -3332,7 +3340,31 @@ static int lfs_fs_pred(lfs_t *lfs, return LFS_ERR_NOENT; } -static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], +struct lfs_fs_parent_match { + lfs_t *lfs; + const lfs_block_t pair[2]; +}; + +static int lfs_fs_parent_match(void *data, + lfs_tag_t tag, const void *buffer) { + struct lfs_fs_parent_match *find = data; + lfs_t *lfs = find->lfs; + const struct lfs_diskoff *disk = buffer; + (void)tag; + + lfs_block_t child[2]; + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + disk->block, disk->off, &child, sizeof(child)); + if (err) { + return err; + } + + lfs_pair_fromle32(child); + return (lfs_pair_cmp(child, find->pair) == 0); +} + +static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *parent) { if (lfs_pair_isnull(lfs->root)) { return LFS_ERR_NOENT; @@ -3340,9 +3372,11 @@ static int32_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], // search for both orderings so we can reuse the find function for (int i = 0; i < 2; i++) { - int32_t tag = lfs_dir_find(lfs, parent, + struct lfs_fs_parent_match match = {lfs, {pair[0], pair[1]}}; + lfs_stag_t tag = lfs_dir_findmatch(lfs, parent, (const lfs_block_t[2]){0, 1}, true, 0x7fc00fff, - LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), pair); + LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), + lfs_fs_parent_match, &match); if (tag != LFS_ERR_NOENT) { return tag; } @@ -3362,7 +3396,7 @@ static int lfs_fs_relocate(lfs_t *lfs, } // update internally tracked dirs - for (lfs_mlist_t *d = lfs->mlist; d; d = d->next) { + for (struct lfs_mlist *d = lfs->mlist; d; d = d->next) { if (lfs_pair_cmp(oldpair, d->m.pair) == 0) { d->m.pair[0] = newpair[0]; d->m.pair[1] = newpair[1]; @@ -3371,7 +3405,7 @@ static int lfs_fs_relocate(lfs_t *lfs, // find parent lfs_mdir_t parent; - int32_t tag = lfs_fs_parent(lfs, oldpair, &parent); + lfs_stag_t tag = lfs_fs_parent(lfs, oldpair, &parent); if (tag < 0 && tag != LFS_ERR_NOENT) { return tag; } @@ -3382,7 +3416,7 @@ static int lfs_fs_relocate(lfs_t *lfs, lfs_pair_tole32(newpair); int err = lfs_dir_commit(lfs, &parent, - &(lfs_mattr_t){.tag=tag, .buffer=newpair}); + &(struct lfs_mattr){.tag=tag, .buffer=newpair}); lfs_pair_fromle32(newpair); if (err) { return err; @@ -3431,7 +3465,7 @@ static int lfs_fs_deorphan(lfs_t *lfs) { if (!pdir.split) { // check if we have a parent lfs_mdir_t parent; - int32_t tag = lfs_fs_parent(lfs, pdir.tail, &parent); + lfs_stag_t tag = lfs_fs_parent(lfs, pdir.tail, &parent); if (tag < 0 && tag != LFS_ERR_NOENT) { return tag; } @@ -3455,7 +3489,7 @@ static int lfs_fs_deorphan(lfs_t *lfs) { } lfs_block_t pair[2]; - int32_t res = lfs_dir_get(lfs, &parent, 0x7ffff000, tag, pair); + lfs_stag_t res = lfs_dir_get(lfs, &parent, 0x7ffff000, tag, pair); if (res < 0) { return res; } diff --git a/lfs.h b/lfs.h index 120a9a38..f8867b3f 100644 --- a/lfs.h +++ b/lfs.h @@ -113,7 +113,7 @@ enum lfs_type { LFS_FROM_REGION = 0x000, LFS_FROM_DISK = 0x200, LFS_FROM_MOVE = 0x0c1, - LFS_FROM_ATTRS = 0x0c2, + LFS_FROM_USERATTRS = 0x0c2, LFS_FROM_SUPERBLOCK = 0x0c3, }; @@ -284,13 +284,7 @@ struct lfs_file_config { }; -/// littlefs data structures /// -typedef struct lfs_mattr { - int32_t tag; - const void *buffer; - const struct lfs_mattr *next; -} lfs_mattr_t; - +/// internal littlefs data structures /// typedef struct lfs_cache { lfs_block_t block; lfs_off_t off; @@ -324,13 +318,7 @@ typedef struct lfs_mdir { lfs_global_t locals; } lfs_mdir_t; -typedef struct lfs_mlist { - struct lfs_mlist *next; - uint16_t id; - uint8_t type; - lfs_mdir_t m; -} lfs_mlist_t; - +// littlefs directory type typedef struct lfs_dir { struct lfs_dir *next; uint16_t id; @@ -341,6 +329,7 @@ typedef struct lfs_dir { lfs_block_t head[2]; } lfs_dir_t; +// littlefs file type typedef struct lfs_file { struct lfs_file *next; uint16_t id; @@ -373,26 +362,29 @@ typedef struct lfs_superblock { lfs_size_t inline_max; } lfs_superblock_t; -typedef struct lfs_free { - lfs_block_t off; - lfs_block_t size; - lfs_block_t i; - lfs_block_t ack; - uint32_t *buffer; -} lfs_free_t; - -// The littlefs type +// The littlefs filesystem type typedef struct lfs { lfs_cache_t rcache; lfs_cache_t pcache; lfs_block_t root[2]; - lfs_mlist_t *mlist; + struct lfs_mlist { + struct lfs_mlist *next; + uint16_t id; + uint8_t type; + lfs_mdir_t m; + } *mlist; uint32_t seed; lfs_global_t globals; lfs_global_t locals; - lfs_free_t free; + struct lfs_free { + lfs_block_t off; + lfs_block_t size; + lfs_block_t i; + lfs_block_t ack; + uint32_t *buffer; + } free; const struct lfs_config *cfg; lfs_size_t block_size; From 7bacf9b1e01b9bf56fec9fc481e2078c4424c2d7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 12 Sep 2018 01:34:03 -0500 Subject: [PATCH 104/139] Removed xored-globals from the mdir struct The xored-globals have a very large footprint. In the worst case, the xored-globals are stored on each metadata-pair, twice in memory. They must be very small, but are also very useful, so at risk of growing in the future (hint global free-list?). Initially we also stored a copy in each mdir structure, since this avoided extra disk access to look up the globals when we need to modify the global state on a metadata-pair. But we can easily just fetch the globals when needed. This is more costly in terms of runtime, but reduces RAM impact of globals, which was previously needed for each open dir and file. --- lfs.c | 204 +++++++++++++++++++++++++++++++--------------------------- lfs.h | 1 - 2 files changed, 108 insertions(+), 97 deletions(-) diff --git a/lfs.c b/lfs.c index 869205ff..45c52fd0 100644 --- a/lfs.c +++ b/lfs.c @@ -394,6 +394,8 @@ static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { /// Internal operations predeclared here /// +static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, + const struct lfs_mattr *attrs); static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], @@ -537,7 +539,7 @@ static int lfs_dir_traverse(lfs_t *lfs, } static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2], + lfs_mdir_t *dir, const lfs_block_t pair[2], lfs_tag_t matchmask, lfs_tag_t matchtag, int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { // find the block with the most recent revision @@ -558,7 +560,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } // now fetch the actual dir (and find match) - lfs_stag_t foundtag = LFS_ERR_NOENT; + lfs_stag_t foundtag = 0; dir->pair[0] = pair[0]; dir->pair[1] = pair[1]; dir->off = 0; @@ -581,7 +583,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, .split = false, .count = 0, }; - lfs_global_zero(&temp.locals); temp.rev = lfs_tole32(temp.rev); uint32_t crc = lfs_crc(0xffffffff, &temp.rev, sizeof(temp.rev)); @@ -669,7 +670,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { temp.count += 1; - if (lfs_tag_isvalid(tempfoundtag) && + if (tempfoundtag && lfs_tag_id(tag) <= lfs_tag_id(tempfoundtag)) { tempfoundtag += LFS_MKTAG(0, 1, 0); } @@ -678,8 +679,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, temp.count -= 1; if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { - tempfoundtag = LFS_ERR_NOENT; - } else if (lfs_tag_isvalid(tempfoundtag) && + tempfoundtag = 0; + } else if (tempfoundtag && lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { tempfoundtag -= LFS_MKTAG(0, 1, 0); } @@ -696,24 +697,12 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } } lfs_pair_fromle32(temp.tail); - } else if (lfs_tag_subtype(tag) == LFS_TYPE_GLOBALS) { - temp.locals.l.deorphaned = (lfs_tag_type(tag) & 1); - err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - block, off+sizeof(tag), - &temp.locals, 10); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - } } if ((tag & matchmask) == (matchtag & matchmask)) { // found a match? if (lfs_tag_isdelete(tag)) { - tempfoundtag = LFS_ERR_NOENT; + tempfoundtag = 0; } else if (cb) { int res = cb(data, tag, &(struct lfs_diskoff){ block, off+sizeof(tag)}); @@ -728,6 +717,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, if (res) { tempfoundtag = tag; } + } else { + tempfoundtag = tag; } } } @@ -741,8 +732,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // synthetic move if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0) { if (lfs->globals.g.moveid == lfs_tag_id(foundtag)) { - foundtag = LFS_ERR_NOENT; - } else if (lfs_tag_isvalid(foundtag) && + foundtag = 0; + } else if (foundtag && lfs->globals.g.moveid < lfs_tag_id(foundtag)) { foundtag -= LFS_MKTAG(0, 1, 0); } @@ -763,12 +754,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, static int lfs_dir_fetch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2]) { - lfs_stag_t res = lfs_dir_fetchmatch(lfs, dir, pair, 0, 0, NULL, NULL); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - - return 0; + return lfs_dir_fetchmatch(lfs, dir, pair, + 0xffffffff, 0x00000000, NULL, NULL); } static lfs_stag_t lfs_dir_findmatch(lfs_t *lfs, @@ -781,7 +768,7 @@ static lfs_stag_t lfs_dir_findmatch(lfs_t *lfs, while ((dir->split || fs) && !lfs_pair_isnull(dir->tail)) { lfs_stag_t tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, findmask, findtag, cb, data); - if (tag != LFS_ERR_NOENT) { + if (tag) { return tag; } } @@ -821,7 +808,7 @@ static int lfs_dir_get_match(void *data, return tag & 0x7fffffff; } -static lfs_stag_t lfs_dir_get(lfs_t *lfs, lfs_mdir_t *dir, +static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, lfs_tag_t getmask, lfs_tag_t gettag, void *buffer) { lfs_stag_t getdiff = 0; if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0 && @@ -993,20 +980,6 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, - lfs_global_t *locals) { - if (lfs_global_iszero(&lfs->locals)) { - return 0; - } - - lfs_global_xor(locals, &lfs->locals); - int err = lfs_commit_attr(lfs, commit, - LFS_MKTAG(LFS_TYPE_GLOBALS + locals->l.deorphaned, 0x3ff, 10), - locals); - lfs_global_xor(locals, &lfs->locals); - return err; -} - static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, bool compacting) { // align to program units @@ -1105,12 +1078,36 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { dir->tail[1] = 0xffffffff; dir->erased = false; dir->split = false; - lfs_global_zero(&dir->locals); // don't write out yet, let caller take care of that return 0; } +static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mdir_t *tail) { + // steal tail + dir->tail[0] = tail->tail[0]; + dir->tail[1] = tail->tail[1]; + dir->split = tail->split; + + // steal state + lfs_global_t locals; + lfs_stag_t res = lfs_dir_get(lfs, tail, 0x7c000000, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + + if (res != LFS_ERR_NOENT) { + locals.l.deorphaned = (lfs_tag_type(res) & 1); + lfs_global_xor(&lfs->locals, &locals); + } + + return lfs_dir_commit(lfs, dir, + LFS_MKATTR(LFS_TYPE_TAIL + dir->split, + 0x3ff, dir->tail, sizeof(dir->tail), + NULL)); +} + static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { @@ -1120,9 +1117,19 @@ static int lfs_dir_compact(lfs_t *lfs, // There's nothing special about our global delta, so feed it back // into the global global delta - lfs_global_xor(&lfs->locals, &dir->locals); - lfs_global_zero(&dir->locals); + lfs_global_t locals; + lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c000000, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + if (res != LFS_ERR_NOENT) { + locals.l.deorphaned = (lfs_tag_type(res) & 1); + lfs_global_xor(&lfs->locals, &locals); + } + + // begin loop to commit compaction to blocks until a compact sticks while (true) { // setup compaction bool splitted = false; @@ -1160,7 +1167,7 @@ static int lfs_dir_compact(lfs_t *lfs, (const lfs_block_t[2]){0, 1}) == 0) { // we're writing too much to the superblock, // should we expand? - lfs_ssize_t res = lfs_fs_size(lfs); + res = lfs_fs_size(lfs); if (res < 0) { return res; } @@ -1235,10 +1242,12 @@ static int lfs_dir_compact(lfs_t *lfs, } } - if (!relocated) { + if (!relocated && !lfs_global_iszero(&lfs->locals)) { // commit any globals, unless we're relocating, // in which case our parent will steal our globals - err = lfs_commit_globals(lfs, &commit, &dir->locals); + err = lfs_commit_attr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_GLOBALS + lfs->locals.l.deorphaned, + 0x3ff, 10), &lfs->locals); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1343,7 +1352,6 @@ static int lfs_dir_compact(lfs_t *lfs, if (!relocated) { // successful commit, update globals - lfs_global_xor(&dir->locals, &lfs->locals); lfs_global_zero(&lfs->locals); } else { // update references if we relocated @@ -1398,15 +1406,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } if (err != LFS_ERR_NOENT && pdir.split) { - // steal tail and global state - pdir.split = dir->split; - pdir.tail[0] = dir->tail[0]; - pdir.tail[1] = dir->tail[1]; - lfs_global_xor(&lfs->locals, &dir->locals); - return lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_TAIL + pdir.split, 0x3ff, - pdir.tail, sizeof(pdir.tail), - NULL)); + return lfs_dir_drop(lfs, &pdir, dir); } } } @@ -1449,15 +1449,34 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } } - int err = lfs_commit_globals(lfs, &commit, &dir->locals); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - goto compact; + // commit any global diffs if we have any + if (!lfs_global_iszero(&lfs->locals)) { + lfs_global_t locals; + lfs_global_zero(&locals); + lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c000000, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + + if (res != LFS_ERR_NOENT) { + locals.l.deorphaned = (lfs_tag_type(res) & 1); + } + + lfs_global_xor(&locals, &lfs->locals); + int err = lfs_commit_attr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_GLOBALS + locals.l.deorphaned, + 0x3ff, 10), &locals); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; + } + return err; } - return err; } - err = lfs_commit_crc(lfs, &commit, false); + // finalize commit with the crc + int err = lfs_commit_crc(lfs, &commit, false); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1469,13 +1488,13 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, dir->off = commit.off; dir->etag = commit.ptag; // successful commit, update globals - lfs_global_xor(&dir->locals, &lfs->locals); lfs_global_zero(&lfs->locals); break; compact: // fall back to compaction lfs_cache_drop(lfs, &lfs->pcache); + err = lfs_dir_compact(lfs, dir, attrs, dir, 0, dir->count); if (err) { return err; @@ -2777,14 +2796,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } - // steal state - cwd.tail[0] = dir.tail[0]; - cwd.tail[1] = dir.tail[1]; - lfs_global_xor(&lfs->locals, &dir.locals); - err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - cwd.tail, sizeof(cwd.tail), - NULL)); + err = lfs_dir_drop(lfs, &cwd, &dir); if (err) { return err; } @@ -2884,14 +2896,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - // steal state - newcwd.tail[0] = prevdir.tail[0]; - newcwd.tail[1] = prevdir.tail[1]; - lfs_global_xor(&lfs->locals, &prevdir.locals); - err = lfs_dir_commit(lfs, &newcwd, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - newcwd.tail, sizeof(newcwd.tail), - NULL)); + err = lfs_dir_drop(lfs, &newcwd, &prevdir); if (err) { return err; } @@ -3207,14 +3212,26 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // scan for any global updates lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pair_isnull(dir.tail)) { - err = lfs_dir_fetch(lfs, &dir, dir.tail); - if (err) { + res = lfs_dir_fetchmatch(lfs, &dir, dir.tail, 0x7c000000, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), NULL, NULL); + if (res < 0) { err = LFS_ERR_INVAL; goto cleanup; } - // xor together indirect deletes - lfs_global_xor(&lfs->locals, &dir.locals); + if (res) { + lfs_global_t locals; + res = lfs_dir_get(lfs, &dir, 0x7c000000, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); + if (res < 0) { + err = res; + goto cleanup; + } + locals.l.deorphaned = (lfs_tag_type(res) & 1); + + // xor together indirect deletes + lfs_global_xor(&lfs->locals, &locals); + } } // update littlefs with globals @@ -3438,8 +3455,8 @@ static int lfs_fs_relocate(lfs_t *lfs, parent.tail[0] = newpair[0]; parent.tail[1] = newpair[1]; err = lfs_dir_commit(lfs, &parent, - LFS_MKATTR(LFS_TYPE_TAIL + parent.split, 0x3ff, - parent.tail, sizeof(parent.tail), + LFS_MKATTR(LFS_TYPE_TAIL + parent.split, + 0x3ff, parent.tail, sizeof(parent.tail), NULL)); if (err) { return err; @@ -3475,12 +3492,7 @@ static int lfs_fs_deorphan(lfs_t *lfs) { LFS_DEBUG("Fixing orphan %"PRIu32" %"PRIu32, pdir.tail[0], pdir.tail[1]); - pdir.tail[0] = dir.tail[0]; - pdir.tail[1] = dir.tail[1]; - err = lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pdir.tail, sizeof(pdir.tail), - NULL)); + err = lfs_dir_drop(lfs, &pdir, &dir); if (err) { return err; } @@ -3503,8 +3515,8 @@ static int lfs_fs_deorphan(lfs_t *lfs) { pdir.tail[0] = pair[0]; pdir.tail[1] = pair[1]; err = lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pdir.tail, sizeof(pdir.tail), + LFS_MKATTR(LFS_TYPE_SOFTTAIL, + 0x3ff, pdir.tail, sizeof(pdir.tail), NULL)); if (err) { return err; diff --git a/lfs.h b/lfs.h index f8867b3f..090041c7 100644 --- a/lfs.h +++ b/lfs.h @@ -315,7 +315,6 @@ typedef struct lfs_mdir { bool erased; bool split; lfs_block_t tail[2]; - lfs_global_t locals; } lfs_mdir_t; // littlefs directory type From cf87ba537598d13b0884751bee8d4ad71e782450 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 12 Sep 2018 01:50:21 -0500 Subject: [PATCH 105/139] Combined superblock scan and fetch of xored-globals during mount Conceptually these are two separate operations. However, they are both only needed during mount, both require iteration over the linked-list of metadata-pairs, and both are independent from each other. Combining these into one gives us a nice code savings. Additionally, this greatly simplifies the lookup of the root directory. Initially we used a flag to indicate which superblock was root, since we didn't want to fetch more pairs than we needed to. But since we're going to fetch all metadata-pairs anyways, we can just use the last superblock we find as the indicator of our root directory. --- lfs.c | 153 ++++++++++++++++++++++++------------------------- lfs.h | 8 +-- tests/debug.py | 1 - 3 files changed, 78 insertions(+), 84 deletions(-) diff --git a/lfs.c b/lfs.c index 45c52fd0..592f9805 100644 --- a/lfs.c +++ b/lfs.c @@ -3116,7 +3116,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_tole32(&superblock); err = lfs_dir_commit(lfs, &root, - LFS_MKATTR(LFS_TYPE_ROOT, 0, &superblock, sizeof(superblock), + LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), NULL)); if (err) { goto cleanup; @@ -3139,97 +3139,94 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - // find root/superblock - lfs_mdir_t root; - lfs_superblock_t superblock; - lfs_stag_t tag = lfs_dir_findmatch(lfs, - &root, (const lfs_block_t[2]){0, 1}, false, 0x7fc00000, - LFS_MKTAG(LFS_TYPE_ROOT, 0, 8), - lfs_dir_find_match, &(struct lfs_dir_find_match){ - lfs, "littlefs", 8}); - if (tag < 0) { - err = tag; - goto cleanup; - } + // scan directory blocks for superblock and any global updates + lfs_mdir_t dir = {.tail = {0, 1}}; + while (!lfs_pair_isnull(dir.tail)) { + // fetch next block in tail list + lfs_stag_t res = lfs_dir_fetchmatch(lfs, &dir, dir.tail, 0x7fc00000, + LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, "littlefs", 8}); + if (res < 0) { + err = res; + goto cleanup; + } - lfs_stag_t res = lfs_dir_get(lfs, &root, 0x7f800000, - LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), - &superblock); - if (res < 0) { - err = res; - goto cleanup; - } - lfs_superblock_fromle32(&superblock); + // has superblock? + if (res) { + // update root + lfs->root[0] = dir.pair[0]; + lfs->root[1] = dir.pair[1]; + + // grab superblock + lfs_superblock_t superblock; + res = lfs_dir_get(lfs, &dir, 0x7f800000, + LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), + &superblock); + if (res < 0) { + err = res; + goto cleanup; + } + lfs_superblock_fromle32(&superblock); - lfs->root[0] = root.pair[0]; - lfs->root[1] = root.pair[1]; + // check version + uint16_t major_version = (0xffff & (superblock.version >> 16)); + uint16_t minor_version = (0xffff & (superblock.version >> 0)); + if ((major_version != LFS_DISK_VERSION_MAJOR || + minor_version > LFS_DISK_VERSION_MINOR)) { + LFS_ERROR("Invalid version %"PRIu32".%"PRIu32, + major_version, minor_version); + err = LFS_ERR_INVAL; + goto cleanup; + } - // check version - uint16_t major_version = (0xffff & (superblock.version >> 16)); - uint16_t minor_version = (0xffff & (superblock.version >> 0)); - if ((major_version != LFS_DISK_VERSION_MAJOR || - minor_version > LFS_DISK_VERSION_MINOR)) { - LFS_ERROR("Invalid version %"PRIu32".%"PRIu32, - major_version, minor_version); - err = LFS_ERR_INVAL; - goto cleanup; - } + // check superblock configuration + if (superblock.attr_max) { + if (superblock.attr_max > lfs->attr_max) { + LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", + superblock.attr_max, lfs->attr_max); + err = LFS_ERR_INVAL; + goto cleanup; + } - // check superblock configuration - if (superblock.attr_max) { - if (superblock.attr_max > lfs->attr_max) { - LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", - superblock.attr_max, lfs->attr_max); - err = LFS_ERR_INVAL; - goto cleanup; - } + lfs->attr_max = superblock.attr_max; + } - lfs->attr_max = superblock.attr_max; - } + if (superblock.name_max) { + if (superblock.name_max > lfs->name_max) { + LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", + superblock.name_max, lfs->name_max); + err = LFS_ERR_INVAL; + goto cleanup; + } - if (superblock.name_max) { - if (superblock.name_max > lfs->name_max) { - LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", - superblock.name_max, lfs->name_max); - err = LFS_ERR_INVAL; - goto cleanup; - } + lfs->name_max = superblock.name_max; + } - lfs->name_max = superblock.name_max; - } + if (superblock.inline_max) { + if (superblock.inline_max > lfs->inline_max) { + LFS_ERROR("Unsupported inline_max (%"PRIu32" > %"PRIu32")", + superblock.inline_max, lfs->inline_max); + err = LFS_ERR_INVAL; + goto cleanup; + } - if (superblock.inline_max) { - if (superblock.inline_max > lfs->inline_max) { - LFS_ERROR("Unsupported inline_max (%"PRIu32" > %"PRIu32")", - superblock.inline_max, lfs->inline_max); - err = LFS_ERR_INVAL; - goto cleanup; + lfs->inline_max = superblock.inline_max; + } } - lfs->inline_max = superblock.inline_max; - } - - // scan for any global updates - lfs_mdir_t dir = {.tail = {0, 1}}; - while (!lfs_pair_isnull(dir.tail)) { - res = lfs_dir_fetchmatch(lfs, &dir, dir.tail, 0x7c000000, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), NULL, NULL); - if (res < 0) { - err = LFS_ERR_INVAL; + // has globals? + lfs_global_t locals; + res = lfs_dir_get(lfs, &dir, 0x7c000000, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); + if (res < 0 && res != LFS_ERR_NOENT) { + err = res; goto cleanup; } - if (res) { - lfs_global_t locals; - res = lfs_dir_get(lfs, &dir, 0x7c000000, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); - if (res < 0) { - err = res; - goto cleanup; - } + if (res != LFS_ERR_NOENT) { locals.l.deorphaned = (lfs_tag_type(res) & 1); - - // xor together indirect deletes + // xor together to find resulting globals lfs_global_xor(&lfs->locals, &locals); } } diff --git a/lfs.h b/lfs.h index 090041c7..5acb484d 100644 --- a/lfs.h +++ b/lfs.h @@ -94,27 +94,25 @@ enum lfs_type { // internally used types LFS_TYPE_USER = 0x100, - LFS_TYPE_SUPERBLOCK = 0x001, - LFS_TYPE_ROOT = 0x000, LFS_TYPE_NAME = 0x000, LFS_TYPE_DELETE = 0x020, LFS_TYPE_STRUCT = 0x040, - LFS_TYPE_GLOBALS = 0x0e0, LFS_TYPE_TAIL = 0x080, LFS_TYPE_SOFTTAIL = 0x080, LFS_TYPE_HARDTAIL = 0x081, LFS_TYPE_CRC = 0x0a0, + LFS_TYPE_SUPERBLOCK = 0x001, + LFS_TYPE_GLOBALS = 0x0e0, LFS_TYPE_DIRSTRUCT = 0x040, LFS_TYPE_INLINESTRUCT = 0x041, LFS_TYPE_CTZSTRUCT = 0x042, // internal chip sources - LFS_FROM_REGION = 0x000, + LFS_FROM_MEM = 0x000, LFS_FROM_DISK = 0x200, LFS_FROM_MOVE = 0x0c1, LFS_FROM_USERATTRS = 0x0c2, - LFS_FROM_SUPERBLOCK = 0x0c3, }; // File open flags diff --git a/tests/debug.py b/tests/debug.py index 3c975c84..2fcc05a0 100755 --- a/tests/debug.py +++ b/tests/debug.py @@ -7,7 +7,6 @@ (0x1ff, 0x002): 'reg', (0x1ff, 0x003): 'dir', (0x1ff, 0x001): 'superblock', - (0x1ff, 0x000): 'root', (0x1ff, 0x020): 'delete', (0x1f0, 0x0e0): 'globals', (0x1ff, 0x080): 'tail soft', From 29b881017df353bb8a387fb59832c34a802db761 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 14 Sep 2018 22:02:39 -0500 Subject: [PATCH 106/139] Revisited xored-globals and related logic Added separate bit for "hasmove", which means we don't need to check the move id, and allows us to add more sync-related global states in the future, as long as they never happen simultaneously (such as orphans and moves). Also refactored some of the logic and removed the union in the global structure, which didn't really add anything of value. --- lfs.c | 578 +++++++++++++++++++++++++++++----------------------------- lfs.h | 23 +-- 2 files changed, 295 insertions(+), 306 deletions(-) diff --git a/lfs.c b/lfs.c index 592f9805..86079464 100644 --- a/lfs.c +++ b/lfs.c @@ -315,52 +315,55 @@ struct lfs_diskoff { }; // operations on set of globals -static inline void lfs_global_xor(lfs_global_t *a, const lfs_global_t *b) { - for (int i = 0; i < sizeof(lfs_global_t)/4; i++) { - a->u32[i] ^= b->u32[i]; +static inline void lfs_global_xor(struct lfs_globals *a, + const struct lfs_globals *b) { + uint32_t *a32 = (uint32_t *)a; + const uint32_t *b32 = (const uint32_t *)b; + for (int i = 0; i < sizeof(struct lfs_globals)/4; i++) { + a32[i] ^= b32[i]; } } -static inline bool lfs_global_iszero(const lfs_global_t *a) { - for (int i = 0; i < sizeof(lfs_global_t)/4; i++) { - if (a->u32[i] != 0) { +static inline bool lfs_global_iszero(const struct lfs_globals *a) { + const uint32_t *a32 = (const uint32_t *)a; + for (int i = 0; i < sizeof(struct lfs_globals)/4; i++) { + if (a32[i] != 0) { return false; } } return true; } -static inline void lfs_global_zero(lfs_global_t *a) { - memset(a, 0, sizeof(lfs_global_t)); +static inline void lfs_global_zero(struct lfs_globals *a) { + lfs_global_xor(a, a); } -static inline void lfs_global_fromle32(lfs_global_t *a) { - lfs_pair_fromle32(a->l.movepair); - a->l.moveid = lfs_fromle16(a->l.moveid); +static inline void lfs_global_fromle32(struct lfs_globals *a) { + lfs_pair_fromle32(a->pair); + a->id = lfs_fromle16(a->id); } -static inline void lfs_global_tole32(lfs_global_t *a) { - lfs_pair_tole32(a->l.movepair); - a->l.moveid = lfs_tole16(a->l.moveid); +static inline void lfs_global_tole32(struct lfs_globals *a) { + lfs_pair_tole32(a->pair); + a->id = lfs_tole16(a->id); } static inline void lfs_global_move(lfs_t *lfs, - const lfs_block_t pair[2], uint16_t id) { - lfs_global_t diff; - lfs_global_zero(&diff); - diff.l.movepair[0] ^= lfs->globals.g.movepair[0] ^ pair[0]; - diff.l.movepair[1] ^= lfs->globals.g.movepair[1] ^ pair[1]; - diff.l.moveid ^= lfs->globals.g.moveid ^ id; + bool hasmove, const lfs_block_t pair[2], uint16_t id) { lfs_global_fromle32(&lfs->locals); - lfs_global_xor(&lfs->locals, &diff); + lfs_global_xor(&lfs->locals, &lfs->globals); + lfs->globals.hasmove = hasmove; + lfs->globals.pair[0] = pair[0]; + lfs->globals.pair[1] = pair[1]; + lfs->globals.id = id; + lfs_global_xor(&lfs->locals, &lfs->globals); lfs_global_tole32(&lfs->locals); - lfs_global_xor(&lfs->globals, &diff); } static inline void lfs_global_orphans(lfs_t *lfs, int8_t orphans) { - lfs->locals.l.deorphaned ^= (lfs->globals.g.orphans == 0); - lfs->locals.l.deorphaned ^= (lfs->globals.g.orphans + orphans == 0); - lfs->globals.g.orphans += orphans; + lfs->locals.orphans ^= (lfs->globals.orphans == 0); + lfs->globals.orphans += orphans; + lfs->locals.orphans ^= (lfs->globals.orphans == 0); } // other endianness operations @@ -730,11 +733,12 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // consider what we have good enough if (dir->off > 0) { // synthetic move - if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0) { - if (lfs->globals.g.moveid == lfs_tag_id(foundtag)) { + if (lfs->globals.hasmove && + lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0) { + if (lfs->globals.id == lfs_tag_id(foundtag)) { foundtag = 0; } else if (foundtag && - lfs->globals.g.moveid < lfs_tag_id(foundtag)) { + lfs->globals.id < lfs_tag_id(foundtag)) { foundtag -= LFS_MKTAG(0, 1, 0); } } @@ -758,24 +762,6 @@ static int lfs_dir_fetch(lfs_t *lfs, 0xffffffff, 0x00000000, NULL, NULL); } -static lfs_stag_t lfs_dir_findmatch(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2], bool fs, - lfs_tag_t findmask, lfs_tag_t findtag, - int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { - dir->split = true; - dir->tail[0] = pair[0]; - dir->tail[1] = pair[1]; - while ((dir->split || fs) && !lfs_pair_isnull(dir->tail)) { - lfs_stag_t tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, - findmask, findtag, cb, data); - if (tag) { - return tag; - } - } - - return LFS_ERR_NOENT; -} - struct lfs_dir_get_match { lfs_t *lfs; void *buffer; @@ -811,8 +797,9 @@ static int lfs_dir_get_match(void *data, static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, lfs_tag_t getmask, lfs_tag_t gettag, void *buffer) { lfs_stag_t getdiff = 0; - if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0 && - lfs_tag_id(gettag) <= lfs->globals.g.moveid) { + if (lfs->globals.hasmove && + lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0 && + lfs_tag_id(gettag) <= lfs->globals.id) { // synthetic moves gettag += LFS_MKTAG(0, 1, 0); getdiff -= LFS_MKTAG(0, 1, 0); @@ -829,6 +816,181 @@ static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, return res ? res : LFS_ERR_NOENT; } +static int lfs_dir_getglobals(lfs_t *lfs, const lfs_mdir_t *dir, + struct lfs_globals *globals) { + struct lfs_globals locals; + lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c000000, + LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); + if (res < 0 && res != LFS_ERR_NOENT) { + return res; + } + + if (res != LFS_ERR_NOENT) { + locals.hasmove = (lfs_tag_type(res) & 2); + locals.orphans = (lfs_tag_type(res) & 1); + // xor together to find resulting globals + lfs_global_xor(globals, &locals); + } + + return 0; +} + +static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, + uint16_t id, struct lfs_info *info) { + if (id == 0x3ff) { + // special case for root + strcpy(info->name, "/"); + info->type = LFS_TYPE_DIR; + return 0; + } + + lfs_stag_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name); + if (tag < 0) { + return tag; + } + + info->type = lfs_tag_type(tag); + + struct lfs_ctz ctz; + tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); + if (tag < 0) { + return tag; + } + lfs_ctz_fromle32(&ctz); + + if (lfs_tag_type(tag) == LFS_TYPE_CTZSTRUCT) { + info->size = ctz.size; + } else if (lfs_tag_type(tag) == LFS_TYPE_INLINESTRUCT) { + info->size = lfs_tag_size(tag); + } + + return 0; +} + +static lfs_stag_t lfs_dir_findmatch(lfs_t *lfs, + lfs_mdir_t *dir, const lfs_block_t pair[2], bool fs, + lfs_tag_t findmask, lfs_tag_t findtag, + int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + dir->split = true; + dir->tail[0] = pair[0]; + dir->tail[1] = pair[1]; + while ((dir->split || fs) && !lfs_pair_isnull(dir->tail)) { + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, + findmask, findtag, cb, data); + if (tag) { + return tag; + } + } + + return LFS_ERR_NOENT; +} + +struct lfs_dir_find_match { + lfs_t *lfs; + const void *name; + lfs_size_t size; +}; + +static int lfs_dir_find_match(void *data, + lfs_tag_t tag, const void *buffer) { + struct lfs_dir_find_match *name = data; + lfs_t *lfs = name->lfs; + const struct lfs_diskoff *disk = buffer; + (void)tag; + + return lfs_bd_cmp(lfs, + NULL, &lfs->rcache, name->size, + disk->block, disk->off, name->name, name->size); +} + +static lfs_stag_t lfs_dir_find(lfs_t *lfs, + lfs_mdir_t *dir, const char **path) { + // we reduce path to a single name if we can find it + const char *name = *path; + *path = NULL; + + // default to root dir + lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); + lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; + + while (true) { +nextname: + // skip slashes + name += strspn(name, "/"); + lfs_size_t namelen = strcspn(name, "/"); + + // skip '.' and root '..' + if ((namelen == 1 && memcmp(name, ".", 1) == 0) || + (namelen == 2 && memcmp(name, "..", 2) == 0)) { + name += namelen; + goto nextname; + } + + // skip if matched by '..' in name + const char *suffix = name + namelen; + lfs_size_t sufflen; + int depth = 1; + while (true) { + suffix += strspn(suffix, "/"); + sufflen = strcspn(suffix, "/"); + if (sufflen == 0) { + break; + } + + if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { + depth -= 1; + if (depth == 0) { + name = suffix + sufflen; + goto nextname; + } + } else { + depth += 1; + } + + suffix += sufflen; + } + + // found path + if (name[0] == '\0') { + return tag; + } + + // update what we've found if path is only a name + if (strchr(name, '/') == NULL) { + *path = name; + } + + // only continue if we hit a directory + if (lfs_tag_type(tag) != LFS_TYPE_DIR) { + return LFS_ERR_NOTDIR; + } + + // grab the entry data + if (lfs_tag_id(tag) != 0x3ff) { + lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); + if (res < 0) { + return res; + } + lfs_pair_fromle32(pair); + } + + // find entry matching name + tag = lfs_dir_findmatch(lfs, dir, pair, false, 0x7c000fff, + LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, name, namelen}); + if (tag < 0) { + return tag; + } + + // to next name + name += namelen; + } +} + // commit logic struct lfs_commit { lfs_block_t block; @@ -980,6 +1142,13 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, return 0; } +static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, + struct lfs_globals *globals) { + return lfs_commit_attr(lfs, commit, + LFS_MKTAG(LFS_TYPE_GLOBALS + 2*globals->hasmove + globals->orphans, + 0x3ff, 10), globals); +} + static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, bool compacting) { // align to program units @@ -1090,18 +1259,12 @@ static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mdir_t *tail) { dir->split = tail->split; // steal state - lfs_global_t locals; - lfs_stag_t res = lfs_dir_get(lfs, tail, 0x7c000000, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - - if (res != LFS_ERR_NOENT) { - locals.l.deorphaned = (lfs_tag_type(res) & 1); - lfs_global_xor(&lfs->locals, &locals); + int err = lfs_dir_getglobals(lfs, tail, &lfs->locals); + if (err) { + return err; } + // update pred's tail return lfs_dir_commit(lfs, dir, LFS_MKATTR(LFS_TYPE_TAIL + dir->split, 0x3ff, dir->tail, sizeof(dir->tail), @@ -1117,16 +1280,9 @@ static int lfs_dir_compact(lfs_t *lfs, // There's nothing special about our global delta, so feed it back // into the global global delta - lfs_global_t locals; - lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c000000, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - - if (res != LFS_ERR_NOENT) { - locals.l.deorphaned = (lfs_tag_type(res) & 1); - lfs_global_xor(&lfs->locals, &locals); + int err = lfs_dir_getglobals(lfs, dir, &lfs->locals); + if (err) { + return err; } // begin loop to commit compaction to blocks until a compact sticks @@ -1167,7 +1323,7 @@ static int lfs_dir_compact(lfs_t *lfs, (const lfs_block_t[2]){0, 1}) == 0) { // we're writing too much to the superblock, // should we expand? - res = lfs_fs_size(lfs); + lfs_stag_t res = lfs_fs_size(lfs); if (res < 0) { return res; } @@ -1187,7 +1343,7 @@ static int lfs_dir_compact(lfs_t *lfs, } // erase block to write to - int err = lfs_bd_erase(lfs, dir->pair[1]); + err = lfs_bd_erase(lfs, dir->pair[1]); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1198,7 +1354,7 @@ static int lfs_dir_compact(lfs_t *lfs, // write out header uint32_t rev = lfs_tole32(dir->rev); - int err = lfs_commit_prog(lfs, &commit, &rev, sizeof(rev)); + err = lfs_commit_prog(lfs, &commit, &rev, sizeof(rev)); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1245,9 +1401,7 @@ static int lfs_dir_compact(lfs_t *lfs, if (!relocated && !lfs_global_iszero(&lfs->locals)) { // commit any globals, unless we're relocating, // in which case our parent will steal our globals - err = lfs_commit_attr(lfs, &commit, - LFS_MKTAG(LFS_TYPE_GLOBALS + lfs->locals.l.deorphaned, - 0x3ff, 10), &lfs->locals); + err = lfs_commit_globals(lfs, &commit, &lfs->locals); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1357,7 +1511,7 @@ static int lfs_dir_compact(lfs_t *lfs, // update references if we relocated LFS_DEBUG("Relocating %"PRIu32" %"PRIu32" to %"PRIu32" %"PRIu32, oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); - int err = lfs_fs_relocate(lfs, oldpair, dir->pair); + err = lfs_fs_relocate(lfs, oldpair, dir->pair); if (err) { return err; } @@ -1369,21 +1523,23 @@ static int lfs_dir_compact(lfs_t *lfs, static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs) { struct lfs_mattr cancelattr; - lfs_global_t canceldiff; - lfs_global_zero(&canceldiff); - if (lfs_pair_cmp(dir->pair, lfs->globals.g.movepair) == 0) { + struct lfs_globals cancels; + lfs_global_zero(&cancels); + if (lfs->globals.hasmove && + lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0) { // Wait, we have the move? Just cancel this out here // We need to, or else the move can become outdated - canceldiff.l.movepair[0] ^= lfs->globals.g.movepair[0] ^ 0xffffffff; - canceldiff.l.movepair[1] ^= lfs->globals.g.movepair[1] ^ 0xffffffff; - canceldiff.l.moveid ^= lfs->globals.g.moveid ^ 0x3ff; - lfs_global_fromle32(&lfs->locals); - lfs_global_xor(&lfs->locals, &canceldiff); - lfs_global_tole32(&lfs->locals); - - cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.l.moveid, 0); + cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.id, 0); cancelattr.next = attrs; attrs = &cancelattr; + + cancels.hasmove = lfs->globals.hasmove; + cancels.pair[0] = lfs->globals.pair[0]; + cancels.pair[1] = lfs->globals.pair[1]; + cancels.id = lfs->globals.id; + lfs_global_fromle32(&lfs->locals); + lfs_global_xor(&lfs->locals, &cancels); + lfs_global_tole32(&lfs->locals); } // calculate new directory size @@ -1451,22 +1607,13 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // commit any global diffs if we have any if (!lfs_global_iszero(&lfs->locals)) { - lfs_global_t locals; - lfs_global_zero(&locals); - lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c000000, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - - if (res != LFS_ERR_NOENT) { - locals.l.deorphaned = (lfs_tag_type(res) & 1); + struct lfs_globals locals = lfs->locals; + int err = lfs_dir_getglobals(lfs, dir, &locals); + if (err) { + return err; } - lfs_global_xor(&locals, &lfs->locals); - int err = lfs_commit_attr(lfs, &commit, - LFS_MKTAG(LFS_TYPE_GLOBALS + locals.l.deorphaned, - 0x3ff, 10), &locals); + err = lfs_commit_globals(lfs, &commit, &locals); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1504,7 +1651,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } // update globals that are affected - lfs_global_xor(&lfs->globals, &canceldiff); + lfs_global_xor(&lfs->globals, &cancels); // update any directories that are affected lfs_mdir_t copy = *dir; @@ -1538,143 +1685,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, return 0; } -struct lfs_dir_find_match { - lfs_t *lfs; - const void *name; - lfs_size_t size; -}; - -static int lfs_dir_find_match(void *data, - lfs_tag_t tag, const void *buffer) { - struct lfs_dir_find_match *name = data; - lfs_t *lfs = name->lfs; - const struct lfs_diskoff *disk = buffer; - (void)tag; - - return lfs_bd_cmp(lfs, - NULL, &lfs->rcache, name->size, - disk->block, disk->off, name->name, name->size); -} - -static lfs_stag_t lfs_dir_find(lfs_t *lfs, - lfs_mdir_t *dir, const char **path) { - // we reduce path to a single name if we can find it - const char *name = *path; - *path = NULL; - - // default to root dir - lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); - lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; - - while (true) { -nextname: - // skip slashes - name += strspn(name, "/"); - lfs_size_t namelen = strcspn(name, "/"); - - // skip '.' and root '..' - if ((namelen == 1 && memcmp(name, ".", 1) == 0) || - (namelen == 2 && memcmp(name, "..", 2) == 0)) { - name += namelen; - goto nextname; - } - - // skip if matched by '..' in name - const char *suffix = name + namelen; - lfs_size_t sufflen; - int depth = 1; - while (true) { - suffix += strspn(suffix, "/"); - sufflen = strcspn(suffix, "/"); - if (sufflen == 0) { - break; - } - - if (sufflen == 2 && memcmp(suffix, "..", 2) == 0) { - depth -= 1; - if (depth == 0) { - name = suffix + sufflen; - goto nextname; - } - } else { - depth += 1; - } - - suffix += sufflen; - } - - // found path - if (name[0] == '\0') { - return tag; - } - - // update what we've found if path is only a name - if (strchr(name, '/') == NULL) { - *path = name; - } - - // only continue if we hit a directory - if (lfs_tag_type(tag) != LFS_TYPE_DIR) { - return LFS_ERR_NOTDIR; - } - - // grab the entry data - if (lfs_tag_id(tag) != 0x3ff) { - lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); - if (res < 0) { - return res; - } - lfs_pair_fromle32(pair); - } - - // find entry matching name - tag = lfs_dir_findmatch(lfs, dir, pair, false, 0x7c000fff, - LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), - lfs_dir_find_match, &(struct lfs_dir_find_match){ - lfs, name, namelen}); - if (tag < 0) { - return tag; - } - - // to next name - name += namelen; - } -} - -static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, - uint16_t id, struct lfs_info *info) { - if (id == 0x3ff) { - // special case for root - strcpy(info->name, "/"); - info->type = LFS_TYPE_DIR; - return 0; - } - - lfs_stag_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name); - if (tag < 0) { - return tag; - } - - info->type = lfs_tag_type(tag); - - struct lfs_ctz ctz; - tag = lfs_dir_get(lfs, dir, 0x7c3ff000, - LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); - if (tag < 0) { - return tag; - } - lfs_ctz_fromle32(&ctz); - - if (lfs_tag_type(tag) == LFS_TYPE_CTZSTRUCT) { - info->size = ctz.size; - } else if (lfs_tag_type(tag) == LFS_TYPE_INLINESTRUCT) { - info->size = lfs_tag_size(tag); - } - - return 0; -} /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { @@ -2865,7 +2875,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // create move to fix later - lfs_global_move(lfs, oldcwd.pair, lfs_tag_id(oldtag)); + lfs_global_move(lfs, true, oldcwd.pair, lfs_tag_id(oldtag)); // move over all attributes err = lfs_dir_commit(lfs, &newcwd, @@ -3052,10 +3062,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[1] = 0xffffffff; lfs->mlist = NULL; lfs->seed = 0; - lfs->globals.g.movepair[0] = 0xffffffff; - lfs->globals.g.movepair[1] = 0xffffffff; - lfs->globals.g.moveid = 0x3ff; - lfs->globals.g.orphans = 0; + lfs_global_zero(&lfs->globals); lfs_global_zero(&lfs->locals); return 0; @@ -3216,18 +3223,9 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } // has globals? - lfs_global_t locals; - res = lfs_dir_get(lfs, &dir, 0x7c000000, - LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); - if (res < 0 && res != LFS_ERR_NOENT) { - err = res; - goto cleanup; - } - - if (res != LFS_ERR_NOENT) { - locals.l.deorphaned = (lfs_tag_type(res) & 1); - // xor together to find resulting globals - lfs_global_xor(&lfs->locals, &locals); + err = lfs_dir_getglobals(lfs, &dir, &lfs->locals); + if (err) { + return err; } } @@ -3235,11 +3233,9 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_global_fromle32(&lfs->locals); lfs_global_xor(&lfs->globals, &lfs->locals); lfs_global_zero(&lfs->locals); - if (!lfs_pair_isnull(lfs->globals.g.movepair)) { + if (lfs->globals.hasmove) { LFS_DEBUG("Found move %"PRIu32" %"PRIu32" %"PRIu32, - lfs->globals.g.movepair[0], - lfs->globals.g.movepair[1], - lfs->globals.g.moveid); + lfs->globals.pair[0], lfs->globals.pair[1], lfs->globals.id); } // setup free lookahead @@ -3463,7 +3459,36 @@ static int lfs_fs_relocate(lfs_t *lfs, return 0; } +static int lfs_fs_demove(lfs_t *lfs) { + if (!lfs->globals.hasmove) { + return 0; + } + + // Fix bad moves + LFS_DEBUG("Fixing move %"PRIu32" %"PRIu32" %"PRIu32, + lfs->globals.pair[0], lfs->globals.pair[1], lfs->globals.id); + + // fetch and delete the moved entry + lfs_mdir_t movedir; + int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.pair); + if (err) { + return err; + } + + // rely on cancel logic inside commit + err = lfs_dir_commit(lfs, &movedir, NULL); + if (err) { + return err; + } + + return 0; +} + static int lfs_fs_deorphan(lfs_t *lfs) { + if (!lfs->globals.orphans) { + return 0; + } + // Fix any orphans lfs_mdir_t pdir = {.split = true}; lfs_mdir_t dir = {.tail = {0, 1}}; @@ -3527,26 +3552,17 @@ static int lfs_fs_deorphan(lfs_t *lfs) { } // mark orphans as fixed - lfs_global_orphans(lfs, -lfs->globals.g.orphans); + lfs_global_orphans(lfs, -lfs->globals.orphans); return 0; } -static int lfs_fs_demove(lfs_t *lfs) { - // Fix bad moves - LFS_DEBUG("Fixing move %"PRIu32" %"PRIu32" %"PRIu32, - lfs->globals.g.movepair[0], - lfs->globals.g.movepair[1], - lfs->globals.g.moveid); - - // fetch and delete the moved entry - lfs_mdir_t movedir; - int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.g.movepair); +static int lfs_fs_forceconsistency(lfs_t *lfs) { + int err = lfs_fs_demove(lfs); if (err) { return err; } - // rely on cancel logic inside commit - err = lfs_dir_commit(lfs, &movedir, NULL); + err = lfs_fs_deorphan(lfs); if (err) { return err; } @@ -3554,24 +3570,6 @@ static int lfs_fs_demove(lfs_t *lfs) { return 0; } -static int lfs_fs_forceconsistency(lfs_t *lfs) { - if (lfs->globals.g.orphans) { - int err = lfs_fs_deorphan(lfs); - if (err) { - return err; - } - } - - if (lfs->globals.g.moveid != 0x3ff) { - int err = lfs_fs_demove(lfs); - if (err) { - return err; - } - } - - return 0; -} - static int lfs_fs_size_count(void *p, lfs_block_t block) { (void)block; lfs_size_t *size = p; diff --git a/lfs.h b/lfs.h index 5acb484d..23b0bc6b 100644 --- a/lfs.h +++ b/lfs.h @@ -290,20 +290,6 @@ typedef struct lfs_cache { uint8_t *buffer; } lfs_cache_t; -typedef union lfs_global { - uint32_t u32[3]; - struct { - lfs_block_t movepair[2]; - uint16_t moveid; - uint8_t deorphaned; - } l; - struct { - lfs_block_t movepair[2]; - uint16_t moveid; - uint8_t orphans; - } g; -} lfs_global_t; - typedef struct lfs_mdir { lfs_block_t pair[2]; uint32_t rev; @@ -373,8 +359,13 @@ typedef struct lfs { } *mlist; uint32_t seed; - lfs_global_t globals; - lfs_global_t locals; + struct lfs_globals { + lfs_block_t pair[2]; + uint16_t id; + bool hasmove; + uint8_t orphans; + } globals, locals; + struct lfs_free { lfs_block_t off; lfs_block_t size; From cafe6ab46603244522916c7580385bfd04a3cff4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 15 Sep 2018 02:28:45 -0500 Subject: [PATCH 107/139] Fixed issue with splitting metadata-pairs in full filesystem Depending on your perspective, this may not be a necessary operation, given that a nearly-full filesystem is already prone to ENOSPC errors, especially a COW filesystem. However, splitting metadata-pairs can happen in really unfortunate situations, such as removing files. The solution here is to allow "overcompaction", that is, a compaction without bounds checking to allow splitting. This unfortunately pushes our metadata-pairs past their reasonable limit of saturation, which means writes get exponentially costly. However it does allow littlefs to continue working in extreme situations. --- lfs.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lfs.c b/lfs.c index 86079464..f7cb7972 100644 --- a/lfs.c +++ b/lfs.c @@ -1290,6 +1290,7 @@ static int lfs_dir_compact(lfs_t *lfs, // setup compaction bool splitted = false; bool exhausted = false; + bool overcompacting = false; struct lfs_commit commit; commit.block = dir->pair[1]; @@ -1310,9 +1311,11 @@ static int lfs_dir_compact(lfs_t *lfs, // cleanup delete, and we cap at half a block to give room // for metadata updates commit.begin = 0; - commit.end = lfs_min( - lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size), - lfs->cfg->block_size - 38); + commit.end = lfs->cfg->block_size - 38; + if (!overcompacting) { + commit.end = lfs_min(commit.end, + lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size)); + } if (!splitted) { // increment revision count @@ -1369,8 +1372,9 @@ static int lfs_dir_compact(lfs_t *lfs, 0x003ff000, LFS_MKTAG(0, id, 0), -LFS_MKTAG(0, begin, 0), source, attrs); - if (err && !(splitted && err == LFS_ERR_NOSPC)) { - if (err == LFS_ERR_NOSPC) { + if (err && !(splitted && !overcompacting && + err == LFS_ERR_NOSPC)) { + if (!overcompacting && err == LFS_ERR_NOSPC) { goto split; } else if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1457,6 +1461,11 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t tail; err = lfs_dir_alloc(lfs, &tail); if (err) { + if (err == LFS_ERR_NOSPC) { + // No space to expand? Try overcompacting + overcompacting = true; + goto commit; + } return err; } From d7e4abad0bbbff05489a85a66302d5adb12cb789 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 15 Sep 2018 02:40:53 -0500 Subject: [PATCH 108/139] Edited tag structure to balance size vs id count This is a minor tweak that resulted from looking at some other use cases for the littlefs data-structure on disk. Consider an implementation that does not need to buffer inline-files in RAM. In this case we should have as large a tag size field as possible. Unfortunately, we don't have much space to work with in the 32-bit tag struct, so we have to make some compromises. These limitations could be removed with a 64-bit tag struct, at the cost of code size. 32-bit tag structure: [--- 32 ---] [1|- 9 -|- 9 -|-- 13 --] ^ ^ ^ ^- entry length | | \-------- file id | \-------------- tag type \------------------ valid bit --- lfs.c | 72 ++++++++++++++++++++++++------------------------ lfs.h | 4 +-- tests/corrupt.py | 2 +- tests/debug.py | 12 ++++---- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/lfs.c b/lfs.c index f7cb7972..4f50fce9 100644 --- a/lfs.c +++ b/lfs.c @@ -265,7 +265,7 @@ typedef uint32_t lfs_tag_t; typedef int32_t lfs_stag_t; #define LFS_MKTAG(type, id, size) \ - (((lfs_tag_t)(type) << 22) | ((lfs_tag_t)(id) << 12) | (lfs_tag_t)(size)) + (((lfs_tag_t)(type) << 22) | ((lfs_tag_t)(id) << 13) | (lfs_tag_t)(size)) static inline bool lfs_tag_isvalid(lfs_tag_t tag) { return !(tag & 0x80000000); @@ -276,7 +276,7 @@ static inline bool lfs_tag_isuser(lfs_tag_t tag) { } static inline bool lfs_tag_isdelete(lfs_tag_t tag) { - return (tag & 0x00000fff) == 0xfff; + return ((int32_t)(tag << 19) >> 19) == -1; } static inline uint16_t lfs_tag_type(lfs_tag_t tag) { @@ -288,11 +288,11 @@ static inline uint16_t lfs_tag_subtype(lfs_tag_t tag) { } static inline uint16_t lfs_tag_id(lfs_tag_t tag) { - return (tag & 0x003ff000) >> 12; + return (tag & 0x003fe000) >> 13; } static inline lfs_size_t lfs_tag_size(lfs_tag_t tag) { - return tag & 0x00000fff; + return tag & 0x00001fff; } static inline lfs_size_t lfs_tag_dsize(lfs_tag_t tag) { @@ -837,14 +837,14 @@ static int lfs_dir_getglobals(lfs_t *lfs, const lfs_mdir_t *dir, static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id, struct lfs_info *info) { - if (id == 0x3ff) { + if (id == 0x1ff) { // special case for root strcpy(info->name, "/"); info->type = LFS_TYPE_DIR; return 0; } - lfs_stag_t tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + lfs_stag_t tag = lfs_dir_get(lfs, dir, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name); if (tag < 0) { return tag; @@ -853,7 +853,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, info->type = lfs_tag_type(tag); struct lfs_ctz ctz; - tag = lfs_dir_get(lfs, dir, 0x7c3ff000, + tag = lfs_dir_get(lfs, dir, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); if (tag < 0) { return tag; @@ -912,7 +912,7 @@ static lfs_stag_t lfs_dir_find(lfs_t *lfs, *path = NULL; // default to root dir - lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); + lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x1ff, 0); lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; while (true) { @@ -968,8 +968,8 @@ static lfs_stag_t lfs_dir_find(lfs_t *lfs, } // grab the entry data - if (lfs_tag_id(tag) != 0x3ff) { - lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c3ff000, + if (lfs_tag_id(tag) != 0x1ff) { + lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -978,7 +978,7 @@ static lfs_stag_t lfs_dir_find(lfs_t *lfs, } // find entry matching name - tag = lfs_dir_findmatch(lfs, dir, pair, false, 0x7c000fff, + tag = lfs_dir_findmatch(lfs, dir, pair, false, 0x7c001fff, LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), lfs_dir_find_match, &(struct lfs_dir_find_match){ lfs, name, namelen}); @@ -1046,7 +1046,7 @@ static int lfs_commit_move_match(void *data, .pair[0] = commit->block, .off = commit->off, .etag = commit->ptag}, NULL, - lfs_tag_isuser(tag) ? 0x7ffff000 : 0x7c3ff000, tag, 0, + lfs_tag_isuser(tag) ? 0x7fffe000 : 0x7c3fe000, tag, 0, lfs_dir_get_match, &(struct lfs_dir_get_match){ lfs, NULL, 0, true}); if (res < 0 && res != LFS_ERR_NOENT) { @@ -1089,7 +1089,7 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, if (lfs_tag_type(tag) == LFS_FROM_MOVE) { // special case for moves return lfs_commit_move(lfs, commit, 1, - 0x003ff000, LFS_MKTAG(0, lfs_tag_size(tag), 0), + 0x003fe000, LFS_MKTAG(0, lfs_tag_size(tag), 0), LFS_MKTAG(0, lfs_tag_id(tag), 0) - LFS_MKTAG(0, lfs_tag_size(tag), 0), buffer, NULL); @@ -1146,7 +1146,7 @@ static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, struct lfs_globals *globals) { return lfs_commit_attr(lfs, commit, LFS_MKTAG(LFS_TYPE_GLOBALS + 2*globals->hasmove + globals->orphans, - 0x3ff, 10), globals); + 0x1ff, 10), globals); } static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, @@ -1166,8 +1166,8 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, // build crc tag bool reset = ~lfs_fromle32(tag) >> 31; - tag = LFS_MKTAG(LFS_TYPE_CRC + (compacting << 1) + reset, - 0x3ff, off - (commit->off+sizeof(lfs_tag_t))); + tag = LFS_MKTAG(LFS_TYPE_CRC + 2*compacting + reset, + 0x1ff, off - (commit->off+sizeof(lfs_tag_t))); // write out crc uint32_t footer[2]; @@ -1267,7 +1267,7 @@ static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mdir_t *tail) { // update pred's tail return lfs_dir_commit(lfs, dir, LFS_MKATTR(LFS_TYPE_TAIL + dir->split, - 0x3ff, dir->tail, sizeof(dir->tail), + 0x1ff, dir->tail, sizeof(dir->tail), NULL)); } @@ -1369,7 +1369,7 @@ static int lfs_dir_compact(lfs_t *lfs, for (uint16_t id = begin; id < end || commit.off < commit.ack; id++) { for (int pass = 0; pass < 2; pass++) { err = lfs_commit_move(lfs, &commit, pass, - 0x003ff000, LFS_MKTAG(0, id, 0), + 0x003fe000, LFS_MKTAG(0, id, 0), -LFS_MKTAG(0, begin, 0), source, attrs); if (err && !(splitted && !overcompacting && @@ -1419,7 +1419,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_pair_tole32(dir->tail); err = lfs_commit_attr(lfs, &commit, LFS_MKTAG(LFS_TYPE_TAIL + dir->split, - 0x3ff, sizeof(dir->tail)), dir->tail); + 0x1ff, sizeof(dir->tail)), dir->tail); lfs_pair_fromle32(dir->tail); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1737,7 +1737,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.tail[1] = dir.pair[1]; lfs_pair_tole32(dir.pair); err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x1ff, cwd.tail, sizeof(cwd.tail), LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, NULL)))); @@ -1760,13 +1760,13 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { } lfs_block_t pair[2]; - if (lfs_tag_id(tag) == 0x3ff) { + if (lfs_tag_id(tag) == 0x1ff) { // handle root dir separately pair[0] = lfs->root[0]; pair[1] = lfs->root[1]; } else { // get dir pair from parent - lfs_stag_t res = lfs_dir_get(lfs, &dir->m, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, &dir->m, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -2163,7 +2163,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, file->flags |= LFS_F_DIRTY; } else { // try to load what's on disk, if it's inlined we'll fix it later - tag = lfs_dir_get(lfs, &file->m, 0x7c3ff000, + tag = lfs_dir_get(lfs, &file->m, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_STRUCT, file->id, 8), &file->ctz); if (tag < 0) { err = tag; @@ -2175,7 +2175,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // fetch attrs for (const struct lfs_attr *a = file->cfg->attrs; a; a = a->next) { if ((file->flags & 3) != LFS_O_WRONLY) { - lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x7ffff000, + lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x7fffe000, LFS_MKTAG(0x100 | a->type, file->id, a->size), a->buffer); if (res < 0 && res != LFS_ERR_NOENT) { err = res; @@ -2218,7 +2218,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // don't always read (may be new/trunc file) if (file->ctz.size > 0) { - lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->ctz.size), file->cache.buffer); if (res < 0) { @@ -2778,7 +2778,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (lfs_tag_type(tag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t pair[2]; - lfs_stag_t res = lfs_dir_get(lfs, &cwd, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, &cwd, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -2862,7 +2862,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } else if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t prevpair[2]; - lfs_stag_t res = lfs_dir_get(lfs, &newcwd, 0x7c3ff000, + lfs_stag_t res = lfs_dir_get(lfs, &newcwd, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair); if (res < 0) { return res; @@ -2933,7 +2933,7 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, } uint16_t id = lfs_tag_id(res); - if (id == 0x3ff) { + if (id == 0x1ff) { // special case for root id = 0; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); @@ -2942,7 +2942,7 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, } } - res = lfs_dir_get(lfs, &cwd, 0x7ffff000, + res = lfs_dir_get(lfs, &cwd, 0x7fffe000, LFS_MKTAG(0x100 | type, id, lfs_min(size, lfs->attr_max)), buffer); if (res < 0) { @@ -2964,7 +2964,7 @@ static int lfs_commitattr(lfs_t *lfs, const char *path, } uint16_t id = lfs_tag_id(res); - if (id == 0x3ff) { + if (id == 0x1ff) { // special case for root id = 0; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); @@ -2988,7 +2988,7 @@ int lfs_setattr(lfs_t *lfs, const char *path, } int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { - return lfs_commitattr(lfs, path, type, NULL, LFS_ATTR_MAX+1); + return lfs_commitattr(lfs, path, type, NULL, 0x1fff); } @@ -3290,7 +3290,7 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { struct lfs_ctz ctz; - lfs_stag_t tag = lfs_dir_get(lfs, &dir, 0x7c3ff000, + lfs_stag_t tag = lfs_dir_get(lfs, &dir, 0x7c3fe000, LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); if (tag < 0) { if (tag == LFS_ERR_NOENT) { @@ -3393,7 +3393,7 @@ static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], for (int i = 0; i < 2; i++) { struct lfs_fs_parent_match match = {lfs, {pair[0], pair[1]}}; lfs_stag_t tag = lfs_dir_findmatch(lfs, parent, - (const lfs_block_t[2]){0, 1}, true, 0x7fc00fff, + (const lfs_block_t[2]){0, 1}, true, 0x7fc01fff, LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), lfs_fs_parent_match, &match); if (tag != LFS_ERR_NOENT) { @@ -3458,7 +3458,7 @@ static int lfs_fs_relocate(lfs_t *lfs, parent.tail[1] = newpair[1]; err = lfs_dir_commit(lfs, &parent, LFS_MKATTR(LFS_TYPE_TAIL + parent.split, - 0x3ff, parent.tail, sizeof(parent.tail), + 0x1ff, parent.tail, sizeof(parent.tail), NULL)); if (err) { return err; @@ -3532,7 +3532,7 @@ static int lfs_fs_deorphan(lfs_t *lfs) { } lfs_block_t pair[2]; - lfs_stag_t res = lfs_dir_get(lfs, &parent, 0x7ffff000, tag, pair); + lfs_stag_t res = lfs_dir_get(lfs, &parent, 0x7fffe000, tag, pair); if (res < 0) { return res; } @@ -3547,7 +3547,7 @@ static int lfs_fs_deorphan(lfs_t *lfs) { pdir.tail[1] = pair[1]; err = lfs_dir_commit(lfs, &pdir, LFS_MKATTR(LFS_TYPE_SOFTTAIL, - 0x3ff, pdir.tail, sizeof(pdir.tail), + 0x1ff, pdir.tail, sizeof(pdir.tail), NULL)); if (err) { return err; diff --git a/lfs.h b/lfs.h index 23b0bc6b..edb70265 100644 --- a/lfs.h +++ b/lfs.h @@ -49,7 +49,7 @@ typedef uint32_t lfs_block_t; // to <= 0xfff. Stored in superblock and must be respected by other // littlefs drivers. #ifndef LFS_ATTR_MAX -#define LFS_ATTR_MAX 0xffe +#define LFS_ATTR_MAX 0x1ffe #endif // Maximum name size in bytes, may be redefined to reduce the size of the @@ -64,7 +64,7 @@ typedef uint32_t lfs_block_t; // block. Limited to <= LFS_ATTR_MAX and <= cache_size. Stored in superblock // and must be respected by other littlefs drivers. #ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 0xffe +#define LFS_INLINE_MAX 0x1ffe #endif // Possible error codes, these are negative to allow diff --git a/tests/corrupt.py b/tests/corrupt.py index 4a49d428..d9943892 100755 --- a/tests/corrupt.py +++ b/tests/corrupt.py @@ -19,7 +19,7 @@ def corrupt(block): break tag ^= ntag - size = (tag & 0xfff) if (tag & 0xfff) != 0xfff else 0 + size = (tag & 0x1fff) if (tag & 0x1fff) != 0x1fff else 0 file.seek(size, os.SEEK_CUR) # lob off last 3 bytes diff --git a/tests/debug.py b/tests/debug.py index 2fcc05a0..0088442c 100755 --- a/tests/debug.py +++ b/tests/debug.py @@ -76,11 +76,11 @@ def main(*blocks): off += 4 type = (tag & 0x7fc00000) >> 22 - id = (tag & 0x003ff000) >> 12 - size = (tag & 0x00000fff) >> 0 + id = (tag & 0x003fe000) >> 13 + size = (tag & 0x00001fff) >> 0 iscrc = (type & 0x1f0) == 0x0f0 - data = file.read(size if size != 0xfff else 0) + data = file.read(size if size != 0x1fff else 0) if iscrc: crc = binascii.crc32(data[:4], crc) else: @@ -89,12 +89,12 @@ def main(*blocks): print '%04x: %08x %-14s %3s %3s %-23s %-8s' % ( off, tag, typeof(type) + (' bad!' if iscrc and ~crc else ''), - id if id != 0x3ff else '.', - size if size != 0xfff else 'x', + id if id != 0x1ff else '.', + size if size != 0x1fff else 'x', ' '.join('%02x' % ord(c) for c in data[:8]), ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8])) - off += size if size != 0xfff else 0 + off += size if size != 0x1fff else 0 if iscrc: crc = 0 tag ^= (type & 1) << 31 From f010d2add1e7ad1907579f3a0358da2662b288b2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 2 Oct 2018 15:42:07 -0500 Subject: [PATCH 109/139] Fixed issue with reads ignoring the pcache The downside of smarter caching is that now there are more complicated corner cases to consider. Here we weren't considering our pcaches when aligning reads to the rcache. This meant if things were unaligned, we would read a cache-line that overlaps the pcache and then proceed to ignore whatever we overlapped. This fix is to determine the limit of an rcache read not from cache alignment but from the available caches, which we check anyways to find cached data. --- lfs.c | 53 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/lfs.c b/lfs.c index 4f50fce9..feb936db 100644 --- a/lfs.c +++ b/lfs.c @@ -44,39 +44,46 @@ static int lfs_bd_read(lfs_t *lfs, } while (size > 0) { + lfs_size_t diff = size; + if (pcache && block == pcache->block && - off >= pcache->off && off < pcache->off + pcache->size) { - // is already in pcache? - lfs_size_t diff = lfs_min(size, pcache->size - (off-pcache->off)); - memcpy(data, &pcache->buffer[off-pcache->off], diff); + if (off >= pcache->off) { + // is already in pcache? + diff = lfs_min(diff, pcache->size - (off-pcache->off)); + memcpy(data, &pcache->buffer[off-pcache->off], diff); - data += diff; - off += diff; - size -= diff; - continue; + data += diff; + off += diff; + size -= diff; + continue; + } + + // pcache takes priority + diff = lfs_min(diff, pcache->off-off); } if (block == rcache->block && - off >= rcache->off && off < rcache->off + rcache->size) { - // is already in rcache? - lfs_size_t diff = lfs_min(size, rcache->size - (off-rcache->off)); - if (pcache && block == pcache->block) { - diff = lfs_min(diff, pcache->off - off); + if (off >= rcache->off) { + // is already in rcache? + diff = lfs_min(diff, rcache->size - (off-rcache->off)); + memcpy(data, &rcache->buffer[off-rcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; } - memcpy(data, &rcache->buffer[off-rcache->off], diff); - data += diff; - off += diff; - size -= diff; - continue; + // rcache takes priority + diff = lfs_min(diff, rcache->off-off); } if (size >= hint && off % lfs->cfg->read_size == 0 && size >= lfs->cfg->read_size) { // bypass cache? - lfs_size_t diff = size - (size % lfs->cfg->read_size); + diff = lfs_aligndown(diff, lfs->cfg->read_size); int err = lfs->cfg->read(lfs->cfg, block, off, data, diff); if (err) { return err; @@ -91,10 +98,10 @@ static int lfs_bd_read(lfs_t *lfs, // load to cache, first condition can no longer fail LFS_ASSERT(block < lfs->cfg->block_count); rcache->block = block; - rcache->off = lfs_aligndown(off, lfs->cfg->prog_size); - rcache->size = lfs_min(lfs_min( - lfs_alignup(off+hint, lfs->cfg->prog_size), - lfs->cfg->block_size) - rcache->off, lfs->cfg->cache_size); + rcache->off = lfs_aligndown(off, lfs->cfg->read_size); + rcache->size = lfs_min(lfs_alignup(off+hint, lfs->cfg->read_size), + lfs_min(lfs->cfg->block_size - rcache->off, + lfs->cfg->cache_size)); int err = lfs->cfg->read(lfs->cfg, rcache->block, rcache->off, rcache->buffer, rcache->size); if (err) { From ad96fca18ffb8787b0fe323d199ac4d5f0fa7eb3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 2 Oct 2018 18:28:37 -0500 Subject: [PATCH 110/139] Changed attr_max to be specific to custom attributes While technically, both system and user attributes share the same disk limitations, that's not what attr_max represents when considered from the user's perspective. To the user, attr_max applies only to custom attributes. This means attr_max should not impact other configurable limitations, such as inline files, and the ordering should be reconsidered with what the user finds most important. --- lfs.c | 41 +++++++++++++++++++-------------------- lfs.h | 61 +++++++++++++++++++++++++++++++---------------------------- 2 files changed, 53 insertions(+), 49 deletions(-) diff --git a/lfs.c b/lfs.c index feb936db..6f7813f8 100644 --- a/lfs.c +++ b/lfs.c @@ -388,18 +388,18 @@ static inline void lfs_superblock_fromle32(lfs_superblock_t *superblock) { superblock->version = lfs_fromle32(superblock->version); superblock->block_size = lfs_fromle32(superblock->block_size); superblock->block_count = lfs_fromle32(superblock->block_count); + superblock->name_max = lfs_fromle32(superblock->name_max); superblock->inline_max = lfs_fromle32(superblock->inline_max); superblock->attr_max = lfs_fromle32(superblock->attr_max); - superblock->name_max = lfs_fromle32(superblock->name_max); } static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { superblock->version = lfs_tole32(superblock->version); superblock->block_size = lfs_tole32(superblock->block_size); superblock->block_count = lfs_tole32(superblock->block_count); + superblock->name_max = lfs_tole32(superblock->name_max); superblock->inline_max = lfs_tole32(superblock->inline_max); superblock->attr_max = lfs_tole32(superblock->attr_max); - superblock->name_max = lfs_tole32(superblock->name_max); } @@ -3054,6 +3054,12 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { } // check that the size limits are sane + LFS_ASSERT(lfs->cfg->name_max <= LFS_NAME_MAX); + lfs->name_max = lfs->cfg->name_max; + if (!lfs->name_max) { + lfs->name_max = LFS_NAME_MAX; + } + LFS_ASSERT(lfs->cfg->inline_max <= LFS_INLINE_MAX); LFS_ASSERT(lfs->cfg->inline_max <= lfs->cfg->cache_size); lfs->inline_max = lfs->cfg->inline_max; @@ -3067,12 +3073,6 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->attr_max = LFS_ATTR_MAX; } - LFS_ASSERT(lfs->cfg->name_max <= LFS_NAME_MAX); - lfs->name_max = lfs->cfg->name_max; - if (!lfs->name_max) { - lfs->name_max = LFS_NAME_MAX; - } - // setup default state lfs->root[0] = 0xffffffff; lfs->root[1] = 0xffffffff; @@ -3132,9 +3132,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .block_size = lfs->cfg->block_size, .block_count = lfs->cfg->block_count, - .attr_max = lfs->attr_max, .name_max = lfs->name_max, .inline_max = lfs->inline_max, + .attr_max = lfs->attr_max, }; lfs_superblock_tole32(&superblock); @@ -3204,17 +3204,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } // check superblock configuration - if (superblock.attr_max) { - if (superblock.attr_max > lfs->attr_max) { - LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", - superblock.attr_max, lfs->attr_max); - err = LFS_ERR_INVAL; - goto cleanup; - } - - lfs->attr_max = superblock.attr_max; - } - if (superblock.name_max) { if (superblock.name_max > lfs->name_max) { LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", @@ -3236,6 +3225,18 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->inline_max = superblock.inline_max; } + + if (superblock.attr_max) { + if (superblock.attr_max > lfs->attr_max) { + LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", + superblock.attr_max, lfs->attr_max); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->attr_max = superblock.attr_max; + } + } // has globals? diff --git a/lfs.h b/lfs.h index edb70265..f4020fa5 100644 --- a/lfs.h +++ b/lfs.h @@ -44,29 +44,28 @@ typedef int32_t lfs_soff_t; typedef uint32_t lfs_block_t; -// Maximum size of all attributes per file in bytes, may be redefined but a -// a smaller LFS_ATTR_MAX has no benefit. Stored in 12-bits and limited -// to <= 0xfff. Stored in superblock and must be respected by other -// littlefs drivers. -#ifndef LFS_ATTR_MAX -#define LFS_ATTR_MAX 0x1ffe -#endif - // Maximum name size in bytes, may be redefined to reduce the size of the -// info struct. Limited to <= LFS_ATTR_MAX. Stored in superblock and must -// be respected by other littlefs drivers. +// info struct. Limited to <= 8190. Stored in superblock and must be +// respected by other littlefs drivers. #ifndef LFS_NAME_MAX #define LFS_NAME_MAX 0xff #endif -// Maximum inline file size in bytes. Large inline files require a larger -// cache size, but if a file can be inline it does not need its own data -// block. Limited to <= LFS_ATTR_MAX and <= cache_size. Stored in superblock -// and must be respected by other littlefs drivers. +// Maximum inline file size in bytes, may be redefined to limit RAM usage, +// but littlefs will automatically limit the LFS_INLINE_MAX to the +// configured cache_size. Limited to <= 8190. Stored in superblock and must +// be respected by other littlefs drivers. #ifndef LFS_INLINE_MAX #define LFS_INLINE_MAX 0x1ffe #endif +// Maximum size of custom attributes in bytes, may be redefined, but there is +// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 8190. Stored +// in superblock and must be respected by other littlefs drivers. +#ifndef LFS_ATTR_MAX +#define LFS_ATTR_MAX 0x1ffe +#endif + // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { @@ -213,24 +212,25 @@ struct lfs_config { // lookahead block. void *lookahead_buffer; - // Optional upper limit on file attributes in bytes. No downside for larger - // attributes size but must be less than LFS_ATTR_MAX. Defaults to - // LFS_ATTR_MAX when zero.Stored in superblock and must be respected by - // other littlefs drivers. - lfs_size_t attr_max; - // Optional upper limit on length of file names in bytes. No downside for // larger names except the size of the info struct which is controlled by // the LFS_NAME_MAX define. Defaults to LFS_NAME_MAX when zero. Stored in // superblock and must be respected by other littlefs drivers. lfs_size_t name_max; - // Optional upper limit on inlined files in bytes. Large inline files - // require a larger cache size, but if a file can be inlined it does not - // need its own data block. Must be smaller than cache_size and less than - // LFS_INLINE_MAX. Defaults to min(LFS_INLINE_MAX, read_size) when zero. - // Stored in superblock and must be respected by other littlefs drivers. + // Optional upper limit on inlined files in bytes. Inline files must be + // backed by RAM, but if a file fits in RAM it can be inlined into its + // directory block without needing its own data block. Must be <= + // cache_size and LFS_INLINE_MAX. Defaults to min(LFS_INLINE_MAX, + // cache_size) when zero. Stored in superblock and must be respected by + // other littlefs drivers. lfs_size_t inline_max; + + // Optional upper limit on custom attributes in bytes. No downside for + // larger attributes size but must be <= LFS_ATTR_MAX. Defaults to + // LFS_ATTR_MAX when zero. Stored in superblock and must be respected by + // other littlefs drivers. + lfs_size_t attr_max; }; // File info structure @@ -238,10 +238,13 @@ struct lfs_info { // Type of the file, either LFS_TYPE_REG or LFS_TYPE_DIR uint8_t type; - // Size of the file, only valid for REG files + // Size of the file, only valid for REG files. Limited to 32-bits. lfs_size_t size; - // Name of the file stored as a null-terminated string + // Name of the file stored as a null-terminated string. Limited to + // LFS_NAME_MAX+1, which can be changed by redefining LFS_NAME_MAX to + // reduce RAM. LFS_NAME_MAX is stored in superblock and must be + // respected by other littlefs drivers. char name[LFS_NAME_MAX+1]; }; @@ -340,9 +343,9 @@ typedef struct lfs_superblock { lfs_size_t block_size; lfs_size_t block_count; - lfs_size_t attr_max; lfs_size_t name_max; lfs_size_t inline_max; + lfs_size_t attr_max; } lfs_superblock_t; // The littlefs filesystem type @@ -377,9 +380,9 @@ typedef struct lfs { const struct lfs_config *cfg; lfs_size_t block_size; lfs_size_t block_count; - lfs_size_t attr_max; lfs_size_t name_max; lfs_size_t inline_max; + lfs_size_t attr_max; } lfs_t; From 7af8b81b81a7e294b902a859b73b9400e771b114 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 2 Oct 2018 20:18:30 -0500 Subject: [PATCH 111/139] Changed lookahead configuration unit to bytes instead of bits The fact that the lookahead buffer uses bits instead of bytes is an internal detail. Poking this through to the user API has caused a decent amount of confusion. Most buffers are provided as bytes and the inconsistency here can be surprising. The use of bytes instead of bits also makes us forward compatible in the case that we want to change the lookahead internal representation (hint segment list). Additionally, we change the configuration name to lookahead_size. This matches other configurations, such as cache_size and read_size, while also notifying the user that something important changed at compile time (by breaking). --- .travis.yml | 2 +- README.md | 3 ++- lfs.c | 15 ++++++++------- lfs.h | 40 +++++++++++++++++++++------------------- tests/template.fmt | 18 +++++++++--------- 5 files changed, 41 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a47a56b..e3f786b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ script: # run tests with a few different configurations - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_CACHE_SIZE=4" - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_CACHE_SIZE=512 -DLFS_BLOCK_CYCLES=16" - - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048" + - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD_SIZE=256" - make clean test QUIET=1 CFLAGS+="-DLFS_INLINE_MAX=0" - make clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS" diff --git a/README.md b/README.md index 623ba0ae..45be6f4b 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,8 @@ const struct lfs_config cfg = { .prog_size = 16, .block_size = 4096, .block_count = 128, - .lookahead = 128, + .cache_size = 16, + .lookahead_size = 16, }; // entry point diff --git a/lfs.c b/lfs.c index 6f7813f8..8759f834 100644 --- a/lfs.c +++ b/lfs.c @@ -462,11 +462,11 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { lfs->free.off = (lfs->free.off + lfs->free.size) % lfs->cfg->block_count; - lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->free.ack); + lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); lfs->free.i = 0; // find mask of free blocks from tree - memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); + memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); int err = lfs_fs_traverse(lfs, lfs_alloc_lookahead, lfs); if (err) { return err; @@ -3041,12 +3041,12 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs_cache_zero(lfs, &lfs->pcache); // setup lookahead, must be multiple of 32-bits - LFS_ASSERT(lfs->cfg->lookahead % 32 == 0); - LFS_ASSERT(lfs->cfg->lookahead > 0); + LFS_ASSERT(lfs->cfg->lookahead_size % 4 == 0); + LFS_ASSERT(lfs->cfg->lookahead_size > 0); if (lfs->cfg->lookahead_buffer) { lfs->free.buffer = lfs->cfg->lookahead_buffer; } else { - lfs->free.buffer = lfs_malloc(lfs->cfg->lookahead/8); + lfs->free.buffer = lfs_malloc(lfs->cfg->lookahead_size); if (!lfs->free.buffer) { err = LFS_ERR_NOMEM; goto cleanup; @@ -3112,9 +3112,10 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { } // create free lookahead - memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); + memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); lfs->free.off = 0; - lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->cfg->block_count); + lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, + lfs->cfg->block_count); lfs->free.i = 0; lfs_alloc_ack(lfs); diff --git a/lfs.h b/lfs.h index f4020fa5..dae59edb 100644 --- a/lfs.h +++ b/lfs.h @@ -177,39 +177,41 @@ struct lfs_config { lfs_size_t prog_size; // Size of an erasable block. This does not impact ram consumption and - // may be larger than the physical erase size. However, this should be - // kept small as each file currently takes up an entire block. - // Must be a multiple of the read, program, and cache sizes. + // may be larger than the physical erase size. However, non-inlined files + // take up at minimum one block. Must be a multiple of the read + // and program sizes. lfs_size_t block_size; // Number of erasable blocks on the device. lfs_size_t block_count; // Number of erase cycles before we should move data to another block. - // May be zero to never move data, in which case no block-level - // wear-leveling is performed. + // May be zero, in which case no block-level wear-leveling is performed. uint32_t block_cycles; // Size of block caches. Each cache buffers a portion of a block in RAM. - // This determines the size of the read cache, the program cache, and a + // The littlefs needs a read cache, a program cache, and one additional // cache per file. Larger caches can improve performance by storing more - // data. Must be a multiple of the read and program sizes. + // data and reducing the number of disk accesses. Must be a multiple of + // the read and program sizes, and a factor of the block size. lfs_size_t cache_size; - // Number of blocks to lookahead during block allocation. A larger - // lookahead reduces the number of passes required to allocate a block. - // The lookahead buffer requires only 1 bit per block so it can be quite - // large with little ram impact. Should be a multiple of 32. - lfs_size_t lookahead; + // Size of the lookahead buffer in bytes. A larger lookahead buffer + // increases the number of blocks found during an allocation pass. The + // lookahead buffer is stored as a compact bitmap, so each byte of RAM + // can track 8 blocks. Must be a multiple of 4. + lfs_size_t lookahead_size; - // Optional, statically allocated read buffer. Must be read sized. + // Optional statically allocated read buffer. Must be cache_size. + // By default lfs_malloc is used to allocate this buffer. void *read_buffer; - // Optional, statically allocated program buffer. Must be program sized. + // Optional statically allocated program buffer. Must be cache_size. + // By default lfs_malloc is used to allocate this buffer. void *prog_buffer; - // Optional, statically allocated lookahead buffer. Must be 1 bit per - // lookahead block. + // Optional statically allocated program buffer. Must be lookahead_size. + // By default lfs_malloc is used to allocate this buffer. void *lookahead_buffer; // Optional upper limit on length of file names in bytes. No downside for @@ -266,11 +268,11 @@ struct lfs_attr { // Optional configuration provided during lfs_file_opencfg struct lfs_file_config { - // Optional, statically allocated buffer for files. Must be program sized. - // If NULL, malloc will be used by default. + // Optional statically allocated file buffer. Must be cache_size. + // By default lfs_malloc is used to allocate this buffer. void *buffer; - // Optional, linked list of custom attributes related to the file. If the + // Optional linked list of custom attributes related to the file. If the // file is opened with read access, the attributes will be read from // during the open call. If the file is opened with write access, the // attributes will be written to disk every file sync or close. This diff --git a/tests/template.fmt b/tests/template.fmt index 85759db6..7fdec7c9 100644 --- a/tests/template.fmt +++ b/tests/template.fmt @@ -85,8 +85,8 @@ uintmax_t test; #define LFS_CACHE_SIZE 64 #endif -#ifndef LFS_LOOKAHEAD -#define LFS_LOOKAHEAD 128 +#ifndef LFS_LOOKAHEAD_SIZE +#define LFS_LOOKAHEAD_SIZE 16 #endif const struct lfs_config cfg = {{ @@ -96,13 +96,13 @@ const struct lfs_config cfg = {{ .erase = &lfs_emubd_erase, .sync = &lfs_emubd_sync, - .read_size = LFS_READ_SIZE, - .prog_size = LFS_PROG_SIZE, - .block_size = LFS_BLOCK_SIZE, - .block_count = LFS_BLOCK_COUNT, - .block_cycles = LFS_BLOCK_CYCLES, - .cache_size = LFS_CACHE_SIZE, - .lookahead = LFS_LOOKAHEAD, + .read_size = LFS_READ_SIZE, + .prog_size = LFS_PROG_SIZE, + .block_size = LFS_BLOCK_SIZE, + .block_count = LFS_BLOCK_COUNT, + .block_cycles = LFS_BLOCK_CYCLES, + .cache_size = LFS_CACHE_SIZE, + .lookahead_size = LFS_LOOKAHEAD_SIZE, }}; From aeca7667b32097c109b4ce84a2f3adb2949643ab Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 4 Oct 2018 14:49:34 -0500 Subject: [PATCH 112/139] Switched to strongly ordered directories Instead of storing files in an arbitrary order, we now store files in ascending lexicographical order by filename. Although a big change, this actually has little impact on how littlefs works internally. We need to support file insertion, and compare file names to find our position. But since we already need to scan the entire directory block, this adds relatively little overhead. What this does allow, is the potential to add B-tree support in the future in a backwards compatible manner. How could you add B-trees to littlefs? 1. Add an optional "child" tag with a pointer that allows you to skip to a position in the metadata-pair list that composes the directory 2. When splitting a metadata-pair (sound familiar?), we either insert a second child tag in our parent, or we create a new root containing the child tags. 3. Each layer needs a bit stored in the tail-pointer to indicate if we're going to the next layer. This can be created trivially when we create a new root. 4. During lookup we keep two pointers containing the bounds of our search. We may need to iterate through multiple metadata-pairs in our linked-list, but this gives us a O(log n) lookup cost in a balanced tree. 5. During deletion we also delete any children pointers. Note that children pointers must come before the actual file entry. This gives us a B-tree implementation that is compatible with the current directory layout (assuming the files are ordered). This means that B-trees could be supported by a host PC and ignored on a small device. And during power-loss, we never end up with a broken filesystem, just a less-than-optimal tree. Note that we don't handle removes, so it's possible for a tree to become unbalanced. But worst case that's the same as the current linked-list implementation. All we need to do now is keep directories ordered. If we decide to drop B-tree support in the future or the B-tree implementation turns out inherently flawed, we can just drop the ordered requirement without breaking compatibility and recover the code cost. --- lfs.c | 242 ++++++++++++++++++++++---------------------- lfs.h | 2 +- tests/test_dirs.sh | 34 +++---- tests/test_files.sh | 14 +-- tests/test_move.sh | 8 +- tests/test_seek.sh | 30 +++--- 6 files changed, 165 insertions(+), 165 deletions(-) diff --git a/lfs.c b/lfs.c index 8759f834..9ba52f78 100644 --- a/lfs.c +++ b/lfs.c @@ -128,11 +128,11 @@ static int lfs_bd_cmp(lfs_t *lfs, } if (dat != data[i]) { - return false; + return (dat < data[i]) ? 1 : 2; } } - return true; + return 0; } static int lfs_bd_flush(lfs_t *lfs, @@ -152,12 +152,8 @@ static int lfs_bd_flush(lfs_t *lfs, int res = lfs_bd_cmp(lfs, NULL, rcache, diff, pcache->block, pcache->off, pcache->buffer, diff); - if (res < 0) { - return res; - } - - if (!res) { - return LFS_ERR_CORRUPT; + if (res) { + return (res < 0) ? res : LFS_ERR_CORRUPT; } } @@ -688,7 +684,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, LFS_ASSERT(temp.count > 0); temp.count -= 1; - if (lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { + if (tempfoundtag && + lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { tempfoundtag = 0; } else if (tempfoundtag && lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { @@ -724,11 +721,10 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, return res; } - if (res) { - tempfoundtag = tag; + if (res && (!tempfoundtag || + lfs_tag_id(res) <= lfs_tag_id(tempfoundtag))) { + tempfoundtag = res; } - } else { - tempfoundtag = tag; } } } @@ -740,12 +736,12 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // consider what we have good enough if (dir->off > 0) { // synthetic move - if (lfs->globals.hasmove && + if (foundtag && + lfs->globals.hasmove && lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0) { if (lfs->globals.id == lfs_tag_id(foundtag)) { foundtag = 0; - } else if (foundtag && - lfs->globals.id < lfs_tag_id(foundtag)) { + } else if (lfs->globals.id < lfs_tag_id(foundtag)) { foundtag -= LFS_MKTAG(0, 1, 0); } } @@ -876,24 +872,6 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, return 0; } -static lfs_stag_t lfs_dir_findmatch(lfs_t *lfs, - lfs_mdir_t *dir, const lfs_block_t pair[2], bool fs, - lfs_tag_t findmask, lfs_tag_t findtag, - int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { - dir->split = true; - dir->tail[0] = pair[0]; - dir->tail[1] = pair[1]; - while ((dir->split || fs) && !lfs_pair_isnull(dir->tail)) { - lfs_stag_t tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, - findmask, findtag, cb, data); - if (tag) { - return tag; - } - } - - return LFS_ERR_NOENT; -} - struct lfs_dir_find_match { lfs_t *lfs; const void *name; @@ -905,22 +883,39 @@ static int lfs_dir_find_match(void *data, struct lfs_dir_find_match *name = data; lfs_t *lfs = name->lfs; const struct lfs_diskoff *disk = buffer; - (void)tag; - return lfs_bd_cmp(lfs, - NULL, &lfs->rcache, name->size, - disk->block, disk->off, name->name, name->size); + lfs_size_t diff = lfs_min(name->size, lfs_tag_size(tag)); + int res = lfs_bd_cmp(lfs, + NULL, &lfs->rcache, diff, + disk->block, disk->off, name->name, diff); + if (res < 0) { + return res; + } + + // found match? + if (res == 0 && (name->size == lfs_tag_size(tag) || + lfs_tag_type(tag) == LFS_TYPE_SUPERBLOCK)) { + return tag; + } + + // a greater name found, exit early + if (res == 2 && lfs_tag_type(tag) != LFS_TYPE_SUPERBLOCK) { + return tag | 0x1fff; + } + + // no match keep looking + return 0; } -static lfs_stag_t lfs_dir_find(lfs_t *lfs, - lfs_mdir_t *dir, const char **path) { +static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, + const char **path, uint16_t *id) { // we reduce path to a single name if we can find it const char *name = *path; - *path = NULL; // default to root dir lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x1ff, 0); - lfs_block_t pair[2] = {lfs->root[0], lfs->root[1]}; + dir->tail[0] = lfs->root[0]; + dir->tail[1] = lfs->root[1]; while (true) { nextname: @@ -964,10 +959,8 @@ static lfs_stag_t lfs_dir_find(lfs_t *lfs, return tag; } - // update what we've found if path is only a name - if (strchr(name, '/') == NULL) { - *path = name; - } + // update what we've found so far + *path = name; // only continue if we hit a directory if (lfs_tag_type(tag) != LFS_TYPE_DIR) { @@ -977,20 +970,42 @@ static lfs_stag_t lfs_dir_find(lfs_t *lfs, // grab the entry data if (lfs_tag_id(tag) != 0x1ff) { lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c3fe000, - LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); + LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), dir->tail); if (res < 0) { return res; } - lfs_pair_fromle32(pair); + lfs_pair_fromle32(dir->tail); } // find entry matching name - tag = lfs_dir_findmatch(lfs, dir, pair, false, 0x7c001fff, - LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), - lfs_dir_find_match, &(struct lfs_dir_find_match){ - lfs, name, namelen}); - if (tag < 0) { - return tag; + while (true) { + tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, + 0x7c000000, LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, name, namelen}); + if (tag < 0) { + return tag; + } + + if (id) { + if (strchr(name, '/') != NULL) { + // if path is not only name we're not valid candidate + // for creation + *id = 0x1ff; + } else if (tag) { + *id = lfs_tag_id(tag); + } else { + *id = dir->count; + } + } + + if (tag && !lfs_tag_isdelete(tag)) { + break; + } + + if (lfs_tag_isdelete(tag) || !dir->split) { + return LFS_ERR_NOENT; + } } // to next name @@ -1560,10 +1575,12 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // calculate new directory size lfs_tag_t deletetag = 0xffffffff; + lfs_tag_t createtag = 0xffffffff; int attrcount = 0; for (const struct lfs_mattr *a = attrs; a; a = a->next) { if (lfs_tag_subtype(a->tag) == LFS_TYPE_NAME) { dir->count += 1; + createtag = a->tag; } else if (lfs_tag_subtype(a->tag) == LFS_TYPE_DELETE) { LFS_ASSERT(dir->count > 0); dir->count -= 1; @@ -1685,6 +1702,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, if (d->type == LFS_TYPE_DIR) { ((lfs_dir_t*)d)->pos -= 1; } + } else if (&d->m != dir && d->id >= lfs_tag_id(createtag)) { + d->id += 1; + if (d->type == LFS_TYPE_DIR) { + ((lfs_dir_t*)d)->pos += 1; + } } while (d->id >= d->m.count && d->m.split) { @@ -1711,9 +1733,10 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } lfs_mdir_t cwd; - lfs_stag_t res = lfs_dir_find(lfs, &cwd, &path); - if (!(res == LFS_ERR_NOENT && path)) { - return (res < 0) ? res : LFS_ERR_EXIST; + uint16_t id; + err = lfs_dir_find(lfs, &cwd, &path, &id); + if (!(err == LFS_ERR_NOENT && id != 0x1ff)) { + return (err < 0) ? err : LFS_ERR_EXIST; } // check that name fits @@ -1739,7 +1762,6 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } // get next slot and commit - uint16_t id = cwd.count; cwd.tail[0] = dir.pair[0]; cwd.tail[1] = dir.pair[1]; lfs_pair_tole32(dir.pair); @@ -1757,7 +1779,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { - lfs_stag_t tag = lfs_dir_find(lfs, &dir->m, &path); + lfs_stag_t tag = lfs_dir_find(lfs, &dir->m, &path, NULL); if (tag < 0) { return tag; } @@ -2114,21 +2136,20 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } // setup simple file details - int err = 0; + int err; file->cfg = cfg; file->flags = flags; file->pos = 0; file->cache.buffer = NULL; // allocate entry for file if it doesn't exist - lfs_stag_t tag = lfs_dir_find(lfs, &file->m, &path); - if (tag < 0 && !(tag == LFS_ERR_NOENT && path)) { + lfs_stag_t tag = lfs_dir_find(lfs, &file->m, &path, &file->id); + if (tag < 0 && !(tag == LFS_ERR_NOENT && file->id != 0x1ff)) { err = tag; goto cleanup; } // get id, add to list of mdirs to catch update changes - file->id = lfs_tag_id(tag); file->type = LFS_TYPE_REG; file->next = (lfs_file_t*)lfs->mlist; lfs->mlist = (struct lfs_mlist*)file; @@ -2147,7 +2168,6 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } // get next slot and create entry to remember name - file->id = file->m.count; err = lfs_dir_commit(lfs, &file->m, LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, NULL, 0, LFS_MKATTR(LFS_TYPE_REG, file->id, path, nlen, @@ -2755,7 +2775,7 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { lfs_mdir_t cwd; - lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path); + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); if (tag < 0) { return tag; } @@ -2771,12 +2791,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } lfs_mdir_t cwd; - err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path); + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); if (tag < 0) { return tag; } @@ -2840,20 +2855,19 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find old entry lfs_mdir_t oldcwd; - lfs_stag_t oldtag = lfs_dir_find(lfs, &oldcwd, &oldpath); + lfs_stag_t oldtag = lfs_dir_find(lfs, &oldcwd, &oldpath, NULL); if (oldtag < 0) { return oldtag; } // find new entry lfs_mdir_t newcwd; - lfs_stag_t prevtag = lfs_dir_find(lfs, &newcwd, &newpath); - if (prevtag < 0 && prevtag != LFS_ERR_NOENT) { - return prevtag; + uint16_t newid; + lfs_stag_t prevtag = lfs_dir_find(lfs, &newcwd, &newpath, &newid); + if (prevtag < 0 && !(prevtag == LFS_ERR_NOENT && newid != 0x1ff)) { + return err; } - uint16_t newid = lfs_tag_id(prevtag); - lfs_mdir_t prevdir; if (prevtag == LFS_ERR_NOENT) { // check that name fits @@ -2861,9 +2875,6 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (nlen > lfs->name_max) { return LFS_ERR_NAMETOOLONG; } - - // get next id - newid = newcwd.count; } else if (lfs_tag_type(prevtag) != lfs_tag_type(oldtag)) { return LFS_ERR_ISDIR; } else if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { @@ -2934,12 +2945,12 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, uint8_t type, void *buffer, lfs_size_t size) { lfs_mdir_t cwd; - lfs_stag_t res = lfs_dir_find(lfs, &cwd, &path); - if (res < 0) { - return res; + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); + if (tag < 0) { + return tag; } - uint16_t id = lfs_tag_id(res); + uint16_t id = lfs_tag_id(tag); if (id == 0x1ff) { // special case for root id = 0; @@ -2949,28 +2960,28 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, } } - res = lfs_dir_get(lfs, &cwd, 0x7fffe000, + tag = lfs_dir_get(lfs, &cwd, 0x7fffe000, LFS_MKTAG(0x100 | type, id, lfs_min(size, lfs->attr_max)), buffer); - if (res < 0) { - if (res == LFS_ERR_NOENT) { + if (tag < 0) { + if (tag == LFS_ERR_NOENT) { return LFS_ERR_NOATTR; } - return res; + return tag; } - return lfs_tag_size(res); + return lfs_tag_size(tag); } static int lfs_commitattr(lfs_t *lfs, const char *path, uint8_t type, const void *buffer, lfs_size_t size) { lfs_mdir_t cwd; - lfs_stag_t res = lfs_dir_find(lfs, &cwd, &path); - if (res < 0) { - return res; + lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); + if (tag < 0) { + return tag; } - uint16_t id = lfs_tag_id(res); + uint16_t id = lfs_tag_id(tag); if (id == 0x1ff) { // special case for root id = 0; @@ -3167,28 +3178,28 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pair_isnull(dir.tail)) { // fetch next block in tail list - lfs_stag_t res = lfs_dir_fetchmatch(lfs, &dir, dir.tail, 0x7fc00000, + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, 0x7fc00000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), lfs_dir_find_match, &(struct lfs_dir_find_match){ lfs, "littlefs", 8}); - if (res < 0) { - err = res; + if (tag < 0) { + err = tag; goto cleanup; } // has superblock? - if (res) { + if (tag && !lfs_tag_isdelete(tag)) { // update root lfs->root[0] = dir.pair[0]; lfs->root[1] = dir.pair[1]; // grab superblock lfs_superblock_t superblock; - res = lfs_dir_get(lfs, &dir, 0x7f800000, + tag = lfs_dir_get(lfs, &dir, 0x7f800000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), &superblock); - if (res < 0) { - err = res; + if (tag < 0) { + err = tag; goto cleanup; } lfs_superblock_fromle32(&superblock); @@ -3277,10 +3288,6 @@ int lfs_unmount(lfs_t *lfs) { /// Filesystem filesystem operations /// int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *data, lfs_block_t block), void *data) { - if (lfs_pair_isnull(lfs->root)) { - return 0; - } - // iterate over metadata pairs lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pair_isnull(dir.tail)) { @@ -3347,10 +3354,6 @@ int lfs_fs_traverse(lfs_t *lfs, static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *pdir) { - if (lfs_pair_isnull(lfs->root)) { - return LFS_ERR_NOENT; - } - // iterate over all directory directory entries pdir->tail[0] = 0; pdir->tail[1] = 1; @@ -3389,23 +3392,20 @@ static int lfs_fs_parent_match(void *data, } lfs_pair_fromle32(child); - return (lfs_pair_cmp(child, find->pair) == 0); + return (lfs_pair_cmp(child, find->pair) == 0) ? tag : 0; } static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], lfs_mdir_t *parent) { - if (lfs_pair_isnull(lfs->root)) { - return LFS_ERR_NOENT; - } - - // search for both orderings so we can reuse the find function - for (int i = 0; i < 2; i++) { - struct lfs_fs_parent_match match = {lfs, {pair[0], pair[1]}}; - lfs_stag_t tag = lfs_dir_findmatch(lfs, parent, - (const lfs_block_t[2]){0, 1}, true, 0x7fc01fff, - LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), - lfs_fs_parent_match, &match); - if (tag != LFS_ERR_NOENT) { + // use fetchmatch with callback to find pairs + parent->tail[0] = 0; + parent->tail[1] = 1; + while (!lfs_pair_isnull(parent->tail)) { + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, parent, parent->tail, + 0x7fc01fff, LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), + lfs_fs_parent_match, &(struct lfs_fs_parent_match){ + lfs, {pair[0], pair[1]}}); + if (tag) { return tag; } } diff --git a/lfs.h b/lfs.h index dae59edb..0dd1c2f7 100644 --- a/lfs.h +++ b/lfs.h @@ -92,7 +92,7 @@ enum lfs_type { LFS_TYPE_DIR = 0x003, // internally used types - LFS_TYPE_USER = 0x100, + LFS_TYPE_USERATTR = 0x100, LFS_TYPE_NAME = 0x000, LFS_TYPE_DELETE = 0x020, LFS_TYPE_STRUCT = 0x040, diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 53d76f7a..4aac9459 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -43,11 +43,11 @@ tests/test.py << TEST strcmp(info.name, "..") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "potato") => 0; - info.type => LFS_TYPE_DIR; - lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "burito") => 0; info.type => LFS_TYPE_REG; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "potato") => 0; + info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; lfs_unmount(&lfs) => 0; @@ -85,10 +85,10 @@ tests/test.py << TEST strcmp(info.name, "baked") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "sweet") => 0; + strcmp(info.name, "fried") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "fried") => 0; + strcmp(info.name, "sweet") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; @@ -100,7 +100,7 @@ tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_mkdir(&lfs, "cactus") => 0; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "cactus/test%d", i); + sprintf((char*)buffer, "cactus/test%03d", i); lfs_mkdir(&lfs, (char*)buffer) => 0; } lfs_unmount(&lfs) => 0; @@ -115,7 +115,7 @@ tests/test.py << TEST strcmp(info.name, "..") => 0; info.type => LFS_TYPE_DIR; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "test%d", i); + sprintf((char*)buffer, "test%03d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, (char*)buffer) => 0; info.type => LFS_TYPE_DIR; @@ -208,10 +208,10 @@ tests/test.py << TEST strcmp(info.name, "baked") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "sweet") => 0; + strcmp(info.name, "fried") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "fried") => 0; + strcmp(info.name, "sweet") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; @@ -241,10 +241,10 @@ tests/test.py << TEST strcmp(info.name, "baked") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "sweet") => 0; + strcmp(info.name, "fried") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "fried") => 0; + strcmp(info.name, "sweet") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; @@ -273,10 +273,10 @@ tests/test.py << TEST strcmp(info.name, "baked") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "sweet") => 0; + strcmp(info.name, "fried") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "fried") => 0; + strcmp(info.name, "sweet") => 0; info.type => LFS_TYPE_DIR; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; @@ -332,7 +332,7 @@ tests/test.py << TEST lfs_remove(&lfs, "cactus") => LFS_ERR_NOTEMPTY; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "cactus/test%d", i); + sprintf((char*)buffer, "cactus/test%03d", i); lfs_remove(&lfs, (char*)buffer) => 0; } @@ -361,7 +361,7 @@ tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_mkdir(&lfs, "prickly-pear") => 0; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "prickly-pear/test%d", i); + sprintf((char*)buffer, "prickly-pear/test%03d", i); lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_WRONLY | LFS_O_CREAT) => 0; size = 6; @@ -381,7 +381,7 @@ tests/test.py << TEST strcmp(info.name, "..") => 0; info.type => LFS_TYPE_DIR; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "test%d", i); + sprintf((char*)buffer, "test%03d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, (char*)buffer) => 0; info.type => LFS_TYPE_REG; @@ -397,7 +397,7 @@ tests/test.py << TEST lfs_remove(&lfs, "prickly-pear") => LFS_ERR_NOTEMPTY; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "prickly-pear/test%d", i); + sprintf((char*)buffer, "prickly-pear/test%03d", i); lfs_remove(&lfs, (char*)buffer) => 0; } diff --git a/tests/test_files.sh b/tests/test_files.sh index bbecea92..5251c61f 100755 --- a/tests/test_files.sh +++ b/tests/test_files.sh @@ -29,7 +29,7 @@ tests/test.py << TEST TEST w_test() { -tests/test.py << TEST +tests/test.py ${4:-} << TEST size = $1; lfs_size_t chunk = 31; srand(0); @@ -115,21 +115,21 @@ tests/test.py << TEST info.type => LFS_TYPE_REG; info.size => strlen("Hello World!\n"); lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "smallavacado") => 0; + strcmp(info.name, "largeavacado") => 0; info.type => LFS_TYPE_REG; - info.size => $SMALLSIZE; + info.size => $LARGESIZE; lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "mediumavacado") => 0; info.type => LFS_TYPE_REG; info.size => $MEDIUMSIZE; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "largeavacado") => 0; - info.type => LFS_TYPE_REG; - info.size => $LARGESIZE; - lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "noavacado") => 0; info.type => LFS_TYPE_REG; info.size => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "smallavacado") => 0; + info.type => LFS_TYPE_REG; + info.size => $SMALLSIZE; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; lfs_unmount(&lfs) => 0; diff --git a/tests/test_move.sh b/tests/test_move.sh index 53f7e156..458ca1ed 100755 --- a/tests/test_move.sh +++ b/tests/test_move.sh @@ -257,10 +257,10 @@ tests/test.py << TEST lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "..") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "hola") => 0; - lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "bonjour") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "hola") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "ohayo") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; @@ -303,10 +303,10 @@ tests/test.py << TEST lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "..") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; - strcmp(info.name, "hola") => 0; - lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "bonjour") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "hola") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, "ohayo") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 0; lfs_dir_close(&lfs, &dir[0]) => 0; diff --git a/tests/test_seek.sh b/tests/test_seek.sh index 0084d42f..609c250e 100755 --- a/tests/test_seek.sh +++ b/tests/test_seek.sh @@ -12,7 +12,7 @@ tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_mkdir(&lfs, "hello") => 0; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "hello/kitty%d", i); + sprintf((char*)buffer, "hello/kitty%03d", i); lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; @@ -39,7 +39,7 @@ tests/test.py << TEST lfs_soff_t pos; int i; for (i = 0; i < $SMALLSIZE; i++) { - sprintf((char*)buffer, "kitty%d", i); + sprintf((char*)buffer, "kitty%03d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, (char*)buffer) => 0; pos = lfs_dir_tell(&lfs, &dir[0]); @@ -47,12 +47,12 @@ tests/test.py << TEST pos >= 0 => 1; lfs_dir_seek(&lfs, &dir[0], pos) => 0; - sprintf((char*)buffer, "kitty%d", i); + sprintf((char*)buffer, "kitty%03d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, (char*)buffer) => 0; lfs_dir_rewind(&lfs, &dir[0]) => 0; - sprintf((char*)buffer, "kitty%d", 0); + sprintf((char*)buffer, "kitty%03d", 0); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, ".") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; @@ -61,7 +61,7 @@ tests/test.py << TEST strcmp(info.name, (char*)buffer) => 0; lfs_dir_seek(&lfs, &dir[0], pos) => 0; - sprintf((char*)buffer, "kitty%d", i); + sprintf((char*)buffer, "kitty%03d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, (char*)buffer) => 0; @@ -81,7 +81,7 @@ tests/test.py << TEST lfs_soff_t pos; int i; for (i = 0; i < $MEDIUMSIZE; i++) { - sprintf((char*)buffer, "kitty%d", i); + sprintf((char*)buffer, "kitty%03d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, (char*)buffer) => 0; pos = lfs_dir_tell(&lfs, &dir[0]); @@ -89,12 +89,12 @@ tests/test.py << TEST pos >= 0 => 1; lfs_dir_seek(&lfs, &dir[0], pos) => 0; - sprintf((char*)buffer, "kitty%d", i); + sprintf((char*)buffer, "kitty%03d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, (char*)buffer) => 0; lfs_dir_rewind(&lfs, &dir[0]) => 0; - sprintf((char*)buffer, "kitty%d", 0); + sprintf((char*)buffer, "kitty%03d", 0); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, ".") => 0; lfs_dir_read(&lfs, &dir[0], &info) => 1; @@ -103,7 +103,7 @@ tests/test.py << TEST strcmp(info.name, (char*)buffer) => 0; lfs_dir_seek(&lfs, &dir[0], pos) => 0; - sprintf((char*)buffer, "kitty%d", i); + sprintf((char*)buffer, "kitty%03d", i); lfs_dir_read(&lfs, &dir[0], &info) => 1; strcmp(info.name, (char*)buffer) => 0; @@ -114,7 +114,7 @@ TEST echo "--- Simple file seek ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDONLY) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty042", LFS_O_RDONLY) => 0; lfs_soff_t pos; size = strlen("kittycatcat"); @@ -163,7 +163,7 @@ TEST echo "--- Large file seek ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDONLY) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty042", LFS_O_RDONLY) => 0; lfs_soff_t pos; size = strlen("kittycatcat"); @@ -212,7 +212,7 @@ TEST echo "--- Simple file seek and write ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDWR) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty042", LFS_O_RDWR) => 0; lfs_soff_t pos; size = strlen("kittycatcat"); @@ -253,7 +253,7 @@ TEST echo "--- Large file seek and write ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDWR) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty042", LFS_O_RDWR) => 0; lfs_soff_t pos; size = strlen("kittycatcat"); @@ -296,7 +296,7 @@ TEST echo "--- Boundary seek and write ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDWR) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty042", LFS_O_RDWR) => 0; size = strlen("hedgehoghog"); const lfs_soff_t offsets[] = {512, 1020, 513, 1021, 511, 1019}; @@ -324,7 +324,7 @@ TEST echo "--- Out-of-bounds seek ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - lfs_file_open(&lfs, &file[0], "hello/kitty42", LFS_O_RDWR) => 0; + lfs_file_open(&lfs, &file[0], "hello/kitty042", LFS_O_RDWR) => 0; size = strlen("kittycatcat"); lfs_file_size(&lfs, &file[0]) => $LARGESIZE*size; From 97a7191814a5900a35f11c3ebac0f6a710a6bb90 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 4 Oct 2018 16:25:19 -0500 Subject: [PATCH 113/139] Fixed issue with creating files named "littlefs" A rather humorous issue, we accidentally ended up mixing our file namespace with our superblocks. This meant if we created a file named "littlefs" it would reference the superblock and all sorts of things would break. Fixing this also highlighted another issue, the fact that the superblock always needs to come before any file entries in the directory. I didn't account for this in the initial B-tree design, but we need a higher ordering for superblocks + children + files than just name. To fix this I added ordering information in the 2 bits currently unused in the tag type. Though note that the size of these fields are flexible. 9-bit type field: [--- 9 ---] [1|- 3 -|- 2 -|- 3 -] ^ ^ ^ ^- type-specific info | | \------- ordering info | \------------- subtype \----------------- user bit --- lfs.c | 62 ++++++++++++++++++++++++--------------------- lfs.h | 12 ++++----- tests/debug.py | 8 +++--- tests/test_paths.sh | 8 ++++++ 4 files changed, 50 insertions(+), 40 deletions(-) diff --git a/lfs.c b/lfs.c index 9ba52f78..0d5f079d 100644 --- a/lfs.c +++ b/lfs.c @@ -287,7 +287,7 @@ static inline uint16_t lfs_tag_type(lfs_tag_t tag) { } static inline uint16_t lfs_tag_subtype(lfs_tag_t tag) { - return ((tag & 0x7c000000) >> 26) << 4; + return ((tag & 0x78000000) >> 26) << 4; } static inline uint16_t lfs_tag_id(lfs_tag_t tag) { @@ -531,7 +531,7 @@ static int lfs_dir_traverse(lfs_t *lfs, } } - if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { + if (lfs_tag_subtype(tag) == LFS_TYPE_CREATE) { // found where something was created if (lfs_tag_id(tag) == lfs_tag_id(matchtag - matchdiff)) { break; @@ -673,7 +673,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } // check for special tags - if (lfs_tag_subtype(tag) == LFS_TYPE_NAME) { + if (lfs_tag_subtype(tag) == LFS_TYPE_CREATE) { temp.count += 1; if (tempfoundtag && @@ -822,7 +822,7 @@ static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, static int lfs_dir_getglobals(lfs_t *lfs, const lfs_mdir_t *dir, struct lfs_globals *globals) { struct lfs_globals locals; - lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c000000, + lfs_stag_t res = lfs_dir_get(lfs, dir, 0x78000000, LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); if (res < 0 && res != LFS_ERR_NOENT) { return res; @@ -848,7 +848,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, } lfs_stag_t tag = lfs_dir_get(lfs, dir, 0x7c3fe000, - LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name); + LFS_MKTAG(LFS_TYPE_DIR, id, lfs->name_max+1), info->name); if (tag < 0) { return tag; } @@ -856,7 +856,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, info->type = lfs_tag_type(tag); struct lfs_ctz ctz; - tag = lfs_dir_get(lfs, dir, 0x7c3fe000, + tag = lfs_dir_get(lfs, dir, 0x783fe000, LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); if (tag < 0) { return tag; @@ -893,13 +893,12 @@ static int lfs_dir_find_match(void *data, } // found match? - if (res == 0 && (name->size == lfs_tag_size(tag) || - lfs_tag_type(tag) == LFS_TYPE_SUPERBLOCK)) { + if (res == 0 && name->size == lfs_tag_size(tag)) { return tag; } // a greater name found, exit early - if (res == 2 && lfs_tag_type(tag) != LFS_TYPE_SUPERBLOCK) { + if (res > 1 && lfs_tag_type(tag) != LFS_TYPE_SUPERBLOCK) { return tag | 0x1fff; } @@ -969,7 +968,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, // grab the entry data if (lfs_tag_id(tag) != 0x1ff) { - lfs_stag_t res = lfs_dir_get(lfs, dir, 0x7c3fe000, + lfs_stag_t res = lfs_dir_get(lfs, dir, 0x783fe000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), dir->tail); if (res < 0) { return res; @@ -980,7 +979,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, // find entry matching name while (true) { tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, - 0x7c000000, LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), + 0x7c000000, LFS_MKTAG(LFS_TYPE_DIR, 0, namelen), lfs_dir_find_match, &(struct lfs_dir_find_match){ lfs, name, namelen}); if (tag < 0) { @@ -1059,7 +1058,7 @@ static int lfs_commit_move_match(void *data, struct lfs_commit *commit = move->commit; if (move->pass == 0) { - if (lfs_tag_subtype(tag) != LFS_TYPE_NAME) { + if (lfs_tag_subtype(tag) != LFS_TYPE_CREATE) { return 0; } } else { @@ -1068,7 +1067,7 @@ static int lfs_commit_move_match(void *data, .pair[0] = commit->block, .off = commit->off, .etag = commit->ptag}, NULL, - lfs_tag_isuser(tag) ? 0x7fffe000 : 0x7c3fe000, tag, 0, + lfs_tag_isuser(tag) ? 0x7fffe000 : 0x783fe000, tag, 0, lfs_dir_get_match, &(struct lfs_dir_get_match){ lfs, NULL, 0, true}); if (res < 0 && res != LFS_ERR_NOENT) { @@ -1578,7 +1577,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_tag_t createtag = 0xffffffff; int attrcount = 0; for (const struct lfs_mattr *a = attrs; a; a = a->next) { - if (lfs_tag_subtype(a->tag) == LFS_TYPE_NAME) { + if (lfs_tag_subtype(a->tag) == LFS_TYPE_CREATE) { dir->count += 1; createtag = a->tag; } else if (lfs_tag_subtype(a->tag) == LFS_TYPE_DELETE) { @@ -1795,7 +1794,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { pair[1] = lfs->root[1]; } else { // get dir pair from parent - lfs_stag_t res = lfs_dir_get(lfs, &dir->m, 0x7c3fe000, + lfs_stag_t res = lfs_dir_get(lfs, &dir->m, 0x783fe000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -2190,7 +2189,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, file->flags |= LFS_F_DIRTY; } else { // try to load what's on disk, if it's inlined we'll fix it later - tag = lfs_dir_get(lfs, &file->m, 0x7c3fe000, + tag = lfs_dir_get(lfs, &file->m, 0x783fe000, LFS_MKTAG(LFS_TYPE_STRUCT, file->id, 8), &file->ctz); if (tag < 0) { err = tag; @@ -2245,7 +2244,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // don't always read (may be new/trunc file) if (file->ctz.size > 0) { - lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x7c3fe000, + lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x783fe000, LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->ctz.size), file->cache.buffer); if (res < 0) { @@ -2800,7 +2799,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (lfs_tag_type(tag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t pair[2]; - lfs_stag_t res = lfs_dir_get(lfs, &cwd, 0x7c3fe000, + lfs_stag_t res = lfs_dir_get(lfs, &cwd, 0x783fe000, LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -2880,7 +2879,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } else if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t prevpair[2]; - lfs_stag_t res = lfs_dir_get(lfs, &newcwd, 0x7c3fe000, + lfs_stag_t res = lfs_dir_get(lfs, &newcwd, 0x783fe000, LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair); if (res < 0) { return res; @@ -3139,9 +3138,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // write one superblock lfs_superblock_t superblock = { - .magic = {"littlefs"}, - .version = LFS_DISK_VERSION, - + .version = LFS_DISK_VERSION, .block_size = lfs->cfg->block_size, .block_count = lfs->cfg->block_count, .name_max = lfs->name_max, @@ -3151,8 +3148,10 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_tole32(&superblock); err = lfs_dir_commit(lfs, &root, - LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, &superblock, sizeof(superblock), - NULL)); + LFS_MKATTR(LFS_TYPE_INLINESTRUCT, 0, + &superblock, sizeof(superblock), + LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, "littlefs", 8, + NULL))); if (err) { goto cleanup; } @@ -3178,7 +3177,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pair_isnull(dir.tail)) { // fetch next block in tail list - lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, 0x7fc00000, + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, 0x7fffe000, LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), lfs_dir_find_match, &(struct lfs_dir_find_match){ lfs, "littlefs", 8}); @@ -3195,8 +3194,8 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // grab superblock lfs_superblock_t superblock; - tag = lfs_dir_get(lfs, &dir, 0x7f800000, - LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, sizeof(superblock)), + tag = lfs_dir_get(lfs, &dir, 0x7fffe000, + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), &superblock); if (tag < 0) { err = tag; @@ -3248,7 +3247,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->attr_max = superblock.attr_max; } - } // has globals? @@ -3258,6 +3256,12 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } } + // found superblock? + if (lfs_pair_isnull(lfs->root)) { + err = LFS_ERR_INVAL; + goto cleanup; + } + // update littlefs with globals lfs_global_fromle32(&lfs->locals); lfs_global_xor(&lfs->globals, &lfs->locals); @@ -3306,7 +3310,7 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { struct lfs_ctz ctz; - lfs_stag_t tag = lfs_dir_get(lfs, &dir, 0x7c3fe000, + lfs_stag_t tag = lfs_dir_get(lfs, &dir, 0x783fe000, LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); if (tag < 0) { if (tag == LFS_ERR_NOENT) { diff --git a/lfs.h b/lfs.h index 0dd1c2f7..10cb768a 100644 --- a/lfs.h +++ b/lfs.h @@ -88,12 +88,12 @@ enum lfs_error { // File types enum lfs_type { // file types - LFS_TYPE_REG = 0x002, - LFS_TYPE_DIR = 0x003, + LFS_TYPE_REG = 0x011, + LFS_TYPE_DIR = 0x010, // internally used types LFS_TYPE_USERATTR = 0x100, - LFS_TYPE_NAME = 0x000, + LFS_TYPE_CREATE = 0x000, LFS_TYPE_DELETE = 0x020, LFS_TYPE_STRUCT = 0x040, LFS_TYPE_TAIL = 0x080, @@ -110,8 +110,8 @@ enum lfs_type { // internal chip sources LFS_FROM_MEM = 0x000, LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x0c1, - LFS_FROM_USERATTRS = 0x0c2, + LFS_FROM_MOVE = 0x061, + LFS_FROM_USERATTRS = 0x062, }; // File open flags @@ -339,9 +339,7 @@ typedef struct lfs_file { } lfs_file_t; typedef struct lfs_superblock { - char magic[8]; uint32_t version; - lfs_size_t block_size; lfs_size_t block_count; diff --git a/tests/debug.py b/tests/debug.py index 0088442c..71f204a7 100755 --- a/tests/debug.py +++ b/tests/debug.py @@ -4,8 +4,8 @@ import binascii TYPES = { - (0x1ff, 0x002): 'reg', - (0x1ff, 0x003): 'dir', + (0x1ff, 0x011): 'create reg', + (0x1ff, 0x010): 'create dir', (0x1ff, 0x001): 'superblock', (0x1ff, 0x020): 'delete', (0x1f0, 0x0e0): 'globals', @@ -23,10 +23,10 @@ def typeof(type): mask = 0x1ff & ~((1 << prefix)-1) if (mask, type & mask) in TYPES: return TYPES[mask, type & mask] + ( - ' [%0*x]' % (prefix/4, type & ((1 << prefix)-1)) + ' %0*x' % (prefix/4, type & ((1 << prefix)-1)) if prefix else '') else: - return '[%02x]' % type + return '%02x' % type def main(*blocks): # find most recent block diff --git a/tests/test_paths.sh b/tests/test_paths.sh index ea5eb216..999001a5 100755 --- a/tests/test_paths.sh +++ b/tests/test_paths.sh @@ -139,6 +139,14 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Superblock conflict test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "littlefs") => 0; + lfs_remove(&lfs, "littlefs") => 0; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Max path test ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; From 795dd8c7ab930892536c6c2ee7c29b8bfac477db Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 5 Oct 2018 18:22:33 -0500 Subject: [PATCH 114/139] Fixed mkdir when inserting into a non-end block This was an oversight on my part when adding strict ordering to directories. Unfortunately now we can't take advantage of the atomic creation of tail+dir entries. Now we need to first create the tail, then create the actually directory entry. If we lose power, the orphan is cleaned up like orphans created during remove. Note that we still take advantage of the atomic tail+dir entries if we are an end block. This is actually because this corner case is complicated to _not_ do atomically, needing to update the directory we just committed to. --- .travis.yml | 2 +- lfs.c | 45 +++++++++++++++++++++++++++++++++++-------- tests/test_corrupt.sh | 2 +- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index e3f786b4..255e84bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,7 +126,7 @@ jobs: - mkdir mount/littlefs - cp -r $(git ls-tree --name-only HEAD) mount/littlefs - cd mount/littlefs - - ls + - ls -flh - make -B test_dirs test_files QUIET=1 # Automatically update releases diff --git a/lfs.c b/lfs.c index 0d5f079d..32e626f1 100644 --- a/lfs.c +++ b/lfs.c @@ -1746,29 +1746,58 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // build up new directory lfs_alloc_ack(lfs); - lfs_mdir_t dir; err = lfs_dir_alloc(lfs, &dir); if (err) { return err; } - dir.tail[0] = cwd.tail[0]; - dir.tail[1] = cwd.tail[1]; + // find end of list + lfs_mdir_t pred = cwd; + while (pred.split) { + err = lfs_dir_fetch(lfs, &pred, pred.tail); + if (err) { + return err; + } + } + + // setup dir + dir.tail[0] = pred.tail[0]; + dir.tail[1] = pred.tail[1]; err = lfs_dir_commit(lfs, &dir, NULL); if (err) { return err; } - // get next slot and commit - cwd.tail[0] = dir.pair[0]; - cwd.tail[1] = dir.pair[1]; + // current block end of list? + if (!cwd.split) { + // update atomically + cwd.tail[0] = dir.pair[0]; + cwd.tail[1] = dir.pair[1]; + } else { + // update tails, this creates a desync + pred.tail[0] = dir.pair[0]; + pred.tail[1] = dir.pair[1]; + lfs_global_orphans(lfs, +1); + err = lfs_dir_commit(lfs, &pred, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x1ff, + pred.tail, sizeof(pred.tail), + NULL)); + if (err) { + return err; + } + lfs_global_orphans(lfs, -1); + } + + // now insert into our parent block lfs_pair_tole32(dir.pair); err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x1ff, cwd.tail, sizeof(cwd.tail), LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, - NULL)))); + (!cwd.split) + ? LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x1ff, + cwd.tail, sizeof(cwd.tail), NULL) + : NULL))); lfs_pair_fromle32(dir.pair); if (err) { return err; diff --git a/tests/test_corrupt.sh b/tests/test_corrupt.sh index a73b7f8c..81b06747 100755 --- a/tests/test_corrupt.sh +++ b/tests/test_corrupt.sh @@ -89,7 +89,7 @@ do rm -rf blocks mkdir blocks lfs_mktree - chmod a-w blocks/$b + chmod a-w blocks/$b || true lfs_mktree lfs_chktree done From 4a1b8ae22277fbb0901b3b34649aaf0135bbdc73 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 21 Oct 2018 11:25:48 -0500 Subject: [PATCH 115/139] Fixed issues found by more aggressive rename tests - Fixed underflow issue caused by search id shortcuts that would result in early termination from lfs_dir_get - Fixed issue where entry file delete would toss out the best id during lfs_dir_fetchmatch - Fixed globals going out of date when canceling in same metadata-pair - Fixed early removal of metadata-pair when attribute list contains creates after deletes bring dir->count to zero --- lfs.c | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/lfs.c b/lfs.c index fc1b7a3d..246acc2e 100644 --- a/lfs.c +++ b/lfs.c @@ -518,7 +518,10 @@ static int lfs_dir_traverse(lfs_t *lfs, if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { lastcommit = 2 & lfs_tag_type(tag); - } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { + } + + if (lfs_tag_id(matchmask) != 0 && + lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { // something was deleted, need to move around it if (lfs_tag_id(tag) <= lfs_tag_id(matchtag - matchdiff)) { matchdiff -= LFS_MKTAG(0, 1, 0); @@ -533,11 +536,13 @@ static int lfs_dir_traverse(lfs_t *lfs, } } - if (lfs_tag_subtype(tag) == LFS_TYPE_CREATE) { + if (lfs_tag_id(matchmask) != 0 && + lfs_tag_subtype(tag) == LFS_TYPE_CREATE) { // found where something was created if (lfs_tag_id(tag) == lfs_tag_id(matchtag - matchdiff)) { break; - } else if (lfs_tag_id(tag) < lfs_tag_id(matchtag - matchdiff)) { + } else if (lfs_tag_id(tag) < + lfs_tag_id(matchtag - matchdiff)) { matchdiff += LFS_MKTAG(0, 1, 0); } } @@ -686,7 +691,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, LFS_ASSERT(temp.count > 0); temp.count -= 1; - if (tempfoundtag && + if (tempfoundtag && !lfs_tag_isdelete(tempfoundtag) && lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { tempfoundtag = 0; } else if (tempfoundtag && @@ -1390,7 +1395,8 @@ static int lfs_dir_compact(lfs_t *lfs, } // commit with a move - for (uint16_t id = begin; id < end || commit.off < commit.ack; id++) { + for (uint16_t id = begin; + id < end || commit.off < commit.ack; id++) { for (int pass = 0; pass < 2; pass++) { err = lfs_commit_move(lfs, &commit, pass, 0x003fe000, LFS_MKTAG(0, id, 0), @@ -1556,6 +1562,7 @@ static int lfs_dir_compact(lfs_t *lfs, static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs) { + // check for globals work struct lfs_mattr cancelattr; struct lfs_globals cancels; lfs_global_zero(&cancels); @@ -1564,7 +1571,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // Wait, we have the move? Just cancel this out here // We need to, or else the move can become outdated cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.id, 0); - cancelattr.next = attrs; // TODO need order + cancelattr.next = attrs; attrs = &cancelattr; cancels.hasmove = lfs->globals.hasmove; @@ -1584,26 +1591,31 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, if (lfs_tag_subtype(a->tag) == LFS_TYPE_CREATE) { dir->count += 1; createtag = a->tag; + + // Oh no, did we tweak globals? We need to fix that too + if (lfs_tag_id(a->tag) <= lfs_tag_id(cancelattr.tag)) { + cancelattr.tag += LFS_MKTAG(0, 1, 0); + } } else if (lfs_tag_subtype(a->tag) == LFS_TYPE_DELETE) { LFS_ASSERT(dir->count > 0); dir->count -= 1; deletetag = a->tag; + } - if (dir->count == 0) { - // should we actually drop the directory block? - lfs_mdir_t pdir; - int err = lfs_fs_pred(lfs, dir->pair, &pdir); - if (err && err != LFS_ERR_NOENT) { - return err; - } + attrcount += 1; + } - if (err != LFS_ERR_NOENT && pdir.split) { - return lfs_dir_drop(lfs, &pdir, dir); - } - } + // should we actually drop the directory block? + if (lfs_tag_isvalid(deletetag) && dir->count == 0) { + lfs_mdir_t pdir; + int err = lfs_fs_pred(lfs, dir->pair, &pdir); + if (err && err != LFS_ERR_NOENT) { + return err; } - attrcount += 1; + if (err != LFS_ERR_NOENT && pdir.split) { + return lfs_dir_drop(lfs, &pdir, dir); + } } if (dir->erased) { From 5b26c68ae2abae93b0fea4fca669765841dd1d2a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 22 Oct 2018 16:42:30 -0500 Subject: [PATCH 116/139] Tweaked tag endianness to catch power-loss after <1 word is written There was an interesting subtlety with the existing layout of tags that could become a problem in the future. Basically, littlefs avoids writing to any region of storage it is not absolutely sure has been erased beforehand. This is a part of limiting the number of assumptions about storage. It's possible a storage technology can't support writes without erases in a way that is undetectable at write time (Maybe changing a bit without an erase decreases the longevity of the information stored on the bit). But the existing layout had a very tiny corner case where this wasn't true. Consider the location of the valid bit in the tag struct: [1|--- 31 ---] ^--- valid bit The responsibility of this bit is to indicate if an attempt has been made to write the following commit. If it is not set (the specific value is dependent on a previous read and identified by the preceeding commit), the assumption is that it is safe to write to the next region because it has been erased previously. If it is set, we check if the next commit is valid, if it isn't (because of CRC failure, likely due to power-loss), we discard the commit. But because an attempt has been made to write to that storage, we must then do a compaction to move to the other block in the metadata-pair. This plan looks good on paper, but what does it look like on storage? The problem is that words in littlefs are in little-endian. So on storage the tag actually looks like this: [- 8 -|- 8 -|- 8 -|1|- 7 -] ^-- valid bit This means that we don't actually set the valid bit before writing the tag! We write the lower bytes first. If we lose power, we may have written 3 bytes without this fact being detectable. We could restructure the tag structure to store the valid bit lower, however because none of the fields are 7 bits, this would make the extraction more costly, and we then lose the ability to check this valid bit with a sign comparison. The simple solution is to just store the tag in big-endian. A small benefit is that this will actually have a negative code cost on big-endian machines. This mixture of endiannesses is frustrating, however it is a pragmatic solution with only a 20-byte code size cost. --- lfs.c | 10 +++++----- lfs_util.h | 46 ++++++++++++++++++++++++++++++++++++++++++++++ tests/corrupt.py | 2 +- tests/debug.py | 2 +- 4 files changed, 53 insertions(+), 7 deletions(-) diff --git a/lfs.c b/lfs.c index 246acc2e..4c914ee4 100644 --- a/lfs.c +++ b/lfs.c @@ -512,7 +512,7 @@ static int lfs_dir_traverse(lfs_t *lfs, return err; } - ntag = lfs_fromle32(ntag) ^ tag; + ntag = lfs_frombe32(ntag) ^ tag; tag |= 0x80000000; } @@ -617,7 +617,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } crc = lfs_crc(crc, &tag, sizeof(tag)); - tag = lfs_fromle32(tag) ^ ptag; + tag = lfs_frombe32(tag) ^ ptag; // next commit not yet programmed if (!lfs_tag_isvalid(tag)) { @@ -1134,7 +1134,7 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, } // write out tag - lfs_tag_t ntag = lfs_tole32((tag & 0x7fffffff) ^ commit->ptag); + lfs_tag_t ntag = lfs_tobe32((tag & 0x7fffffff) ^ commit->ptag); int err = lfs_commit_prog(lfs, commit, &ntag, sizeof(ntag)); if (err) { return err; @@ -1193,13 +1193,13 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, } // build crc tag - bool reset = ~lfs_fromle32(tag) >> 31; + bool reset = ~lfs_frombe32(tag) >> 31; tag = LFS_MKTAG(LFS_TYPE_CRC + 2*compacting + reset, 0x1ff, off - (commit->off+sizeof(lfs_tag_t))); // write out crc uint32_t footer[2]; - footer[0] = lfs_tole32(tag ^ commit->ptag); + footer[0] = lfs_tobe32(tag ^ commit->ptag); commit->crc = lfs_crc(commit->crc, &footer[0], sizeof(footer[0])); footer[1] = lfs_tole32(commit->crc); err = lfs_bd_prog(lfs, diff --git a/lfs_util.h b/lfs_util.h index 8a65694c..6e2f0020 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -186,6 +186,52 @@ static inline uint16_t lfs_tole16(uint16_t a) { return lfs_fromle16(a); } +// Convert between 32-bit big-endian and native order +static inline uint32_t lfs_frombe32(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + return __builtin_bswap32(a); +#elif !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + return a; +#else + return (((uint8_t*)&a)[0] << 24) | + (((uint8_t*)&a)[1] << 16) | + (((uint8_t*)&a)[2] << 8) | + (((uint8_t*)&a)[3] << 0); +#endif +} + +static inline uint32_t lfs_tobe32(uint32_t a) { + return lfs_frombe32(a); +} + +// Convert between 16-bit big-endian and native order +static inline uint16_t lfs_frombe16(uint16_t a) { +#if !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + return __builtin_bswap16(a); +#elif !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + return a; +#else + return (((uint8_t*)&a)[0] << 8) | + (((uint8_t*)&a)[1] << 0); +#endif +} + +static inline uint16_t lfs_tobe16(uint16_t a) { + return lfs_frombe16(a); +} + // Calculate CRC-32 with polynomial = 0x04c11db7 uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size); diff --git a/tests/corrupt.py b/tests/corrupt.py index d9943892..b6671ff6 100755 --- a/tests/corrupt.py +++ b/tests/corrupt.py @@ -14,7 +14,7 @@ def corrupt(block): tag = 0xffffffff while True: try: - ntag, = struct.unpack('I', file.read(4)) except struct.error: break diff --git a/tests/debug.py b/tests/debug.py index 71f204a7..c76dc3d3 100755 --- a/tests/debug.py +++ b/tests/debug.py @@ -68,7 +68,7 @@ def main(*blocks): try: data = file.read(4) crc = binascii.crc32(data, crc) - ntag, = struct.unpack('I', data) except struct.error: break From dc507a7b5f73278604d1b11921acbec8f58d358a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 22 Oct 2018 17:58:47 -0500 Subject: [PATCH 117/139] Changed required alignment of lookahead_size to 64 bits This is to prepare for future compatibility with other implementations of the allocator's lookahead that are under consideration. The most promising design so far is a sort of segments-list data structure that stores pointer+size pairs, requiring 64-bits of alignment. Changing this now takes advantage of the major version to avoid a compatibility break in the future. If we end up not changing the allocator or don't need 64-bit alignment we can easily drop this requirement without breaking anyone's code. --- lfs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lfs.c b/lfs.c index 4c914ee4..31297c15 100644 --- a/lfs.c +++ b/lfs.c @@ -3095,8 +3095,8 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs_cache_zero(lfs, &lfs->rcache); lfs_cache_zero(lfs, &lfs->pcache); - // setup lookahead, must be multiple of 32-bits - LFS_ASSERT(lfs->cfg->lookahead_size % 4 == 0); + // setup lookahead, must be multiple of 64-bits + LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0); LFS_ASSERT(lfs->cfg->lookahead_size > 0); if (lfs->cfg->lookahead_buffer) { lfs->free.buffer = lfs->cfg->lookahead_buffer; From a548ce68c116cb2ac67c266e4a4b1aa9fb3f8e45 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 26 Dec 2018 20:27:34 -0600 Subject: [PATCH 118/139] Switched to traversal-based compact logic This simplifies some of the interactions between reading and writing inside the commit logic. Unfortunately this change didn't decrease code size as was initially hoped, but it does offer a nice runtime improvement for the common case and should improve debugability. Before, the compact logic required three iterations: 1. iterate through all the ids in a directory 2. scan attrs bound to each id in the directory 3. lookup attrs in the in-progress commit The code for this, while terse and complicated, did have some nice side effect. The directory lookup logic could be reused for looking up in the in-progress commit, and iterating through each id allows us to know exactly how many ids we can fit during a compact. Giving us a O(n^3) compact and O(n^3) split. However, this was complicated by a few things. First, this compact logic doesn't handle deleted attrs. To work around this, I added a marker for the last commit (or first based on your perspective) which would indicate if a delete should be copied over. This worked but was a bit hacky and meant deletes weren't cleaned up on the first compact. Second, we can't actually figure out our compacted size until we compact. This worked ok except for the fact that splits will always have a failed compact. This means we waste an erase which could very expensive. It is possible to work around this by keeping our work, but with only a single prog cache this was very tricky and also somewhat hacky. Third, the interactions between reading and writing to the same block were tricky and error-prone. They should mostly be working now, but seeing this requirement go away does not make me sad. The new compact logic fixes these issues by moving the complexity into a general-purpose lfs_dir_traverse function which has much fewer side effects on the system. We can even use it for dry-runs to precompute our estimated size. How does it work? 1. iterate through all attr in the directory 2. for each attr, scan the rest of the directory to figure out the attr's history, this will change the attr based on dir modifications and may even exit early if the attr was deleted. The end result is a traversal function that gives us the resulting state of each attr in only O(n^2). To make this complete, we allow a bounded recursion into mcu-side move attrs, although this ends up being O(n^3) unlike moves in the original solution (however moves are less common. This gives us a nice traversal function we can use for compacts and moves, handles deletes, and is overall simpler to reason about. Two minor hiccups: 1. We need to handle create attrs specially, since this algorithm doesn't care or id order, which can cause problems since attr insertion are order sensitive. We can fix this by simply looking up each create (since there is only one per file) in order at the beginning of our traversal. This is oddly complimentary to the move logic, which also handles create attrs separately. 2. We no longer know exactly how many ids we can write to a dir during splits. However, since we can do a dry-run traversal, we can use that to simply binary search for the mid-point. This gives us a O(n^2) compact and O(n^2 log n) split, which is a nice minor improvement (remember n is bounded by block size). --- lfs.c | 757 +++++++++++++++++++++++++++++++--------------------------- lfs.h | 1 + 2 files changed, 411 insertions(+), 347 deletions(-) diff --git a/lfs.c b/lfs.c index 31297c15..b4d5d520 100644 --- a/lfs.c +++ b/lfs.c @@ -404,6 +404,9 @@ static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { /// Internal operations predeclared here /// static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs); +static int lfs_dir_compact(lfs_t *lfs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, + lfs_mdir_t *source, uint16_t begin, uint16_t end); static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], @@ -478,15 +481,14 @@ static void lfs_alloc_ack(lfs_t *lfs) { /// Metadata pair and directory operations /// -static int lfs_dir_traverse(lfs_t *lfs, +static int lfs_dir_traverseget(lfs_t *lfs, const lfs_mdir_t *dir, const struct lfs_mattr *attrs, - lfs_tag_t matchmask, lfs_tag_t matchtag, lfs_stag_t matchdiff, + lfs_tag_t getmask, lfs_tag_t gettag, lfs_stag_t getdiff, int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { lfs_block_t block = dir->pair[0]; lfs_off_t off = dir->off; lfs_tag_t ntag = dir->etag; - bool lastcommit = false; - matchtag += matchdiff; + gettag += getdiff; // iterate over dir block backwards (for faster lookups) while (attrs || off >= sizeof(lfs_tag_t) + lfs_tag_dsize(ntag)) { @@ -516,34 +518,199 @@ static int lfs_dir_traverse(lfs_t *lfs, tag |= 0x80000000; } - if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { - lastcommit = 2 & lfs_tag_type(tag); - } - - if (lfs_tag_id(matchmask) != 0 && + if (lfs_tag_id(getmask) != 0 && lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { // something was deleted, need to move around it - if (lfs_tag_id(tag) <= lfs_tag_id(matchtag - matchdiff)) { - matchdiff -= LFS_MKTAG(0, 1, 0); + if (lfs_tag_id(tag) <= lfs_tag_id(gettag - getdiff)) { + getdiff -= LFS_MKTAG(0, 1, 0); } } - if ((tag & matchmask) == ((matchtag - matchdiff) & matchmask) && - !(lfs_tag_isdelete(tag) && lastcommit)) { - int res = cb(data, tag + matchdiff, buffer); - if (res) { - return res; + if ((tag & getmask) == ((gettag - getdiff) & getmask)) { + if (lfs_tag_isdelete(tag)) { + return LFS_ERR_NOENT; } + + return cb(data, tag + getdiff, buffer); } - if (lfs_tag_id(matchmask) != 0 && + if (lfs_tag_id(getmask) != 0 && lfs_tag_subtype(tag) == LFS_TYPE_CREATE) { // found where something was created - if (lfs_tag_id(tag) == lfs_tag_id(matchtag - matchdiff)) { + if (lfs_tag_id(tag) == lfs_tag_id(gettag - getdiff)) { break; } else if (lfs_tag_id(tag) < - lfs_tag_id(matchtag - matchdiff)) { - matchdiff += LFS_MKTAG(0, 1, 0); + lfs_tag_id(gettag - getdiff)) { + getdiff += LFS_MKTAG(0, 1, 0); + } + } + } + + return LFS_ERR_NOENT; +} + +struct lfs_dir_get_read { + lfs_t *lfs; + void *buffer; + lfs_size_t size; +}; + +static int lfs_dir_get_read(void *data, + lfs_tag_t tag, const void *buffer) { + struct lfs_dir_get_read *get = data; + lfs_t *lfs = get->lfs; + const struct lfs_diskoff *disk = buffer; + if (get->buffer) { + lfs_size_t diff = lfs_min(lfs_tag_size(tag), get->size); + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, diff, + disk->block, disk->off, get->buffer, diff); + if (err) { + return err; + } + + memset((uint8_t*)get->buffer + diff, 0, get->size - diff); + } + + return tag & 0x7fffffff; +} + +static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, + lfs_tag_t getmask, lfs_tag_t gettag, void *buffer) { + lfs_stag_t getdiff = 0; + if (lfs->globals.hasmove && + lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0 && + lfs_tag_id(gettag) <= lfs->globals.id) { + // synthetic moves + gettag += LFS_MKTAG(0, 1, 0); + getdiff -= LFS_MKTAG(0, 1, 0); + } + + return lfs_dir_traverseget(lfs, dir, NULL, getmask, gettag, getdiff, + lfs_dir_get_read, &(struct lfs_dir_get_read){ + lfs, buffer, lfs_tag_size(gettag)}); +} + +static int lfs_dir_traverse_filter(void *p, + lfs_tag_t tag, const void *buffer) { + lfs_tag_t *filtertag = p; + (void)buffer; + + // check for redundancy + lfs_tag_t mask = (lfs_tag_isuser(tag) ? 0x7fffe000 : 0x783fe000); + if (!lfs_tag_subtype(tag) == LFS_TYPE_CREATE && ( + (mask & tag) == (mask & *filtertag) || + (lfs_tag_subtype(tag) == LFS_TYPE_DELETE && + lfs_tag_id(tag) == lfs_tag_id(*filtertag)))) { + return true; + } + + // check if we need to adjust for created/deleted tags + if (lfs_tag_subtype(tag) == LFS_TYPE_CREATE && + lfs_tag_id(tag) <= lfs_tag_id(*filtertag)) { + *filtertag += LFS_MKTAG(0, 1, 0); + } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE && + lfs_tag_id(tag) < lfs_tag_id(*filtertag)) { + *filtertag -= LFS_MKTAG(0, 1, 0); + } + + return false; +} + +static int lfs_dir_traverseall(lfs_t *lfs, const lfs_mdir_t *dir, + const struct lfs_mattr *attrs, + lfs_off_t off, lfs_tag_t ptag, int i, + uint16_t begin, uint16_t end, lfs_stag_t diff, + int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + // precompute length of attrs so we can iterate backwards, this + // lets us "append" commits + int attrcount = 0; + for (const struct lfs_mattr *a = attrs; a; a = a->next) { + attrcount += 1; + } + + // iterate over directory and attrs + while (off+lfs_tag_dsize(ptag) < dir->off || i < attrcount) { + lfs_tag_t tag; + const void *buffer; + struct lfs_diskoff disk; + if (off+lfs_tag_dsize(ptag) < dir->off) { + off += lfs_tag_dsize(ptag); + + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(tag), + dir->pair[0], off, &tag, sizeof(tag)); + if (err) { + return err; + } + + tag = (lfs_frombe32(tag) ^ ptag) | 0x80000000; + disk.block = dir->pair[0]; + disk.off = off+sizeof(lfs_tag_t); + buffer = &disk; + ptag = tag; + } else { + const struct lfs_mattr *a = attrs; + for (int j = 0; j < attrcount-i-1; j++) { + a = a->next; + } + + tag = a->tag; + buffer = a->buffer; + i += 1; + } + + // do we need to filter? inlining the filtering logic here allows + // for some minor optimizations + if (end <= 0x1ff) { + if (!lfs_tag_isuser(tag) && + lfs_tag_subtype(tag) != LFS_TYPE_STRUCT && + lfs_tag_subtype(tag) != LFS_TYPE_FROM) { + continue; + } + + // scan for duplicates and update tag based on creates/deletes + int filter = lfs_dir_traverseall(lfs, dir, attrs, + off, ptag, i, 0, -1, 0, + lfs_dir_traverse_filter, &tag); + if (filter < 0) { + return filter; + } + + if (filter) { + continue; + } + + // in filter range? + if (!(lfs_tag_id(tag) >= begin && lfs_tag_id(tag) < end)) { + continue; + } + } + + // handle special cases for mcu-side operations + if (lfs_tag_type(tag) == LFS_FROM_MOVE) { + uint16_t fromid = lfs_tag_size(tag); + uint16_t toid = lfs_tag_id(tag); + int err = lfs_dir_traverseall(lfs, buffer, NULL, + 0, 0xffffffff, 0, fromid, fromid+1, + LFS_MKTAG(0, toid - fromid, 0) + diff, + cb, data); + if (err) { + return err; + } + } else if (lfs_tag_type(tag) == LFS_FROM_USERATTRS) { + for (const struct lfs_attr *a = buffer; a; a = a->next) { + int err = cb(data, + LFS_MKTAG(0x100 | a->type, lfs_tag_id(tag), a->size) + + diff, a->buffer); + if (err) { + return err; + } + } + } else { + int err = cb(data, tag + diff, buffer); + if (err) { + return err; } } } @@ -551,6 +718,26 @@ static int lfs_dir_traverse(lfs_t *lfs, return 0; } +static int lfs_dir_traverse(lfs_t *lfs, const lfs_mdir_t *dir, + const struct lfs_mattr *attrs, + uint16_t begin, uint16_t end, lfs_stag_t diff, + int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + // grab create tags first, this is needed because we can't really order + // tags by ids otherwise, and our insertion cleverness becomes a problem + for (uint16_t i = begin; i < end; i++) { + int err = lfs_dir_traverseget(lfs, dir, attrs, + 0x783fe000, LFS_MKTAG(LFS_TYPE_CREATE, i, 0), + diff, cb, data); + if (err) { + return err; + } + } + + // then just grab every other tag in range, order is no longer a concern + return lfs_dir_traverseall(lfs, dir, attrs, + 0, 0xffffffff, 0, begin, end, diff, cb, data); +} + static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2], lfs_tag_t matchmask, lfs_tag_t matchtag, @@ -772,60 +959,6 @@ static int lfs_dir_fetch(lfs_t *lfs, 0xffffffff, 0x00000000, NULL, NULL); } -struct lfs_dir_get_match { - lfs_t *lfs; - void *buffer; - lfs_size_t size; - bool compacting; -}; - -static int lfs_dir_get_match(void *data, - lfs_tag_t tag, const void *buffer) { - struct lfs_dir_get_match *get = data; - lfs_t *lfs = get->lfs; - const struct lfs_diskoff *disk = buffer; - - if (lfs_tag_isdelete(tag) && !get->compacting) { - return LFS_ERR_NOENT; - } - - if (get->buffer) { - lfs_size_t diff = lfs_min(lfs_tag_size(tag), get->size); - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, diff, - disk->block, disk->off, get->buffer, diff); - if (err) { - return err; - } - - memset((uint8_t*)get->buffer + diff, 0, get->size - diff); - } - - return tag & 0x7fffffff; -} - -static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, - lfs_tag_t getmask, lfs_tag_t gettag, void *buffer) { - lfs_stag_t getdiff = 0; - if (lfs->globals.hasmove && - lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0 && - lfs_tag_id(gettag) <= lfs->globals.id) { - // synthetic moves - gettag += LFS_MKTAG(0, 1, 0); - getdiff -= LFS_MKTAG(0, 1, 0); - } - - lfs_stag_t res = lfs_dir_traverse(lfs, dir, NULL, - getmask, gettag, getdiff, - lfs_dir_get_match, &(struct lfs_dir_get_match){ - lfs, buffer, lfs_tag_size(gettag)}); - if (res < 0) { - return res; - } - - return res ? res : LFS_ERR_NOENT; -} - static int lfs_dir_getglobals(lfs_t *lfs, const lfs_mdir_t *dir, struct lfs_globals *globals) { struct lfs_globals locals; @@ -1028,105 +1161,25 @@ struct lfs_commit { lfs_off_t begin; lfs_off_t end; - lfs_off_t ack; }; -static int lfs_commit_prog(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_dir_commitprog(lfs_t *lfs, struct lfs_commit *commit, const void *buffer, lfs_size_t size) { - lfs_off_t skip = lfs_min(lfs_max(commit->ack, commit->off) - - commit->off, size); int err = lfs_bd_prog(lfs, &lfs->pcache, &lfs->rcache, false, - commit->block, commit->off + skip, - (const uint8_t*)buffer + skip, size - skip); + commit->block, commit->off , + (const uint8_t*)buffer, size); if (err) { return err; } commit->crc = lfs_crc(commit->crc, buffer, size); commit->off += size; - commit->ack = lfs_max(commit->off, commit->ack); return 0; } -static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, - lfs_tag_t tag, const void *buffer); - -struct lfs_commit_move_match { - lfs_t *lfs; - struct lfs_commit *commit; - int pass; -}; - -static int lfs_commit_move_match(void *data, +static int lfs_dir_commitattr(lfs_t *lfs, struct lfs_commit *commit, lfs_tag_t tag, const void *buffer) { - struct lfs_commit_move_match *move = data; - lfs_t *lfs = move->lfs; - struct lfs_commit *commit = move->commit; - - if (move->pass == 0) { - if (lfs_tag_subtype(tag) != LFS_TYPE_CREATE) { - return 0; - } - } else { - // check if type has already been committed - lfs_stag_t res = lfs_dir_traverse(lfs, &(const lfs_mdir_t){ - .pair[0] = commit->block, - .off = commit->off, - .etag = commit->ptag}, NULL, - lfs_tag_isuser(tag) ? 0x7fffe000 : 0x783fe000, tag, 0, - lfs_dir_get_match, &(struct lfs_dir_get_match){ - lfs, NULL, 0, true}); - if (res < 0 && res != LFS_ERR_NOENT) { - return res; - } - - if (res > 0) { - return 0; - } - } - - // update id and commit, as we are currently unique - return lfs_commit_attr(lfs, commit, tag, buffer); -} - -static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit, int pass, - lfs_tag_t frommask, lfs_tag_t fromtag, lfs_stag_t fromdiff, - const lfs_mdir_t *dir, const struct lfs_mattr *attrs) { - return lfs_dir_traverse(lfs, dir, attrs, - frommask, fromtag, fromdiff, - lfs_commit_move_match, &(struct lfs_commit_move_match){ - lfs, commit, pass}); -} - -static int lfs_commit_userattrs(lfs_t *lfs, struct lfs_commit *commit, - uint16_t id, const struct lfs_attr *attrs) { - for (const struct lfs_attr *a = attrs; a; a = a->next) { - int err = lfs_commit_attr(lfs, commit, - LFS_MKTAG(0x100 | a->type, id, a->size), a->buffer); - if (err) { - return err; - } - } - - return 0; -} - -static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, - lfs_tag_t tag, const void *buffer) { - if (lfs_tag_type(tag) == LFS_FROM_MOVE) { - // special case for moves - return lfs_commit_move(lfs, commit, 1, - 0x003fe000, LFS_MKTAG(0, lfs_tag_size(tag), 0), - LFS_MKTAG(0, lfs_tag_id(tag), 0) - - LFS_MKTAG(0, lfs_tag_size(tag), 0), - buffer, NULL); - } else if (lfs_tag_type(tag) == LFS_FROM_USERATTRS) { - // special case for custom attributes - return lfs_commit_userattrs(lfs, commit, - lfs_tag_id(tag), buffer); - } - // check if we fit lfs_size_t dsize = lfs_tag_dsize(tag); if (commit->off + dsize > commit->end) { @@ -1135,14 +1188,14 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, // write out tag lfs_tag_t ntag = lfs_tobe32((tag & 0x7fffffff) ^ commit->ptag); - int err = lfs_commit_prog(lfs, commit, &ntag, sizeof(ntag)); + int err = lfs_dir_commitprog(lfs, commit, &ntag, sizeof(ntag)); if (err) { return err; } if (!(tag & 0x80000000)) { // from memory - err = lfs_commit_prog(lfs, commit, buffer, dsize-sizeof(tag)); + err = lfs_dir_commitprog(lfs, commit, buffer, dsize-sizeof(tag)); if (err) { return err; } @@ -1159,7 +1212,7 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, return err; } - err = lfs_commit_prog(lfs, commit, &dat, 1); + err = lfs_dir_commitprog(lfs, commit, &dat, 1); if (err) { return err; } @@ -1170,15 +1223,14 @@ static int lfs_commit_attr(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_commit_globals(lfs_t *lfs, struct lfs_commit *commit, +static int lfs_dir_commitglobals(lfs_t *lfs, struct lfs_commit *commit, struct lfs_globals *globals) { - return lfs_commit_attr(lfs, commit, + return lfs_dir_commitattr(lfs, commit, LFS_MKTAG(LFS_TYPE_GLOBALS + 2*globals->hasmove + globals->orphans, 0x1ff, 10), globals); } -static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, - bool compacting) { +static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { // align to program units lfs_off_t off = lfs_alignup(commit->off + 2*sizeof(uint32_t), lfs->cfg->prog_size); @@ -1194,7 +1246,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit, // build crc tag bool reset = ~lfs_frombe32(tag) >> 31; - tag = LFS_MKTAG(LFS_TYPE_CRC + 2*compacting + reset, + tag = LFS_MKTAG(LFS_TYPE_CRC + reset, 0x1ff, off - (commit->off+sizeof(lfs_tag_t))); // write out crc @@ -1299,79 +1351,155 @@ static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mdir_t *tail) { NULL)); } +static int lfs_dir_split(lfs_t *lfs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, + lfs_mdir_t *source, uint16_t split, uint16_t end) { + // create tail directory + lfs_mdir_t tail; + int err = lfs_dir_alloc(lfs, &tail); + if (err) { + return err; + } + + tail.split = dir->split; + tail.tail[0] = dir->tail[0]; + tail.tail[1] = dir->tail[1]; + + err = lfs_dir_compact(lfs, &tail, attrs, source, split, end); + if (err) { + return err; + } + + dir->tail[0] = tail.pair[0]; + dir->tail[1] = tail.pair[1]; + dir->split = true; + + // update root if needed + if (lfs_pair_cmp(dir->pair, lfs->root) == 0 && split == 0) { + lfs->root[0] = tail.pair[0]; + lfs->root[1] = tail.pair[1]; + } + + return 0; +} + +static int lfs_dir_commit_size(void *p, lfs_tag_t tag, const void *buffer) { + lfs_size_t *size = p; + (void)buffer; + + *size += lfs_tag_dsize(tag); + return 0; +} + +struct lfs_dir_commit_commit { + lfs_t *lfs; + struct lfs_commit *commit; +}; + +static int lfs_dir_commit_commit(void *p, lfs_tag_t tag, const void *buffer) { + struct lfs_dir_commit_commit *commit = p; + return lfs_dir_commitattr(commit->lfs, commit->commit, tag, buffer); +} + static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; + struct lfs_globals globals, locals; bool relocated = false; + bool exhausted = false; - // There's nothing special about our global delta, so feed it back - // into the global global delta - int err = lfs_dir_getglobals(lfs, dir, &lfs->locals); - if (err) { - return err; - } - - // begin loop to commit compaction to blocks until a compact sticks while (true) { - // setup compaction - bool splitted = false; - bool exhausted = false; - bool overcompacting = false; - - struct lfs_commit commit; - commit.block = dir->pair[1]; - commit.ack = 0; - -commit: - // setup erase state - exhausted = false; - dir->count = end - begin; - int16_t ackid = -1; - - // setup commit state - commit.off = 0; - commit.crc = 0xffffffff; - commit.ptag = 0xffffffff; + // find size + lfs_size_t size = 0; + int err = lfs_dir_traverse(lfs, source, attrs, begin, end, 0, + lfs_dir_commit_size, &size); + if (err) { + return err; + } // space is complicated, we need room for tail, crc, globals, // cleanup delete, and we cap at half a block to give room // for metadata updates - commit.begin = 0; - commit.end = lfs->cfg->block_size - 38; - if (!overcompacting) { - commit.end = lfs_min(commit.end, - lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size)); - } - - if (!splitted) { - // increment revision count - dir->rev += 1; - if (lfs->cfg->block_cycles && - dir->rev % lfs->cfg->block_cycles == 0) { - if (lfs_pair_cmp(dir->pair, - (const lfs_block_t[2]){0, 1}) == 0) { - // we're writing too much to the superblock, - // should we expand? - lfs_ssize_t res = lfs_fs_size(lfs); - if (res < 0) { - return res; - } + if (size <= lfs_min(lfs->cfg->block_size - 38, + lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size))) { + break; + } - // do we have enough space to expand? - if ((lfs_size_t)res < lfs->cfg->block_count/2) { - LFS_DEBUG("Expanding superblock at rev %"PRIu32, - dir->rev); - exhausted = true; - goto split; - } - } else { - // we're writing too much, time to relocate - exhausted = true; - goto relocate; + // can't fit, need to split, we should really be finding the + // largest size that fits with a small binary search, but right now + // it's not worth the code size + uint16_t split = (end - begin) / 2; + err = lfs_dir_split(lfs, dir, attrs, source, begin+split, end); + if (err) { + // if we fail to split, we may be able to overcompact, unless + // we're too big for even the full block, in which case our + // only option is to error + if (err == LFS_ERR_NOSPC && size <= lfs->cfg->block_size - 38) { + break; + } + return err; + } + + end = begin + split; + } + + // increment revision count + dir->rev += 1; + if (lfs->cfg->block_cycles && dir->rev % lfs->cfg->block_cycles == 0) { + if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { + // oh no! we're writing too much to the superblock, + // should we expand? + lfs_ssize_t res = lfs_fs_size(lfs); + if (res < 0) { + return res; + } + + // do we have extra space? littlefs can't reclaim this space + // by itself, so expand cautiously + if ((lfs_size_t)res < lfs->cfg->block_count/2) { + LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); + int err = lfs_dir_split(lfs, dir, attrs, source, begin, end); + if (err && err != LFS_ERR_NOSPC) { + return err; + } + + // welp, we tried, if we ran out of space there's not much + // we can do, we'll error later if we've become frozen + if (!err) { + end = begin; } } + } else { + // we're writing too much, time to relocate + exhausted = true; + goto relocate; + } + } + + // begin loop to commit compaction to blocks until a compact sticks + while (true) { + if (true) { + // There's nothing special about our global delta, so feed it into + // our local global delta + globals = lfs->globals; + locals = lfs->locals; + int err = lfs_dir_getglobals(lfs, dir, &locals); + if (err) { + return err; + } + + // setup commit state + struct lfs_commit commit = { + .block = dir->pair[1], + .off = 0, + .ptag = 0xffffffff, + .crc = 0xffffffff, + + .begin = 0, + .end = lfs->cfg->block_size - 8, + }; // erase block to write to err = lfs_bd_erase(lfs, dir->pair[1]); @@ -1381,12 +1509,12 @@ static int lfs_dir_compact(lfs_t *lfs, } return err; } - } - if (true) { // write out header - uint32_t rev = lfs_tole32(dir->rev); - err = lfs_commit_prog(lfs, &commit, &rev, sizeof(rev)); + dir->rev = lfs_tole32(dir->rev); + err = lfs_dir_commitprog(lfs, &commit, + &dir->rev, sizeof(dir->rev)); + dir->rev = lfs_fromle32(dir->rev); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1395,59 +1523,35 @@ static int lfs_dir_compact(lfs_t *lfs, } // commit with a move - for (uint16_t id = begin; - id < end || commit.off < commit.ack; id++) { - for (int pass = 0; pass < 2; pass++) { - err = lfs_commit_move(lfs, &commit, pass, - 0x003fe000, LFS_MKTAG(0, id, 0), - -LFS_MKTAG(0, begin, 0), - source, attrs); - if (err && !(splitted && !overcompacting && - err == LFS_ERR_NOSPC)) { - if (!overcompacting && err == LFS_ERR_NOSPC) { - goto split; - } else if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - } - - ackid = id; - } - - // reopen reserved space at the end - commit.end = lfs->cfg->block_size - 8; - - if (ackid >= end) { - // extra garbage attributes were written out during split, - // need to clean up - err = lfs_commit_attr(lfs, &commit, - LFS_MKTAG(LFS_TYPE_DELETE, ackid, 0), NULL); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; + err = lfs_dir_traverse(lfs, source, attrs, + begin, end, -LFS_MKTAG(0, begin, 0), + lfs_dir_commit_commit, &(struct lfs_dir_commit_commit){ + lfs, &commit}); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; } + return err; } - if (!relocated && !lfs_global_iszero(&lfs->locals)) { + if (!relocated && !lfs_global_iszero(&locals)) { // commit any globals, unless we're relocating, // in which case our parent will steal our globals - err = lfs_commit_globals(lfs, &commit, &lfs->locals); + err = lfs_dir_commitglobals(lfs, &commit, &locals); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } + + lfs_global_zero(&locals); } if (!lfs_pair_isnull(dir->tail)) { // commit tail, which may be new after last size check lfs_pair_tole32(dir->tail); - err = lfs_commit_attr(lfs, &commit, + err = lfs_dir_commitattr(lfs, &commit, LFS_MKTAG(LFS_TYPE_TAIL + dir->split, 0x1ff, sizeof(dir->tail)), dir->tail); lfs_pair_fromle32(dir->tail); @@ -1459,7 +1563,7 @@ static int lfs_dir_compact(lfs_t *lfs, } } - err = lfs_commit_crc(lfs, &commit, true); + err = lfs_dir_commitcrc(lfs, &commit); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1469,58 +1573,13 @@ static int lfs_dir_compact(lfs_t *lfs, // successful compaction, swap dir pair to indicate most recent lfs_pair_swap(dir->pair); + dir->count = end - begin; dir->off = commit.off; dir->etag = commit.ptag; dir->erased = true; } break; -split: - // commit no longer fits, need to split dir, - // drop caches and create tail - splitted = !exhausted; - if (lfs->pcache.block != 0xffffffff) { - commit.ack -= lfs->pcache.size; - lfs_cache_drop(lfs, &lfs->pcache); - } - - if (!exhausted && ackid < 0) { - // If we can't fit in this block, we won't fit in next block - return LFS_ERR_NOSPC; - } - - lfs_mdir_t tail; - err = lfs_dir_alloc(lfs, &tail); - if (err) { - if (err == LFS_ERR_NOSPC) { - // No space to expand? Try overcompacting - overcompacting = true; - goto commit; - } - return err; - } - - tail.split = dir->split; - tail.tail[0] = dir->tail[0]; - tail.tail[1] = dir->tail[1]; - - err = lfs_dir_compact(lfs, &tail, attrs, source, ackid+1, end); - if (err) { - return err; - } - - end = ackid+1; - dir->tail[0] = tail.pair[0]; - dir->tail[1] = tail.pair[1]; - dir->split = true; - - if (exhausted) { - lfs->root[0] = tail.pair[0]; - lfs->root[1] = tail.pair[1]; - } - - goto commit; - relocate: // commit was corrupted, drop caches and prepare to relocate block relocated = true; @@ -1536,7 +1595,7 @@ static int lfs_dir_compact(lfs_t *lfs, } // relocate half of pair - err = lfs_alloc(lfs, &dir->pair[1]); + int err = lfs_alloc(lfs, &dir->pair[1]); if (err && (err != LFS_ERR_NOSPC && !exhausted)) { return err; } @@ -1544,14 +1603,15 @@ static int lfs_dir_compact(lfs_t *lfs, continue; } - if (!relocated) { - // successful commit, update globals - lfs_global_zero(&lfs->locals); - } else { + // successful commit, update globals + lfs->globals = globals; + lfs->locals = locals; + + if (relocated) { // update references if we relocated LFS_DEBUG("Relocating %"PRIu32" %"PRIu32" to %"PRIu32" %"PRIu32, oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); - err = lfs_fs_relocate(lfs, oldpair, dir->pair); + int err = lfs_fs_relocate(lfs, oldpair, dir->pair); if (err) { return err; } @@ -1583,6 +1643,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_global_tole32(&lfs->locals); } + struct lfs_globals globals = lfs->globals; + struct lfs_globals locals = lfs->locals; + // calculate new directory size lfs_tag_t deletetag = 0xffffffff; lfs_tag_t createtag = 0xffffffff; @@ -1591,11 +1654,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, if (lfs_tag_subtype(a->tag) == LFS_TYPE_CREATE) { dir->count += 1; createtag = a->tag; - - // Oh no, did we tweak globals? We need to fix that too - if (lfs_tag_id(a->tag) <= lfs_tag_id(cancelattr.tag)) { - cancelattr.tag += LFS_MKTAG(0, 1, 0); - } } else if (lfs_tag_subtype(a->tag) == LFS_TYPE_DELETE) { LFS_ASSERT(dir->count > 0); dir->count -= 1; @@ -1623,51 +1681,47 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, struct lfs_commit commit = { .block = dir->pair[0], .off = dir->off, - .crc = 0xffffffff, .ptag = dir->etag, + .crc = 0xffffffff, .begin = dir->off, .end = lfs->cfg->block_size - 8, - .ack = 0, }; - // iterate over commits backwards, this lets us "append" commits - for (int i = 0; i < attrcount; i++) { - const struct lfs_mattr *a = attrs; - for (int j = 0; j < attrcount-i-1; j++) { - a = a->next; - } - - lfs_pair_tole32(dir->tail); - int err = lfs_commit_attr(lfs, &commit, a->tag, a->buffer); - lfs_pair_fromle32(dir->tail); - if (err) { - if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { - goto compact; - } - return err; + // traverse attrs that need to be written out + lfs_pair_tole32(dir->tail); + int err = lfs_dir_traverseall(lfs, dir, attrs, + dir->off, dir->etag, 0, 0, -1, 0, + lfs_dir_commit_commit, &(struct lfs_dir_commit_commit){ + lfs, &commit}); + lfs_pair_fromle32(dir->tail); + if (err) { + if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { + goto compact; } + return err; } // commit any global diffs if we have any - if (!lfs_global_iszero(&lfs->locals)) { - struct lfs_globals locals = lfs->locals; - int err = lfs_dir_getglobals(lfs, dir, &locals); + if (!lfs_global_iszero(&locals)) { + err = lfs_dir_getglobals(lfs, dir, &locals); if (err) { return err; } - err = lfs_commit_globals(lfs, &commit, &locals); + err = lfs_dir_commitglobals(lfs, &commit, &locals); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; } return err; } + + lfs_global_zero(&locals); } // finalize commit with the crc - int err = lfs_commit_crc(lfs, &commit, false); + err = lfs_dir_commitcrc(lfs, &commit); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1679,7 +1733,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, dir->off = commit.off; dir->etag = commit.ptag; // successful commit, update globals - lfs_global_zero(&lfs->locals); + lfs->globals = globals; + lfs->locals = locals; } else { compact: // fall back to compaction @@ -2946,7 +3001,15 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // create move to fix later - lfs_global_move(lfs, true, oldcwd.pair, lfs_tag_id(oldtag)); + uint16_t newoldtagid = lfs_tag_id(oldtag); + if (lfs_pair_cmp(oldcwd.pair, newcwd.pair) == 0 && + prevtag == LFS_ERR_NOENT && newid <= newoldtagid) { + // there is a small chance we are being renamed in the same directory + // to an id less than our old id, the global update to handle this + // is a bit messy + newoldtagid += 1; + } + lfs_global_move(lfs, true, oldcwd.pair, newoldtagid); // move over all attributes err = lfs_dir_commit(lfs, &newcwd, diff --git a/lfs.h b/lfs.h index a8071a0c..85c31e39 100644 --- a/lfs.h +++ b/lfs.h @@ -118,6 +118,7 @@ enum lfs_type { LFS_TYPE_CTZSTRUCT = 0x042, // internal chip sources + LFS_TYPE_FROM = 0x060, LFS_FROM_MEM = 0x000, LFS_FROM_DISK = 0x200, LFS_FROM_MOVE = 0x061, From b989b4a89f6c69b0219b64404de89be63923ca44 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 29 Dec 2018 07:53:12 -0600 Subject: [PATCH 119/139] Cleaned up tag encoding, now with clear chunk field Before, the tag format's type field was limited to 9-bits. This sounds like a lot, but this field needed to encode up to 256 user-specified types. This limited the flexibility of the encoded types. As time went on, more bits in the type field were repurposed for various things, leaving a rather fragile type field. Here we make the jump to full 11-bit type fields. This comes at the cost of a smaller length field, however the use of the length field was always going to come with a RAM limitation. Rather than putting pressure on RAM for inline files, the new type field lets us encode a chunk number, splitting up inline files into multiple updatable units. This actually pushes the theoretical inline max from 8KiB to 256KiB! (Note that we only allow a single 1KiB chunk for now, chunky inline files is just a theoretical future improvement). Here is the new 32-bit tag format, note that there are multiple levels of types which break down into more info: [---- 32 ----] [1|-- 11 --|-- 10 --|-- 10 --] ^. ^ . ^ ^- entry length |. | . \------------ file id chunk info |. \-----.------------------ type info (type3) \.-----------.------------------ valid bit [-3-|-- 8 --] ^ ^- chunk info \------- type info (type1) Additionally, I've split the CREATE tag into separate SPLICE and NAME tags. This simplified the new compact logic a bit. For now, littlefs still follows the rule that a NAME tag precedes any other tags related to a file, but this can change in the future. --- lfs.c | 733 ++++++++++++++++++++----------------------- lfs.h | 58 ++-- tests/corrupt.py | 2 +- tests/debug.py | 54 ++-- tests/test_alloc.sh | 6 +- tests/test_format.sh | 3 +- 6 files changed, 411 insertions(+), 445 deletions(-) diff --git a/lfs.c b/lfs.c index b4d5d520..3d3aea05 100644 --- a/lfs.c +++ b/lfs.c @@ -112,6 +112,12 @@ static int lfs_bd_read(lfs_t *lfs, return 0; } +enum { + LFS_CMP_EQ = 0, + LFS_CMP_LT = 1, + LFS_CMP_GT = 2, +}; + static int lfs_bd_cmp(lfs_t *lfs, const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, lfs_block_t block, lfs_off_t off, @@ -128,11 +134,11 @@ static int lfs_bd_cmp(lfs_t *lfs, } if (dat != data[i]) { - return (dat < data[i]) ? 1 : 2; + return (dat < data[i]) ? LFS_CMP_LT : LFS_CMP_GT; } } - return 0; + return LFS_CMP_EQ; } static int lfs_bd_flush(lfs_t *lfs, @@ -152,8 +158,12 @@ static int lfs_bd_flush(lfs_t *lfs, int res = lfs_bd_cmp(lfs, NULL, rcache, diff, pcache->block, pcache->off, pcache->buffer, diff); - if (res) { - return (res < 0) ? res : LFS_ERR_CORRUPT; + if (res < 0) { + return res; + } + + if (res != LFS_CMP_EQ) { + return LFS_ERR_CORRUPT; } } @@ -268,34 +278,38 @@ typedef uint32_t lfs_tag_t; typedef int32_t lfs_stag_t; #define LFS_MKTAG(type, id, size) \ - (((lfs_tag_t)(type) << 22) | ((lfs_tag_t)(id) << 13) | (lfs_tag_t)(size)) + (((lfs_tag_t)(type) << 20) | ((lfs_tag_t)(id) << 10) | (lfs_tag_t)(size)) static inline bool lfs_tag_isvalid(lfs_tag_t tag) { return !(tag & 0x80000000); } -static inline bool lfs_tag_isuser(lfs_tag_t tag) { - return (tag & 0x40000000); +static inline bool lfs_tag_isdelete(lfs_tag_t tag) { + return ((int32_t)(tag << 22) >> 22) == -1; } -static inline bool lfs_tag_isdelete(lfs_tag_t tag) { - return ((int32_t)(tag << 19) >> 19) == -1; +static inline uint16_t lfs_tag_type1(lfs_tag_t tag) { + return (tag & 0x70000000) >> 20; +} + +static inline uint16_t lfs_tag_type3(lfs_tag_t tag) { + return (tag & 0x7ff00000) >> 20; } -static inline uint16_t lfs_tag_type(lfs_tag_t tag) { - return (tag & 0x7fc00000) >> 22; +static inline uint8_t lfs_tag_chunk(lfs_tag_t tag) { + return (tag & 0x0ff00000) >> 20; } -static inline uint16_t lfs_tag_subtype(lfs_tag_t tag) { - return ((tag & 0x78000000) >> 26) << 4; +static inline int8_t lfs_tag_splice(lfs_tag_t tag) { + return (int8_t)lfs_tag_chunk(tag); } static inline uint16_t lfs_tag_id(lfs_tag_t tag) { - return (tag & 0x003fe000) >> 13; + return (tag & 0x000ffc00) >> 10; } static inline lfs_size_t lfs_tag_size(lfs_tag_t tag) { - return tag & 0x00001fff; + return tag & 0x000003ff; } static inline lfs_size_t lfs_tag_dsize(lfs_tag_t tag) { @@ -405,7 +419,7 @@ static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs); static int lfs_dir_compact(lfs_t *lfs, - lfs_mdir_t *dir, const struct lfs_mattr *attrs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount, lfs_mdir_t *source, uint16_t begin, uint16_t end); static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); @@ -481,114 +495,67 @@ static void lfs_alloc_ack(lfs_t *lfs) { /// Metadata pair and directory operations /// -static int lfs_dir_traverseget(lfs_t *lfs, - const lfs_mdir_t *dir, const struct lfs_mattr *attrs, - lfs_tag_t getmask, lfs_tag_t gettag, lfs_stag_t getdiff, - int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { - lfs_block_t block = dir->pair[0]; +static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, + lfs_tag_t gmask, lfs_tag_t gtag, void *buffer) { lfs_off_t off = dir->off; lfs_tag_t ntag = dir->etag; - gettag += getdiff; + lfs_stag_t gdiff = 0; + + if (lfs->globals.hasmove && + lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0 && + lfs_tag_id(gtag) <= lfs->globals.id) { + // synthetic moves + gdiff -= LFS_MKTAG(0, 1, 0); + } // iterate over dir block backwards (for faster lookups) - while (attrs || off >= sizeof(lfs_tag_t) + lfs_tag_dsize(ntag)) { - lfs_tag_t tag; - const void *buffer; - struct lfs_diskoff disk; - if (attrs) { - tag = attrs->tag; - buffer = attrs->buffer; - attrs = attrs->next; - } else { - off -= lfs_tag_dsize(ntag); + while (off >= sizeof(lfs_tag_t) + lfs_tag_dsize(ntag)) { + off -= lfs_tag_dsize(ntag); + lfs_tag_t tag = ntag; + int err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, sizeof(ntag), + dir->pair[0], off, &ntag, sizeof(ntag)); + if (err) { + return err; + } - tag = ntag; - buffer = &disk; - disk.block = block; - disk.off = off + sizeof(tag); + ntag = (lfs_frombe32(ntag) ^ tag) & 0x7fffffff; - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(ntag), - block, off, &ntag, sizeof(ntag)); - if (err) { - return err; + if (lfs_tag_id(gmask) != 0 && + lfs_tag_type1(tag) == LFS_TYPE_SPLICE && + lfs_tag_id(tag) <= lfs_tag_id(gtag - gdiff)) { + if (tag == (LFS_MKTAG(LFS_TYPE_CREATE, 0, 0) | + (LFS_MKTAG(0, 0x3ff, 0) & (gtag - gdiff)))) { + // found where we were created + return LFS_ERR_NOENT; } - ntag = lfs_frombe32(ntag) ^ tag; - tag |= 0x80000000; + // move around splices + gdiff += LFS_MKTAG(0, lfs_tag_splice(tag), 0); } - if (lfs_tag_id(getmask) != 0 && - lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { - // something was deleted, need to move around it - if (lfs_tag_id(tag) <= lfs_tag_id(gettag - getdiff)) { - getdiff -= LFS_MKTAG(0, 1, 0); - } - } - - if ((tag & getmask) == ((gettag - getdiff) & getmask)) { + if ((gmask & tag) == (gmask & (gtag - gdiff))) { if (lfs_tag_isdelete(tag)) { return LFS_ERR_NOENT; } - return cb(data, tag + getdiff, buffer); - } - - if (lfs_tag_id(getmask) != 0 && - lfs_tag_subtype(tag) == LFS_TYPE_CREATE) { - // found where something was created - if (lfs_tag_id(tag) == lfs_tag_id(gettag - getdiff)) { - break; - } else if (lfs_tag_id(tag) < - lfs_tag_id(gettag - getdiff)) { - getdiff += LFS_MKTAG(0, 1, 0); + lfs_size_t diff = lfs_min(lfs_tag_size(tag), + lfs_tag_size(gtag)); + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, diff, + dir->pair[0], off+sizeof(tag), buffer, diff); + if (err) { + return err; } - } - } - - return LFS_ERR_NOENT; -} -struct lfs_dir_get_read { - lfs_t *lfs; - void *buffer; - lfs_size_t size; -}; + memset((uint8_t*)buffer + diff, 0, + lfs_tag_size(gtag) - diff); -static int lfs_dir_get_read(void *data, - lfs_tag_t tag, const void *buffer) { - struct lfs_dir_get_read *get = data; - lfs_t *lfs = get->lfs; - const struct lfs_diskoff *disk = buffer; - if (get->buffer) { - lfs_size_t diff = lfs_min(lfs_tag_size(tag), get->size); - int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, diff, - disk->block, disk->off, get->buffer, diff); - if (err) { - return err; + return tag + gdiff; } - - memset((uint8_t*)get->buffer + diff, 0, get->size - diff); } - return tag & 0x7fffffff; -} - -static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, - lfs_tag_t getmask, lfs_tag_t gettag, void *buffer) { - lfs_stag_t getdiff = 0; - if (lfs->globals.hasmove && - lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0 && - lfs_tag_id(gettag) <= lfs->globals.id) { - // synthetic moves - gettag += LFS_MKTAG(0, 1, 0); - getdiff -= LFS_MKTAG(0, 1, 0); - } - - return lfs_dir_traverseget(lfs, dir, NULL, getmask, gettag, getdiff, - lfs_dir_get_read, &(struct lfs_dir_get_read){ - lfs, buffer, lfs_tag_size(gettag)}); + return LFS_ERR_NOENT; } static int lfs_dir_traverse_filter(void *p, @@ -597,46 +564,36 @@ static int lfs_dir_traverse_filter(void *p, (void)buffer; // check for redundancy - lfs_tag_t mask = (lfs_tag_isuser(tag) ? 0x7fffe000 : 0x783fe000); - if (!lfs_tag_subtype(tag) == LFS_TYPE_CREATE && ( - (mask & tag) == (mask & *filtertag) || - (lfs_tag_subtype(tag) == LFS_TYPE_DELETE && - lfs_tag_id(tag) == lfs_tag_id(*filtertag)))) { + uint32_t mask = LFS_MKTAG(0x7ff, 0x3ff, 0); + if ((mask & tag) == (mask & *filtertag) || + (mask & tag) == (LFS_MKTAG(LFS_TYPE_DELETE, 0, 0) | + (LFS_MKTAG(0, 0x3ff, 0) & *filtertag))) { return true; } // check if we need to adjust for created/deleted tags - if (lfs_tag_subtype(tag) == LFS_TYPE_CREATE && + if (lfs_tag_type1(tag) == LFS_TYPE_SPLICE && lfs_tag_id(tag) <= lfs_tag_id(*filtertag)) { - *filtertag += LFS_MKTAG(0, 1, 0); - } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE && - lfs_tag_id(tag) < lfs_tag_id(*filtertag)) { - *filtertag -= LFS_MKTAG(0, 1, 0); + *filtertag += LFS_MKTAG(0, lfs_tag_splice(tag), 0); } return false; } -static int lfs_dir_traverseall(lfs_t *lfs, const lfs_mdir_t *dir, - const struct lfs_mattr *attrs, - lfs_off_t off, lfs_tag_t ptag, int i, - uint16_t begin, uint16_t end, lfs_stag_t diff, +static int lfs_dir_traverse(lfs_t *lfs, + const lfs_mdir_t *dir, lfs_off_t off, lfs_tag_t ptag, + const struct lfs_mattr *attrs, int attrcount, + lfs_tag_t tmask, lfs_tag_t ttag, + uint16_t begin, uint16_t end, int16_t diff, int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { - // precompute length of attrs so we can iterate backwards, this - // lets us "append" commits - int attrcount = 0; - for (const struct lfs_mattr *a = attrs; a; a = a->next) { - attrcount += 1; - } - - // iterate over directory and attrs - while (off+lfs_tag_dsize(ptag) < dir->off || i < attrcount) { + // iterate over directory and attrs, we iterate over attrs in reverse order + // which lets us "append" commits + while (off+lfs_tag_dsize(ptag) < dir->off || attrcount > 0) { lfs_tag_t tag; const void *buffer; struct lfs_diskoff disk; if (off+lfs_tag_dsize(ptag) < dir->off) { off += lfs_tag_dsize(ptag); - int err = lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, sizeof(tag), dir->pair[0], off, &tag, sizeof(tag)); @@ -651,27 +608,27 @@ static int lfs_dir_traverseall(lfs_t *lfs, const lfs_mdir_t *dir, ptag = tag; } else { const struct lfs_mattr *a = attrs; - for (int j = 0; j < attrcount-i-1; j++) { + for (int j = 0; j < attrcount-1; j++) { a = a->next; } tag = a->tag; buffer = a->buffer; - i += 1; + attrcount -= 1; + } + + lfs_tag_t mask = LFS_MKTAG(0x7ff, 0, 0); + if ((mask & tmask & tag) != (mask & tmask & ttag)) { + continue; } // do we need to filter? inlining the filtering logic here allows // for some minor optimizations - if (end <= 0x1ff) { - if (!lfs_tag_isuser(tag) && - lfs_tag_subtype(tag) != LFS_TYPE_STRUCT && - lfs_tag_subtype(tag) != LFS_TYPE_FROM) { - continue; - } - + if (lfs_tag_id(tmask) != 0) { // scan for duplicates and update tag based on creates/deletes - int filter = lfs_dir_traverseall(lfs, dir, attrs, - off, ptag, i, 0, -1, 0, + int filter = lfs_dir_traverse(lfs, + dir, off, ptag, attrs, attrcount, + 0, 0, 0, 0, 0, lfs_dir_traverse_filter, &tag); if (filter < 0) { return filter; @@ -688,27 +645,28 @@ static int lfs_dir_traverseall(lfs_t *lfs, const lfs_mdir_t *dir, } // handle special cases for mcu-side operations - if (lfs_tag_type(tag) == LFS_FROM_MOVE) { + if (lfs_tag_type3(tag) == LFS_FROM_MOVE) { uint16_t fromid = lfs_tag_size(tag); uint16_t toid = lfs_tag_id(tag); - int err = lfs_dir_traverseall(lfs, buffer, NULL, - 0, 0xffffffff, 0, fromid, fromid+1, - LFS_MKTAG(0, toid - fromid, 0) + diff, + int err = lfs_dir_traverse(lfs, + buffer, 0, 0xffffffff, NULL, 0, + LFS_MKTAG(0x600, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_STRUCT, 0, 0), + fromid, fromid+1, toid-fromid+diff, cb, data); if (err) { return err; } - } else if (lfs_tag_type(tag) == LFS_FROM_USERATTRS) { + } else if (lfs_tag_type3(tag) == LFS_FROM_USERATTRS) { for (const struct lfs_attr *a = buffer; a; a = a->next) { - int err = cb(data, - LFS_MKTAG(0x100 | a->type, lfs_tag_id(tag), a->size) - + diff, a->buffer); + int err = cb(data, LFS_MKTAG(LFS_TYPE_USERATTR + a->type, + lfs_tag_id(tag) + diff, a->size), a->buffer); if (err) { return err; } } } else { - int err = cb(data, tag + diff, buffer); + int err = cb(data, tag + LFS_MKTAG(0, diff, 0), buffer); if (err) { return err; } @@ -718,30 +676,14 @@ static int lfs_dir_traverseall(lfs_t *lfs, const lfs_mdir_t *dir, return 0; } -static int lfs_dir_traverse(lfs_t *lfs, const lfs_mdir_t *dir, - const struct lfs_mattr *attrs, - uint16_t begin, uint16_t end, lfs_stag_t diff, - int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { - // grab create tags first, this is needed because we can't really order - // tags by ids otherwise, and our insertion cleverness becomes a problem - for (uint16_t i = begin; i < end; i++) { - int err = lfs_dir_traverseget(lfs, dir, attrs, - 0x783fe000, LFS_MKTAG(LFS_TYPE_CREATE, i, 0), - diff, cb, data); - if (err) { - return err; - } - } - - // then just grab every other tag in range, order is no longer a concern - return lfs_dir_traverseall(lfs, dir, attrs, - 0, 0xffffffff, 0, begin, end, diff, cb, data); -} - static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2], - lfs_tag_t matchmask, lfs_tag_t matchtag, + lfs_tag_t fmask, lfs_tag_t ftag, uint16_t *id, int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { + // we can find tag very efficiently during a fetch, since we're already + // scanning the entire directory + lfs_stag_t besttag = -1; + // find the block with the most recent revision uint32_t revs[2]; int r = 0; @@ -759,41 +701,32 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } } - // now fetch the actual dir (and find match) - lfs_stag_t foundtag = 0; - dir->pair[0] = pair[0]; - dir->pair[1] = pair[1]; - dir->off = 0; - if (r != 0) { - lfs_pair_swap(dir->pair); - lfs_pair_swap(revs); - } + dir->pair[0] = pair[(r+0)%2]; + dir->pair[1] = pair[(r+1)%2]; + dir->rev = revs[(r+0)%2]; + dir->off = 0; // nonzero = found some commits - // scan tags and check crcs + // now scan tags to fetch the actual dir and find possible match for (int i = 0; i < 2; i++) { - lfs_block_t block = dir->pair[0]; - lfs_off_t off = sizeof(uint32_t); + lfs_off_t off = 0; lfs_tag_t ptag = 0xffffffff; - lfs_tag_t tempfoundtag = foundtag; - lfs_mdir_t temp = { - .pair = {dir->pair[0], dir->pair[1]}, - .rev = revs[0], - .tail = {0xffffffff, 0xffffffff}, - .split = false, - .count = 0, - }; + uint16_t tempcount = 0; + lfs_block_t temptail[2] = {0xffffffff, 0xffffffff}; + bool tempsplit = false; + lfs_stag_t tempbesttag = besttag; - temp.rev = lfs_tole32(temp.rev); - uint32_t crc = lfs_crc(0xffffffff, &temp.rev, sizeof(temp.rev)); - temp.rev = lfs_fromle32(temp.rev); + dir->rev = lfs_tole32(dir->rev); + uint32_t crc = lfs_crc(0xffffffff, &dir->rev, sizeof(dir->rev)); + dir->rev = lfs_fromle32(dir->rev); while (true) { // extract next tag lfs_tag_t tag; + off += lfs_tag_dsize(ptag); int err = lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - block, off, &tag, sizeof(tag)); + dir->pair[0], off, &tag, sizeof(tag)); if (err) { if (err == LFS_ERR_CORRUPT) { // can't continue? @@ -806,24 +739,21 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, crc = lfs_crc(crc, &tag, sizeof(tag)); tag = lfs_frombe32(tag) ^ ptag; - // next commit not yet programmed - if (!lfs_tag_isvalid(tag)) { - dir->erased = (lfs_tag_subtype(ptag) == LFS_TYPE_CRC); + // next commit not yet programmed or we're not in valid range + if (!lfs_tag_isvalid(tag) || + off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { + dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC); break; } - // check we're in valid range - if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { - dir->erased = false; - break; - } + ptag = tag; - if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) { + if (lfs_tag_type1(tag) == LFS_TYPE_CRC) { // check the crc attr uint32_t dcrc; err = lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - block, off+sizeof(tag), &dcrc, sizeof(dcrc)); + dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { if (err == LFS_ERR_CORRUPT) { dir->erased = false; @@ -839,113 +769,127 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } // reset the next bit if we need to - tag ^= (lfs_tag_type(tag) & 1) << 31; + ptag ^= (lfs_tag_chunk(tag) & 1) << 31; + + // toss our crc into the filesystem seed for + // pseudorandom numbers lfs->seed ^= crc; - crc = 0xffffffff; // update with what's found so far - foundtag = tempfoundtag; - *dir = temp; + besttag = tempbesttag; dir->off = off + lfs_tag_dsize(tag); - dir->etag = tag; - } else { - // crc the entry first, leaving it in the cache - for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) { - uint8_t dat; - err = lfs_bd_read(lfs, - NULL, &lfs->rcache, lfs->cfg->block_size, - block, off+j, &dat, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - return err; - } + dir->etag = ptag; + dir->count = tempcount; + dir->tail[0] = temptail[0]; + dir->tail[1] = temptail[1]; + dir->split = tempsplit; - crc = lfs_crc(crc, &dat, 1); - } - - // check for special tags - if (lfs_tag_subtype(tag) == LFS_TYPE_CREATE) { - temp.count += 1; + // reset crc + crc = 0xffffffff; + continue; + } - if (tempfoundtag && - lfs_tag_id(tag) <= lfs_tag_id(tempfoundtag)) { - tempfoundtag += LFS_MKTAG(0, 1, 0); - } - } else if (lfs_tag_subtype(tag) == LFS_TYPE_DELETE) { - LFS_ASSERT(temp.count > 0); - temp.count -= 1; - - if (tempfoundtag && !lfs_tag_isdelete(tempfoundtag) && - lfs_tag_id(tag) == lfs_tag_id(tempfoundtag)) { - tempfoundtag = 0; - } else if (tempfoundtag && - lfs_tag_id(tag) < lfs_tag_id(tempfoundtag)) { - tempfoundtag -= LFS_MKTAG(0, 1, 0); - } - } else if (lfs_tag_subtype(tag) == LFS_TYPE_TAIL) { - temp.split = (lfs_tag_type(tag) & 1); - err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - block, off+sizeof(tag), - &temp.tail, sizeof(temp.tail)); - if (err) { - if (err == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } + // crc the entry first, hopefully leaving it in the cache + for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) { + uint8_t dat; + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+j, &dat, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + dir->erased = false; + break; } - lfs_pair_fromle32(temp.tail); + return err; } - if ((tag & matchmask) == (matchtag & matchmask)) { - // found a match? - if (lfs_tag_isdelete(tag)) { - tempfoundtag = 0; - } else if (cb) { - int res = cb(data, tag, &(struct lfs_diskoff){ - block, off+sizeof(tag)}); - if (res < 0) { - if (res == LFS_ERR_CORRUPT) { - dir->erased = false; - break; - } - return res; - } + crc = lfs_crc(crc, &dat, 1); + } - if (res && (!tempfoundtag || - lfs_tag_id(res) <= lfs_tag_id(tempfoundtag))) { - tempfoundtag = res; - } + // directory modification tags? + if (lfs_tag_type1(tag) == LFS_TYPE_NAME) { + // increase count of files if necessary + if (lfs_tag_id(tag) >= tempcount) { + tempcount = lfs_tag_id(tag) + 1; + } + } else if (lfs_tag_type1(tag) == LFS_TYPE_SPLICE) { + tempcount += lfs_tag_splice(tag); + + if (tag == (LFS_MKTAG(LFS_TYPE_DELETE, 0, 0) | + (LFS_MKTAG(0, 0x3ff, 0) & tempbesttag))) { + tempbesttag |= 0x80000000; + } else if (tempbesttag != -1 && + lfs_tag_id(tag) <= lfs_tag_id(tempbesttag)) { + tempbesttag += LFS_MKTAG(0, lfs_tag_splice(tag), 0); + } + } else if (lfs_tag_type1(tag) == LFS_TYPE_TAIL) { + tempsplit = (lfs_tag_chunk(tag) & 1); + err = lfs_bd_read(lfs, + &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + dir->pair[0], off+sizeof(tag), + &temptail, sizeof(temptail)); + if (err) { + if (err == LFS_ERR_CORRUPT) { + dir->erased = false; + break; } } + lfs_pair_fromle32(temptail); } - ptag = tag; - off += lfs_tag_dsize(tag); + // found a match for our fetcher? + if ((fmask & tag) == (fmask & ftag)) { + int res = cb(data, tag, &(struct lfs_diskoff){ + dir->pair[0], off+sizeof(tag)}); + if (res < 0) { + if (res == LFS_ERR_CORRUPT) { + dir->erased = false; + break; + } + return res; + } + + if (res == LFS_CMP_EQ) { + // found a match + tempbesttag = tag; + } else if (res == LFS_CMP_GT && + lfs_tag_id(tag) <= lfs_tag_id(tempbesttag)) { + // found a greater match, keep track to keep things sorted + tempbesttag = tag | 0x80000000; + } + } } // consider what we have good enough if (dir->off > 0) { // synthetic move - if (foundtag && - lfs->globals.hasmove && + if (lfs->globals.hasmove && lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0) { - if (lfs->globals.id == lfs_tag_id(foundtag)) { - foundtag = 0; - } else if (lfs->globals.id < lfs_tag_id(foundtag)) { - foundtag -= LFS_MKTAG(0, 1, 0); + if (lfs->globals.id == lfs_tag_id(besttag)) { + besttag |= 0x80000000; + } else if (besttag != -1 && + lfs->globals.id < lfs_tag_id(besttag)) { + besttag -= LFS_MKTAG(0, 1, 0); } } - return foundtag; + // found tag? or found best id? + if (id) { + *id = lfs_min(lfs_tag_id(besttag), dir->count); + } + + if (lfs_tag_isvalid(besttag)) { + return besttag; + } else if (lfs_tag_id(besttag) < dir->count) { + return LFS_ERR_NOENT; + } else { + return 0; + } } - // failed, try the other crc? + // failed, try the other block? lfs_pair_swap(dir->pair); - lfs_pair_swap(revs); + dir->rev = revs[(r+1)%2]; } LFS_ERROR("Corrupted dir pair at %"PRIu32" %"PRIu32, @@ -955,22 +899,23 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, static int lfs_dir_fetch(lfs_t *lfs, lfs_mdir_t *dir, const lfs_block_t pair[2]) { - return lfs_dir_fetchmatch(lfs, dir, pair, - 0xffffffff, 0x00000000, NULL, NULL); + // note, mask=-1, tag=0 can never match a tag since this + // pattern has the invalid bit set + return lfs_dir_fetchmatch(lfs, dir, pair, -1, 0, NULL, NULL, NULL); } static int lfs_dir_getglobals(lfs_t *lfs, const lfs_mdir_t *dir, struct lfs_globals *globals) { struct lfs_globals locals; - lfs_stag_t res = lfs_dir_get(lfs, dir, 0x78000000, + lfs_stag_t res = lfs_dir_get(lfs, dir, LFS_MKTAG(0x700, 0, 0), LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); if (res < 0 && res != LFS_ERR_NOENT) { return res; } if (res != LFS_ERR_NOENT) { - locals.hasmove = (lfs_tag_type(res) & 2); - locals.orphans = (lfs_tag_type(res) & 1); + locals.hasmove = (lfs_tag_type3(res) & 2); + locals.orphans = (lfs_tag_type3(res) & 1); // xor together to find resulting globals lfs_global_xor(globals, &locals); } @@ -980,32 +925,32 @@ static int lfs_dir_getglobals(lfs_t *lfs, const lfs_mdir_t *dir, static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir, uint16_t id, struct lfs_info *info) { - if (id == 0x1ff) { + if (id == 0x3ff) { // special case for root strcpy(info->name, "/"); info->type = LFS_TYPE_DIR; return 0; } - lfs_stag_t tag = lfs_dir_get(lfs, dir, 0x7c3fe000, - LFS_MKTAG(LFS_TYPE_DIR, id, lfs->name_max+1), info->name); + lfs_stag_t tag = lfs_dir_get(lfs, dir, LFS_MKTAG(0x780, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name); if (tag < 0) { return tag; } - info->type = lfs_tag_type(tag); + info->type = lfs_tag_type3(tag); struct lfs_ctz ctz; - tag = lfs_dir_get(lfs, dir, 0x783fe000, + tag = lfs_dir_get(lfs, dir, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); if (tag < 0) { return tag; } lfs_ctz_fromle32(&ctz); - if (lfs_tag_type(tag) == LFS_TYPE_CTZSTRUCT) { + if (lfs_tag_type3(tag) == LFS_TYPE_CTZSTRUCT) { info->size = ctz.size; - } else if (lfs_tag_type(tag) == LFS_TYPE_INLINESTRUCT) { + } else if (lfs_tag_type3(tag) == LFS_TYPE_INLINESTRUCT) { info->size = lfs_tag_size(tag); } @@ -1024,35 +969,34 @@ static int lfs_dir_find_match(void *data, lfs_t *lfs = name->lfs; const struct lfs_diskoff *disk = buffer; + // compare with disk lfs_size_t diff = lfs_min(name->size, lfs_tag_size(tag)); int res = lfs_bd_cmp(lfs, NULL, &lfs->rcache, diff, disk->block, disk->off, name->name, diff); - if (res < 0) { + if (res != LFS_CMP_EQ) { return res; } - // found match? - if (res == 0 && name->size == lfs_tag_size(tag)) { - return tag; + // only equal if our size is still the same + if (name->size != lfs_tag_size(tag)) { + return (name->size < lfs_tag_size(tag)) ? LFS_CMP_LT : LFS_CMP_GT; } - // a greater name found, exit early - if (res > 1 && lfs_tag_type(tag) != LFS_TYPE_SUPERBLOCK) { - return tag | 0x1fff; - } - - // no match keep looking - return 0; + // found a match! + return LFS_CMP_EQ; } static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, const char **path, uint16_t *id) { // we reduce path to a single name if we can find it const char *name = *path; + if (id) { + *id = 0x3ff; + } // default to root dir - lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x1ff, 0); + lfs_stag_t tag = LFS_MKTAG(LFS_TYPE_DIR, 0x3ff, 0); dir->tail[0] = lfs->root[0]; dir->tail[1] = lfs->root[1]; @@ -1102,13 +1046,13 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, *path = name; // only continue if we hit a directory - if (lfs_tag_type(tag) != LFS_TYPE_DIR) { + if (lfs_tag_type3(tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } // grab the entry data - if (lfs_tag_id(tag) != 0x1ff) { - lfs_stag_t res = lfs_dir_get(lfs, dir, 0x783fe000, + if (lfs_tag_id(tag) != 0x3ff) { + lfs_stag_t res = lfs_dir_get(lfs, dir, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), dir->tail); if (res < 0) { return res; @@ -1119,30 +1063,21 @@ static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir, // find entry matching name while (true) { tag = lfs_dir_fetchmatch(lfs, dir, dir->tail, - 0x7c000000, LFS_MKTAG(LFS_TYPE_DIR, 0, namelen), + LFS_MKTAG(0x780, 0, 0), + LFS_MKTAG(LFS_TYPE_NAME, 0, namelen), + // are we last name? + (strchr(name, '/') == NULL) ? id : NULL, lfs_dir_find_match, &(struct lfs_dir_find_match){ lfs, name, namelen}); if (tag < 0) { return tag; } - if (id) { - if (strchr(name, '/') != NULL) { - // if path is not only name we're not valid candidate - // for creation - *id = 0x1ff; - } else if (tag) { - *id = lfs_tag_id(tag); - } else { - *id = dir->count; - } - } - - if (tag && !lfs_tag_isdelete(tag)) { + if (tag) { break; } - if (lfs_tag_isdelete(tag) || !dir->split) { + if (!dir->split) { return LFS_ERR_NOENT; } } @@ -1227,7 +1162,7 @@ static int lfs_dir_commitglobals(lfs_t *lfs, struct lfs_commit *commit, struct lfs_globals *globals) { return lfs_dir_commitattr(lfs, commit, LFS_MKTAG(LFS_TYPE_GLOBALS + 2*globals->hasmove + globals->orphans, - 0x1ff, 10), globals); + 0x3ff, 10), globals); } static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { @@ -1247,7 +1182,7 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { // build crc tag bool reset = ~lfs_frombe32(tag) >> 31; tag = LFS_MKTAG(LFS_TYPE_CRC + reset, - 0x1ff, off - (commit->off+sizeof(lfs_tag_t))); + 0x3ff, off - (commit->off+sizeof(lfs_tag_t))); // write out crc uint32_t footer[2]; @@ -1347,12 +1282,12 @@ static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mdir_t *tail) { // update pred's tail return lfs_dir_commit(lfs, dir, LFS_MKATTR(LFS_TYPE_TAIL + dir->split, - 0x1ff, dir->tail, sizeof(dir->tail), + 0x3ff, dir->tail, sizeof(dir->tail), NULL)); } static int lfs_dir_split(lfs_t *lfs, - lfs_mdir_t *dir, const struct lfs_mattr *attrs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount, lfs_mdir_t *source, uint16_t split, uint16_t end) { // create tail directory lfs_mdir_t tail; @@ -1365,7 +1300,7 @@ static int lfs_dir_split(lfs_t *lfs, tail.tail[0] = dir->tail[0]; tail.tail[1] = dir->tail[1]; - err = lfs_dir_compact(lfs, &tail, attrs, source, split, end); + err = lfs_dir_compact(lfs, &tail, attrs, attrcount, source, split, end); if (err) { return err; } @@ -1402,7 +1337,7 @@ static int lfs_dir_commit_commit(void *p, lfs_tag_t tag, const void *buffer) { } static int lfs_dir_compact(lfs_t *lfs, - lfs_mdir_t *dir, const struct lfs_mattr *attrs, + lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; @@ -1413,7 +1348,11 @@ static int lfs_dir_compact(lfs_t *lfs, while (true) { // find size lfs_size_t size = 0; - int err = lfs_dir_traverse(lfs, source, attrs, begin, end, 0, + int err = lfs_dir_traverse(lfs, + source, 0, 0xffffffff, attrs, attrcount, + LFS_MKTAG(0x400, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_NAME, 0, 0), + begin, end, -begin, lfs_dir_commit_size, &size); if (err) { return err; @@ -1431,7 +1370,8 @@ static int lfs_dir_compact(lfs_t *lfs, // largest size that fits with a small binary search, but right now // it's not worth the code size uint16_t split = (end - begin) / 2; - err = lfs_dir_split(lfs, dir, attrs, source, begin+split, end); + err = lfs_dir_split(lfs, dir, attrs, attrcount, + source, begin+split, end); if (err) { // if we fail to split, we may be able to overcompact, unless // we're too big for even the full block, in which case our @@ -1460,7 +1400,8 @@ static int lfs_dir_compact(lfs_t *lfs, // by itself, so expand cautiously if ((lfs_size_t)res < lfs->cfg->block_count/2) { LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); - int err = lfs_dir_split(lfs, dir, attrs, source, begin, end); + int err = lfs_dir_split(lfs, dir, attrs, attrcount, + source, begin, end); if (err && err != LFS_ERR_NOSPC) { return err; } @@ -1522,9 +1463,12 @@ static int lfs_dir_compact(lfs_t *lfs, return err; } - // commit with a move - err = lfs_dir_traverse(lfs, source, attrs, - begin, end, -LFS_MKTAG(0, begin, 0), + // traverse the directory, this time writing out all unique tags + err = lfs_dir_traverse(lfs, + source, 0, 0xffffffff, attrs, attrcount, + LFS_MKTAG(0x400, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_NAME, 0, 0), + begin, end, -begin, lfs_dir_commit_commit, &(struct lfs_dir_commit_commit){ lfs, &commit}); if (err) { @@ -1553,7 +1497,7 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_pair_tole32(dir->tail); err = lfs_dir_commitattr(lfs, &commit, LFS_MKTAG(LFS_TYPE_TAIL + dir->split, - 0x1ff, sizeof(dir->tail)), dir->tail); + 0x3ff, sizeof(dir->tail)), dir->tail); lfs_pair_fromle32(dir->tail); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1651,10 +1595,10 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, lfs_tag_t createtag = 0xffffffff; int attrcount = 0; for (const struct lfs_mattr *a = attrs; a; a = a->next) { - if (lfs_tag_subtype(a->tag) == LFS_TYPE_CREATE) { + if (lfs_tag_type3(a->tag) == LFS_TYPE_CREATE) { dir->count += 1; createtag = a->tag; - } else if (lfs_tag_subtype(a->tag) == LFS_TYPE_DELETE) { + } else if (lfs_tag_type3(a->tag) == LFS_TYPE_DELETE) { LFS_ASSERT(dir->count > 0); dir->count -= 1; deletetag = a->tag; @@ -1690,8 +1634,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // traverse attrs that need to be written out lfs_pair_tole32(dir->tail); - int err = lfs_dir_traverseall(lfs, dir, attrs, - dir->off, dir->etag, 0, 0, -1, 0, + int err = lfs_dir_traverse(lfs, + dir, dir->off, dir->etag, attrs, attrcount, + 0, 0, 0, 0, 0, lfs_dir_commit_commit, &(struct lfs_dir_commit_commit){ lfs, &commit}); lfs_pair_fromle32(dir->tail); @@ -1740,7 +1685,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // fall back to compaction lfs_cache_drop(lfs, &lfs->pcache); - int err = lfs_dir_compact(lfs, dir, attrs, dir, 0, dir->count); + int err = lfs_dir_compact(lfs, dir, attrs, attrcount, + dir, 0, dir->count); if (err) { return err; } @@ -1798,7 +1744,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { lfs_mdir_t cwd; uint16_t id; err = lfs_dir_find(lfs, &cwd, &path, &id); - if (!(err == LFS_ERR_NOENT && id != 0x1ff)) { + if (!(err == LFS_ERR_NOENT && id != 0x3ff)) { return (err < 0) ? err : LFS_ERR_EXIST; } @@ -1844,7 +1790,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { pred.tail[1] = dir.pair[1]; lfs_global_orphans(lfs, +1); err = lfs_dir_commit(lfs, &pred, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x1ff, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, pred.tail, sizeof(pred.tail), NULL)); if (err) { @@ -1858,10 +1804,11 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { err = lfs_dir_commit(lfs, &cwd, LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, + LFS_MKATTR(LFS_TYPE_CREATE, id, NULL, 0, (!cwd.split) - ? LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x1ff, + ? LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, cwd.tail, sizeof(cwd.tail), NULL) - : NULL))); + : NULL)))); lfs_pair_fromle32(dir.pair); if (err) { return err; @@ -1876,18 +1823,18 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { return tag; } - if (lfs_tag_type(tag) != LFS_TYPE_DIR) { + if (lfs_tag_type3(tag) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } lfs_block_t pair[2]; - if (lfs_tag_id(tag) == 0x1ff) { + if (lfs_tag_id(tag) == 0x3ff) { // handle root dir separately pair[0] = lfs->root[0]; pair[1] = lfs->root[1]; } else { // get dir pair from parent - lfs_stag_t res = lfs_dir_get(lfs, &dir->m, 0x783fe000, + lfs_stag_t res = lfs_dir_get(lfs, &dir->m, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -2238,7 +2185,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // allocate entry for file if it doesn't exist lfs_stag_t tag = lfs_dir_find(lfs, &file->m, &path, &file->id); - if (tag < 0 && !(tag == LFS_ERR_NOENT && file->id != 0x1ff)) { + if (tag < 0 && !(tag == LFS_ERR_NOENT && file->id != 0x3ff)) { err = tag; goto cleanup; } @@ -2265,7 +2212,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, err = lfs_dir_commit(lfs, &file->m, LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, NULL, 0, LFS_MKATTR(LFS_TYPE_REG, file->id, path, nlen, - NULL))); + LFS_MKATTR(LFS_TYPE_CREATE, file->id, NULL, 0, + NULL)))); if (err) { err = LFS_ERR_NAMETOOLONG; goto cleanup; @@ -2275,7 +2223,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } else if (flags & LFS_O_EXCL) { err = LFS_ERR_EXIST; goto cleanup; - } else if (lfs_tag_type(tag) != LFS_TYPE_REG) { + } else if (lfs_tag_type3(tag) != LFS_TYPE_REG) { err = LFS_ERR_ISDIR; goto cleanup; } else if (flags & LFS_O_TRUNC) { @@ -2284,7 +2232,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, file->flags |= LFS_F_DIRTY; } else { // try to load what's on disk, if it's inlined we'll fix it later - tag = lfs_dir_get(lfs, &file->m, 0x783fe000, + tag = lfs_dir_get(lfs, &file->m, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, file->id, 8), &file->ctz); if (tag < 0) { err = tag; @@ -2296,8 +2244,10 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // fetch attrs for (const struct lfs_attr *a = file->cfg->attrs; a; a = a->next) { if ((file->flags & 3) != LFS_O_WRONLY) { - lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x7fffe000, - LFS_MKTAG(0x100 | a->type, file->id, a->size), a->buffer); + lfs_stag_t res = lfs_dir_get(lfs, &file->m, + LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_USERATTR + a->type, file->id, a->size), + a->buffer); if (res < 0 && res != LFS_ERR_NOENT) { err = res; goto cleanup; @@ -2328,7 +2278,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // zero to avoid information leak lfs_cache_zero(lfs, &file->cache); - if (lfs_tag_type(tag) == LFS_TYPE_INLINESTRUCT) { + if (lfs_tag_type3(tag) == LFS_TYPE_INLINESTRUCT) { // load inline files file->ctz.head = 0xfffffffe; file->ctz.size = lfs_tag_size(tag); @@ -2339,7 +2289,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // don't always read (may be new/trunc file) if (file->ctz.size > 0) { - lfs_stag_t res = lfs_dir_get(lfs, &file->m, 0x783fe000, + lfs_stag_t res = lfs_dir_get(lfs, &file->m, + LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->ctz.size), file->cache.buffer); if (res < 0) { @@ -2896,10 +2847,10 @@ int lfs_remove(lfs_t *lfs, const char *path) { } lfs_mdir_t dir; - if (lfs_tag_type(tag) == LFS_TYPE_DIR) { + if (lfs_tag_type3(tag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t pair[2]; - lfs_stag_t res = lfs_dir_get(lfs, &cwd, 0x783fe000, + lfs_stag_t res = lfs_dir_get(lfs, &cwd, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair); if (res < 0) { return res; @@ -2927,7 +2878,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { return err; } - if (lfs_tag_type(tag) == LFS_TYPE_DIR) { + if (lfs_tag_type3(tag) == LFS_TYPE_DIR) { // fix orphan lfs_global_orphans(lfs, -1); @@ -2963,7 +2914,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_mdir_t newcwd; uint16_t newid; lfs_stag_t prevtag = lfs_dir_find(lfs, &newcwd, &newpath, &newid); - if (prevtag < 0 && !(prevtag == LFS_ERR_NOENT && newid != 0x1ff)) { + if (prevtag < 0 && !(prevtag == LFS_ERR_NOENT && newid != 0x3ff)) { return err; } @@ -2974,12 +2925,12 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (nlen > lfs->name_max) { return LFS_ERR_NAMETOOLONG; } - } else if (lfs_tag_type(prevtag) != lfs_tag_type(oldtag)) { + } else if (lfs_tag_type3(prevtag) != lfs_tag_type3(oldtag)) { return LFS_ERR_ISDIR; - } else if (lfs_tag_type(prevtag) == LFS_TYPE_DIR) { + } else if (lfs_tag_type3(prevtag) == LFS_TYPE_DIR) { // must be empty before removal lfs_block_t prevpair[2]; - lfs_stag_t res = lfs_dir_get(lfs, &newcwd, 0x783fe000, + lfs_stag_t res = lfs_dir_get(lfs, &newcwd, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair); if (res < 0) { return res; @@ -3014,10 +2965,11 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // move over all attributes err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTR(LFS_FROM_MOVE, newid, &oldcwd, lfs_tag_id(oldtag), - LFS_MKATTR(lfs_tag_type(oldtag), newid, newpath, strlen(newpath), + LFS_MKATTR(lfs_tag_type3(oldtag), newid, newpath, strlen(newpath), + LFS_MKATTR(LFS_TYPE_CREATE, newid, NULL, 0, (prevtag != LFS_ERR_NOENT) ? LFS_MKATTR(LFS_TYPE_DELETE, newid, NULL, 0, NULL) - : NULL))); + : NULL)))); if (err) { return err; } @@ -3031,7 +2983,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } } - if (prevtag != LFS_ERR_NOENT && lfs_tag_type(prevtag) == LFS_TYPE_DIR) { + if (prevtag != LFS_ERR_NOENT && lfs_tag_type3(prevtag) == LFS_TYPE_DIR) { // fix orphan lfs_global_orphans(lfs, -1); @@ -3058,7 +3010,7 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, } uint16_t id = lfs_tag_id(tag); - if (id == 0x1ff) { + if (id == 0x3ff) { // special case for root id = 0; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); @@ -3067,8 +3019,9 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path, } } - tag = lfs_dir_get(lfs, &cwd, 0x7fffe000, - LFS_MKTAG(0x100 | type, id, lfs_min(size, lfs->attr_max)), + tag = lfs_dir_get(lfs, &cwd, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_USERATTR + type, + id, lfs_min(size, lfs->attr_max)), buffer); if (tag < 0) { if (tag == LFS_ERR_NOENT) { @@ -3089,7 +3042,7 @@ static int lfs_commitattr(lfs_t *lfs, const char *path, } uint16_t id = lfs_tag_id(tag); - if (id == 0x1ff) { + if (id == 0x3ff) { // special case for root id = 0; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); @@ -3099,7 +3052,7 @@ static int lfs_commitattr(lfs_t *lfs, const char *path, } return lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(0x100 | type, id, buffer, size, + LFS_MKATTR(LFS_TYPE_USERATTR + type, id, buffer, size, NULL)); } @@ -3113,7 +3066,7 @@ int lfs_setattr(lfs_t *lfs, const char *path, } int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { - return lfs_commitattr(lfs, path, type, NULL, 0x1fff); + return lfs_commitattr(lfs, path, type, NULL, 0x3ff); } @@ -3268,7 +3221,8 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { LFS_MKATTR(LFS_TYPE_INLINESTRUCT, 0, &superblock, sizeof(superblock), LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, "littlefs", 8, - NULL))); + LFS_MKATTR(LFS_TYPE_CREATE, 0, NULL, 0, + NULL)))); if (err) { goto cleanup; } @@ -3295,8 +3249,10 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs_mdir_t dir = {.tail = {0, 1}}; while (!lfs_pair_isnull(dir.tail)) { // fetch next block in tail list - lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, 0x7fffe000, + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, + LFS_MKTAG(0x7ff, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), + NULL, lfs_dir_find_match, &(struct lfs_dir_find_match){ lfs, "littlefs", 8}); if (tag < 0) { @@ -3312,7 +3268,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // grab superblock lfs_superblock_t superblock; - tag = lfs_dir_get(lfs, &dir, 0x7fffe000, + tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), &superblock); if (tag < 0) { @@ -3439,7 +3395,7 @@ int lfs_fs_traverse(lfs_t *lfs, for (uint16_t id = 0; id < dir.count; id++) { struct lfs_ctz ctz; - lfs_stag_t tag = lfs_dir_get(lfs, &dir, 0x783fe000, + lfs_stag_t tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x700, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz); if (tag < 0) { if (tag == LFS_ERR_NOENT) { @@ -3449,7 +3405,7 @@ int lfs_fs_traverse(lfs_t *lfs, } lfs_ctz_fromle32(&ctz); - if (lfs_tag_type(tag) == LFS_TYPE_CTZSTRUCT) { + if (lfs_tag_type3(tag) == LFS_TYPE_CTZSTRUCT) { err = lfs_ctz_traverse(lfs, NULL, &lfs->rcache, ctz.head, ctz.size, cb, data); if (err) { @@ -3525,7 +3481,7 @@ static int lfs_fs_parent_match(void *data, } lfs_pair_fromle32(child); - return (lfs_pair_cmp(child, find->pair) == 0) ? tag : 0; + return (lfs_pair_cmp(child, find->pair) == 0) ? LFS_CMP_EQ : LFS_CMP_LT; } static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], @@ -3535,10 +3491,12 @@ static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], parent->tail[1] = 1; while (!lfs_pair_isnull(parent->tail)) { lfs_stag_t tag = lfs_dir_fetchmatch(lfs, parent, parent->tail, - 0x7fc01fff, LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), + LFS_MKTAG(0x7ff, 0, 0x3ff), + LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 0, 8), + NULL, lfs_fs_parent_match, &(struct lfs_fs_parent_match){ lfs, {pair[0], pair[1]}}); - if (tag) { + if (tag && tag != LFS_ERR_NOENT) { return tag; } } @@ -3600,7 +3558,7 @@ static int lfs_fs_relocate(lfs_t *lfs, parent.tail[1] = newpair[1]; err = lfs_dir_commit(lfs, &parent, LFS_MKATTR(LFS_TYPE_TAIL + parent.split, - 0x1ff, parent.tail, sizeof(parent.tail), + 0x3ff, parent.tail, sizeof(parent.tail), NULL)); if (err) { return err; @@ -3674,7 +3632,8 @@ static int lfs_fs_deorphan(lfs_t *lfs) { } lfs_block_t pair[2]; - lfs_stag_t res = lfs_dir_get(lfs, &parent, 0x7fffe000, tag, pair); + lfs_stag_t res = lfs_dir_get(lfs, &parent, + LFS_MKTAG(0x7ff, 0x3ff, 0), tag, pair); if (res < 0) { return res; } @@ -3689,7 +3648,7 @@ static int lfs_fs_deorphan(lfs_t *lfs) { pdir.tail[1] = pair[1]; err = lfs_dir_commit(lfs, &pdir, LFS_MKATTR(LFS_TYPE_SOFTTAIL, - 0x1ff, pdir.tail, sizeof(pdir.tail), + 0x3ff, pdir.tail, sizeof(pdir.tail), NULL)); if (err) { return err; @@ -3735,5 +3694,5 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs) { return err; } - return size; + return size; } diff --git a/lfs.h b/lfs.h index 85c31e39..05ba5405 100644 --- a/lfs.h +++ b/lfs.h @@ -45,25 +45,25 @@ typedef int32_t lfs_soff_t; typedef uint32_t lfs_block_t; // Maximum name size in bytes, may be redefined to reduce the size of the -// info struct. Limited to <= 8190. Stored in superblock and must be +// info struct. Limited to <= 1022. Stored in superblock and must be // respected by other littlefs drivers. #ifndef LFS_NAME_MAX -#define LFS_NAME_MAX 0xff +#define LFS_NAME_MAX 255 #endif // Maximum inline file size in bytes, may be redefined to limit RAM usage, // but littlefs will automatically limit the LFS_INLINE_MAX to the -// configured cache_size. Limited to <= 8190. Stored in superblock and must +// configured cache_size. Limited to <= 1022. Stored in superblock and must // be respected by other littlefs drivers. #ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 0x1ffe +#define LFS_INLINE_MAX 1022 #endif // Maximum size of custom attributes in bytes, may be redefined, but there is -// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 8190. Stored +// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 1022. Stored // in superblock and must be respected by other littlefs drivers. #ifndef LFS_ATTR_MAX -#define LFS_ATTR_MAX 0x1ffe +#define LFS_ATTR_MAX 1022 #endif // Maximum size of a file in bytes, may be redefined to limit to support other @@ -98,31 +98,33 @@ enum lfs_error { // File types enum lfs_type { // file types - LFS_TYPE_REG = 0x011, - LFS_TYPE_DIR = 0x010, + LFS_TYPE_REG = 0x001, + LFS_TYPE_DIR = 0x002, // internally used types - LFS_TYPE_USERATTR = 0x100, - LFS_TYPE_CREATE = 0x000, - LFS_TYPE_DELETE = 0x020, - LFS_TYPE_STRUCT = 0x040, - LFS_TYPE_TAIL = 0x080, - LFS_TYPE_SOFTTAIL = 0x080, - LFS_TYPE_HARDTAIL = 0x081, - LFS_TYPE_CRC = 0x0a0, - LFS_TYPE_SUPERBLOCK = 0x001, - LFS_TYPE_GLOBALS = 0x0e0, - - LFS_TYPE_DIRSTRUCT = 0x040, - LFS_TYPE_INLINESTRUCT = 0x041, - LFS_TYPE_CTZSTRUCT = 0x042, + LFS_TYPE_SPLICE = 0x400, + LFS_TYPE_NAME = 0x000, + LFS_TYPE_STRUCT = 0x200, + LFS_TYPE_USERATTR = 0x300, + LFS_TYPE_FROM = 0x100, + LFS_TYPE_TAIL = 0x600, + LFS_TYPE_GLOBALS = 0x700, + LFS_TYPE_CRC = 0x500, + + // internally used type specializations + LFS_TYPE_CREATE = 0x401, + LFS_TYPE_DELETE = 0x4ff, + LFS_TYPE_SUPERBLOCK = 0x0ff, + LFS_TYPE_DIRSTRUCT = 0x200, + LFS_TYPE_CTZSTRUCT = 0x202, + LFS_TYPE_INLINESTRUCT = 0x201, + LFS_TYPE_SOFTTAIL = 0x600, + LFS_TYPE_HARDTAIL = 0x601, + LFS_TYPE_MOVESTATE = 0x7ff, // internal chip sources - LFS_TYPE_FROM = 0x060, - LFS_FROM_MEM = 0x000, - LFS_FROM_DISK = 0x200, - LFS_FROM_MOVE = 0x061, - LFS_FROM_USERATTRS = 0x062, + LFS_FROM_MOVE = 0x101, + LFS_FROM_USERATTRS = 0x102, }; // File open flags @@ -314,8 +316,8 @@ typedef struct lfs_cache { typedef struct lfs_mdir { lfs_block_t pair[2]; uint32_t rev; - uint32_t etag; lfs_off_t off; + uint32_t etag; uint16_t count; bool erased; bool split; diff --git a/tests/corrupt.py b/tests/corrupt.py index b6671ff6..44f4f667 100755 --- a/tests/corrupt.py +++ b/tests/corrupt.py @@ -19,7 +19,7 @@ def corrupt(block): break tag ^= ntag - size = (tag & 0x1fff) if (tag & 0x1fff) != 0x1fff else 0 + size = (tag & 0x3ff) if (tag & 0x3ff) != 0x3ff else 0 file.seek(size, os.SEEK_CUR) # lob off last 3 bytes diff --git a/tests/debug.py b/tests/debug.py index c76dc3d3..f8c0484a 100755 --- a/tests/debug.py +++ b/tests/debug.py @@ -4,23 +4,29 @@ import binascii TYPES = { - (0x1ff, 0x011): 'create reg', - (0x1ff, 0x010): 'create dir', - (0x1ff, 0x001): 'superblock', - (0x1ff, 0x020): 'delete', - (0x1f0, 0x0e0): 'globals', - (0x1ff, 0x080): 'tail soft', - (0x1ff, 0x081): 'tail hard', - (0x1f0, 0x0a0): 'crc', - (0x1ff, 0x040): 'struct dir', - (0x1ff, 0x041): 'struct inline', - (0x1ff, 0x042): 'struct ctz', - (0x100, 0x100): 'attr', + (0x700, 0x400): 'splice', + (0x7ff, 0x401): 'create', + (0x7ff, 0x4ff): 'delete', + (0x700, 0x000): 'name', + (0x7ff, 0x001): 'name reg', + (0x7ff, 0x002): 'name dir', + (0x7ff, 0x0ff): 'name superblock', + (0x700, 0x200): 'struct', + (0x7ff, 0x200): 'struct dir', + (0x7ff, 0x202): 'struct ctz', + (0x7ff, 0x201): 'struct inline', + (0x700, 0x300): 'userattr', + (0x700, 0x600): 'tail', + (0x7ff, 0x600): 'tail soft', + (0x7ff, 0x601): 'tail hard', + (0x700, 0x700): 'gstate', + (0x7ff, 0x7ff): 'gstate move', + (0x700, 0x500): 'crc', } def typeof(type): - for prefix in range(9): - mask = 0x1ff & ~((1 << prefix)-1) + for prefix in range(12): + mask = 0x7ff & ~((1 << prefix)-1) if (mask, type & mask) in TYPES: return TYPES[mask, type & mask] + ( ' %0*x' % (prefix/4, type & ((1 << prefix)-1)) @@ -59,7 +65,7 @@ def main(*blocks): print "--- %s ---" % ', '.join(v for _,v in sorted(versions, reverse=True)) # go through each tag, print useful information - print "%-4s %-8s %-14s %3s %3s %s" % ( + print "%-4s %-8s %-14s %3s %4s %s" % ( 'off', 'tag', 'type', 'id', 'len', 'dump') tag = 0xffffffff @@ -75,26 +81,26 @@ def main(*blocks): tag ^= ntag off += 4 - type = (tag & 0x7fc00000) >> 22 - id = (tag & 0x003fe000) >> 13 - size = (tag & 0x00001fff) >> 0 - iscrc = (type & 0x1f0) == 0x0f0 + type = (tag & 0x7ff00000) >> 20 + id = (tag & 0x000ffc00) >> 10 + size = (tag & 0x000003ff) >> 0 + iscrc = (type & 0x700) == 0x500 - data = file.read(size if size != 0x1fff else 0) + data = file.read(size if size != 0x3ff else 0) if iscrc: crc = binascii.crc32(data[:4], crc) else: crc = binascii.crc32(data, crc) - print '%04x: %08x %-14s %3s %3s %-23s %-8s' % ( + print '%04x: %08x %-15s %3s %4s %-23s %-8s' % ( off, tag, typeof(type) + (' bad!' if iscrc and ~crc else ''), - id if id != 0x1ff else '.', - size if size != 0x1fff else 'x', + id if id != 0x3ff else '.', + size if size != 0x3ff else 'x', ' '.join('%02x' % ord(c) for c in data[:8]), ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8])) - off += size if size != 0x1fff else 0 + off += size if size != 0x3ff else 0 if iscrc: crc = 0 tag ^= (type & 1) << 31 diff --git a/tests/test_alloc.sh b/tests/test_alloc.sh index f924a29f..3f993e9f 100755 --- a/tests/test_alloc.sh +++ b/tests/test_alloc.sh @@ -253,7 +253,7 @@ tests/test.py << TEST // find out max file size lfs_mkdir(&lfs, "exhaustiondir") => 0; - for (int i = 0; i < 9; i++) { + for (int i = 0; i < 10; i++) { sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); lfs_mkdir(&lfs, (char*)buffer) => 0; } @@ -275,7 +275,7 @@ tests/test.py << TEST lfs_remove(&lfs, "exhaustion") => 0; lfs_remove(&lfs, "exhaustiondir") => 0; - for (int i = 0; i < 9; i++) { + for (int i = 0; i < 10; i++) { sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); lfs_remove(&lfs, (char*)buffer) => 0; } @@ -287,7 +287,7 @@ tests/test.py << TEST } lfs_file_sync(&lfs, &file[0]) => 0; - for (int i = 0; i < 9; i++) { + for (int i = 0; i < 10; i++) { sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i); lfs_mkdir(&lfs, (char*)buffer) => 0; } diff --git a/tests/test_format.sh b/tests/test_format.sh index d885f835..5a115359 100755 --- a/tests/test_format.sh +++ b/tests/test_format.sh @@ -12,8 +12,7 @@ TEST echo "--- Basic mounting ---" tests/test.py << TEST lfs_format(&lfs, &cfg) => 0; -TEST -tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; lfs_unmount(&lfs) => 0; TEST From 66d751544d3b63d6650ed66ae2cdb0cdab25a0f3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 4 Jan 2019 17:23:36 -0600 Subject: [PATCH 120/139] Modified global state format to work with new tag format The main difference here is a change from encoding "hasorphans" and "hasmove" bits in the tag itself. This worked with the old format, but in the new format the space these bits take up must be consistent for each tag type. The tradeoff is that the new tag format allows for up to 256 different global states which may be useful in the future (for example, a global free list). The new format encodes this info in the data blob, using an additional word of storage. This word is actually formatted the same as though it was a tag, which simplified internal handling and may allow other tag types in the future. Format for global state: [---- 96 bits ----] [1|- 11 -|- 10 -|- 10 -|--- 64 ---] ^ ^ ^ ^ ^- move dir pair | | | \-------------------------- unused, must be 0s | | \--------------------------------- move id | \---------------------------------------- type, 0xfff for move \--------------------------------------------- has orphans This also included another iteration over globals (renamed to gstate) with some simplifications to how globals are handled. --- lfs.c | 395 +++++++++++++++++++++++++++-------------------------- lfs.h | 8 +- lfs_util.h | 44 ------ 3 files changed, 208 insertions(+), 239 deletions(-) diff --git a/lfs.c b/lfs.c index 3d3aea05..d0c89f9d 100644 --- a/lfs.c +++ b/lfs.c @@ -331,56 +331,48 @@ struct lfs_diskoff { lfs_off_t off; }; -// operations on set of globals -static inline void lfs_global_xor(struct lfs_globals *a, - const struct lfs_globals *b) { - uint32_t *a32 = (uint32_t *)a; - const uint32_t *b32 = (const uint32_t *)b; - for (unsigned i = 0; i < sizeof(struct lfs_globals)/4; i++) { - a32[i] ^= b32[i]; +// operations on global state +static inline void lfs_gstate_xor(struct lfs_gstate *a, + const struct lfs_gstate *b) { + for (int i = 0; i < 3; i++) { + ((uint32_t*)a)[i] ^= ((const uint32_t*)b)[i]; } } -static inline bool lfs_global_iszero(const struct lfs_globals *a) { - const uint32_t *a32 = (const uint32_t *)a; - for (unsigned i = 0; i < sizeof(struct lfs_globals)/4; i++) { - if (a32[i] != 0) { +static inline bool lfs_gstate_iszero(const struct lfs_gstate *a) { + for (int i = 0; i < 3; i++) { + if (((uint32_t*)a)[i] != 0) { return false; } } return true; } -static inline void lfs_global_zero(struct lfs_globals *a) { - lfs_global_xor(a, a); +static inline bool lfs_gstate_hasorphans(const struct lfs_gstate *a) { + return lfs_tag_size(a->tag); } -static inline void lfs_global_fromle32(struct lfs_globals *a) { - lfs_pair_fromle32(a->pair); - a->id = lfs_fromle16(a->id); +static inline uint8_t lfs_gstate_getorphans(const struct lfs_gstate *a) { + return lfs_tag_size(a->tag); } -static inline void lfs_global_tole32(struct lfs_globals *a) { - lfs_pair_tole32(a->pair); - a->id = lfs_tole16(a->id); +static inline bool lfs_gstate_hasmove(const struct lfs_gstate *a) { + return lfs_tag_type1(a->tag); } -static inline void lfs_global_move(lfs_t *lfs, - bool hasmove, const lfs_block_t pair[2], uint16_t id) { - lfs_global_fromle32(&lfs->locals); - lfs_global_xor(&lfs->locals, &lfs->globals); - lfs->globals.hasmove = hasmove; - lfs->globals.pair[0] = pair[0]; - lfs->globals.pair[1] = pair[1]; - lfs->globals.id = id; - lfs_global_xor(&lfs->locals, &lfs->globals); - lfs_global_tole32(&lfs->locals); +static inline bool lfs_gstate_hasmovehere(const struct lfs_gstate *a, + const lfs_block_t *pair) { + return lfs_tag_type1(a->tag) && lfs_pair_cmp(a->pair, pair) == 0; } -static inline void lfs_global_orphans(lfs_t *lfs, int8_t orphans) { - lfs->locals.orphans ^= (lfs->globals.orphans == 0); - lfs->globals.orphans += orphans; - lfs->locals.orphans ^= (lfs->globals.orphans == 0); +static inline void lfs_gstate_fromle32(struct lfs_gstate *a) { + for (int i = 0; i < 3; i++) { + ((uint32_t*)a)[i] = lfs_fromle32(((uint32_t*)a)[i]); + } +} + +static inline void lfs_gstate_tole32(struct lfs_gstate *a) { + lfs_gstate_fromle32(a); } // other endianness operations @@ -427,6 +419,9 @@ static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent); static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]); +static void lfs_fs_preporphans(lfs_t *lfs, int8_t orphans); +static void lfs_fs_prepmove(lfs_t *lfs, + uint16_t id, const lfs_block_t pair[2]); static int lfs_fs_forceconsistency(lfs_t *lfs); static int lfs_deinit(lfs_t *lfs); @@ -501,9 +496,8 @@ static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, lfs_tag_t ntag = dir->etag; lfs_stag_t gdiff = 0; - if (lfs->globals.hasmove && - lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0 && - lfs_tag_id(gtag) <= lfs->globals.id) { + if (lfs_gstate_hasmovehere(&lfs->gstate, dir->pair) && + lfs_tag_id(gtag) <= lfs_tag_id(lfs->gstate.tag)) { // synthetic moves gdiff -= LFS_MKTAG(0, 1, 0); } @@ -582,13 +576,12 @@ static int lfs_dir_traverse_filter(void *p, static int lfs_dir_traverse(lfs_t *lfs, const lfs_mdir_t *dir, lfs_off_t off, lfs_tag_t ptag, - const struct lfs_mattr *attrs, int attrcount, + const struct lfs_mattr *attrs, int attrcount, bool hasseenmove, lfs_tag_t tmask, lfs_tag_t ttag, uint16_t begin, uint16_t end, int16_t diff, int (*cb)(void *data, lfs_tag_t tag, const void *buffer), void *data) { - // iterate over directory and attrs, we iterate over attrs in reverse order - // which lets us "append" commits - while (off+lfs_tag_dsize(ptag) < dir->off || attrcount > 0) { + // iterate over directory and attrs + while (true) { lfs_tag_t tag; const void *buffer; struct lfs_diskoff disk; @@ -606,7 +599,7 @@ static int lfs_dir_traverse(lfs_t *lfs, disk.off = off+sizeof(lfs_tag_t); buffer = &disk; ptag = tag; - } else { + } else if (attrcount > 0) { const struct lfs_mattr *a = attrs; for (int j = 0; j < attrcount-1; j++) { a = a->next; @@ -615,6 +608,15 @@ static int lfs_dir_traverse(lfs_t *lfs, tag = a->tag; buffer = a->buffer; attrcount -= 1; + } else if (!hasseenmove && + lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { + // Wait, we have pending move? Handle this here (we need to + // or else we risk letting moves fall out of date) + tag = lfs->gpending.tag & LFS_MKTAG(0x7ff, 0x3ff, 0); + buffer = NULL; + hasseenmove = true; + } else { + return 0; } lfs_tag_t mask = LFS_MKTAG(0x7ff, 0, 0); @@ -627,7 +629,7 @@ static int lfs_dir_traverse(lfs_t *lfs, if (lfs_tag_id(tmask) != 0) { // scan for duplicates and update tag based on creates/deletes int filter = lfs_dir_traverse(lfs, - dir, off, ptag, attrs, attrcount, + dir, off, ptag, attrs, attrcount, hasseenmove, 0, 0, 0, 0, 0, lfs_dir_traverse_filter, &tag); if (filter < 0) { @@ -649,7 +651,7 @@ static int lfs_dir_traverse(lfs_t *lfs, uint16_t fromid = lfs_tag_size(tag); uint16_t toid = lfs_tag_id(tag); int err = lfs_dir_traverse(lfs, - buffer, 0, 0xffffffff, NULL, 0, + buffer, 0, 0xffffffff, NULL, 0, true, LFS_MKTAG(0x600, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_STRUCT, 0, 0), fromid, fromid+1, toid-fromid+diff, @@ -672,8 +674,6 @@ static int lfs_dir_traverse(lfs_t *lfs, } } } - - return 0; } static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, @@ -824,10 +824,10 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } } else if (lfs_tag_type1(tag) == LFS_TYPE_TAIL) { tempsplit = (lfs_tag_chunk(tag) & 1); + err = lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, - dir->pair[0], off+sizeof(tag), - &temptail, sizeof(temptail)); + dir->pair[0], off+sizeof(tag), &temptail, 8); if (err) { if (err == LFS_ERR_CORRUPT) { dir->erased = false; @@ -863,12 +863,11 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // consider what we have good enough if (dir->off > 0) { // synthetic move - if (lfs->globals.hasmove && - lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0) { - if (lfs->globals.id == lfs_tag_id(besttag)) { + if (lfs_gstate_hasmovehere(&lfs->gstate, dir->pair)) { + if (lfs_tag_id(lfs->gstate.tag) == lfs_tag_id(besttag)) { besttag |= 0x80000000; } else if (besttag != -1 && - lfs->globals.id < lfs_tag_id(besttag)) { + lfs_tag_id(lfs->gstate.tag) < lfs_tag_id(besttag)) { besttag -= LFS_MKTAG(0, 1, 0); } } @@ -904,20 +903,18 @@ static int lfs_dir_fetch(lfs_t *lfs, return lfs_dir_fetchmatch(lfs, dir, pair, -1, 0, NULL, NULL, NULL); } -static int lfs_dir_getglobals(lfs_t *lfs, const lfs_mdir_t *dir, - struct lfs_globals *globals) { - struct lfs_globals locals; - lfs_stag_t res = lfs_dir_get(lfs, dir, LFS_MKTAG(0x700, 0, 0), - LFS_MKTAG(LFS_TYPE_GLOBALS, 0, 10), &locals); +static int lfs_dir_getgstate(lfs_t *lfs, const lfs_mdir_t *dir, + struct lfs_gstate *gstate) { + struct lfs_gstate temp; + lfs_stag_t res = lfs_dir_get(lfs, dir, LFS_MKTAG(0x7ff, 0, 0), + LFS_MKTAG(LFS_TYPE_MOVESTATE, 0, sizeof(temp)), &temp); if (res < 0 && res != LFS_ERR_NOENT) { return res; } if (res != LFS_ERR_NOENT) { - locals.hasmove = (lfs_tag_type3(res) & 2); - locals.orphans = (lfs_tag_type3(res) & 1); - // xor together to find resulting globals - lfs_global_xor(globals, &locals); + // xor together to find resulting gstate + lfs_gstate_xor(gstate, &temp); } return 0; @@ -1158,13 +1155,6 @@ static int lfs_dir_commitattr(lfs_t *lfs, struct lfs_commit *commit, return 0; } -static int lfs_dir_commitglobals(lfs_t *lfs, struct lfs_commit *commit, - struct lfs_globals *globals) { - return lfs_dir_commitattr(lfs, commit, - LFS_MKTAG(LFS_TYPE_GLOBALS + 2*globals->hasmove + globals->orphans, - 0x3ff, 10), globals); -} - static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { // align to program units lfs_off_t off = lfs_alignup(commit->off + 2*sizeof(uint32_t), @@ -1181,8 +1171,8 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { // build crc tag bool reset = ~lfs_frombe32(tag) >> 31; - tag = LFS_MKTAG(LFS_TYPE_CRC + reset, - 0x3ff, off - (commit->off+sizeof(lfs_tag_t))); + tag = LFS_MKTAG(LFS_TYPE_CRC + reset, 0x3ff, + off - (commit->off+sizeof(lfs_tag_t))); // write out crc uint32_t footer[2]; @@ -1267,23 +1257,24 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { return 0; } -static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, const lfs_mdir_t *tail) { - // steal tail - dir->tail[0] = tail->tail[0]; - dir->tail[1] = tail->tail[1]; - dir->split = tail->split; - +static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, lfs_mdir_t *tail) { // steal state - int err = lfs_dir_getglobals(lfs, tail, &lfs->locals); + int err = lfs_dir_getgstate(lfs, tail, &lfs->gdelta); if (err) { return err; } - // update pred's tail - return lfs_dir_commit(lfs, dir, - LFS_MKATTR(LFS_TYPE_TAIL + dir->split, - 0x3ff, dir->tail, sizeof(dir->tail), + // steal tail + lfs_pair_tole32(tail->tail); + err = lfs_dir_commit(lfs, dir, + LFS_MKATTR(LFS_TYPE_TAIL + tail->split, 0x3ff, tail->tail, 8, NULL)); + lfs_pair_fromle32(tail->tail); + if (err) { + return err; + } + + return 0; } static int lfs_dir_split(lfs_t *lfs, @@ -1341,7 +1332,6 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *source, uint16_t begin, uint16_t end) { // save some state in case block is bad const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; - struct lfs_globals globals, locals; bool relocated = false; bool exhausted = false; @@ -1349,7 +1339,7 @@ static int lfs_dir_compact(lfs_t *lfs, // find size lfs_size_t size = 0; int err = lfs_dir_traverse(lfs, - source, 0, 0xffffffff, attrs, attrcount, + source, 0, 0xffffffff, attrs, attrcount, false, LFS_MKTAG(0x400, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_NAME, 0, 0), begin, end, -begin, @@ -1358,10 +1348,10 @@ static int lfs_dir_compact(lfs_t *lfs, return err; } - // space is complicated, we need room for tail, crc, globals, + // space is complicated, we need room for tail, crc, gstate, // cleanup delete, and we cap at half a block to give room // for metadata updates - if (size <= lfs_min(lfs->cfg->block_size - 38, + if (size <= lfs_min(lfs->cfg->block_size - 36, lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size))) { break; } @@ -1376,7 +1366,7 @@ static int lfs_dir_compact(lfs_t *lfs, // if we fail to split, we may be able to overcompact, unless // we're too big for even the full block, in which case our // only option is to error - if (err == LFS_ERR_NOSPC && size <= lfs->cfg->block_size - 38) { + if (err == LFS_ERR_NOSPC && size <= lfs->cfg->block_size - 36) { break; } return err; @@ -1424,9 +1414,7 @@ static int lfs_dir_compact(lfs_t *lfs, if (true) { // There's nothing special about our global delta, so feed it into // our local global delta - globals = lfs->globals; - locals = lfs->locals; - int err = lfs_dir_getglobals(lfs, dir, &locals); + int err = lfs_dir_getgstate(lfs, dir, &lfs->gdelta); if (err) { return err; } @@ -1465,7 +1453,7 @@ static int lfs_dir_compact(lfs_t *lfs, // traverse the directory, this time writing out all unique tags err = lfs_dir_traverse(lfs, - source, 0, 0xffffffff, attrs, attrcount, + source, 0, 0xffffffff, attrs, attrcount, false, LFS_MKTAG(0x400, 0x3ff, 0), LFS_MKTAG(LFS_TYPE_NAME, 0, 0), begin, end, -begin, @@ -1478,27 +1466,32 @@ static int lfs_dir_compact(lfs_t *lfs, return err; } - if (!relocated && !lfs_global_iszero(&locals)) { - // commit any globals, unless we're relocating, - // in which case our parent will steal our globals - err = lfs_dir_commitglobals(lfs, &commit, &locals); + // commit tail, which may be new after last size check + if (!lfs_pair_isnull(dir->tail)) { + lfs_pair_tole32(dir->tail); + err = lfs_dir_commitattr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_TAIL + dir->split, 0x3ff, 8), + dir->tail); + lfs_pair_fromle32(dir->tail); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } + } - lfs_global_zero(&locals); + // need to update gstate now that we've acknowledged moves + if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { + lfs_fs_prepmove(lfs, 0x3ff, NULL); } - if (!lfs_pair_isnull(dir->tail)) { - // commit tail, which may be new after last size check - lfs_pair_tole32(dir->tail); + if (!relocated && !lfs_gstate_iszero(&lfs->gdelta)) { + // commit any globals, unless we're relocating, + // in which case our parent will steal our globals err = lfs_dir_commitattr(lfs, &commit, - LFS_MKTAG(LFS_TYPE_TAIL + dir->split, - 0x3ff, sizeof(dir->tail)), dir->tail); - lfs_pair_fromle32(dir->tail); + LFS_MKTAG(LFS_TYPE_MOVESTATE, 0x3ff, + sizeof(lfs->gdelta)), &lfs->gdelta); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1547,11 +1540,10 @@ static int lfs_dir_compact(lfs_t *lfs, continue; } - // successful commit, update globals - lfs->globals = globals; - lfs->locals = locals; - - if (relocated) { + if (!relocated) { + lfs->gstate = lfs->gpending; + lfs->gdelta = (struct lfs_gstate){0}; + } else { // update references if we relocated LFS_DEBUG("Relocating %"PRIu32" %"PRIu32" to %"PRIu32" %"PRIu32, oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]); @@ -1566,47 +1558,35 @@ static int lfs_dir_compact(lfs_t *lfs, static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs) { - // check for globals work - struct lfs_mattr cancelattr; - struct lfs_globals cancels; - lfs_global_zero(&cancels); - if (lfs->globals.hasmove && - lfs_pair_cmp(dir->pair, lfs->globals.pair) == 0) { - // Wait, we have the move? Just cancel this out here - // We need to, or else the move can become outdated - cancelattr.tag = LFS_MKTAG(LFS_TYPE_DELETE, lfs->globals.id, 0); - cancelattr.next = attrs; - attrs = &cancelattr; - - cancels.hasmove = lfs->globals.hasmove; - cancels.pair[0] = lfs->globals.pair[0]; - cancels.pair[1] = lfs->globals.pair[1]; - cancels.id = lfs->globals.id; - lfs_global_fromle32(&lfs->locals); - lfs_global_xor(&lfs->locals, &cancels); - lfs_global_tole32(&lfs->locals); - } - - struct lfs_globals globals = lfs->globals; - struct lfs_globals locals = lfs->locals; - - // calculate new directory size + // calculate changes to the directory lfs_tag_t deletetag = 0xffffffff; lfs_tag_t createtag = 0xffffffff; int attrcount = 0; for (const struct lfs_mattr *a = attrs; a; a = a->next) { if (lfs_tag_type3(a->tag) == LFS_TYPE_CREATE) { - dir->count += 1; createtag = a->tag; + dir->count += 1; } else if (lfs_tag_type3(a->tag) == LFS_TYPE_DELETE) { + deletetag = a->tag; LFS_ASSERT(dir->count > 0); dir->count -= 1; - deletetag = a->tag; + } else if (lfs_tag_type1(a->tag) == LFS_TYPE_TAIL) { + dir->tail[0] = ((lfs_block_t*)a->buffer)[0]; + dir->tail[1] = ((lfs_block_t*)a->buffer)[1]; + dir->split = (lfs_tag_chunk(a->tag) & 1); + lfs_pair_fromle32(dir->tail); } attrcount += 1; } + // do we have a pending move? + if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { + deletetag = lfs->gpending.tag & LFS_MKTAG(0x7ff, 0x3ff, 0); + LFS_ASSERT(dir->count > 0); + dir->count -= 1; + } + // should we actually drop the directory block? if (lfs_tag_isvalid(deletetag) && dir->count == 0) { lfs_mdir_t pdir; @@ -1635,7 +1615,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // traverse attrs that need to be written out lfs_pair_tole32(dir->tail); int err = lfs_dir_traverse(lfs, - dir, dir->off, dir->etag, attrs, attrcount, + dir, dir->off, dir->etag, attrs, attrcount, false, 0, 0, 0, 0, 0, lfs_dir_commit_commit, &(struct lfs_dir_commit_commit){ lfs, &commit}); @@ -1647,22 +1627,27 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, return err; } + // need to update gstate now that we've acknowledged moves + if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { + lfs_fs_prepmove(lfs, 0x3ff, NULL); + } + // commit any global diffs if we have any - if (!lfs_global_iszero(&locals)) { - err = lfs_dir_getglobals(lfs, dir, &locals); + if (!lfs_gstate_iszero(&lfs->gdelta)) { + err = lfs_dir_getgstate(lfs, dir, &lfs->gdelta); if (err) { return err; } - err = lfs_dir_commitglobals(lfs, &commit, &locals); + err = lfs_dir_commitattr(lfs, &commit, + LFS_MKTAG(LFS_TYPE_MOVESTATE, 0x3ff, + sizeof(lfs->gdelta)), &lfs->gdelta); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; } return err; } - - lfs_global_zero(&locals); } // finalize commit with the crc @@ -1677,9 +1662,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // successful commit, update dir dir->off = commit.off; dir->etag = commit.ptag; - // successful commit, update globals - lfs->globals = globals; - lfs->locals = locals; + // successful commit, update gstate + lfs->gstate = lfs->gpending; + lfs->gdelta = (struct lfs_gstate){0}; } else { compact: // fall back to compaction @@ -1692,9 +1677,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } } - // update globals that are affected - lfs_global_xor(&lfs->globals, &cancels); - // update any directories that are affected lfs_mdir_t copy = *dir; @@ -1772,31 +1754,28 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { } // setup dir - dir.tail[0] = pred.tail[0]; - dir.tail[1] = pred.tail[1]; - err = lfs_dir_commit(lfs, &dir, NULL); + lfs_pair_tole32(pred.tail); + err = lfs_dir_commit(lfs, &dir, + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, pred.tail, 8, + NULL)); + lfs_pair_fromle32(pred.tail); if (err) { return err; } // current block end of list? - if (!cwd.split) { - // update atomically - cwd.tail[0] = dir.pair[0]; - cwd.tail[1] = dir.pair[1]; - } else { + if (cwd.split) { // update tails, this creates a desync - pred.tail[0] = dir.pair[0]; - pred.tail[1] = dir.pair[1]; - lfs_global_orphans(lfs, +1); + lfs_fs_preporphans(lfs, +1); + lfs_pair_tole32(dir.pair); err = lfs_dir_commit(lfs, &pred, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - pred.tail, sizeof(pred.tail), + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, dir.pair, 8, NULL)); + lfs_pair_fromle32(dir.pair); if (err) { return err; } - lfs_global_orphans(lfs, -1); + lfs_fs_preporphans(lfs, -1); } // now insert into our parent block @@ -1806,8 +1785,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, LFS_MKATTR(LFS_TYPE_CREATE, id, NULL, 0, (!cwd.split) - ? LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, - cwd.tail, sizeof(cwd.tail), NULL) + ? LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, dir.pair, 8, + NULL) : NULL)))); lfs_pair_fromle32(dir.pair); if (err) { @@ -2867,7 +2846,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // mark fs as orphaned - lfs_global_orphans(lfs, +1); + lfs_fs_preporphans(lfs, +1); } // delete the entry @@ -2880,7 +2859,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { if (lfs_tag_type3(tag) == LFS_TYPE_DIR) { // fix orphan - lfs_global_orphans(lfs, -1); + lfs_fs_preporphans(lfs, -1); err = lfs_fs_pred(lfs, dir.pair, &cwd); if (err) { @@ -2948,7 +2927,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // mark fs as orphaned - lfs_global_orphans(lfs, +1); + lfs_fs_preporphans(lfs, +1); } // create move to fix later @@ -2960,7 +2939,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // is a bit messy newoldtagid += 1; } - lfs_global_move(lfs, true, oldcwd.pair, newoldtagid); + + lfs_fs_prepmove(lfs, newoldtagid, oldcwd.pair); // move over all attributes err = lfs_dir_commit(lfs, &newcwd, @@ -2985,7 +2965,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (prevtag != LFS_ERR_NOENT && lfs_tag_type3(prevtag) == LFS_TYPE_DIR) { // fix orphan - lfs_global_orphans(lfs, -1); + lfs_fs_preporphans(lfs, -1); err = lfs_fs_pred(lfs, prevdir.pair, &newcwd); if (err) { @@ -3155,8 +3135,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[1] = 0xffffffff; lfs->mlist = NULL; lfs->seed = 0; - lfs_global_zero(&lfs->globals); - lfs_global_zero(&lfs->locals); + lfs->gstate = (struct lfs_gstate){0}; + lfs->gpending = (struct lfs_gstate){0}; + lfs->gdelta = (struct lfs_gstate){0}; return 0; @@ -3334,8 +3315,8 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } } - // has globals? - err = lfs_dir_getglobals(lfs, &dir, &lfs->locals); + // has gstate? + err = lfs_dir_getgstate(lfs, &dir, &lfs->gpending); if (err) { return err; } @@ -3347,13 +3328,14 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { goto cleanup; } - // update littlefs with globals - lfs_global_fromle32(&lfs->locals); - lfs_global_xor(&lfs->globals, &lfs->locals); - lfs_global_zero(&lfs->locals); - if (lfs->globals.hasmove) { + // update littlefs with gstate + lfs_gstate_fromle32(&lfs->gpending); + lfs->gstate = lfs->gpending; + if (lfs_gstate_hasmove(&lfs->gstate)) { LFS_DEBUG("Found move %"PRIu32" %"PRIu32" %"PRIu32, - lfs->globals.pair[0], lfs->globals.pair[1], lfs->globals.id); + lfs->gstate.pair[0], + lfs->gstate.pair[1], + lfs_tag_id(lfs->gstate.tag)); } // setup free lookahead @@ -3531,7 +3513,7 @@ static int lfs_fs_relocate(lfs_t *lfs, if (tag != LFS_ERR_NOENT) { // update disk, this creates a desync - lfs_global_orphans(lfs, +1); + lfs_fs_preporphans(lfs, +1); lfs_pair_tole32(newpair); int err = lfs_dir_commit(lfs, &parent, @@ -3542,7 +3524,7 @@ static int lfs_fs_relocate(lfs_t *lfs, } // next step, clean up orphans - lfs_global_orphans(lfs, -1); + lfs_fs_preporphans(lfs, -1); } // find pred @@ -3554,12 +3536,11 @@ static int lfs_fs_relocate(lfs_t *lfs, // if we can't find dir, it must be new if (err != LFS_ERR_NOENT) { // replace bad pair, either we clean up desync, or no desync occured - parent.tail[0] = newpair[0]; - parent.tail[1] = newpair[1]; + lfs_pair_tole32(newpair); err = lfs_dir_commit(lfs, &parent, - LFS_MKATTR(LFS_TYPE_TAIL + parent.split, - 0x3ff, parent.tail, sizeof(parent.tail), + LFS_MKATTR(LFS_TYPE_TAIL + parent.split, 0x3ff, newpair, 8, NULL)); + lfs_pair_fromle32(newpair); if (err) { return err; } @@ -3568,18 +3549,52 @@ static int lfs_fs_relocate(lfs_t *lfs, return 0; } +static void lfs_fs_preporphans(lfs_t *lfs, int8_t orphans) { + lfs_gstate_fromle32(&lfs->gdelta); + lfs->gdelta.tag ^= lfs_gstate_hasorphans(&lfs->gpending); + + lfs->gpending.tag += orphans; + + lfs->gdelta.tag ^= lfs_gstate_hasorphans(&lfs->gpending); + lfs_gstate_tole32(&lfs->gdelta); +} + +static void lfs_fs_prepmove(lfs_t *lfs, + uint16_t id, const lfs_block_t pair[2]) { + lfs_gstate_fromle32(&lfs->gdelta); + lfs_gstate_xor(&lfs->gdelta, &lfs->gpending); + + if (id != 0x3ff) { + lfs->gpending.tag = LFS_MKTAG(LFS_TYPE_DELETE, id, + lfs_gstate_getorphans(&lfs->gpending)); + lfs->gpending.pair[0] = pair[0]; + lfs->gpending.pair[1] = pair[1]; + } else { + lfs->gpending.tag = LFS_MKTAG(0, 0, + lfs_gstate_getorphans(&lfs->gpending)); + lfs->gpending.pair[0] = 0; + lfs->gpending.pair[1] = 0; + } + + lfs_gstate_xor(&lfs->gdelta, &lfs->gpending); + lfs_gstate_tole32(&lfs->gdelta); +} + + static int lfs_fs_demove(lfs_t *lfs) { - if (!lfs->globals.hasmove) { + if (!lfs_gstate_hasmove(&lfs->gstate)) { return 0; } // Fix bad moves LFS_DEBUG("Fixing move %"PRIu32" %"PRIu32" %"PRIu32, - lfs->globals.pair[0], lfs->globals.pair[1], lfs->globals.id); + lfs->gstate.pair[0], + lfs->gstate.pair[1], + lfs_tag_id(lfs->gstate.tag)); // fetch and delete the moved entry lfs_mdir_t movedir; - int err = lfs_dir_fetch(lfs, &movedir, lfs->globals.pair); + int err = lfs_dir_fetch(lfs, &movedir, lfs->gstate.pair); if (err) { return err; } @@ -3594,7 +3609,7 @@ static int lfs_fs_demove(lfs_t *lfs) { } static int lfs_fs_deorphan(lfs_t *lfs) { - if (!lfs->globals.orphans) { + if (!lfs_gstate_hasorphans(&lfs->gstate)) { return 0; } @@ -3644,12 +3659,11 @@ static int lfs_fs_deorphan(lfs_t *lfs) { LFS_DEBUG("Fixing half-orphan %"PRIu32" %"PRIu32, pair[0], pair[1]); - pdir.tail[0] = pair[0]; - pdir.tail[1] = pair[1]; + lfs_pair_tole32(pair); err = lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, - 0x3ff, pdir.tail, sizeof(pdir.tail), + LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, pair, 8, NULL)); + lfs_pair_fromle32(pair); if (err) { return err; } @@ -3662,7 +3676,8 @@ static int lfs_fs_deorphan(lfs_t *lfs) { } // mark orphans as fixed - lfs_global_orphans(lfs, -lfs->globals.orphans); + lfs_fs_preporphans(lfs, -lfs_gstate_getorphans(&lfs->gstate)); + lfs->gstate = lfs->gpending; return 0; } diff --git a/lfs.h b/lfs.h index 05ba5405..4dba2b7d 100644 --- a/lfs.h +++ b/lfs.h @@ -381,12 +381,10 @@ typedef struct lfs { } *mlist; uint32_t seed; - struct lfs_globals { + struct lfs_gstate { + uint32_t tag; lfs_block_t pair[2]; - uint16_t id; - bool hasmove; - uint8_t orphans; - } globals, locals; + } gstate, gpending, gdelta; struct lfs_free { lfs_block_t off; diff --git a/lfs_util.h b/lfs_util.h index 6e2f0020..1dc3b0fd 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -164,28 +164,6 @@ static inline uint32_t lfs_tole32(uint32_t a) { return lfs_fromle32(a); } -// Convert between 16-bit little-endian and native order -static inline uint16_t lfs_fromle16(uint16_t a) { -#if !defined(LFS_NO_INTRINSICS) && ( \ - (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ - (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ - (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) - return a; -#elif !defined(LFS_NO_INTRINSICS) && ( \ - (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ - (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ - (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) - return __builtin_bswap16(a); -#else - return (((uint8_t*)&a)[0] << 0) | - (((uint8_t*)&a)[1] << 8); -#endif -} - -static inline uint16_t lfs_tole16(uint16_t a) { - return lfs_fromle16(a); -} - // Convert between 32-bit big-endian and native order static inline uint32_t lfs_frombe32(uint32_t a) { #if !defined(LFS_NO_INTRINSICS) && ( \ @@ -210,28 +188,6 @@ static inline uint32_t lfs_tobe32(uint32_t a) { return lfs_frombe32(a); } -// Convert between 16-bit big-endian and native order -static inline uint16_t lfs_frombe16(uint16_t a) { -#if !defined(LFS_NO_INTRINSICS) && ( \ - (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ - (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ - (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) - return __builtin_bswap16(a); -#elif !defined(LFS_NO_INTRINSICS) && ( \ - (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ - (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ - (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) - return a; -#else - return (((uint8_t*)&a)[0] << 8) | - (((uint8_t*)&a)[1] << 0); -#endif -} - -static inline uint16_t lfs_tobe16(uint16_t a) { - return lfs_frombe16(a); -} - // Calculate CRC-32 with polynomial = 0x04c11db7 uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size); From 51b2c7e4b64a692d3c71396e1cf2230245516f76 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 8 Jan 2019 08:52:03 -0600 Subject: [PATCH 121/139] Changed custom attribute descriptors to used arrays While linked-lists do have some minor benefits, arrays are more idiomatic in C and may provide a more intuitive API. Initially the linked-list approach was more beneficial than it is now, since it allowed custom attributes to be chained to internal linked lists of attributes. However, this was dropped because exposing the internal attribute list in this way created a rather messy user interface that required strictly encoding the attributes with the on-disk tag format. Minor downside, users can no longer introduce custom attributes in different layers (think OS vs app). Minor upside, the code size and stack usage was reduced a bit. Fortunately, this API can always be changed in the future without breaking anything (except maybe API compatibility). --- lfs.c | 164 ++++++++++++++++++++------------------------ lfs.h | 14 ++-- tests/test_attrs.sh | 56 ++++++++------- 3 files changed, 115 insertions(+), 119 deletions(-) diff --git a/lfs.c b/lfs.c index d0c89f9d..3a02348a 100644 --- a/lfs.c +++ b/lfs.c @@ -320,17 +320,17 @@ static inline lfs_size_t lfs_tag_dsize(lfs_tag_t tag) { struct lfs_mattr { lfs_tag_t tag; const void *buffer; - const struct lfs_mattr *next; }; -#define LFS_MKATTR(type, id, buffer, size, next) \ - &(const struct lfs_mattr){LFS_MKTAG(type, id, size), (buffer), (next)} - struct lfs_diskoff { lfs_block_t block; lfs_off_t off; }; +#define LFS_MKATTRS(...) \ + (struct lfs_mattr[]){__VA_ARGS__}, \ + sizeof((struct lfs_mattr[]){__VA_ARGS__}) / sizeof(struct lfs_mattr) + // operations on global state static inline void lfs_gstate_xor(struct lfs_gstate *a, const struct lfs_gstate *b) { @@ -409,7 +409,7 @@ static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { /// Internal operations predeclared here /// static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, - const struct lfs_mattr *attrs); + const struct lfs_mattr *attrs, int attrcount); static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount, lfs_mdir_t *source, uint16_t begin, uint16_t end); @@ -600,13 +600,9 @@ static int lfs_dir_traverse(lfs_t *lfs, buffer = &disk; ptag = tag; } else if (attrcount > 0) { - const struct lfs_mattr *a = attrs; - for (int j = 0; j < attrcount-1; j++) { - a = a->next; - } - - tag = a->tag; - buffer = a->buffer; + tag = attrs[0].tag; + buffer = attrs[0].buffer; + attrs += 1; attrcount -= 1; } else if (!hasseenmove && lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { @@ -647,7 +643,9 @@ static int lfs_dir_traverse(lfs_t *lfs, } // handle special cases for mcu-side operations - if (lfs_tag_type3(tag) == LFS_FROM_MOVE) { + if (lfs_tag_type3(tag) == LFS_FROM_NOOP) { + // do nothing + } else if (lfs_tag_type3(tag) == LFS_FROM_MOVE) { uint16_t fromid = lfs_tag_size(tag); uint16_t toid = lfs_tag_id(tag); int err = lfs_dir_traverse(lfs, @@ -660,9 +658,10 @@ static int lfs_dir_traverse(lfs_t *lfs, return err; } } else if (lfs_tag_type3(tag) == LFS_FROM_USERATTRS) { - for (const struct lfs_attr *a = buffer; a; a = a->next) { - int err = cb(data, LFS_MKTAG(LFS_TYPE_USERATTR + a->type, - lfs_tag_id(tag) + diff, a->size), a->buffer); + for (unsigned i = 0; i < lfs_tag_size(tag); i++) { + const struct lfs_attr *a = buffer; + int err = cb(data, LFS_MKTAG(LFS_TYPE_USERATTR + a[i].type, + lfs_tag_id(tag) + diff, a[i].size), a[i].buffer); if (err) { return err; } @@ -1266,9 +1265,8 @@ static int lfs_dir_drop(lfs_t *lfs, lfs_mdir_t *dir, lfs_mdir_t *tail) { // steal tail lfs_pair_tole32(tail->tail); - err = lfs_dir_commit(lfs, dir, - LFS_MKATTR(LFS_TYPE_TAIL + tail->split, 0x3ff, tail->tail, 8, - NULL)); + err = lfs_dir_commit(lfs, dir, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_TAIL + tail->split, 0x3ff, 8), tail->tail})); lfs_pair_fromle32(tail->tail); if (err) { return err; @@ -1557,27 +1555,24 @@ static int lfs_dir_compact(lfs_t *lfs, } static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, - const struct lfs_mattr *attrs) { + const struct lfs_mattr *attrs, int attrcount) { // calculate changes to the directory lfs_tag_t deletetag = 0xffffffff; lfs_tag_t createtag = 0xffffffff; - int attrcount = 0; - for (const struct lfs_mattr *a = attrs; a; a = a->next) { - if (lfs_tag_type3(a->tag) == LFS_TYPE_CREATE) { - createtag = a->tag; + for (int i = 0; i < attrcount; i++) { + if (lfs_tag_type3(attrs[i].tag) == LFS_TYPE_CREATE) { + createtag = attrs[i].tag; dir->count += 1; - } else if (lfs_tag_type3(a->tag) == LFS_TYPE_DELETE) { - deletetag = a->tag; + } else if (lfs_tag_type3(attrs[i].tag) == LFS_TYPE_DELETE) { + deletetag = attrs[i].tag; LFS_ASSERT(dir->count > 0); dir->count -= 1; - } else if (lfs_tag_type1(a->tag) == LFS_TYPE_TAIL) { - dir->tail[0] = ((lfs_block_t*)a->buffer)[0]; - dir->tail[1] = ((lfs_block_t*)a->buffer)[1]; - dir->split = (lfs_tag_chunk(a->tag) & 1); + } else if (lfs_tag_type1(attrs[i].tag) == LFS_TYPE_TAIL) { + dir->tail[0] = ((lfs_block_t*)attrs[i].buffer)[0]; + dir->tail[1] = ((lfs_block_t*)attrs[i].buffer)[1]; + dir->split = (lfs_tag_chunk(attrs[i].tag) & 1); lfs_pair_fromle32(dir->tail); } - - attrcount += 1; } // do we have a pending move? @@ -1755,9 +1750,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // setup dir lfs_pair_tole32(pred.tail); - err = lfs_dir_commit(lfs, &dir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, pred.tail, 8, - NULL)); + err = lfs_dir_commit(lfs, &dir, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 8), pred.tail})); lfs_pair_fromle32(pred.tail); if (err) { return err; @@ -1768,9 +1762,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // update tails, this creates a desync lfs_fs_preporphans(lfs, +1); lfs_pair_tole32(dir.pair); - err = lfs_dir_commit(lfs, &pred, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, dir.pair, 8, - NULL)); + err = lfs_dir_commit(lfs, &pred, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 8), dir.pair})); lfs_pair_fromle32(dir.pair); if (err) { return err; @@ -1780,14 +1773,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // now insert into our parent block lfs_pair_tole32(dir.pair); - err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_DIRSTRUCT, id, dir.pair, sizeof(dir.pair), - LFS_MKATTR(LFS_TYPE_DIR, id, path, nlen, - LFS_MKATTR(LFS_TYPE_CREATE, id, NULL, 0, - (!cwd.split) - ? LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, dir.pair, 8, - NULL) - : NULL)))); + err = lfs_dir_commit(lfs, &cwd, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, id, 0)}, + {LFS_MKTAG(LFS_TYPE_DIR, id, nlen), path}, + {LFS_MKTAG(LFS_TYPE_DIRSTRUCT, id, 8), dir.pair}, + {!cwd.split + ? LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 8) + : LFS_MKTAG(LFS_FROM_NOOP, 0, 0), dir.pair})); lfs_pair_fromle32(dir.pair); if (err) { return err; @@ -2188,11 +2180,10 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } // get next slot and create entry to remember name - err = lfs_dir_commit(lfs, &file->m, - LFS_MKATTR(LFS_TYPE_INLINESTRUCT, file->id, NULL, 0, - LFS_MKATTR(LFS_TYPE_REG, file->id, path, nlen, - LFS_MKATTR(LFS_TYPE_CREATE, file->id, NULL, 0, - NULL)))); + err = lfs_dir_commit(lfs, &file->m, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0)}, + {LFS_MKTAG(LFS_TYPE_REG, file->id, nlen), path}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0)})); if (err) { err = LFS_ERR_NAMETOOLONG; goto cleanup; @@ -2221,12 +2212,13 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } // fetch attrs - for (const struct lfs_attr *a = file->cfg->attrs; a; a = a->next) { + for (unsigned i = 0; i < file->cfg->attr_count; i++) { if ((file->flags & 3) != LFS_O_WRONLY) { lfs_stag_t res = lfs_dir_get(lfs, &file->m, LFS_MKTAG(0x7ff, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_USERATTR + a->type, file->id, a->size), - a->buffer); + LFS_MKTAG(LFS_TYPE_USERATTR + file->cfg->attrs[i].type, + file->id, file->cfg->attrs[i].size), + file->cfg->attrs[i].buffer); if (res < 0 && res != LFS_ERR_NOENT) { err = res; goto cleanup; @@ -2234,7 +2226,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } if ((file->flags & 3) != LFS_O_RDONLY) { - if (a->size > lfs->attr_max) { + if (file->cfg->attrs[i].size > lfs->attr_max) { err = LFS_ERR_NOSPC; goto cleanup; } @@ -2476,11 +2468,10 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } // commit file data and attributes - err = lfs_dir_commit(lfs, &file->m, - LFS_MKATTR(LFS_FROM_USERATTRS, - file->id, file->cfg->attrs, 0, - LFS_MKATTR(type, file->id, buffer, size, - NULL))); + err = lfs_dir_commit(lfs, &file->m, LFS_MKATTRS( + {LFS_MKTAG(type, file->id, size), buffer}, + {LFS_MKTAG(LFS_FROM_USERATTRS, file->id, + file->cfg->attr_count), file->cfg->attrs})); if (err) { if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) { goto relocate; @@ -2850,9 +2841,8 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // delete the entry - err = lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_DELETE, lfs_tag_id(tag), NULL, 0, - NULL)); + err = lfs_dir_commit(lfs, &cwd, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(tag), 0)})); if (err) { return err; } @@ -2943,13 +2933,14 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_fs_prepmove(lfs, newoldtagid, oldcwd.pair); // move over all attributes - err = lfs_dir_commit(lfs, &newcwd, - LFS_MKATTR(LFS_FROM_MOVE, newid, &oldcwd, lfs_tag_id(oldtag), - LFS_MKATTR(lfs_tag_type3(oldtag), newid, newpath, strlen(newpath), - LFS_MKATTR(LFS_TYPE_CREATE, newid, NULL, 0, - (prevtag != LFS_ERR_NOENT) - ? LFS_MKATTR(LFS_TYPE_DELETE, newid, NULL, 0, NULL) - : NULL)))); + err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTRS( + {prevtag != LFS_ERR_NOENT + ? LFS_MKTAG(LFS_TYPE_DELETE, newid, 0) + : LFS_MKTAG(LFS_FROM_NOOP, 0, 0)}, + {LFS_MKTAG(LFS_TYPE_CREATE, newid, 0)}, + {LFS_MKTAG(lfs_tag_type3(oldtag), newid, strlen(newpath)), + newpath}, + {LFS_MKTAG(LFS_FROM_MOVE, newid, lfs_tag_id(oldtag)), &oldcwd})); if (err) { return err; } @@ -2957,7 +2948,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // let commit clean up after move (if we're different! otherwise move // logic already fixed it for us) if (lfs_pair_cmp(oldcwd.pair, newcwd.pair) != 0) { - err = lfs_dir_commit(lfs, &oldcwd, NULL); + err = lfs_dir_commit(lfs, &oldcwd, NULL, 0); if (err) { return err; } @@ -3031,9 +3022,8 @@ static int lfs_commitattr(lfs_t *lfs, const char *path, } } - return lfs_dir_commit(lfs, &cwd, - LFS_MKATTR(LFS_TYPE_USERATTR + type, id, buffer, size, - NULL)); + return lfs_dir_commit(lfs, &cwd, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_USERATTR + type, id, size), buffer})); } int lfs_setattr(lfs_t *lfs, const char *path, @@ -3198,12 +3188,11 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { }; lfs_superblock_tole32(&superblock); - err = lfs_dir_commit(lfs, &root, - LFS_MKATTR(LFS_TYPE_INLINESTRUCT, 0, - &superblock, sizeof(superblock), - LFS_MKATTR(LFS_TYPE_SUPERBLOCK, 0, "littlefs", 8, - LFS_MKATTR(LFS_TYPE_CREATE, 0, NULL, 0, - NULL)))); + err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0)}, + {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})); if (err) { goto cleanup; } @@ -3516,8 +3505,7 @@ static int lfs_fs_relocate(lfs_t *lfs, lfs_fs_preporphans(lfs, +1); lfs_pair_tole32(newpair); - int err = lfs_dir_commit(lfs, &parent, - &(struct lfs_mattr){.tag=tag, .buffer=newpair}); + int err = lfs_dir_commit(lfs, &parent, LFS_MKATTRS({tag, newpair})); lfs_pair_fromle32(newpair); if (err) { return err; @@ -3537,9 +3525,8 @@ static int lfs_fs_relocate(lfs_t *lfs, if (err != LFS_ERR_NOENT) { // replace bad pair, either we clean up desync, or no desync occured lfs_pair_tole32(newpair); - err = lfs_dir_commit(lfs, &parent, - LFS_MKATTR(LFS_TYPE_TAIL + parent.split, 0x3ff, newpair, 8, - NULL)); + err = lfs_dir_commit(lfs, &parent, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_TAIL + parent.split, 0x3ff, 8), newpair})); lfs_pair_fromle32(newpair); if (err) { return err; @@ -3600,7 +3587,7 @@ static int lfs_fs_demove(lfs_t *lfs) { } // rely on cancel logic inside commit - err = lfs_dir_commit(lfs, &movedir, NULL); + err = lfs_dir_commit(lfs, &movedir, NULL, 0); if (err) { return err; } @@ -3660,9 +3647,8 @@ static int lfs_fs_deorphan(lfs_t *lfs) { pair[0], pair[1]); lfs_pair_tole32(pair); - err = lfs_dir_commit(lfs, &pdir, - LFS_MKATTR(LFS_TYPE_SOFTTAIL, 0x3ff, pair, 8, - NULL)); + err = lfs_dir_commit(lfs, &pdir, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 8), pair})); lfs_pair_fromle32(pair); if (err) { return err; diff --git a/lfs.h b/lfs.h index 4dba2b7d..61ef66ff 100644 --- a/lfs.h +++ b/lfs.h @@ -123,6 +123,7 @@ enum lfs_type { LFS_TYPE_MOVESTATE = 0x7ff, // internal chip sources + LFS_FROM_NOOP = 0x000, LFS_FROM_MOVE = 0x101, LFS_FROM_USERATTRS = 0x102, }; @@ -268,7 +269,8 @@ struct lfs_info { char name[LFS_NAME_MAX+1]; }; -// Custom attribute structure +// Custom attribute structure, used to describe custom attributes +// committed atomically during file writes. struct lfs_attr { // 8-bit type of attribute, provided by user and used to // identify the attribute @@ -279,9 +281,6 @@ struct lfs_attr { // Size of attribute in bytes, limited to LFS_ATTR_MAX lfs_size_t size; - - // Pointer to next attribute in linked list - struct lfs_attr *next; }; // Optional configuration provided during lfs_file_opencfg @@ -290,8 +289,8 @@ struct lfs_file_config { // By default lfs_malloc is used to allocate this buffer. void *buffer; - // Optional linked list of custom attributes related to the file. If the - // file is opened with read access, the attributes will be read from + // Optional list of custom attributes related to the file. If the file + // is opened with read access, these attributes will be read from disk // during the open call. If the file is opened with write access, the // attributes will be written to disk every file sync or close. This // write occurs atomically with update to the file's contents. @@ -302,6 +301,9 @@ struct lfs_file_config { // is larger, then it will be silently truncated. If the attribute is not // found, it will be created implicitly. struct lfs_attr *attrs; + + // Number of custom attributes in the list + lfs_size_t attr_count; }; diff --git a/tests/test_attrs.sh b/tests/test_attrs.sh index e9b2227d..e4ff4ce6 100755 --- a/tests/test_attrs.sh +++ b/tests/test_attrs.sh @@ -156,10 +156,12 @@ TEST echo "--- Set/get file attribute ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - struct lfs_attr a1 = {'A', buffer, 4}; - struct lfs_attr b1 = {'B', buffer+4, 6, &a1}; - struct lfs_attr c1 = {'C', buffer+10, 5, &b1}; - struct lfs_file_config cfg1 = {.attrs = &c1}; + struct lfs_attr attrs1[] = { + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}, + }; + struct lfs_file_config cfg1 = {.attrs=attrs1, .attr_count=3}; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; memcpy(buffer, "aaaa", 4); @@ -173,53 +175,55 @@ tests/test.py << TEST memcmp(buffer+4, "bbbbbb", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - b1.size = 0; + attrs1[1].size = 0; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; lfs_file_close(&lfs, &file[0]) => 0; memset(buffer, 0, 15); - b1.size = 6; + attrs1[1].size = 6; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; lfs_file_close(&lfs, &file[0]) => 0; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - b1.size = 6; + attrs1[1].size = 6; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; memcpy(buffer+4, "dddddd", 6); lfs_file_close(&lfs, &file[0]) => 0; memset(buffer, 0, 15); - b1.size = 6; + attrs1[1].size = 6; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; lfs_file_close(&lfs, &file[0]) => 0; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "dddddd", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - b1.size = 3; + attrs1[1].size = 3; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; memcpy(buffer+4, "eee", 3); lfs_file_close(&lfs, &file[0]) => 0; memset(buffer, 0, 15); - b1.size = 6; + attrs1[1].size = 6; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; lfs_file_close(&lfs, &file[0]) => 0; memcmp(buffer, "aaaa", 4) => 0; memcmp(buffer+4, "eee\0\0\0", 6) => 0; memcmp(buffer+10, "ccccc", 5) => 0; - a1.size = LFS_ATTR_MAX+1; + attrs1[0].size = LFS_ATTR_MAX+1; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => LFS_ERR_NOSPC; - struct lfs_attr a2 = {'A', buffer, 4}; - struct lfs_attr b2 = {'B', buffer+4, 9, &a2}; - struct lfs_attr c2 = {'C', buffer+13, 5, &b2}; - struct lfs_file_config cfg2 = {.attrs = &c2}; + struct lfs_attr attrs2[] = { + {'A', buffer, 4}, + {'B', buffer+4, 9}, + {'C', buffer+13, 5}, + }; + struct lfs_file_config cfg2 = {.attrs=attrs2, .attr_count=3}; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDWR, &cfg2) => 0; memcpy(buffer+4, "fffffffff", 9); lfs_file_close(&lfs, &file[0]) => 0; - a1.size = 4; + attrs1[0].size = 4; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg1) => 0; lfs_file_close(&lfs, &file[0]) => 0; @@ -227,10 +231,12 @@ tests/test.py << TEST TEST tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - struct lfs_attr a2 = {'A', buffer, 4}; - struct lfs_attr b2 = {'B', buffer+4, 9, &a2}; - struct lfs_attr c2 = {'C', buffer+13, 5, &b2}; - struct lfs_file_config cfg2 = {.attrs = &c2}; + struct lfs_attr attrs2[] = { + {'A', buffer, 4}, + {'B', buffer+4, 9}, + {'C', buffer+13, 5}, + }; + struct lfs_file_config cfg2 = {.attrs=attrs2, .attr_count=3}; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_RDONLY, &cfg2) => 0; lfs_file_close(&lfs, &file[0]) => 0; @@ -248,10 +254,12 @@ TEST echo "--- Deferred file attributes ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - struct lfs_attr a1 = {'B', "gggg", 4}; - struct lfs_attr b1 = {'C', "", 0, &a1}; - struct lfs_attr c1 = {'D', "hhhh", 4, &b1}; - struct lfs_file_config cfg1 = {.attrs = &c1}; + struct lfs_attr attrs1[] = { + {'B', "gggg", 4}, + {'C', "", 0}, + {'D', "hhhh", 4}, + }; + struct lfs_file_config cfg1 = {.attrs=attrs1, .attr_count=3}; lfs_file_opencfg(&lfs, &file[0], "hello/hello", LFS_O_WRONLY, &cfg1) => 0; From e1f9d2bc09b194ef9a262b54513fb6cd7a626dee Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 13 Jan 2019 11:08:42 -0600 Subject: [PATCH 122/139] Added support for RAM-independent reading of inline files One of the new features in LittleFS is "inline files", which is the inlining of small files in the parent directory. Inline files have a big limitation in that they no longer have a dedicated scratch area to write out data before commit-time. This is fine as long as inline files are small enough to fit in RAM. However, this dependency on RAM creates an uncomfortable situation for portability, with larger devices able to create larger files than smaller devices. This problem is especially important on embedded systems, where RAM is at a premium. Recently, I realized this RAM requirement is necessary for _writing_ inline files, but not for _reading_ inline files. By allowing fetches of specific slices of inline files it's possible to read inline files without the RAM to back it. However however, this creates a conflict with COW semantics. Normally, when a file is open twice, it is referenced by a COW data structure that can be updated independently. Inlines files that fit in RAM also allows independent updates, but the moment an inline file can't fit in RAM, any updates to that directory block could corrupt open files referencing the inline file. The fact that this behaviour is only inconsistent for inline files created on a different device with more RAM creates a potential nightmare for user experience. Fortunately, there is a workaround for this. When we are commiting to a directory, any open files needs to live in a COW structure or in RAM. While we could move large inline files to COW structures at open time, this would break the separation of read/write operations and could lead to write errors at read time (ie ENOSPC). But since this is only an issue for commits, we can defer the move to a COW structure to any commits to that directory. This means when committing to a directory we need to find any _open_ large inline files and evict them from the directory, leaving the file with a new COW structure even if it was opened read only. While complicated, the end result is inline files that can use the MAX RAM that is available, but can be read with MIN RAM, even with multiple write operations happening to the underlying directory block. This prevents users from needing to learn the idiosyncrasies of inline files to use the filesystem portably. --- lfs.c | 255 ++++++++++++++++++++++++++++++++++++++-------------------- lfs.h | 51 ++++-------- 2 files changed, 184 insertions(+), 122 deletions(-) diff --git a/lfs.c b/lfs.c index 3a02348a..08ba3f04 100644 --- a/lfs.c +++ b/lfs.c @@ -366,13 +366,15 @@ static inline bool lfs_gstate_hasmovehere(const struct lfs_gstate *a, } static inline void lfs_gstate_fromle32(struct lfs_gstate *a) { - for (int i = 0; i < 3; i++) { - ((uint32_t*)a)[i] = lfs_fromle32(((uint32_t*)a)[i]); - } + a->tag = lfs_fromle32(a->tag); + a->pair[0] = lfs_fromle32(a->pair[0]); + a->pair[1] = lfs_fromle32(a->pair[1]); } static inline void lfs_gstate_tole32(struct lfs_gstate *a) { - lfs_gstate_fromle32(a); + a->tag = lfs_tole32(a->tag); + a->pair[0] = lfs_tole32(a->pair[0]); + a->pair[1] = lfs_tole32(a->pair[1]); } // other endianness operations @@ -391,9 +393,8 @@ static inline void lfs_superblock_fromle32(lfs_superblock_t *superblock) { superblock->block_size = lfs_fromle32(superblock->block_size); superblock->block_count = lfs_fromle32(superblock->block_count); superblock->name_max = lfs_fromle32(superblock->name_max); - superblock->inline_max = lfs_fromle32(superblock->inline_max); - superblock->attr_max = lfs_fromle32(superblock->attr_max); superblock->file_max = lfs_fromle32(superblock->file_max); + superblock->attr_max = lfs_fromle32(superblock->attr_max); } static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { @@ -401,9 +402,8 @@ static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) { superblock->block_size = lfs_tole32(superblock->block_size); superblock->block_count = lfs_tole32(superblock->block_count); superblock->name_max = lfs_tole32(superblock->name_max); - superblock->inline_max = lfs_tole32(superblock->inline_max); - superblock->attr_max = lfs_tole32(superblock->attr_max); superblock->file_max = lfs_tole32(superblock->file_max); + superblock->attr_max = lfs_tole32(superblock->attr_max); } @@ -413,19 +413,20 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, static int lfs_dir_compact(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount, lfs_mdir_t *source, uint16_t begin, uint16_t end); +static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file); +static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file); +static void lfs_fs_preporphans(lfs_t *lfs, int8_t orphans); +static void lfs_fs_prepmove(lfs_t *lfs, + uint16_t id, const lfs_block_t pair[2]); static int lfs_fs_pred(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *pdir); static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t dir[2], lfs_mdir_t *parent); static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]); -static void lfs_fs_preporphans(lfs_t *lfs, int8_t orphans); -static void lfs_fs_prepmove(lfs_t *lfs, - uint16_t id, const lfs_block_t pair[2]); static int lfs_fs_forceconsistency(lfs_t *lfs); static int lfs_deinit(lfs_t *lfs); - /// Block allocator /// static int lfs_alloc_lookahead(void *p, lfs_block_t block) { lfs_t *lfs = (lfs_t*)p; @@ -490,8 +491,9 @@ static void lfs_alloc_ack(lfs_t *lfs) { /// Metadata pair and directory operations /// -static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, - lfs_tag_t gmask, lfs_tag_t gtag, void *buffer) { +static lfs_stag_t lfs_dir_getslice(lfs_t *lfs, const lfs_mdir_t *dir, + lfs_tag_t gmask, lfs_tag_t gtag, + lfs_off_t goff, void *gbuffer, lfs_size_t gsize) { lfs_off_t off = dir->off; lfs_tag_t ntag = dir->etag; lfs_stag_t gdiff = 0; @@ -507,7 +509,7 @@ static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, off -= lfs_tag_dsize(ntag); lfs_tag_t tag = ntag; int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(ntag), + NULL, &lfs->rcache, sizeof(ntag), dir->pair[0], off, &ntag, sizeof(ntag)); if (err) { return err; @@ -533,16 +535,15 @@ static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, return LFS_ERR_NOENT; } - lfs_size_t diff = lfs_min(lfs_tag_size(tag), - lfs_tag_size(gtag)); + lfs_size_t diff = lfs_min(lfs_tag_size(tag), gsize); err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, diff, - dir->pair[0], off+sizeof(tag), buffer, diff); + NULL, &lfs->rcache, diff, + dir->pair[0], off+sizeof(tag)+goff, gbuffer, diff); if (err) { return err; } - memset((uint8_t*)buffer + diff, 0, + memset((uint8_t*)gbuffer + diff, 0, lfs_tag_size(gtag) - diff); return tag + gdiff; @@ -552,6 +553,74 @@ static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, return LFS_ERR_NOENT; } +static lfs_stag_t lfs_dir_get(lfs_t *lfs, const lfs_mdir_t *dir, + lfs_tag_t gmask, lfs_tag_t gtag, void *buffer) { + return lfs_dir_getslice(lfs, dir, + gmask, gtag, + 0, buffer, lfs_tag_size(gtag)); +} + +static int lfs_dir_getread(lfs_t *lfs, const lfs_mdir_t *dir, + const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, + lfs_tag_t gmask, lfs_tag_t gtag, + lfs_off_t off, void *buffer, lfs_size_t size) { + uint8_t *data = buffer; + if (off+size > lfs->cfg->block_size) { + return LFS_ERR_CORRUPT; + } + + while (size > 0) { + lfs_size_t diff = size; + + if (pcache && pcache->block == 0xfffffffe && + off < pcache->off + pcache->size) { + if (off >= pcache->off) { + // is already in pcache? + diff = lfs_min(diff, pcache->size - (off-pcache->off)); + memcpy(data, &pcache->buffer[off-pcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; + } + + // pcache takes priority + diff = lfs_min(diff, pcache->off-off); + } + + if (rcache->block == 0xfffffffe && + off < rcache->off + rcache->size) { + if (off >= rcache->off) { + // is already in rcache? + diff = lfs_min(diff, rcache->size - (off-rcache->off)); + memcpy(data, &rcache->buffer[off-rcache->off], diff); + + data += diff; + off += diff; + size -= diff; + continue; + } + + // rcache takes priority + diff = lfs_min(diff, rcache->off-off); + } + + // load to cache, first condition can no longer fail + rcache->block = 0xfffffffe; + rcache->off = lfs_aligndown(off, lfs->cfg->read_size); + rcache->size = lfs_min(lfs_alignup(off+hint, lfs->cfg->read_size), + lfs->cfg->cache_size); + int err = lfs_dir_getslice(lfs, dir, gmask, gtag, + rcache->off, rcache->buffer, rcache->size); + if (err) { + return err; + } + } + + return 0; +} + static int lfs_dir_traverse_filter(void *p, lfs_tag_t tag, const void *buffer) { lfs_tag_t *filtertag = p; @@ -588,7 +657,7 @@ static int lfs_dir_traverse(lfs_t *lfs, if (off+lfs_tag_dsize(ptag) < dir->off) { off += lfs_tag_dsize(ptag); int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(tag), + NULL, &lfs->rcache, sizeof(tag), dir->pair[0], off, &tag, sizeof(tag)); if (err) { return err; @@ -688,7 +757,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, int r = 0; for (int i = 0; i < 2; i++) { int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(revs[i]), + NULL, &lfs->rcache, sizeof(revs[i]), pair[i], 0, &revs[i], sizeof(revs[i])); revs[i] = lfs_fromle32(revs[i]); if (err && err != LFS_ERR_CORRUPT) { @@ -724,7 +793,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_tag_t tag; off += lfs_tag_dsize(ptag); int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + NULL, &lfs->rcache, lfs->cfg->block_size, dir->pair[0], off, &tag, sizeof(tag)); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -751,7 +820,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // check the crc attr uint32_t dcrc; err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + NULL, &lfs->rcache, lfs->cfg->block_size, dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -825,7 +894,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, tempsplit = (lfs_tag_chunk(tag) & 1); err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + NULL, &lfs->rcache, lfs->cfg->block_size, dir->pair[0], off+sizeof(tag), &temptail, 8); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1137,7 +1206,7 @@ static int lfs_dir_commitattr(lfs_t *lfs, struct lfs_commit *commit, // rely on caching to make this efficient uint8_t dat; err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, dsize-sizeof(tag)-i, + NULL, &lfs->rcache, dsize-sizeof(tag)-i, disk->block, disk->off+i, &dat, 1); if (err) { return err; @@ -1162,7 +1231,7 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) { // read erased state from next program unit lfs_tag_t tag; int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(tag), + NULL, &lfs->rcache, sizeof(tag), commit->block, off, &tag, sizeof(tag)); if (err && err != LFS_ERR_CORRUPT) { return err; @@ -1232,7 +1301,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { // rather than clobbering one of the blocks we just pretend // the revision may be valid int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, sizeof(dir->rev), + NULL, &lfs->rcache, sizeof(dir->rev), dir->pair[0], 0, &dir->rev, sizeof(dir->rev)); if (err) { return err; @@ -1556,6 +1625,27 @@ static int lfs_dir_compact(lfs_t *lfs, static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount) { + // check for any inline files that aren't RAM backed and + // forcefully evict them, needed for filesystem consistency + for (lfs_file_t *f = (lfs_file_t*)lfs->mlist; f; f = f->next) { + if (dir != &f->m && lfs_pair_cmp(f->m.pair, dir->pair) == 0 && + f->type == LFS_TYPE_REG && (f->flags & LFS_F_INLINE) && + f->ctz.size > lfs->cfg->cache_size) { + f->flags &= ~LFS_F_READING; + f->off = 0; + + int err = lfs_file_relocate(lfs, f); + if (err) { + return err; + } + + err = lfs_file_flush(lfs, f); + if (err) { + return err; + } + } + } + // calculate changes to the directory lfs_tag_t deletetag = 0xffffffff; lfs_tag_t createtag = 0xffffffff; @@ -2262,7 +2352,7 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, if (file->ctz.size > 0) { lfs_stag_t res = lfs_dir_get(lfs, &file->m, LFS_MKTAG(0x700, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->ctz.size), + LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->cache.size), file->cache.buffer); if (res < 0) { err = res; @@ -2306,6 +2396,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { + lfs_alloc_ack(lfs); while (true) { // just relocate what exists into new block lfs_block_t nblock; @@ -2325,11 +2416,23 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { // either read from dirty cache or disk for (lfs_off_t i = 0; i < file->off; i++) { uint8_t data; - err = lfs_bd_read(lfs, - &file->cache, &lfs->rcache, file->off-i, - file->block, i, &data, 1); - if (err) { - return err; + if (file->flags & LFS_F_INLINE) { + err = lfs_dir_getread(lfs, &file->m, + // note we evict inline files before they can be dirty + NULL, &file->cache, file->off-i, + LFS_MKTAG(0xfff, 0x1ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), + i, &data, 1); + if (err) { + return err; + } + } else { + err = lfs_bd_read(lfs, + &file->cache, &lfs->rcache, file->off-i, + file->block, i, &data, 1); + if (err) { + return err; + } } err = lfs_bd_prog(lfs, @@ -2351,6 +2454,8 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { lfs_cache_zero(lfs, &lfs->pcache); file->block = nblock; + file->flags &= ~LFS_F_INLINE; + file->flags |= LFS_F_WRITING; return 0; relocate: @@ -2362,9 +2467,7 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { - if (file->flags & LFS_F_READING) { - file->flags &= ~LFS_F_READING; - } + file->flags &= ~LFS_F_READING; if (file->flags & LFS_F_WRITING) { lfs_off_t pos = file->pos; @@ -2403,8 +2506,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { // write out what we have while (true) { - int err = lfs_bd_flush(lfs, - &file->cache, &lfs->rcache, true); + int err = lfs_bd_flush(lfs, &file->cache, &lfs->rcache, true); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -2486,17 +2588,11 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { relocate: // inline file doesn't fit anymore - file->block = 0xfffffffe; file->off = file->pos; - - lfs_alloc_ack(lfs); err = lfs_file_relocate(lfs, file); if (err) { return err; } - - file->flags &= ~LFS_F_INLINE; - file->flags |= LFS_F_WRITING; } } @@ -2546,11 +2642,22 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, // read as much as we can in current block lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); - int err = lfs_bd_read(lfs, - NULL, &file->cache, lfs->cfg->block_size, - file->block, file->off, data, diff); - if (err) { - return err; + if (file->flags & LFS_F_INLINE) { + int err = lfs_dir_getread(lfs, &file->m, + NULL, &file->cache, lfs->cfg->block_size, + LFS_MKTAG(0xfff, 0x1ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), + file->off, data, diff); + if (err) { + return err; + } + } else { + int err = lfs_bd_read(lfs, + NULL, &file->cache, lfs->cfg->block_size, + file->block, file->off, data, diff); + if (err) { + return err; + } } file->pos += diff; @@ -2602,20 +2709,15 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } if ((file->flags & LFS_F_INLINE) && - file->pos + nsize > lfs->inline_max) { + lfs_max(file->pos+nsize, file->ctz.size) > + lfs_min(lfs->cfg->cache_size, LFS_ATTR_MAX)) { // inline file doesn't fit anymore - file->block = 0xfffffffe; file->off = file->pos; - - lfs_alloc_ack(lfs); int err = lfs_file_relocate(lfs, file); if (err) { file->flags |= LFS_F_ERRED; return err; } - - file->flags &= ~LFS_F_INLINE; - file->flags |= LFS_F_WRITING; } while (nsize > 0) { @@ -3101,11 +3203,10 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->name_max = LFS_NAME_MAX; } - LFS_ASSERT(lfs->cfg->inline_max <= LFS_INLINE_MAX); - LFS_ASSERT(lfs->cfg->inline_max <= lfs->cfg->cache_size); - lfs->inline_max = lfs->cfg->inline_max; - if (!lfs->inline_max) { - lfs->inline_max = lfs_min(LFS_INLINE_MAX, lfs->cfg->cache_size); + LFS_ASSERT(lfs->cfg->file_max <= LFS_FILE_MAX); + lfs->file_max = lfs->cfg->file_max; + if (!lfs->file_max) { + lfs->file_max = LFS_FILE_MAX; } LFS_ASSERT(lfs->cfg->attr_max <= LFS_ATTR_MAX); @@ -3114,12 +3215,6 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->attr_max = LFS_ATTR_MAX; } - LFS_ASSERT(lfs->cfg->file_max <= LFS_FILE_MAX); - lfs->file_max = lfs->cfg->file_max; - if (!lfs->file_max) { - lfs->file_max = LFS_FILE_MAX; - } - // setup default state lfs->root[0] = 0xffffffff; lfs->root[1] = 0xffffffff; @@ -3182,9 +3277,8 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { .block_size = lfs->cfg->block_size, .block_count = lfs->cfg->block_count, .name_max = lfs->name_max, - .inline_max = lfs->inline_max, - .attr_max = lfs->attr_max, .file_max = lfs->file_max, + .attr_max = lfs->attr_max, }; lfs_superblock_tole32(&superblock); @@ -3270,15 +3364,15 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->name_max = superblock.name_max; } - if (superblock.inline_max) { - if (superblock.inline_max > lfs->inline_max) { - LFS_ERROR("Unsupported inline_max (%"PRIu32" > %"PRIu32")", - superblock.inline_max, lfs->inline_max); + if (superblock.file_max) { + if (superblock.file_max > lfs->file_max) { + LFS_ERROR("Unsupported file_max (%"PRIu32" > %"PRIu32")", + superblock.file_max, lfs->file_max); err = LFS_ERR_INVAL; goto cleanup; } - lfs->inline_max = superblock.inline_max; + lfs->file_max = superblock.file_max; } if (superblock.attr_max) { @@ -3291,17 +3385,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->attr_max = superblock.attr_max; } - - if (superblock.file_max) { - if (superblock.file_max > lfs->file_max) { - LFS_ERROR("Unsupported file_max (%"PRIu32" > %"PRIu32")", - superblock.file_max, lfs->file_max); - err = LFS_ERR_INVAL; - goto cleanup; - } - - lfs->file_max = superblock.file_max; - } } // has gstate? diff --git a/lfs.h b/lfs.h index 61ef66ff..ec41c2f9 100644 --- a/lfs.h +++ b/lfs.h @@ -51,30 +51,21 @@ typedef uint32_t lfs_block_t; #define LFS_NAME_MAX 255 #endif -// Maximum inline file size in bytes, may be redefined to limit RAM usage, -// but littlefs will automatically limit the LFS_INLINE_MAX to the -// configured cache_size. Limited to <= 1022. Stored in superblock and must -// be respected by other littlefs drivers. -#ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 1022 -#endif - -// Maximum size of custom attributes in bytes, may be redefined, but there is -// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 1022. Stored -// in superblock and must be respected by other littlefs drivers. -#ifndef LFS_ATTR_MAX -#define LFS_ATTR_MAX 1022 -#endif - // Maximum size of a file in bytes, may be redefined to limit to support other // drivers. Limited on disk to <= 4294967296. However, above 2147483647 the // functions lfs_file_seek, lfs_file_size, and lfs_file_tell will return -// incorrect values due to signed sizes. Stored in superblock and must be -// respected by other littlefs drivers. +// incorrect values due to using signed integers. Stored in superblock and +// must be respected by other littlefs drivers. #ifndef LFS_FILE_MAX #define LFS_FILE_MAX 2147483647 #endif +// Maximum size of custom attributes in bytes, may be redefined, but there is +// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 1022. +#ifndef LFS_ATTR_MAX +#define LFS_ATTR_MAX 1022 +#endif + // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { @@ -234,24 +225,15 @@ struct lfs_config { // superblock and must be respected by other littlefs drivers. lfs_size_t name_max; - // Optional upper limit on inlined files in bytes. Inline files must be - // backed by RAM, but if a file fits in RAM it can be inlined into its - // directory block without needing its own data block. Must be <= - // cache_size and LFS_INLINE_MAX. Defaults to min(LFS_INLINE_MAX, - // cache_size) when zero. Stored in superblock and must be respected by - // other littlefs drivers. - lfs_size_t inline_max; - - // Optional upper limit on custom attributes in bytes. No downside for - // larger attributes size but must be <= LFS_ATTR_MAX. Defaults to - // LFS_ATTR_MAX when zero. Stored in superblock and must be respected by - // other littlefs drivers. - lfs_size_t attr_max; - // Optional upper limit on files in bytes. No downside for larger files // but must be <= LFS_FILE_MAX. Defaults to LFS_FILE_MAX when zero. Stored // in superblock and must be respected by other littlefs drivers. lfs_size_t file_max; + + // Optional upper limit on custom attributes in bytes. No downside for + // larger attributes size but must be <= LFS_ATTR_MAX. Defaults to + // LFS_ATTR_MAX when zero. + lfs_size_t attr_max; }; // File info structure @@ -362,11 +344,9 @@ typedef struct lfs_superblock { uint32_t version; lfs_size_t block_size; lfs_size_t block_count; - lfs_size_t name_max; - lfs_size_t inline_max; - lfs_size_t attr_max; lfs_size_t file_max; + lfs_size_t attr_max; } lfs_superblock_t; // The littlefs filesystem type @@ -398,9 +378,8 @@ typedef struct lfs { const struct lfs_config *cfg; lfs_size_t name_max; - lfs_size_t inline_max; - lfs_size_t attr_max; lfs_size_t file_max; + lfs_size_t attr_max; } lfs_t; From 916b30855878edb7d5af77d9cd2b661a3cd4f733 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 14 Jan 2019 17:53:41 -0600 Subject: [PATCH 123/139] Fixed excessive waste from overly large inline files Before this, there were some safety limits, but there was no real default limit to the size of inline files other than the amount of RAM available. On PCs, this meant that inline files were free to fill up directory blocks to a little under the block size. However this is very wasteful in terms of storage space. Because of splitting limits to keep the compact runtime reasonable, each byte of an inline files uses 4x the amount. Fortunately we can find an optimal inline limit: Inline file waste for n bytes = 3n CTZ file waste for n bytes = B - n Where B = block size Solving for n = B/4 So the optimal inline limit is B/4. However, this assumes a perfect inline file and no metadata. We can decrease this to B/8 to give a bit more breathing room for directory+file metadata. --- lfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lfs.c b/lfs.c index 08ba3f04..5d00d780 100644 --- a/lfs.c +++ b/lfs.c @@ -2710,7 +2710,8 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, if ((file->flags & LFS_F_INLINE) && lfs_max(file->pos+nsize, file->ctz.size) > - lfs_min(lfs->cfg->cache_size, LFS_ATTR_MAX)) { + lfs_min(LFS_ATTR_MAX, lfs_min( + lfs->cfg->cache_size, lfs->cfg->block_size/8))) { // inline file doesn't fit anymore file->off = file->pos; int err = lfs_file_relocate(lfs, file); From 5fb8fa9f06e65abb3cc77c85998aa0678ddfb9df Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 22 Jan 2019 16:09:51 -0600 Subject: [PATCH 124/139] Fixed issue with global state updates being lost during relocates Caught during power resilience testing, this was a bug that only occurs when we need to compact in the middle of a move commit and we find that the destination block is bad, forcing a relocate. This series of events would cause littlefs to clear the "gpending" state in preparation for fixing the move atomically, but this fix never gets written out because of the relocate. The fix here is to separate the update to the "gdelta" and "gpending" state, marking "gdelta" in preparation for the move, but waiting to update "gpending" until after our commit completes. This keeps our disk state in sync without prematurely dropping moves. --- lfs.c | 76 ++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/lfs.c b/lfs.c index 5d00d780..a93a5f34 100644 --- a/lfs.c +++ b/lfs.c @@ -365,6 +365,19 @@ static inline bool lfs_gstate_hasmovehere(const struct lfs_gstate *a, return lfs_tag_type1(a->tag) && lfs_pair_cmp(a->pair, pair) == 0; } +static inline void lfs_gstate_xororphans(struct lfs_gstate *a, + const struct lfs_gstate *b, bool orphans) { + a->tag ^= LFS_MKTAG(0x800, 0, 0) & (b->tag ^ (orphans << 31)); +} + +static inline void lfs_gstate_xormove(struct lfs_gstate *a, + const struct lfs_gstate *b, uint16_t id, const lfs_block_t pair[2]) { + a->tag ^= LFS_MKTAG(0x7ff, 0x3ff, 0) & (b->tag ^ ( + (id != 0x3ff) ? LFS_MKTAG(LFS_TYPE_DELETE, id, 0) : 0)); + a->pair[0] ^= b->pair[0] ^ ((id != 0x3ff) ? pair[0] : 0); + a->pair[1] ^= b->pair[1] ^ ((id != 0x3ff) ? pair[1] : 0); +} + static inline void lfs_gstate_fromle32(struct lfs_gstate *a) { a->tag = lfs_fromle32(a->tag); a->pair[0] = lfs_fromle32(a->pair[0]); @@ -982,6 +995,7 @@ static int lfs_dir_getgstate(lfs_t *lfs, const lfs_mdir_t *dir, if (res != LFS_ERR_NOENT) { // xor together to find resulting gstate + lfs_gstate_fromle32(&temp); lfs_gstate_xor(gstate, &temp); } @@ -1548,17 +1562,14 @@ static int lfs_dir_compact(lfs_t *lfs, } } - // need to update gstate now that we've acknowledged moves - if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { - lfs_fs_prepmove(lfs, 0x3ff, NULL); - } - if (!relocated && !lfs_gstate_iszero(&lfs->gdelta)) { // commit any globals, unless we're relocating, // in which case our parent will steal our globals + lfs_gstate_tole32(&lfs->gdelta); err = lfs_dir_commitattr(lfs, &commit, LFS_MKTAG(LFS_TYPE_MOVESTATE, 0x3ff, sizeof(lfs->gdelta)), &lfs->gdelta); + lfs_gstate_fromle32(&lfs->gdelta); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1581,6 +1592,12 @@ static int lfs_dir_compact(lfs_t *lfs, dir->off = commit.off; dir->etag = commit.ptag; dir->erased = true; + // note we able to have already handled move here + if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { + lfs_gstate_xormove(&lfs->gpending, + &lfs->gpending, 0x3ff, NULL); + } + } break; @@ -1670,6 +1687,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, deletetag = lfs->gpending.tag & LFS_MKTAG(0x7ff, 0x3ff, 0); LFS_ASSERT(dir->count > 0); dir->count -= 1; + + // mark gdelta so we reflect the move we will fix + lfs_gstate_xormove(&lfs->gdelta, &lfs->gpending, 0x3ff, NULL); } // should we actually drop the directory block? @@ -1712,11 +1732,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, return err; } - // need to update gstate now that we've acknowledged moves - if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { - lfs_fs_prepmove(lfs, 0x3ff, NULL); - } - // commit any global diffs if we have any if (!lfs_gstate_iszero(&lfs->gdelta)) { err = lfs_dir_getgstate(lfs, dir, &lfs->gdelta); @@ -1724,9 +1739,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, return err; } + lfs_gstate_tole32(&lfs->gdelta); err = lfs_dir_commitattr(lfs, &commit, LFS_MKTAG(LFS_TYPE_MOVESTATE, 0x3ff, sizeof(lfs->gdelta)), &lfs->gdelta); + lfs_gstate_fromle32(&lfs->gdelta); if (err) { if (err == LFS_ERR_NOSPC || err == LFS_ERR_CORRUPT) { goto compact; @@ -1747,7 +1764,13 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, // successful commit, update dir dir->off = commit.off; dir->etag = commit.ptag; - // successful commit, update gstate + + // note we able to have already handled move here + if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { + lfs_gstate_xormove(&lfs->gpending, &lfs->gpending, 0x3ff, NULL); + } + + // update gstate lfs->gstate = lfs->gpending; lfs->gdelta = (struct lfs_gstate){0}; } else { @@ -3402,7 +3425,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } // update littlefs with gstate - lfs_gstate_fromle32(&lfs->gpending); + lfs->gpending.tag += !lfs_tag_isvalid(lfs->gpending.tag); lfs->gstate = lfs->gpending; if (lfs_gstate_hasmove(&lfs->gstate)) { LFS_DEBUG("Found move %"PRIu32" %"PRIu32" %"PRIu32, @@ -3621,34 +3644,17 @@ static int lfs_fs_relocate(lfs_t *lfs, } static void lfs_fs_preporphans(lfs_t *lfs, int8_t orphans) { - lfs_gstate_fromle32(&lfs->gdelta); - lfs->gdelta.tag ^= lfs_gstate_hasorphans(&lfs->gpending); - lfs->gpending.tag += orphans; - - lfs->gdelta.tag ^= lfs_gstate_hasorphans(&lfs->gpending); - lfs_gstate_tole32(&lfs->gdelta); + lfs_gstate_xororphans(&lfs->gdelta, &lfs->gpending, + lfs_gstate_hasorphans(&lfs->gpending)); + lfs_gstate_xororphans(&lfs->gpending, &lfs->gpending, + lfs_gstate_hasorphans(&lfs->gpending)); } static void lfs_fs_prepmove(lfs_t *lfs, uint16_t id, const lfs_block_t pair[2]) { - lfs_gstate_fromle32(&lfs->gdelta); - lfs_gstate_xor(&lfs->gdelta, &lfs->gpending); - - if (id != 0x3ff) { - lfs->gpending.tag = LFS_MKTAG(LFS_TYPE_DELETE, id, - lfs_gstate_getorphans(&lfs->gpending)); - lfs->gpending.pair[0] = pair[0]; - lfs->gpending.pair[1] = pair[1]; - } else { - lfs->gpending.tag = LFS_MKTAG(0, 0, - lfs_gstate_getorphans(&lfs->gpending)); - lfs->gpending.pair[0] = 0; - lfs->gpending.pair[1] = 0; - } - - lfs_gstate_xor(&lfs->gdelta, &lfs->gpending); - lfs_gstate_tole32(&lfs->gdelta); + lfs_gstate_xormove(&lfs->gdelta, &lfs->gpending, id, pair); + lfs_gstate_xormove(&lfs->gpending, &lfs->gpending, id, pair); } From 8cca1b6a868f6a4a0587b887c25d1756e345d1ef Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 22 Jan 2019 16:21:16 -0600 Subject: [PATCH 125/139] Fixed several small issues found during wider testing - Fixed cache tarnishing issue where flush did not clean up read caches - Removed extra alloc acks which would prevent file relocations from resolving on an exhausted filesystem - Removed unsigned comparison < 0 from changed in file seek - Fixed bug in lfs_dir_getslice with using gtag's size - Removed warnings around PRIu32 used 16-bit types in debug info --- lfs.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lfs.c b/lfs.c index a93a5f34..d33b282e 100644 --- a/lfs.c +++ b/lfs.c @@ -556,8 +556,7 @@ static lfs_stag_t lfs_dir_getslice(lfs_t *lfs, const lfs_mdir_t *dir, return err; } - memset((uint8_t*)gbuffer + diff, 0, - lfs_tag_size(gtag) - diff); + memset((uint8_t*)gbuffer + diff, 0, gsize - diff); return tag + gdiff; } @@ -1651,6 +1650,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, f->flags &= ~LFS_F_READING; f->off = 0; + lfs_alloc_ack(lfs); int err = lfs_file_relocate(lfs, f); if (err) { return err; @@ -2419,7 +2419,6 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { - lfs_alloc_ack(lfs); while (true) { // just relocate what exists into new block lfs_block_t nblock; @@ -2490,7 +2489,12 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { - file->flags &= ~LFS_F_READING; + if (file->flags & LFS_F_READING) { + if (!(file->flags & LFS_F_INLINE)) { + lfs_cache_drop(lfs, &file->cache); + } + file->flags &= ~LFS_F_READING; + } if (file->flags & LFS_F_WRITING) { lfs_off_t pos = file->pos; @@ -2737,6 +2741,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, lfs->cfg->cache_size, lfs->cfg->block_size/8))) { // inline file doesn't fit anymore file->off = file->pos; + lfs_alloc_ack(lfs); int err = lfs_file_relocate(lfs, file); if (err) { file->flags |= LFS_F_ERRED; @@ -2832,7 +2837,7 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, npos = file->ctz.size + off; } - if (npos < 0 || npos > lfs->file_max) { + if (npos > lfs->file_max) { // file position out of range return LFS_ERR_INVAL; } @@ -3370,7 +3375,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { uint16_t minor_version = (0xffff & (superblock.version >> 0)); if ((major_version != LFS_DISK_VERSION_MAJOR || minor_version > LFS_DISK_VERSION_MINOR)) { - LFS_ERROR("Invalid version %"PRIu32".%"PRIu32, + LFS_ERROR("Invalid version %"PRIu16".%"PRIu16, major_version, minor_version); err = LFS_ERR_INVAL; goto cleanup; @@ -3428,7 +3433,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->gpending.tag += !lfs_tag_isvalid(lfs->gpending.tag); lfs->gstate = lfs->gpending; if (lfs_gstate_hasmove(&lfs->gstate)) { - LFS_DEBUG("Found move %"PRIu32" %"PRIu32" %"PRIu32, + LFS_DEBUG("Found move %"PRIu32" %"PRIu32" %"PRIu16, lfs->gstate.pair[0], lfs->gstate.pair[1], lfs_tag_id(lfs->gstate.tag)); @@ -3664,7 +3669,7 @@ static int lfs_fs_demove(lfs_t *lfs) { } // Fix bad moves - LFS_DEBUG("Fixing move %"PRIu32" %"PRIu32" %"PRIu32, + LFS_DEBUG("Fixing move %"PRIu32" %"PRIu32" %"PRIu16, lfs->gstate.pair[0], lfs->gstate.pair[1], lfs_tag_id(lfs->gstate.tag)); From 173c21215159236b2d93caf2856636d29680ef79 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 25 Jan 2019 17:07:17 -0600 Subject: [PATCH 126/139] Added scripts/prefix.py for automatically prefixing version numbers Example: ./scripts/prefix.py lfs2 Will convert the following: lfs_* -> lfs2_* LFS_* -> LFS2_* -DLFS_* -> -DLFS2_* --- scripts/prefix.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 scripts/prefix.py diff --git a/scripts/prefix.py b/scripts/prefix.py new file mode 100755 index 00000000..ca547b69 --- /dev/null +++ b/scripts/prefix.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# This script replaces prefixes of files, and symbols in that file. +# Useful for creating different versions of the codebase that don't +# conflict at compile time. +# +# example: +# $ ./scripts/prefix.py lfs2 + +import os +import os.path +import re +import glob +import itertools +import tempfile +import shutil +import subprocess + +DEFAULT_PREFIX = "lfs" + +def subn(from_prefix, to_prefix, name): + name, count1 = re.subn('\\b'+from_prefix, to_prefix, name) + name, count2 = re.subn('\\b'+from_prefix.upper(), to_prefix.upper(), name) + name, count3 = re.subn('\\B-D'+from_prefix.upper(), + '-D'+to_prefix.upper(), name) + return name, count1+count2+count3 + +def main(from_prefix, to_prefix=None, files=None): + if not to_prefix: + from_prefix, to_prefix = DEFAULT_PREFIX, from_prefix + + if not files: + files = subprocess.check_output([ + 'git', 'ls-tree', '-r', '--name-only', 'HEAD']).split() + + for oldname in files: + # Rename any matching file names + newname, namecount = subn(from_prefix, to_prefix, oldname) + if namecount: + subprocess.check_call(['git', 'mv', oldname, newname]) + + # Rename any prefixes in file + count = 0 + with open(newname+'~', 'w') as tempf: + with open(newname) as newf: + for line in newf: + line, n = subn(from_prefix, to_prefix, line) + count += n + tempf.write(line) + shutil.copystat(newname, newname+'~') + os.rename(newname+'~', newname) + subprocess.check_call(['git', 'add', newname]) + + # Summary + print '%s: %d replacements' % ( + '%s -> %s' % (oldname, newname) if namecount else oldname, + count) + +if __name__ == "__main__": + import sys + sys.exit(main(*sys.argv[1:])) From 95c1a6339ddd9e0dbbc19bf0be0c37f629af2b5d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 29 Jan 2019 21:53:56 -0600 Subject: [PATCH 127/139] Fixed corner case in block_cycles eviction logic The problem was when we allocate a dir-pair, it's possible for the revision count to immediately overflow and the dir-pair be evicted and returned to the unused blocks without being written even once. In the case that block_cycles = 1, this made it impossible to ever create a dir-pair, even in lfs_format. I've also added a bit of logic to lfs_dir_alloc that will prevent any immediate evictions because of the revision count. found by TheLoneWolfling --- lfs.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lfs.c b/lfs.c index d33b282e..d4ccb41f 100644 --- a/lfs.c +++ b/lfs.c @@ -1316,15 +1316,14 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) { int err = lfs_bd_read(lfs, NULL, &lfs->rcache, sizeof(dir->rev), dir->pair[0], 0, &dir->rev, sizeof(dir->rev)); - if (err) { - return err; - } - dir->rev = lfs_fromle32(dir->rev); if (err && err != LFS_ERR_CORRUPT) { return err; } + // make sure we don't immediately evict + dir->rev += dir->rev & 1; + // set defaults dir->off = sizeof(dir->rev); dir->etag = 0xffffffff; @@ -1457,7 +1456,8 @@ static int lfs_dir_compact(lfs_t *lfs, // increment revision count dir->rev += 1; - if (lfs->cfg->block_cycles && dir->rev % lfs->cfg->block_cycles == 0) { + if (lfs->cfg->block_cycles && + (dir->rev % (lfs->cfg->block_cycles+1) == 0)) { if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) { // oh no! we're writing too much to the superblock, // should we expand? From 10dfc36f08081274e37133107fff3a14d180b5e4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 31 Jan 2019 14:54:47 -0600 Subject: [PATCH 128/139] Fixed issue with long names causing unbounded recursion This was caused by any commit containing entries large enough to _always_ force a compaction. This would cause littlefs to think that it would need to split infinitely because there was no base case. The fix here is pretty simple: treat any commit with only a single entry as unsplittable. This forces littlefs to first try overcompacting (fitting more in a block than what has optimal runtime), and then failing that return LFS_ERR_NOSPC for higher layers to handle. found by TheLoneWolfling --- lfs.c | 5 +++-- tests/test_paths.sh | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/lfs.c b/lfs.c index d4ccb41f..c6ff2e98 100644 --- a/lfs.c +++ b/lfs.c @@ -1414,7 +1414,8 @@ static int lfs_dir_compact(lfs_t *lfs, bool relocated = false; bool exhausted = false; - while (true) { + // should we split? + while (end - begin > 1) { // find size lfs_size_t size = 0; int err = lfs_dir_traverse(lfs, @@ -1429,7 +1430,7 @@ static int lfs_dir_compact(lfs_t *lfs, // space is complicated, we need room for tail, crc, gstate, // cleanup delete, and we cap at half a block to give room - // for metadata updates + // for metadata updates. if (size <= lfs_min(lfs->cfg->block_size - 36, lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size))) { break; diff --git a/tests/test_paths.sh b/tests/test_paths.sh index 999001a5..b1d41a7f 100755 --- a/tests/test_paths.sh +++ b/tests/test_paths.sh @@ -165,5 +165,29 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Really big path test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + memset(buffer, 'w', LFS_NAME_MAX); + buffer[LFS_NAME_MAX+1] = '\0'; + lfs_mkdir(&lfs, (char*)buffer) => 0; + lfs_remove(&lfs, (char*)buffer) => 0; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_remove(&lfs, (char*)buffer) => 0; + + memcpy(buffer, "coffee/", strlen("coffee/")); + memset(buffer+strlen("coffee/"), 'w', LFS_NAME_MAX); + buffer[strlen("coffee/")+LFS_NAME_MAX+1] = '\0'; + lfs_mkdir(&lfs, (char*)buffer) => 0; + lfs_remove(&lfs, (char*)buffer) => 0; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_remove(&lfs, (char*)buffer) => 0; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Results ---" tests/stats.py From 512930c856bb38f984512e0263252fdfdccb8021 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 1 Feb 2019 09:16:00 -0600 Subject: [PATCH 129/139] Updated SPEC.md to reflect v2 changes --- SPEC.md | 1052 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 736 insertions(+), 316 deletions(-) diff --git a/SPEC.md b/SPEC.md index 2a1f9eca..8949204f 100644 --- a/SPEC.md +++ b/SPEC.md @@ -2,9 +2,9 @@ This is the technical specification of the little filesystem. This document covers the technical details of how the littlefs is stored on disk for -introspection and tooling development. This document assumes you are -familiar with the design of the littlefs, for more info on how littlefs -works check out [DESIGN.md](DESIGN.md). +introspection and tool development. This document assumes you are familiar +with the design of the littlefs, for more info on how littlefs works check +out [DESIGN.md](DESIGN.md). ``` | | | .---._____ @@ -15,356 +15,776 @@ works check out [DESIGN.md](DESIGN.md). | | | ``` -## Some important details +## Some quick notes -- The littlefs is a block-based filesystem. This is, the disk is divided into - an array of evenly sized blocks that are used as the logical unit of storage - in littlefs. Block pointers are stored in 32 bits. +- littlefs is a block-based filesystem. The disk is divided into an array of + evenly sized blocks that are used as the logical unit of storage. Block + pointers are stored in 32 bits. -- There is no explicit free-list stored on disk, the littlefs only knows what - is in use in the filesystem. +- In addition to the logical block size (which usually matches the erase + block size), littlefs also uses a program block size and read block size. + These determine the alignment of block device operations, but aren't needed + for portability. -- The littlefs uses the value of 0xffffffff to represent a null block-pointer. +- By default, any values in littlefs are stored in little-endian byte order. -- All values in littlefs are stored in little-endian byte order. +- The littlefs uses the value of `0xffffffff` to represent a null + block address. ## Directories / Metadata pairs Metadata pairs form the backbone of the littlefs and provide a system for -atomic updates. Even the superblock is stored in a metadata pair. +distributed atomic updates. Even the superblock is stored in a metadata pair. As their name suggests, a metadata pair is stored in two blocks, with one block -acting as a redundant backup in case the other is corrupted. These two blocks -could be anywhere in the disk and may not be next to each other, so any -pointers to directory pairs need to be stored as two block pointers. - -Here's the layout of metadata blocks on disk: - -| offset | size | description | -|--------|---------------|----------------| -| 0x00 | 32 bits | revision count | -| 0x04 | 32 bits | dir size | -| 0x08 | 64 bits | tail pointer | -| 0x10 | size-16 bytes | dir entries | -| 0x00+s | 32 bits | CRC | - -**Revision count** - Incremented every update, only the uncorrupted -metadata-block with the most recent revision count contains the valid metadata. -Comparison between revision counts must use sequence comparison since the -revision counts may overflow. - -**Dir size** - Size in bytes of the contents in the current metadata block, -including the metadata-pair metadata. Additionally, the highest bit of the -dir size may be set to indicate that the directory's contents continue on the -next metadata-pair pointed to by the tail pointer. - -**Tail pointer** - Pointer to the next metadata-pair in the filesystem. -A null pair-pointer (0xffffffff, 0xffffffff) indicates the end of the list. -If the highest bit in the dir size is set, this points to the next -metadata-pair in the current directory, otherwise it points to an arbitrary -metadata-pair. Starting with the superblock, the tail-pointers form a -linked-list containing all metadata-pairs in the filesystem. - -**CRC** - 32 bit CRC used to detect corruption from power-lost, from block -end-of-life, or just from noise on the storage bus. The CRC is appended to -the end of each metadata-block. The littlefs uses the standard CRC-32, which -uses a polynomial of 0x04c11db7, initialized with 0xffffffff. - -Here's an example of a simple directory stored on disk: -``` -(32 bits) revision count = 10 (0x0000000a) -(32 bits) dir size = 154 bytes, end of dir (0x0000009a) -(64 bits) tail pointer = 37, 36 (0x00000025, 0x00000024) -(32 bits) CRC = 0xc86e3106 - -00000000: 0a 00 00 00 9a 00 00 00 25 00 00 00 24 00 00 00 ........%...$... -00000010: 22 08 00 03 05 00 00 00 04 00 00 00 74 65 61 22 "...........tea" -00000020: 08 00 06 07 00 00 00 06 00 00 00 63 6f 66 66 65 ...........coffe -00000030: 65 22 08 00 04 09 00 00 00 08 00 00 00 73 6f 64 e"...........sod -00000040: 61 22 08 00 05 1d 00 00 00 1c 00 00 00 6d 69 6c a"...........mil -00000050: 6b 31 22 08 00 05 1f 00 00 00 1e 00 00 00 6d 69 k1"...........mi -00000060: 6c 6b 32 22 08 00 05 21 00 00 00 20 00 00 00 6d lk2"...!... ...m -00000070: 69 6c 6b 33 22 08 00 05 23 00 00 00 22 00 00 00 ilk3"...#..."... -00000080: 6d 69 6c 6b 34 22 08 00 05 25 00 00 00 24 00 00 milk4"...%...$.. -00000090: 00 6d 69 6c 6b 35 06 31 6e c8 .milk5.1n. -``` - -A note about the tail pointer linked-list: Normally, this linked-list is -threaded through the entire filesystem. However, after power-loss this -linked-list may become out of sync with the rest of the filesystem. -- The linked-list may contain a directory that has actually been removed -- The linked-list may contain a metadata pair that has not been updated after - a block in the pair has gone bad. - -The threaded linked-list must be checked for these errors before it can be -used reliably. Fortunately, the threaded linked-list can simply be ignored -if littlefs is mounted read-only. - -## Entries - -Each metadata block contains a series of entries that follow a standard -layout. An entry contains the type of the entry, along with a section for -entry-specific data, attributes, and a name. - -Here's the layout of entries on disk: - -| offset | size | description | -|---------|------------------------|----------------------------| -| 0x0 | 8 bits | entry type | -| 0x1 | 8 bits | entry length | -| 0x2 | 8 bits | attribute length | -| 0x3 | 8 bits | name length | -| 0x4 | entry length bytes | entry-specific data | -| 0x4+e | attribute length bytes | system-specific attributes | -| 0x4+e+a | name length bytes | entry name | - -**Entry type** - Type of the entry, currently this is limited to the following: -- 0x11 - file entry -- 0x22 - directory entry -- 0x2e - superblock entry - -Additionally, the type is broken into two 4 bit nibbles, with the upper nibble -specifying the type's data structure used when scanning the filesystem. The -lower nibble clarifies the type further when multiple entries share the same -data structure. - -The highest bit is reserved for marking the entry as "moved". If an entry -is marked as "moved", the entry may also exist somewhere else in the -filesystem. If the entry exists elsewhere, this entry must be treated as -though it does not exist. - -**Entry length** - Length in bytes of the entry-specific data. This does -not include the entry type size, attributes, or name. The full size in bytes -of the entry is 4 + entry length + attribute length + name length. - -**Attribute length** - Length of system-specific attributes in bytes. Since -attributes are system specific, there is not much guarantee on the values in -this section, and systems are expected to work even when it is empty. See the -[attributes](#entry-attributes) section for more details. - -**Name length** - Length of the entry name. Entry names are stored as UTF8, -although most systems will probably only support ASCII. Entry names can not -contain '/' and can not be '.' or '..' as these are a part of the syntax of -filesystem paths. - -Here's an example of a simple entry stored on disk: -``` -(8 bits) entry type = file (0x11) -(8 bits) entry length = 8 bytes (0x08) -(8 bits) attribute length = 0 bytes (0x00) -(8 bits) name length = 12 bytes (0x0c) -(8 bytes) entry data = 05 00 00 00 20 00 00 00 -(12 bytes) entry name = smallavacado - -00000000: 11 08 00 0c 05 00 00 00 20 00 00 00 73 6d 61 6c ........ ...smal -00000010: 6c 61 76 61 63 61 64 6f lavacado -``` - -## Superblock - -The superblock is the anchor for the littlefs. The superblock is stored as -a metadata pair containing a single superblock entry. It is through the -superblock that littlefs can access the rest of the filesystem. - -The superblock can always be found in blocks 0 and 1, however fetching the -superblock requires knowing the block size. The block size can be guessed by -searching the beginning of disk for the string "littlefs", although currently -the filesystems relies on the user providing the correct block size. - -The superblock is the most valuable block in the filesystem. It is updated -very rarely, only during format or when the root directory must be moved. It -is encouraged to always write out both superblock pairs even though it is not -required. - -Here's the layout of the superblock entry: - -| offset | size | description | -|--------|------------------------|----------------------------------------| -| 0x00 | 8 bits | entry type (0x2e for superblock entry) | -| 0x01 | 8 bits | entry length (20 bytes) | -| 0x02 | 8 bits | attribute length | -| 0x03 | 8 bits | name length (8 bytes) | -| 0x04 | 64 bits | root directory | -| 0x0c | 32 bits | block size | -| 0x10 | 32 bits | block count | -| 0x14 | 32 bits | version | -| 0x18 | attribute length bytes | system-specific attributes | -| 0x18+a | 8 bytes | magic string ("littlefs") | - -**Root directory** - Pointer to the root directory's metadata pair. - -**Block size** - Size of the logical block size used by the filesystem. - -**Block count** - Number of blocks in the filesystem. - -**Version** - The littlefs version encoded as a 32 bit value. The upper 16 bits -encodes the major version, which is incremented when a breaking-change is -introduced in the filesystem specification. The lower 16 bits encodes the -minor version, which is incremented when a backwards-compatible change is -introduced. Non-standard Attribute changes do not change the version. This -specification describes version 1.1 (0x00010001), which is the first version -of littlefs. - -**Magic string** - The magic string "littlefs" takes the place of an entry -name. - -Here's an example of a complete superblock: -``` -(32 bits) revision count = 3 (0x00000003) -(32 bits) dir size = 52 bytes, end of dir (0x00000034) -(64 bits) tail pointer = 3, 2 (0x00000003, 0x00000002) -(8 bits) entry type = superblock (0x2e) -(8 bits) entry length = 20 bytes (0x14) -(8 bits) attribute length = 0 bytes (0x00) -(8 bits) name length = 8 bytes (0x08) -(64 bits) root directory = 3, 2 (0x00000003, 0x00000002) -(32 bits) block size = 512 bytes (0x00000200) -(32 bits) block count = 1024 blocks (0x00000400) -(32 bits) version = 1.1 (0x00010001) -(8 bytes) magic string = littlefs -(32 bits) CRC = 0xc50b74fa - -00000000: 03 00 00 00 34 00 00 00 03 00 00 00 02 00 00 00 ....4........... -00000010: 2e 14 00 08 03 00 00 00 02 00 00 00 00 02 00 00 ................ -00000020: 00 04 00 00 01 00 01 00 6c 69 74 74 6c 65 66 73 ........littlefs -00000030: fa 74 0b c5 .t.. -``` - -## Directory entries - -Directories are stored in entries with a pointer to the first metadata pair -in the directory. Keep in mind that a directory may be composed of multiple -metadata pairs connected by the tail pointer when the highest bit in the dir -size is set. +providing a backup during erase cycles in case power is lost. These two blocks +are not necessarily sequential and may be anywhere on disk, so a "pointer" to a +metadata pair is stored as two block pointers. -Here's the layout of a directory entry: - -| offset | size | description | -|--------|------------------------|-----------------------------------------| -| 0x0 | 8 bits | entry type (0x22 for directory entries) | -| 0x1 | 8 bits | entry length (8 bytes) | -| 0x2 | 8 bits | attribute length | -| 0x3 | 8 bits | name length | -| 0x4 | 64 bits | directory pointer | -| 0xc | attribute length bytes | system-specific attributes | -| 0xc+a | name length bytes | directory name | +On top of this, each metadata block behaves as an appendable log, containing a +variable number of commits. Commits can be appended to the metadata log in +order to update the metadata without requiring an erase cycles. Note that +successive commits may supersede the metadata in previous commits. Only the +most recent metadata should be considered valid. + +The high-level layout of a metadata block is fairly simple: + +``` + .---------------------------------------. +.-| revision count | entries | \ +| |-------------------+ | | +| | | | +| | | +-- 1st commit +| | | | +| | +-------------------| | +| | | CRC | / +| |-------------------+-------------------| +| | entries | \ +| | | | +| | | +-- 2nd commit +| | +-------------------+--------------| | +| | | CRC | padding | / +| |----+-------------------+--------------| +| | entries | \ +| | | | +| | | +-- 3rd commit +| | +-------------------+---------| | +| | | CRC | | / +| |---------+-------------------+ | +| | unwritten storage | more commits +| | | | +| | | v +| | | +| | | +| '---------------------------------------' +'---------------------------------------' +``` + +Each metadata block contains a 32-bit revision count followed by a number of +commits. Each commit contains a variable number of metadata entries followed +by a 32-bit CRC. + +Note also that entries aren't necessarily word-aligned. This allows us to +store metadata more compactly, however we can only write to addresses that are +aligned to our program block size. This means each commit may have padding for +alignment. + +Metadata block fields: + +- **Revision count (32-bits)** - Incremented every erase cycle. If both blocks + contain valid commits, only the block with the most recent revision count + should be used. Sequence comparison must be used to avoid issues with + integer overflow. + +- **CRC (32-bits)** - Detects corruption from power-loss or other write + issues. Uses a CRC-32 with a polynomial of `0x04c11db7` initialized + with `0xffffffff`. + +Entries themselves are stored as a 32-bit tag followed by a variable length +blob of data. But exactly how these tags are stored is a little bit tricky. + +Metadata blocks support both forward and backward iteration. In order to do +this without duplicating the space for each tag, neighboring entries have their +tags XORed together, starting with `0xffffffff`. -**Directory pointer** - Pointer to the first metadata pair in the directory. - -Here's an example of a directory entry: ``` -(8 bits) entry type = directory (0x22) -(8 bits) entry length = 8 bytes (0x08) -(8 bits) attribute length = 0 bytes (0x00) -(8 bits) name length = 3 bytes (0x03) -(64 bits) directory pointer = 5, 4 (0x00000005, 0x00000004) -(3 bytes) name = tea - -00000000: 22 08 00 03 05 00 00 00 04 00 00 00 74 65 61 "...........tea -``` - -## File entries - -Files are stored in entries with a pointer to the head of the file and the -size of the file. This is enough information to determine the state of the -CTZ skip-list that is being referenced. - -How files are actually stored on disk is a bit complicated. The full -explanation of CTZ skip-lists can be found in [DESIGN.md](DESIGN.md#ctz-skip-lists). - -A terribly quick summary: For every nth block where n is divisible by 2^x, -the block contains a pointer to block n-2^x. These pointers are stored in -increasing order of x in each block of the file preceding the data in the + Forward iteration Backward iteration + +.-------------------. 0xffffffff .-------------------. +| revision count | | | revision count | +|-------------------| v |-------------------| +| tag ~A |---> xor -> tag A | tag ~A |---> xor -> 0xffffffff +|-------------------| | |-------------------| ^ +| data A | | | data A | | +| | | | | | +| | | | | | +|-------------------| v |-------------------| | +| tag AxB |---> xor -> tag B | tag AxB |---> xor -> tag A +|-------------------| | |-------------------| ^ +| data B | | | data B | | +| | | | | | +| | | | | | +|-------------------| v |-------------------| | +| tag BxC |---> xor -> tag C | tag BxC |---> xor -> tag B +|-------------------| |-------------------| ^ +| data C | | data C | | +| | | | tag C +| | | | +| | | | +'-------------------' '-------------------' +``` + +One last thing to note before we get into the details around tag encoding. Each +tag contains a valid bit used to indicate if the tag and containing commit is +valid. This valid bit is the first bit found in the tag and the commit and can +be used to tell if we've attempted to write to the remaining space in the block. -The maximum number of pointers in a block is bounded by the maximum file size -divided by the block size. With 32 bits for file size, this results in a -minimum block size of 104 bytes. +Here's a more complete example of metadata block containing 4 entries: -Here's the layout of a file entry: +``` + .---------------------------------------. +.-| revision count | tag ~A | \ +| |-------------------+-------------------| | +| | data A | | +| | | | +| |-------------------+-------------------| | +| | tag AxB | data B | <--. | +| |-------------------+ | | | +| | | | +-- 1st commit +| | +-------------------+---------| | | +| | | tag BxC | | <-.| | +| |---------+-------------------+ | || | +| | data C | || | +| | | || | +| |-------------------+-------------------| || | +| | tag CxCRC | CRC | || / +| |-------------------+-------------------| || +| | tag CRCxA' | data A' | || \ +| |-------------------+ | || | +| | | || | +| | +-------------------+----| || +-- 2nd commit +| | | tag CRCxA' | | || | +| |--------------+-------------------+----| || | +| | CRC | padding | || / +| |--------------+----+-------------------| || +| | tag CRCxA'' | data A'' | <---. \ +| |-------------------+ | ||| | +| | | ||| | +| | +-------------------+---------| ||| | +| | | tag A''xD | | < ||| | +| |---------+-------------------+ | |||| +-- 3rd commit +| | data D | |||| | +| | +---------| |||| | +| | | tag Dx| |||| | +| |---------+-------------------+---------| |||| | +| |CRC | CRC | | |||| / +| |---------+-------------------+ | |||| +| | unwritten storage | |||| more commits +| | | |||| | +| | | |||| v +| | | |||| +| | | |||| +| '---------------------------------------' |||| +'---------------------------------------' |||'- most recent A + ||'-- most recent B + |'--- most recent C + '---- most recent D +``` -| offset | size | description | -|--------|------------------------|------------------------------------| -| 0x0 | 8 bits | entry type (0x11 for file entries) | -| 0x1 | 8 bits | entry length (8 bytes) | -| 0x2 | 8 bits | attribute length | -| 0x3 | 8 bits | name length | -| 0x4 | 32 bits | file head | -| 0x8 | 32 bits | file size | -| 0xc | attribute length bytes | system-specific attributes | -| 0xc+a | name length bytes | directory name | +## Metadata tags -**File head** - Pointer to the block that is the head of the file's CTZ -skip-list. +So in littlefs, 32-bit tags describe every type of metadata. And this means +_every_ type of metadata, including file entries, directory fields, and +global state. Even the CRCs used to mark the end of commits get their own tag. -**File size** - Size of file in bytes. +Because of this, the tag format contains some densely packed informtaion. Note +that there are multiple levels of types which break down into more info: -Here's an example of a file entry: ``` -(8 bits) entry type = file (0x11) -(8 bits) entry length = 8 bytes (0x08) -(8 bits) attribute length = 0 bytes (0x00) -(8 bits) name length = 12 bytes (0x03) -(32 bits) file head = 543 (0x0000021f) -(32 bits) file size = 256 KB (0x00040000) -(12 bytes) name = largeavacado +[---- 32 ----] +[1|-- 11 --|-- 10 --|-- 10 --] + ^. ^ . ^ ^- length + |. | . '------------ id + |. '-----.------------------ type (type3) + '.-----------.------------------ valid bit + [-3-|-- 8 --] + ^ ^- chunk + '------- type (type1) +``` + + +Before we go further, there's one VERY important thing to note. These tags are +NOT stored in little-endian. Tags stored in commits are actually stored in +big-endian (and is the only thing in littlefs stored in big-endian). This +little bit of craziness comes from the fact that the valid bit must be the +first bit in a commit, and when converted to little-endian, the valid bit finds +itself in byte 4. We could restructure the tag to store the valid bit lower, +but, because none of the fields are byte-aligned, this would be more +complicated than just storing the tag in big-endian. + +Another thing to note is that both the tags `0x00000000` and `0xffffffff` are +invalid and can be used for null values. + +Metadata tag fields: + +- **Valid bit (1-bit)** - Indicates if the tag is valid. + +- **Type3 (11-bits)** - Type of the tag. This field is broken down further + into a 3-bit abstract type and an 8-bit chunk field. Note that the value + `0x000` is invalid and not assigned a type. -00000000: 11 08 00 0c 1f 02 00 00 00 00 04 00 6c 61 72 67 ............larg -00000010: 65 61 76 61 63 61 64 6f eavacado +- **Type1 (3-bits)** - Abstract type of the tag. Groups the tags into + 8 categories that facilitate bitmasked lookups. + +- **Chunk (8-bits)** - Chunk field used for various purposes by the different + abstract types. type1+chunk+id form a unique identifier for each tag in the + metadata block. + +- **Id (10-bits)** - File id associated with the tag. Each file in a metadata + block gets a unique id which is used to associate tags with that file. The + special value `0x3ff` is used for any tags that are not associated with a + file, such as directory and global metadata. + +- **Length (10-bits)** - Length of the data in bytes. The special value + `0x3ff` indicates that this tag has been deleted. + +## Metadata types + +What follows is an exhaustive list of metadata in littlefs. + +--- +#### `0x401` LFS_TYPE_CREATE + +Creates a new file with this id. Note that files in a metadata block +don't necessarily need a create tag. All a create does is move over any +files using this id. In this sense a create is similar to insertion into +an imaginary array of files. + +The create and delete tags allow littlefs to keep files in a directory +ordered alphabetically by filename. + +--- +#### `0x4ff` LFS_TYPE_DELETE + +Deletes the file with this id. An inverse to create, this tag moves over +any files neighboring this id similar to a deletion from an imaginary +array of files. + +--- +#### `0x0xx` LFS_TYPE_NAME + +Associates the id with a file name and file type. + +The data contains the file name stored as an ASCII string (may be expanded to +UTF8 in the future). + +The chunk field in this tag indicates an 8-bit file type which can be one of +the following. + +Currently, the name tag must precede any other tags associated with the id and +can not be reassigned without deleting the file. + +Layout of the name tag: + +``` + tag data +[-- 32 --][--- variable length ---] +[1| 3| 8 | 10 | 10 ][--- (size) ---] + ^ ^ ^ ^ ^- size ^- file name + | | | '------ id + | | '----------- file type + | '-------------- type1 (0x0) + '----------------- valid bit ``` -## Entry attributes +Name fields: + +- **file type (8-bits)** - Type of the file. + +- **file name** - File name stored as an ASCII string. + +--- +#### `0x001` LFS_TYPE_REG -Each dir entry can have up to 256 bytes of system-specific attributes. Since -these attributes are system-specific, they may not be portable between -different systems. For this reason, all attributes must be optional. A minimal -littlefs driver must be able to get away with supporting no attributes at all. +Initializes the id + name as a regular file. -For some level of portability, littlefs has a simple scheme for attributes. -Each attribute is prefixes with an 8-bit type that indicates what the attribute -is. The length of attributes may also be determined from this type. Attributes -in an entry should be sorted based on portability, since attribute parsing -will end when it hits the first attribute it does not understand. +How each file is stored depends on its struct tag, which is described below. -Each system should choose a 4-bit value to prefix all attribute types with to -avoid conflicts with other systems. Additionally, littlefs drivers that support -attributes should provide a "ignore attributes" flag to users in case attribute -conflicts do occur. +--- +#### `0x002` LFS_TYPE_DIR -Attribute types prefixes with 0x0 and 0xf are currently reserved for future -standard attributes. Standard attributes will be added to this document in -that case. +Initializes the id + name as a directory. -Here's an example of non-standard time attribute: +Directories in littlefs are stored on disk as a linked-list of metadata pairs, +each pair containing any number of files in alphabetical order. A pointer to +the directory is stored in the struct tag, which is described below. + +--- +#### `0x0ff` LFS_TYPE_SUPERBLOCK + +Initializes the id as a superblock entry. + +The superblock entry is a special entry used to store format-time configuration +and identify the filesystem. + +The name is a bit of a misnomer. While the superblock entry serves the same +purpose as a superblock found in other filesystems, in littlefs the superblock +does not get a dedicated block. Instead, the superblock entry is duplicated +across a linked-list of metadata pairs rooted on the blocks 0 and 1. The last +metadata pair doubles as the root directory of the filesystem. + +``` +.--------. .--------. .--------. .--------. .--------. +| super |->| super |->| super |->| super |->| file B | +| block | | block | | block | | block | | file C | +| | | | | | | file A | | file D | +'--------' '--------' '--------' '--------' '--------' + +\---------------+----------------/ \---------+---------/ + superblock pairs root directory ``` -(8 bits) attribute type = time (0xc1) -(72 bits) time in seconds = 1506286115 (0x0059c81a23) -00000000: c1 23 1a c8 59 00 .#..Y. +The filesystem starts with only the root directory. The superblock metadata +pairs grow every time the root pair is compacted in order to prolong the +life of the device exponentially. + +The contents of the superblock entry are stored in a name tag with the +superblock type and an inline-struct tag. The name tag contains the magic +string "littlefs", while the inline-struct tag contains version and +configuration information. + +Layout of the superblock name tag and inline-struct tag: + ``` + tag data +[-- 32 --][-- 32 --|-- 32 --] +[1|- 11 -| 10 | 10 ][--- 64 ---] + ^ ^ ^ ^- size (8) ^- magic string ("littlefs") + | | '------ id (0) + | '------------ type (0x0ff) + '----------------- valid bit + + tag data +[-- 32 --][-- 32 --|-- 32 --| +[1|- 11 -| 10 | 10 ][-- 32 --|-- 32 --| + ^ ^ ^ ^ ^- version ^- block size + | | | '- size (24) + | | '------ id (0) + | '------------ type (0x201) + '----------------- valid bit + + data (cont) +|-- 32 --|-- 32 --|-- 32 --| +|-- 32 --|-- 32 --|-- 32 --| + ^- block count ^- name max ^- file max + + data (cont) +|-- 32 --] +|-- 32 --] + ^- attr max +``` + +Superblock fields: + +- **Magic string (8-bytes)** - Magic string indicating the presence of littlefs + on the device. Must be the string "littlefs". + +- **Version (32-bits)** - The version of littlefs at format time. The version + is encoded in a 32-bit value with the upper 16-bits containing the major + version, and the lower 16-bits containing the minor version. + + This specification describes version 2.0 (`0x00020000`). + +- **Block size (32-bits)** - Size of the logical block size used by the + filesystem in bytes. + +- **Block count (32-bits)** - Number of blocks in the filesystem. + +- **Name max (32-bits)** - Maximum size of file names in bytes. + +- **File max (32-bits)** - Maximum size of files in bytes. + +- **Attr max (32-bits)** - Maximum size of file attributes in bytes. + +The superblock must always be the first entry (id 0) in a metdata pair as well +as be the first entry written to the block. This means that the superblock +entry can be read from a device using offsets alone. + +--- +#### `0x2xx` LFS_TYPE_STRUCT + +Associates the id with an on-disk data structure. -Here's an example of non-standard permissions attribute: +The exact layout of the data depends on the data structure type stored in the +chunk field and can be one of the following. + +Any type of struct supercedes all other structs associated with the id. For +example, appending a ctz-struct replaces an inline-struct on the same file. + +--- +#### `0x200` LFS_TYPE_DIRSTRUCT + +Gives the id a directory data structure. + +Directories in littlefs are stored on disk as a linked-list of metadata pairs, +each pair containing any number of files in alphabetical order. + +``` + | + v +.--------. .--------. .--------. .--------. .--------. .--------. +| file A |->| file D |->| file G |->| file I |->| file J |->| file M | +| file B | | file E | | file H | | | | file K | | file N | +| file C | | file F | | | | | | file L | | | +'--------' '--------' '--------' '--------' '--------' '--------' ``` -(8 bits) attribute type = permissions (0xc2) -(16 bits) permission bits = rw-rw-r-- (0x01b4) -00000000: c2 b4 01 ... +The dir-struct tag contains only the pointer to the first metadata-pair in the +directory. The directory size is not known without traversing the directory. + +The pointer to the next metadata-pair in the directory is stored in a tail tag, +which is described below. + +Layout of the dir-struct tag: + ``` + tag data +[-- 32 --][-- 32 --|-- 32 --] +[1|- 11 -| 10 | 10 ][--- 64 ---] + ^ ^ ^ ^- size (8) ^- metadata pair + | | '------ id + | '------------ type (0x200) + '----------------- valid bit +``` + +Dir-struct fields: + +- **Metadata pair (8-bytes)** - Pointer to the first metadata-pair + in the directory. + +--- +#### `0x201` LFS_TYPE_INLINESTRUCT + +Gives the id an inline data structure. + +Inline structs store small files that can fit in the metdata pair. In this +case, the file data is stored directly in the tag's data area. + +Layout of the inline-struct tag: -Here's what a dir entry may look like with these attributes: ``` -(8 bits) entry type = file (0x11) -(8 bits) entry length = 8 bytes (0x08) -(8 bits) attribute length = 9 bytes (0x09) -(8 bits) name length = 12 bytes (0x0c) -(8 bytes) entry data = 05 00 00 00 20 00 00 00 -(8 bits) attribute type = time (0xc1) -(72 bits) time in seconds = 1506286115 (0x0059c81a23) -(8 bits) attribute type = permissions (0xc2) -(16 bits) permission bits = rw-rw-r-- (0x01b4) -(12 bytes) entry name = smallavacado + tag data +[-- 32 --][--- variable length ---] +[1|- 11 -| 10 | 10 ][--- (size) ---] + ^ ^ ^ ^- size ^- inline data + | | '------ id + | '------------ type (0x201) + '----------------- valid bit +``` + +Inline-struct fields: + +- **Inline data** - File data stored directly in the metadata-pair. + +--- +#### `0x202` LFS_TYPE_CTZSTRUCT + +Gives the id a CTZ skip-list data structure. + +CTZ skip-lists store files that can not fit in the metadata pair. These files +are stored in a skip-list in reverse, with a pointer to the head of the +skip-list. Note that the head of the skip-list and the file size is enough +information to read the file. + +How exactly CTZ skip-lists work is a bit complicted. A full explanation can be +found in the [DESIGN.md](DESIGN.md#ctz-skip-lists). + +A quick summary: For every nth block where n is divisible by 2^x, the block +contains a pointer to block n-2^x. These pointers are stored in increasing +order of x in each block of the file before the actual data. + +``` + | + v +.--------. .--------. .--------. .--------. .--------. .--------. +| A |<-| D |<-| G |<-| J |<-| M |<-| P | +| B |<-| E |--| H |<-| K |--| N | | Q | +| C |<-| F |--| I |--| L |--| O | | | +'--------' '--------' '--------' '--------' '--------' '--------' + block 0 block 1 block 2 block 3 block 4 block 5 + 1 skip 2 skips 1 skip 3 skips 1 skip +``` + +Note that the maximum number of pointers in a block is bounded by the maximum +file size divided by the block size. With 32 bits for file size, this results +in a minimum block size of 104 bytes. + +Layout of the CTZ-struct tag: + +``` + tag data +[-- 32 --][-- 32 --|-- 32 --] +[1|- 11 -| 10 | 10 ][-- 32 --|-- 32 --] + ^ ^ ^ ^ ^ ^- file size + | | | | '-------------------- file head + | | | '- size (8) + | | '------ id + | '------------ type (0x202) + '----------------- valid bit +``` + +CTZ-struct fields: + +- **File head (32-bits)** - Pointer to the block that is the head of the + file's CTZ skip-list. + +- **File size (32-bits)** - Size of the file in bytes. + +--- +#### `0x3xx` LFS_TYPE_USERATTR + +Attaches a user attribute to an id. + +littlefs has a concept of "user attributes". These are small user-provided +attributes that can be used to store things like timestamps, hashes, +permissions, etc. + +Each user attribute is uniquely identified by an 8-bit type which is stored in +the chunk field, and the user attribute itself can be found in the tag's data. + +There are currently no standard user attributes and a portable littlefs +implementation should work with any user attributes missing. + +Layout of the user-attr tag: -00000000: 11 08 09 0c 05 00 00 00 20 00 00 00 c1 23 1a c8 ........ ....#.. -00000010: 59 00 c2 b4 01 73 6d 61 6c 6c 61 76 61 63 61 64 Y....smallavacad -00000020: 6f o ``` + tag data +[-- 32 --][--- variable length ---] +[1| 3| 8 | 10 | 10 ][--- (size) ---] + ^ ^ ^ ^ ^- size ^- attr data + | | | '------ id + | | '----------- attr type + | '-------------- type1 (0x3) + '----------------- valid bit +``` + +User-attr fields: + +- **Attr type (8-bits)** - Type of the user attributes. + +- **Attr data** - The data associated with the user attribute. + +--- +#### `0x6xx` LFS_TYPE_TAIL + +Provides the tail pointer for the metadata pair itself. + +The metadata pair's tail pointer is used in littlefs for a linked-list +containing all metadata pairs. The chunk field contains the type of the tail, +which indicates if the following metadata pair is a part of the directory +(hard-tail) or only used to traverse the filesystem (soft-tail). + +``` + .--------. + | dir A |-. + |softtail| | +.--------| |-' +| '--------' +| .-' '-------------. +| v v +| .--------. .--------. .--------. +'->| dir B |->| dir B |->| dir C | + |hardtail| |softtail| | | + | | | | | | + '--------' '--------' '--------' +``` + +Currently any type supercedes any other preceding tails in the metadata pair, +but this may change if additional metadata pair state is added. + +A note about the metadata pair linked-list: Normally, this linked-list contains +every metadata pair in the filesystem. However, there are some operations that +can cause this linked-list to become out of sync if a power-loss were to occur. +When this happens, littlefs sets the "sync" flag in the global state. How +exactly this flag is stored is described below. + +When the sync flag is set: + +- The linked-list may contain an orphaned directory that has been removed in + the filesystem. +- The linked-list may contain a metadata pair with a bad block that has been + replaced in the filesystem. + +If the sync flag is set, the threaded linked-list must be checked for these +errors before it can be used reliably. Note that the threaded linked-list can +be ignored if littlefs is mounted read-only. + +Layout of the tail tag: + +``` + tag data +[-- 32 --][-- 32 --|-- 32 --] +[1| 3| 8 | 10 | 10 ][--- 64 ---] + ^ ^ ^ ^ ^- size (8) ^- metadata pair + | | | '------ id + | | '---------- tail type + | '------------- type1 (0x6) + '---------------- valid bit +``` + +Tail fields: + +- **Tail type (8-bits)** - Type of the tail pointer. + +- **Metadata pair (8-bytes)** - Pointer to the next metadata-pair. + +--- +#### `0x600` LFS_TYPE_SOFTTAIL + +Provides a tail pointer that points to the next metadata pair in the +filesystem. + +In this case, the next metadata pair is not a part of our current directory +and should only be followed when traversing the entire filesystem. + +--- +#### `0x601` LFS_TYPE_HARDTAIL + +Provides a tail pointer that points to the next metadata pair in the +directory. + +In this case, the next metadata pair belongs to the current directory. Note +that because directories in littlefs are sorted alphabetically, the next +metadata pair should only contain filenames greater than any filename in the +current pair. + +--- +#### `0x7xx` LFS_TYPE_GSTATE + +Provides delta bits for global state entries. + +littlefs has a concept of "global state". This is a small set of state that +can be updated by a commit to _any_ metadata pair in the filesystem. + +The way this works is that the global state is stored as a set of deltas +distributed across the filesystem such that the global state can by found by +the xor-sum of these deltas. + +``` +.--------. .--------. .--------. .--------. .--------. +| |->| gstate |->| |->| gstate |->| gstate | +| | | 0x23 | | | | 0xff | | 0xce | +| | | | | | | | | | +'--------' '--------' '--------' '--------' '--------' + | | | + v v v + 0x00 --> xor ------------------> xor ------> xor --> gstate 0x12 +``` + +Note that storing globals this way is very expensive in terms of storage usage, +so any global state should be kept very small. + +The size and format of each piece of global state depends on the type, which +is stored in the chunk field. Currently, the only global state is move state, +which is outlined below. + +--- +#### `0x7ff` LFS_TYPE_MOVESTATE + +Provides delta bits for the global move state. + +The move state in littlefs is used to store info about operations that could +cause to filesystem to go out of sync if the power is lost. The operations +where this could occur is moves of files between metadata pairs and any +operation that changes metadata pairs on the threaded linked-list. + +In the case of moves, the move state contains a tag + metadata pair describing +the source of the ongoing move. If this tag is non-zero, that means that power +was lost during a move, and the file exists in two different locations. If this +happens, the source of the move should be considered deleted, and the move +should be completed (the source should be deleted) before any other write +operations to the filesystem. + +In the case of operations to the threaded linked-list, a single "sync" bit is +used to indicate that a modification is ongoing. If this sync flag is set, the +threaded linked-list will need to be checked for errors before it can be used +reliably. The exact cases to check for are described above in the tail tag. + +Layout of the move state: + +``` + tag data +[-- 32 --][-- 32 --|-- 32 --|-- 32 --] +[1|- 11 -| 10 | 10 ][1|- 11 -| 10 | 10 |--- 64 ---] + ^ ^ ^ ^ ^ ^ ^ ^- padding (0) ^- metadata pair + | | | | | | '------ move id + | | | | | '------------ move type + | | | | '----------------- sync bit + | | | | + | | | '- size (12) + | | '------ id (0x3ff) + | '------------ type (0x7ff) + '----------------- valid bit +``` + +Move state fields: + +- **Sync bit (1-bit)** - Indicates if the metadata pair threaded linked-list is + in-sync. If set, the threaded linked-list should be checked for errors. + +- **Move type (11-bits)** - Type of move being performed. Must be either + `0x000`, indicating no move, or `0x4ff` indicating the source file should + be deleted. + +- **Move id (10-bits)** - The file id being moved. + +- **Metadata pair (8-bytes)** - Pointer to the metadata-pair containing + the move. + +--- +#### `0x5xx` LFS_TYPE_CRC + +Last but not least, the CRC tag marks the end of a commit and provides a +checksum for any commits to the metadata block. + +The first 32-bits of the data contain a CRC-32 with a polynomial of +`0x04c11db7` initialized with `0xffffffff`. This CRC provides a checksum for +all metadata since the previous CRC tag, including the CRC tag itself. For +the first commit, this includes the revision count for the metadata block. + +However, the size of the data is not limited to 32-bits. The data field may +larger to pad the commit to the next program-aligned boundary. + +In addition, the CRC tag's chunk field contains a set of flags which can +change the behaviour of commits. Currently the only flag in use is the lowest +bit, which determines the expected state of the valid bit for any following +tags. This is used to guarantee that unwritten storage in a metadata block +will be detected as invalid. + +Layout of the CRC tag: + +``` + tag data +[-- 32 --][-- 32 --|--- variable length ---] +[1| 3| 8 | 10 | 10 ][-- 32 --|--- (size) ---] + ^ ^ ^ ^ ^ ^- crc ^- padding + | | | | '- size (12) + | | | '------ id (0x3ff) + | | '----------- valid state + | '-------------- type1 (0x5) + '----------------- valid bit +``` + +CRC fields: + +- **Valid state (1-bit)** - Indicates the expected value of the valid bit for + any tags in the next commit. + +- **CRC (32-bits)** - CRC-32 with a polynomial of `0x04c11db7` initialized with + `0xffffffff`. + +- **Padding** - Padding to the next program-aligned boundary. No guarantees are + made about the contents. + +--- From a0644794ca63ae63bb52fcbd26d3a50bff03a778 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 12 Feb 2019 00:01:28 -0600 Subject: [PATCH 130/139] Fixed several small issues - Fixed uninitialized values found by valgrind. - Fixed uninitialized value in lfs_dir_fetchmatch when handling revision counts. - Fixed mess left by lfs_dir_find when attempting to find the root directory in lfs_rename and lfs_remove. - Fixed corner case with definitions of lfs->cfg->block_cycles. - Added test cases around different forms of the root directory. I think all of these were found by TheLoneWolfling, so props! --- lfs.c | 21 +++++++++++++-------- tests/test_paths.sh | 8 ++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/lfs.c b/lfs.c index c6ff2e98..c4832d99 100644 --- a/lfs.c +++ b/lfs.c @@ -765,7 +765,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_stag_t besttag = -1; // find the block with the most recent revision - uint32_t revs[2]; + uint32_t revs[2] = {0, 0}; int r = 0; for (int i = 0; i < 2; i++) { int err = lfs_bd_read(lfs, @@ -776,7 +776,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, return err; } - if (lfs_scmp(revs[i], revs[(i+1)%2]) > 0 || err == LFS_ERR_CORRUPT) { + if (err != LFS_ERR_CORRUPT && + lfs_scmp(revs[i], revs[(i+1)%2]) > 0) { r = i; } } @@ -2944,8 +2945,8 @@ int lfs_remove(lfs_t *lfs, const char *path) { lfs_mdir_t cwd; lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL); - if (tag < 0) { - return tag; + if (tag < 0 || lfs_tag_id(tag) == 0x3ff) { + return (tag < 0) ? tag : LFS_ERR_INVAL; } lfs_mdir_t dir; @@ -3007,16 +3008,17 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find old entry lfs_mdir_t oldcwd; lfs_stag_t oldtag = lfs_dir_find(lfs, &oldcwd, &oldpath, NULL); - if (oldtag < 0) { - return oldtag; + if (oldtag < 0 || lfs_tag_id(oldtag) == 0x3ff) { + return (oldtag < 0) ? oldtag : LFS_ERR_INVAL; } // find new entry lfs_mdir_t newcwd; uint16_t newid; lfs_stag_t prevtag = lfs_dir_find(lfs, &newcwd, &newpath, &newid); - if (prevtag < 0 && !(prevtag == LFS_ERR_NOENT && newid != 0x3ff)) { - return err; + if ((prevtag < 0 || lfs_tag_id(prevtag) == 0x3ff) && + !(prevtag == LFS_ERR_NOENT && newid != 0x3ff)) { + return (prevtag < 0) ? prevtag : LFS_ERR_INVAL; } lfs_mdir_t prevdir; @@ -3187,6 +3189,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) <= lfs->cfg->block_size); + // we don't support some corner cases + LFS_ASSERT(lfs->cfg->block_cycles < 0xffffffff); + // setup read cache if (lfs->cfg->read_buffer) { lfs->rcache.buffer = lfs->cfg->read_buffer; diff --git a/tests/test_paths.sh b/tests/test_paths.sh index b1d41a7f..3cffcfea 100755 --- a/tests/test_paths.sh +++ b/tests/test_paths.sh @@ -128,6 +128,14 @@ tests/test.py << TEST lfs_mkdir(&lfs, "/") => LFS_ERR_EXIST; lfs_file_open(&lfs, &file[0], "/", LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_ISDIR; + + // more corner cases + lfs_remove(&lfs, "") => LFS_ERR_INVAL; + lfs_remove(&lfs, ".") => LFS_ERR_INVAL; + lfs_remove(&lfs, "..") => LFS_ERR_INVAL; + lfs_remove(&lfs, "/") => LFS_ERR_INVAL; + lfs_remove(&lfs, "//") => LFS_ERR_INVAL; + lfs_remove(&lfs, "./") => LFS_ERR_INVAL; lfs_unmount(&lfs) => 0; TEST From 7d8f8ced03ea7b0afd7f26c340fc1c2474360ab5 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 12 Feb 2019 00:11:01 -0600 Subject: [PATCH 131/139] Enabled -Wextra This only required adding NULLs where commit statements were not fully initialized. Unfortunately we still need -Wno-missing-field-initializers because of a bug in GCC that persists on Travis. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=60784 Found by apmorton --- Makefile | 4 +++- lfs.c | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 79d91565..185d8e59 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,9 @@ override CFLAGS += -m$(WORD) endif override CFLAGS += -I. override CFLAGS += -std=c99 -Wall -pedantic -override CFLAGS += -Wshadow -Wunused-parameter -Wjump-misses-init -Wsign-compare +override CFLAGS += -Wextra -Wshadow -Wjump-misses-init +# Remove missing-field-initializers because of GCC bug +override CFLAGS += -Wno-missing-field-initializers all: $(TARGET) diff --git a/lfs.c b/lfs.c index c4832d99..7db5dba8 100644 --- a/lfs.c +++ b/lfs.c @@ -1889,7 +1889,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { // now insert into our parent block lfs_pair_tole32(dir.pair); err = lfs_dir_commit(lfs, &cwd, LFS_MKATTRS( - {LFS_MKTAG(LFS_TYPE_CREATE, id, 0)}, + {LFS_MKTAG(LFS_TYPE_CREATE, id, 0), NULL}, {LFS_MKTAG(LFS_TYPE_DIR, id, nlen), path}, {LFS_MKTAG(LFS_TYPE_DIRSTRUCT, id, 8), dir.pair}, {!cwd.split @@ -2296,9 +2296,9 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, // get next slot and create entry to remember name err = lfs_dir_commit(lfs, &file->m, LFS_MKATTRS( - {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0)}, + {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0), NULL}, {LFS_MKTAG(LFS_TYPE_REG, file->id, nlen), path}, - {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0)})); + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), NULL})); if (err) { err = LFS_ERR_NAMETOOLONG; goto cleanup; @@ -2975,7 +2975,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { // delete the entry err = lfs_dir_commit(lfs, &cwd, LFS_MKATTRS( - {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(tag), 0)})); + {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(tag), 0), NULL})); if (err) { return err; } @@ -3070,8 +3070,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTRS( {prevtag != LFS_ERR_NOENT ? LFS_MKTAG(LFS_TYPE_DELETE, newid, 0) - : LFS_MKTAG(LFS_FROM_NOOP, 0, 0)}, - {LFS_MKTAG(LFS_TYPE_CREATE, newid, 0)}, + : LFS_MKTAG(LFS_FROM_NOOP, 0, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_CREATE, newid, 0), NULL}, {LFS_MKTAG(lfs_tag_type3(oldtag), newid, strlen(newpath)), newpath}, {LFS_MKTAG(LFS_FROM_MOVE, newid, lfs_tag_id(oldtag)), &oldcwd})); @@ -3318,7 +3318,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_tole32(&superblock); err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( - {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0)}, + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL}, {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), &superblock})); From 4ad09d6c4ec95c126909b905900237cdc829c6b0 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 22 Feb 2019 21:34:03 -0600 Subject: [PATCH 132/139] Added migration from littlefs v1 This is the help the introduction of littlefs v2, which is disk incompatible with littlefs v1. While v2 can't mount v1, what we can do is provide an optional migration, which can convert v1 into v2 partially in-place. At worse, we only need to carry over the readonly operations on v1, which are much less complicated than the write operations, so the extra code cost may be as low as 25% of the v1 code size. Also, because v2 contains only metadata changes, it's possible to avoid copying file data during the update. Enabling the migration requires two steps 1. Defining LFS_MIGRATE 2. Call lfs_migrate (only available with the above macro) Each macro multiplies the number of configurations needed to be tested, so I've been avoiding macro controlled features since there's still work to be done around testing the single configuration that's already available. However, here the cost would be too high if we included migration code in the standard build. We can't use the lfs_migrate function for link time gc because of a dependency between the allocator and v1 data structures. So how does lfs_migrate work? It turned out to be a bit complicated, but the answer is a multistep process that relies on mounting v1 readonly and building the metadata skeleton needed by v2. 1. For each directory, create a v2 directory 2. Copy over v1 entries into v2 directory, including the soft-tail entry 3. Move head block of v2 directory into the unused metadata block in v1 directory. This results in both a v1 and v2 directory sharing the same metadata pair. 4. Finally, create a new superblock in the unused metadata block of the v1 superblock. Just like with normal metadata updates, the completion of the write to the second metadata block marks a succesful migration that can be mounted with littlefs v2. And all of this can occur atomically, enabling complete fallback if power is lost of an error occurs. Note there are several limitations with this solution. 1. While migration doesn't duplicate file data, it does temporarily duplicate all metadata. This can cause a device to run out of space if storage is tight and the filesystem as many files. If the device was created with >~2x the expected storage, it should be fine. 2. The current implementation is not able to recover if the metadata pairs develop bad blocks. It may be possilbe to workaround this, but it creates the problem that directories may change location during the migration. The other solutions I've looked at are complicated and require superlinear runtime. Currently I don't think it's worth fixing this limitation. 3. Enabling the migration requires additional code size. Currently this looks like it's roughly 11% at least on x86. And, if any failure does occur, no harm is done to the original v1 filesystem on disk. --- .travis.yml | 3 +- lfs.c | 647 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lfs.h | 19 ++ 3 files changed, 668 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ccbcaa87..aa9663d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -113,7 +113,7 @@ jobs: - mkdir mount - sudo chmod a+rw /dev/loop0 - - dd if=/dev/zero bs=512 count=2048 of=disk + - dd if=/dev/zero bs=512 count=4096 of=disk - losetup /dev/loop0 disk script: # self-host test @@ -126,6 +126,7 @@ jobs: - mkdir mount/littlefs - cp -r $(git ls-tree --name-only HEAD) mount/littlefs - cd mount/littlefs + - stat . - ls -flh - make -B test_dirs test_files QUIET=1 diff --git a/lfs.c b/lfs.c index 7db5dba8..c0867187 100644 --- a/lfs.c +++ b/lfs.c @@ -439,6 +439,10 @@ static int lfs_fs_relocate(lfs_t *lfs, const lfs_block_t oldpair[2], lfs_block_t newpair[2]); static int lfs_fs_forceconsistency(lfs_t *lfs); static int lfs_deinit(lfs_t *lfs); +#ifdef LFS_MIGRATE +static int lfs1_traverse(lfs_t *lfs, + int (*cb)(void*, lfs_block_t), void *data); +#endif /// Block allocator /// static int lfs_alloc_lookahead(void *p, lfs_block_t block) { @@ -3258,6 +3262,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->gstate = (struct lfs_gstate){0}; lfs->gpending = (struct lfs_gstate){0}; lfs->gdelta = (struct lfs_gstate){0}; +#ifdef LFS_MIGRATE + lfs->lfs1 = NULL; +#endif return 0; @@ -3468,6 +3475,20 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *data, lfs_block_t block), void *data) { // iterate over metadata pairs lfs_mdir_t dir = {.tail = {0, 1}}; + +#ifdef LFS_MIGRATE + // also consider v1 blocks during migration + if (lfs->lfs1) { + int err = lfs1_traverse(lfs, cb, data); + if (err) { + return err; + } + + dir.tail[0] = lfs->root[0]; + dir.tail[1] = lfs->root[1]; + } +#endif + while (!lfs_pair_isnull(dir.tail)) { for (int i = 0; i < 2; i++) { int err = cb(data, dir.tail[i]); @@ -3798,3 +3819,629 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs) { return size; } + +#ifdef LFS_MIGRATE +////// Migration from littelfs v1 below this ////// + +/// Version info /// + +// Software library version +// Major (top-nibble), incremented on backwards incompatible changes +// Minor (bottom-nibble), incremented on feature additions +#define LFS1_VERSION 0x00010007 +#define LFS1_VERSION_MAJOR (0xffff & (LFS1_VERSION >> 16)) +#define LFS1_VERSION_MINOR (0xffff & (LFS1_VERSION >> 0)) + +// Version of On-disk data structures +// Major (top-nibble), incremented on backwards incompatible changes +// Minor (bottom-nibble), incremented on feature additions +#define LFS1_DISK_VERSION 0x00010001 +#define LFS1_DISK_VERSION_MAJOR (0xffff & (LFS1_DISK_VERSION >> 16)) +#define LFS1_DISK_VERSION_MINOR (0xffff & (LFS1_DISK_VERSION >> 0)) + + +/// v1 Definitions /// + +// File types +enum lfs1_type { + LFS1_TYPE_REG = 0x11, + LFS1_TYPE_DIR = 0x22, + LFS1_TYPE_SUPERBLOCK = 0x2e, +}; + +typedef struct lfs1 { + lfs_block_t root[2]; +} lfs1_t; + +typedef struct lfs1_entry { + lfs_off_t off; + + struct lfs1_disk_entry { + uint8_t type; + uint8_t elen; + uint8_t alen; + uint8_t nlen; + union { + struct { + lfs_block_t head; + lfs_size_t size; + } file; + lfs_block_t dir[2]; + } u; + } d; +} lfs1_entry_t; + +typedef struct lfs1_dir { + struct lfs1_dir *next; + lfs_block_t pair[2]; + lfs_off_t off; + + lfs_block_t head[2]; + lfs_off_t pos; + + struct lfs1_disk_dir { + uint32_t rev; + lfs_size_t size; + lfs_block_t tail[2]; + } d; +} lfs1_dir_t; + +typedef struct lfs1_superblock { + lfs_off_t off; + + struct lfs1_disk_superblock { + uint8_t type; + uint8_t elen; + uint8_t alen; + uint8_t nlen; + lfs_block_t root[2]; + uint32_t block_size; + uint32_t block_count; + uint32_t version; + char magic[8]; + } d; +} lfs1_superblock_t; + + +/// Low-level wrappers v1->v2 /// +void lfs1_crc(uint32_t *crc, const void *buffer, size_t size) { + *crc = lfs_crc(*crc, buffer, size); +} + +static int lfs1_bd_read(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size) { + // if we ever do more than writes to alternating pairs, + // this may need to consider pcache + return lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, size, + block, off, buffer, size); +} + +static int lfs1_bd_crc(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, lfs_size_t size, uint32_t *crc) { + for (lfs_off_t i = 0; i < size; i++) { + uint8_t c; + int err = lfs1_bd_read(lfs, block, off+i, &c, 1); + if (err) { + return err; + } + + lfs1_crc(crc, &c, 1); + } + + return 0; +} + + +/// Endian swapping functions /// +static void lfs1_dir_fromle32(struct lfs1_disk_dir *d) { + d->rev = lfs_fromle32(d->rev); + d->size = lfs_fromle32(d->size); + d->tail[0] = lfs_fromle32(d->tail[0]); + d->tail[1] = lfs_fromle32(d->tail[1]); +} + +static void lfs1_dir_tole32(struct lfs1_disk_dir *d) { + d->rev = lfs_tole32(d->rev); + d->size = lfs_tole32(d->size); + d->tail[0] = lfs_tole32(d->tail[0]); + d->tail[1] = lfs_tole32(d->tail[1]); +} + +static void lfs1_entry_fromle32(struct lfs1_disk_entry *d) { + d->u.dir[0] = lfs_fromle32(d->u.dir[0]); + d->u.dir[1] = lfs_fromle32(d->u.dir[1]); +} + +static void lfs1_entry_tole32(struct lfs1_disk_entry *d) { + d->u.dir[0] = lfs_tole32(d->u.dir[0]); + d->u.dir[1] = lfs_tole32(d->u.dir[1]); +} + +static void lfs1_superblock_fromle32(struct lfs1_disk_superblock *d) { + d->root[0] = lfs_fromle32(d->root[0]); + d->root[1] = lfs_fromle32(d->root[1]); + d->block_size = lfs_fromle32(d->block_size); + d->block_count = lfs_fromle32(d->block_count); + d->version = lfs_fromle32(d->version); +} + + +///// Metadata pair and directory operations /// +static inline lfs_size_t lfs1_entry_size(const lfs1_entry_t *entry) { + return 4 + entry->d.elen + entry->d.alen + entry->d.nlen; +} + +static int lfs1_dir_fetch(lfs_t *lfs, + lfs1_dir_t *dir, const lfs_block_t pair[2]) { + // copy out pair, otherwise may be aliasing dir + const lfs_block_t tpair[2] = {pair[0], pair[1]}; + bool valid = false; + + // check both blocks for the most recent revision + for (int i = 0; i < 2; i++) { + struct lfs1_disk_dir test; + int err = lfs1_bd_read(lfs, tpair[i], 0, &test, sizeof(test)); + lfs1_dir_fromle32(&test); + if (err) { + if (err == LFS_ERR_CORRUPT) { + continue; + } + return err; + } + + if (valid && lfs_scmp(test.rev, dir->d.rev) < 0) { + continue; + } + + if ((0x7fffffff & test.size) < sizeof(test)+4 || + (0x7fffffff & test.size) > lfs->cfg->block_size) { + continue; + } + + uint32_t crc = 0xffffffff; + lfs1_dir_tole32(&test); + lfs1_crc(&crc, &test, sizeof(test)); + lfs1_dir_fromle32(&test); + err = lfs1_bd_crc(lfs, tpair[i], sizeof(test), + (0x7fffffff & test.size) - sizeof(test), &crc); + if (err) { + if (err == LFS_ERR_CORRUPT) { + continue; + } + return err; + } + + if (crc != 0) { + continue; + } + + valid = true; + + // setup dir in case it's valid + dir->pair[0] = tpair[(i+0) % 2]; + dir->pair[1] = tpair[(i+1) % 2]; + dir->off = sizeof(dir->d); + dir->d = test; + } + + if (!valid) { + LFS_ERROR("Corrupted dir pair at %" PRIu32 " %" PRIu32 , + tpair[0], tpair[1]); + return LFS_ERR_CORRUPT; + } + + return 0; +} + +static int lfs1_dir_next(lfs_t *lfs, lfs1_dir_t *dir, lfs1_entry_t *entry) { + while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) { + if (!(0x80000000 & dir->d.size)) { + entry->off = dir->off; + return LFS_ERR_NOENT; + } + + int err = lfs1_dir_fetch(lfs, dir, dir->d.tail); + if (err) { + return err; + } + + dir->off = sizeof(dir->d); + dir->pos += sizeof(dir->d) + 4; + } + + int err = lfs1_bd_read(lfs, dir->pair[0], dir->off, + &entry->d, sizeof(entry->d)); + lfs1_entry_fromle32(&entry->d); + if (err) { + return err; + } + + entry->off = dir->off; + dir->off += lfs1_entry_size(entry); + dir->pos += lfs1_entry_size(entry); + return 0; +} + +/// littlefs v1 specific operations /// +int lfs1_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { + if (lfs_pair_isnull(lfs->lfs1->root)) { + return 0; + } + + // iterate over metadata pairs + lfs1_dir_t dir; + lfs1_entry_t entry; + lfs_block_t cwd[2] = {0, 1}; + + while (true) { + for (int i = 0; i < 2; i++) { + int err = cb(data, cwd[i]); + if (err) { + return err; + } + } + + int err = lfs1_dir_fetch(lfs, &dir, cwd); + if (err) { + return err; + } + + // iterate over contents + while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { + err = lfs1_bd_read(lfs, dir.pair[0], dir.off, + &entry.d, sizeof(entry.d)); + lfs1_entry_fromle32(&entry.d); + if (err) { + return err; + } + + dir.off += lfs1_entry_size(&entry); + if ((0x70 & entry.d.type) == (0x70 & LFS1_TYPE_REG)) { + err = lfs_ctz_traverse(lfs, NULL, &lfs->rcache, + entry.d.u.file.head, entry.d.u.file.size, cb, data); + if (err) { + return err; + } + } + } + + // we also need to check if we contain a threaded v2 directory + lfs_mdir_t dir2 = {.split=true, .tail={cwd[0], cwd[1]}}; + while (dir2.split) { + err = lfs_dir_fetch(lfs, &dir2, dir2.tail); + if (err) { + break; + } + + for (int i = 0; i < 2; i++) { + err = cb(data, dir2.pair[i]); + if (err) { + return err; + } + } + } + + cwd[0] = dir.d.tail[0]; + cwd[1] = dir.d.tail[1]; + + if (lfs_pair_isnull(cwd)) { + break; + } + } + + return 0; +} + +static int lfs1_moved(lfs_t *lfs, const void *e) { + if (lfs_pair_isnull(lfs->lfs1->root)) { + return 0; + } + + // skip superblock + lfs1_dir_t cwd; + int err = lfs1_dir_fetch(lfs, &cwd, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + // iterate over all directory directory entries + lfs1_entry_t entry; + while (!lfs_pair_isnull(cwd.d.tail)) { + err = lfs1_dir_fetch(lfs, &cwd, cwd.d.tail); + if (err) { + return err; + } + + while (true) { + err = lfs1_dir_next(lfs, &cwd, &entry); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + if (err == LFS_ERR_NOENT) { + break; + } + + if (!(0x80 & entry.d.type) && + memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) { + return true; + } + } + } + + return false; +} + +/// Filesystem operations /// +static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1, + const struct lfs_config *cfg) { + int err = 0; + if (true) { + err = lfs_init(lfs, cfg); + if (err) { + return err; + } + + lfs->lfs1 = lfs1; + lfs->lfs1->root[0] = 0xffffffff; + lfs->lfs1->root[1] = 0xffffffff; + + // setup free lookahead + lfs->free.off = 0; + lfs->free.size = 0; + lfs->free.i = 0; + lfs_alloc_ack(lfs); + + // load superblock + lfs1_dir_t dir; + lfs1_superblock_t superblock; + err = lfs1_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + if (err && err != LFS_ERR_CORRUPT) { + goto cleanup; + } + + if (!err) { + err = lfs1_bd_read(lfs, dir.pair[0], sizeof(dir.d), + &superblock.d, sizeof(superblock.d)); + lfs1_superblock_fromle32(&superblock.d); + if (err) { + goto cleanup; + } + + lfs->lfs1->root[0] = superblock.d.root[0]; + lfs->lfs1->root[1] = superblock.d.root[1]; + } + + if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) { + LFS_ERROR("Invalid superblock at %d %d", 0, 1); + err = LFS_ERR_CORRUPT; + goto cleanup; + } + + uint16_t major_version = (0xffff & (superblock.d.version >> 16)); + uint16_t minor_version = (0xffff & (superblock.d.version >> 0)); + if ((major_version != LFS1_DISK_VERSION_MAJOR || + minor_version > LFS1_DISK_VERSION_MINOR)) { + LFS_ERROR("Invalid version %d.%d", major_version, minor_version); + err = LFS_ERR_INVAL; + goto cleanup; + } + + return 0; + } + +cleanup: + lfs_deinit(lfs); + return err; +} + +static int lfs1_unmount(lfs_t *lfs) { + return lfs_deinit(lfs); +} + +/// v1 migration /// +int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { + struct lfs1 lfs1; + int err = lfs1_mount(lfs, &lfs1, cfg); + if (err) { + return err; + } + + if (true) { + // iterate through each directory, copying over entries + // into new directory + lfs1_dir_t dir1; + lfs_mdir_t dir2; + dir1.d.tail[0] = lfs->lfs1->root[0]; + dir1.d.tail[1] = lfs->lfs1->root[1]; + while (!lfs_pair_isnull(dir1.d.tail)) { + // iterate old dir + err = lfs1_dir_fetch(lfs, &dir1, dir1.d.tail); + if (err) { + goto cleanup; + } + + // create new dir and bind as temporary pretend root + err = lfs_dir_alloc(lfs, &dir2); + if (err) { + goto cleanup; + } + + dir2.rev = dir1.d.rev; + lfs->root[0] = dir2.pair[0]; + lfs->root[1] = dir2.pair[1]; + + err = lfs_dir_commit(lfs, &dir2, NULL, 0); + if (err) { + goto cleanup; + } + + while (true) { + lfs1_entry_t entry1; + err = lfs1_dir_next(lfs, &dir1, &entry1); + if (err && err != LFS_ERR_NOENT) { + goto cleanup; + } + + if (err == LFS_ERR_NOENT) { + break; + } + + // check that entry has not been moved + if (entry1.d.type & 0x80) { + int moved = lfs1_moved(lfs, &entry1.d.u); + if (moved < 0) { + err = moved; + goto cleanup; + } + + if (moved) { + continue; + } + + entry1.d.type &= ~0x80; + } + + // also fetch name + char name[LFS_NAME_MAX+1]; + memset(name, 0, sizeof(name)); + err = lfs1_bd_read(lfs, dir1.pair[0], + entry1.off + 4+entry1.d.elen+entry1.d.alen, + name, entry1.d.nlen); + if (err) { + goto cleanup; + } + + bool isdir = (entry1.d.type == LFS1_TYPE_DIR); + + // create entry in new dir + err = lfs_dir_fetch(lfs, &dir2, lfs->root); + if (err) { + goto cleanup; + } + + uint16_t id; + err = lfs_dir_find(lfs, &dir2, &(const char*){name}, &id); + if (!(err == LFS_ERR_NOENT && id != 0x3ff)) { + err = (err < 0) ? err : LFS_ERR_EXIST; + goto cleanup; + } + + lfs1_entry_tole32(&entry1.d); + err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, id, 0), NULL}, + {LFS_MKTAG( + isdir ? LFS_TYPE_DIR : LFS_TYPE_REG, + id, entry1.d.nlen), name}, + {LFS_MKTAG( + isdir ? LFS_TYPE_DIRSTRUCT : LFS_TYPE_CTZSTRUCT, + id, sizeof(&entry1.d.u)), &entry1.d.u})); + lfs1_entry_fromle32(&entry1.d); + if (err) { + goto cleanup; + } + } + + if (!lfs_pair_isnull(dir1.d.tail)) { + // find last block and update tail to thread into fs + err = lfs_dir_fetch(lfs, &dir2, lfs->root); + if (err) { + goto cleanup; + } + + while (dir2.split) { + err = lfs_dir_fetch(lfs, &dir2, dir2.tail); + if (err) { + goto cleanup; + } + } + + lfs_pair_tole32(dir2.pair); + err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 0), + dir1.d.tail})); + lfs_pair_fromle32(dir2.pair); + if (err) { + goto cleanup; + } + } + + // Copy over first block to thread into fs. Unfortunately + // if this fails there is not much we can do. + err = lfs_bd_erase(lfs, dir1.pair[1]); + if (err) { + goto cleanup; + } + + err = lfs_dir_fetch(lfs, &dir2, lfs->root); + if (err) { + goto cleanup; + } + + for (lfs_off_t i = 0; i < dir2.off; i++) { + uint8_t dat; + err = lfs_bd_read(lfs, + NULL, &lfs->rcache, dir2.off, + dir2.pair[0], i, &dat, 1); + if (err) { + goto cleanup; + } + + err = lfs_bd_prog(lfs, + &lfs->pcache, &lfs->rcache, true, + dir1.pair[1], i, &dat, 1); + if (err) { + goto cleanup; + } + } + } + + // Create new superblock. This marks a successful migration! + err = lfs1_dir_fetch(lfs, &dir1, (const lfs_block_t[2]){0, 1}); + if (err) { + goto cleanup; + } + + dir2.pair[0] = dir1.pair[0]; + dir2.pair[1] = dir1.pair[1]; + dir2.rev = dir1.d.rev; + dir2.off = sizeof(dir2.rev); + dir2.etag = 0xffffffff; + dir2.count = 0; + dir2.tail[0] = lfs->lfs1->root[0]; + dir2.tail[1] = lfs->lfs1->root[1]; + dir2.erased = false; + dir2.split = true; + + lfs_superblock_t superblock = { + .version = LFS_DISK_VERSION, + .block_size = lfs->cfg->block_size, + .block_count = lfs->cfg->block_count, + .name_max = lfs->name_max, + .file_max = lfs->file_max, + .attr_max = lfs->attr_max, + }; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})); + if (err) { + goto cleanup; + } + + // sanity check that fetch works + err = lfs_dir_fetch(lfs, &dir2, (const lfs_block_t[2]){0, 1}); + if (err) { + goto cleanup; + } + } + +cleanup: + lfs1_unmount(lfs); + return err; +} + +#endif diff --git a/lfs.h b/lfs.h index ec41c2f9..f797f667 100644 --- a/lfs.h +++ b/lfs.h @@ -380,6 +380,10 @@ typedef struct lfs { lfs_size_t name_max; lfs_size_t file_max; lfs_size_t attr_max; + +#ifdef LFS_MIGRATE + struct lfs1 *lfs1; +#endif } lfs_t; @@ -617,6 +621,21 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // Returns a negative error code on failure. int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); +#ifdef LFS_MIGRATE +// Attempts to migrate a previous version of littlefs +// +// Behaves similarly to the lfs_format function. Attempts to mount +// the previous version of littlefs and update the filesystem so it can be +// mounted with the current version of littlefs. +// +// Requires a littlefs object and config struct. This clobbers the littlefs +// object, and does not leave the filesystem mounted. The config struct must +// be zeroed for defaults and backwards compatibility. +// +// Returns a negative error code on failure. +int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg); +#endif + #ifdef __cplusplus } /* extern "C" */ From bdff4bc59eb69bc403e0f5878e72a96a0a21a190 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 2 Mar 2019 19:30:05 -0600 Subject: [PATCH 133/139] Updated DESIGN.md to reflect v2 changes Now with graphs! Images are stored on the branch gh-images in an effort to avoid binary bloat in the git history. Also spruced up SPEC.md and README.md and ran a spellechecker over the documentation. Favorite typo so far was dependendent, which is, in fact, not a word. --- DESIGN.md | 2965 +++++++++++++++++++++++++++++++++++------------------ README.md | 173 ++-- SPEC.md | 271 +++-- 3 files changed, 2204 insertions(+), 1205 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 3afb0a20..da693f44 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,6 +1,6 @@ -## The design of the little filesystem +## The design of littlefs -A little fail-safe filesystem designed for embedded systems. +A little fail-safe filesystem designed for microcontrollers. ``` | | | .---._____ @@ -11,211 +11,581 @@ A little fail-safe filesystem designed for embedded systems. | | | ``` -For a bit of backstory, the littlefs was developed with the goal of learning -more about filesystem design by tackling the relative unsolved problem of -managing a robust filesystem resilient to power loss on devices -with limited RAM and ROM. - -The embedded systems the littlefs is targeting are usually 32 bit -microcontrollers with around 32KB of RAM and 512KB of ROM. These are -often paired with SPI NOR flash chips with about 4MB of flash storage. - -Flash itself is a very interesting piece of technology with quite a bit of -nuance. Unlike most other forms of storage, writing to flash requires two -operations: erasing and programming. The programming operation is relatively -cheap, and can be very granular. For NOR flash specifically, byte-level -programs are quite common. Erasing, however, requires an expensive operation -that forces the state of large blocks of memory to reset in a destructive -reaction that gives flash its name. The [Wikipedia entry](https://en.wikipedia.org/wiki/Flash_memory) -has more information if you are interested in how this works. - -This leaves us with an interesting set of limitations that can be simplified -to three strong requirements: - -1. **Power-loss resilient** - This is the main goal of the littlefs and the - focus of this project. - - Embedded systems are usually designed without a shutdown routine and a - notable lack of user interface for recovery, so filesystems targeting - embedded systems must be prepared to lose power at any given time. - - Despite this state of things, there are very few embedded filesystems that - handle power loss in a reasonable manner, and most can become corrupted if - the user is unlucky enough. - -2. **Wear leveling** - Due to the destructive nature of flash, most flash - chips have a limited number of erase cycles, usually in the order of around - 100,000 erases per block for NOR flash. Filesystems that don't take wear - into account can easily burn through blocks used to store frequently updated - metadata. - - Consider the [FAT filesystem](https://en.wikipedia.org/wiki/Design_of_the_FAT_file_system), - which stores a file allocation table (FAT) at a specific offset from the - beginning of disk. Every block allocation will update this table, and after - 100,000 updates, the block will likely go bad, rendering the filesystem - unusable even if there are many more erase cycles available on the storage - as a whole. - -3. **Bounded RAM/ROM** - Even with the design difficulties presented by the - previous two limitations, we have already seen several flash filesystems - developed on PCs that handle power loss just fine, such as the - logging filesystems. However, these filesystems take advantage of the - relatively cheap access to RAM, and use some rather... opportunistic... - techniques, such as reconstructing the entire directory structure in RAM. - These operations make perfect sense when the filesystem's only concern is - erase cycles, but the idea is a bit silly on embedded systems. - - To cater to embedded systems, the littlefs has the simple limitation of - using only a bounded amount of RAM and ROM. That is, no matter what is - written to the filesystem, and no matter how large the underlying storage - is, the littlefs will always use the same amount of RAM and ROM. This - presents a very unique challenge, and makes presumably simple operations, - such as iterating through the directory tree, surprisingly difficult. +littlefs was originally built as an experiment to learn about filesystem design +in the context of microcontrollers. The question was: How would you build a +filesystem that is resilient to power-loss and flash wear without using +unbounded memory? + +This document covers the high-level design of littlefs, how it is different +than other filesystems, and the design decisions that got us here. For the +low-level details covering every bit on disk, check out [SPEC.md](SPEC.md). + +## The problem + +The embedded systems littlefs targets are usually 32-bit microcontrollers with +around 32 KiB of RAM and 512 KiB of ROM. These are often paired with SPI NOR +flash chips with about 4 MiB of flash storage. These devices are too small for +Linux and most existing filesystems, requiring code written specifically with +size in mind. + +Flash itself is an interesting piece of technology with its own quirks and +nuance. Unlike other forms of storage, writing to flash requires two +operations: erasing and programming. Programming (setting bits to 0) is +relatively cheap and can be very granular. Erasing however (setting bits to 1), +requires an expensive and destructive operation which gives flash its name. +[Wikipedia][wikipedia-flash] has more information on how exactly flash works. + +To make the situation more annoying, it's very common for these embedded +systems to lose power at any time. Usually, microcontroller code is simple and +reactive, with no concept of a shutdown routine. This presents a big challenge +for persistent storage, where an unlucky power loss can corrupt the storage and +leave a device unrecoverable. + +This leaves us with three major requirements for an embedded filesystem. + +1. **Power-loss resilience** - On these systems, power can be lost at any time. + If a power loss corrupts any persistent data structures, this can cause the + device to become unrecoverable. An embedded filesystem must be designed to + recover from a power loss during any write operation. + +1. **Wear leveling** - Writing to flash is destructive. If a filesystem + repeatedly writes to the same block, eventually that block will wear out. + Filesystems that don't take wear into account can easily burn through blocks + used to store frequently updated metadata and cause a device's early death. + +1. **Bounded RAM/ROM** - If the above requirements weren't enough, these + systems also have very limited amounts of memory. This prevents many + existing filesystem designs, which can lean on relatively large amounts of + RAM to temporarily store filesystem metadata. + + For ROM, this means we need to keep our design simple and reuse code paths + were possible. For RAM we have a stronger requirement, all RAM usage is + bounded. This means RAM usage does not grow as the filesystem changes in + size or number of files. This creates a unique challenge as even presumably + simple operations, such as traversing the filesystem, become surprisingly + difficult. ## Existing designs? -There are of course, many different existing filesystem. Here is a very rough -summary of the general ideas behind some of them. - -Most of the existing filesystems fall into the one big category of filesystem -designed in the early days of spinny magnet disks. While there is a vast amount -of interesting technology and ideas in this area, the nature of spinny magnet -disks encourage properties, such as grouping writes near each other, that don't -make as much sense on recent storage types. For instance, on flash, write -locality is not important and can actually increase wear. - -One of the most popular designs for flash filesystems is called the -[logging filesystem](https://en.wikipedia.org/wiki/Log-structured_file_system). -The flash filesystems [jffs](https://en.wikipedia.org/wiki/JFFS) -and [yaffs](https://en.wikipedia.org/wiki/YAFFS) are good examples. In a -logging filesystem, data is not stored in a data structure on disk, but instead -the changes to the files are stored on disk. This has several neat advantages, -such as the fact that the data is written in a cyclic log format and naturally -wear levels as a side effect. And, with a bit of error detection, the entire -filesystem can easily be designed to be resilient to power loss. The -journaling component of most modern day filesystems is actually a reduced -form of a logging filesystem. However, logging filesystems have a difficulty -scaling as the size of storage increases. And most filesystems compensate by -caching large parts of the filesystem in RAM, a strategy that is inappropriate -for embedded systems. - -Another interesting filesystem design technique is that of [copy-on-write (COW)](https://en.wikipedia.org/wiki/Copy-on-write). -A good example of this is the [btrfs](https://en.wikipedia.org/wiki/Btrfs) -filesystem. COW filesystems can easily recover from corrupted blocks and have -natural protection against power loss. However, if they are not designed with -wear in mind, a COW filesystem could unintentionally wear down the root block -where the COW data structures are synchronized. +So, what's already out there? There are, of course, many different filesystems, +however they often share and borrow feature from each other. If we look at +power-loss resilience and wear leveling, we can narrow these down to a handful +of designs. -## Metadata pairs +1. First we have the non-resilient, block based filesystems, such as [FAT] and + [ext2]. These are the earliest filesystem designs and often the most simple. + Here storage is divided into blocks, with each file being stored in a + collection of blocks. Without modifications, these filesystems are not + power-loss resilient, so updating a file is a simple as rewriting the blocks + in place. + + ``` + .--------. + | root | + | | + | | + '--------' + .-' '-. + v v + .--------. .--------. + | A | | B | + | | | | + | | | | + '--------' '--------' + .-' .-' '-. + v v v + .--------. .--------. .--------. + | C | | D | | E | + | | | | | | + | | | | | | + '--------' '--------' '--------' + ``` + + Because of their simplicity, these filesystems are usually both the fastest + and smallest. However the lack of power resilience is not great, and the + binding relationship of storage location and data removes the filesystem's + ability to manage wear. + +2. In a completely different direction, we have logging filesystems, such as + [JFFS], [YAFFS], and [SPIFFS], storage location is not bound to a piece of + data, instead the entire storage is used for a circular log which is + appended with every change made to the filesystem. Writing appends new + changes, while reading requires traversing the log to reconstruct a file. + Some logging filesystems cache files to avoid the read cost, but this comes + at a tradeoff of RAM. + + ``` + v + .--------.--------.--------.--------.--------.--------.--------.--------. + | C | new B | new A | | A | B | + | | | |-> | | | + | | | | | | | + '--------'--------'--------'--------'--------'--------'--------'--------' + ``` + + Logging filesystem are beautifully elegant. With a checksum, we can easily + detect power-loss and fall back to the previous state by ignoring failed + appends. And if that wasn't good enough, their cyclic nature means that + logging filesystems distribute wear across storage perfectly. + + The main downside is performance. If we look at garbage collection, the + process of cleaning up outdated data from the end of the log, I've yet to + see a pure logging filesystem that does not have one of these two costs: + + 1. _O(n²)_ runtime + 2. _O(n)_ RAM + + SPIFFS is a very interesting case here, as it uses the fact that repeated + programs to NOR flash is both atomic and masking. This is a very neat + solution, however it limits the type of storage you can support. + +3. Perhaps the most common type of filesystem, a journaling filesystem is the + offspring that happens when you mate a block based filesystem with a logging + filesystem. [ext4] and [NTFS] are good examples. Here, we take a normal + block based filesystem and add a bounded log where we note every change + before it occurs. + + ``` + journal + .--------.--------. + .--------. | C'| D'| | E'| + | root |-->| | |-> | | + | | | | | | | + | | '--------'--------' + '--------' + .-' '-. + v v + .--------. .--------. + | A | | B | + | | | | + | | | | + '--------' '--------' + .-' .-' '-. + v v v + .--------. .--------. .--------. + | C | | D | | E | + | | | | | | + | | | | | | + '--------' '--------' '--------' + ``` + + + This sort of filesystem takes the best from both worlds. Performance can be + as fast as a block based filesystem (though updating the journal does have + a small cost), and atomic updates to the journal allow the filesystem to + recover in the event of a power loss. + + Unfortunately, journaling filesystems have a couple of problems. They are + fairly complex, since there are effectively two filesystems running in + parallel, which comes with a code size cost. They also offer no protection + against wear because of the strong relationship between storage location + and data. + +4. Last but not least we have copy-on-write (COW) filesystems, such as + [btrfs] and [ZFS]. These are very similar to other block based filesystems, + but instead of updating block inplace, all updates are performed by creating + a copy with the changes and replacing any references to the old block with + our new block. This recursively pushes all of our problems upwards until we + reach the root of our filesystem, which is often stored in a very small log. + + ``` + .--------. .--------. + | root | write |new root| + | | ==> | | + | | | | + '--------' '--------' + .-' '-. | '-. + | .-------|------------------' v + v v v .--------. + .--------. .--------. | new B | + | A | | B | | | + | | | | | | + | | | | '--------' + '--------' '--------' .-' | + .-' .-' '-. .------------|------' + | | | | v + v v v v .--------. + .--------. .--------. .--------. | new D | + | C | | D | | E | | | + | | | | | | | | + | | | | | | '--------' + '--------' '--------' '--------' + ``` + + COW filesystems are interesting. They offer very similar performance to + block based filesystems while managing to pull off atomic updates without + storing data changes directly in a log. They even disassociate the storage + location of data, which creates an opportunity for wear leveling. + + Well, almost. The unbounded upwards movement of updates causes some + problems. Because updates to a COW filesystem don't stop until they've + reached the root, an update can cascade into a larger set of writes than + would be needed for the original data. On top of this, the upward motion + focuses these writes into the block, which can wear out much earlier than + the rest of the filesystem. + +## littlefs + +So what does littlefs do? + +If we look at existing filesystems, there are two interesting design patterns +that stand out, but each have their own set of problems. Logging, which +provides independent atomicity, has poor runtime performance. And COW data +structures, which perform well, push the atomicity problem upwards. + +Can we work around these limitations? + +Consider logging. It has either a _O(n²)_ runtime or _O(n)_ RAM cost. We +can't avoid these costs, _but_ if we put an upper bound on the size we can at +least prevent the theoretical cost from becoming problem. This relies on the +super secret computer science hack where you can pretend any algorithmic +complexity is _O(1)_ by bounding the input. + +In the case of COW data structures, we can try twisting the definition a bit. +Let's say that our COW structure doesn't copy after a single write, but instead +copies after _n_ writes. This doesn't change most COW properties (assuming you +can write atomically!), but what it does do is prevent the upward motion of +wear. This sort of copy-on-bounded-writes (CObW) still focuses wear, but at +each level we divide the propagation of wear by _n_. With a sufficiently +large _n_ (> branching factor) wear propagation is no longer a problem. + +See where this is going? Separate, logging and COW are imperfect solutions and +have weaknesses that limit their usefulness. But if we merge the two they can +mutually solve each other's limitations. + +This is the idea behind littlefs. At the sub-block level, littlefs is built +out of small, two blocks logs that provide atomic updates to metadata anywhere +on the filesystem. At the super-block level, littlefs is a CObW tree of blocks +that can be evicted on demand. -The core piece of technology that provides the backbone for the littlefs is -the concept of metadata pairs. The key idea here is that any metadata that -needs to be updated atomically is stored on a pair of blocks tagged with -a revision count and checksum. Every update alternates between these two -pairs, so that at any time there is always a backup containing the previous -state of the metadata. - -Consider a small example where each metadata pair has a revision count, -a number as data, and the XOR of the block as a quick checksum. If -we update the data to a value of 9, and then to a value of 5, here is -what the pair of blocks may look like after each update: ``` - block 1 block 2 block 1 block 2 block 1 block 2 -.---------.---------. .---------.---------. .---------.---------. -| rev: 1 | rev: 0 | | rev: 1 | rev: 2 | | rev: 3 | rev: 2 | -| data: 3 | data: 0 | -> | data: 3 | data: 9 | -> | data: 5 | data: 9 | -| xor: 2 | xor: 0 | | xor: 2 | xor: 11 | | xor: 6 | xor: 11 | -'---------'---------' '---------'---------' '---------'---------' - let data = 9 let data = 5 + root + .--------.--------. + | A'| B'| | + | | |-> | + | | | | + '--------'--------' + .----' '--------------. + A v B v + .--------.--------. .--------.--------. + | C'| D'| | | E'|new| | + | | |-> | | | E'|-> | + | | | | | | | | + '--------'--------' '--------'--------' + .-' '--. | '------------------. + v v .-' v +.--------. .--------. v .--------. +| C | | D | .--------. write | new E | +| | | | | E | ==> | | +| | | | | | | | +'--------' '--------' | | '--------' + '--------' .-' | + .-' '-. .-------------|------' + v v v v + .--------. .--------. .--------. + | F | | G | | new F | + | | | | | | + | | | | | | + '--------' '--------' '--------' ``` -After each update, we can find the most up to date value of data by looking -at the revision count. +There are still some minor issues. Small logs can be expensive in terms of +storage, in the worst case a small log costs 4x the size of the original data. +CObW structures require an efficient block allocator since allocation occurs +every _n_ writes. And there is still the challenge of keeping the RAM usage +constant. -Now consider what the blocks may look like if we suddenly lose power while -changing the value of data to 5: -``` - block 1 block 2 block 1 block 2 block 1 block 2 -.---------.---------. .---------.---------. .---------.---------. -| rev: 1 | rev: 0 | | rev: 1 | rev: 2 | | rev: 3 | rev: 2 | -| data: 3 | data: 0 | -> | data: 3 | data: 9 | -x | data: 3 | data: 9 | -| xor: 2 | xor: 0 | | xor: 2 | xor: 11 | | xor: 2 | xor: 11 | -'---------'---------' '---------'---------' '---------'---------' - let data = 9 let data = 5 - powerloss!!! -``` +## Metadata pairs + +Metadata pairs are the backbone of littlefs. These are small, two block logs +that allow atomic updates anywhere in the filesystem. + +Why two blocks? Well, logs work by appending entries to a circular buffer +stored on disk. But remember that flash has limited write granularity. We can +incrementally program new data onto erased blocks, but we need to erase a full +block at a time. This means that in order for our circular buffer to work, we +need more than one block. + +We could make our logs larger than two blocks, but the next challenge is how +do we store references to these logs? Because the blocks themselves are erased +during writes, using a data structure to track these blocks is complicated. +The simple solution here is to store a two block addresses for every metadata +pair. This has the added advantage that we can change out blocks in the +metadata pair independently, and we don't reduce our block granularity for +other operations. + +In order to determine which metadata block is the most recent, we store a +revision count that we compare using [sequence arithmetic][wikipedia-sna] +(very handy for avoiding problems with integer overflow). Conveniently, this +revision count also gives us a rough idea of how many erases have occurred on +the block. -In this case, block 1 was partially written with a new revision count, but -the littlefs hadn't made it to updating the value of data. However, if we -check our checksum we notice that block 1 was corrupted. So we fall back to -block 2 and use the value 9. - -Using this concept, the littlefs is able to update metadata blocks atomically. -There are a few other tweaks, such as using a 32 bit CRC and using sequence -arithmetic to handle revision count overflow, but the basic concept -is the same. These metadata pairs define the backbone of the littlefs, and the -rest of the filesystem is built on top of these atomic updates. - -## Non-meta data - -Now, the metadata pairs do come with some drawbacks. Most notably, each pair -requires two blocks for each block of data. I'm sure users would be very -unhappy if their storage was suddenly cut in half! Instead of storing -everything in these metadata blocks, the littlefs uses a COW data structure -for files which is in turn pointed to by a metadata block. When -we update a file, we create copies of any blocks that are modified until -the metadata blocks are updated with the new copy. Once the metadata block -points to the new copy, we deallocate the old blocks that are no longer in use. - -Here is what updating a one-block file may look like: ``` - block 1 block 2 block 1 block 2 block 1 block 2 -.---------.---------. .---------.---------. .---------.---------. -| rev: 1 | rev: 0 | | rev: 1 | rev: 0 | | rev: 1 | rev: 2 | -| file: 4 | file: 0 | -> | file: 4 | file: 0 | -> | file: 4 | file: 5 | -| xor: 5 | xor: 0 | | xor: 5 | xor: 0 | | xor: 5 | xor: 7 | -'---------'---------' '---------'---------' '---------'---------' - | | | - v v v - block 4 block 4 block 5 block 4 block 5 -.--------. .--------. .--------. .--------. .--------. -| old | | old | | new | | old | | new | -| data | | data | | data | | data | | data | -| | | | | | | | | | -'--------' '--------' '--------' '--------' '--------' - update data in file update metadata pair +metadata pair pointer: {block 0, block 1} + | '--------------------. + '-. | +disk v v +.--------.--------.--------.--------.--------.--------.--------.--------. +| | |metadata| |metadata| | +| | |block 0 | |block 1 | | +| | | | | | | +'--------'--------'--------'--------'--------'--------'--------'--------' + '--. .----' + v v + metadata pair .----------------.----------------. + | revision 11 | revision 12 | + block 1 is |----------------|----------------| + most recent | A | A'' | + |----------------|----------------| + | checksum | checksum | + |----------------|----------------| + | B | A''' | <- most recent A + |----------------|----------------| + | A'' | checksum | + |----------------|----------------| + | checksum | | | + |----------------| v | + '----------------'----------------' ``` -It doesn't matter if we lose power while writing new data to block 5, -since the old data remains unmodified in block 4. This example also -highlights how the atomic updates of the metadata blocks provide a -synchronization barrier for the rest of the littlefs. - -At this point, it may look like we are wasting an awfully large amount -of space on the metadata. Just looking at that example, we are using -three blocks to represent a file that fits comfortably in one! So instead -of giving each file a metadata pair, we actually store the metadata for -all files contained in a single directory in a single metadata block. - -Now we could just leave files here, copying the entire file on write -provides the synchronization without the duplicated memory requirements -of the metadata blocks. However, we can do a bit better. +So how do we atomically update our metadata pairs? Atomicity (a type of +power-loss resilience) requires two parts: redundancy and error detection. +Error detection can be provided with a checksum, and in littlefs's case we +use a 32-bit [CRC][wikipedia-crc]. Maintaining redundancy, on the other hand, +requires multiple stages. + +1. If our block is not full and the program size is small enough to let us + append more entries, we can simply append the entries to the log. Because + we don't overwrite the original entries (remember rewriting flash requires + an erase), we still have the original entries if we lose power during the + append. + + ``` + commit A + .----------------.----------------. .----------------.----------------. + | revision 1 | revision 0 | => | revision 1 | revision 0 | + |----------------|----------------| |----------------|----------------| + | | | | | A | | + | v | | |----------------| | + | | | | checksum | | + | | | |----------------| | + | | | | | | | + | | | | v | | + | | | | | | + | | | | | | + | | | | | | + | | | | | | + '----------------'----------------' '----------------'----------------' + ``` + + Note that littlefs doesn't maintain a checksum for each entry. Many logging + filesystems do this, but it limits what you can update in a single atomic + operation. What we can do instead is group multiple entries into a commit + that shares a single checksum. This lets us update multiple unrelated pieces + of metadata as long as they reside on the same metadata pair. + + ``` + commit B and A' + .----------------.----------------. .----------------.----------------. + | revision 1 | revision 0 | => | revision 1 | revision 0 | + |----------------|----------------| |----------------|----------------| + | A | | | A | | + |----------------| | |----------------| | + | checksum | | | checksum | | + |----------------| | |----------------| | + | | | | | B | | + | v | | |----------------| | + | | | | A' | | + | | | |----------------| | + | | | | checksum | | + | | | |----------------| | + '----------------'----------------' '----------------'----------------' + ``` + +2. If our block _is_ full of entries, we need to somehow remove outdated + entries to make space for new ones. This process is called garbage + collection, but because littlefs has multiple garbage collectors, we + also call this specific case compaction. + + Compared to other filesystems, littlefs's garbage collector is relatively + simple. We want to avoid RAM consumption, so we use a sort of brute force + solution where for each entry we check to see if a newer entry has been + written. If the entry is the most recent we append it to our new block. This + is where having two blocks becomes important, if we lose power we still have + everything in our original block. + + During this compaction step we also erase the metadata block and increment + the revision count. Because we can commit multiple entries at once, we can + write all of these changes to the second block without worrying about power + loss. It's only when the commit's checksum is written that the compacted + entries and revision count become committed and readable. + + ``` + commit B', need to compact + .----------------.----------------. .----------------.----------------. + | revision 1 | revision 0 | => | revision 1 | revision 2 | + |----------------|----------------| |----------------|----------------| + | A | | | A | A' | + |----------------| | |----------------|----------------| + | checksum | | | checksum | B' | + |----------------| | |----------------|----------------| + | B | | | B | checksum | + |----------------| | |----------------|----------------| + | A' | | | A' | | | + |----------------| | |----------------| v | + | checksum | | | checksum | | + |----------------| | |----------------| | + '----------------'----------------' '----------------'----------------' + ``` + +3. If our block is full of entries _and_ we can't find any garbage, then what? + At this point, most logging filesystems would return an error indicating no + more space is available, but because we have small logs, overflowing a log + isn't really an error condition. + + Instead, we split our original metadata pair into two metadata pairs, each + containing half of the entries, connected by a tail pointer. Instead of + increasing the size of the log and dealing with the scalability issues + associated with larger logs, we form a linked list of small bounded logs. + This is a tradeoff as this approach does use more storage space, but at the + benefit of improved scalability. + + Despite writing to two metadata pairs, we can still maintain power + resilience during this split step by first preparing the new metadata pair, + and then inserting the tail pointer during the commit to the original + metadata pair. + + ``` + commit C and D, need to split + .----------------.----------------. .----------------.----------------. + | revision 1 | revision 2 | => | revision 3 | revision 2 | + |----------------|----------------| |----------------|----------------| + | A | A' | | A' | A' | + |----------------|----------------| |----------------|----------------| + | checksum | B' | | B' | B' | + |----------------|----------------| |----------------|----------------| + | B | checksum | | tail ---------------------. + |----------------|----------------| |----------------|----------------| | + | A' | | | | checksum | | | + |----------------| v | |----------------| | | + | checksum | | | | | | | + |----------------| | | v | | | + '----------------'----------------' '----------------'----------------' | + .----------------.---------' + v v + .----------------.----------------. + | revision 1 | revision 0 | + |----------------|----------------| + | C | | + |----------------| | + | D | | + |----------------| | + | checksum | | + |----------------| | + | | | | + | v | | + | | | + | | | + '----------------'----------------' + ``` + +There is another complexity the crops up when dealing with small logs. The +amortized runtime cost of garbage collection is not only dependent on its +one time cost (_O(n²)_ for littlefs), but also depends on how often +garbage collection occurs. + +Consider two extremes: + +1. Log is empty, garbage collection occurs once every _n_ updates +2. Log is full, garbage collection occurs **every** update + +Clearly we need to be more aggressive than waiting for our metadata pair to +be full. As the metadata pair approaches fullness the frequency of compactions +grows very rapidly. + +Looking at the problem generically, consider a log with ![n] bytes for each +entry, ![d] dynamic entries (entries that are outdated during garbage +collection), and ![s] static entries (entries that need to be copied during +garbage collection). If we look at the amortized runtime complexity of updating +this log we get this formula: + +![cost = n + n (s / d+1)][metadata-formula1] + +If we let ![r] be the ratio of static space to the size of our log in bytes, we +find an alternative representation of the number of static and dynamic entries: + +![s = r (size/n)][metadata-formula2] + +![d = (1 - r) (size/n)][metadata-formula3] + +Substituting these in for ![d] and ![s] gives us a nice formula for the cost of +updating an entry given how full the log is: + +![cost = n + n (r (size/n) / ((1-r) (size/n) + 1))][metadata-formula4] + +Assuming 100 byte entries in a 4 KiB log, we can graph this using the entry +size to find a multiplicative cost: + +![Metadata pair update cost graph][metadata-cost-graph] + +So at 50% usage, we're seeing an average of 2x cost per update, and at 75% +usage, we're already at an average of 4x cost per update. + +To avoid this exponential growth, instead of waiting for our metadata pair +to be full, we split the metadata pair once we exceed 50% capacity. We do this +lazily, waiting until we need to compact before checking if we fit in our 50% +limit. This limits the overhead of garbage collection to 2x the runtime cost, +giving us an amortized runtime complexity of _O(1)_. + +--- + +If we look at metadata pairs and linked-lists of metadata pairs at a high +level, they have fairly nice runtime costs. Assuming _n_ metadata pairs, +each containing _m_ metadata entries, the _lookup_ cost for a specific +entry has a worst case runtime complexity of _O(nm)_. For _updating_ a specific +entry, the worst case complexity is _O(nm²)_, with an amortized complexity +of only _O(nm)_. + +However, splitting at 50% capacity does mean that in the best case our +metadata pairs will only be 1/2 full. If we include the overhead of the second +block in our metadata pair, each metadata entry has an effective storage cost +of 4x the original size. I imagine users would not be happy if they found +that they can only use a quarter of their original storage. Metadata pairs +provide a mechanism for performing atomic updates, but we need a separate +mechanism for storing the bulk of our data. ## CTZ skip-lists -There are many different data structures for representing the actual -files in filesystems. Of these, the littlefs uses a rather unique [COW](https://upload.wikimedia.org/wikipedia/commons/0/0c/Cow_female_black_white.jpg) -data structure that allows the filesystem to reuse unmodified parts of the -file without additional metadata pairs. - -First lets consider storing files in a simple linked-list. What happens when we -append a block? We have to change the last block in the linked-list to point -to this new block, which means we have to copy out the last block, and change -the second-to-last block, and then the third-to-last, and so on until we've -copied out the entire file. +Metadata pairs provide efficient atomic updates but unfortunately have a large +storage cost. But we can work around this storage cost by only using the +metadata pairs to store references to more dense, copy-on-write (COW) data +structures. + +[Copy-on-write data structures][wikipedia-cow], also called purely functional +data structures, are a category of data structures where the underlying +elements are immutable. Making changes to the data requires creating new +elements containing a copy of the updated data and replacing any references +with references to the new elements. Generally, the performance of a COW data +structure depends on how many old elements can be reused after replacing parts +of the data. + +littlefs has several requirements of its COW structures. They need to be +efficient to read and write, but most frustrating, they need to be traversable +with a constant amount of RAM. Notably this rules out +[B-trees][wikipedia-B-tree], which can not be traversed with constant RAM, and +[B+-trees][wikipedia-B+-tree], which are not possible to update with COW +operations. + +--- + +So, what can we do? First let's consider storing files in a simple COW +linked-list. Appending a block, which is the basis for writing files, means we +have to update the last block to point to our new block. This requires a COW +operation, which means we need to update the second-to-last block, and then the +third-to-last, and so on until we've copied out the entire file. ``` -Exhibit A: A linked-list +A linked-list .--------. .--------. .--------. .--------. .--------. .--------. | data 0 |->| data 1 |->| data 2 |->| data 4 |->| data 5 |->| data 6 | | | | | | | | | | | | | @@ -223,17 +593,15 @@ Exhibit A: A linked-list '--------' '--------' '--------' '--------' '--------' '--------' ``` -To get around this, the littlefs, at its heart, stores files backwards. Each -block points to its predecessor, with the first block containing no pointers. -If you think about for a while, it starts to make a bit of sense. Appending -blocks just point to their predecessor and no other blocks need to be updated. -If we update a block in the middle, we will need to copy out the blocks that -follow, but can reuse the blocks before the modified block. Since most file -operations either reset the file each write or append to files, this design -avoids copying the file in the most common cases. +To avoid a full copy during appends, we can store the data backwards. Appending +blocks just requires adding the new block and no other blocks need to be +updated. If we update a block in the middle, we still need to copy the +following blocks, but can reuse any blocks before it. Since most file writes +are linear, this design gambles that appends are the most common type of data +update. ``` -Exhibit B: A backwards linked-list +A backwards linked-list .--------. .--------. .--------. .--------. .--------. .--------. | data 0 |<-| data 1 |<-| data 2 |<-| data 4 |<-| data 5 |<-| data 6 | | | | | | | | | | | | | @@ -241,25 +609,28 @@ Exhibit B: A backwards linked-list '--------' '--------' '--------' '--------' '--------' '--------' ``` -However, a backwards linked-list does come with a rather glaring problem. -Iterating over a file _in order_ has a runtime cost of O(n^2). Gah! A quadratic -runtime to just _read_ a file? That's awful. Keep in mind reading files is -usually the most common filesystem operation. +However, a backwards linked-list does have a rather glaring problem. Iterating +over a file _in order_ has a runtime cost of _O(n²)_. A quadratic runtime +just to read a file! That's awful. + +Fortunately we can do better. Instead of a singly linked list, littlefs +uses a multilayered linked-list often called a +[skip-list][wikipedia-skip-list]. However, unlike the most common type of +skip-list, littlefs's skip-lists are strictly deterministic built around some +interesting properties of the count-trailing-zeros (CTZ) instruction. -To avoid this problem, the littlefs uses a multilayered linked-list. For -every nth block where n is divisible by 2^x, the block contains a pointer -to block n-2^x. So each block contains anywhere from 1 to log2(n) pointers -that skip to various sections of the preceding list. If you're familiar with -data-structures, you may have recognized that this is a type of deterministic +The rules CTZ skip-lists follow are that for every _n_‍th block where _n_ +is divisible by 2‍_ˣ_, that block contains a pointer to block +_n_-2‍_ˣ_. This means that each block contains anywhere from 1 to +log₂_n_ pointers that skip to different preceding elements of the skip-list. -The name comes from the use of the -[count trailing zeros (CTZ)](https://en.wikipedia.org/wiki/Count_trailing_zeros) -instruction, which allows us to calculate the power-of-two factors efficiently. -For a given block n, the block contains ctz(n)+1 pointers. +The name comes from heavy use of the [CTZ instruction][wikipedia-ctz], which +lets us calculate the power-of-two factors efficiently. For a give block _n_, +that block contains ctz(_n_)+1 pointers. ``` -Exhibit C: A backwards CTZ skip-list +A backwards CTZ skip-list .--------. .--------. .--------. .--------. .--------. .--------. | data 0 |<-| data 1 |<-| data 2 |<-| data 3 |<-| data 4 |<-| data 5 | | |<-| |--| |<-| |--| | | | @@ -267,11 +638,11 @@ Exhibit C: A backwards CTZ skip-list '--------' '--------' '--------' '--------' '--------' '--------' ``` -The additional pointers allow us to navigate the data-structure on disk -much more efficiently than in a singly linked-list. +The additional pointers let us navigate the data-structure on disk much more +efficiently than in a singly linked list. -Taking exhibit C for example, here is the path from data block 5 to data -block 1. You can see how data block 3 was completely skipped: +Consider a path from data block 5 to data block 1. You can see how data block 3 +was completely skipped: ``` .--------. .--------. .--------. .--------. .--------. .--------. | data 0 | | data 1 |<-| data 2 | | data 3 | | data 4 |<-| data 5 | @@ -280,7 +651,7 @@ block 1. You can see how data block 3 was completely skipped: '--------' '--------' '--------' '--------' '--------' '--------' ``` -The path to data block 0 is even more quick, requiring only two jumps: +The path to data block 0 is even faster, requiring only two jumps: ``` .--------. .--------. .--------. .--------. .--------. .--------. | data 0 | | data 1 | | data 2 | | data 3 | | data 4 |<-| data 5 | @@ -291,193 +662,171 @@ The path to data block 0 is even more quick, requiring only two jumps: We can find the runtime complexity by looking at the path to any block from the block containing the most pointers. Every step along the path divides -the search space for the block in half. This gives us a runtime of O(log n). -To get to the block with the most pointers, we can perform the same steps -backwards, which puts the runtime at O(2 log n) = O(log n). The interesting -part about this data structure is that this optimal path occurs naturally -if we greedily choose the pointer that covers the most distance without passing -our target block. - -So now we have a representation of files that can be appended trivially with -a runtime of O(1), and can be read with a worst case runtime of O(n log n). -Given that the the runtime is also divided by the amount of data we can store -in a block, this is pretty reasonable. - -Unfortunately, the CTZ skip-list comes with a few questions that aren't -straightforward to answer. What is the overhead? How do we handle more -pointers than we can store in a block? How do we store the skip-list in -a directory entry? - -One way to find the overhead per block is to look at the data structure as -multiple layers of linked-lists. Each linked-list skips twice as many blocks -as the previous linked-list. Another way of looking at it is that each -linked-list uses half as much storage per block as the previous linked-list. -As we approach infinity, the number of pointers per block forms a geometric -series. Solving this geometric series gives us an average of only 2 pointers -per block. - -![overhead_per_block](https://latex.codecogs.com/svg.latex?%5Clim_%7Bn%5Cto%5Cinfty%7D%5Cfrac%7B1%7D%7Bn%7D%5Csum_%7Bi%3D0%7D%5E%7Bn%7D%5Cleft%28%5Ctext%7Bctz%7D%28i%29+1%5Cright%29%20%3D%20%5Csum_%7Bi%3D0%7D%5Cfrac%7B1%7D%7B2%5Ei%7D%20%3D%202) - -Finding the maximum number of pointers in a block is a bit more complicated, -but since our file size is limited by the integer width we use to store the -size, we can solve for it. Setting the overhead of the maximum pointers equal -to the block size we get the following equation. Note that a smaller block size -results in more pointers, and a larger word width results in larger pointers. - -![maximum overhead](https://latex.codecogs.com/svg.latex?B%20%3D%20%5Cfrac%7Bw%7D%7B8%7D%5Cleft%5Clceil%5Clog_2%5Cleft%28%5Cfrac%7B2%5Ew%7D%7BB-2%5Cfrac%7Bw%7D%7B8%7D%7D%5Cright%29%5Cright%5Crceil) - -where: -B = block size in bytes -w = word width in bits - -Solving the equation for B gives us the minimum block size for various word -widths: -32 bit CTZ skip-list = minimum block size of 104 bytes -64 bit CTZ skip-list = minimum block size of 448 bytes - -Since littlefs uses a 32 bit word size, we are limited to a minimum block -size of 104 bytes. This is a perfectly reasonable minimum block size, with most -block sizes starting around 512 bytes. So we can avoid additional logic to -avoid overflowing our block's capacity in the CTZ skip-list. - -So, how do we store the skip-list in a directory entry? A naive approach would -be to store a pointer to the head of the skip-list, the length of the file -in bytes, the index of the head block in the skip-list, and the offset in the -head block in bytes. However this is a lot of information, and we can observe -that a file size maps to only one block index + offset pair. So it should be -sufficient to store only the pointer and file size. - -But there is one problem, calculating the block index + offset pair from a -file size doesn't have an obvious implementation. - -We can start by just writing down an equation. The first idea that comes to -mind is to just use a for loop to sum together blocks until we reach our -file size. We can write this equation as a summation: - -![summation1](https://latex.codecogs.com/svg.latex?N%20%3D%20%5Csum_i%5En%5Cleft%5BB-%5Cfrac%7Bw%7D%7B8%7D%5Cleft%28%5Ctext%7Bctz%7D%28i%29+1%5Cright%29%5Cright%5D) - -where: -B = block size in bytes -w = word width in bits -n = block index in skip-list -N = file size in bytes - -And this works quite well, but is not trivial to calculate. This equation -requires O(n) to compute, which brings the entire runtime of reading a file -to O(n^2 log n). Fortunately, the additional O(n) does not need to touch disk, -so it is not completely unreasonable. But if we could solve this equation into -a form that is easily computable, we can avoid a big slowdown. - -Unfortunately, the summation of the CTZ instruction presents a big challenge. -How would you even begin to reason about integrating a bitwise instruction? -Fortunately, there is a powerful tool I've found useful in these situations: -The [On-Line Encyclopedia of Integer Sequences (OEIS)](https://oeis.org/). -If we work out the first couple of values in our summation, we find that CTZ -maps to [A001511](https://oeis.org/A001511), and its partial summation maps -to [A005187](https://oeis.org/A005187), and surprisingly, both of these -sequences have relatively trivial equations! This leads us to a rather -unintuitive property: - -![mindblown](https://latex.codecogs.com/svg.latex?%5Csum_i%5En%5Cleft%28%5Ctext%7Bctz%7D%28i%29+1%5Cright%29%20%3D%202n-%5Ctext%7Bpopcount%7D%28n%29) - -where: -ctz(x) = the number of trailing bits that are 0 in x -popcount(x) = the number of bits that are 1 in x - -It's a bit bewildering that these two seemingly unrelated bitwise instructions -are related by this property. But if we start to dissect this equation we can -see that it does hold. As n approaches infinity, we do end up with an average -overhead of 2 pointers as we find earlier. And popcount seems to handle the -error from this average as it accumulates in the CTZ skip-list. - -Now we can substitute into the original equation to get a trivial equation -for a file size: - -![summation2](https://latex.codecogs.com/svg.latex?N%20%3D%20Bn%20-%20%5Cfrac%7Bw%7D%7B8%7D%5Cleft%282n-%5Ctext%7Bpopcount%7D%28n%29%5Cright%29) - -Unfortunately, we're not quite done. The popcount function is non-injective, -so we can only find the file size from the block index, not the other way -around. However, we can solve for an n' block index that is greater than n -with an error bounded by the range of the popcount function. We can then -repeatedly substitute this n' into the original equation until the error -is smaller than the integer division. As it turns out, we only need to -perform this substitution once. Now we directly calculate our block index: - -![formulaforn](https://latex.codecogs.com/svg.latex?n%20%3D%20%5Cleft%5Clfloor%5Cfrac%7BN-%5Cfrac%7Bw%7D%7B8%7D%5Cleft%28%5Ctext%7Bpopcount%7D%5Cleft%28%5Cfrac%7BN%7D%7BB-2%5Cfrac%7Bw%7D%7B8%7D%7D-1%5Cright%29+2%5Cright%29%7D%7BB-2%5Cfrac%7Bw%7D%7B8%7D%7D%5Cright%5Crfloor) - -Now that we have our block index n, we can just plug it back into the above -equation to find the offset. However, we do need to rearrange the equation -a bit to avoid integer overflow: - -![formulaforoff](https://latex.codecogs.com/svg.latex?%5Cmathit%7Boff%7D%20%3D%20N%20-%20%5Cleft%28B-2%5Cfrac%7Bw%7D%7B8%7D%5Cright%29n%20-%20%5Cfrac%7Bw%7D%7B8%7D%5Ctext%7Bpopcount%7D%28n%29) - -The solution involves quite a bit of math, but computers are very good at math. -Now we can solve for both the block index and offset from the file size in O(1). - -Here is what it might look like to update a file stored with a CTZ skip-list: +the search space for the block in half, giving us a runtime of _O(log n)_. +To get _to_ the block with the most pointers, we can perform the same steps +backwards, which puts the runtime at _O(2 log n)_ = _O(log n)_. An interesting +note is that this optimal path occurs naturally if we greedily choose the +pointer that covers the most distance without passing our target. + +So now we have a [COW] data structure that is cheap to append with a runtime +of _O(1)_, and can be read with a worst case runtime of _O(n log n)_. Given +that this runtime is also divided by the amount of data we can store in a +block, this cost is fairly reasonable. + +--- + +This is a new data structure, so we still have several questions. What is the +storage overage? Can the number of pointers exceed the size of a block? How do +we store a CTZ skip-list in our metadata pairs? + +To find the storage overhead, we can look at the data structure as multiple +linked-lists. Each linked-list skips twice as many blocks as the previous, +or from another perspective, each linked-list uses half as much storage as +the previous. As we approach infinity, the storage overhead forms a geometric +series. Solving this tells us that on average our storage overhead is only +2 pointers per block. + +![lim,n->inf((1/n)sum,i,0->n(ctz(i)+1)) = sum,i,0->inf(1/2^i) = 2][ctz-formula1] + +Because our file size is limited the word width we use to store sizes, we can +also solve for the maximum number of pointers we would ever need to store in a +block. If we set the overhead of pointers equal to the block size, we get the +following equation. Note that both a smaller block size (![B][bigB]) and larger +word width (![w]) result in more storage overhead. + +![B = (w/8)ceil(log2(2^w / (B-2w/8)))][ctz-formula2] + +Solving the equation for ![B][bigB] gives us the minimum block size for some +common word widths: + +1. 32-bit CTZ skip-list => minimum block size of 104 bytes +2. 64-bit CTZ skip-list => minimum block size of 448 bytes + +littlefs uses a 32-bit word width, so our blocks can only overflow with +pointers if they are smaller than 104 bytes. This is an easy requirement, as +in practice, most block sizes start at 512 bytes. As long as our block size +is larger than 104 bytes, we can avoid the extra logic needed to handle +pointer overflow. + +This last question is how do we store CTZ skip-lists? We need a pointer to the +head block, the size of the skip-list, the index of the head block, and our +offset in the head block. But it's worth noting that each size maps to a unique +index + offset pair. So in theory we can store only a single pointer and size. + +However, calculating the index + offset pair from the size is a bit +complicated. We can start with a summation that loops through all of the blocks +up until our given size. Let ![B][bigB] be the block size in bytes, ![w] be the +word width in bits, ![n] be the index of the block in the skip-list, and +![N][bigN] be the file size in bytes: + +![N = sum,i,0->n(B-(w/8)(ctz(i)+1))][ctz-formula3] + +This works quite well, but requires _O(n)_ to compute, which brings the full +runtime of reading a file up to _O(n² log n)_. Fortunately, that summation +doesn't need to touch the disk, so the practical impact is minimal. + +However, despite the integration of a bitwise operation, we can actually reduce +this equation to a _O(1)_ form. While browsing the amazing resource that is +the [On-Line Encyclopedia of Integer Sequences (OEIS)][oeis], I managed to find +[A001511], which matches the iteration of the CTZ instruction, +and [A005187], which matches its partial summation. Much to my +surprise, these both result from simple equations, leading us to a rather +unintuitive property that ties together two seemingly unrelated bitwise +instructions: + +![sum,i,0->n(ctz(i)+1) = 2n-popcount(n)][ctz-formula4] + +where: + +1. ctz(![x]) = the number of trailing bits that are 0 in ![x] +2. popcount(![x]) = the number of bits that are 1 in ![x] + +Initial tests of this surprising property seem to hold. As ![n] approaches +infinity, we end up with an average overhead of 2 pointers, which matches what +our assumption from earlier. During iteration, the popcount function seems to +handle deviations from this average. Of course, just to make sure I wrote a +quick script that verified this property for all 32-bit integers. + +Now we can substitute into our original equation to find a more efficient +equation for file size: + +![N = Bn - (w/8)(2n-popcount(n))][ctz-formula5] + +Unfortunately, the popcount function is non-injective, so we can't solve this +equation for our index. But what we can do is solve for an ![n'] index that +is greater than ![n] with error bounded by the range of the popcount function. +We can repeatedly substitute ![n'] into the original equation until the error +is smaller than our integer resolution. As it turns out, we only need to +perform this substitution once, which gives us this formula for our index: + +![n = floor((N-(w/8)popcount(N/(B-2w/8))) / (B-2w/8))][ctz-formula6] + +Now that we have our index ![n], we can just plug it back into the above +equation to find the offset. We run into a bit of a problem with integer +overflow, but we can avoid this by rearranging the equation a bit: + +![off = N - (B-2w/8)n - (w/8)popcount(n)][ctz-formula7] + +Our solution requires quite a bit of math, but computer are very good at math. +Now we can find both our block index and offset from a size in _O(1)_, letting +us store CTZ skip-lists with only a pointer and size. + +CTZ skip-lists give us a COW data structure that is easily traversable in +_O(n)_, can be appended in _O(1)_, and can be read in _O(n log n)_. All of +these operations work in a bounded amount of RAM and require only two words of +storage overhead per block. In combination with metadata pairs, CTZ skip-lists +provide power resilience and compact storage of data. + ``` - block 1 block 2 - .---------.---------. - | rev: 1 | rev: 0 | - | file: 6 | file: 0 | - | size: 4 | size: 0 | - | xor: 3 | xor: 0 | - '---------'---------' - | + .--------. + .|metadata| + || | + || | + |'--------' + '----|---' v - block 3 block 4 block 5 block 6 .--------. .--------. .--------. .--------. | data 0 |<-| data 1 |<-| data 2 |<-| data 3 | | |<-| |--| | | | | | | | | | | | '--------' '--------' '--------' '--------' -| update data in file -v - - block 1 block 2 - .---------.---------. - | rev: 1 | rev: 0 | - | file: 6 | file: 0 | - | size: 4 | size: 0 | - | xor: 3 | xor: 0 | - '---------'---------' - | +write data to disk, create copies +=> + .--------. + .|metadata| + || | + || | + |'--------' + '----|---' v - block 3 block 4 block 5 block 6 .--------. .--------. .--------. .--------. -| data 0 |<-| data 1 |<-| old |<-| old | -| |<-| |--| data 2 | | data 3 | +| data 0 |<-| data 1 |<-| data 2 |<-| data 3 | +| |<-| |--| | | | | | | | | | | | '--------' '--------' '--------' '--------' ^ ^ ^ - | | | block 7 block 8 block 9 block 10 | | | .--------. .--------. .--------. .--------. | | '----| new |<-| new |<-| new |<-| new | | '----------------| data 2 |<-| data 3 |--| data 4 | | data 5 | '------------------| |--| |--| | | | '--------' '--------' '--------' '--------' -| update metadata pair -v - - block 1 block 2 - .---------.---------. - | rev: 1 | rev: 2 | - | file: 6 | file: 10| - | size: 4 | size: 6 | - | xor: 3 | xor: 14 | - '---------'---------' - | +commit to metadata pair +=> + .--------. + .|new | + ||metadata| + || | + |'--------' + '----|---' | - block 3 block 4 block 5 block 6 | .--------. .--------. .--------. .--------. | -| data 0 |<-| data 1 |<-| old |<-| old | | -| |<-| |--| data 2 | | data 3 | | +| data 0 |<-| data 1 |<-| data 2 |<-| data 3 | | +| |<-| |--| | | | | | | | | | | | | | '--------' '--------' '--------' '--------' | ^ ^ ^ v - | | | block 7 block 8 block 9 block 10 | | | .--------. .--------. .--------. .--------. | | '----| new |<-| new |<-| new |<-| new | | '----------------| data 2 |<-| data 3 |--| data 4 | | data 5 | @@ -485,68 +834,98 @@ v '--------' '--------' '--------' '--------' ``` -## Block allocation - -So those two ideas provide the grounds for the filesystem. The metadata pairs -give us directories, and the CTZ skip-lists give us files. But this leaves -one big [elephant](https://upload.wikimedia.org/wikipedia/commons/3/37/African_Bush_Elephant.jpg) -of a question. How do we get those blocks in the first place? - -One common strategy is to store unallocated blocks in a big free list, and -initially the littlefs was designed with this in mind. By storing a reference -to the free list in every single metadata pair, additions to the free list -could be updated atomically at the same time the replacement blocks were -stored in the metadata pair. During boot, every metadata pair had to be -scanned to find the most recent free list, but once the list was found the -state of all free blocks becomes known. - -However, this approach had several issues: - -- There was a lot of nuanced logic for adding blocks to the free list without - modifying the blocks, since the blocks remain active until the metadata is - updated. -- The free list had to support both additions and removals in FIFO order while - minimizing block erases. -- The free list had to handle the case where the file system completely ran - out of blocks and may no longer be able to add blocks to the free list. -- If we used a revision count to track the most recently updated free list, - metadata blocks that were left unmodified were ticking time bombs that would - cause the system to go haywire if the revision count overflowed. -- Every single metadata block wasted space to store these free list references. - -Actually, to simplify, this approach had one massive glaring issue: complexity. - -> Complexity leads to fallibility. -> Fallibility leads to unmaintainability. -> Unmaintainability leads to suffering. - -Or at least, complexity leads to increased code size, which is a problem -for embedded systems. - -In the end, the littlefs adopted more of a "drop it on the floor" strategy. -That is, the littlefs doesn't actually store information about which blocks -are free on the storage. The littlefs already stores which files _are_ in -use, so to find a free block, the littlefs just takes all of the blocks that -exist and subtract the blocks that are in use. - -Of course, it's not quite that simple. Most filesystems that adopt this "drop -it on the floor" strategy either rely on some properties inherent to the -filesystem, such as the cyclic-buffer structure of logging filesystems, -or use a bitmap or table stored in RAM to track free blocks, which scales -with the size of storage and is problematic when you have limited RAM. You -could iterate through every single block in storage and check it against -every single block in the filesystem on every single allocation, but that -would have an abhorrent runtime. - -So the littlefs compromises. It doesn't store a bitmap the size of the storage, -but it does store a little bit-vector that contains a fixed set lookahead -for block allocations. During a block allocation, the lookahead vector is -checked for any free blocks. If there are none, the lookahead region jumps -forward and the entire filesystem is scanned for free blocks. +## The block allocator + +So we now have the framework for an atomic, wear leveling filesystem. Small two +block metadata pairs provide atomic updates, while CTZ skip-lists provide +compact storage of data in COW blocks. + +But now we need to look at the [elephant] in the room. Where do all these +blocks come from? + +Deciding which block to use next is the responsibility of the block allocator. +In filesystem design, block allocation is often a second-class citizen, but in +a COW filesystem its role becomes much more important as it is needed for +nearly every write to the filesystem. + +Normally, block allocation involves some sort of free list or bitmap stored on +the filesystem that is updated with free blocks. However, with power +resilience, keeping these structure consistent becomes difficult. It doesn't +help that any mistake in updating these structures can result in lost blocks +that are impossible to recover. + +littlefs takes a cautious approach. Instead of trusting a free list on disk, +littlefs relies on the fact that the filesystem on disk is a mirror image of +the free blocks on the disk. The block allocator operates much like a garbage +collector in a scripting language, scanning for unused blocks on demand. + +``` + .----. + |root| + | | + '----' + v-------' '-------v +.----. . . .----. +| A | . . | B | +| | . . | | +'----' . . '----' +. . . . v--' '------------v---------v +. . . .----. . .----. .----. +. . . | C | . | D | | E | +. . . | | . | | | | +. . . '----' . '----' '----' +. . . . . . . . . . +.----.----.----.----.----.----.----.----.----.----.----.----. +| A | |root| C | B | | D | | E | | +| | | | | | | | | | | +'----'----'----'----'----'----'----'----'----'----'----'----' + ^ ^ ^ ^ ^ + '-------------------'----'-------------------'----'-- free blocks +``` + +While this approach may sound complicated, the decision to not maintain a free +list greatly simplifies the overall design of littlefs. Unlike programming +languages, there are only a handful of data structures we need to traverse. +And block deallocation, which occurs nearly as often as block allocation, +is simply a noop. This "drop it on the floor" strategy greatly reduces the +complexity of managing on disk data structures, especially when handling +high-risk error conditions. + +--- + +Our block allocator needs to find free blocks efficiently. You could traverse +through every block on storage and check each one against our filesystem tree, +however the runtime would be abhorrent. We need to somehow collect multiple +blocks each traversal. + +Looking at existing designs, some larger filesystems that use a similar "drop +it on the floor" strategy store a bitmap of the entire storage in [RAM]. This +works well because bitmaps are surprisingly compact. We can't use the same +strategy here, as it violates our constant RAM requirement, but we may be able +to modify the idea into a workable solution. + +``` +.----.----.----.----.----.----.----.----.----.----.----.----. +| A | |root| C | B | | D | | E | | +| | | | | | | | | | | +'----'----'----'----'----'----'----'----'----'----'----'----' + 1 0 1 1 1 0 0 1 0 1 0 0 + \---------------------------+----------------------------/ + v + bitmap: 0xb94 (0b101110010100) +``` + +The block allocator in littlefs is a compromise between a disk-sized bitmap and +a brute force traversal. Instead of a bitmap the size of storage, we keep track +of a small, fixed-size bitmap called the lookahead buffer. During block +allocation, we take blocks from the lookahead buffer. If the lookahead buffer +is empty, we scan the filesystem for more free blocks, populating our lookahead +buffer. Each scan we use an increasing offset, circling the storage as blocks +are allocated. Here's what it might look like to allocate 4 blocks on a decently busy -filesystem with a 32bit lookahead and a total of -128 blocks (512Kbytes of storage if blocks are 4Kbyte): +filesystem with a 32 bit lookahead and a total of 128 blocks (512 KiB +of storage if blocks are 4 KiB): ``` boot... lookahead: fs blocks: fffff9fffffffffeffffffffffff0000 @@ -570,40 +949,557 @@ alloc = 112 lookahead: ffff8000 fs blocks: ffffffffffffffffffffffffffff8000 ``` -While this lookahead approach still has an asymptotic runtime of O(n^2) to -scan all of storage, the lookahead reduces the practical runtime to a -reasonable amount. Bit-vectors are surprisingly compact, given only 16 bytes, -the lookahead could track 128 blocks. For a 4Mbyte flash chip with 4Kbyte -blocks, the littlefs would only need 8 passes to scan the entire storage. +This lookahead approach has a runtime complexity of _O(n²)_ to completely +scan storage, however, bitmaps are surprisingly compact, and in practice only +one or two passes are usually needed to find free blocks. Additionally, the +performance of the allocator can be optimized by adjusting the block size or +size of the lookahead buffer, trading either write granularity or RAM for +allocator performance. + +## Wear leveling + +The block allocator has a secondary role: wear leveling. + +Wear leveling is the process of distributing wear across all blocks in the +storage to prevent the filesystem from experiencing an early death due to +wear on a single block in the storage. + +littlefs has two methods of protecting against wear: +1. Detection and recovery from bad blocks +2. Evenly distributing wear across dynamic blocks + +--- + +Recovery from bad blocks doesn't actually have anything to do with the block +allocator itself. Instead, it relies on the ability of the filesystem to detect +and evict bad blocks when they occur. + +In littlefs, it is fairly straightforward to detect bad blocks at write time. +All writes must be sourced by some form of data in RAM, so immediately after we +write to a block, we can read the data back and verify that it was written +correctly. If we find that the data on disk does not match the copy we have in +RAM, a write error has occurred and we most likely have a bad block. + +Once we detect a bad block, we need to recover from it. In the case of write +errors, we have a copy of the corrupted data in RAM, so all we need to do is +evict the bad block, allocate a new, hopefully good block, and repeat the write +that previously failed. + +The actual act of evicting the bad block and replacing it with a new block is +left up to the filesystem's copy-on-bounded-writes (CObW) data structures. One +property of CObW data structures is that any block can be replaced during a +COW operation. The bounded-writes part is normally triggered by a counter, but +nothing prevents us from triggering a COW operation as soon as we find a bad +block. + +``` + .----. + |root| + | | + '----' + v--' '----------------------v +.----. .----. +| A | | B | +| | | | +'----' '----' +. . v---' . +. . .----. . +. . | C | . +. . | | . +. . '----' . +. . . . . +.----.----.----.----.----.----.----.----.----.----. +| A |root| | C | B | | +| | | | | | | +'----'----'----'----'----'----'----'----'----'----' + +update C +=> + .----. + |root| + | | + '----' + v--' '----------------------v +.----. .----. +| A | | B | +| | | | +'----' '----' +. . v---' . +. . .----. . +. . |bad | . +. . |blck| . +. . '----' . +. . . . . +.----.----.----.----.----.----.----.----.----.----. +| A |root| |bad | B | | +| | | |blck| | | +'----'----'----'----'----'----'----'----'----'----' + +oh no! bad block! relocate C +=> + .----. + |root| + | | + '----' + v--' '----------------------v +.----. .----. +| A | | B | +| | | | +'----' '----' +. . v---' . +. . .----. . +. . |bad | . +. . |blck| . +. . '----' . +. . . . . +.----.----.----.----.----.----.----.----.----.----. +| A |root| |bad | B |bad | | +| | | |blck| |blck| | +'----'----'----'----'----'----'----'----'----'----' + ---------> +oh no! bad block! relocate C +=> + .----. + |root| + | | + '----' + v--' '----------------------v +.----. .----. +| A | | B | +| | | | +'----' '----' +. . v---' . +. . .----. . .----. +. . |bad | . | C' | +. . |blck| . | | +. . '----' . '----' +. . . . . . . +.----.----.----.----.----.----.----.----.----.----. +| A |root| |bad | B |bad | C' | | +| | | |blck| |blck| | | +'----'----'----'----'----'----'----'----'----'----' + --------------> +successfully relocated C, update B +=> + .----. + |root| + | | + '----' + v--' '----------------------v +.----. .----. +| A | |bad | +| | |blck| +'----' '----' +. . v---' . +. . .----. . .----. +. . |bad | . | C' | +. . |blck| . | | +. . '----' . '----' +. . . . . . . +.----.----.----.----.----.----.----.----.----.----. +| A |root| |bad |bad |bad | C' | | +| | | |blck|blck|blck| | | +'----'----'----'----'----'----'----'----'----'----' + +oh no! bad block! relocate B +=> + .----. + |root| + | | + '----' + v--' '----------------------v +.----. .----. .----. +| A | |bad | |bad | +| | |blck| |blck| +'----' '----' '----' +. . v---' . . . +. . .----. . .----. . +. . |bad | . | C' | . +. . |blck| . | | . +. . '----' . '----' . +. . . . . . . . +.----.----.----.----.----.----.----.----.----.----. +| A |root| |bad |bad |bad | C' |bad | +| | | |blck|blck|blck| |blck| +'----'----'----'----'----'----'----'----'----'----' + --------------> +oh no! bad block! relocate B +=> + .----. + |root| + | | + '----' + v--' '----------------------v +.----. .----. .----. +| A | | B' | |bad | +| | | | |blck| +'----' '----' '----' +. . . | . .---' . +. . . '--------------v-------------v +. . . . .----. . .----. +. . . . |bad | . | C' | +. . . . |blck| . | | +. . . . '----' . '----' +. . . . . . . . . +.----.----.----.----.----.----.----.----.----.----. +| A |root| B' | |bad |bad |bad | C' |bad | +| | | | |blck|blck|blck| |blck| +'----'----'----'----'----'----'----'----'----'----' +------------> ------------------ +successfully relocated B, update root +=> + .----. + |root| + | | + '----' + v--' '--v +.----. .----. +| A | | B' | +| | | | +'----' '----' +. . . '---------------------------v +. . . . .----. +. . . . | C' | +. . . . | | +. . . . '----' +. . . . . . +.----.----.----.----.----.----.----.----.----.----. +| A |root| B' | |bad |bad |bad | C' |bad | +| | | | |blck|blck|blck| |blck| +'----'----'----'----'----'----'----'----'----'----' +``` + +We may find that the new block is also bad, but hopefully after repeating this +cycle we'll eventually find a new block where a write succeeds. If we don't, +that means that all blocks in our storage are bad, and we've reached the end of +our device's usable life. At this point, littlefs will return an "out of space" +error, which is technically true, there are no more good blocks, but as an +added benefit also matches the error condition expected by users of dynamically +sized data. + +--- + +Read errors, on the other hand, are quite a bit more complicated. We don't have +a copy of the data lingering around in RAM, so we need a way to reconstruct the +original data even after it has been corrupted. One such mechanism for this is +[error-correction-codes (ECC)][wikipedia-ecc]. + +ECC is an extension to the idea of a checksum. Where a checksum such as CRC can +detect that an error has occurred in the data, ECC can detect and actually +correct some amount of errors. However, there is a limit to how many errors ECC +can detect, call the [Hamming bound][wikipedia-hamming-bound]. As the number of +errors approaches the Hamming bound, we may still be able to detect errors, but +can no longer fix the data. If we've reached this point the block is +unrecoverable. + +littlefs by itself does **not** provide ECC. The block nature and relatively +large footprint of ECC does not work well with the dynamically sized data of +filesystems, correcting errors without RAM is complicated, and ECC fits better +with the geometry of block devices. In fact, several NOR flash chips have extra +storage intended for ECC, and many NAND chips can even calculate ECC on the +chip itself. + +In littlefs, ECC is entirely optional. Read errors can instead be prevented +proactively by wear leveling. But it's important to note that ECC can be used +at the block device level to modestly extend the life of a device. littlefs +respects any errors reported by the block device, allow a block device to +provide additional aggressive error detection. + +--- + +To avoid read errors, we need to be proactive, as opposed to reactive as we +were with write errors. + +One way to do this is to detect when the number of errors in a block exceeds +some threshold, but is still recoverable. With ECC we can do this at write +time, and treat the error as a write error, evicting the block before fatal +read errors have a chance to develop. + +A different, more generic strategy, is to proactively distribute wear across +all blocks in the storage, with the hope that no single block fails before the +rest of storage is approaching the end of its usable life. This is called +wear leveling. + +Generally, wear leveling algorithms fall into one of two categories: + +1. [Dynamic wear leveling][wikipedia-dynamic-wear-leveling], where we + distribute wear over "dynamic" blocks. The can be accomplished by + only considering unused blocks. + +2. [Static wear leveling][wikipedia-static-wear-leveling], where we + distribute wear over both "dynamic" and "static" blocks. To make this work, + we need to consider all blocks, including blocks that already contain data. + +As a tradeoff for code size and complexity, littlefs (currently) only provides +dynamic wear leveling. This is a best efforts solution. Wear is not distributed +perfectly, but it is distributed among the free blocks and greatly extends the +life of a device. + +On top of this, littlefs uses a statistical wear leveling algorithm. What this +means is that we don’t actively track wear, instead we rely on a uniform +distribution of wear across storage to approximate a dynamic wear leveling +algorithm. Despite the long name, this is actually a simplification of dynamic +wear leveling. + +The uniform distribution of wear is left up to the block allocator, which +creates a uniform distribution in two parts. The easy part is when the device +is powered, in which case we allocate the blocks linearly, circling the device. +The harder part is what to do when the device loses power. We can't just +restart the allocator at the beginning of storage, as this would bias the wear. +Instead, we start the allocator as a random offset every time we mount the +filesystem. As long as this random offset is uniform, the combined allocation +pattern is also a uniform distribution. + +![Cumulative wear distribution graph][wear-distribution-graph] + +Initially, this approach to wear leveling looks like it creates a difficult +dependency on a power-independent random number generator, which must return +different random numbers on each boot. However, the filesystem is in a +relatively unique situation in that it is sitting on top of a large of amount +of entropy that persists across power loss. + +We can actually use the data on disk to directly drive our random number +generator. In practice, this is implemented by xoring the checksums of each +metadata pair, which is already calculated to fetch and mount the filesystem. + +``` + .--------. \ probably random + .|metadata| | ^ + || | +-> crc ----------------------> xor + || | | ^ + |'--------' / | + '---|--|-' | + .-' '-------------------------. | + | | | + | .--------------> xor ------------> xor + | | ^ | ^ + v crc crc v crc + .--------. \ ^ .--------. \ ^ .--------. \ ^ + .|metadata|-|--|-->|metadata| | | .|metadata| | | + || | +--' || | +--' || | +--' + || | | || | | || | | + |'--------' / |'--------' / |'--------' / + '---|--|-' '----|---' '---|--|-' + .-' '-. | .-' '-. + v v v v v +.--------. .--------. .--------. .--------. .--------. +| data | | data | | data | | data | | data | +| | | | | | | | | | +| | | | | | | | | | +'--------' '--------' '--------' '--------' '--------' +``` + +Note that this random number generator is not perfect. It only returns unique +random numbers when the filesystem is modified. This is exactly what we want +for distributing wear in the allocator, but means this random number generator +is not useful for general use. + +--- + +Together, bad block detection and dynamic wear leveling provide a best effort +solution for avoiding the early death of a filesystem due to wear. Importantly, +littlefs's wear leveling algorithm provides a key feature: You can increase the +life of a device simply by increasing the size of storage. And if more +aggressive wear leveling is desired, you can always combine littlefs with a +[flash translation layer (FTL)][wikipedia-ftl] to get a small power resilient +filesystem with static wear leveling. + +## Files + +Now that we have our building blocks out of the way, we can start looking at +our filesystem as a whole. + +The first step: How do we actually store our files? + +We've determined that CTZ skip-lists are pretty good at storing data compactly, +so following the precedent found in other filesystems we could give each file +a skip-list stored in a metadata pair that acts as an inode for the file. + + +``` + .--------. + .|metadata| + || | + || | + |'--------' + '----|---' + v +.--------. .--------. .--------. .--------. +| data 0 |<-| data 1 |<-| data 2 |<-| data 3 | +| |<-| |--| | | | +| | | | | | | | +'--------' '--------' '--------' '--------' +``` + +However, this doesn't work well when files are small, which is common for +embedded systems. Compared to PCs, _all_ data in an embedded system is small. + +Consider a small 4-byte file. With a two block metadata-pair and one block for +the CTZ skip-list, we find ourselves using a full 3 blocks. On most NOR flash +with 4 KiB blocks, this is 12 KiB of overhead. A ridiculous 3072x increase. + +``` +file stored as inode, 4 bytes costs ~12 KiB + + .----------------. \ +.| revision | | +||----------------| \ | +|| skiplist ---. +- metadata | +||----------------| | / 4x8 bytes | +|| checksum | | 32 bytes | +||----------------| | | +|| | | | +- metadata pair +|| v | | | 2x4 KiB +|| | | | 8 KiB +|| | | | +|| | | | +|| | | | +|'----------------' | | +'----------------' | / + .--------' + v + .----------------. \ \ + | data | +- data | + |----------------| / 4 bytes | + | | | + | | | + | | | + | | +- data block + | | | 4 KiB + | | | + | | | + | | | + | | | + | | | + '----------------' / +``` + +We can make several improvements. First, instead of giving each file its own +metadata pair, we can store multiple files in a single metadata pair. One way +to do this is to directly associate a directory with a metadata pair (or a +linked list of metadata pairs). This makes it easy for multiple files to share +the directory's metadata pair for logging and reduce the collective storage +overhead. + +The strict binding of metadata pairs and directories also gives users +direct control over storage utilization depending on how they organize their +directories. + +``` +multiple files stored in metadata pair, 4 bytes costs ~4 KiB + + .----------------. + .| revision | + ||----------------| + || A name | + || A skiplist -----. + ||----------------| | \ + || B name | | +- metadata + || B skiplist ---. | | 4x8 bytes + ||----------------| | | / 32 bytes + || checksum | | | + ||----------------| | | + || | | | | + || v | | | + |'----------------' | | + '----------------' | | + .----------------' | + v v +.----------------. .----------------. \ \ +| A data | | B data | +- data | +| | |----------------| / 4 bytes | +| | | | | +| | | | | +| | | | | +| | | | + data block +| | | | | 4 KiB +| | | | | +|----------------| | | | +| | | | | +| | | | | +| | | | | +'----------------' '----------------' / +``` + +The second improvement we can make is noticing that for very small files, our +attempts to use CTZ skip-lists for compact storage backfires. Metadata pairs +have a ~4x storage cost, so if our file is smaller than 1/4 the block size, +there's actually no benefit in storing our file outside of our metadata pair. + +In this case, we can store the file directly in our directory's metadata pair. +We call this an inline file, and it allows a directory to store many small +files quite efficiently. Our previous 4 byte file now only takes up a +theoretical 16 bytes on disk. + +``` +inline files stored in metadata pair, 4 bytes costs ~16 bytes + + .----------------. +.| revision | +||----------------| +|| A name | +|| A skiplist ---. +||----------------| | \ +|| B name | | +- data +|| B data | | | 4x4 bytes +||----------------| | / 16 bytes +|| checksum | | +||----------------| | +|| | | | +|| v | | +|'----------------' | +'----------------' | + .---------' + v + .----------------. + | A data | + | | + | | + | | + | | + | | + | | + | | + |----------------| + | | + | | + | | + '----------------' +``` + +Once the file exceeds 1/4 the block size, we switch to a CTZ skip-list. This +means that our files never use more than 4x storage overhead, decreasing as +the file grows in size. -The real benefit of this approach is just how much it simplified the design -of the littlefs. Deallocating blocks is as simple as simply forgetting they -exist, and there is absolutely no concern of bugs in the deallocation code -causing difficult to detect memory leaks. +![File storage cost graph][file-cost-graph] ## Directories -Now we just need directories to store our files. Since we already have -metadata blocks that store information about files, lets just use these -metadata blocks as the directories. Maybe turn the directories into linked -lists of metadata blocks so it isn't limited by the number of files that fit -in a single block. Add entries that represent other nested directories. -Drop "." and ".." entries, cause who needs them. Dust off our hands and -we now have a directory tree. +Now we just need directories to store our files. As mentioned above we want +a strict binding of directories and metadata pairs, but there are a few +complications we need to sort out. + +On their own, each directory is a linked-list of metadata pairs. This lets us +store an unlimited number of files in each directory, and we don't need to +worry about the runtime complexity of unbounded logs. We can store other +directory pointers in our metadata pairs, which gives us a directory tree, much +like what you find on other filesystems. ``` .--------. - |root dir| - | pair 0 | - | | - '--------' + .| root | + || | + || | + |'--------' + '---|--|-' .-' '-------------------------. v v .--------. .--------. .--------. - | dir A |------->| dir A | | dir B | - | pair 0 | | pair 1 | | pair 0 | - | | | | | | - '--------' '--------' '--------' + .| dir A |------->| dir A | .| dir B | + || | || | || | + || | || | || | + |'--------' |'--------' |'--------' + '---|--|-' '----|---' '---|--|-' .-' '-. | .-' '-. v v v v v .--------. .--------. .--------. .--------. .--------. @@ -613,34 +1509,30 @@ we now have a directory tree. '--------' '--------' '--------' '--------' '--------' ``` -Unfortunately it turns out it's not that simple. See, iterating over a -directory tree isn't actually all that easy, especially when you're trying -to fit in a bounded amount of RAM, which rules out any recursive solution. -And since our block allocator involves iterating over the entire filesystem -tree, possibly multiple times in a single allocation, iteration needs to be -efficient. - -So, as a solution, the littlefs adopted a sort of threaded tree. Each -directory not only contains pointers to all of its children, but also a -pointer to the next directory. These pointers create a linked-list that -is threaded through all of the directories in the filesystem. Since we -only use this linked list to check for existence, the order doesn't actually -matter. As an added plus, we can repurpose the pointer for the individual -directory linked-lists and avoid using any additional space. +The main complication is, once again, traversal with a constant amount of +[RAM]. The directory tree is a tree, and the unfortunate fact is you can't +traverse a tree with constant RAM. + +Fortunately, the elements of our tree are metadata pairs, so unlike CTZ +skip-lists, we're not limited to strict COW operations. One thing we can do is +thread a linked-list through our tree, explicitly enabling cheap traversal +over the entire filesystem. ``` .--------. - |root dir|-. - | pair 0 | | - .--------| |-' - | '--------' + .| root |-. + || | | + .-------|| |-' + | |'--------' + | '---|--|-' | .-' '-------------------------. | v v | .--------. .--------. .--------. '->| dir A |------->| dir A |------->| dir B | - | pair 0 | | pair 1 | | pair 0 | - | | | | | | - '--------' '--------' '--------' + || | || | || | + || | || | || | + |'--------' |'--------' |'--------' + '---|--|-' '----|---' '---|--|-' .-' '-. | .-' '-. v v v v v .--------. .--------. .--------. .--------. .--------. @@ -650,577 +1542,632 @@ directory linked-lists and avoid using any additional space. '--------' '--------' '--------' '--------' '--------' ``` -This threaded tree approach does come with a few tradeoffs. Now, anytime we -want to manipulate the directory tree, we find ourselves having to update two -pointers instead of one. For anyone familiar with creating atomic data -structures this should set off a whole bunch of red flags. +Unfortunately, not sticking to pure COW operations creates some problems. Now, +whenever we want to manipulate the directory tree, multiple pointers need to be +updated. If you're familiar with designing atomic data structures this should +set off a bunch of red flags. -But unlike the data structure guys, we can update a whole block atomically! So -as long as we're really careful (and cheat a little bit), we can still -manipulate the directory tree in a way that is resilient to power loss. - -Consider how we might add a new directory. Since both pointers that reference -it can come from the same directory, we only need a single atomic update to -finagle the directory into the filesystem: -``` - .--------. - |root dir|-. - | pair 0 | | -.--| |-' -| '--------' -| | -| v -| .--------. -'->| dir A | - | pair 0 | - | | - '--------' +To work around this, our threaded linked-list has a bit of leeway. Instead of +only containing metadata pairs found in our filesystem, it is allowed to +contain metadata pairs that have no parent because of a power loss. These are +called orphaned metadata pairs. -| create the new directory block -v +With the possibility of orphans, we can build power loss resilient operations +that maintain a filesystem tree threaded with a linked-list for traversal. - .--------. - |root dir|-. - | pair 0 | | - .--| |-' - | '--------' - | | - | v - | .--------. -.--------. '->| dir A | -| dir B |---->| pair 0 | -| pair 0 | | | -| | '--------' -'--------' - -| update root to point to directory B -v +Adding a directory to our tree: +``` .--------. - |root dir|-. - | pair 0 | | -.--------| |-' -| '--------' + .| root |-. + || | | +.-------|| |-' +| |'--------' +| '---|--|-' | .-' '-. | v v | .--------. .--------. -'->| dir B |->| dir A | - | pair 0 | | pair 0 | - | | | | - '--------' '--------' -``` +'->| dir A |->| dir C | + || | || | + || | || | + |'--------' |'--------' + '--------' '--------' -Note that even though directory B was added after directory A, we insert -directory B before directory A in the linked-list because it is convenient. +allocate dir B +=> + .--------. + .| root |-. + || | | +.-------|| |-' +| |'--------' +| '---|--|-' +| .-' '-. +| v v +| .--------. .--------. +'->| dir A |--->| dir C | + || | .->| | + || | | || | + |'--------' | |'--------' + '--------' | '--------' + | + .--------. | + .| dir B |-' + || | + || | + |'--------' + '--------' + +insert dir B into threaded linked-list, creating an orphan +=> + .--------. + .| root |-. + || | | +.-------|| |-' +| |'--------' +| '---|--|-' +| .-' '-------------. +| v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + || | || orphan!| || | + || | || | || | + |'--------' |'--------' |'--------' + '--------' '--------' '--------' -Now how about removal: -``` - .--------. .--------. - |root dir|------->|root dir|-. - | pair 0 | | pair 1 | | -.--------| |--------| |-' -| '--------' '--------' -| .-' '-. | +add dir B to parent directory +=> + .--------. + .| root |-. + || | | +.-------------|| |-' +| |'--------' +| '--|-|-|-' +| .------' | '-------. | v v v | .--------. .--------. .--------. '->| dir A |->| dir B |->| dir C | - | pair 0 | | pair 0 | | pair 0 | - | | | | | | - '--------' '--------' '--------' + || | || | || | + || | || | || | + |'--------' |'--------' |'--------' + '--------' '--------' '--------' +``` -| update root to no longer contain directory B -v +Removing a directory: - .--------. .--------. - |root dir|------------->|root dir|-. - | pair 0 | | pair 1 | | -.--| |--------------| |-' -| '--------' '--------' -| | | -| v v +``` + .--------. + .| root |-. + || | | +.-------------|| |-' +| |'--------' +| '--|-|-|-' +| .------' | '-------. +| v v v | .--------. .--------. .--------. '->| dir A |->| dir B |->| dir C | - | pair 0 | | pair 0 | | pair 0 | - | | | | | | - '--------' '--------' '--------' + || | || | || | + || | || | || | + |'--------' |'--------' |'--------' + '--------' '--------' '--------' -| remove directory B from the linked-list -v +remove dir B from parent directory, creating an orphan +=> + .--------. + .| root |-. + || | | +.-------|| |-' +| |'--------' +| '---|--|-' +| .-' '-------------. +| v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + || | || orphan!| || | + || | || | || | + |'--------' |'--------' |'--------' + '--------' '--------' '--------' - .--------. .--------. - |root dir|->|root dir|-. - | pair 0 | | pair 1 | | -.--| |--| |-' -| '--------' '--------' -| | | -| v v +remove dir B from threaded linked-list, returning dir B to free blocks +=> + .--------. + .| root |-. + || | | +.-------|| |-' +| |'--------' +| '---|--|-' +| .-' '-. +| v v | .--------. .--------. '->| dir A |->| dir C | - | pair 0 | | pair 0 | - | | | | - '--------' '--------' + || | || | + || | || | + |'--------' |'--------' + '--------' '--------' ``` -Wait, wait, wait, that's not atomic at all! If power is lost after removing -directory B from the root, directory B is still in the linked-list. We've -just created a memory leak! +In addition to normal directory tree operations, we can use orphans to evict +blocks in a metadata pair when the block goes bad or exceeds its allocated +erases. If we lose power while evicting a metadata block we may end up with +a situation where the filesystem references the replacement block while the +threaded linked-list still contains the evicted block. We call this a +half-orphan. -And to be honest, I don't have a clever solution for this case. As a -side-effect of using multiple pointers in the threaded tree, the littlefs -can end up with orphan blocks that have no parents and should have been -removed. +``` + .--------. + .| root |-. + || | | +.-------------|| |-' +| |'--------' +| '--|-|-|-' +| .------' | '-------. +| v v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + || | || | || | + || | || | || | + |'--------' |'--------' |'--------' + '--------' '--------' '--------' + +try to write to dir B +=> + .--------. + .| root |-. + || | | +.----------------|| |-' +| |'--------' +| '-|-||-|-' +| .--------' || '-----. +| v |v v +| .--------. .--------. .--------. +'->| dir A |---->| dir B |->| dir C | + || |-. | | || | + || | | | | || | + |'--------' | '--------' |'--------' + '--------' | v '--------' + | .--------. + '->| dir B | + | bad | + | block! | + '--------' + +oh no! bad block detected, allocate replacement +=> + .--------. + .| root |-. + || | | +.----------------|| |-' +| |'--------' +| '-|-||-|-' +| .--------' || '-------. +| v |v v +| .--------. .--------. .--------. +'->| dir A |---->| dir B |--->| dir C | + || |-. | | .->| | + || | | | | | || | + |'--------' | '--------' | |'--------' + '--------' | v | '--------' + | .--------. | + '->| dir B | | + | bad | | + | block! | | + '--------' | + | + .--------. | + | dir B |--' + | | + | | + '--------' + +insert replacement in threaded linked-list, creating a half-orphan +=> + .--------. + .| root |-. + || | | +.----------------|| |-' +| |'--------' +| '-|-||-|-' +| .--------' || '-------. +| v |v v +| .--------. .--------. .--------. +'->| dir A |---->| dir B |--->| dir C | + || |-. | | .->| | + || | | | | | || | + |'--------' | '--------' | |'--------' + '--------' | v | '--------' + | .--------. | + | | dir B | | + | | bad | | + | | block! | | + | '--------' | + | | + | .--------. | + '->| dir B |--' + | half | + | orphan!| + '--------' + +fix reference in parent directory +=> + .--------. + .| root |-. + || | | +.-------------|| |-' +| |'--------' +| '--|-|-|-' +| .------' | '-------. +| v v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + || | || | || | + || | || | || | + |'--------' |'--------' |'--------' + '--------' '--------' '--------' +``` + +Finding orphans and half-orphans is expensive, requiring a _O(n²)_ +comparison of every metadata pair with every directory entry. But the tradeoff +is a power resilient filesystem that works with only a bounded amount of RAM. +Fortunately, we only need to check for orphans on the first allocation after +boot, and a read-only littlefs can ignore the threaded linked-list entirely. -To keep these orphan blocks from becoming a problem, the littlefs has a -deorphan step that simply iterates through every directory in the linked-list -and checks it against every directory entry in the filesystem to see if it -has a parent. The deorphan step occurs on the first block allocation after -boot, so orphans should never cause the littlefs to run out of storage -prematurely. Note that the deorphan step never needs to run in a read-only -filesystem. +If we only had some sort of global state, then we could also store a flag and +avoid searching for orphans unless we knew we were specifically interrupted +while manipulating the directory tree (foreshadowing!). ## The move problem -Now we have a real problem. How do we move things between directories while -remaining power resilient? Even looking at the problem from a high level, -it seems impossible. We can update directory blocks atomically, but atomically -updating two independent directory blocks is not an atomic operation. +We have one last challenge. The move problem. Phrasing the problem is simple: + +How do you atomically move a file between two directories? + +In littlefs we can atomically commit to directories, but we can't create +an atomic commit that span multiple directories. The filesystem must go +through a minimum of two distinct states to complete a move. + +To make matters worse, file moves are a common form of synchronization for +filesystems. As a filesystem designed for power-loss, it's important we get +atomic moves right. + +So what can we do? + +- We definitely can't just let power-loss result in duplicated or lost files. + This could easily break user's code and would only reveal itself in extreme + cases. We were only able to be lazy about the threaded linked-list because + it isn't user facing and we can handle the corner cases internally. + +- Some filesystems propagate COW operations up the tree until finding a common + parent. Unfortunately this interacts poorly with our threaded tree and brings + back the issue of upward propagation of wear. + +- In a previous version of littlefs we tried to solve this problem by going + back and forth between the source and destination, marking and unmarking the + file as moving in order to make the move atomic from the user perspective. + This worked, but not well. Finding failed moves was expensive and required + a unique identifier for each file. + +In the end, solving the move problem required creating a new mechanism for +sharing knowledge between multiple metadata pairs. In littlefs this led to the +introduction of a mechanism called "global state". + +--- + +Global state is a small set of state that can be updated from _any_ metadata +pair. Combining global state with metadata pair's ability to update multiple +entries in one commit gives us a powerful tool for crafting complex atomic +operations. + +How does global state work? + +Global state exists as a set of deltas that are distributed across the metadata +pairs in the filesystem. The actual global state can be built out of these +deltas by xoring together all of the deltas in the filesystem. -Here's the steps the filesystem may go through to move a directory: ``` - .--------. - |root dir|-. - | pair 0 | | -.--------| |-' -| '--------' -| .-' '-. -| v v -| .--------. .--------. -'->| dir A |->| dir B | - | pair 0 | | pair 0 | - | | | | - '--------' '--------' + .--------. .--------. .--------. .--------. .--------. +.| |->| gdelta |->| |->| gdelta |->| gdelta | +|| | || 0x23 | || | || 0xff | || 0xce | +|| | || | || | || | || | +|'--------' |'--------' |'--------' |'--------' |'--------' +'--------' '----|---' '--------' '----|---' '----|---' + v v v + 0x00 --> xor ------------------> xor ------> xor --> gstate 0x12 +``` -| update directory B to point to directory A -v +To update the global state from a metadata pair, we take the global state we +know and xor it with both our changes and any existing delta in the metadata +pair. Committing this new delta to the metadata pair commits the changes to +the filesystem's global state. - .--------. - |root dir|-. - | pair 0 | | -.--------| |-' -| '--------' -| .-----' '-. -| | v -| | .--------. -| | .->| dir B | -| | | | pair 0 | -| | | | | -| | | '--------' -| | .-------' -| v v | -| .--------. | -'->| dir A |-' - | pair 0 | +``` + .--------. .--------. .--------. .--------. .--------. +.| |->| gdelta |->| |->| gdelta |->| gdelta | +|| | || 0x23 | || | || 0xff | || 0xce | +|| | || | || | || | || | +|'--------' |'--------' |'--------' |'--------' |'--------' +'--------' '----|---' '--------' '--|---|-' '----|---' + v v | v + 0x00 --> xor ----------------> xor -|------> xor --> gstate = 0x12 + | | + | | +change gstate to 0xab --> xor <------------|--------------------------' +=> | v + '------------> xor + | + v + .--------. .--------. .--------. .--------. .--------. +.| |->| gdelta |->| |->| gdelta |->| gdelta | +|| | || 0x23 | || | || 0x46 | || 0xce | +|| | || | || | || | || | +|'--------' |'--------' |'--------' |'--------' |'--------' +'--------' '----|---' '--------' '----|---' '----|---' + v v v + 0x00 --> xor ------------------> xor ------> xor --> gstate = 0xab +``` + +To make this efficient, we always keep a copy of the global state in RAM. We +only need to iterate over our metadata pairs and build the global state when +the filesystem is mounted. + +You may have noticed that global state is very expensive. We keep a copy in +RAM and a delta in an unbounded number of metadata pairs. Even if we reset +the global state to its initial value we can't easily clean up the deltas on +disk. For this reason, it's very important that we keep the size of global +state bounded and extremely small. But, even with a strict budget, global +state is incredibly valuable. + +--- + +Now we can solve the move problem. We can create global state describing our +move atomically with the creation of the new file, and we can clear this move +state atomically with the removal of the old file. + +``` + .--------. gstate = no move + .| root |-. + || | | +.-------------|| |-' +| |'--------' +| '--|-|-|-' +| .------' | '-------. +| v v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + || | || | || | + || | || | || | + |'--------' |'--------' |'--------' + '----|---' '--------' '--------' + v + .--------. + | file D | + | | | | '--------' -| update root to no longer contain directory A -v +begin move, add reference in dir C, change gstate to have move +=> + .--------. gstate = moving file D in dir A (m1) + .| root |-. + || | | +.-------------|| |-' +| |'--------' +| '--|-|-|-' +| .------' | '-------. +| v v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + || | || | || gdelta | + || | || | || =m1 | + |'--------' |'--------' |'--------' + '----|---' '--------' '----|---' + | .----------------' + v v .--------. - |root dir|-. - | pair 0 | | -.----| |-' -| '--------' -| | -| v -| .--------. -| .->| dir B | -| | | pair 0 | -| '--| |-. -| '--------' | -| | | -| v | -| .--------. | -'--->| dir A |-' - | pair 0 | + | file D | + | | | | '--------' + +complete move, remove reference in dir A, change gstate to no move +=> + .--------. gstate = no move (m1^~m1) + .| root |-. + || | | +.-------------|| |-' +| |'--------' +| '--|-|-|-' +| .------' | '-------. +| v v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + || gdelta | || | || gdelta | + || =~m1 | || | || =m1 | + |'--------' |'--------' |'--------' + '--------' '--------' '----|---' + v + .--------. + | file D | + | | + | | + '--------' ``` -We can leave any orphans up to the deorphan step to collect, but that doesn't -help the case where dir A has both dir B and the root dir as parents if we -lose power inconveniently. - -Initially, you might think this is fine. Dir A _might_ end up with two parents, -but the filesystem will still work as intended. But then this raises the -question of what do we do when the dir A wears out? For other directory blocks -we can update the parent pointer, but for a dir with two parents we would need -work out how to update both parents. And the check for multiple parents would -need to be carried out for every directory, even if the directory has never -been moved. - -It also presents a bad user-experience, since the condition of ending up with -two parents is rare, it's unlikely user-level code will be prepared. Just think -about how a user would recover from a multi-parented directory. They can't just -remove one directory, since remove would report the directory as "not empty". - -Other atomic filesystems simple COW the entire directory tree. But this -introduces a significant bit of complexity, which leads to code size, along -with a surprisingly expensive runtime cost during what most users assume is -a single pointer update. - -Another option is to update the directory block we're moving from to point -to the destination with a sort of predicate that we have moved if the -destination exists. Unfortunately, the omnipresent concern of wear could -cause any of these directory entries to change blocks, and changing the -entry size before a move introduces complications if it spills out of -the current directory block. - -So how do we go about moving a directory atomically? - -We rely on the improbableness of power loss. - -Power loss during a move is certainly possible, but it's actually relatively -rare. Unless a device is writing to a filesystem constantly, it's unlikely that -a power loss will occur during filesystem activity. We still need to handle -the condition, but runtime during a power loss takes a back seat to the runtime -during normal operations. - -So what littlefs does is inelegantly simple. When littlefs moves a file, it -marks the file as "moving". This is stored as a single bit in the directory -entry and doesn't take up much space. Then littlefs moves the directory, -finishing with the complete remove of the "moving" directory entry. -``` - .--------. - |root dir|-. - | pair 0 | | -.--------| |-' -| '--------' -| .-' '-. -| v v -| .--------. .--------. -'->| dir A |->| dir B | - | pair 0 | | pair 0 | - | | | | - '--------' '--------' - -| update root directory to mark directory A as moving -v - - .----------. - |root dir |-. - | pair 0 | | -.-------| moving A!|-' -| '----------' -| .-' '-. -| v v -| .--------. .--------. -'->| dir A |->| dir B | - | pair 0 | | pair 0 | - | | | | - '--------' '--------' - -| update directory B to point to directory A -v - - .----------. - |root dir |-. - | pair 0 | | -.-------| moving A!|-' -| '----------' -| .-----' '-. -| | v -| | .--------. -| | .->| dir B | -| | | | pair 0 | -| | | | | -| | | '--------' -| | .-------' -| v v | -| .--------. | -'->| dir A |-' - | pair 0 | - | | - '--------' +If, after building our global state during mount, we find information +describing an ongoing move, we know we lost power during a move and the file +is duplicated in both the source and destination directories. If this happens, +we can resolve the move using the information in the global state to remove +one of the files. -| update root to no longer contain directory A -v +``` + .--------. gstate = moving file D in dir A (m1) + .| root |-. ^ + || |------------> xor +.---------------|| |-' ^ +| |'--------' | +| '--|-|-|-' | +| .--------' | '---------. | +| | | | | +| | .----------> xor --------> xor +| v | v ^ v ^ +| .--------. | .--------. | .--------. | +'->| dir A |-|->| dir B |-|->| dir C | | + || |-' || |-' || gdelta |-' + || | || | || =m1 | + |'--------' |'--------' |'--------' + '----|---' '--------' '----|---' + | .---------------------' + v v .--------. - |root dir|-. - | pair 0 | | -.----| |-' -| '--------' -| | -| v -| .--------. -| .->| dir B | -| | | pair 0 | -| '--| |-. -| '--------' | -| | | -| v | -| .--------. | -'--->| dir A |-' - | pair 0 | + | file D | + | | | | '--------' ``` -Now, if we run into a directory entry that has been marked as "moved", one -of two things is possible. Either the directory entry exists elsewhere in the -filesystem, or it doesn't. This is a O(n) operation, but only occurs in the -unlikely case we lost power during a move. - -And we can easily fix the "moved" directory entry. Since we're already scanning -the filesystem during the deorphan step, we can also check for moved entries. -If we find one, we either remove the "moved" marking or remove the whole entry -if it exists elsewhere in the filesystem. - -## Wear awareness - -So now that we have all of the pieces of a filesystem, we can look at a more -subtle attribute of embedded storage: The wear down of flash blocks. - -The first concern for the littlefs, is that perfectly valid blocks can suddenly -become unusable. As a nice side-effect of using a COW data-structure for files, -we can simply move on to a different block when a file write fails. All -modifications to files are performed in copies, so we will only replace the -old file when we are sure none of the new file has errors. Directories, on -the other hand, need a different strategy. - -The solution to directory corruption in the littlefs relies on the redundant -nature of the metadata pairs. If an error is detected during a write to one -of the metadata pairs, we seek out a new block to take its place. Once we find -a block without errors, we iterate through the directory tree, updating any -references to the corrupted metadata pair to point to the new metadata block. -Just like when we remove directories, we can lose power during this operation -and end up with a desynchronized metadata pair in our filesystem. And just like -when we remove directories, we leave the possibility of a desynchronized -metadata pair up to the deorphan step to clean up. - -Here's what encountering a directory error may look like with all of -the directories and directory pointers fully expanded: +We can also move directories the same way we move files. There is the threaded +linked-list to consider, but leaving the threaded linked-list unchanged works +fine as the order doesn't really matter. + ``` - root dir - block 1 block 2 - .---------.---------. - | rev: 1 | rev: 0 |--. - | | |-.| -.------| | |-|' -|.-----| | |-' -|| '---------'---------' -|| |||||'--------------------------------------------------. -|| ||||'-----------------------------------------. | -|| |||'-----------------------------. | | -|| ||'--------------------. | | | -|| |'-------. | | | | -|| v v v v v v -|| dir A dir B dir C -|| block 3 block 4 block 5 block 6 block 7 block 8 -|| .---------.---------. .---------.---------. .---------.---------. -|'->| rev: 1 | rev: 0 |->| rev: 1 | rev: 0 |->| rev: 1 | rev: 0 | -'-->| | |->| | |->| | | - | | | | | | | - | | | | | | | | | - '---------'---------' '---------'---------' '---------'---------' - -| update directory B -v - - root dir - block 1 block 2 - .---------.---------. - | rev: 1 | rev: 0 |--. - | | |-.| -.------| | |-|' -|.-----| | |-' -|| '---------'---------' -|| |||||'--------------------------------------------------. -|| ||||'-----------------------------------------. | -|| |||'-----------------------------. | | -|| ||'--------------------. | | | -|| |'-------. | | | | -|| v v v v v v -|| dir A dir B dir C -|| block 3 block 4 block 5 block 6 block 7 block 8 -|| .---------.---------. .---------.---------. .---------.---------. -|'->| rev: 1 | rev: 0 |->| rev: 1 | rev: 2 |->| rev: 1 | rev: 0 | -'-->| | |->| | corrupt!|->| | | - | | | | | corrupt!| | | | - | | | | | corrupt!| | | | - '---------'---------' '---------'---------' '---------'---------' - -| oh no! corruption detected -v allocate a replacement block - - root dir - block 1 block 2 - .---------.---------. - | rev: 1 | rev: 0 |--. - | | |-.| -.------| | |-|' -|.-----| | |-' -|| '---------'---------' -|| |||||'----------------------------------------------------. -|| ||||'-------------------------------------------. | -|| |||'-----------------------------. | | -|| ||'--------------------. | | | -|| |'-------. | | | | -|| v v v v v v -|| dir A dir B dir C -|| block 3 block 4 block 5 block 6 block 7 block 8 -|| .---------.---------. .---------.---------. .---------.---------. -|'->| rev: 1 | rev: 0 |->| rev: 1 | rev: 2 |--->| rev: 1 | rev: 0 | -'-->| | |->| | corrupt!|--->| | | - | | | | | corrupt!| .->| | | - | | | | | corrupt!| | | | | - '---------'---------' '---------'---------' | '---------'---------' - block 9 | - .---------. | - | rev: 2 |-' - | | - | | - | | - '---------' - -| update root directory to contain block 9 -v - - root dir - block 1 block 2 - .---------.---------. - | rev: 1 | rev: 2 |--. - | | |-.| -.-----| | |-|' -|.----| | |-' -|| '---------'---------' -|| .--------'||||'----------------------------------------------. -|| | |||'-------------------------------------. | -|| | ||'-----------------------. | | -|| | |'------------. | | | -|| | | | | | | -|| v v v v v v -|| dir A dir B dir C -|| block 3 block 4 block 5 block 9 block 7 block 8 -|| .---------.---------. .---------. .---------. .---------.---------. -|'->| rev: 1 | rev: 0 |-->| rev: 1 |-| rev: 2 |--->| rev: 1 | rev: 0 | -'-->| | |-. | | | |--->| | | - | | | | | | | | .->| | | - | | | | | | | | | | | | - '---------'---------' | '---------' '---------' | '---------'---------' - | block 6 | - | .---------. | - '------------>| rev: 2 |-' - | corrupt!| - | corrupt!| - | corrupt!| - '---------' - -| remove corrupted block from linked-list -v - - root dir - block 1 block 2 - .---------.---------. - | rev: 1 | rev: 2 |--. - | | |-.| -.-----| | |-|' -|.----| | |-' -|| '---------'---------' -|| .--------'||||'-----------------------------------------. -|| | |||'--------------------------------. | -|| | ||'--------------------. | | -|| | |'-----------. | | | -|| | | | | | | -|| v v v v v v -|| dir A dir B dir C -|| block 3 block 4 block 5 block 9 block 7 block 8 -|| .---------.---------. .---------.---------. .---------.---------. -|'->| rev: 1 | rev: 2 |->| rev: 1 | rev: 2 |->| rev: 1 | rev: 0 | -'-->| | |->| | |->| | | - | | | | | | | | | - | | | | | | | | | - '---------'---------' '---------'---------' '---------'---------' + .--------. gstate = no move (m1^~m1) + .| root |-. + || | | +.-------------|| |-' +| |'--------' +| '--|-|-|-' +| .------' | '-------. +| v v v +| .--------. .--------. .--------. +'->| dir A |->| dir B |->| dir C | + || gdelta | || | || gdelta | + || =~m1 | || | || =m1 | + |'--------' |'--------' |'--------' + '--------' '--------' '----|---' + v + .--------. + | file D | + | | + | | + '--------' + +begin move, add reference in dir C, change gstate to have move +=> + .--------. gstate = moving dir B in root (m1^~m1^m2) + .| root |-. + || | | +.--------------|| |-' +| |'--------' +| '--|-|-|-' +| .-------' | '----------. +| v | v +| .--------. | .--------. +'->| dir A |-. | .->| dir C | + || gdelta | | | | || gdelta | + || =~m1 | | | | || =m1^m2 | + |'--------' | | | |'--------' + '--------' | | | '---|--|-' + | | .-------' | + | v v | v + | .--------. | .--------. + '->| dir B |-' | file D | + || | | | + || | | | + |'--------' '--------' + '--------' + +complete move, remove reference in root, change gstate to no move +=> + .--------. gstate = no move (m1^~m1^m2^~m2) + .| root |-. + || gdelta | | +.-----------|| =~m2 |-' +| |'--------' +| '---|--|-' +| .-----' '-----. +| v v +| .--------. .--------. +'->| dir A |-. .->| dir C | + || gdelta | | | || gdelta | + || =~m1 | | '-|| =m1^m2 |-------. + |'--------' | |'--------' | + '--------' | '---|--|-' | + | .-' '-. | + | v v | + | .--------. .--------. | + '->| dir B |--| file D |-' + || | | | + || | | | + |'--------' '--------' + '--------' ``` -Also one question I've been getting is, what about the root directory? -It can't move so wouldn't the filesystem die as soon as the root blocks -develop errors? And you would be correct. So instead of storing the root -in the first few blocks of the storage, the root is actually pointed to -by the superblock. The superblock contains a few bits of static data, but -outside of when the filesystem is formatted, it is only updated when the root -develops errors and needs to be moved. - -## Wear leveling - -The second concern for the littlefs is that blocks in the filesystem may wear -unevenly. In this situation, a filesystem may meet an early demise where -there are no more non-corrupted blocks that aren't in use. It's common to -have files that were written once and left unmodified, wasting the potential -erase cycles of the blocks it sits on. - -Wear leveling is a term that describes distributing block writes evenly to -avoid the early termination of a flash part. There are typically two levels -of wear leveling: -1. Dynamic wear leveling - Wear is distributed evenly across all **dynamic** - blocks. Usually this is accomplished by simply choosing the unused block - with the lowest amount of wear. Note this does not solve the problem of - static data. -2. Static wear leveling - Wear is distributed evenly across all **dynamic** - and **static** blocks. Unmodified blocks may be evicted for new block - writes. This does handle the problem of static data but may lead to - wear amplification. - -In littlefs's case, it's possible to use the revision count on metadata pairs -to approximate the wear of a metadata block. And combined with the COW nature -of files, littlefs could provide your usual implementation of dynamic wear -leveling. - -However, the littlefs does not. This is for a few reasons. Most notably, even -if the littlefs did implement dynamic wear leveling, this would still not -handle the case of write-once files, and near the end of the lifetime of a -flash device, you would likely end up with uneven wear on the blocks anyways. - -As a flash device reaches the end of its life, the metadata blocks will -naturally be the first to go since they are updated most often. In this -situation, the littlefs is designed to simply move on to another set of -metadata blocks. This travelling means that at the end of a flash device's -life, the filesystem will have worn the device down nearly as evenly as the -usual dynamic wear leveling could. More aggressive wear leveling would come -with a code-size cost for marginal benefit. - - -One important takeaway to note, if your storage stack uses highly sensitive -storage such as NAND flash, static wear leveling is the only valid solution. -In most cases you are going to be better off using a full [flash translation layer (FTL)](https://en.wikipedia.org/wiki/Flash_translation_layer). -NAND flash already has many limitations that make it poorly suited for an -embedded system: low erase cycles, very large blocks, errors that can develop -even during reads, errors that can develop during writes of neighboring blocks. -Managing sensitive storage such as NAND flash is out of scope for the littlefs. -The littlefs does have some properties that may be beneficial on top of a FTL, -such as limiting the number of writes where possible, but if you have the -storage requirements that necessitate the need of NAND flash, you should have -the RAM to match and just use an FTL or flash filesystem. - -## Summary - -So, to summarize: - -1. The littlefs is composed of directory blocks -2. Each directory is a linked-list of metadata pairs -3. These metadata pairs can be updated atomically by alternating which - metadata block is active -4. Directory blocks contain either references to other directories or files -5. Files are represented by copy-on-write CTZ skip-lists which support O(1) - append and O(n log n) reading -6. Blocks are allocated by scanning the filesystem for used blocks in a - fixed-size lookahead region that is stored in a bit-vector -7. To facilitate scanning the filesystem, all directories are part of a - linked-list that is threaded through the entire filesystem -8. If a block develops an error, the littlefs allocates a new block, and - moves the data and references of the old block to the new. -9. Any case where an atomic operation is not possible, mistakes are resolved - by a deorphan step that occurs on the first allocation after boot - -That's the little filesystem. Thanks for reading! - +Global state gives us a powerful tool we can use to solve the move problem. +And the result is surprisingly performant, only needing the minimum number +of states and using the same number of commits as a naive move. Additionally, +global state gives us a bit of persistent state we can use for some other +small improvements. + +## Conclusion + +And that's littlefs, thanks for reading! + + +[wikipedia-flash]: https://en.wikipedia.org/wiki/Flash_memory +[wikipedia-sna]: https://en.wikipedia.org/wiki/Serial_number_arithmetic +[wikipedia-crc]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check +[wikipedia-cow]: https://en.wikipedia.org/wiki/Copy-on-write +[wikipedia-B-tree]: https://en.wikipedia.org/wiki/B-tree +[wikipedia-B+-tree]: https://en.wikipedia.org/wiki/B%2B_tree +[wikipedia-skip-list]: https://en.wikipedia.org/wiki/Skip_list +[wikipedia-ctz]: https://en.wikipedia.org/wiki/Count_trailing_zeros +[wikipedia-ecc]: https://en.wikipedia.org/wiki/Error_correction_code +[wikipedia-hamming-bound]: https://en.wikipedia.org/wiki/Hamming_bound +[wikipedia-dynamic-wear-leveling]: https://en.wikipedia.org/wiki/Wear_leveling#Dynamic_wear_leveling +[wikipedia-static-wear-leveling]: https://en.wikipedia.org/wiki/Wear_leveling#Static_wear_leveling +[wikipedia-ftl]: https://en.wikipedia.org/wiki/Flash_translation_layer + +[oeis]: https://oeis.org +[A001511]: https://oeis.org/A001511 +[A005187]: https://oeis.org/A005187 + +[fat]: https://en.wikipedia.org/wiki/Design_of_the_FAT_file_system +[ext2]: http://e2fsprogs.sourceforge.net/ext2intro.html +[jffs]: https://www.sourceware.org/jffs2/jffs2-html +[yaffs]: https://yaffs.net/documents/how-yaffs-works +[spiffs]: https://github.com/pellepl/spiffs/blob/master/docs/TECH_SPEC +[ext4]: https://ext4.wiki.kernel.org/index.php/Ext4_Design +[ntfs]: https://en.wikipedia.org/wiki/NTFS +[btrfs]: https://btrfs.wiki.kernel.org/index.php/Btrfs_design +[zfs]: https://en.wikipedia.org/wiki/ZFS + +[cow]: https://upload.wikimedia.org/wikipedia/commons/0/0c/Cow_female_black_white.jpg +[elephant]: https://upload.wikimedia.org/wikipedia/commons/3/37/African_Bush_Elephant.jpg +[ram]: https://upload.wikimedia.org/wikipedia/commons/9/97/New_Mexico_Bighorn_Sheep.JPG + +[metadata-formula1]: https://latex.codecogs.com/svg.latex?cost%20%3D%20n%20+%20n%20%5Cfrac%7Bs%7D%7Bd+1%7D +[metadata-formula2]: https://latex.codecogs.com/svg.latex?s%20%3D%20r%20%5Cfrac%7Bsize%7D%7Bn%7D +[metadata-formula3]: https://latex.codecogs.com/svg.latex?d%20%3D%20%281-r%29%20%5Cfrac%7Bsize%7D%7Bn%7D +[metadata-formula4]: https://latex.codecogs.com/svg.latex?cost%20%3D%20n%20+%20n%20%5Cfrac%7Br%5Cfrac%7Bsize%7D%7Bn%7D%7D%7B%281-r%29%5Cfrac%7Bsize%7D%7Bn%7D+1%7D + +[ctz-formula1]: https://latex.codecogs.com/svg.latex?%5Clim_%7Bn%5Cto%5Cinfty%7D%5Cfrac%7B1%7D%7Bn%7D%5Csum_%7Bi%3D0%7D%5E%7Bn%7D%5Cleft%28%5Ctext%7Bctz%7D%28i%29+1%5Cright%29%20%3D%20%5Csum_%7Bi%3D0%7D%5Cfrac%7B1%7D%7B2%5Ei%7D%20%3D%202 +[ctz-formula2]: https://latex.codecogs.com/svg.latex?B%20%3D%20%5Cfrac%7Bw%7D%7B8%7D%5Cleft%5Clceil%5Clog_2%5Cleft%28%5Cfrac%7B2%5Ew%7D%7BB-2%5Cfrac%7Bw%7D%7B8%7D%7D%5Cright%29%5Cright%5Crceil +[ctz-formula3]: https://latex.codecogs.com/svg.latex?N%20%3D%20%5Csum_i%5En%5Cleft%5BB-%5Cfrac%7Bw%7D%7B8%7D%5Cleft%28%5Ctext%7Bctz%7D%28i%29+1%5Cright%29%5Cright%5D +[ctz-formula4]: https://latex.codecogs.com/svg.latex?%5Csum_i%5En%5Cleft%28%5Ctext%7Bctz%7D%28i%29+1%5Cright%29%20%3D%202n-%5Ctext%7Bpopcount%7D%28n%29 +[ctz-formula5]: https://latex.codecogs.com/svg.latex?N%20%3D%20Bn%20-%20%5Cfrac%7Bw%7D%7B8%7D%5Cleft%282n-%5Ctext%7Bpopcount%7D%28n%29%5Cright%29 +[ctz-formula6]: https://latex.codecogs.com/svg.latex?n%20%3D%20%5Cleft%5Clfloor%5Cfrac%7BN-%5Cfrac%7Bw%7D%7B8%7D%5Cleft%28%5Ctext%7Bpopcount%7D%5Cleft%28%5Cfrac%7BN%7D%7BB-2%5Cfrac%7Bw%7D%7B8%7D%7D-1%5Cright%29+2%5Cright%29%7D%7BB-2%5Cfrac%7Bw%7D%7B8%7D%7D%5Cright%5Crfloor +[ctz-formula7]: https://latex.codecogs.com/svg.latex?%5Cmathit%7Boff%7D%20%3D%20N%20-%20%5Cleft%28B-2%5Cfrac%7Bw%7D%7B8%7D%5Cright%29n%20-%20%5Cfrac%7Bw%7D%7B8%7D%5Ctext%7Bpopcount%7D%28n%29 + +[bigB]: https://latex.codecogs.com/svg.latex?B +[d]: https://latex.codecogs.com/svg.latex?d +[m]: https://latex.codecogs.com/svg.latex?m +[bigN]: https://latex.codecogs.com/svg.latex?N +[n]: https://latex.codecogs.com/svg.latex?n +[n']: https://latex.codecogs.com/svg.latex?n%27 +[r]: https://latex.codecogs.com/svg.latex?r +[s]: https://latex.codecogs.com/svg.latex?s +[w]: https://latex.codecogs.com/svg.latex?w +[x]: https://latex.codecogs.com/svg.latex?x + +[metadata-cost-graph]: https://raw.githubusercontent.com/geky/littlefs/gh-images/metadata-cost.svg?sanitize=true +[wear-distribution-graph]: https://raw.githubusercontent.com/geky/littlefs/gh-images/wear-distribution.svg?sanitize=true +[file-cost-graph]: https://raw.githubusercontent.com/geky/littlefs/gh-images/file-cost.svg?sanitize=true diff --git a/README.md b/README.md index d8c2c7ba..22f4c23f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## The little filesystem +## littlefs -A little fail-safe filesystem designed for embedded systems. +A little fail-safe filesystem designed for microcontrollers. ``` | | | .---._____ @@ -11,17 +11,19 @@ A little fail-safe filesystem designed for embedded systems. | | | ``` -**Bounded RAM/ROM** - The littlefs is designed to work with a limited amount -of memory. Recursion is avoided and dynamic memory is limited to configurable -buffers that can be provided statically. +**Power-loss resilience** - littlefs is designed to handle random power +failures. All file operations have strong copy-on-write guarantees and if +power is lost the filesystem will fall back to the last known good state. -**Power-loss resilient** - The littlefs is designed for systems that may have -random power failures. The littlefs has strong copy-on-write guarantees and -storage on disk is always kept in a valid state. +**Dynamic wear leveling** - littlefs is designed with flash in mind, and +provides wear leveling over dynamic blocks. Additionally, littlefs can +detect bad blocks and work around them. -**Wear leveling** - Since the most common form of embedded storage is erodible -flash memories, littlefs provides a form of dynamic wear leveling for systems -that can not fit a full flash translation layer. +**Bounded RAM/ROM** - littlefs is designed to work with a small amount of +memory. RAM usage is strictly bounded, which means RAM consumption does not +change as the filesystem grows. The filesystem contains no unbounded +recursion and dynamic memory is limited to configurable buffers that can be +provided statically. ## Example @@ -91,11 +93,11 @@ int main(void) { Detailed documentation (or at least as much detail as is currently available) can be found in the comments in [lfs.h](lfs.h). -As you may have noticed, littlefs takes in a configuration structure that -defines how the filesystem operates. The configuration struct provides the -filesystem with the block device operations and dimensions, tweakable -parameters that tradeoff memory usage for performance, and optional -static buffers if the user wants to avoid dynamic memory. +littlefs takes in a configuration structure that defines how the filesystem +operates. The configuration struct provides the filesystem with the block +device operations and dimensions, tweakable parameters that tradeoff memory +usage for performance, and optional static buffers if the user wants to avoid +dynamic memory. The state of the littlefs is stored in the `lfs_t` type which is left up to the user to allocate, allowing multiple filesystems to be in use @@ -107,14 +109,14 @@ directory functions, with the deviation that the allocation of filesystem structures must be provided by the user. All POSIX operations, such as remove and rename, are atomic, even in event -of power-loss. Additionally, no file updates are actually committed to the -filesystem until sync or close is called on the file. +of power-loss. Additionally, no file updates are not actually committed to +the filesystem until sync or close is called on the file. ## Other notes -All littlefs have the potential to return a negative error code. The errors -can be either one of those found in the `enum lfs_error` in [lfs.h](lfs.h), -or an error returned by the user's block device operations. +All littlefs functions have the potential to return a negative error code. The +errors can be either one of those found in the `enum lfs_error` in +[lfs.h](lfs.h), or an error returned by the user's block device operations. In the configuration struct, the `prog` and `erase` function provided by the user may return a `LFS_ERR_CORRUPT` error if the implementation already can @@ -128,14 +130,60 @@ from memory, otherwise data integrity can not be guaranteed. If the `write` function does not perform caching, and therefore each `read` or `write` call hits the memory, the `sync` function can simply return 0. -## Reference material +## Design -[DESIGN.md](DESIGN.md) - DESIGN.md contains a fully detailed dive into how -littlefs actually works. I would encourage you to read it since the -solutions and tradeoffs at work here are quite interesting. +At a high level, littlefs is a block based filesystem that uses small logs to +store metadata and larger copy-on-write (COW) structures to store file data. -[SPEC.md](SPEC.md) - SPEC.md contains the on-disk specification of littlefs -with all the nitty-gritty details. Can be useful for developing tooling. +In littlefs, these ingredients form a sort of two-layered cake, with the small +logs (called metadata pairs) providing fast updates to metadata anywhere on +storage, while the COW structures store file data compactly and without any +wear amplification cost. + +Both of these data structures are built out of blocks, which are fed by a +common block allocator. By limiting the number of erases allowed on a block +per allocation, the allocator provides dynamic wear leveling over the entire +filesystem. + +``` + root + .--------.--------. + | A'| B'| | + | | |-> | + | | | | + '--------'--------' + .----' '--------------. + A v B v + .--------.--------. .--------.--------. + | C'| D'| | | E'|new| | + | | |-> | | | E'|-> | + | | | | | | | | + '--------'--------' '--------'--------' + .-' '--. | '------------------. + v v .-' v +.--------. .--------. v .--------. +| C | | D | .--------. write | new E | +| | | | | E | ==> | | +| | | | | | | | +'--------' '--------' | | '--------' + '--------' .-' | + .-' '-. .-------------|------' + v v v v + .--------. .--------. .--------. + | F | | G | | new F | + | | | | | | + | | | | | | + '--------' '--------' '--------' +``` + +More details on how littlefs works can be found in [DESIGN.md](DESIGN.md) and +[SPEC.md](SPEC.md). + +- [DESIGN.md](DESIGN.md) - A fully detailed dive into how littlefs works. + I would suggest reading it as the tradeoffs at work are quite interesting. + +- [SPEC.md](SPEC.md) - The on-disk specification of littlefs with all the + nitty-gritty details. May be useful for tooling development. ## Testing @@ -149,9 +197,9 @@ make test ## License -The littlefs is provided under the [BSD-3-Clause](https://spdx.org/licenses/BSD-3-Clause.html) -license. See [LICENSE.md](LICENSE.md) for more information. Contributions to -this project are accepted under the same license. +The littlefs is provided under the [BSD-3-Clause] license. See +[LICENSE.md](LICENSE.md) for more information. Contributions to this project +are accepted under the same license. Individual files contain the following tag instead of the full license text. @@ -162,32 +210,39 @@ License Identifiers that are here available: http://spdx.org/licenses/ ## Related projects -[Mbed OS](https://github.com/ARMmbed/mbed-os/tree/master/features/filesystem/littlefs) - -The easiest way to get started with littlefs is to jump into [Mbed](https://os.mbed.com/), -which already has block device drivers for most forms of embedded storage. The -littlefs is available in Mbed OS as the [LittleFileSystem](https://os.mbed.com/docs/latest/reference/littlefilesystem.html) -class. - -[littlefs-fuse](https://github.com/geky/littlefs-fuse) - A [FUSE](https://github.com/libfuse/libfuse) -wrapper for littlefs. The project allows you to mount littlefs directly on a -Linux machine. Can be useful for debugging littlefs if you have an SD card -handy. - -[littlefs-js](https://github.com/geky/littlefs-js) - A javascript wrapper for -littlefs. I'm not sure why you would want this, but it is handy for demos. -You can see it in action [here](http://littlefs.geky.net/demo.html). - -[mklfs](https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src) - -A command line tool built by the [Lua RTOS](https://github.com/whitecatboard/Lua-RTOS-ESP32) -guys for making littlefs images from a host PC. Supports Windows, Mac OS, -and Linux. - -[SPIFFS](https://github.com/pellepl/spiffs) - Another excellent embedded -filesystem for NOR flash. As a more traditional logging filesystem with full -static wear-leveling, SPIFFS will likely outperform littlefs on small -memories such as the internal flash on microcontrollers. - -[Dhara](https://github.com/dlbeer/dhara) - An interesting NAND flash -translation layer designed for small MCUs. It offers static wear-leveling and -power-resilience with only a fixed O(|address|) pointer structure stored on -each block and in RAM. +- [littlefs-fuse] - A [FUSE] wrapper for littlefs. The project allows you to + mount littlefs directly on a Linux machine. Can be useful for debugging + littlefs if you have an SD card handy. + +- [littlefs-js] - A javascript wrapper for littlefs. I'm not sure why you would + want this, but it is handy for demos. You can see it in action + [here][littlefs-js-demo]. + +- [mklfs] - A command line tool built by the [Lua RTOS] guys for making + littlefs images from a host PC. Supports Windows, Mac OS, and Linux. + +- [Mbed OS] - The easiest way to get started with littlefs is to jump into Mbed + which already has block device drivers for most forms of embedded storage. + littlefs is available in Mbed OS as the [LittleFileSystem] class. + +- [SPIFFS] - Another excellent embedded filesystem for NOR flash. As a more + traditional logging filesystem with full static wear-leveling, SPIFFS will + likely outperform littlefs on small memories such as the internal flash on + microcontrollers. + +- [Dhara] - An interesting NAND flash translation layer designed for small + MCUs. It offers static wear-leveling and power-resilience with only a fixed + _O(|address|)_ pointer structure stored on each block and in RAM. + + +[BSD-3-Clause]: https://spdx.org/licenses/BSD-3-Clause.html +[littlefs-fuse]: https://github.com/geky/littlefs-fuse +[FUSE]: https://github.com/libfuse/libfuse +[littlefs-js]: https://github.com/geky/littlefs-js +[littlefs-js-demo]:http://littlefs.geky.net/demo.html +[mklfs]: https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src +[Lua RTOS]: https://github.com/whitecatboard/Lua-RTOS-ESP32 +[Mbed OS]: https://github.com/armmbed/mbed-os +[LittleFileSystem]: https://os.mbed.com/docs/mbed-os/v5.12/apis/littlefilesystem.html +[SPIFFS]: https://github.com/pellepl/spiffs +[Dhara]: https://github.com/dlbeer/dhara diff --git a/SPEC.md b/SPEC.md index 8949204f..e6622d38 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,9 +1,9 @@ -## The little filesystem technical specification +## littlefs technical specification This is the technical specification of the little filesystem. This document covers the technical details of how the littlefs is stored on disk for -introspection and tool development. This document assumes you are familiar -with the design of the littlefs, for more info on how littlefs works check +introspection and tooling. This document assumes you are familiar with the +design of the littlefs, for more info on how littlefs works check out [DESIGN.md](DESIGN.md). ``` @@ -18,22 +18,21 @@ out [DESIGN.md](DESIGN.md). ## Some quick notes - littlefs is a block-based filesystem. The disk is divided into an array of - evenly sized blocks that are used as the logical unit of storage. Block - pointers are stored in 32 bits. + evenly sized blocks that are used as the logical unit of storage. + +- Block pointers are stored in 32 bits, with the special value `0xffffffff` + representing a null block address. - In addition to the logical block size (which usually matches the erase block size), littlefs also uses a program block size and read block size. - These determine the alignment of block device operations, but aren't needed - for portability. - -- By default, any values in littlefs are stored in little-endian byte order. + These determine the alignment of block device operations, but don't need + to be consistent for portability. -- The littlefs uses the value of `0xffffffff` to represent a null - block address. +- By default, all values in littlefs are stored in little-endian byte order. ## Directories / Metadata pairs -Metadata pairs form the backbone of the littlefs and provide a system for +Metadata pairs form the backbone of littlefs and provide a system for distributed atomic updates. Even the superblock is stored in a metadata pair. As their name suggests, a metadata pair is stored in two blocks, with one block @@ -91,14 +90,14 @@ alignment. Metadata block fields: -- **Revision count (32-bits)** - Incremented every erase cycle. If both blocks - contain valid commits, only the block with the most recent revision count - should be used. Sequence comparison must be used to avoid issues with - integer overflow. +1. **Revision count (32-bits)** - Incremented every erase cycle. If both blocks + contain valid commits, only the block with the most recent revision count + should be used. Sequence comparison must be used to avoid issues with + integer overflow. -- **CRC (32-bits)** - Detects corruption from power-loss or other write - issues. Uses a CRC-32 with a polynomial of `0x04c11db7` initialized - with `0xffffffff`. +2. **CRC (32-bits)** - Detects corruption from power-loss or other write + issues. Uses a CRC-32 with a polynomial of `0x04c11db7` initialized + with `0xffffffff`. Entries themselves are stored as a 32-bit tag followed by a variable length blob of data. But exactly how these tags are stored is a little bit tricky. @@ -159,7 +158,7 @@ Here's a more complete example of metadata block containing 4 entries: | | | || | | |-------------------+-------------------| || | | | tag CxCRC | CRC | || / -| |-------------------+-------------------| || +| |-------------------+-------------------| || | | tag CRCxA' | data A' | || \ | |-------------------+ | || | | | | || | @@ -167,7 +166,7 @@ Here's a more complete example of metadata block containing 4 entries: | | | tag CRCxA' | | || | | |--------------+-------------------+----| || | | | CRC | padding | || / -| |--------------+----+-------------------| || +| |--------------+----+-------------------| || | | tag CRCxA'' | data A'' | <---. \ | |-------------------+ | ||| | | | | ||| | @@ -179,12 +178,12 @@ Here's a more complete example of metadata block containing 4 entries: | | | tag Dx| |||| | | |---------+-------------------+---------| |||| | | |CRC | CRC | | |||| / -| |---------+-------------------+ | |||| +| |---------+-------------------+ | |||| | | unwritten storage | |||| more commits | | | |||| | -| | | |||| v -| | | |||| -| | | |||| +| | | |||| v +| | | |||| +| | | |||| | '---------------------------------------' |||| '---------------------------------------' |||'- most recent A ||'-- most recent B @@ -198,7 +197,7 @@ So in littlefs, 32-bit tags describe every type of metadata. And this means _every_ type of metadata, including file entries, directory fields, and global state. Even the CRCs used to mark the end of commits get their own tag. -Because of this, the tag format contains some densely packed informtaion. Note +Because of this, the tag format contains some densely packed information. Note that there are multiple levels of types which break down into more info: ``` @@ -214,9 +213,9 @@ that there are multiple levels of types which break down into more info: ``` -Before we go further, there's one VERY important thing to note. These tags are -NOT stored in little-endian. Tags stored in commits are actually stored in -big-endian (and is the only thing in littlefs stored in big-endian). This +Before we go further, there's one important thing to note. These tags are +**not** stored in little-endian. Tags stored in commits are actually stored +in big-endian (and is the only thing in littlefs stored in big-endian). This little bit of craziness comes from the fact that the valid bit must be the first bit in a commit, and when converted to little-endian, the valid bit finds itself in byte 4. We could restructure the tag to store the valid bit lower, @@ -228,26 +227,26 @@ invalid and can be used for null values. Metadata tag fields: -- **Valid bit (1-bit)** - Indicates if the tag is valid. +1. **Valid bit (1-bit)** - Indicates if the tag is valid. -- **Type3 (11-bits)** - Type of the tag. This field is broken down further - into a 3-bit abstract type and an 8-bit chunk field. Note that the value - `0x000` is invalid and not assigned a type. +2. **Type3 (11-bits)** - Type of the tag. This field is broken down further + into a 3-bit abstract type and an 8-bit chunk field. Note that the value + `0x000` is invalid and not assigned a type. -- **Type1 (3-bits)** - Abstract type of the tag. Groups the tags into - 8 categories that facilitate bitmasked lookups. +3. **Type1 (3-bits)** - Abstract type of the tag. Groups the tags into + 8 categories that facilitate bitmasked lookups. -- **Chunk (8-bits)** - Chunk field used for various purposes by the different - abstract types. type1+chunk+id form a unique identifier for each tag in the - metadata block. +4. **Chunk (8-bits)** - Chunk field used for various purposes by the different + abstract types. type1+chunk+id form a unique identifier for each tag in the + metadata block. -- **Id (10-bits)** - File id associated with the tag. Each file in a metadata - block gets a unique id which is used to associate tags with that file. The - special value `0x3ff` is used for any tags that are not associated with a - file, such as directory and global metadata. +5. **Id (10-bits)** - File id associated with the tag. Each file in a metadata + block gets a unique id which is used to associate tags with that file. The + special value `0x3ff` is used for any tags that are not associated with a + file, such as directory and global metadata. -- **Length (10-bits)** - Length of the data in bytes. The special value - `0x3ff` indicates that this tag has been deleted. +6. **Length (10-bits)** - Length of the data in bytes. The special value + `0x3ff` indicates that this tag has been deleted. ## Metadata types @@ -274,7 +273,7 @@ array of files. --- #### `0x0xx` LFS_TYPE_NAME -Associates the id with a file name and file type. +Associates the id with a file name and file type. The data contains the file name stored as an ASCII string (may be expanded to UTF8 in the future). @@ -300,9 +299,9 @@ Layout of the name tag: Name fields: -- **file type (8-bits)** - Type of the file. +1. **file type (8-bits)** - Type of the file. -- **file name** - File name stored as an ASCII string. +2. **file name** - File name stored as an ASCII string. --- #### `0x001` LFS_TYPE_REG @@ -335,14 +334,15 @@ across a linked-list of metadata pairs rooted on the blocks 0 and 1. The last metadata pair doubles as the root directory of the filesystem. ``` -.--------. .--------. .--------. .--------. .--------. -| super |->| super |->| super |->| super |->| file B | -| block | | block | | block | | block | | file C | -| | | | | | | file A | | file D | + .--------. .--------. .--------. .--------. .--------. +.| super |->| super |->| super |->| super |->| file B | +|| block | || block | || block | || block | || file C | +|| | || | || | || file A | || file D | +|'--------' |'--------' |'--------' |'--------' |'--------' '--------' '--------' '--------' '--------' '--------' -\---------------+----------------/ \---------+---------/ - superblock pairs root directory +\----------------+----------------/ \----------+----------/ + superblock pairs root directory ``` The filesystem starts with only the root directory. The superblock metadata @@ -366,48 +366,41 @@ Layout of the superblock name tag and inline-struct tag: '----------------- valid bit tag data -[-- 32 --][-- 32 --|-- 32 --| -[1|- 11 -| 10 | 10 ][-- 32 --|-- 32 --| - ^ ^ ^ ^ ^- version ^- block size +[-- 32 --][-- 32 --|-- 32 --|-- 32 --] +[1|- 11 -| 10 | 10 ][-- 32 --|-- 32 --|-- 32 --] + ^ ^ ^ ^ ^- version ^- block size ^- block count + | | | | [-- 32 --|-- 32 --|-- 32 --] + | | | | [-- 32 --|-- 32 --|-- 32 --] + | | | | ^- name max ^- file max ^- attr max | | | '- size (24) | | '------ id (0) | '------------ type (0x201) '----------------- valid bit - - data (cont) -|-- 32 --|-- 32 --|-- 32 --| -|-- 32 --|-- 32 --|-- 32 --| - ^- block count ^- name max ^- file max - - data (cont) -|-- 32 --] -|-- 32 --] - ^- attr max ``` Superblock fields: -- **Magic string (8-bytes)** - Magic string indicating the presence of littlefs - on the device. Must be the string "littlefs". +1. **Magic string (8-bytes)** - Magic string indicating the presence of + littlefs on the device. Must be the string "littlefs". -- **Version (32-bits)** - The version of littlefs at format time. The version - is encoded in a 32-bit value with the upper 16-bits containing the major - version, and the lower 16-bits containing the minor version. +2. **Version (32-bits)** - The version of littlefs at format time. The version + is encoded in a 32-bit value with the upper 16-bits containing the major + version, and the lower 16-bits containing the minor version. - This specification describes version 2.0 (`0x00020000`). + This specification describes version 2.0 (`0x00020000`). -- **Block size (32-bits)** - Size of the logical block size used by the - filesystem in bytes. +3. **Block size (32-bits)** - Size of the logical block size used by the + filesystem in bytes. -- **Block count (32-bits)** - Number of blocks in the filesystem. +4. **Block count (32-bits)** - Number of blocks in the filesystem. -- **Name max (32-bits)** - Maximum size of file names in bytes. +5. **Name max (32-bits)** - Maximum size of file names in bytes. -- **File max (32-bits)** - Maximum size of files in bytes. +6. **File max (32-bits)** - Maximum size of files in bytes. -- **Attr max (32-bits)** - Maximum size of file attributes in bytes. +7. **Attr max (32-bits)** - Maximum size of file attributes in bytes. -The superblock must always be the first entry (id 0) in a metdata pair as well +The superblock must always be the first entry (id 0) in a metadata pair as well as be the first entry written to the block. This means that the superblock entry can be read from a device using offsets alone. @@ -419,7 +412,7 @@ Associates the id with an on-disk data structure. The exact layout of the data depends on the data structure type stored in the chunk field and can be one of the following. -Any type of struct supercedes all other structs associated with the id. For +Any type of struct supersedes all other structs associated with the id. For example, appending a ctz-struct replaces an inline-struct on the same file. --- @@ -431,12 +424,13 @@ Directories in littlefs are stored on disk as a linked-list of metadata pairs, each pair containing any number of files in alphabetical order. ``` - | - v -.--------. .--------. .--------. .--------. .--------. .--------. -| file A |->| file D |->| file G |->| file I |->| file J |->| file M | -| file B | | file E | | file H | | | | file K | | file N | -| file C | | file F | | | | | | file L | | | + | + v + .--------. .--------. .--------. .--------. .--------. .--------. +.| file A |->| file D |->| file G |->| file I |->| file J |->| file M | +|| file B | || file E | || file H | || | || file K | || file N | +|| file C | || file F | || | || | || file L | || | +|'--------' |'--------' |'--------' |'--------' |'--------' |'--------' '--------' '--------' '--------' '--------' '--------' '--------' ``` @@ -460,15 +454,15 @@ Layout of the dir-struct tag: Dir-struct fields: -- **Metadata pair (8-bytes)** - Pointer to the first metadata-pair - in the directory. +1. **Metadata pair (8-bytes)** - Pointer to the first metadata-pair + in the directory. --- #### `0x201` LFS_TYPE_INLINESTRUCT Gives the id an inline data structure. -Inline structs store small files that can fit in the metdata pair. In this +Inline structs store small files that can fit in the metadata pair. In this case, the file data is stored directly in the tag's data area. Layout of the inline-struct tag: @@ -485,7 +479,7 @@ Layout of the inline-struct tag: Inline-struct fields: -- **Inline data** - File data stored directly in the metadata-pair. +1. **Inline data** - File data stored directly in the metadata-pair. --- #### `0x202` LFS_TYPE_CTZSTRUCT @@ -497,12 +491,13 @@ are stored in a skip-list in reverse, with a pointer to the head of the skip-list. Note that the head of the skip-list and the file size is enough information to read the file. -How exactly CTZ skip-lists work is a bit complicted. A full explanation can be +How exactly CTZ skip-lists work is a bit complicated. A full explanation can be found in the [DESIGN.md](DESIGN.md#ctz-skip-lists). -A quick summary: For every nth block where n is divisible by 2^x, the block -contains a pointer to block n-2^x. These pointers are stored in increasing -order of x in each block of the file before the actual data. +A quick summary: For every _n_‍th block where _n_ is divisible by +2‍_ˣ_, that block contains a pointer to block _n_-2‍_ˣ_. +These pointers are stored in increasing order of _x_ in each block of the file +before the actual data. ``` | @@ -536,15 +531,15 @@ Layout of the CTZ-struct tag: CTZ-struct fields: -- **File head (32-bits)** - Pointer to the block that is the head of the - file's CTZ skip-list. +1. **File head (32-bits)** - Pointer to the block that is the head of the + file's CTZ skip-list. -- **File size (32-bits)** - Size of the file in bytes. +2. **File size (32-bits)** - Size of the file in bytes. --- #### `0x3xx` LFS_TYPE_USERATTR -Attaches a user attribute to an id. +Attaches a user attribute to an id. littlefs has a concept of "user attributes". These are small user-provided attributes that can be used to store things like timestamps, hashes, @@ -571,9 +566,9 @@ Layout of the user-attr tag: User-attr fields: -- **Attr type (8-bits)** - Type of the user attributes. +1. **Attr type (8-bits)** - Type of the user attributes. -- **Attr data** - The data associated with the user attribute. +2. **Attr data** - The data associated with the user attribute. --- #### `0x6xx` LFS_TYPE_TAIL @@ -586,21 +581,23 @@ which indicates if the following metadata pair is a part of the directory (hard-tail) or only used to traverse the filesystem (soft-tail). ``` - .--------. - | dir A |-. - |softtail| | + .--------. + .| dir A |-. + ||softtail| | .--------| |-' -| '--------' +| |'--------' +| '---|--|-' | .-' '-------------. | v v | .--------. .--------. .--------. '->| dir B |->| dir B |->| dir C | - |hardtail| |softtail| | | - | | | | | | - '--------' '--------' '--------' + ||hardtail| ||softtail| || | + || | || | || | + |'--------' |'--------' |'--------' + '--------' '--------' '--------' ``` -Currently any type supercedes any other preceding tails in the metadata pair, +Currently any type supersedes any other preceding tails in the metadata pair, but this may change if additional metadata pair state is added. A note about the metadata pair linked-list: Normally, this linked-list contains @@ -611,10 +608,10 @@ exactly this flag is stored is described below. When the sync flag is set: -- The linked-list may contain an orphaned directory that has been removed in - the filesystem. -- The linked-list may contain a metadata pair with a bad block that has been - replaced in the filesystem. +1. The linked-list may contain an orphaned directory that has been removed in + the filesystem. +2. The linked-list may contain a metadata pair with a bad block that has been + replaced in the filesystem. If the sync flag is set, the threaded linked-list must be checked for these errors before it can be used reliably. Note that the threaded linked-list can @@ -635,9 +632,9 @@ Layout of the tail tag: Tail fields: -- **Tail type (8-bits)** - Type of the tail pointer. +1. **Tail type (8-bits)** - Type of the tail pointer. -- **Metadata pair (8-bytes)** - Pointer to the next metadata-pair. +2. **Metadata pair (8-bytes)** - Pointer to the next metadata-pair. --- #### `0x600` LFS_TYPE_SOFTTAIL @@ -668,18 +665,18 @@ littlefs has a concept of "global state". This is a small set of state that can be updated by a commit to _any_ metadata pair in the filesystem. The way this works is that the global state is stored as a set of deltas -distributed across the filesystem such that the global state can by found by +distributed across the filesystem such that the global state can be found by the xor-sum of these deltas. ``` -.--------. .--------. .--------. .--------. .--------. -| |->| gstate |->| |->| gstate |->| gstate | -| | | 0x23 | | | | 0xff | | 0xce | -| | | | | | | | | | -'--------' '--------' '--------' '--------' '--------' - | | | - v v v - 0x00 --> xor ------------------> xor ------> xor --> gstate 0x12 + .--------. .--------. .--------. .--------. .--------. +.| |->| gdelta |->| |->| gdelta |->| gdelta | +|| | || 0x23 | || | || 0xff | || 0xce | +|| | || | || | || | || | +|'--------' |'--------' |'--------' |'--------' |'--------' +'--------' '----|---' '--------' '----|---' '----|---' + v v v + 0x00 --> xor ------------------> xor ------> xor --> gstate = 0x12 ``` Note that storing globals this way is very expensive in terms of storage usage, @@ -730,17 +727,17 @@ Layout of the move state: Move state fields: -- **Sync bit (1-bit)** - Indicates if the metadata pair threaded linked-list is - in-sync. If set, the threaded linked-list should be checked for errors. +1. **Sync bit (1-bit)** - Indicates if the metadata pair threaded linked-list + is in-sync. If set, the threaded linked-list should be checked for errors. -- **Move type (11-bits)** - Type of move being performed. Must be either - `0x000`, indicating no move, or `0x4ff` indicating the source file should - be deleted. +2. **Move type (11-bits)** - Type of move being performed. Must be either + `0x000`, indicating no move, or `0x4ff` indicating the source file should + be deleted. -- **Move id (10-bits)** - The file id being moved. +3. **Move id (10-bits)** - The file id being moved. -- **Metadata pair (8-bytes)** - Pointer to the metadata-pair containing - the move. +4. **Metadata pair (8-bytes)** - Pointer to the metadata-pair containing + the move. --- #### `0x5xx` LFS_TYPE_CRC @@ -778,13 +775,13 @@ Layout of the CRC tag: CRC fields: -- **Valid state (1-bit)** - Indicates the expected value of the valid bit for - any tags in the next commit. +1. **Valid state (1-bit)** - Indicates the expected value of the valid bit for + any tags in the next commit. -- **CRC (32-bits)** - CRC-32 with a polynomial of `0x04c11db7` initialized with - `0xffffffff`. +2. **CRC (32-bits)** - CRC-32 with a polynomial of `0x04c11db7` initialized + with `0xffffffff`. -- **Padding** - Padding to the next program-aligned boundary. No guarantees are - made about the contents. +3. **Padding** - Padding to the next program-aligned boundary. No guarantees + are made about the contents. --- From 9568f8ee2da496b1f3b7372baa735ba61c46fbae Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 1 Apr 2019 22:12:08 -0500 Subject: [PATCH 134/139] Added v1->v2 migration into CI Also fixed issue where migration would not handle large dirs due to v1 iteration changing the pair of the directory. --- .travis.yml | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- lfs.c | 9 +++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index aa9663d0..572037c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -130,7 +130,58 @@ jobs: - ls -flh - make -B test_dirs test_files QUIET=1 - # Automatically update releases + # self-host with littlefs-fuse for fuzz test + - stage: test + env: + - STAGE=test + - NAME=littlefs-migration + install: + - sudo apt-get install libfuse-dev + - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha v2 + - git clone --depth 1 https://github.com/geky/littlefs-fuse v1 + - fusermount -V + - gcc --version + before_script: + # setup disk for littlefs-fuse + - rm -rf v2/littlefs/* + - cp -r $(git ls-tree --name-only HEAD) v2/littlefs + + - mkdir mount + - sudo chmod a+rw /dev/loop0 + - dd if=/dev/zero bs=512 count=4096 of=disk + - losetup /dev/loop0 disk + script: + # compile v1 and v2 + - make -C v1 + - make -C v2 + + # run self-host test with v1 + - v1/lfs --format /dev/loop0 + - v1/lfs /dev/loop0 mount + + - ls mount + - mkdir mount/littlefs + - cp -r $(git ls-tree --name-only HEAD) mount/littlefs + - cd mount/littlefs + - stat . + - ls -flh + - make -B test_dirs test_files QUIET=1 + + # attempt to migrate + - cd ../.. + - fusermount -u mount + + - v2/lfs --migrate /dev/loop0 + - v2/lfs /dev/loop0 mount + + # run self-host test with v2 right where we left off + - ls mount + - cd mount/littlefs + - stat . + - ls -flh + - make -B test_dirs test_files QUIET=1 + + # Automatically create releases - stage: deploy env: - STAGE=deploy diff --git a/lfs.c b/lfs.c index c0867187..7c51117c 100644 --- a/lfs.c +++ b/lfs.c @@ -4268,6 +4268,8 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { } dir2.rev = dir1.d.rev; + dir1.head[0] = dir1.pair[0]; + dir1.head[1] = dir1.pair[1]; lfs->root[0] = dir2.pair[0]; lfs->root[1] = dir2.pair[1]; @@ -4368,7 +4370,10 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { // Copy over first block to thread into fs. Unfortunately // if this fails there is not much we can do. - err = lfs_bd_erase(lfs, dir1.pair[1]); + LFS_DEBUG("Migrating %"PRIu32" %"PRIu32" -> %"PRIu32" %"PRIu32, + lfs->root[0], lfs->root[1], dir1.head[0], dir1.head[1]); + + err = lfs_bd_erase(lfs, dir1.head[1]); if (err) { goto cleanup; } @@ -4389,7 +4394,7 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { err = lfs_bd_prog(lfs, &lfs->pcache, &lfs->rcache, true, - dir1.pair[1], i, &dat, 1); + dir1.head[1], i, &dat, 1); if (err) { goto cleanup; } From 0b76635f10664e74dda365b8788b8356501fd77a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 9 Apr 2019 16:06:43 -0500 Subject: [PATCH 135/139] Added better handling of large program sizes (> 1024) The issue here is how commits handle padding to the nearest program size. This is done by exploiting the size field of the LFS_TYPE_CRC tag that completes the commit. Unfortunately, during developement, the size field shrank in size to make room for more type information, limiting the size field to 1024. Normally this isn't a problem, as very rarely do program sizes exceed 1024 bytes. However, using a simulated block device, user earlephilhower found that exceeding 1024 caused littlefs to crash. To make this corner case behave in a more user friendly manner, I've modified this situtation to treat >1024 program sizes as small commits that don't match the prog size. As a part of this, littlefs also needed to understand that non-matching commits indicate an "unerased" dir block, which would be needed for portability (something which notably lacks testing). This raises the question of if the tag size field size needs to be reconsidered, but to change that at this point would need a new major version. found by earlephilhower --- lfs.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lfs.c b/lfs.c index 7c51117c..8ef03e2d 100644 --- a/lfs.c +++ b/lfs.c @@ -827,7 +827,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // next commit not yet programmed or we're not in valid range if (!lfs_tag_isvalid(tag) || off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { - dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC); + dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC && + dir->off % lfs->cfg->prog_size == 0); break; } @@ -1596,7 +1597,7 @@ static int lfs_dir_compact(lfs_t *lfs, dir->count = end - begin; dir->off = commit.off; dir->etag = commit.ptag; - dir->erased = true; + dir->erased = (dir->off % lfs->cfg->prog_size == 0); // note we able to have already handled move here if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) { lfs_gstate_xormove(&lfs->gpending, @@ -2381,7 +2382,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, if (file->ctz.size > 0) { lfs_stag_t res = lfs_dir_get(lfs, &file->m, LFS_MKTAG(0x700, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->cache.size), + LFS_MKTAG(LFS_TYPE_STRUCT, file->id, + lfs_min(file->cache.size, 0x3fe)), file->cache.buffer); if (res < 0) { err = res; From c2c2ce6b97d96ab1e13008392188e5a0360dd85a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 9 Apr 2019 17:41:26 -0500 Subject: [PATCH 136/139] Fixed issue with handling block device errors in lfs_file_sync lfs_file_sync was not correctly setting the LFS_F_ERRED flag. Fortunately this is a relatively easy fix. LFS_F_ERRED prevents further issues from occuring when cleaning up resources with lfs_file_close. found by TheLoneWolfling --- lfs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lfs.c b/lfs.c index 8ef03e2d..639e0160 100644 --- a/lfs.c +++ b/lfs.c @@ -2578,6 +2578,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { while (true) { int err = lfs_file_flush(lfs, file); if (err) { + file->flags |= LFS_F_ERRED; return err; } @@ -2613,6 +2614,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) { goto relocate; } + file->flags |= LFS_F_ERRED; return err; } @@ -2626,6 +2628,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { file->off = file->pos; err = lfs_file_relocate(lfs, file); if (err) { + file->flags |= LFS_F_ERRED; return err; } } From 1ff643229817ce1ec7bebd3bbf99f2dc4f2ab381 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 9 Apr 2019 18:07:44 -0500 Subject: [PATCH 137/139] Added clarification on buffer alignment. In v2, the lookahead_buffer was changed from requiring 4-byte alignment to requiring 8-byte alignment. This was not documented as well as it could be, and as FabianInostroza noted, this also implies that lfs_malloc must provide 8-byte alignment. To protect against this, I've also added an assert on the alignment of both the lookahead_size and lookahead_buffer. found by FabianInostroza and amitv87 --- lfs.c | 3 ++- lfs.h | 5 +++-- lfs_util.h | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lfs.c b/lfs.c index 639e0160..57a51de9 100644 --- a/lfs.c +++ b/lfs.c @@ -3228,8 +3228,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs_cache_zero(lfs, &lfs->pcache); // setup lookahead, must be multiple of 64-bits - LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0); LFS_ASSERT(lfs->cfg->lookahead_size > 0); + LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0 && + (uintptr_t)lfs->cfg->lookahead_buffer % 8 == 0); if (lfs->cfg->lookahead_buffer) { lfs->free.buffer = lfs->cfg->lookahead_buffer; } else { diff --git a/lfs.h b/lfs.h index f797f667..ae991e07 100644 --- a/lfs.h +++ b/lfs.h @@ -215,8 +215,9 @@ struct lfs_config { // By default lfs_malloc is used to allocate this buffer. void *prog_buffer; - // Optional statically allocated program buffer. Must be lookahead_size. - // By default lfs_malloc is used to allocate this buffer. + // Optional statically allocated lookahead buffer. Must be lookahead_size + // and aligned to a 64-bit boundary. By default lfs_malloc is used to + // allocate this buffer. void *lookahead_buffer; // Optional upper limit on length of file names in bytes. No downside for diff --git a/lfs_util.h b/lfs_util.h index 1dc3b0fd..28b14005 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -192,6 +192,7 @@ static inline uint32_t lfs_tobe32(uint32_t a) { uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size); // Allocate memory, only used if buffers are not provided to littlefs +// Note, memory must be 64-bit aligned static inline void *lfs_malloc(size_t size) { #ifndef LFS_NO_MALLOC return malloc(size); From 651e14e79683be0cab30b91858018c19159968c1 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 9 Apr 2019 18:37:53 -0500 Subject: [PATCH 138/139] Cleaned up a couple of warnings - Shifting signed 32-bit value by 31 bits is undefined behaviour This was an interesting one as on initial inspection, `uint8_t & 1` looks like it will result in an unsigned variable. However, due to uint8_t being "smaller" than int, this actually results in a signed int, causing an undefined shift operation. - Identical inner 'if' condition is always true (outer condition is 'true' and inner condition is 'true'). This was caused by the use of `if (true) {` to avoid "goto bypasses variable initialization" warnings. Using just `{` instead seems to avoid this problem. found by keck-in-space and armandas --- lfs.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lfs.c b/lfs.c index 57a51de9..96a78693 100644 --- a/lfs.c +++ b/lfs.c @@ -855,7 +855,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, } // reset the next bit if we need to - ptag ^= (lfs_tag_chunk(tag) & 1) << 31; + ptag ^= (lfs_tag_chunk(tag) & 1U) << 31; // toss our crc into the filesystem seed for // pseudorandom numbers @@ -1498,7 +1498,7 @@ static int lfs_dir_compact(lfs_t *lfs, // begin loop to commit compaction to blocks until a compact sticks while (true) { - if (true) { + { // There's nothing special about our global delta, so feed it into // our local global delta int err = lfs_dir_getgstate(lfs, dir, &lfs->gdelta); @@ -1603,7 +1603,6 @@ static int lfs_dir_compact(lfs_t *lfs, lfs_gstate_xormove(&lfs->gpending, &lfs->gpending, 0x3ff, NULL); } - } break; @@ -2123,7 +2122,7 @@ static int lfs_ctz_extend(lfs_t *lfs, } LFS_ASSERT(nblock >= 2 && nblock <= lfs->cfg->block_count); - if (true) { + { err = lfs_bd_erase(lfs, nblock); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -3298,7 +3297,7 @@ static int lfs_deinit(lfs_t *lfs) { int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { int err = 0; - if (true) { + { err = lfs_init(lfs, cfg); if (err) { return err; @@ -4182,7 +4181,7 @@ static int lfs1_moved(lfs_t *lfs, const void *e) { static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1, const struct lfs_config *cfg) { int err = 0; - if (true) { + { err = lfs_init(lfs, cfg); if (err) { return err; @@ -4253,7 +4252,7 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - if (true) { + { // iterate through each directory, copying over entries // into new directory lfs1_dir_t dir1; From 48bd2bff821ae70e85c82d7f6c6c6bcf2ee23dae Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 9 Apr 2019 18:56:53 -0500 Subject: [PATCH 139/139] Artificially limited number of file ids per metadata block This is an expirement to determine which field in the tag structure is the most critical: tag id or tag size. This came from looking at NAND storage and discussions around behaviour of large prog_sizes. Initial exploration indicates that prog_sizes around 2KiB are not _that_ uncommon, and the 1KiB limitation is surprising. It's possible to increase the lfs_tag size to 12-bits (4096), but at the cost of only 8-bit ids (256). [---- 32 ----] a [1|-3-|-- 8 --|-- 10 --|-- 10 --] b [1|-3-|-- 8 --|-- 8 --|-- 12 --] This requires more investigation, but in order to allow us to change the tag sizes with minimal impact I've artificially limited the number of file ids to 0xfe (255) different file ids per metadata pair. If 12-bit lengths turn out to be a bad idea, we can remove the artificial limit without backwards incompatible changes. To avoid breaking users already on v2-alpha, this change will refuse _creating_ file ids > 255, but should read file ids > 255 without issues. --- lfs.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lfs.c b/lfs.c index 96a78693..25c88c60 100644 --- a/lfs.c +++ b/lfs.c @@ -1437,8 +1437,10 @@ static int lfs_dir_compact(lfs_t *lfs, // space is complicated, we need room for tail, crc, gstate, // cleanup delete, and we cap at half a block to give room // for metadata updates. - if (size <= lfs_min(lfs->cfg->block_size - 36, - lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size))) { + if (end - begin < 0xff && + size <= lfs_min(lfs->cfg->block_size - 36, + lfs_alignup(lfs->cfg->block_size/2, + lfs->cfg->prog_size))) { break; } @@ -1711,7 +1713,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir, } } - if (dir->erased) { + if (dir->erased || dir->count >= 0xff) { // try to commit struct lfs_commit commit = { .block = dir->pair[0],