From d1d22684b7f07a0cedfc3b7c2ecbaf37cc40e22d Mon Sep 17 00:00:00 2001
From: Kirill Kouzoubov <Kirill888@gmail.com>
Date: Tue, 21 Nov 2023 05:59:43 +1100
Subject: [PATCH] Fix nested subdirectory listing #1430 (#1433)

---
 fsspec/implementations/reference.py           | 20 +++++++++++++------
 .../implementations/tests/test_reference.py   | 11 +++++++++-
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/fsspec/implementations/reference.py b/fsspec/implementations/reference.py
index b69e5483e..cfbd39fc1 100644
--- a/fsspec/implementations/reference.py
+++ b/fsspec/implementations/reference.py
@@ -985,16 +985,24 @@ def _dircache_from_items(self):
             elif len(part) == 1:
                 size = None
             else:
-                _, start, size = part
+                _, _, size = part
             par = path.rsplit("/", 1)[0] if "/" in path else ""
             par0 = par
+            subdirs = [par0]
             while par0 and par0 not in self.dircache:
-                # build parent directories
-                self.dircache[par0] = []
-                self.dircache.setdefault(
-                    par0.rsplit("/", 1)[0] if "/" in par0 else "", []
-                ).append({"name": par0, "type": "directory", "size": 0})
+                # collect parent directories
                 par0 = self._parent(par0)
+                subdirs.append(par0)
+
+            subdirs = subdirs[::-1]
+            for parent, child in zip(subdirs, subdirs[1:]):
+                # register newly discovered directories
+                assert child not in self.dircache
+                assert parent in self.dircache
+                self.dircache[parent].append(
+                    {"name": child, "type": "directory", "size": 0}
+                )
+                self.dircache[child] = []
 
             self.dircache[par].append({"name": path, "type": "file", "size": size})
 
diff --git a/fsspec/implementations/tests/test_reference.py b/fsspec/implementations/tests/test_reference.py
index c553b35a0..efdca6598 100644
--- a/fsspec/implementations/tests/test_reference.py
+++ b/fsspec/implementations/tests/test_reference.py
@@ -10,7 +10,6 @@
 
 
 def test_simple(server):  # noqa: F811
-
     refs = {
         "a": b"data",
         "b": (realfile, 0, 5),
@@ -53,6 +52,16 @@ def test_ls(server):  # noqa: F811
     }
 
 
+def test_nested_dirs_ls():
+    # issue #1430
+    refs = {"a": "A", "B/C/b": "B", "B/C/d": "d", "B/_": "_"}
+    fs = fsspec.filesystem("reference", fo=refs)
+    assert len(fs.ls("")) == 2
+    assert set(e["name"] for e in fs.ls("")) == set(["a", "B"])
+    assert len(fs.ls("B")) == 2
+    assert set(e["name"] for e in fs.ls("B")) == set(["B/C", "B/_"])
+
+
 def test_info(server):  # noqa: F811
     refs = {
         "a": b"data",