Resolve conflict

README.md

@@ -4,6 +4,7 @@
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/fsspec/badges/version.svg)](https://anaconda.org/conda-forge/fsspec)
 ![Build](https://github.com/fsspec/filesystem_spec/workflows/CI/badge.svg)
 [![Docs](https://readthedocs.org/projects/filesystem-spec/badge/?version=latest)](https://filesystem-spec.readthedocs.io/en/latest/?badge=latest)
+[![PyPi downloads](https://img.shields.io/pypi/dm/fsspec?label=pypi%20downloads&style=flat)](https://pepy.tech/project/fsspec)
 
 A specification for pythonic filesystems.
 

fsspec/archive.py

@@ -13,7 +13,7 @@ class AbstractArchiveFileSystem(AbstractFileSystem):
     """
 
     def __str__(self):
-        return "<Archive-like object %s at %s>" % (type(self).__name__, id(self))
+        return f"<Archive-like object {type(self).__name__} at {id(self)}>"
 
     __repr__ = __str__
 

fsspec/asyn.py

@@ -426,7 +426,7 @@ async def _process_limits(self, url, start, end):
                 end = ""
             if isinstance(end, numbers.Integral):
                 end -= 1  # bytes range is inclusive
-        return "bytes=%s-%s" % (start, end)
+        return f"bytes={start}-{end}"
 
     async def _cat_file(self, path, start=None, end=None, **kwargs):
         raise NotImplementedError

fsspec/caching.py

@@ -222,8 +222,9 @@ def __init__(self, blocksize, fetcher, size, maxblocks=32):
         self._fetch_block_cached = functools.lru_cache(maxblocks)(self._fetch_block)
 
     def __repr__(self):
-        return "<BlockCache blocksize={}, size={}, nblocks={}>".format(
-            self.blocksize, self.size, self.nblocks
+        return (
+            f"<BlockCache blocksize={self.blocksize}, "
+            f"size={self.size}, nblocks={self.nblocks}>"
         )
 
     def cache_info(self):
@@ -277,9 +278,8 @@ def _fetch_block(self, block_number):
         """
         if block_number > self.nblocks:
             raise ValueError(
-                "'block_number={}' is greater than the number of blocks ({})".format(
-                    block_number, self.nblocks
-                )
+                f"'block_number={block_number}' is greater than "
+                f"the number of blocks ({self.nblocks})"
             )
 
         start = block_number * self.blocksize
@@ -606,8 +606,9 @@ def __init__(self, blocksize, fetcher, size, maxblocks=32):
         self._fetch_future_lock = threading.Lock()
 
     def __repr__(self):
-        return "<BackgroundBlockCache blocksize={}, size={}, nblocks={}>".format(
-            self.blocksize, self.size, self.nblocks
+        return (
+            f"<BackgroundBlockCache blocksize={self.blocksize}, "
+            f"size={self.size}, nblocks={self.nblocks}>"
         )
 
     def cache_info(self):
@@ -719,9 +720,8 @@ def _fetch_block(self, block_number, log_info="sync"):
         """
         if block_number > self.nblocks:
             raise ValueError(
-                "'block_number={}' is greater than the number of blocks ({})".format(
-                    block_number, self.nblocks
-                )
+                f"'block_number={block_number}' is greater than "
+                f"the number of blocks ({self.nblocks})"
             )
 
         start = block_number * self.blocksize

fsspec/compression.py

@@ -39,13 +39,11 @@ def register_compression(name, callback, extensions, force=False):
 
     # Validate registration
     if name in compr and not force:
-        raise ValueError("Duplicate compression registration: %s" % name)
+        raise ValueError(f"Duplicate compression registration: {name}")
 
     for ext in extensions:
         if ext in fsspec.utils.compressions and not force:
-            raise ValueError(
-                "Duplicate compression file extension: %s (%s)" % (ext, name)
-            )
+            raise ValueError(f"Duplicate compression file extension: {ext} ({name})")
 
     compr[name] = callback
 

fsspec/core.py

@@ -92,7 +92,7 @@ def __reduce__(self):
         )
 
     def __repr__(self):
-        return "<OpenFile '{}'>".format(self.path)
+        return f"<OpenFile '{self.path}'>"
 
     def __enter__(self):
         mode = self.mode.replace("t", "").replace("b", "") + "b"
@@ -195,7 +195,7 @@ def __getitem__(self, item):
         return out
 
     def __repr__(self):
-        return "<List of %s OpenFile instances>" % len(self)
+        return f"<List of {len(self)} OpenFile instances>"
 
 
 def open_files(
@@ -498,7 +498,7 @@ def get_compression(urlpath, compression):
     if compression == "infer":
         compression = infer_compression(urlpath)
     if compression is not None and compression not in compr:
-        raise ValueError("Compression type %s not supported" % compression)
+        raise ValueError(f"Compression type {compression} not supported")
     return compression
 
 

fsspec/fuse.py

@@ -275,7 +275,7 @@ def format_help(self):
     for item in args.option or []:
         key, sep, value = item.partition("=")
         if not sep:
-            parser.error(message="Wrong option: {!r}".format(item))
+            parser.error(message=f"Wrong option: {item!r}")
         val = value.lower()
         if val.endswith("[int]"):
             value = int(value[: -len("[int]")])

fsspec/gui.py

@@ -70,7 +70,7 @@ class which owns it.
             same name.
         """
         if name not in self.signals:
-            raise ValueError("Attempt to assign an undeclared signal: %s" % name)
+            raise ValueError(f"Attempt to assign an undeclared signal: {name}")
         self._sigs[name] = {
             "widget": widget,
             "callbacks": [],
@@ -141,7 +141,7 @@ def _emit(self, sig, value=None):
 
         Calling of callbacks will halt whenever one returns False.
         """
-        logger.log(self._sigs[sig]["log"], "{}: {}".format(sig, value))
+        logger.log(self._sigs[sig]["log"], f"{sig}: {value}")
         for callback in self._sigs[sig]["callbacks"]:
             if isinstance(callback, str):
                 self._emit(callback)
@@ -319,7 +319,7 @@ def fs(self):
     def urlpath(self):
         """URL of currently selected item"""
         return (
-            (self.protocol.value + "://" + self.main.value[0])
+            (f"{self.protocol.value}://{self.main.value[0]}")
             if self.main.value
             else None
         )

fsspec/implementations/cached.py

@@ -128,6 +128,11 @@ def __init__(
         self.expiry = expiry_time
         self.compression = compression
 
+        # Size of cache in bytes. If None then the size is unknown and will be
+        # recalculated the next time cache_size() is called. On writes to the
+        # cache this is reset to None.
+        self._cache_size = None
+
         if same_names is not None and cache_mapper is not None:
             raise ValueError(
                 "Cannot specify both same_names and cache_mapper in "
@@ -165,6 +170,17 @@ def _remove_tempdir(tempdir):
     def _mkcache(self):
         os.makedirs(self.storage[-1], exist_ok=True)
 
+    def cache_size(self):
+        """Return size of cache in bytes.
+
+        If more than one cache directory is in use, only the size of the last
+        one (the writable cache directory) is returned.
+        """
+        if self._cache_size is None:
+            cache_dir = self.storage[-1]
+            self._cache_size = filesystem("file").du(cache_dir, withdirs=True)
+        return self._cache_size
+
     def load_cache(self):
         """Read set of stored blocks from file"""
         self._metadata.load()
@@ -176,6 +192,7 @@ def save_cache(self):
         self._mkcache()
         self._metadata.save()
         self.last_cache = time.time()
+        self._cache_size = None
 
     def _check_cache(self):
         """Reload caches if time elapsed or any disappeared"""
@@ -202,6 +219,7 @@ def clear_cache(self):
         """
         rmtree(self.storage[-1])
         self.load_cache()
+        self._cache_size = None
 
     def clear_expired_cache(self, expiry_time=None):
         """Remove all expired files and metadata from the cache
@@ -231,6 +249,8 @@ def clear_expired_cache(self, expiry_time=None):
             rmtree(self.storage[-1])
             self.load_cache()
 
+        self._cache_size = None
+
     def pop_from_cache(self, path):
         """Remove cached version of given file
 
@@ -242,6 +262,7 @@ def pop_from_cache(self, path):
         fn = self._metadata.pop_file(path)
         if fn is not None:
             os.remove(fn)
+        self._cache_size = None
 
     def _open(
         self,
@@ -283,10 +304,10 @@ def _open(
             hash, blocks = detail["fn"], detail["blocks"]
             if blocks is True:
                 # stored file is complete
-                logger.debug("Opening local copy of %s" % path)
+                logger.debug("Opening local copy of %s", path)
                 return open(fn, mode)
             # TODO: action where partial file exists in read-only cache
-            logger.debug("Opening partially cached copy of %s" % path)
+            logger.debug("Opening partially cached copy of %s", path)
         else:
             hash = self._mapper(path)
             fn = os.path.join(self.storage[-1], hash)
@@ -299,7 +320,7 @@ def _open(
                 "uid": self.fs.ukey(path),
             }
             self._metadata.update_file(path, detail)
-            logger.debug("Creating local sparse file for %s" % path)
+            logger.debug("Creating local sparse file for %s", path)
 
         # call target filesystems open
         self._mkcache()
@@ -322,9 +343,9 @@ def _open(
         if "blocksize" in detail:
             if detail["blocksize"] != f.blocksize:
                 raise BlocksizeMismatchError(
-                    "Cached file must be reopened with same block"
-                    "size as original (old: %i, new %i)"
-                    "" % (detail["blocksize"], f.blocksize)
+                    f"Cached file must be reopened with same block"
+                    f" size as original (old: {detail['blocksize']},"
+                    f" new {f.blocksize})"
                 )
         else:
             detail["blocksize"] = f.blocksize
@@ -389,6 +410,7 @@ def __getattribute__(self, item):
             "__hash__",
             "__eq__",
             "to_json",
+            "cache_size",
         ]:
             # all the methods defined in this class. Note `open` here, since
             # it calls `_open`, but is actually in superclass
@@ -535,6 +557,7 @@ def commit_many(self, open_files):
                 os.remove(f.name)
             except FileNotFoundError:
                 pass
+        self._cache_size = None
 
     def _make_local_details(self, path):
         hash = self._mapper(path)
@@ -547,7 +570,7 @@ def _make_local_details(self, path):
             "uid": self.fs.ukey(path),
         }
         self._metadata.update_file(path, detail)
-        logger.debug("Copying %s to local cache" % path)
+        logger.debug("Copying %s to local cache", path)
         return fn
 
     def cat(
@@ -604,7 +627,7 @@ def _open(self, path, mode="rb", **kwargs):
             detail, fn = detail
             _, blocks = detail["fn"], detail["blocks"]
             if blocks is True:
-                logger.debug("Opening local copy of %s" % path)
+                logger.debug("Opening local copy of %s", path)
 
                 # In order to support downstream filesystems to be able to
                 # infer the compression from the original filename, like
@@ -616,8 +639,8 @@ def _open(self, path, mode="rb", **kwargs):
                 return f
             else:
                 raise ValueError(
-                    "Attempt to open partially cached file %s"
-                    "as a wholly cached file" % path
+                    f"Attempt to open partially cached file {path}"
+                    f" as a wholly cached file"
                 )
         else:
             fn = self._make_local_details(path)
@@ -700,10 +723,11 @@ def _open(self, path, mode="rb", **kwargs):
 
         sha = self._mapper(path)
         fn = os.path.join(self.storage[-1], sha)
-        logger.debug("Copying %s to local cache" % path)
+        logger.debug("Copying %s to local cache", path)
         kwargs["mode"] = mode
 
         self._mkcache()
+        self._cache_size = None
         if self.compression:
             with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2:
                 if isinstance(f, AbstractBufferedFile):

fsspec/implementations/ftp.py

@@ -156,7 +156,7 @@ def cb(x):
             outfile.write(x)
 
         self.ftp.retrbinary(
-            "RETR %s" % rpath,
+            f"RETR {rpath}",
             blocksize=self.blocksize,
             callback=cb,
         )
@@ -172,7 +172,7 @@ def cb(x):
             out.append(x)
 
         self.ftp.retrbinary(
-            "RETR %s" % path,
+            f"RETR {path}",
             blocksize=self.blocksize,
             rest=start,
             callback=cb,
@@ -321,7 +321,7 @@ def callback(x):
 
         try:
             self.fs.ftp.retrbinary(
-                "RETR %s" % self.path,
+                f"RETR {self.path}",
                 blocksize=self.blocksize,
                 rest=start,
                 callback=callback,
@@ -339,7 +339,7 @@ def callback(x):
     def _upload_chunk(self, final=False):
         self.buffer.seek(0)
         self.fs.ftp.storbinary(
-            "STOR " + self.path, self.buffer, blocksize=self.blocksize, rest=self.offset
+            f"STOR {self.path}", self.buffer, blocksize=self.blocksize, rest=self.offset
         )
         return True
 

fsspec/implementations/git.py

@@ -81,7 +81,7 @@ def ls(self, path, detail=True, ref=None, **kwargs):
                             "type": "directory",
                             "name": "/".join([path, obj.name]).lstrip("/"),
                             "hex": obj.hex,
-                            "mode": "%o" % obj.filemode,
+                            "mode": f"{obj.filemode:o}",
                             "size": 0,
                         }
                     )
@@ -91,7 +91,7 @@ def ls(self, path, detail=True, ref=None, **kwargs):
                             "type": "file",
                             "name": "/".join([path, obj.name]).lstrip("/"),
                             "hex": obj.hex,
-                            "mode": "%o" % obj.filemode,
+                            "mode": f"{obj.filemode:o}",
                             "size": obj.size,
                         }
                     )
@@ -102,7 +102,7 @@ def ls(self, path, detail=True, ref=None, **kwargs):
                     "type": "file",
                     "name": obj.name,
                     "hex": obj.hex,
-                    "mode": "%o" % obj.filemode,
+                    "mode": f"{obj.filemode:o}",
                     "size": obj.size,
                 }
             ]