test: s3fs pytest unraisable exception (#1012)

* install optional fsspec backends in the CI for some python versions

* update submit interface

* annotation

* do not use named arguments for the path/url

* Future init

* add s3fs and sshfs as test dependencies (fsspec beckends)

* test fsspec s3 for more combination of parameters

* remove pip install from ci

* style: pre-commit fixes

* revert test order

* remove dependencies as a test

* add s3fs to test

* exclude s3fs to python version 3.12

* add sshfs test (skipped)

* fix pytest

* asyncio not available in 3.12

* asyncio not available in 3.12

* add comment for fsspec threads

* attempt to close resources

* handle s3fs case separate for now

* attempt to pass tests

* attempt to pass tests

* simplified

* remove support for use_threads option, run non-async fs in threads using asyncio

* stop the loop on resource shutdown

* add skip for xrootd due to server issues

* remove skip for xrootd

* remove shutdown

* understand ci fail

* enable test that uses s3fs

* skip test for CI

* skip tests on problematic python versions

* test S3Source only in its test file

* fix pytest parameter

* rename test

* rename test function (duplicate name)

* imports

* rerun on timeout

* rerun on timeout

* remove minio

* skip s3fs test instead

* rename test

* no s3 in test names

* improve pytest regex

* python 3.8 skip

* skip s3_fail when timeout

* error tuple

* line jump

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Jim Pivarski <[email protected]>

tests/test_0029-more-string-types.py → tests/test_0029_more_string_types.py

@@ -1,9 +1,5 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE
 
-import json
-import sys
-
-import numpy
 import pytest
 import skhep_testdata
 
@@ -82,7 +78,7 @@ def test_parse_typename():
         parse_typename("map<string, int>>")
 
 
-def test_strings1():
+def test_strings_1():
     with uproot.open(skhep_testdata.data_path("uproot-small-evnt-tree-fullsplit.root"))[
         "tree"
     ] as tree:
@@ -166,23 +162,23 @@ def test_top_level_vectors():
         ]
 
 
-def test_strings1():
+def test_strings_2():
     with uproot.open(skhep_testdata.data_path("uproot-issue31.root"))[
         "T/name"
     ] as branch:
         result = branch.array(library="np")
         assert result.tolist() == ["one", "two", "three", "four", "five"]
 
 
-def test_strings2():
+def test_strings_3():
     with uproot.open(skhep_testdata.data_path("uproot-small-evnt-tree-fullsplit.root"))[
         "tree/Str"
     ] as branch:
         result = branch.array(library="np")
         assert result.tolist() == [f"evt-{i:03d}" for i in range(100)]
 
 
-def test_strings3():
+def test_strings_4():
     with uproot.open(skhep_testdata.data_path("uproot-small-evnt-tree-fullsplit.root"))[
         "tree/StdStr"
     ] as branch:

tests/test_0341-manipulate-streamer-info.py → tests/test_0341_manipulate_streamer_info.py

@@ -2,7 +2,6 @@
 
 import os
 
-import numpy as np
 import pytest
 
 import uproot
@@ -80,7 +79,7 @@ def test_with_mkdir(tmp_path):
         f6.mkdir("three")
 
 
-def test_add_streamers1(tmp_path):
+def test_add_streamers_1(tmp_path):
     has_TObjString = os.path.join(tmp_path, "has_TObjString.root")
 
     f_TObjString = ROOT.TFile(has_TObjString, "recreate")
@@ -120,7 +119,7 @@ def test_add_streamers1(tmp_path):
     f4.Close()
 
 
-def test_add_streamers2(tmp_path):
+def test_add_streamers_2(tmp_path):
     has_histogram = os.path.join(tmp_path, "has_histogram.root")
 
     f_histogram = ROOT.TFile(has_histogram, "recreate")
@@ -208,7 +207,7 @@ def test_add_streamers2(tmp_path):
     f3.Close()
 
 
-def test_add_streamers3(tmp_path):
+def test_add_streamers_3(tmp_path):
     has_TObjString = os.path.join(tmp_path, "has_TObjString.root")
 
     f_TObjString = ROOT.TFile(has_TObjString, "recreate")

tests/test_0692_fsspec_reading.py

@@ -9,6 +9,7 @@
 import fsspec
 import requests
 import os
+import sys
 
 
 def test_open_fsspec_http(server):
@@ -53,13 +54,17 @@ def test_open_fsspec_local():
 @pytest.mark.parametrize(
     "handler",
     [
-        # uproot.source.fsspec.FSSpecSource,
+        uproot.source.fsspec.FSSpecSource,
         uproot.source.s3.S3Source,
         None,
     ],
 )
 def test_open_fsspec_s3(handler):
     pytest.importorskip("s3fs")
+    if sys.version_info < (3, 11):
+        pytest.skip(
+            "https://github.com/scikit-hep/uproot5/pull/1012",
+        )
 
     with uproot.open(
         "s3://pivarski-princeton/pythia_ppZee_run17emb.picoDst.root:PicoDst",

tests/test_0916_read_from_s3.py

@@ -1,17 +1,18 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE
 
+import socket
+
 import pytest
 
 import uproot
 
 pytest.importorskip("minio")
 
 
-@pytest.mark.skip("https://github.com/scikit-hep/uproot5/pull/1012")
 @pytest.mark.network
 def test_s3_fail():
-    # TODO: fix this! Something not closing properly.
-    with pytest.raises(FileNotFoundError):
+    with pytest.raises((FileNotFoundError, TimeoutError, socket.timeout)):
+        # Sometimes this raises a timeout error that doesn't go away for a long time, we might as well skip it.
         with uproot.source.s3.S3Source(
             "s3://pivarski-princeton/does-not-exist", timeout=0.1
         ) as source: