diff --git a/tests/fs/test_docker_fsspec_dir_function.py b/tests/fs/test_docker_fsspec_dir_function.py new file mode 100644 index 0000000..bb1bdaa --- /dev/null +++ b/tests/fs/test_docker_fsspec_dir_function.py @@ -0,0 +1,48 @@ +import logging + +from alluxiofs import AlluxioFileSystem +from tests.fs.test_docker_fsspec_file_function import ALLUXIO_PREFIX +from tests.fs.test_docker_fsspec_file_function import check_file_info + +LOGGER = logging.getLogger(__name__) + +DIR_PATH = "/opt/alluxio/ufs" +SUB_DIR_PATH = "/opt/alluxio/ufs/hash_res" +FILE_PREFIX = "file://" + + +def check_dir_info(dir_info, dir_path): + assert dir_info.get("name") == dir_path + assert dir_info.get("type") == "directory" + assert not dir_info.get("size") + + +def alluxio_fsspec_test_dir(alluxio_file_system, alluxio_dir_path): + file_list = alluxio_file_system.ls(alluxio_dir_path) + assert len(file_list) == 2 + if file_list[0].get("type") == "file": + check_file_info(file_list[0]) + check_dir_info(file_list[1], SUB_DIR_PATH) + elif file_list[0].get("type") == "directory": + check_dir_info(file_list[0], SUB_DIR_PATH) + check_file_info(file_list[1]) + else: + raise AssertionError("File type is invalid.") + dir_info = alluxio_file_system.info(alluxio_dir_path) + check_dir_info(dir_info, DIR_PATH) + assert alluxio_file_system.isdir(alluxio_dir_path) + assert not alluxio_file_system.isfile(alluxio_dir_path) + + +def test_alluxio_fsspec_dir_function(alluxio_file_system: AlluxioFileSystem): + alluxio_fsspec_test_dir(alluxio_file_system, DIR_PATH) + alluxio_fsspec_test_dir(alluxio_file_system, FILE_PREFIX + DIR_PATH) + alluxio_fsspec_test_dir( + alluxio_file_system, ALLUXIO_PREFIX + FILE_PREFIX + DIR_PATH + ) + + +def test_etcd_alluxio_fsspec_dir_function( + etcd_alluxio_file_system: AlluxioFileSystem, +): + test_alluxio_fsspec_dir_function(etcd_alluxio_file_system) diff --git a/tests/fs/test_docker_fsspec_cat.py b/tests/fs/test_docker_fsspec_file_function.py similarity index 66% rename from tests/fs/test_docker_fsspec_cat.py rename to tests/fs/test_docker_fsspec_file_function.py index 729272c..466b2b6 100644 --- a/tests/fs/test_docker_fsspec_cat.py +++ b/tests/fs/test_docker_fsspec_file_function.py @@ -11,6 +11,9 @@ LOGGER = logging.getLogger(__name__) +FILE_PATH = "/opt/alluxio/ufs/test.csv" +ALLUXIO_PREFIX = "alluxio::" + def validate_read_range( alluxio_file_system: AlluxioFileSystem, @@ -42,10 +45,29 @@ def validate_read_range( raise AssertionError(error_message) +def check_file_info(file_info): + assert file_info.get("name") == FILE_PATH + assert file_info.get("type") == "file" + assert file_info.get("size") == 6569089 + + def alluxio_fsspec_cat_file(alluxio_file_system, alluxio_path, local_path): file_size = os.path.getsize(local_path) - alluxio_file_system.ls(alluxio_path) + file_list = alluxio_file_system.ls(alluxio_path) + assert len(file_list) == 1 + check_file_info(file_list[0]) + file_info = alluxio_file_system.info(alluxio_path) + check_file_info(file_info) + assert not alluxio_file_system.isdir(alluxio_path) + assert alluxio_file_system.isfile(alluxio_path) + + with alluxio_file_system.open(alluxio_path) as f: + alluxio_file_data = f.read() + + with open(local_path, "rb") as local_file: + local_file_data = local_file.read() + assert local_file_data == alluxio_file_data # Validate normal case max_length = 13 * 1024 @@ -81,16 +103,19 @@ def alluxio_fsspec_cat_file(alluxio_file_system, alluxio_path, local_path): LOGGER.debug("Passed corner test cases") -def test_alluxio_fsspec_cat_file(alluxio_file_system: AlluxioFileSystem): +def test_alluxio_fsspec_file_function(alluxio_file_system: AlluxioFileSystem): alluxio_fsspec_cat_file( alluxio_file_system, ALLUXIO_FILE_PATH, LOCAL_FILE_PATH ) alluxio_fsspec_cat_file( - alluxio_file_system, "alluxio::" + ALLUXIO_FILE_PATH, LOCAL_FILE_PATH + alluxio_file_system, + ALLUXIO_PREFIX + ALLUXIO_FILE_PATH, + LOCAL_FILE_PATH, ) + alluxio_fsspec_cat_file(alluxio_file_system, FILE_PATH, LOCAL_FILE_PATH) -def test_etcd_alluxio_fsspec_cat_file( +def test_etcd_alluxio_fsspec_file_function( etcd_alluxio_file_system: AlluxioFileSystem, ): - test_alluxio_fsspec_cat_file(etcd_alluxio_file_system) + test_alluxio_fsspec_file_function(etcd_alluxio_file_system)