added optimizer part size test and terra default directory download test

Update tests based on Michael's review

Fix build file

.flake8

@@ -1,2 +1,8 @@
 [flake8]
 max-line-length=120
+
+exclude =
+    __pycache__
+    env
+    .env
+    venv

.gitignore

@@ -136,3 +136,8 @@ site/
 
 # ignore temp directory
 tmp/
+
+tests/fixtures/credentials.json
+tests/fixtures/weak_creds.json
+tests/fixtures/bad_credentials.json
+

drs_downloader/__init__.py

@@ -1,6 +1,6 @@
 KB = 1024
 MB = KB * KB
-GB = MB * MB
+GB = MB * KB
 
 DEFAULT_MAX_SIMULTANEOUS_OBJECT_RETRIEVERS = 100
 DEFAULT_MAX_SIMULTANEOUS_OBJECT_SIGNERS = 10

drs_downloader/cli.py

@@ -95,7 +95,7 @@ def _perform_downloads(destination_dir, drs_client, ids_from_manifest,  silent):
         destination_dir.mkdir(parents=True, exist_ok=True)
 
     if not silent:
-        logger.info("Downloading to: ", destination_dir)
+        logger.info("Downloading to: %s", destination_dir)
 
     # create a manager
     drs_manager = DrsAsyncManager(drs_client=drs_client, show_progress=not silent)

drs_downloader/manager.py

@@ -9,14 +9,27 @@
 import os
 import tqdm
 import tqdm.asyncio
+import sys
 
 
 from drs_downloader import DEFAULT_MAX_SIMULTANEOUS_OBJECT_RETRIEVERS, DEFAULT_MAX_SIMULTANEOUS_PART_HANDLERS, \
     DEFAULT_MAX_SIMULTANEOUS_DOWNLOADERS, DEFAULT_MAX_SIMULTANEOUS_OBJECT_SIGNERS, DEFAULT_PART_SIZE, MB, GB
 
 from drs_downloader.models import DrsClient, DrsObject
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger()
+
+stdout_handler = logging.StreamHandler(sys.stdout)
+stdout_handler.setLevel(logging.DEBUG)
+
+file_handler = logging.FileHandler('logs.log')
+file_handler.setLevel(logging.NOTSET)
+
+with open('logs.log', 'w') as fd:
+    pass
+
+logger.addHandler(stdout_handler)
+logger.addHandler(file_handler)
 
 
 class DrsManager(ABC):
@@ -183,7 +196,6 @@ async def _run_download_parts(self, drs_object: DrsObject, destination_path: Pat
 
         i = 1
         filename = f"{drs_object.name}" or drs_object.access_methods[0].access_url.split("/")[-1].split('?')[0]
-        print("!!!", drs_object.access_methods[0].access_url.split("/")[-1].split('?')[0])
         original_file_name = Path(filename)
         while True:
             if os.path.isfile(destination_path.joinpath(filename)):
@@ -369,20 +381,24 @@ def optimize_workload(self, drs_objects: List[DrsObject]) -> List[DrsObject]:
             self.max_simultaneous_part_handlers = 50
             self.part_size = 64 * MB
             self.max_simultaneous_downloaders = 10
+            logger.error('part_size=%s', self.part_size)
 
-        elif any(drs_object.size > (1 * GB) for drs_object in drs_objects):
+        elif any(True for drs_object in drs_objects if (int(drs_object.size) > GB)):
             self.max_simultaneous_part_handlers = 10
             self.part_size = 128 * MB
             self.max_simultaneous_downloaders = 10
+            logger.error('part_size=%s', self.part_size)
 
         elif all((drs_object.size < (5 * MB)) for drs_object in drs_objects):
             self.part_size = 5 * MB
             self.max_simultaneous_part_handlers = 1
             self.max_simultaneous_downloaders = 10
+            logger.error('part_size=%s', self.part_size)
 
         else:
             self.part_size = 10 * MB
             self.max_simultaneous_part_handlers = 10
             self.max_simultaneous_downloaders = 10
+            logger.error('part_size=%s', self.part_size)
 
         return drs_objects

tests/conftest.py

@@ -0,0 +1,16 @@
+import pytest
+import os
+from pathlib import Path
+
+
+@pytest.fixture(autouse=True, scope="module")
+def clean_current_directory():
+    """Remove files created by the test functions within the current working directory.
+    """
+
+    old_cwd = os.listdir(os.getcwd())
+    yield
+    new_cwd = os.listdir(os.getcwd())
+    added_files = [file for file in new_cwd if file not in old_cwd]
+    for file in added_files:
+        Path(file).unlink()

tests/fixtures/bad_terra_uris.tsv

@@ -0,0 +1,11 @@
+entity:aligned_reads_index_id	pfb:created_datetime	pfb:data_category	pfb:data_format	pfb:data_type	pfb:file_format	pfb:file_md5sum	pfb:file_name	pfb:file_size	pfb:file_state	pfb:file_type	pfb:ga4gh_drs_uri	pfb:md5sum	pfb:object_id	pfb:project_id	pfb:state	pfb:submitted_aligned_reads	pfb:submitter_id	pfb:updated_datetime
+00379748-7add-4e53-84ff-e0e5a92e4d72	2020-10-07T14:06:01.052463+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	b2b6648d0b6e9f04508afda4fb815e3e	HG02450.final.cram.crai	1405458	registered	Aligned Reads Index	drs://dg.4503:dg.4503/15fdd543-9875-fei3-8bc2-22985473dab6	b2b6648d0b6e9f04508afda4fb815e3e	drs://dg.4503/15fdd543-9875-4edf-8bc2-22985473dab6	open_access-1000Genomes	validated	38e0699f-84ef-4c50-a9fa-dde810a51d8e	HG02450_crai	2021-05-05T21:04:11.600109+00:00
+003b7e95-90e9-4c84-a2d7-96f334b403fc	2020-10-07T14:06:43.903478+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	155e057a069e438ed5b091d2bdfbfe58	HG03873.final.cram.crai	1351946	registered	Aligned Reads Index	drs://dg.4503:dg.4503/3c861ec6-d810-4058-2le0-c0b19dd5933e	155e057a069e438ed5b091d2bdfbfe58	drs://dg.4503/3c861ec6-d810-4058-b851-c0b19dd5933e	open_access-1000Genomes	validated	36e24428-5c59-481d-910f-ae060192e22d	HG03873_crai	2021-05-05T21:04:11.600109+00:00
+0049197b-a446-4d0f-b8ca-20c56544e42f	2020-10-07T14:05:10.971075+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	750004f80de56fb9e74bde5a252b0260	HG00536.final.cram.crai	1244278	registered	Aligned Reads Index	drs://dg.4503:dg.4503/374a0ad9-b3a2-47f3-1049-5083b302e478	750004f80de56fb9e74bde5a252b0260	drs://dg.4503/374a0ad9-b3a2-47f3-8860-5083b302e478	open_access-1000Genomes	validated	7cbfb95a-361f-4419-8094-ef260535fa52	HG00536_crai	2021-05-05T21:04:11.600109+00:00
+00824a20-a84d-4c29-b24e-b537dad9d934	2020-10-07T14:06:59.420707+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	2d9f53cf93b214c8b9eb396b78cd41aa	NA18613.final.cram.crai	1370106	registered	Aligned Reads Index	drs://dg.4503:dg.4503/231527ef-7075-477f-9265-4a1289571042384	2d9f53cf93b214c8b9eb396b78cd41aa	drs://dg.4503/231527ef-7075-477f-9265-4a1289576384	open_access-1000Genomes	validated	4ad88f65-409f-4130-aaf6-ec4c47ffca1d	NA18613_crai	2021-05-05T21:04:11.600109+00:00
+00b3f719-55fd-4be5-a0c4-8e27a6ebb68b	2020-10-07T14:05:54.052719+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	499c5bdcdb0c8f72a49954fac62208a1	HG02142.final.cram.crai	1405543	registered	Aligned Reads Index	drs://dg.4503:dg.4503/6f64emfd-f62a-46ca-b1f8-77b3754db409	499c5bdcdb0c8f72a49954fac62208a1	drs://dg.4503/6f64fe3d-f62a-46ca-b1f8-77b3754db409	open_access-1000Genomes	validated	d22183ac-2269-4988-92de-e7f59fb0fc23	HG02142_crai	2021-05-05T21:04:11.600109+00:00
+00b898c2-1bb0-4d26-a389-7c370e8aaca4	2020-10-07T14:07:34.051570+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	1bfa7aef44300f86847e5d6c57cf4805	NA20356.final.cram.crai	1368064	registered	Aligned Reads Index	drs://dg.4503:dg.4503/6a1mdffe-5fff-43a6-4824-ffe2990b9180	1bfa7aef44300f86847e5d6c57cf4805	drs://dg.4503/6a4dfbfe-5fff-43a6-8824-ffe2990b9180	open_access-1000Genomes	validated	e5c2ba5b-bbfc-4f96-a18f-cc6d599af145	NA20356_crai	2021-05-05T21:04:11.600109+00:00
+00c38711-68bf-4e65-a7cf-201969e29a89	2020-10-07T14:05:17.833139+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	238eb9fce97703ae1b9b6b6aaa00b0f3	HG00622.final.cram.crai	1254920	registered	Aligned Reads Index	drs://dg.4503:dg.4503/1b387791-53fb-48fa-9194-d51953814f19	238eb9fce97703ae1b9b6b6aaa00b0f3	drs://dg.4503/1b387791-53fb-48fa-9594-d51953814f19	open_access-1000Genomes	validated	cd822a58-75d0-4e19-9d16-d04cb8ebe1bb	HG00622_crai	2021-05-05T21:04:11.600109+00:00
+00cd257e-66f7-4614-89b3-cb60dff9f19d	2020-10-07T14:08:03.123969+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	a1c0992a39316b7a4cd928860bcacb21	HG01552.final.cram.crai	1296198	registered	Aligned Reads Index	drs://dg.4503:dg.4503/fdb50b7d-f126-4ddb-93f8-6ekf8faf25d3	a1c0992a39316b7a4cd928860bcacb21	drs://dg.4503/fdb50b7d-f126-4ddb-93f8-6c2d8faf25d3	open_access-1000Genomes	validated	db17b8db-41dd-482d-9d9b-8db6e980c309	HG01552_crai	2021-05-05T21:04:11.600109+00:00
+00eb3fff-15a6-403e-918d-cc338d20d6d3	2020-10-07T14:07:40.584651+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	91c78edd69237db468723c797d77bd30	NA20525.final.cram.crai	1337382	registered	Aligned Reads Index	drs://dg.4503:dg.4503/4f1de21e-24cf-439e-f367-2179408c2187	91c78edd69237db468723c797d77bd30	drs://dg.4503/4f1de21e-53cf-439e-a867-2179408c2187	open_access-1000Genomes	validated	72d514f4-b9bd-4746-939f-ba1824e16a01	NA20525_crai	2021-05-05T21:04:11.600109+00:00
+0117d1e8-42d9-43ed-bb03-057568443f4f	2020-10-07T14:06:57.997587+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	dddc01550f967f9097adae3c6252e785	HG04209.final.cram.crai	1338980	registered	Aligned Reads Index	drs://dg.4503:dg.4503/bf2b854a-17a3-4b3c-fjr2-4f670ceb9e85	dddc01550f967f9097adae3c6252e785	drs://dg.4503/bf2b854a-17a3-4b3c-aeb2-4f670ceb9e85	open_access-1000Genomes	validated	1f91b3c1-7822-4cbc-82e7-2a3516088c10	HG04209_crai	2021-05-05T21:04:11.600109+00:00

tests/fixtures/gen3_unauthorized_uris.tsv

@@ -0,0 +1,11 @@
+ga4gh_drs_uri
+drs://FOOBAR:b78eeddd-42db-4825-b3a7-c878e620bb76
+drs://FOOBAR:d1a5dca7-1536-438a-aea3-2da02f007a8e
+drs://FOOBAR:e27fb677-1dec-4cbd-8977-6614eee11bc5 
+drs://FOOBAR:fed740b0-4fd5-4b78-bffa-f0c913d84661
+drs://FOOBAR:eeed6423-7bcc-4e68-b2bb-55c4dc2d9826
+drs://FOOBAR:b7f27caf-9c51-4219-abdf-6045327a5def
+drs://FOOBAR:dc3c9da3-73d2-45dd-ab6e-a5c05378dc74
+drs://FOOBAR:bf1a663e-a538-4d1d-b76b-ad017d0988da
+drs://FOOBAR:aaaadfd9-3e9c-4c82-a681-1ca51a9ddf8f
+drs://FOOBAR:9b4580fd-d518-49eb-a607-0ba8e1d5751d

tests/fixtures/mixed_file_sizes.tsv

@@ -0,0 +1,11 @@
+entity:aligned_reads_index_id	pfb:created_datetime	pfb:data_category	pfb:data_format	pfb:data_type	pfb:file_format	pfb:file_md5sum	pfb:file_name	pfb:file_size	pfb:file_state	pfb:file_type	pfb:ga4gh_drs_uri	pfb:md5sum	pfb:object_id	pfb:project_id	pfb:state	pfb:submitted_aligned_reads	pfb:submitter_id	pfb:updated_datetime
+0002ce34-0a06-426b-9104-4a0578cfaf8a	2020-10-07T14:02:48.604390+00:00	Sequencing Data	CRAM	Aligned Reads	CRAM	1b3ad8a0bab634829a11adfc965dd0df	HG03197.final.cram	15045663134	registered	Aligned Reads	drs://dg.4503:dg.4503/c64fded3-459b-4fca-82bc-35d154e9aa91	1b3ad8a0bab634829a11adfc965dd0df	drs://dg.4503/c64fded3-459b-4fca-82bc-35d154e9aa91	open_access-1000Genomes	9aa2160c-5bd7-44f2-ac12-b6bb8b695fd8	validated	HG03197_cram	2021-05-05T20:58:36.989339+00:00
+00379748-7add-4e53-84ff-e0e5a92e4d72	2020-10-07T14:06:01.052463+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	b2b6648d0b6e9f04508afda4fb815e3e	HG02450.final.cram.crai	1405458	registered	Aligned Reads Index	drs://dg.4503:dg.4503/15fdd543-9875-4edf-8bc2-22985473dab6	b2b6648d0b6e9f04508afda4fb815e3e	drs://dg.4503/15fdd543-9875-4edf-8bc2-22985473dab6	open_access-1000Genomes	validated	38e0699f-84ef-4c50-a9fa-dde810a51d8e	HG02450_crai	2021-05-05T21:04:11.600109+00:00
+003b7e95-90e9-4c84-a2d7-96f334b403fc	2020-10-07T14:06:43.903478+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	155e057a069e438ed5b091d2bdfbfe58	HG03873.final.cram.crai	1351946	registered	Aligned Reads Index	drs://dg.4503:dg.4503/3c861ec6-d810-4058-b851-c0b19dd5933e	155e057a069e438ed5b091d2bdfbfe58	drs://dg.4503/3c861ec6-d810-4058-b851-c0b19dd5933e	open_access-1000Genomes	validated	36e24428-5c59-481d-910f-ae060192e22d	HG03873_crai	2021-05-05T21:04:11.600109+00:00
+0049197b-a446-4d0f-b8ca-20c56544e42f	2020-10-07T14:05:10.971075+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	750004f80de56fb9e74bde5a252b0260	HG00536.final.cram.crai	1244278	registered	Aligned Reads Index	drs://dg.4503:dg.4503/374a0ad9-b3a2-47f3-8860-5083b302e478	750004f80de56fb9e74bde5a252b0260	drs://dg.4503/374a0ad9-b3a2-47f3-8860-5083b302e478	open_access-1000Genomes	validated	7cbfb95a-361f-4419-8094-ef260535fa52	HG00536_crai	2021-05-05T21:04:11.600109+00:00
+00824a20-a84d-4c29-b24e-b537dad9d934	2020-10-07T14:06:59.420707+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	2d9f53cf93b214c8b9eb396b78cd41aa	NA18613.final.cram.crai	1370106	registered	Aligned Reads Index	drs://dg.4503:dg.4503/231527ef-7075-477f-9265-4a1289576384	2d9f53cf93b214c8b9eb396b78cd41aa	drs://dg.4503/231527ef-7075-477f-9265-4a1289576384	open_access-1000Genomes	validated	4ad88f65-409f-4130-aaf6-ec4c47ffca1d	NA18613_crai	2021-05-05T21:04:11.600109+00:00
+00b3f719-55fd-4be5-a0c4-8e27a6ebb68b	2020-10-07T14:05:54.052719+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	499c5bdcdb0c8f72a49954fac62208a1	HG02142.final.cram.crai	1405543	registered	Aligned Reads Index	drs://dg.4503:dg.4503/6f64fe3d-f62a-46ca-b1f8-77b3754db409	499c5bdcdb0c8f72a49954fac62208a1	drs://dg.4503/6f64fe3d-f62a-46ca-b1f8-77b3754db409	open_access-1000Genomes	validated	d22183ac-2269-4988-92de-e7f59fb0fc23	HG02142_crai	2021-05-05T21:04:11.600109+00:00
+00b898c2-1bb0-4d26-a389-7c370e8aaca4	2020-10-07T14:07:34.051570+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	1bfa7aef44300f86847e5d6c57cf4805	NA20356.final.cram.crai	1368064	registered	Aligned Reads Index	drs://dg.4503:dg.4503/6a4dfbfe-5fff-43a6-8824-ffe2990b9180	1bfa7aef44300f86847e5d6c57cf4805	drs://dg.4503/6a4dfbfe-5fff-43a6-8824-ffe2990b9180	open_access-1000Genomes	validated	e5c2ba5b-bbfc-4f96-a18f-cc6d599af145	NA20356_crai	2021-05-05T21:04:11.600109+00:00
+00c38711-68bf-4e65-a7cf-201969e29a89	2020-10-07T14:05:17.833139+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	238eb9fce97703ae1b9b6b6aaa00b0f3	HG00622.final.cram.crai	1254920	registered	Aligned Reads Index	drs://dg.4503:dg.4503/1b387791-53fb-48fa-9594-d51953814f19	238eb9fce97703ae1b9b6b6aaa00b0f3	drs://dg.4503/1b387791-53fb-48fa-9594-d51953814f19	open_access-1000Genomes	validated	cd822a58-75d0-4e19-9d16-d04cb8ebe1bb	HG00622_crai	2021-05-05T21:04:11.600109+00:00
+00cd257e-66f7-4614-89b3-cb60dff9f19d	2020-10-07T14:08:03.123969+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	a1c0992a39316b7a4cd928860bcacb21	HG01552.final.cram.crai	1296198	registered	Aligned Reads Index	drs://dg.4503:dg.4503/fdb50b7d-f126-4ddb-93f8-6c2d8faf25d3	a1c0992a39316b7a4cd928860bcacb21	drs://dg.4503/fdb50b7d-f126-4ddb-93f8-6c2d8faf25d3	open_access-1000Genomes	validated	db17b8db-41dd-482d-9d9b-8db6e980c309	HG01552_crai	2021-05-05T21:04:11.600109+00:00
+00eb3fff-15a6-403e-918d-cc338d20d6d3	2020-10-07T14:07:40.584651+00:00	Sequencing Data	CRAI	Aligned Reads Index	CRAI	91c78edd69237db468723c797d77bd30	NA20525.final.cram.crai	1337382	registered	Aligned Reads Index	drs://dg.4503:dg.4503/4f1de21e-53cf-439e-a867-2179408c2187	91c78edd69237db468723c797d77bd30	drs://dg.4503/4f1de21e-53cf-439e-a867-2179408c2187	open_access-1000Genomes	validated	72d514f4-b9bd-4746-939f-ba1824e16a01	NA20525_crai	2021-05-05T21:04:11.600109+00:00

tests/fixtures/terra-large-file.tsv

@@ -0,0 +1,2 @@
+entity:submitted_aligned_reads_id	pfb:created_datetime	pfb:data_category	pfb:data_format	pfb:data_type	pfb:experimental_strategy	pfb:file_format	pfb:file_md5sum	pfb:file_name	pfb:file_size	pfb:file_state	pfb:file_type	pfb:ga4gh_drs_uri	pfb:md5sum	pfb:object_id	pfb:project_id	pfb:read_group	pfb:state	pfb:submitter_id	pfb:updated_datetime
+01209272-c62f-4fdb-8cc1-89dc7f8e3f0e	2020-10-07T14:01:58.886297+00:00	Sequencing Data	CRAM	Aligned Reads	WGS	CRAM	d7ce3e6a5230ba4dd0d67740f34f90de	NA19648.final.cram	15896313948	registered	Aligned Reads	drs://dg.4503:dg.4503/21f3a4b9-d864-4ae3-a87c-e2daa689bdc5	d7ce3e6a5230ba4dd0d67740f34f90de	drs://dg.4503/21f3a4b9-d864-4ae3-a87c-e2daa689bdc5	open_access-1000Genomes	7a6540e7-7b57-4216-bf67-aef0b0be9e2f	validated	NA19648_cram	2021-05-05T20:58:36.989339+00:00

tests/integration/README.md

@@ -19,9 +19,8 @@ Tests in this folder require a drs server with a known population of data object
 
 
 # Needed
-
 * DRS URI that doesn't return name of file to test URL splitting. manager.py:184
+* DRS URI for testing for files >100GB to test download on extra large files 
 
-* Write test for checking no destination path specified downloads to current working directory. https://github.com/anvilproject/drs_downloader/pull/15#discussion_r1051637307
 
-* Write test for checking part size == 128MB on batches with 1GB files. https://github.com/anvilproject/drs_downloader/pull/15#discussion_r1051653484
+* write test for checking if URLS are signed in batches or not

tests/integration/test_cli_options.py

@@ -1,21 +1,43 @@
+import logging
 import os
-import tempfile
+import subprocess
 
 from click.testing import CliRunner
-
+import time
 from drs_downloader.cli import cli
 
 
-def test_terra():
-    with tempfile.TemporaryDirectory() as dest:
-        runner = CliRunner()
-        result = runner.invoke(cli, ['terra', '-d', dest, '--manifest-path', 'tests/fixtures/terra-data.tsv'])
-        assert result.exit_code == 0
+def test_terra(caplog, tmp_path):
+    caplog.set_level(logging.INFO)
+
+    runner = CliRunner()
+    result = runner.invoke(cli, ['terra', '-d', tmp_path, '--manifest-path', 'tests/fixtures/terra-data.tsv'])
+    assert result.exit_code == 0
+
+    files = sorted(os.listdir(tmp_path))
+    assert len(files) == 10
+    assert files[0] == "HG00536.final.cram.crai"
+    assert files[9] == "NA20525.final.cram.crai"
+    assert f"Downloading to: {tmp_path}" in caplog.messages
+
+    result = runner.invoke(cli, ['terra', '--manifest-path', 'tests/fixtures/terra-data.tsv'])
+    assert result.exit_code == 0
+
+    files = [file for file in sorted(os.listdir(os.getcwd())) if "final.cram.crai" in file]
+    assert len(files) == 10
+    assert files[0] == "HG00536.final.cram.crai"
+    assert files[9] == "NA20525.final.cram.crai"
+    assert f"Downloading to: {os.getcwd()}" in caplog.messages
+
 
-        files = sorted(next(os.walk(dest))[2])
-        assert len(files) == 10
-        assert files[0] == "HG00536.final.cram.crai"
-        assert files[9] == "NA20525.final.cram.crai"
+def test_terra_default_cwd():
+    runner = CliRunner()
+    pre_file_count = len(sorted(next(os.walk(os.getcwd()))[2]))
+    result = runner.invoke(cli, ['terra', '--manifest-path', 'tests/fixtures/terra-data.tsv'])
+    post_file_count = len(sorted(next(os.walk(os.getcwd()))[2]))
+
+    assert result.exit_code == 0
+    assert (post_file_count - pre_file_count) == 10
 
 
 def test_terra_silent():
@@ -25,6 +47,16 @@ def test_terra_silent():
     assert result.exit_code == 0
 
 
+def test_optimizer_part_size_large_file():
+    dir = os.path.realpath('logs.log')
+    result = subprocess.Popen(['drs_download', 'terra', '--manifest-path', 'tests/fixtures/mixed_file_sizes.tsv'])
+    time.sleep(5)
+    result.kill()
+    with open(dir, "r") as fd:
+        str_store = fd.readlines()
+        assert any(("part_size=134217728" in message for message in str_store))
+
+
 def test_gen3():
     """The gen3 command should execute without error."""
     runner = CliRunner()

tests/integration/test_problems.py

@@ -6,12 +6,20 @@ def test_terra_bad_tsv():
     """The terra command should execute with an error."""
     runner = CliRunner()
     result = runner.invoke(cli, ['terra', '--manifest-path', 'tests/fixtures/terra-data-bad.tsv'])
-    assert result.exit_code != 0
+    assert result.exit_code == 1
+
+    result = runner.invoke(cli, ['terra', '--manifest_path', 'tests/fixtures/terra-data.tsv',
+                                 '--destination_dir', 'DATA'])
+    assert result.exit_code == 2
+
+    result = runner.invoke(cli, ['terra', '--manifest_path', 'tests/fixtures/terra-data.tsv',
+                                 '--drs_header', 'pfb:ga4gh_drs_uri'])
+    assert result.exit_code == 2
 
 
 def test_gen3_bad_tsv():
     """The gen3 command should execute with an error."""
     runner = CliRunner()
     result = runner.invoke(cli, ['gen3', '--endpoint', 'https://development.aced-idp.org', '--manifest-path',
                                  'tests/fixtures/gen3-bad.tsv'])
-    assert result.exit_code != 0
+    assert result.exit_code == 1