Skip to content

Commit

Permalink
replace shutil.make_archive by tarfile and gzip
Browse files Browse the repository at this point in the history
this provides greater control and allows us to create reproducible gzip files
i.e. without filename in the header, and with mtime set to zero
  • Loading branch information
dennisvang committed Nov 21, 2023
1 parent a5261d1 commit 65d5345
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 17 deletions.
28 changes: 13 additions & 15 deletions src/tufup/repo/__init__.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import tempfile
from copy import deepcopy
from datetime import datetime, timedelta
import inspect
import json
import logging
import pathlib
import tarfile

try:
# workaround for PyInstaller issue 6911 (setuptools issue 3089)
Expand Down Expand Up @@ -37,7 +39,7 @@
)
from tuf.api.serialization.json import JSONSerializer

from tufup.common import Patcher, SUFFIX_ARCHIVE, SUFFIX_PATCH, TargetMeta
from tufup.common import Patcher, SUFFIX_PATCH, TargetMeta
from tufup.utils.platform_specific import _patched_resolve

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -79,12 +81,7 @@ def make_gztar_archive(
dst_dir: Union[pathlib.Path, str],
app_name: str,
version: str,
**kwargs, # allowed kwargs are passed on to shutil.make_archive
) -> Optional[TargetMeta]:
# remove disallowed kwargs
for key in ['base_name', 'root_dir', 'format']:
if kwargs.pop(key, None):
logger.warning(f'{key} ignored: using default')
# ensure paths
src_dir = pathlib.Path(src_dir)
dst_dir = pathlib.Path(dst_dir)
Expand All @@ -97,15 +94,16 @@ def make_gztar_archive(
if input(f'Found existing archive: {archive_path}.\nOverwrite? [n]/y') != 'y':
print('Using existing archive.')
return TargetMeta(archive_path)
# make archive
base_name = str(dst_dir / archive_filename.replace(SUFFIX_ARCHIVE, ''))
archive_path_str = shutil.make_archive(
base_name=base_name, # archive file path, no suffix
root_dir=str(src_dir), # paths in archive will be relative to root_dir
format='gztar',
**kwargs,
)
return TargetMeta(target_path=archive_path_str)
# create archive
with tempfile.NamedTemporaryFile(mode='wb') as temp_file:
# make temporary tar archive
with tarfile.open(fileobj=temp_file, mode='w') as tar:
for path in src_dir.iterdir():
tar.add(name=path, arcname=path.relative_to(src_dir), recursive=True)
temp_file_path = pathlib.Path(temp_file.name)
# compress tar archive using gzip (force mtime to zero for reproducibility)
Patcher.gzip(src_path=temp_file_path, dst_path=archive_path, mtime=0)
return TargetMeta(target_path=archive_path)


class RolesDict(TypedDict):
Expand Down
12 changes: 10 additions & 2 deletions tests/test_repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import json
import logging
import pathlib
import struct
import tarfile
from tempfile import TemporaryDirectory
from time import sleep
from unittest.mock import Mock, patch
Expand Down Expand Up @@ -96,14 +98,20 @@ def test_make_gztar_archive(self):
dst_dir=self.temp_dir_path,
app_name=app_name,
version=version,
base_dir='.', # this kwarg is allowed
root_dir='some path', # this kwarg is removed
)
self.assertIsInstance(archive, TargetMeta)
self.assertEqual(exists, mock_input_no.called)
self.assertTrue(archive.path.exists())
self.assertTrue(app_name in str(archive.path))
self.assertTrue(version in str(archive.path))
# check mtime in archive gzip header (see RFC 1952 and test_common.py)
mtime = struct.unpack('<I', archive.path.read_bytes()[4:8])[0]
self.assertEqual(0, mtime)
# check archive content
with tarfile.open(archive.path) as tar:
self.assertEqual(
{'root.txt', 'sub', 'sub/sub.txt'}, set(item.name for item in tar)
)


class BaseTests(TempDirTestCase):
Expand Down

0 comments on commit 65d5345

Please sign in to comment.