Support custom metadata objects. (#100)

* add custom arg to Roles.add_or_update_target()

and amend corresponding test

* add optional custom_metadata args to Repository.add_bundle()

and amend corresponding test

* todo check for custom info

* use is for type comparison

* add a TargetMeta.custom attribute

and assign data from the targets in Client.trusted_target_metas

* handle nested mutable types in TargetMeta.custom

Introduces an _immutable() function that handles the basic mutable sequence types.
This is necessary because the new TargetMeta.custom attribute contains a dict, possibly with other nested mutable items.

* add custom metadata to repo workflow example

* generate new test data from repo workflow example

* improve docstring for repo workflow example

* additional test for custom metdata

* cleanup after custom metadata

examples/repo/repo_workflow_example.py

@@ -1,5 +1,6 @@
 import copy
 import logging
+import os
 import pathlib
 import secrets  # from python 3.9+ we can use random.randbytes
 import shutil
@@ -16,17 +17,23 @@
 
 """
 
+This script was based on the python-tuf basic repo example [1]. The script generates a 
+complete example repository, including key pairs and example data. It illustrates 
+some common repository operations.
+
+NOTE: This script creates subdirectories and files in the tufup/examples/repo directory.
+
+NOTE: This script was also used to generate the test data in tests/data.
+
 NOTE: The repo content can be served for local testing as follows:
 
     python -m http.server -d examples/repo/repository
 
-NOTE: This script creates subdirectories and files in the 
-tufup/examples/repo directory. 
-
 NOTE: When running this script in PyCharm, ensure "Emulate terminal in output 
-console" is enabled in the run configuration, otherwise the encryption 
-passwords cannot be entered. 
+console" is enabled in the run configuration, otherwise the encryption passwords 
+cannot be entered.
 
+[1]: https://github.com/theupdateframework/python-tuf/blob/develop/examples/manual_repo/basic_repo.py
 """
 
 logger = logging.getLogger(__name__)
@@ -45,12 +52,23 @@
 TARGETS_DIR = REPO_DIR / DEFAULT_TARGETS_DIR_NAME
 
 # Settings
-EXPIRATION_DAYS = dict(root=365, targets=100, snapshot=7, timestamp=1)
+_TEST_EXPIRATION = int(os.getenv('TEST_EXPIRATION', 0))  # for creating test repo data
+if _TEST_EXPIRATION:
+    logger.warning(f'using TEST_EXPIRATION: {_TEST_EXPIRATION} days')
+EXPIRATION_DAYS = dict(
+    root=_TEST_EXPIRATION or 365,
+    targets=_TEST_EXPIRATION or 100,
+    snapshot=_TEST_EXPIRATION or 7,
+    timestamp=_TEST_EXPIRATION or 1,
+)
 THRESHOLDS = dict(root=2, targets=1, snapshot=1, timestamp=1)
 KEY_MAP = copy.deepcopy(DEFAULT_KEY_MAP)
 KEY_MAP['root'].append('root_two')  # use two keys for root
 ENCRYPTED_KEYS = ['root', 'root_two', 'targets']
 
+# Custom metadata
+DUMMY_METADATA = dict(whatever='important')
+
 # Create repository instance
 repo = Repository(
     app_name=APP_NAME,
@@ -122,9 +140,13 @@
         dummy_file_content += secrets.token_bytes(dummy_delta_size)
     dummy_file_path.write_bytes(dummy_file_content)
 
-    # Create archive and patch and register the new update (here we sign
-    # everything at once, for convenience)
-    repo.add_bundle(new_version=new_version, new_bundle_dir=dummy_bundle_dir)
+    # Create archive and patch and register the new update (here we sign everything
+    # at once, for convenience)
+    repo.add_bundle(
+        new_version=new_version,
+        new_bundle_dir=dummy_bundle_dir,
+        custom_metadata_for_patch=DUMMY_METADATA,  # just to point out the option
+    )
     repo.publish_changes(private_key_dirs=[OFFLINE_DIR_1, OFFLINE_DIR_2, ONLINE_DIR])
 
 # Time goes by

src/tufup/client.py

@@ -66,8 +66,8 @@ def trusted_target_metas(self) -> list:
         _trusted_target_metas = []
         if self._trusted_set.targets:
             _trusted_target_metas = [
-                TargetMeta(target_path=key)
-                for key in self._trusted_set.targets.signed.targets.keys()
+                TargetMeta(target_path=key, custom=target.custom)
+                for key, target in self._trusted_set.targets.signed.targets.items()
             ]
             logger.debug(f'{len(_trusted_target_metas)} TargetMeta objects created')
         else:

src/tufup/common.py

@@ -12,6 +12,29 @@
 SUFFIX_PATCH = '.patch'
 
 
+def _immutable(value):
+    """
+    Make value immutable, recursively, so the result is hashable.
+
+    Applies to (nested) dict, list, set, and bytearray [1] mutable sequence types.
+    Everything else is passed through unaltered, so the more exotic mutable types are
+    not supported.
+
+    [1]: https://peps.python.org/pep-3137/
+    """
+    # recursive cases
+    if isinstance(value, dict):
+        return tuple((k, _immutable(v)) for k, v in value.items())
+    elif isinstance(value, list):
+        return tuple(_immutable(v) for v in value)
+    elif isinstance(value, set):
+        return frozenset(_immutable(v) for v in value)
+    elif isinstance(value, bytearray):
+        return bytes(value)
+    # base case
+    return value
+
+
 class TargetMeta(object):
     filename_pattern = '{name}-{version}{suffix}'
     filename_regex = re.compile(
@@ -24,6 +47,7 @@ def __init__(
         name: Optional[str] = None,
         version: Optional[str] = None,
         is_archive: Optional[bool] = True,
+        custom: Optional[dict] = None,
     ):
         """
         Initialize either with target_path, or with name, version, archive.
@@ -42,6 +66,7 @@ def __init__(
             logger.critical(
                 f'invalid filename "{self.filename}": whitespace not allowed'
             )
+        self.custom = custom
 
     def __str__(self):
         return str(self.target_path_str)
@@ -57,10 +82,10 @@ def __hash__(self):
         https://docs.python.org/3/glossary.html#term-hashable
 
         """
-        return hash(tuple(self.__dict__.items()))
+        return hash(_immutable(self.__dict__))
 
     def __eq__(self, other):
-        if type(other) != type(self):
+        if type(other) is not type(self):
             return NotImplemented
         return vars(self) == vars(other)
 
@@ -70,7 +95,7 @@ def __lt__(self, other):
         without having to specify an explicit sorting key. Note this
         disregards app name, platform, and suffixes.
         """
-        if type(other) != type(self):
+        if type(other) is not type(self):
             return NotImplemented
         return self.version < other.version
 

src/tufup/repo/__init__.py

@@ -358,16 +358,21 @@ def add_or_update_target(
         self,
         local_path: Union[pathlib.Path, str],
         url_path_segments: Optional[List[str]] = None,
+        custom: Optional[dict] = None,
     ):
         # based on python-tuf basic_repo.py
         local_path = pathlib.Path(local_path)
         # build url path
         url_path_segments = url_path_segments or []
         url_path_segments.append(local_path.name)
         url_path = '/'.join(url_path_segments)
+        # create targetfile instance
         target_file_info = TargetFile.from_file(
             target_file_path=url_path, local_path=str(local_path)
         )
+        if custom:
+            # todo: should we verify that custom is a dict?
+            target_file_info.unrecognized_fields['custom'] = custom
         # note we assume self.targets has been initialized
         self.targets.signed.targets[url_path] = target_file_info
 
@@ -709,6 +714,8 @@ def add_bundle(
         new_bundle_dir: Union[pathlib.Path, str],
         new_version: Optional[str] = None,
         skip_patch: bool = False,
+        custom_metadata_for_archive: Optional[dict] = None,
+        custom_metadata_for_patch: Optional[dict] = None,
     ):
         """
         Adds a new application bundle to the local repository.
@@ -740,14 +747,18 @@ def add_bundle(
         latest_archive = self.roles.get_latest_archive()
         if not latest_archive or latest_archive.version < new_archive.version:
             # register new archive
-            self.roles.add_or_update_target(local_path=new_archive.path)
+            self.roles.add_or_update_target(
+                local_path=new_archive.path, custom=custom_metadata_for_archive
+            )
             # create patch, if possible, and register that too
             if latest_archive and not skip_patch:
                 patch_path = Patcher.create_patch(
                     src_path=self.targets_dir / latest_archive.path,
                     dst_path=self.targets_dir / new_archive.path,
                 )
-                self.roles.add_or_update_target(local_path=patch_path)
+                self.roles.add_or_update_target(
+                    local_path=patch_path, custom=custom_metadata_for_patch
+                )
 
     def remove_latest_bundle(self):
         """

tests/data/README.md

@@ -1,3 +1,3 @@
-Large parts of the test data were copied verbatim from the `python-tuf` [repository_data][1] folder.
+These test data were generated using the examples/repo/repo_workflow_example.py script.
 
-[1]: https://github.com/theupdateframework/python-tuf/tree/develop/tests/repository_data
+(expiration dates were set to some time far in the future)

tests/data/keystore/root.pub

@@ -1 +1 @@
-{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "0e492fadf5643a11049e2d7e59db6b8fc766945315f5bdc5648bd94fe2b427cb"}}
+{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "f5033e2659886185ceedec69e2cfee0f348ea63dfffafd5f8566d001b45c470d"}}

tests/data/keystore/root_two.pub

@@ -0,0 +1 @@
+{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "c8eaa5bf0f26e7247c965388a7ce7d3a25113899139c3d9bd2dbbb5e95577397"}}

tests/data/keystore/snapshot.pub

@@ -1 +1 @@
-{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "f3be5f4d498ca80145c84f6ca1d443b139efcbd2dde91219396aa3a0b5d7a987"}}
+{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "41bf1adabf1f564de734fa5fb584a65b943317978a4dcbe39bab03ee722ee73f"}}

tests/data/keystore/targets.pub

@@ -1 +1 @@
-{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "2c12e0cd2837cfe0448d77c93c0258ba8cbc2af89351b9b0bad3aae43e6433bf"}}
+{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "a27a0209711787a4227cbfed23735a75b5f7f5cb0cd6acbf7a239fa2c3535434"}}

tests/data/keystore/timestamp.pub

@@ -1 +1 @@
-{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "405dc59918fab6c3c489e781f06e8a57dd0061d4c62fc071c7fc5b17c4c70209"}}
+{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "2ec5e87c77fe70d918d92a1d849f4ec12907a34cf208123bbbc6d1e4bd584885"}}

tests/data/repository/metadata/1.root.json

@@ -1,71 +1,83 @@
 {
  "signatures": [
   {
-   "keyid": "bd7b600ecf8443b36566a170785cb66a400fee7b5af2c6c693e60fe4f8207cce",
-   "sig": "db5d5329bdf0fddc9e353d882276ddddd74ce5e33d0a3f8abe451f5498e1a09af866259d82a41d7b8afa2b7c9eb2de7d9bd81b08114d6c04cb419593d3884a06"
+   "keyid": "104c43225506bf7637a0061775a0d23ca8693e6bb4b270bc9ee9664259eb77d8",
+   "sig": "aa37e6a5e46938eb7c72054f2f2ff929e949283be67149c2a4fe481e51b91d8cc16876cbce03619af1d0b331ebf1d72ec368069ca49cca8d95a96eeaa06bfc07"
+  },
+  {
+   "keyid": "eb456bc4372b9aef1aea4790911d748a741d27ad0bd0eabcfe41e7fe3c6e9a8f",
+   "sig": "b70196c013a883d0ae5fede183e1c49556ee26fecb0798968e41a391121c39ab229ed2e1f7067760232aeac0b709ecf48a29df34f0184349c5d96f4e9be91703"
   }
  ],
  "signed": {
   "_type": "root",
   "consistent_snapshot": false,
-  "expires": "2032-05-07T15:17:51Z",
+  "expires": "2051-06-24T09:37:39Z",
   "keys": {
-   "0fac4d0180fffcecd7fb6832487e314fbf3cee050e3aed64a4bf60879a053659": {
+   "0eb56770be481c3a117f0487e7b6762edd0eaac7860ba85530dba400edf7de03": {
+    "keytype": "ed25519",
+    "keyval": {
+     "public": "2ec5e87c77fe70d918d92a1d849f4ec12907a34cf208123bbbc6d1e4bd584885"
+    },
+    "scheme": "ed25519"
+   },
+   "104c43225506bf7637a0061775a0d23ca8693e6bb4b270bc9ee9664259eb77d8": {
     "keytype": "ed25519",
     "keyval": {
-     "public": "f3be5f4d498ca80145c84f6ca1d443b139efcbd2dde91219396aa3a0b5d7a987"
+     "public": "c8eaa5bf0f26e7247c965388a7ce7d3a25113899139c3d9bd2dbbb5e95577397"
     },
     "scheme": "ed25519"
    },
-   "40e032f119d90855f540d23cbd364388de5f622cf868cf5c767df661a8678bcb": {
+   "3515ef592c09ddb3a09da0096802afc26852dc7a1978cb1c99fbe3a6f5c0c1a1": {
     "keytype": "ed25519",
     "keyval": {
-     "public": "2c12e0cd2837cfe0448d77c93c0258ba8cbc2af89351b9b0bad3aae43e6433bf"
+     "public": "a27a0209711787a4227cbfed23735a75b5f7f5cb0cd6acbf7a239fa2c3535434"
     },
     "scheme": "ed25519"
    },
-   "8e7d4ee2d147b4db84a208fc1e7eac3e586916afd1e3679c9d42dc89b58438e4": {
+   "5fcbe7c4faa87ab25bea551c0c4b0ac6e47a07caf5e7633314a784c54ad2ea8a": {
     "keytype": "ed25519",
     "keyval": {
-     "public": "405dc59918fab6c3c489e781f06e8a57dd0061d4c62fc071c7fc5b17c4c70209"
+     "public": "41bf1adabf1f564de734fa5fb584a65b943317978a4dcbe39bab03ee722ee73f"
     },
     "scheme": "ed25519"
    },
-   "bd7b600ecf8443b36566a170785cb66a400fee7b5af2c6c693e60fe4f8207cce": {
+   "eb456bc4372b9aef1aea4790911d748a741d27ad0bd0eabcfe41e7fe3c6e9a8f": {
     "keytype": "ed25519",
     "keyval": {
-     "public": "0e492fadf5643a11049e2d7e59db6b8fc766945315f5bdc5648bd94fe2b427cb"
+     "public": "f5033e2659886185ceedec69e2cfee0f348ea63dfffafd5f8566d001b45c470d"
     },
     "scheme": "ed25519"
    }
   },
   "roles": {
    "root": {
     "keyids": [
-     "bd7b600ecf8443b36566a170785cb66a400fee7b5af2c6c693e60fe4f8207cce"
+     "eb456bc4372b9aef1aea4790911d748a741d27ad0bd0eabcfe41e7fe3c6e9a8f",
+     "104c43225506bf7637a0061775a0d23ca8693e6bb4b270bc9ee9664259eb77d8"
     ],
-    "threshold": 1
+    "threshold": 2
    },
    "snapshot": {
     "keyids": [
-     "0fac4d0180fffcecd7fb6832487e314fbf3cee050e3aed64a4bf60879a053659"
+     "5fcbe7c4faa87ab25bea551c0c4b0ac6e47a07caf5e7633314a784c54ad2ea8a"
     ],
     "threshold": 1
    },
    "targets": {
     "keyids": [
-     "40e032f119d90855f540d23cbd364388de5f622cf868cf5c767df661a8678bcb"
+     "3515ef592c09ddb3a09da0096802afc26852dc7a1978cb1c99fbe3a6f5c0c1a1"
     ],
     "threshold": 1
    },
    "timestamp": {
     "keyids": [
-     "8e7d4ee2d147b4db84a208fc1e7eac3e586916afd1e3679c9d42dc89b58438e4"
+     "0eb56770be481c3a117f0487e7b6762edd0eaac7860ba85530dba400edf7de03"
     ],
     "threshold": 1
    }
   },
-  "spec_version": "1.0.29",
+  "spec_version": "1.0.31",
   "version": 1
  }
 }