Flesh out Images integration testing

* Add __version__ via api/version.py
* Add PodmanConfig to parse containers.conf, will support reading
  connections
* Added "Images" logger to Image and ImagesManager
* Update Makefile with integration target
* Update client to support unix:// and http+unix:// schemes
* Add Images integration tests
* Add Network unit and integration tests
* Refactor PodmanLauncher to log service output to log file to capture
  debugging events
* Add option to allow podman service to log debugging events during
  tests. Triggered by DEBUG environment variable
* Refactor Image, Network, ImagesManager and NetworksManager as needed
  now integration tests running
* Refactor unittests where as-built differs from swagger
* Switch Podman service to vfs storage driver

Signed-off-by: Jhon Honce <[email protected]>

.cirrus.yml

@@ -39,8 +39,7 @@ gating_task:
         - make
         - go get github.com/vbatts/git-validation
         - make validate
-        - make unittest
-
+        - make tests
 
 success_task:
     # This task is a required-pass in github settings,

Makefile

@@ -8,7 +8,7 @@ DESTDIR ?=
 EPOCH_TEST_COMMIT ?= $(shell git merge-base $${DEST_BRANCH:-master} HEAD)
 HEAD ?= HEAD
 
-export PODMAN_VERSION ?= '1.80'
+export PODMAN_VERSION ?= '3.0.0'
 
 .PHONY: podman-py
 podman-py: env
@@ -17,22 +17,28 @@ podman-py: env
 
 .PHONY: env
 env:
-	dnf install python3-coverage python3-pylint python3-requests python3-requests-mock python3-fixtures -y
-	# -- or --
-	# $(PYTHON) -m pip install tox
-	# -- or --
+	# see contrib/cirrus/gating/Dockerfile
+	dnf install python3-coverage python3-pylint python3-requests python3-requests-mock python3-fixtures \
+		podman -y
+
 .PHONY: lint
 lint:
 	$(PYTHON) -m pylint podman || exit $$(($$? % 4));
 
+.PHONY: tests
+tests:
+	DEBUG=1 coverage run -m unittest discover -s podman/tests
+	coverage report -m --skip-covered --fail-under=80 --omit=./podman/tests/* --omit=.tox/* --omit=/usr/lib/*
+
 .PHONY: unittest
 unittest:
 	coverage run -m unittest discover -s podman/tests/unit
 	coverage report -m --skip-covered --fail-under=80 --omit=./podman/tests/* --omit=.tox/* --omit=/usr/lib/*
 
-# .PHONY: integration
-# integration:
-# 	test/integration/test_runner.sh -v
+.PHONY: integration
+integration:
+	coverage run -m unittest discover -s podman/tests/integration
+	coverage report -m --skip-covered --fail-under=80 --omit=./podman/tests/* --omit=.tox/* --omit=/usr/lib/*
 
 # .PHONY: install
 HEAD ?= HEAD

contrib/cirrus/gating/Dockerfile

@@ -8,5 +8,7 @@ RUN dnf -y install golang \
 			python3-requests-mock \
 			python3-coverage \
 			python3-pylint \
-			python3-fixtures && \
+			python3-fixtures \
+			podman \
+			&& \
 			dnf -y clean all

podman/__init__.py

@@ -1,6 +1,10 @@
 """Podman client module."""
+import logging
+
 from podman.api_connection import ApiConnection
 from podman.client import PodmanClient, from_env
 
+from podman.api.version import __version__
+
 # isort: unique-list
-__all__ = ['ApiConnection', 'PodmanClient', 'from_env']
+__all__ = ['ApiConnection', 'PodmanClient', '__version__', 'from_env']

podman/api/__init__.py

@@ -1,23 +1,37 @@
 """Tools for connecting to a Podman service."""
-
 from podman.api.client import APIClient
-from podman.api.parse_utils import decode_header, parse_repository, prepare_body, prepare_timestamp
-from podman.api.tar_utils import create_tar, prepare_dockerfile, prepare_dockerignore
+from podman.api.parse_utils import (
+    decode_header,
+    parse_repository,
+    prepare_body,
+    prepare_cidr,
+    prepare_timestamp,
+)
+from podman.api.tar_utils import create_tar, prepare_containerfile, prepare_dockerignore
 from podman.api.url_utils import prepare_filters
 
-DEFAULT_TIMEOUT = APIClient.default_timeout
+from . import version
+
+DEFAULT_TIMEOUT: float = 60.0
 DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024
 
+API_VERSION: str = version.__version__
+COMPATIBLE_VERSION: str = version.__compatible_version__
+
 # isort: unique-list
 __all__ = [
     'APIClient',
+    'API_VERSION',
+    'COMPATIBLE_VERSION',
+    'DEFAULT_CHUNK_SIZE',
     'DEFAULT_TIMEOUT',
     'create_tar',
     'decode_header',
-    'prepare_filters',
     'parse_repository',
     'prepare_body',
-    'prepare_dockerfile',
+    'prepare_cidr',
+    'prepare_containerfile',
     'prepare_dockerignore',
+    'prepare_filters',
     'prepare_timestamp',
 ]

podman/api/client.py

@@ -1,10 +1,11 @@
 """APIClient for connecting to Podman service."""
 import urllib.parse
-from typing import IO, Any, ClassVar, Dict, Iterable, List, Mapping, Optional, Tuple, Union
+from typing import IO, Any, ClassVar, Iterable, List, Mapping, Optional, Tuple, Union
 
 import requests
 from requests import Response
 
+from podman import api
 from podman.api.uds import UDSAdapter
 from podman.errors.exceptions import APIError
 from podman.tlsconfig import TLSConfig
@@ -30,24 +31,8 @@ class APIClient(requests.Session):
     # pylint: disable=arguments-differ
     # pylint: disable=too-many-instance-attributes
 
-    # TODO pull version from a future Version library
-    api_version: str = "3.0.0"
-    compatible_version: str = "1.40"
-
-    default_timeout: float = 60.0
-
     supported_schemes: ClassVar[List[str]] = ("unix", "http+unix")
 
-    required_headers: ClassVar[Dict[str, str]] = {
-        "User-Agent": f"PodmanPy/{api_version}",
-    }
-    """These headers are included in all requests to service."""
-
-    default_headers: ClassVar[Dict[str, str]] = {}
-    """These headers are included in all requests to service.
-       Headers provided in request will override.
-    """
-
     def __init__(
         self,
         base_url: str = None,
@@ -68,31 +53,37 @@ def __init__(
 
         _ = tls
 
-        # FIXME map scheme unix:// -> http+unix://,
-        # FIXME url quote slashes in netloc
-        self.base_url = base_url
-
-        self.version = version or APIClient.api_version
-        self.path_prefix = f"/v{self.version}/libpod"
-        self.compatible_version = (
-            kwargs.get("compatible_version", None) or APIClient.compatible_version
-        )
-        self.compatible_prefix = f"/v{self.compatible_version}"
-
-        uri = urllib.parse.urlparse(self.base_url)
+        uri = urllib.parse.urlparse(base_url)
         if uri.scheme not in APIClient.supported_schemes:
             raise ValueError(
                 f"The scheme '{uri.scheme}' is not supported, only {APIClient.supported_schemes}"
             )
 
-        self.uri = uri
-        self.timeout = timeout or APIClient.default_timeout
-        self.user_agent = user_agent or f"PodmanPy/{self.version}"
+        if uri.scheme == "unix":
+            uri = uri._replace(scheme="http+unix")
+
+        if uri.netloc == "":
+            uri = uri._replace(netloc=uri.path)._replace(path="")
+
+        if "/" in uri.netloc:
+            uri = uri._replace(netloc=urllib.parse.quote_plus(uri.netloc))
+
+        self.base_url = uri.geturl()
+
+        if uri.scheme == "http+unix":
+            self.mount(uri.scheme, UDSAdapter())
+
+        self.version = version or api.API_VERSION
+        self.path_prefix = f"/v{self.version}/libpod"
+        self.compatible_version = kwargs.get("compatible_version") or api.COMPATIBLE_VERSION
+        self.compatible_prefix = f"/v{self.compatible_version}"
+
+        self.timeout = timeout or api.DEFAULT_TIMEOUT
         self.pool_maxsize = num_pools or requests.adapters.DEFAULT_POOLSIZE
         self.credstore_env = credstore_env or {}
 
-        if uri.scheme == "http+unix" or "unix":
-            self.mount(uri.scheme, UDSAdapter())
+        self.user_agent = user_agent or f"PodmanPy/{self.version}"
+        self.headers.update({"User-Agent": self.user_agent})
 
     def delete(
         self,
@@ -298,35 +289,24 @@ def _request(
             APIError: when service returns an Error.
         """
         if timeout is None:
-            timeout = APIClient.default_timeout
+            timeout = api.DEFAULT_TIMEOUT
 
         if not path.startswith("/"):
             path = f"/{path}"
 
         compatible = kwargs.get("compatible", False)
         path_prefix = self.compatible_prefix if compatible else self.path_prefix
+        uri = self.base_url + path_prefix + path
 
         try:
             return self.request(
                 method.upper(),
-                self.base_url + path_prefix + path,
+                uri,
                 params=params,
                 data=data,
-                headers=self._headers(headers or {}),
+                headers=(headers or {}),
                 timeout=timeout,
                 stream=stream,
             )
         except OSError as e:
-            raise APIError(path) from e
-
-    @classmethod
-    def _headers(cls, headers: Mapping[str, str]) -> Dict[str, str]:
-        """Generate header dictionary for request.
-
-        Args:
-            headers: headers unique to this request.
-        """
-        hdrs = APIClient.default_headers.copy()
-        hdrs.update(headers)
-        hdrs.update(APIClient.required_headers)
-        return hdrs
+            raise APIError(uri, explanation=f"{method.upper()} operation failed") from e

podman/api/parse_utils.py

@@ -1,5 +1,6 @@
 """Helper functions for parsing strings."""
 import base64
+import ipaddress
 import json
 from datetime import datetime
 from typing import Any, Dict, MutableMapping, Optional, Tuple, Union
@@ -59,3 +60,12 @@ def prepare_timestamp(value: Union[datetime, int, None]) -> Optional[int]:
         return delta.seconds + delta.days * 24 * 3600
 
     raise ValueError(f"Type '{type(value)}' is not supported by prepare_timestamp()")
+
+
+def prepare_cidr(value: Union[ipaddress.IPv4Network, ipaddress.IPv6Network]) -> (str, str):
+    """Returns network address and Base64 encoded netmask from CIDR.
+
+    Notes:
+        The return values are dictated by the Go JSON decoder.
+    """
+    return str(value.network_address), base64.b64encode(value.netmask.packed).decode("utf-8")

podman/api/tar_utils.py

@@ -25,7 +25,7 @@ def prepare_dockerignore(anchor: str) -> List[str]:
         )
 
 
-def prepare_dockerfile(anchor: str, dockerfile: str) -> str:
+def prepare_containerfile(anchor: str, dockerfile: str) -> str:
     """Ensure that Dockerfile or a proxy Dockerfile is in context_dir.
 
     Args:
@@ -41,15 +41,22 @@ def prepare_dockerfile(anchor: str, dockerfile: str) -> str:
     if dockerfile_path.parent.samefile(anchor_path):
         return dockerfile
 
-    proxy_path = anchor_path / f".dockerfile.{random.getrandbits(160):x}"
+    proxy_path = anchor_path / f".containerfile.{random.getrandbits(160):x}"
     shutil.copy2(dockerfile_path, proxy_path, follow_symlinks=False)
-    return str(proxy_path)
+    return proxy_path.name
 
 
 def create_tar(
-    anchor: str, name: Optional[str] = None, exclude: List[str] = None, gzip: bool = False
+    anchor: str, name: str = None, exclude: List[str] = None, gzip: bool = False
 ) -> BinaryIO:
-    """Create a tarfile from context_dir to send to Podman service"""
+    """Create a tarfile from context_dir to send to Podman service.
+
+    Args:
+        anchor: Directory to use as root of tar file.
+        name: Name of tar file.
+        exclude: List of patterns for files to exclude from tar file.
+        gzip: When True, gzip compress tar file.
+    """
 
     def add_filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
         """Filter files targeted to be added to tarfile.
@@ -64,6 +71,7 @@ def add_filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
         Notes:
             exclude is captured from parent
         """
+
         if not (info.isfile() or info.isdir() or info.issym()):
             return None
 
@@ -80,21 +88,27 @@ def add_filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
 
         if sys.platform == 'win32':
             info.mode = info.mode & 0o755 | 0o111
+
         return info
 
     if name is None:
-        name = tempfile.NamedTemporaryFile()
+        name = tempfile.NamedTemporaryFile(prefix="podman_build_context", suffix=".tar")
     else:
         name = pathlib.Path(name)
 
+    if exclude is None:
+        exclude = list()
+    else:
+        exclude = exclude.copy()
+
     exclude.append(".dockerignore")
-    exclude.append("!" + str(name))
+    exclude.append(name.name)
 
     mode = "w:gz" if gzip else "w"
-    with tarfile.open(name, mode) as tar:
-        tar.add(anchor, arcname=os.path.basename(anchor), recursive=True, filter=add_filter)
+    with tarfile.open(name.name, mode) as tar:
+        tar.add(anchor, arcname="", recursive=True, filter=add_filter)
 
-    return open(name, "rb")
+    return open(name.name, "rb")
 
 
 def _exclude_matcher(path: str, exclude: List[str]) -> bool:

podman/api/url_utils.py

@@ -39,6 +39,7 @@ def _format_dict(filters, criteria):
     for key, value in filters.items():
         if value is None:
             continue
+        value = str(value)
 
         if key in criteria:
             criteria[key].append(value)

podman/api/version.py

@@ -0,0 +1,3 @@
+"""Version of PodmanPy."""
+__version__ = "3.0.0"
+__compatible_version__ = "1.40"