Skip to content

Commit

Permalink
Adding support for file-based reads and externalized type dependencies (
Browse files Browse the repository at this point in the history
#103)

This is non-breaking change that adds a new public method:

read_files - a file-oriented entry point to the front end. This takes a
list of target DSDL files allowing the user to maintain an explicit list
instead of depending on globular filesystem discovery. Furthermore this
method returns a set which is the transitive closure of types depended
on by the target list of types. This allows consumers to track
dependencies and compiler back ends to generate .d files and otherwise
support incremental builds.

The new method may increase performance for systems with large pools of
messages when generating code for a small sub-set as it only considers
the target and dependencies of the target when parsing dsdl files.

---------

Co-authored-by: Pavel Kirienko <[email protected]>
  • Loading branch information
thirtytwobits and pavel-kirienko authored Jun 28, 2024
1 parent 98453a4 commit c7d8ef9
Show file tree
Hide file tree
Showing 19 changed files with 1,581 additions and 320 deletions.
24 changes: 24 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"name": "Python dev environment",
"image": "ghcr.io/opencyphal/toxic:tx22.4.2",
"workspaceFolder": "/workspace",
"workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=delegated",
"mounts": [
"source=root-vscode-server,target=/root/.vscode-server/extensions,type=volume",
"source=pydsdl-tox,target=/workspace/.nox,type=volume"
],
"customizations": {
"vscode": {
"extensions": [
"uavcan.dsdl",
"wholroyd.jinja",
"streetsidesoftware.code-spell-checker",
"ms-python.python",
"ms-python.mypy-type-checker",
"ms-python.black-formatter",
"ms-python.pylint"
]
}
},
"postCreateCommand": "git clone --depth 1 [email protected]:OpenCyphal/public_regulated_data_types.git .dsdl-test && nox -e test-3.12"
}
24 changes: 14 additions & 10 deletions .github/workflows/test-and-release.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: 'Test and Release PyDSDL'
on: push
on: [ push, pull_request ]

# Ensures that only one workflow is running at a time
concurrency:
Expand All @@ -9,11 +9,14 @@ concurrency:
jobs:
pydsdl-test:
name: Test PyDSDL
# Run on push OR on 3rd-party PR.
# https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=edited#pull_request
if: (github.event_name == 'push') || github.event.pull_request.head.repo.fork
strategy:
fail-fast: false
matrix:
os: [ ubuntu-latest, macos-latest ]
python: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
python: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
include:
- os: windows-2019
python: '3.10'
Expand All @@ -22,10 +25,10 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- name: Check out pydsdl
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Check out public_regulated_data_types
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
repository: OpenCyphal/public_regulated_data_types
path: .dsdl-test
Expand All @@ -50,27 +53,28 @@ jobs:
- name: Run build and test
run: |
if [ "$RUNNER_OS" == "Linux" ]; then
nox --non-interactive --error-on-missing-interpreters --session test test_eol pristine lint --python ${{ matrix.python }}
nox --non-interactive --error-on-missing-interpreters --session test pristine lint --python ${{ matrix.python }}
nox --non-interactive --session docs
elif [ "$RUNNER_OS" == "Windows" ]; then
nox --forcecolor --non-interactive --error-on-missing-interpreters --session test test_eol pristine lint
nox --forcecolor --non-interactive --error-on-missing-interpreters --session test pristine lint
elif [ "$RUNNER_OS" == "macOS" ]; then
nox --non-interactive --error-on-missing-interpreters --session test test_eol pristine lint --python ${{ matrix.python }}
nox --non-interactive --error-on-missing-interpreters --session test pristine lint --python ${{ matrix.python }}
else
echo "${{ runner.os }} not supported"
exit 1
fi
python -c "import pydsdl; pydsdl.read_namespace('.dsdl-test/uavcan', [])"
shell: bash

pydsdl-release:
name: Release PyDSDL
if: contains(github.event.head_commit.message, '#release') || contains(github.ref, '/master')
if: >
(github.event_name == 'push') &&
(contains(github.event.head_commit.message, '#release') || contains(github.ref, '/master'))
needs: pydsdl-test
runs-on: ubuntu-latest
steps:
- name: Check out
uses: actions/checkout@v3
uses: actions/checkout@v4

- name: Build distribution
run: |
Expand Down
7 changes: 7 additions & 0 deletions .readthedocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@

version: 2

build:
os: ubuntu-22.04
tools:
python: "3.12"
apt_packages:
- graphviz

sphinx:
configuration: docs/conf.py
fail_on_warning: true
Expand Down
63 changes: 63 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#
# Copyright (C) OpenCyphal Development Team <opencyphal.org>
# Copyright Amazon.com Inc. or its affiliates.
# SPDX-License-Identifier: MIT
#
"""
Configuration for pytest tests including fixtures and hooks.
"""

import tempfile
from pathlib import Path
from typing import Any, Optional

import pytest


# +-------------------------------------------------------------------------------------------------------------------+
# | TEST FIXTURES
# +-------------------------------------------------------------------------------------------------------------------+
class TemporaryDsdlContext:
"""
Powers the temp_dsdl_factory test fixture.
"""
def __init__(self) -> None:
self._base_dir: Optional[Any] = None

def new_file(self, file_path: Path, text: Optional[str] = None) -> Path:
if file_path.is_absolute():
raise ValueError(f"{file_path} is an absolute path. The test fixture requires relative paths to work.")
file = self.base_dir / file_path
file.parent.mkdir(parents=True, exist_ok=True)
if text is not None:
file.write_text(text)
return file

@property
def base_dir(self) -> Path:
if self._base_dir is None:
self._base_dir = tempfile.TemporaryDirectory()
return Path(self._base_dir.name).resolve()

def _test_path_finalizer(self) -> None:
"""
Finalizer to clean up any temporary directories created during the test.
"""
if self._base_dir is not None:
self._base_dir.cleanup()
del self._base_dir
self._base_dir = None

@pytest.fixture(scope="function")
def temp_dsdl_factory(request: pytest.FixtureRequest) -> Any: # pylint: disable=unused-argument
"""
Fixture for pydsdl tests that have to create files as part of the test. This object stays in-scope for a given
test method and does not requires a context manager in the test itself.
Call `new_file(path)` to create a new file path in the fixture's temporary directory. This will create all
uncreated parent directories but will _not_ create the file unless text is provided: `new_file(path, "hello")`
"""
f = TemporaryDsdlContext()
request.addfinalizer(f._test_path_finalizer) # pylint: disable=protected-access
return f

1 change: 1 addition & 0 deletions docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Contents
--------

.. toctree::
:maxdepth: 2

pages/installation
pages/pydsdl
Expand Down
5 changes: 3 additions & 2 deletions docs/pages/pydsdl.rst
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@ You can find a practical usage example in the Nunavut code generation library th
:local:


The main function
+++++++++++++++++
The main functions
++++++++++++++++++

.. autofunction:: pydsdl.read_namespace
.. autofunction:: pydsdl.read_files


Type model
Expand Down
4 changes: 2 additions & 2 deletions docs/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
sphinx == 4.4.0
sphinx_rtd_theme == 1.0.0
sphinx == 7.1.2 # this is the last version that supports Python 3.8
sphinx_rtd_theme == 2.0.0
sphinx-computron >= 0.2, < 2.0
25 changes: 9 additions & 16 deletions noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import nox


PYTHONS = ["3.8", "3.9", "3.10", "3.11"]
PYTHONS = ["3.8", "3.9", "3.10", "3.11", "3.12"]
"""The newest supported Python shall be listed LAST."""

nox.options.error_on_external_run = True
Expand All @@ -33,7 +33,6 @@ def clean(session):
"*.log",
"*.tmp",
".nox",
".dsdl-test",
]
for w in wildcards:
for f in Path.cwd().glob(w):
Expand All @@ -49,9 +48,9 @@ def test(session):
session.log("Using the newest supported Python: %s", is_latest_python(session))
session.install("-e", ".")
session.install(
"pytest ~= 7.3",
"pytest-randomly ~= 3.12",
"coverage ~= 7.2",
"pytest ~= 8.1",
"pytest-randomly ~= 3.15",
"coverage ~= 7.5",
)
session.run("coverage", "run", "-m", "pytest")
session.run("coverage", "report", "--fail-under=95")
Expand All @@ -61,14 +60,6 @@ def test(session):
session.log(f"OPEN IN WEB BROWSER: file://{report_file}")


@nox.session(python=["3.7"])
def test_eol(session):
"""This is a minimal test session for those old Pythons that have EOLed."""
session.install("-e", ".")
session.install("pytest")
session.run("pytest")


@nox.session(python=PYTHONS)
def pristine(session):
"""
Expand All @@ -85,8 +76,9 @@ def pristine(session):
def lint(session):
session.log("Using the newest supported Python: %s", is_latest_python(session))
session.install(
"mypy ~= 1.2.0",
"pylint ~= 2.17.2",
"mypy ~= 1.10",
"types-parsimonious",
"pylint ~= 3.2",
)
session.run(
"mypy",
Expand All @@ -105,7 +97,8 @@ def lint(session):
},
)
if is_latest_python(session):
session.install("black ~= 23.3")
# we run black only on the newest Python version to ensure that the code is formatted with the latest version
session.install("black ~= 24.4")
session.run("black", "--check", ".")


Expand Down
5 changes: 3 additions & 2 deletions pydsdl/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import sys as _sys
from pathlib import Path as _Path

__version__ = "1.20.1"
__version__ = "1.21.0"
__version_info__ = tuple(map(int, __version__.split(".")[:3]))
__license__ = "MIT"
__author__ = "OpenCyphal"
Expand All @@ -25,8 +25,9 @@
_sys.path = [str(_Path(__file__).parent / "third_party")] + _sys.path

# Never import anything that is not available here - API stability guarantees are only provided for the exposed items.
from ._dsdl import PrintOutputHandler as PrintOutputHandler
from ._namespace import read_namespace as read_namespace
from ._namespace import PrintOutputHandler as PrintOutputHandler
from ._namespace import read_files as read_files

# Error model.
from ._error import FrontendError as FrontendError
Expand Down
20 changes: 8 additions & 12 deletions pydsdl/_bit_length_set/_symbolic_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
# This software is distributed under the terms of the MIT License.
# Author: Pavel Kirienko <[email protected]>

import typing
import random
import itertools
from ._symbolic import NullaryOperator, validate_numerically
Expand Down Expand Up @@ -140,7 +139,7 @@ def _unittest_repetition() -> None:
)
assert op.min == 7 * 3
assert op.max == 17 * 3
assert set(op.expand()) == set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3))) # type: ignore
assert set(op.expand()) == set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))
assert set(op.expand()) == {21, 25, 29, 31, 33, 35, 39, 41, 45, 51}
assert set(op.modulo(7)) == {0, 1, 2, 3, 4, 5, 6}
assert set(op.modulo(8)) == {1, 3, 5, 7}
Expand All @@ -149,15 +148,15 @@ def _unittest_repetition() -> None:
for _ in range(1):
child = NullaryOperator(random.randint(0, 100) for _ in range(random.randint(1, 10)))
k = random.randint(0, 10)
ref = set(map(sum, itertools.combinations_with_replacement(child.expand(), k))) # type: ignore
ref = set(map(sum, itertools.combinations_with_replacement(child.expand(), k)))
op = RepetitionOperator(child, k)
assert set(op.expand()) == ref

assert op.min == min(child.expand()) * k
assert op.max == max(child.expand()) * k

div = random.randint(1, 64)
assert set(op.modulo(div)) == {typing.cast(int, x) % div for x in ref}
assert set(op.modulo(div)) == {x % div for x in ref}

validate_numerically(op)

Expand All @@ -173,9 +172,9 @@ def _unittest_range_repetition() -> None:
assert op.max == 17 * 3
assert set(op.expand()) == (
{0}
| set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 1))) # type: ignore
| set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 2))) # type: ignore
| set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3))) # type: ignore
| set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 1)))
| set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 2)))
| set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))
)
assert set(op.expand()) == {0, 7, 11, 14, 17, 18, 21, 22, 24, 25, 28, 29, 31, 33, 34, 35, 39, 41, 45, 51}
assert set(op.modulo(7)) == {0, 1, 2, 3, 4, 5, 6}
Expand All @@ -197,10 +196,7 @@ def _unittest_range_repetition() -> None:
k_max = random.randint(0, 10)
ref = set(
itertools.chain(
*(
map(sum, itertools.combinations_with_replacement(child.expand(), k)) # type: ignore
for k in range(k_max + 1)
)
*(map(sum, itertools.combinations_with_replacement(child.expand(), k)) for k in range(k_max + 1))
)
)
op = RangeRepetitionOperator(child, k_max)
Expand All @@ -210,7 +206,7 @@ def _unittest_range_repetition() -> None:
assert op.max == max(child.expand()) * k_max

div = random.randint(1, 64)
assert set(op.modulo(div)) == {typing.cast(int, x) % div for x in ref}
assert set(op.modulo(div)) == {x % div for x in ref}

validate_numerically(op)

Expand Down
Loading

0 comments on commit c7d8ef9

Please sign in to comment.