diff --git a/.github/workflows/buildcheck.yml b/.github/workflows/buildcheck.yml index 4c7735f5..0339e7c1 100644 --- a/.github/workflows/buildcheck.yml +++ b/.github/workflows/buildcheck.yml @@ -76,7 +76,6 @@ jobs: mkdir /tmp/pypi_vss_test cd /tmp/pypi_vss_test # Just verify that we can start the tools - vspec2x.py --help vspec2csv.py --help vspec2json.py --help vspec2yaml.py --help diff --git a/README-PYPI.md b/README-PYPI.md index c992c152..7737b2c4 100644 --- a/README-PYPI.md +++ b/README-PYPI.md @@ -24,7 +24,7 @@ If you just want the latest version this should be sufficient: pip install vss-tools ``` -When installed tools like `vspec2x.py` shall be available on your path. +When installed tools like `vspec2json.py` shall be available on your path. For more information see the [VSS-Tools wiki](https://github.com/COVESA/vss-tools/wiki/PyPI-packing) diff --git a/README.md b/README.md index 67d74be2..458281e1 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,10 @@ Examples on tool usage can be found in the [VSS Makefile](https://github.com/COV Tool | Description | Tool Category | Documentation | | ------------------ | ----------- |-----------------------|-----------------------------------------------------------------------------------------------------------------------| -| [vspec2x.py](vspec2x.py) | Parses and expands VSS into different text based output formats. Currently supports `json`, `yaml`,`csv`,`idl` | Community Supported | Try `./vspec2x --help` or check [vspec2x documentation](docs/vspec2x.md) | -[vspec2csv.py](vspec2csv.py) | Shortcut for [vspec2x.py](vspec2x.py) generating CSV output | Community Supported | Check [vspec2x documentation](docs/vspec2x.md) | -[vspec2ddsidl.py](vspec2ddsidl.py) | Shortcut for [vspec2x.py](vspec2x.py) generating DDS-IDL output | Community Supported | [VSS2DDSIDL Documentation](docs/VSS2DDSIDL.md). For general parameters check [vspec2x documentation](docs/vspec2x.md) | -[vspec2json.py](vspec2json.py) | Shortcut for [vspec2x.py](vspec2x.py) generating JSON output | Community Supported | Check [vspec2x documentation](docs/vspec2x.md) | -[vspec2yaml.py](vspec2yaml.py) | Shortcut for [vspec2x.py](vspec2x.py) generating flattened YAML output | Community Supported | Check [vspec2x documentation](docs/vspec2x.md) | +[vspec2csv.py](vspec2csv.py) | Generating CSV output | Community Supported | Check [vspec2x documentation](docs/vspec2x.md) | +[vspec2ddsidl.py](vspec2ddsidl.py) | Generating DDS-IDL output | Community Supported | [VSS2DDSIDL Documentation](docs/VSS2DDSIDL.md). For general parameters check [vspec2x documentation](docs/vspec2x.md) | +[vspec2json.py](vspec2json.py) | Generating JSON output | Community Supported | Check [vspec2x documentation](docs/vspec2x.md) | +[vspec2yaml.py](vspec2yaml.py) | Generating flattened YAML output | Community Supported | Check [vspec2x documentation](docs/vspec2x.md) | [vspec2binary.py](vspec2binary.py) | The binary toolset consists of a tool that translates the VSS YAML specification to the binary file format (see below), and two libraries that provides methods that are likely to be needed by a server that manages the VSS tree, one written in C, and one in Go | Community Supported | [vspec2binary Documentation](binary/README.md). For general parameters check [vspec2x documentation](docs/vspec2x.md) | [vspec2franca.py](vspec2franca.py) | Parses and expands a VSS and generates a Franca IDL specification | Community Supported | Check [vspec2x documentation](docs/vspec2x.md) | [vspec2c.py](obsolete/vspec2c.py) | The vspec2c tooling allows a vehicle signal specification to be translated from its source YAML file to native C code that has the entire specification encoded in it. | Obsolete (2022-11-01) | [Documentation](obsolete/vspec2c/README.md) | diff --git a/docs/vspec2x.md b/docs/vspec2x.md index c97511a3..eaff17d2 100644 --- a/docs/vspec2x.md +++ b/docs/vspec2x.md @@ -1,58 +1,55 @@ -# vspec2x converters +# Vspec2x-based generators -vspec2x is a family of VSS converters that share a common codebase. +[vspec2x](../vspec/vspec2x.py) is a generator framework that offers functionality to parse one or more vspec files and +transform that to an internal VSS model, which can be used as input for generators. -As a consequence it provides general commandline parameters guiding the parsing of vspec, as well as parameters specific to specific output formats. +Vspec2x-based generators have a number of common arguments. +It is partially configurable which arguments that shall be available for each generator, +so all arguments described in this documents are not available for all generators. +In addition to this generator-specific arguments may be supported -You can get a description of supported commandline parameters by running `vspec2x.py --help`. - -This documentation will give some examples and elaborate more on specific parameters. +You can get a description of supported commandline arguments by running ` --help`, e.g. `vss2json.py --help`. +In this document `vspec2json.py` is generally used as example, but the same syntax is typically available also for other tools. The supported arguments might look like this - ``` -usage: vspec2x.py [-h] [-I dir] [-e EXTENDED_ATTRIBUTES] [-s] [--abort-on-unknown-attribute] [--abort-on-name-style] - [--format format] [--uuid] [--no_expand] [-o overlays] [-u unit_file] [-q quantity_file] - [-vt vspec_types_file] [-ot ] - [--json-all-extended-attributes] [--json-pretty] - [--yaml-all-extended-attributes] [-v version] [--all-idl-features] [--gqlfield GQLFIELD GQLFIELD] - +``` +$ vspec2json.py --help +usage: vspec2json.py [-h] [-I dir] [-e EXTENDED_ATTRIBUTES] [-s] [--abort-on-unknown-attribute] [--abort-on-name-style] [--uuid] [--no-expand] + [-o overlays] [-q quantity_file] [-u unit_file] [-vt vspec_types_file] + [-ot ] [--json-all-extended-attributes] [--json-pretty] + ``` An example command line to convert the VSS standard catalog into a JSON file is ``` -% python vspec2x.py --format json -I ../spec -u ../spec/units.yaml ../spec/VehicleSignalSpecification.vspec vss.json -Output to json format -Known extended attributes: -Reading unit definitions from ../spec/units.yaml -Loading vspec from ../spec/VehicleSignalSpecification.vspec... -Calling exporter... -Generating JSON output... -Serializing compact JSON... -All done. +$ vspec2json.py -I ../spec -u ../vehicle_signal_specification/spec/units.yaml ../vehicle_signal_specification/spec/VehicleSignalSpecification.vspec vss.json +INFO Known extended attributes: +INFO Added 29 quantities from /home/erik/vehicle_signal_specification/spec/quantities.yaml +INFO Added 61 units from ../vehicle_signal_specification/spec/units.yaml +INFO Loading vspec from ../vehicle_signal_specification/spec/VehicleSignalSpecification.vspec... +INFO Calling exporter... +INFO Generating JSON output... +INFO Serializing compact JSON... +INFO All done. + ``` -This assumes you checked out the [COVESA Vehicle Signal Specification](https://github.com/covesa/vehicle_signal_specification) which contains vss-tools including vspec2x as a submodule. +This assumes you checked out the [COVESA Vehicle Signal Specification](https://github.com/covesa/vehicle_signal_specification) which contains vss-tools as a submodule. The `-I` parameter adds a directory to search for includes referenced in you `.vspec` files. `-I` can be used multiple times to specify more include directories. The `-u` parameter specifies the unit file(s) to use. The `-q` parameter specifies quantity file(s) to use. The first positional argument - `../spec/VehicleSignalSpecification.vspec` in the example - gives the (root) `.vspec` file to be converted. The second positional argument - `vss.json` in the example - is the output file. -The `--format` parameter determines the output format, `JSON` in our example. If format is omitted `vspec2x` tries to guess the correct output format based on the extension of the second positional argument. Alternatively vss-tools supports *shortcuts* for community supported exporters, e.g. `vspec2json.py` for generating JSON. The shortcuts really only add the `--format` parameter for you, so - -``` -python vspec2json.py -I ../spec -u ../spec/units.yaml ../spec/VehicleSignalSpecification.vspec vss.json -``` - -is equivalent to the example above. +It is the file `vspec2json.py` that specified which generator to use, i.e. which output to generate. ## General parameters ### --abort-on-unknown-attribute Terminates parsing when an unknown attribute is encountered, that is an attribute that is not defined in the [VSS standard catalogue](https://covesa.github.io/vehicle_signal_specification/rule_set/), and not whitelisted using the extended attribute parameter `-e` (see below). -*Note*: Here an *attribute* refers to VSS signal metadata sich as "datatype", "min", "max", ... and not to the VSS signal type attribute +*Note*: Here an *attribute* refers to VSS signal metadata such as "datatype", "min", "max", ... and not to the VSS signal type attribute ### --abort-on-name-style Terminates parsing, when the name of a signal does not follow [VSS recomendations](https://covesa.github.io/vehicle_signal_specification/rule_set/basics/#naming-conventions). @@ -214,7 +211,7 @@ When deciding which quantities to use the tooling use the following logic: As of today use of quantity files is optional, and tooling will only give a warning if a unit use a quantity not specified in a quantity file. ## Handling of overlays and extensions -`vspec2x` allows composition of several overlays on top of a base vspec, to extend the model or overwrite certain metadata. Check [VSS documentation](https://covesa.github.io/vehicle_signal_specification/introduction/) on the concept of overlays. +The generator framework allows composition of several overlays on top of a base vspec, to extend the model or overwrite certain metadata. Check [VSS documentation](https://covesa.github.io/vehicle_signal_specification/introduction/) on the concept of overlays. Overlays are in general injected before the VSS tree is expanded. Expansion is the process where branches with instances are transformed into multiple branches. An example is the `Vehicle.Cabin.Door` branch which during expansion get transformed into `Vehicle.Cabin.Door.Row1.Left`, `Vehicle.Cabin.Door.Row1.Right`, `Vehicle.Cabin.Door.Row2.Left`and `Vehicle.Cabin.Door.Row2.Right`. @@ -357,23 +354,15 @@ Will also generate non-payload const attributes such as unit/datatype. Default i Add additional fields to the nodes in the graphql schema. use: -## Writing your own exporter -This is easy. Put the code in file in the [vssexporters directory](../vspec/vssexporters/). +## Writing your own generator -Mandatory functions to be implemented are -```python -def add_arguments(parser: argparse.ArgumentParser): -``` - -and +This is done by creating a subclass of `class Vss2X`, see [vss2x.py](../vspec/vss2x.py). +You need at least to implement the abstract method `generate`. +In addition you may want to customize which generic arguments that shall be used and add +generator specific arguments, if needed. -```python -def export(config: argparse.Namespace, root: VSSNode): -``` -See one of the existing exporters for an example. -Add your exporter module to the `Exporter` class in [vspec2x.py](../vspec2x.py). +A generic implementation example is available as a [test case](../tests/generators). -## Design Decisions and Architecture Please see [vspec2x architecture document](vspec2x_arch.md). diff --git a/setup.py b/setup.py index a3ce29f0..2d714cdb 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ url='https://github.com/COVESA/vss-tools', license='Mozilla Public License 2.0', packages=find_packages(exclude=('tests', 'contrib')), - scripts=['vspec2csv.py', 'vspec2x.py', 'vspec2franca.py', 'vspec2json.py', 'vspec2jsonschema.py', + scripts=['vspec2csv.py', 'vspec2franca.py', 'vspec2json.py', 'vspec2jsonschema.py', 'vspec2ddsidl.py', 'vspec2yaml.py', 'vspec2protobuf.py', 'vspec2graphql.py', 'vspec2id.py'], python_requires='>=3.8', diff --git a/tests/generators/example_generator.py b/tests/generators/example_generator.py new file mode 100755 index 00000000..9d9b82ba --- /dev/null +++ b/tests/generators/example_generator.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 Contributors to COVESA +# +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License 2.0 which is available at +# https://www.mozilla.org/en-US/MPL/2.0/ +# +# SPDX-License-Identifier: MPL-2.0 + +import sys +import logging +from vspec.model.vsstree import VSSNode +import argparse +from typing import Optional +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vspec2x import Vspec2X + + +class ExampleGenerator(Vss2X): + """ + This is an example on how easy you can write your own generator + """ + + def __init__(self, vspec2vss_config: Vspec2VssConfig, keyword: str): + # Change default configs + # By default Vspec2X requires that an output file is needed and specified + # That is not needed for the ExampleGenerator as it just use stdout + # By that reason we change default config to indicate that no output arguments is needed + # That also implies that there will be an error if an output argument is specified + vspec2vss_config.output_file_required = False + + # A lot of other things we do not care about, this reduces number or arguments shown + # if you do "./example_generator.py --help" + vspec2vss_config.uuid_supported = False + vspec2vss_config.extended_attributes_supported = False + vspec2vss_config.type_tree_supported = False + + # The rest here is generator-specific initializations + self.keyword = str.lower(keyword) + self.count = 0 + + def handle_node(self, node: VSSNode): + if self.keyword in str.lower(node.comment): + self.count += 1 + for child in node.children: + self.handle_node(child) + + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + """ + It is required to implement the generate method + """ + self.handle_node(signal_root) + + logging.info("Generating Example output...") + logging.info("I found %d comments with %s", self.count, self.keyword) + + +if __name__ == "__main__": + vspec2vss_config = Vspec2VssConfig() + # The generator shall know nothing about vspec processing or vspec2vss arguments! + # (Even if it may have some expectations on how the model look like) + generator = ExampleGenerator(vspec2vss_config, "VSS contributor") + vspec2x = Vspec2X(generator, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/tests/generators/test.vspec b/tests/generators/test.vspec new file mode 100644 index 00000000..51b6248e --- /dev/null +++ b/tests/generators/test.vspec @@ -0,0 +1,34 @@ +# Copyright (c) 2023 Contributors to COVESA +# +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License 2.0 which is available at +# https://www.mozilla.org/en-US/MPL/2.0/ +# +# SPDX-License-Identifier: MPL-2.0 + +A: + type: branch + description: Branch A. + +############ Testing Single Line Comments ############## + +A.Erik: + datatype: float + type: sensor + unit: km + description: A sensor. + comment: A VSS contributor! + +A.Sebastian: + datatype: float + type: sensor + unit: km + description: A sensor. + comment: A VSS contributor! + +A.ChuckNorris: + datatype: float + type: sensor + unit: km + description: A sensor. + comment: An actor! diff --git a/tests/generators/test_generator.py b/tests/generators/test_generator.py new file mode 100644 index 00000000..1618b63a --- /dev/null +++ b/tests/generators/test_generator.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Contributors to COVESA +# +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License 2.0 which is available at +# https://www.mozilla.org/en-US/MPL/2.0/ +# +# SPDX-License-Identifier: MPL-2.0 + +import pytest +import os + + +@pytest.fixture +def change_test_dir(request, monkeypatch): + # To make sure we run from test directory + monkeypatch.chdir(request.fspath.dirname) + + +def test_generator(change_test_dir): + + test_str = "./example_generator.py -u ../vspec/test_units.yaml test.vspec > out.txt 2>&1" + result = os.system(test_str) + assert os.WIFEXITED(result) + assert os.WEXITSTATUS(result) == 0 + + test_str = 'grep \"I found 2 comments with vss contributor\" out.txt > /dev/null' + result = os.system(test_str) + assert os.WIFEXITED(result) + assert os.WEXITSTATUS(result) == 0 + os.system("rm -f out.txt") diff --git a/tests/vspec/test_multiple_type_trees/test_multiple_type_trees.py b/tests/vspec/test_multiple_type_trees/test_multiple_type_trees.py index b16a7f4f..6512b1e0 100644 --- a/tests/vspec/test_multiple_type_trees/test_multiple_type_trees.py +++ b/tests/vspec/test_multiple_type_trees/test_multiple_type_trees.py @@ -22,7 +22,7 @@ def test_error(change_test_dir): """ Verify that you cannot have multiple type trees, ie. both TypesA and TypesB, there must be a single root """ - test_str = "../../../vspec2x.py --format csv -vt struct1.vspec -vt struct2.vspec -u ../test_units.yaml " \ + test_str = "../../../vspec2csv.py -vt struct1.vspec -vt struct2.vspec -u ../test_units.yaml " \ "test.vspec out.csv 1> out.txt 2>&1" result = os.system(test_str) assert os.WIFEXITED(result) diff --git a/tests/vspec/test_no_expand/test_no_expand.py b/tests/vspec/test_no_expand/test_no_expand.py index e7908767..ab8186d1 100644 --- a/tests/vspec/test_no_expand/test_no_expand.py +++ b/tests/vspec/test_no_expand/test_no_expand.py @@ -20,17 +20,17 @@ def change_test_dir(request, monkeypatch): monkeypatch.chdir(request.fspath.dirname) -@pytest.mark.parametrize("format, output_file, comparison_file, is_warning_expected", [ +@pytest.mark.parametrize("format, output_file, comparison_file, is_error_expected", [ ('json', 'out.json', 'expected.json', False), ('yaml', 'out.yaml', 'expected.yaml', False), ('csv', 'out.csv', 'expected.csv', False), ('protobuf', 'out.proto', 'expected.proto', True), - ('idl', 'out.idl', 'expected.idl', True), + ('ddsidl', 'out.idl', 'expected.idl', True), ('franca', 'out.fidl', 'expected.fidl', True), ('graphql', 'out.graphql', 'expected.graphql', True)]) -def test_no_expand(format, output_file, comparison_file, is_warning_expected: bool, change_test_dir): +def test_no_expand(format, output_file, comparison_file, is_error_expected: bool, change_test_dir): - args = ["../../../vspec2x.py", "--no-expand", "--format", format] + args = ["../../../vspec2" + format + ".py", "--no-expand"] if format == 'json': args.append('--json-pretty') args.extend(["-u", "../test_units.yaml", @@ -40,22 +40,24 @@ def test_no_expand(format, output_file, comparison_file, is_warning_expected: bo result = os.system(test_str) os.system("cat out.txt") assert os.WIFEXITED(result) - assert os.WEXITSTATUS(result) == 0 + if is_error_expected: + assert os.WEXITSTATUS(result) != 0 + else: + assert os.WEXITSTATUS(result) == 0 - # For exporters not supporting "no-expand" a warning shall be given - test_str = 'grep \"no_expand not supported by exporter\" out.txt > /dev/null' + # For exporters not supporting "no-expand" an error shall be given + test_str = 'grep \"error: unrecognized arguments: --no-expand\" out.txt > /dev/null' result = os.system(test_str) assert os.WIFEXITED(result) - if is_warning_expected: + if is_error_expected: assert os.WEXITSTATUS(result) == 0 else: assert os.WEXITSTATUS(result) == 1 - - test_str = f"diff {output_file} {comparison_file}" - result = os.system(test_str) - os.system("rm -f out.txt") - assert os.WIFEXITED(result) - assert os.WEXITSTATUS(result) == 0 + test_str = f"diff {output_file} {comparison_file}" + result = os.system(test_str) + os.system("rm -f out.txt") + assert os.WIFEXITED(result) + assert os.WEXITSTATUS(result) == 0 os.system(f"rm -f {output_file}") @@ -67,12 +69,12 @@ def test_no_expand(format, output_file, comparison_file, is_warning_expected: bo (True, 'expected_overlay_no_expand.json')]) def test_json_overlay(no_expand, comparison_file, change_test_dir): """Test with overlay and expansion (for reference/comparison)""" - args = ["../../../vspec2x.py"] + args = ["../../../vspec2json.py"] if no_expand: args.append('--no-expand') - args.extend(["--format", "json", "--json-pretty", "-u", "../test_units.yaml", + args.extend(["--json-pretty", "-u", "../test_units.yaml", "test.vspec", "-o", "overlay.vspec", "out.json", "1>", "out.txt", "2>&1"]) test_str = " ".join(args) diff --git a/tests/vspec/test_overlay_struct/test_overlay_struct.py b/tests/vspec/test_overlay_struct/test_overlay_struct.py index 237b72ce..8a898795 100644 --- a/tests/vspec/test_overlay_struct/test_overlay_struct.py +++ b/tests/vspec/test_overlay_struct/test_overlay_struct.py @@ -29,7 +29,7 @@ def test_overlay_struct(format, signals_out, expected_signal, change_test_dir): """ Test that data types provided in vspec format are converted correctly """ - args = ["../../../vspec2x.py", "--format", format] + args = ["../../../vspec2" + format + ".py"] if format == 'json': args.append('--json-pretty') args.extend(["-vt", "struct1.vspec", "-vt", "struct2.vspec", "-u", "../test_units.yaml", @@ -64,7 +64,7 @@ def test_overlay_struct_using_struct(format, signals_out, expected_signal, chang """ Test that data types provided in vspec format are converted correctly """ - args = ["../../../vspec2x.py", "--format", format] + args = ["../../../vspec2" + format + ".py"] if format == 'json': args.append('--json-pretty') args.extend(["-vt", "struct1.vspec", "-vt", "struct2_using_struct1.vspec", "-u", "../test_units.yaml", diff --git a/tests/vspec/test_overlay_struct_array/test_overlay_struct_array.py b/tests/vspec/test_overlay_struct_array/test_overlay_struct_array.py index 4f83b3d0..8a41407c 100644 --- a/tests/vspec/test_overlay_struct_array/test_overlay_struct_array.py +++ b/tests/vspec/test_overlay_struct_array/test_overlay_struct_array.py @@ -29,7 +29,7 @@ def test_overlay_struct_array(format, signals_out, expected_signal, change_test_ """ Test that data types provided in vspec format are converted correctly """ - args = ["../../../vspec2x.py", "--format", format] + args = ["../../../vspec2" + format + ".py"] if format == 'json': args.append('--json-pretty') args.extend(["-vt", "struct1.vspec", "-u", "../test_units.yaml", diff --git a/tests/vspec/test_static_uids/test_static_uids.py b/tests/vspec/test_static_uids/test_static_uids.py index 2aa8a6c5..68e40a04 100644 --- a/tests/vspec/test_static_uids/test_static_uids.py +++ b/tests/vspec/test_static_uids/test_static_uids.py @@ -15,7 +15,7 @@ import shlex import vspec import vspec.vssexporters.vss2id as vss2id -import vspec2x +import vspec2id import yaml from typing import Dict @@ -180,7 +180,7 @@ def test_duplicate_hash(caplog: pytest.LogCaptureFixture, children_names: list): def test_full_script(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 0 @@ -189,7 +189,7 @@ def test_full_script(caplog: pytest.LogCaptureFixture): def test_semantic(caplog: pytest.LogCaptureFixture): validation_file: str = "./validation_vspecs/validation_semantic_change.vspec" clas = shlex.split(get_cla_validation(validation_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 1 and all( log.levelname == "WARNING" for log in caplog.records @@ -202,7 +202,7 @@ def test_semantic(caplog: pytest.LogCaptureFixture): def test_vss_path(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test_vss_path.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 1 and all( log.levelname == "WARNING" for log in caplog.records @@ -215,7 +215,7 @@ def test_vss_path(caplog: pytest.LogCaptureFixture): def test_unit(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test_unit.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 2 and all( log.levelname == "WARNING" for log in caplog.records @@ -228,7 +228,7 @@ def test_unit(caplog: pytest.LogCaptureFixture): def test_datatype(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test_datatype.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 1 and all( log.levelname == "WARNING" for log in caplog.records @@ -241,7 +241,7 @@ def test_datatype(caplog: pytest.LogCaptureFixture): def test_name_datatype(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test_name_datatype.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 2 and all( log.levelname == "WARNING" for log in caplog.records @@ -257,7 +257,7 @@ def test_name_datatype(caplog: pytest.LogCaptureFixture): def test_deprecation(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test_deprecation.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 1 and all( log.levelname == "WARNING" for log in caplog.records @@ -270,7 +270,7 @@ def test_deprecation(caplog: pytest.LogCaptureFixture): def test_description(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test_description.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 1 and all( log.levelname == "WARNING" for log in caplog.records @@ -283,7 +283,7 @@ def test_description(caplog: pytest.LogCaptureFixture): def test_added_attribute(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test_added_attribute.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 1 and all( log.levelname == "WARNING" for log in caplog.records @@ -296,7 +296,7 @@ def test_added_attribute(caplog: pytest.LogCaptureFixture): def test_deleted_attribute(caplog: pytest.LogCaptureFixture): test_file: str = "./test_vspecs/test_deleted_attribute.vspec" clas = shlex.split(get_cla_test(test_file)) - vspec2x.main(["--format", "idgen"] + clas[1:]) + vspec2id.main(clas[1:]) assert len(caplog.records) == 1 and all( log.levelname == "WARNING" for log in caplog.records diff --git a/tests/vspec/test_struct_as_root/test_struct_as_root.py b/tests/vspec/test_struct_as_root/test_struct_as_root.py index 4d7dc329..395eda84 100644 --- a/tests/vspec/test_struct_as_root/test_struct_as_root.py +++ b/tests/vspec/test_struct_as_root/test_struct_as_root.py @@ -19,7 +19,7 @@ def change_test_dir(request, monkeypatch): def test_struct_as_root(change_test_dir): - test_str = "../../../vspec2x.py --format csv -vt struct1.vspec -o overlay.vspec" + \ + test_str = "../../../vspec2csv.py -vt struct1.vspec -o overlay.vspec" + \ " -u ../test_units.yaml test.vspec out.csv > out.txt 2>&1" result = os.system(test_str) assert os.WIFEXITED(result) diff --git a/tests/vspec/test_structs/test_commandline.py b/tests/vspec/test_structs/test_commandline.py index 797f3b31..6cb7370d 100644 --- a/tests/vspec/test_structs/test_commandline.py +++ b/tests/vspec/test_structs/test_commandline.py @@ -20,7 +20,7 @@ def change_test_dir(request, monkeypatch): def test_error_when_data_types_file_is_missing(change_test_dir): # test that program fails due to parser error - cmdline = '../../../vspec2x.py -u ../test_units.yaml -ot output_types_file.json test.vspec output_file.json' + cmdline = '../../../vspec2json.py -u ../test_units.yaml -ot output_types_file.json test.vspec output_file.json' test_str = cmdline + " 1> out.txt 2>&1" result = os.system(test_str) assert os.WIFEXITED(result) @@ -39,15 +39,16 @@ def test_error_when_data_types_file_is_missing(change_test_dir): @pytest.mark.parametrize("format", ["binary", "franca", "graphql"]) def test_error_with_non_compatible_formats(format, change_test_dir): # test that program fails due to parser error - cmdline = ('../../../vspec2x.py -u ../test_units.yaml -vt VehicleDataTypes.vspec -ot output_types_file.json' - f' --format {format} test.vspec output_file.json') + cmdline = ('../../../vspec2' + format + '.py -u ../test_units.yaml -vt VehicleDataTypes.vspec ' + '-ot output_types_file.json' + 'test.vspec output_file.json') test_str = cmdline + " 1> out.txt 2>&1" result = os.system(test_str) assert os.WIFEXITED(result) assert os.WEXITSTATUS(result) != 0 # test that the expected error is outputted - test_str = f'grep \"error: {format} format is not yet supported in vspec struct/data type support feature\" ' + \ + test_str = 'grep \"error: unrecognized arguments: -vt\" ' + \ 'out.txt > /dev/null' result = os.system(test_str) os.system("cat out.txt") diff --git a/tests/vspec/test_structs/test_data_type_parsing.py b/tests/vspec/test_structs/test_data_type_parsing.py index 025cd068..e8ff7a6c 100644 --- a/tests/vspec/test_structs/test_data_type_parsing.py +++ b/tests/vspec/test_structs/test_data_type_parsing.py @@ -28,7 +28,7 @@ def test_data_types_export_single_file(format, signals_out, expected_signal, typ """ Test that data types provided in vspec format are converted correctly """ - args = ["../../../vspec2x.py", "--format", format] + args = ["../../../vspec2" + format + ".py"] if format == 'json': args.append('--json-pretty') args.extend(["-vt", type_file, "-u", "../test_units.yaml", @@ -58,7 +58,7 @@ def test_data_types_export_multi_file(format, signals_out, data_types_out, """ Test that data types provided in vspec format are converted correctly """ - args = ["../../../vspec2x.py", "--format", format] + args = ["../../../vspec2" + format + ".py"] if format == 'json': args.append('--json-pretty') args.extend(["-vt", "VehicleDataTypes.vspec", "-u", "../test_units.yaml", "-ot", data_types_out, @@ -103,7 +103,7 @@ def test_data_types_export_to_proto(signal_vspec_file, type_vspec_file, expected """ data_types_out = Path.cwd() / "unused.proto" - args = ["../../../vspec2x.py", "--format", "protobuf", + args = ["../../../vspec2protobuf.py", "-vt", type_vspec_file, "-u", "../test_units.yaml", "-ot", str(data_types_out), signal_vspec_file, actual_signal_file, "1>", "out.txt", "2>&1"] test_str = " ".join(args) @@ -148,7 +148,7 @@ def test_data_types_invalid_reference_in_data_type_tree( """ Test that errors are surfaced when data type name references are invalid within the data type tree """ - test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--format", "json", + test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--json-pretty", "-vt", types_file, "-ot", "VehicleDataTypes.json", "test.vspec", "out.json", "1>", "out.txt", "2>&1"]) result = os.system(test_str) @@ -171,7 +171,7 @@ def test_data_types_orphan_properties( """ Test that errors are surfaced when a property is not defined under a struct """ - test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--format", "json", + test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--json-pretty", "-vt", types_file, "-ot", "VehicleDataTypes.json", "test.vspec", "out.json", "1>", "out.txt", "2>&1"]) result = os.system(test_str) @@ -190,7 +190,7 @@ def test_data_types_invalid_reference_in_signal_tree(change_test_dir): """ Test that errors are surfaced when data type name references are invalid in the signal tree """ - test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--format", "json", + test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--json-pretty", "-vt", "VehicleDataTypes.vspec", "-ot", "VehicleDataTypes.json", "test-invalid-datatypes.vspec", "out.json", "1>", "out.txt", "2>&1"]) @@ -213,7 +213,7 @@ def test_error_when_no_user_defined_data_types_are_provided(change_test_dir): Test that error message is provided when user-defined types are specified in the signal tree but no data type tree is provided. """ - test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--format", "json", + test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--json-pretty", "test.vspec", "out.json", "1>", "out.txt", "2>&1"]) result = os.system(test_str) assert os.WIFEXITED(result) @@ -241,7 +241,7 @@ def test_faulty_use_of_standard_attributes( """ Test faulty use of datatype and unit for structs """ - test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--format", "json", + test_str = " ".join(["../../../vspec2json.py", "-u", "../test_units.yaml", "--json-pretty", "-vt", types_file, "-ot", "VehicleDataTypes.json", vspec_file, "out.json", "1>", "out.txt", "2>&1"]) result = os.system(test_str) diff --git a/tests/vspec/test_types_with_uuid/expected_no_uuid.graphql b/tests/vspec/test_types_with_uuid/expected_no_uuid.graphql deleted file mode 100644 index 552c7f19..00000000 --- a/tests/vspec/test_types_with_uuid/expected_no_uuid.graphql +++ /dev/null @@ -1,59 +0,0 @@ -type Query { - vehicle( - """VIN of the vehicle that you want to request data for.""" - id: String! - - """ - Filter data to only provide information that was sent from the vehicle after that timestamp. - """ - after: String - ): A -} - -"""Branch A.""" -type A { - """A sensor.""" - sensor: A_Sensor - - """An attribute.""" - attribute: A_Attribute - - """An actuator.""" - actuator: A_Actuator -} - -"""A sensor.""" -type A_Sensor { - """Value: A sensor.""" - value: Float - - """Timestamp: A sensor.""" - timestamp: String - - """Unit of A sensor.""" - unit: String -} - -"""An attribute.""" -type A_Attribute { - """Value: An attribute.""" - value: Float - - """Timestamp: An attribute.""" - timestamp: String - - """Unit of An attribute.""" - unit: String -} - -"""An actuator.""" -type A_Actuator { - """Value: An actuator.""" - value: Float - - """Timestamp: An actuator.""" - timestamp: String - - """Unit of An actuator.""" - unit: String -} diff --git a/tests/vspec/test_types_with_uuid/expected_uuid.graphql b/tests/vspec/test_types_with_uuid/expected_uuid.graphql deleted file mode 100644 index 552c7f19..00000000 --- a/tests/vspec/test_types_with_uuid/expected_uuid.graphql +++ /dev/null @@ -1,59 +0,0 @@ -type Query { - vehicle( - """VIN of the vehicle that you want to request data for.""" - id: String! - - """ - Filter data to only provide information that was sent from the vehicle after that timestamp. - """ - after: String - ): A -} - -"""Branch A.""" -type A { - """A sensor.""" - sensor: A_Sensor - - """An attribute.""" - attribute: A_Attribute - - """An actuator.""" - actuator: A_Actuator -} - -"""A sensor.""" -type A_Sensor { - """Value: A sensor.""" - value: Float - - """Timestamp: A sensor.""" - timestamp: String - - """Unit of A sensor.""" - unit: String -} - -"""An attribute.""" -type A_Attribute { - """Value: An attribute.""" - value: Float - - """Timestamp: An attribute.""" - timestamp: String - - """Unit of An attribute.""" - unit: String -} - -"""An actuator.""" -type A_Actuator { - """Value: An actuator.""" - value: Float - - """Timestamp: An actuator.""" - timestamp: String - - """Unit of An actuator.""" - unit: String -} diff --git a/tests/vspec/test_types_with_uuid/test_uuid.py b/tests/vspec/test_types_with_uuid/test_uuid.py index 6bdb6f88..db45d056 100644 --- a/tests/vspec/test_types_with_uuid/test_uuid.py +++ b/tests/vspec/test_types_with_uuid/test_uuid.py @@ -33,21 +33,20 @@ def run_exporter(exporter, argument, compare_suffix): def test_uuid(change_test_dir): - # Run all "supported" exporters, i.e. not those in contrib + # Run all "supported" exporters that supports uuid, i.e. not those in contrib # Exception is "binary", as it is assumed output may vary depending on # target - exporters = ["json", "ddsidl", "csv", "yaml", "franca", "graphql"] + exporters = ["json", "ddsidl", "csv", "yaml", "franca"] for exporter in exporters: run_exporter(exporter, "--uuid", "uuid") - # Same behavior expected if no argument run_exporter(exporter, "", "no_uuid") -def run_obsolete_arg_test(argument, obsolete_arg_expected: bool): - test_str = "../../../vspec2json.py " + argument + " -u ../test_units.yaml test.vspec out.json 1> out.txt 2>&1" +def run_error_test(tool, argument, arg_error_expected: bool): + test_str = "../../../" + tool + " " + argument + " -u ../test_units.yaml test.vspec out.json 1> out.txt 2>&1" result = os.system(test_str) assert os.WIFEXITED(result) - if obsolete_arg_expected: + if arg_error_expected: assert os.WEXITSTATUS(result) != 0 else: assert os.WEXITSTATUS(result) == 0 @@ -56,13 +55,23 @@ def run_obsolete_arg_test(argument, obsolete_arg_expected: bool): os.system("cat out.txt") os.system("rm -f out.json out.txt") assert os.WIFEXITED(result) - if obsolete_arg_expected: + if arg_error_expected: assert os.WEXITSTATUS(result) == 0 else: assert os.WEXITSTATUS(result) != 0 def test_obsolete_arg(change_test_dir): - run_obsolete_arg_test("", False) - run_obsolete_arg_test("--uuid", False) - run_obsolete_arg_test("--no-uuid", True) + """ + Check that obsolete argument --no-uuid results in error + """ + run_error_test("vspec2json.py", "", False) + run_error_test("vspec2json.py", "--uuid", False) + run_error_test("vspec2json.py", "--no-uuid", True) + + +def test_uuid_unsupported(change_test_dir): + """ + Test that we get an error if using --uuid for tools not supporting it + """ + run_error_test("vspec2graphql.py", "--uuid", True) diff --git a/vspec/vspec2vss_config.py b/vspec/vspec2vss_config.py new file mode 100644 index 00000000..b564af4a --- /dev/null +++ b/vspec/vspec2vss_config.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 Contributors to COVESA +# +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License 2.0 which is available at +# https://www.mozilla.org/en-US/MPL/2.0/ +# +# SPDX-License-Identifier: MPL-2.0 + +class Vspec2VssConfig: + """ + This class is intended to be a container for settings related to the conversion + from *.vspec files to the internal VSS model as well as for some generic vspec2x features. + A user may change settings before vspec2x.main is called. + Then vspec2x will use those settings to control which command line arguments + that will be accepted or considered, and based on those some of the settings in this + file may be overridden. + The config instance is later passed to the vss2x instance, so that it can consider + the settings as well. + """ + + def __init__(self): + """ + Setting default/recommended features for vss-tools. + """ + + # Input to model parsing + + # Is it possible to request that uuid shall be generated + self.uuid_supported = True + # Shall it be possible to specify which extended attributes to consider? + self.extended_attributes_supported = True + # Shall it be possible to give a type tree + self.type_tree_supported = True + + # shall vspec2vss expand the model (by default) + self.expand_model = True + # if so shall there anyway be an option to NOT expand + self.no_expand_option_supported = True + + # Is an output file required as part of command line arguments + self.output_file_required = True + + # As of now we have only one type of nodes in the type tree + # and that is structs, so if we support type tree we assume structs are supported + + # Default values for features + # These are typically updated by vspec2x when reading command line arguments + self.generate_uuid = False diff --git a/vspec/vspec2x.py b/vspec/vspec2x.py new file mode 100755 index 00000000..8a9e4719 --- /dev/null +++ b/vspec/vspec2x.py @@ -0,0 +1,180 @@ + + +# Copyright (c) 2016 Contributors to COVESA +# +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License 2.0 which is available at +# https://www.mozilla.org/en-US/MPL/2.0/ +# +# SPDX-License-Identifier: MPL-2.0 + +# +# Convert vspec files to various other formats +# + +from vspec.model.vsstree import VSSNode +from vspec.model.constants import VSSTreeType +from vspec.loggingconfig import initLogging +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig +import argparse +import logging +import sys +import vspec + + +class Vspec2X(): + """ + Framework for translating from *.vspec files to first an internal VSS model, + and then to something else called X. + Users must provide a Vss2X generator that can generate X if they get a VSS model as input. + """ + def __init__(self, generator: Vss2X, vspec2vss_config: Vspec2VssConfig): + + self.generator = generator + self.vspec2vss_config = vspec2vss_config + + def main(self, arguments): + initLogging() + parser = argparse.ArgumentParser(description="Convert vspec to other formats.") + + parser.add_argument('-I', '--include-dir', action='append', metavar='dir', type=str, default=[], + help='Add include directory to search for included vspec files.') + if self.vspec2vss_config.extended_attributes_supported: + parser.add_argument('-e', '--extended-attributes', type=str, default="", + help='Whitelisted extended attributes as comma separated list.') + parser.add_argument('-s', '--strict', action='store_true', + help='Use strict checking: Terminate when anything not covered or not recommended ' + 'by VSS language or extensions is found.') + parser.add_argument('--abort-on-unknown-attribute', action='store_true', + help=" Terminate when an unknown attribute is found.") + parser.add_argument('--abort-on-name-style', action='store_true', + help=" Terminate naming style not follows recommendations.") + if self.vspec2vss_config.uuid_supported: + parser.add_argument('--uuid', action='store_true', + help='Include uuid in generated files.') + if self.vspec2vss_config.expand_model and self.vspec2vss_config.no_expand_option_supported: + parser.add_argument('--no-expand', action='store_true', + help='Do not expand tree.') + parser.add_argument('-o', '--overlays', action='append', metavar='overlays', type=str, default=[], + help='Add overlay that will be layered on top of the VSS file in the order they appear.') + parser.add_argument('-q', '--quantity-file', action='append', metavar='quantity_file', type=str, default=[], + help='Quantity file to be used for generation. Argument -uqmay be used multiple times.') + parser.add_argument('-u', '--unit-file', action='append', metavar='unit_file', type=str, default=[], + help='Unit file to be used for generation. Argument -u may be used multiple times.') + parser.add_argument('vspec_file', metavar='', + help='The vehicle specification file to convert.') + if self.vspec2vss_config.output_file_required: + parser.add_argument('output_file', metavar='', + help='The file to write output to.') + + if self.vspec2vss_config.type_tree_supported: + type_group = parser.add_argument_group('VSS Data Type Tree arguments', + 'Arguments related to struct/type support') + + type_group.add_argument('-vt', '--vspec-types-file', action='append', metavar='vspec_types_file', type=str, + default=[], + help='Data types file in vspec format.') + type_group.add_argument('-ot', '--types-output-file', metavar='', + help='Output file for writing data types from vspec file. ' + + 'If not specified, a single file is used where applicable. ' + + 'In case of JSON and YAML, the data is exported under a ' + + 'special key - "ComplexDataTypes"') + + self.generator.add_arguments(parser.add_argument_group( + "Exporter specific arguments", "")) + + args = parser.parse_args(arguments) + + include_dirs = ["."] + include_dirs.extend(args.include_dir) + + abort_on_unknown_attribute = False + abort_on_namestyle = False + + if args.abort_on_unknown_attribute or args.strict: + abort_on_unknown_attribute = True + if args.abort_on_name_style or args.strict: + abort_on_namestyle = True + + if self.vspec2vss_config.extended_attributes_supported: + known_extended_attributes_list = args.extended_attributes.split(",") + if len(known_extended_attributes_list) > 0: + vspec.model.vsstree.VSSNode.whitelisted_extended_attributes = known_extended_attributes_list + logging.info(f"Known extended attributes: {', '.join(known_extended_attributes_list)}") + else: + known_extended_attributes_list = list() + + self.vspec2vss_config.generate_uuid = self.vspec2vss_config.uuid_supported and args.uuid + + self.vspec2vss_config.expand_model = (self.vspec2vss_config.expand_model and not + (self.vspec2vss_config.no_expand_option_supported and args.no_expand)) + + vspec.load_quantities(args.vspec_file, args.quantity_file) + vspec.load_units(args.vspec_file, args.unit_file) + + # process data type tree + data_type_tree = None + if self.vspec2vss_config.type_tree_supported: + if args.types_output_file is not None and not args.vspec_types_file: + parser.error("An output file for data types was provided. Please also provide " + "the input vspec file for data types") + if args.vspec_types_file: + data_type_tree = self.processDataTypeTree( + parser, args, include_dirs, abort_on_namestyle) + vspec.verify_mandatory_attributes(data_type_tree, abort_on_unknown_attribute) + + try: + logging.info(f"Loading vspec from {args.vspec_file}...") + tree = vspec.load_tree( + args.vspec_file, include_dirs, VSSTreeType.SIGNAL_TREE, + break_on_name_style_violation=abort_on_namestyle, + expand_inst=False, data_type_tree=data_type_tree) + + for overlay in args.overlays: + logging.info(f"Applying VSS overlay from {overlay}...") + othertree = vspec.load_tree(overlay, include_dirs, VSSTreeType.SIGNAL_TREE, + break_on_name_style_violation=abort_on_namestyle, expand_inst=False, + data_type_tree=data_type_tree) + vspec.merge_tree(tree, othertree) + + vspec.check_type_usage(tree, VSSTreeType.SIGNAL_TREE, data_type_tree) + if self.vspec2vss_config.expand_model: + vspec.expand_tree_instances(tree) + + vspec.clean_metadata(tree) + vspec.verify_mandatory_attributes(tree, abort_on_unknown_attribute) + logging.info("Calling exporter...") + + self.generator.generate(args, tree, self.vspec2vss_config, data_type_tree) + logging.info("All done.") + except vspec.VSpecError as e: + logging.error(f"Error: {e}") + sys.exit(255) + + def processDataTypeTree(self, parser: argparse.ArgumentParser, args, include_dirs, + abort_on_namestyle: bool) -> VSSNode: + """ + Helper function to process command line arguments and invoke logic for processing data + type information provided in vspec format + """ + if args.types_output_file is None: + logging.info("Sensors and custom data types will be consolidated into one file.") + # Nope, we should not know this + # if args.format == Exporter.protobuf: + # logging.info("Proto files will be written to the current working directory") + + logging.warning("All exports do not yet support structs. Please check documentation for your exporter!") + + first_tree = True + for type_file in args.vspec_types_file: + logging.info(f"Loading and processing struct/data type tree from {type_file}") + new_tree = vspec.load_tree(type_file, include_dirs, VSSTreeType.DATA_TYPE_TREE, + break_on_name_style_violation=abort_on_namestyle, expand_inst=False) + if first_tree: + tree = new_tree + first_tree = False + else: + vspec.merge_tree(tree, new_tree) + vspec.check_type_usage(tree, VSSTreeType.DATA_TYPE_TREE) + return tree diff --git a/vspec/vss2x.py b/vspec/vss2x.py new file mode 100644 index 00000000..be15483c --- /dev/null +++ b/vspec/vss2x.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023 Contributors to COVESA +# +# This program and the accompanying materials are made available under the +# terms of the Mozilla Public License 2.0 which is available at +# https://www.mozilla.org/en-US/MPL/2.0/ +# +# SPDX-License-Identifier: MPL-2.0 + +from abc import ABC, abstractmethod +import argparse +from typing import Optional +from vspec.model.vsstree import VSSNode +from vspec.vspec2vss_config import Vspec2VssConfig + + +class Vss2X(ABC): + """ + Abstract class for something that takes a VSS model as input + (signal tree plus optionally a type tree) + and does "something". For now it is assumed that output is either files + or text output. + The generator may use both command line config as well as specific configs used for + creating the VSS model to control the generation. + """ + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.description = "This generator does not support any additional arguments." + + @abstractmethod + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + """ + Previously called export, but changing to a more generic name. + Must be defined by the tool. + """ + pass diff --git a/vspec/vssexporters/vss2binary.py b/vspec/vssexporters/vss2binary.py index f34cab7b..030f267b 100644 --- a/vspec/vssexporters/vss2binary.py +++ b/vspec/vssexporters/vss2binary.py @@ -14,17 +14,15 @@ import logging import ctypes import os.path +from typing import Optional from vspec.model.vsstree import VSSNode, VSSType +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig out_file = "" _cbinary = None -def feature_supported(feature_name: str): - """Return true for supported optional arguments/features""" - return False - - def createBinaryCnode(fname, nodename, nodetype, uuid, description, nodedatatype, nodemin, nodemax, unit, allowed, defaultAllowed, validate, children): global _cbinary @@ -54,10 +52,6 @@ def intToHexChar(hexInt): return chr(hexInt - 10 + ord('A')) -def add_arguments(parser: argparse.ArgumentParser): - parser.description = "The binary exporter does not support any additional arguments." - - def export_node(node, generate_uuid, out_file): nodename = str(node.name) b_nodename = nodename.encode('utf-8') @@ -124,23 +118,30 @@ def export_node(node, generate_uuid, out_file): export_node(child, generate_uuid, out_file) -def export(config: argparse.Namespace, root: VSSNode, print_uuid): - global _cbinary - dllName = "../../binary/binarytool.so" - dllAbsPath = os.path.dirname(os.path.abspath(__file__)) + os.path.sep + dllName - if not os.path.isfile(dllAbsPath): - logging.error("The required library binarytool.so is not available, exiting!") - logging.info("You must build the library, " - "see https://github.com/COVESA/vss-tools/blob/master/binary/README.md!") - return - _cbinary = ctypes.CDLL(dllAbsPath) - - _cbinary.createBinaryCnode.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, - ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, - ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, - ctypes.c_int) - - logging.info("Generating binary output...") - out_file = config.output_file - export_node(root, print_uuid, out_file) - logging.info("Binary output generated in " + out_file) +class Vss2Binary(Vss2X): + + def __init__(self, vspec2vss_config: Vspec2VssConfig): + vspec2vss_config.type_tree_supported = False + vspec2vss_config.no_expand_option_supported = False + + def generate(self, config: argparse.Namespace, root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + global _cbinary + dllName = "../../binary/binarytool.so" + dllAbsPath = os.path.dirname(os.path.abspath(__file__)) + os.path.sep + dllName + if not os.path.isfile(dllAbsPath): + logging.error("The required library binarytool.so is not available, exiting!") + logging.info("You must build the library, " + "see https://github.com/COVESA/vss-tools/blob/master/binary/README.md!") + return + _cbinary = ctypes.CDLL(dllAbsPath) + + _cbinary.createBinaryCnode.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, + ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, + ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, + ctypes.c_int) + + logging.info("Generating binary output...") + out_file = config.output_file + export_node(root, vspec2vss_config.generate_uuid, out_file) + logging.info("Binary output generated in " + out_file) diff --git a/vspec/vssexporters/vss2csv.py b/vspec/vssexporters/vss2csv.py index d554c1bc..f4e2437c 100644 --- a/vspec/vssexporters/vss2csv.py +++ b/vspec/vssexporters/vss2csv.py @@ -15,20 +15,12 @@ import logging from vspec.model.vsstree import VSSNode from anytree import PreOrderIter # type: ignore[import] -from vspec.loggingconfig import initLogging from typing import AnyStr +from typing import Optional +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig -def feature_supported(feature_name: str): - """Return true for supported arguments/features""" - if feature_name in ['no_expand']: - return True - return False - - -def add_arguments(parser: argparse.ArgumentParser): - parser.description = "The csv exporter does not support any additional arguments." - # Write the header line @@ -68,23 +60,23 @@ def print_csv_content(file, tree: VSSNode, uuid, include_instance_column: bool): file.write(format_csv_line(arg_list)) -def export(config: argparse.Namespace, signal_root: VSSNode, print_uuid, data_type_root: VSSNode): - logging.info("Generating CSV output...") - - # generic entry should be written when both data types and signals are being written to the same file - generic_entry = data_type_root is not None and config.types_output_file is None - with open(config.output_file, 'w') as f: - signal_entry_type = "Node" if generic_entry else "Signal" - print_csv_header(f, print_uuid, signal_entry_type, config.no_expand) - print_csv_content(f, signal_root, print_uuid, config.no_expand) - if data_type_root is not None and generic_entry is True: - print_csv_content(f, data_type_root, print_uuid, config.no_expand) +class Vss2Csv(Vss2X): - if data_type_root is not None and generic_entry is False: - with open(config.types_output_file, 'w') as f: - print_csv_header(f, print_uuid, "Node", config.no_expand) - print_csv_content(f, data_type_root, print_uuid, config.no_expand) + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + logging.info("Generating CSV output...") + # generic entry should be written when both data types and signals are being written to the same file + generic_entry = data_type_root is not None and config.types_output_file is None + include_instance_column = not vspec2vss_config.expand_model + with open(config.output_file, 'w') as f: + signal_entry_type = "Node" if generic_entry else "Signal" + print_csv_header(f, vspec2vss_config.generate_uuid, signal_entry_type, include_instance_column) + print_csv_content(f, signal_root, vspec2vss_config.generate_uuid, include_instance_column) + if data_type_root is not None and generic_entry is True: + print_csv_content(f, data_type_root, vspec2vss_config.generate_uuid, include_instance_column) -if __name__ == "__main__": - initLogging() + if data_type_root is not None and generic_entry is False: + with open(config.types_output_file, 'w') as f: + print_csv_header(f, vspec2vss_config.generate_uuid, "Node", include_instance_column) + print_csv_content(f, data_type_root, vspec2vss_config.generate_uuid, include_instance_column) diff --git a/vspec/vssexporters/vss2ddsidl.py b/vspec/vssexporters/vss2ddsidl.py index 7a85bb2e..0fdc8360 100644 --- a/vspec/vssexporters/vss2ddsidl.py +++ b/vspec/vssexporters/vss2ddsidl.py @@ -15,21 +15,11 @@ import argparse import keyword import logging +from typing import Optional -from vspec.loggingconfig import initLogging from vspec.model.vsstree import VSSNode, VSSType - - -def feature_supported(feature_name: str): - """Return true for supported optional arguments/features""" - return False - - -def add_arguments(parser: argparse.ArgumentParser): - parser.description = "The DDS-IDL exporter" - parser.add_argument('--all-idl-features', action='store_true', - help='Generate all features based on DDS IDL 4.2 specification') - +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig c_keywords = [ "auto", "break", "case", "char", "const", "continue", "default", "do", "double", "else", "enum", "extern", "float", @@ -278,16 +268,24 @@ def export_idl(file, root, generate_uuids=True, generate_all_idl_features=False) logging.info("IDL file generated at location : " + file.name) -def export(config: argparse.Namespace, signal_root: VSSNode, print_uuid, data_type_root: VSSNode): - logging.info("Generating DDS-IDL output...") +class Vss2DdsIdl(Vss2X): + + def __init__(self, vspec2vss_config: Vspec2VssConfig): + vspec2vss_config.no_expand_option_supported = False + + def add_arguments(self, parser: argparse.ArgumentParser): + parser.description = "The DDS-IDL exporter" + parser.add_argument('--all-idl-features', action='store_true', + help='Generate all features based on DDS IDL 4.2 specification') - if data_type_root is not None: - exporter = StructExporter() - with open(config.output_file, 'w') as idl_out: - idl_out.write(exporter.export(data_type_root)) + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + logging.info("Generating DDS-IDL output...") - with open(config.output_file, 'a' if data_type_root is not None else 'w') as idl_out: - export_idl(idl_out, signal_root, print_uuid, config.all_idl_features) + if data_type_root is not None: + exporter = StructExporter() + with open(config.output_file, 'w') as idl_out: + idl_out.write(exporter.export(data_type_root)) - if __name__ == "__main__": - initLogging() + with open(config.output_file, 'a' if data_type_root is not None else 'w') as idl_out: + export_idl(idl_out, signal_root, vspec2vss_config.generate_uuid, config.all_idl_features) diff --git a/vspec/vssexporters/vss2franca.py b/vspec/vssexporters/vss2franca.py index 9773b7ce..1228b139 100644 --- a/vspec/vssexporters/vss2franca.py +++ b/vspec/vssexporters/vss2franca.py @@ -12,19 +12,13 @@ import argparse +from typing import Optional from vspec.model.vsstree import VSSNode from anytree import PreOrderIter # type: ignore[import] +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig -def feature_supported(feature_name: str): - """Return true for supported optional arguments/features""" - return False - - -def add_arguments(parser: argparse.ArgumentParser): - # no additional output for Franca at this moment - parser.add_argument('-v', metavar='version', help=" Add version information to franca file.") - # Write the header line @@ -82,10 +76,23 @@ def print_franca_content(file, tree, uuid): file.write(f"{output}") -def export(config: argparse.Namespace, root: VSSNode, print_uuid): - print("Generating Franca output...") - outfile = open(config.output_file, 'w') - print_franca_header(outfile, config.v) - print_franca_content(outfile, root, print_uuid) - outfile.write("\n]") - outfile.close() +class Vss2Franca(Vss2X): + + def __init__(self, vspec2vss_config: Vspec2VssConfig): + vspec2vss_config.no_expand_option_supported = False + vspec2vss_config.type_tree_supported = False + + def add_arguments(self, parser: argparse.ArgumentParser): + # Renamed from -v to --franca-vss-version to avoid conflict when using + # -vt (Otherwise -vt would be interpreted as "-v t") + parser.add_argument('--franca-vss-version', metavar='franca_vss_version', + help=" Add version information to franca file.") + + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + print("Generating Franca output...") + outfile = open(config.output_file, 'w') + print_franca_header(outfile, config.franca_vss_version) + print_franca_content(outfile, signal_root, vspec2vss_config.generate_uuid) + outfile.write("\n]") + outfile.close() diff --git a/vspec/vssexporters/vss2graphql.py b/vspec/vssexporters/vss2graphql.py index a0b86825..c4f5dd65 100644 --- a/vspec/vssexporters/vss2graphql.py +++ b/vspec/vssexporters/vss2graphql.py @@ -14,6 +14,9 @@ from vspec.model.constants import VSSDataType from vspec import VSpecError from typing import Dict +from typing import Optional +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig from graphql import ( GraphQLSchema, @@ -57,19 +60,6 @@ } -def feature_supported(feature_name: str): - """Return true for supported optional arguments/features""" - return False - - -def add_arguments(parser: argparse.ArgumentParser): - # no additional output for graphql at this moment - parser.description = "The graphql exporter never generates uuid, i.e. the --uuid option has no effect." - parser.add_argument('--gqlfield', action='append', nargs=2, - help=" Add additional fields to the nodes in the graphql schema. " - "use: ") - - def get_schema_from_tree(root_node: VSSNode, additional_leaf_fields: list) -> str: """Takes a VSSNode and additional fields for the leafs. Returns a graphql schema as string.""" args = dict( @@ -132,9 +122,22 @@ def field(node: VSSNode, description_prefix="", type=GraphQLString) -> GraphQLFi ) -def export(config: argparse.Namespace, root: VSSNode, print_uuid): - print("Generating graphql output...") - outfile = open(config.output_file, 'w') - outfile.write(get_schema_from_tree(root, config.gqlfield)) - outfile.write("\n") - outfile.close() +class Vss2Graphql(Vss2X): + + def __init__(self, vspec2vss_config: Vspec2VssConfig): + vspec2vss_config.no_expand_option_supported = False + vspec2vss_config.type_tree_supported = False + vspec2vss_config.uuid_supported = False + + def add_arguments(self, parser: argparse.ArgumentParser): + parser.add_argument('--gqlfield', action='append', nargs=2, + help=" Add additional fields to the nodes in the graphql schema. " + "use: ") + + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + print("Generating graphql output...") + outfile = open(config.output_file, 'w') + outfile.write(get_schema_from_tree(signal_root, config.gqlfield)) + outfile.write("\n") + outfile.close() diff --git a/vspec/vssexporters/vss2id.py b/vspec/vssexporters/vss2id.py index 93f87d57..d700c74e 100644 --- a/vspec/vssexporters/vss2id.py +++ b/vspec/vssexporters/vss2id.py @@ -15,9 +15,9 @@ import os import sys from typing import Dict, Tuple +from typing import Optional from vspec import load_tree from vspec.model.constants import VSSTreeType -from vspec.loggingconfig import initLogging from vspec.model.vsstree import VSSNode from vspec.utils import vss2id_val from vspec.utils.idgen_utils import ( @@ -25,25 +25,11 @@ fnv1_32_hash, get_all_keys_values, ) +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig import yaml -def add_arguments(parser: argparse.ArgumentParser) -> None: - """Adds command line arguments to a pre-existing argument parser - - @param parser: the pre-existing argument parser - """ - parser.add_argument( - "--validate-static-uid", type=str, default="", help="Path to validation file." - ) - parser.add_argument( - "--only-validate-no-export", - action="store_true", - default=False, - help="For pytests and pipelines you can skip the export of the vspec file.", - ) - - def generate_split_id(node: VSSNode, id_counter: int) -> Tuple[str, int]: """Generates static UIDs using 4-byte FNV-1 hash. @@ -123,38 +109,53 @@ def export_node(yaml_dict, node, id_counter) -> Tuple[int, int]: return id_counter, id_counter -def export(config: argparse.Namespace, signal_root: VSSNode, print_uuid): - """Main export function used to generate the output id vspec. +class Vss2Id(Vss2X): - @param config: Command line arguments it was run with - @param signal_root: root of the signal tree - @param print_uuid: Not used here but needed by main script - """ - logging.info("Generating YAML output...") - - id_counter: int = 0 - signals_yaml_dict: Dict[str, str] = {} # Use str for ID values - id_counter, _ = export_node(signals_yaml_dict, signal_root, id_counter) - - if config.validate_static_uid: - logging.info( - f"Now validating nodes, static UIDs, types, units and description with " - f"file '{config.validate_static_uid}'" + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + """ + Adds command line arguments to a pre-existing argument parser + @param parser: the pre-existing argument parser + """ + parser.add_argument( + "--validate-static-uid", type=str, default="", help="Path to validation file." ) - if os.path.isabs(config.validate_static_uid): - other_path = config.validate_static_uid - else: - other_path = os.path.join(os.getcwd(), config.validate_static_uid) - - validation_tree = load_tree( - other_path, ["."], tree_type=VSSTreeType.SIGNAL_TREE + parser.add_argument( + "--only-validate-no-export", + action="store_true", + default=False, + help="For pytests and pipelines you can skip the export of the vspec file.", ) - vss2id_val.validate_static_uids(signals_yaml_dict, validation_tree, config) - - if not config.only_validate_no_export: - with open(config.output_file, "w") as f: - yaml.dump(signals_yaml_dict, f) - -if __name__ == "__main__": - initLogging() + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + + """Main export function used to generate the output id vspec. + + @param config: Command line arguments it was run with + @param signal_root: root of the signal tree + @param print_uuid: Not used here but needed by main script + """ + logging.info("Generating YAML output...") + + id_counter: int = 0 + signals_yaml_dict: Dict[str, str] = {} # Use str for ID values + id_counter, _ = export_node(signals_yaml_dict, signal_root, id_counter) + + if config.validate_static_uid: + logging.info( + f"Now validating nodes, static UIDs, types, units and description with " + f"file '{config.validate_static_uid}'" + ) + if os.path.isabs(config.validate_static_uid): + other_path = config.validate_static_uid + else: + other_path = os.path.join(os.getcwd(), config.validate_static_uid) + + validation_tree = load_tree( + other_path, ["."], tree_type=VSSTreeType.SIGNAL_TREE + ) + vss2id_val.validate_static_uids(signals_yaml_dict, validation_tree, config) + + if not config.only_validate_no_export: + with open(config.output_file, "w") as f: + yaml.dump(signals_yaml_dict, f) diff --git a/vspec/vssexporters/vss2json.py b/vspec/vssexporters/vss2json.py index 1766a90e..927343a8 100644 --- a/vspec/vssexporters/vss2json.py +++ b/vspec/vssexporters/vss2json.py @@ -15,22 +15,9 @@ import json import logging from typing import Dict, Any -from vspec.loggingconfig import initLogging - - -def feature_supported(feature_name: str): - """Return true for supported arguments/features""" - if feature_name in ['no_expand']: - return True - return False - - -def add_arguments(parser: argparse.ArgumentParser): - parser.add_argument('--json-all-extended-attributes', action='store_true', - help="Generate all extended attributes found in the model " - "(default is generating only those given by the -e/--extended-attributes parameter).") - parser.add_argument('--json-pretty', action='store_true', - help=" Pretty print JSON output.") +from typing import Optional +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig def export_node(json_dict, node, config, print_uuid): @@ -93,31 +80,37 @@ def export_node(json_dict, node, config, print_uuid): child, config, print_uuid) -def export(config: argparse.Namespace, signal_root: VSSNode, print_uuid, data_type_root: VSSNode): - logging.info("Generating JSON output...") - indent = None - if config.json_pretty: - logging.info("Serializing pretty JSON...") - indent = 2 - else: - logging.info("Serializing compact JSON...") - - signals_json_dict: Dict[str, Any] = {} - export_node(signals_json_dict, signal_root, config, print_uuid) - - if data_type_root is not None: - data_types_json_dict: Dict[str, Any] = {} - export_node(data_types_json_dict, data_type_root, config, print_uuid) - if config.types_output_file is None: - logging.info("Adding custom data types to signal dictionary") - signals_json_dict["ComplexDataTypes"] = data_types_json_dict - else: - with open(config.types_output_file, 'w') as f: - json.dump(data_types_json_dict, f, indent=indent, sort_keys=True) - - with open(config.output_file, 'w') as f: - json.dump(signals_json_dict, f, indent=indent, sort_keys=True) +class Vss2Json(Vss2X): + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('--json-all-extended-attributes', action='store_true', + help="Generate all extended attributes found in the model " + "(default is generating only those given by the -e/--extended-attributes parameter).") + parser.add_argument('--json-pretty', action='store_true', + help=" Pretty print JSON output.") -if __name__ == "__main__": - initLogging() + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + logging.info("Generating JSON output...") + indent = None + if config.json_pretty: + logging.info("Serializing pretty JSON...") + indent = 2 + else: + logging.info("Serializing compact JSON...") + + signals_json_dict: Dict[str, Any] = {} + export_node(signals_json_dict, signal_root, config, vspec2vss_config.generate_uuid) + + if data_type_root is not None: + data_types_json_dict: Dict[str, Any] = {} + export_node(data_types_json_dict, data_type_root, config, vspec2vss_config.generate_uuid) + if config.types_output_file is None: + logging.info("Adding custom data types to signal dictionary") + signals_json_dict["ComplexDataTypes"] = data_types_json_dict + else: + with open(config.types_output_file, 'w') as f: + json.dump(data_types_json_dict, f, indent=indent, sort_keys=True) + + with open(config.output_file, 'w') as f: + json.dump(signals_json_dict, f, indent=indent, sort_keys=True) diff --git a/vspec/vssexporters/vss2jsonschema.py b/vspec/vssexporters/vss2jsonschema.py index 75c85af1..03f4a7ce 100644 --- a/vspec/vssexporters/vss2jsonschema.py +++ b/vspec/vssexporters/vss2jsonschema.py @@ -14,8 +14,10 @@ import json import logging from typing import Dict, Any +from typing import Optional from vspec.model.vsstree import VSSNode -from vspec.loggingconfig import initLogging +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig type_map = { "int8": "integer", @@ -45,19 +47,6 @@ } -def add_arguments(parser: argparse.ArgumentParser): - """Check for input arguments.""" - parser.add_argument('--jsonschema-all-extended-attributes', action='store_true', - help="Generate all extended attributes found in the model." - "Should not be used with strict mode JSON Schema validators.") - parser.add_argument("--jsonschema-disallow-additional-properties", action='store_true', - help="Do not allow properties not defined in VSS tree") - parser.add_argument("--jsonschema-require-all-properties", action='store_true', - help="Require all elements defined in VSS tree for a valid object") - parser.add_argument('--jsonschema-pretty', action='store_true', - help=" Pretty print JSON Schema output.") - - def export_node(json_dict, node, config, print_uuid): """Preparing nodes for JSON schema output.""" # keyword with X- sign are left for extensions and they are not part of official JSON schema @@ -122,38 +111,49 @@ def export_node(json_dict, node, config, print_uuid): export_node(json_dict[node.name]["properties"], child, config, print_uuid) -def export(config: argparse.Namespace, signal_root: VSSNode, print_uuid, data_type_root: VSSNode): - """Export function for generating JSON schema file.""" - logging.info("Generating JSON schema...") - indent = None - if config.jsonschema_pretty: - logging.info("Serializing pretty JSON schema...") - indent = 2 - - signals_json_schema: Dict[str, Any] = {} - export_node(signals_json_schema, signal_root, config, print_uuid) - - # Add data types to the schema - if data_type_root is not None: - data_types_json_schema: Dict[str, Any] = {} - export_node(data_types_json_schema, data_type_root, config, print_uuid) - if config.jsonschema_all_extended_attributes: - signals_json_schema["x-ComplexDataTypes"] = data_types_json_schema - - # VSS models only have one root, so there should only be one - # key in the dict - assert (len(signals_json_schema.keys()) == 1) - top_node_name = list(signals_json_schema.keys())[0] - signals_json_schema = signals_json_schema.pop(top_node_name) - - json_schema = { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": top_node_name, - "type": "object", - **signals_json_schema} - with open(config.output_file, 'w', encoding="utf-8") as output_file: - json.dump(json_schema, output_file, indent=indent, sort_keys=False) - - -if __name__ == "__main__": - initLogging() +class Vss2JsonSchema(Vss2X): + + def add_arguments(self, parser: argparse.ArgumentParser): + """Check for input arguments.""" + parser.add_argument('--jsonschema-all-extended-attributes', action='store_true', + help="Generate all extended attributes found in the model." + "Should not be used with strict mode JSON Schema validators.") + parser.add_argument("--jsonschema-disallow-additional-properties", action='store_true', + help="Do not allow properties not defined in VSS tree") + parser.add_argument("--jsonschema-require-all-properties", action='store_true', + help="Require all elements defined in VSS tree for a valid object") + parser.add_argument('--jsonschema-pretty', action='store_true', + help=" Pretty print JSON Schema output.") + + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + """Export function for generating JSON schema file.""" + logging.info("Generating JSON schema...") + indent = None + if config.jsonschema_pretty: + logging.info("Serializing pretty JSON schema...") + indent = 2 + + signals_json_schema: Dict[str, Any] = {} + export_node(signals_json_schema, signal_root, config, vspec2vss_config.generate_uuid) + + # Add data types to the schema + if data_type_root is not None: + data_types_json_schema: Dict[str, Any] = {} + export_node(data_types_json_schema, data_type_root, config, vspec2vss_config.generate_uuid) + if config.jsonschema_all_extended_attributes: + signals_json_schema["x-ComplexDataTypes"] = data_types_json_schema + + # VSS models only have one root, so there should only be one + # key in the dict + assert (len(signals_json_schema.keys()) == 1) + top_node_name = list(signals_json_schema.keys())[0] + signals_json_schema = signals_json_schema.pop(top_node_name) + + json_schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": top_node_name, + "type": "object", + **signals_json_schema} + with open(config.output_file, 'w', encoding="utf-8") as output_file: + json.dump(json_schema, output_file, indent=indent, sort_keys=False) diff --git a/vspec/vssexporters/vss2protobuf.py b/vspec/vssexporters/vss2protobuf.py index b9bc2049..d1dd54e1 100755 --- a/vspec/vssexporters/vss2protobuf.py +++ b/vspec/vssexporters/vss2protobuf.py @@ -17,10 +17,12 @@ import sys from pathlib import Path from typing import Set +from typing import Optional from anytree import PreOrderIter # type: ignore[import] -from vspec.loggingconfig import initLogging from vspec.model.vsstree import VSSNode +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig # Add path to main py vspec parser myDir = os.path.dirname(os.path.realpath(__file__)) @@ -38,15 +40,6 @@ } -def feature_supported(feature_name: str): - """Return true for supported optional arguments/features""" - return False - - -def add_arguments(parser: argparse.ArgumentParser): - parser.description = "The protobuf exporter does not support any additional arguments." - - class ProtoExporter(object): def __init__(self, out_dir: Path): self.out_files: Set[Path] = set() @@ -148,21 +141,31 @@ def print_message_body(nodes, proto_file): proto_file.write(f" {data_type} {node.name} = {i};" + "\n") -def export(config: argparse.Namespace, signal_root: VSSNode, print_uuid, data_type_root: VSSNode): - logging.info("Generating protobuf output...") - if data_type_root is not None: - if config.types_output_file is not None: - fp = Path(config.types_output_file) - exporter_path = Path(os.path.dirname(fp)) - else: - exporter_path = Path(Path.cwd()) - logging.debug(f"Will use {exporter_path} for type exports") - exporter = ProtoExporter(exporter_path) - exporter.traverse_data_type_tree(data_type_root) - - with open(config.output_file, 'w') as f: - traverse_signal_tree(signal_root, f) - - -if __name__ == "__main__": - initLogging() +class Vss2Protobuf(Vss2X): + + def __init__(self, vspec2vss_config: Vspec2VssConfig): + vspec2vss_config.no_expand_option_supported = False + vspec2vss_config.uuid_supported = False + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('--json-all-extended-attributes', action='store_true', + help="Generate all extended attributes found in the model " + "(default is generating only those given by the -e/--extended-attributes parameter).") + parser.add_argument('--json-pretty', action='store_true', + help=" Pretty print JSON output.") + + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: + logging.info("Generating protobuf output...") + if data_type_root is not None: + if config.types_output_file is not None: + fp = Path(config.types_output_file) + exporter_path = Path(os.path.dirname(fp)) + else: + exporter_path = Path(Path.cwd()) + logging.debug(f"Will use {exporter_path} for type exports") + exporter = ProtoExporter(exporter_path) + exporter.traverse_data_type_tree(data_type_root) + + with open(config.output_file, 'w') as f: + traverse_signal_tree(signal_root, f) diff --git a/vspec/vssexporters/vss2yaml.py b/vspec/vssexporters/vss2yaml.py index d3663cf9..e3768d12 100644 --- a/vspec/vssexporters/vss2yaml.py +++ b/vspec/vssexporters/vss2yaml.py @@ -17,22 +17,10 @@ from vspec.model.vsstree import VSSNode import yaml import logging -from vspec.loggingconfig import initLogging from typing import Dict, Any - - -def feature_supported(feature_name: str): - """Return true for supported arguments/features""" - if feature_name in ['no_expand']: - return True - return False - - -def add_arguments(parser: argparse.ArgumentParser): - parser.add_argument('--yaml-all-extended-attributes', action='store_true', - help=("Generate all extended attributes found in the model " - "(default is generating only those given by the " - "-e/--extended-attributes parameter).")) +from typing import Optional +from vspec.vss2x import Vss2X +from vspec.vspec2vss_config import Vspec2VssConfig def export_node(yaml_dict, node, config, print_uuid): @@ -106,23 +94,29 @@ def write_line_break(self, data=None): super().write_line_break() -def export(config: argparse.Namespace, signal_root: VSSNode, print_uuid, data_type_root: VSSNode): - logging.info("Generating YAML output...") +class Vss2Yaml(Vss2X): + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('--yaml-all-extended-attributes', action='store_true', + help=("Generate all extended attributes found in the model " + "(default is generating only those given by the " + "-e/--extended-attributes parameter).")) - signals_yaml_dict: Dict[str, Any] = {} - export_node(signals_yaml_dict, signal_root, config, print_uuid) + def generate(self, config: argparse.Namespace, signal_root: VSSNode, vspec2vss_config: Vspec2VssConfig, + data_type_root: Optional[VSSNode] = None) -> None: - if data_type_root is not None: - data_types_yaml_dict: Dict[str, Any] = {} - export_node(data_types_yaml_dict, data_type_root, config, print_uuid) - if config.types_output_file is None: - logging.info("Adding custom data types to signal dictionary") - signals_yaml_dict["ComplexDataTypes"] = data_types_yaml_dict - else: - export_yaml(config.types_output_file, data_types_yaml_dict) + logging.info("Generating YAML output...") - export_yaml(config.output_file, signals_yaml_dict) + signals_yaml_dict: Dict[str, Any] = {} + export_node(signals_yaml_dict, signal_root, config, vspec2vss_config.generate_uuid) + if data_type_root is not None: + data_types_yaml_dict: Dict[str, Any] = {} + export_node(data_types_yaml_dict, data_type_root, config, vspec2vss_config.generate_uuid) + if config.types_output_file is None: + logging.info("Adding custom data types to signal dictionary") + signals_yaml_dict["ComplexDataTypes"] = data_types_yaml_dict + else: + export_yaml(config.types_output_file, data_types_yaml_dict) -if __name__ == "__main__": - initLogging() + export_yaml(config.output_file, signals_yaml_dict) diff --git a/vspec2binary.py b/vspec2binary.py index fb8f5ec4..63549707 100755 --- a/vspec2binary.py +++ b/vspec2binary.py @@ -13,7 +13,12 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2binary import Vss2Binary if __name__ == "__main__": - vspec2x.main(["--format", "binary"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2binary = Vss2Binary(vspec2vss_config) + vspec2x = Vspec2X(vss2binary, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/vspec2csv.py b/vspec2csv.py index 73b26f49..1cfee950 100755 --- a/vspec2csv.py +++ b/vspec2csv.py @@ -13,7 +13,12 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2csv import Vss2Csv if __name__ == "__main__": - vspec2x.main(["--format", "csv"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2csv = Vss2Csv() + vspec2x = Vspec2X(vss2csv, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/vspec2ddsidl.py b/vspec2ddsidl.py index d6c84a7f..111aa2cb 100755 --- a/vspec2ddsidl.py +++ b/vspec2ddsidl.py @@ -13,7 +13,12 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2ddsidl import Vss2DdsIdl if __name__ == "__main__": - vspec2x.main(["--format", "idl"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2json = Vss2DdsIdl(vspec2vss_config) + vspec2x = Vspec2X(vss2json, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/vspec2franca.py b/vspec2franca.py index b1206f93..37f13fa3 100755 --- a/vspec2franca.py +++ b/vspec2franca.py @@ -13,7 +13,12 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2franca import Vss2Franca if __name__ == "__main__": - vspec2x.main(["--format", "franca"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2franca = Vss2Franca(vspec2vss_config) + vspec2x = Vspec2X(vss2franca, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/vspec2graphql.py b/vspec2graphql.py index 4ae02565..49e82875 100755 --- a/vspec2graphql.py +++ b/vspec2graphql.py @@ -13,7 +13,12 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2graphql import Vss2Graphql if __name__ == "__main__": - vspec2x.main(["--format", "graphql"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2graphql = Vss2Graphql(vspec2vss_config) + vspec2x = Vspec2X(vss2graphql, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/vspec2id.py b/vspec2id.py index 399468fd..5674aad7 100755 --- a/vspec2id.py +++ b/vspec2id.py @@ -12,7 +12,17 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2id import Vss2Id + + +def main(arguments): + vspec2vss_config = Vspec2VssConfig() + vss2json = Vss2Id() + vspec2x = Vspec2X(vss2json, vspec2vss_config) + vspec2x.main(arguments) + if __name__ == "__main__": - vspec2x.main(["--format", "idgen"] + sys.argv[1:]) + main(sys.argv[1:]) diff --git a/vspec2json.py b/vspec2json.py index f6eca381..54b6460d 100755 --- a/vspec2json.py +++ b/vspec2json.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2016 Contributors to COVESA +# Copyright (c) 2023 Contributors to COVESA # # This program and the accompanying materials are made available under the # terms of the Mozilla Public License 2.0 which is available at @@ -13,7 +13,12 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2json import Vss2Json if __name__ == "__main__": - vspec2x.main(["--format", "json"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2json = Vss2Json() + vspec2x = Vspec2X(vss2json, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/vspec2jsonschema.py b/vspec2jsonschema.py index 5ac3d6e6..42bad653 100755 --- a/vspec2jsonschema.py +++ b/vspec2jsonschema.py @@ -13,7 +13,12 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2jsonschema import Vss2JsonSchema if __name__ == "__main__": - vspec2x.main(["--format", "jsonschema"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2jsonschema = Vss2JsonSchema() + vspec2x = Vspec2X(vss2jsonschema, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/vspec2protobuf.py b/vspec2protobuf.py index a8bee678..f7e90e50 100755 --- a/vspec2protobuf.py +++ b/vspec2protobuf.py @@ -13,7 +13,13 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2protobuf import Vss2Protobuf + if __name__ == "__main__": - vspec2x.main(["--format", "protobuf"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2protobuf = Vss2Protobuf(vspec2vss_config) + vspec2x = Vspec2X(vss2protobuf, vspec2vss_config) + vspec2x.main(sys.argv[1:]) diff --git a/vspec2x.py b/vspec2x.py deleted file mode 100755 index f31a6461..00000000 --- a/vspec2x.py +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2016 Contributors to COVESA -# -# This program and the accompanying materials are made available under the -# terms of the Mozilla Public License 2.0 which is available at -# https://www.mozilla.org/en-US/MPL/2.0/ -# -# SPDX-License-Identifier: MPL-2.0 - -# -# Convert vspec files to various other formats -# - -from enum import Enum -from vspec.model.vsstree import VSSNode -from vspec.model.constants import VSSTreeType -from vspec.loggingconfig import initLogging -import argparse -import logging -import sys -import vspec - - -from vspec.vssexporters import vss2json, vss2csv, vss2yaml, \ - vss2binary, vss2franca, vss2ddsidl, vss2graphql, vss2protobuf, vss2jsonschema, vss2id - -SUPPORTED_STRUCT_EXPORT_FORMATS = set(["json", "yaml", "csv", "protobuf", "jsonschema", "idl"]) - - -class Exporter(Enum): - """ - You can add new exporters here. Put the code in vssexporters and add it here - See one of the existing exporters for an example. - Mandatory functions are - def add_arguments(parser: argparse.ArgumentParser) - def export(config: argparse.Namespace, root: VSSNode): - """ - json = vss2json - csv = vss2csv - yaml = vss2yaml - binary = vss2binary - franca = vss2franca - idl = vss2ddsidl - graphql = vss2graphql - protobuf = vss2protobuf - - jsonschema = vss2jsonschema - idgen = vss2id - - def __str__(self): - return self.name - - @staticmethod - def from_string(s): - try: - return Exporter[s] - except KeyError: - raise ValueError() - - -def main(arguments): - parser = argparse.ArgumentParser(description="Convert vspec to other formats.") - - initLogging() - - parser.add_argument('-I', '--include-dir', action='append', metavar='dir', type=str, default=[], - help='Add include directory to search for included vspec files.') - parser.add_argument('-e', '--extended-attributes', type=str, default="", - help='Whitelisted extended attributes as comma separated list. ' - 'Note, that not all exporters will support (all) extended attributes.') - parser.add_argument('-s', '--strict', action='store_true', - help='Use strict checking: Terminate when anything not covered or not recommended ' - 'by VSS language or extensions is found.') - parser.add_argument('--abort-on-unknown-attribute', action='store_true', - help=" Terminate when an unknown attribute is found.") - parser.add_argument('--abort-on-name-style', action='store_true', - help=" Terminate naming style not follows recommendations.") - parser.add_argument('--format', metavar='format', type=Exporter.from_string, choices=list(Exporter), - help='Output format, choose one from ' + - str(Exporter._member_names_) + # pylint: disable=no-member - ". If omitted we try to guess form output_file suffix.") - parser.add_argument('--uuid', action='store_true', - help='Include uuid in generated files.') - parser.add_argument('--no-expand', action='store_true', - help='Do not expand tree.') - parser.add_argument('-o', '--overlays', action='append', metavar='overlays', type=str, default=[], - help='Add overlay that will be layered on top of the VSS file in the order they appear.') - parser.add_argument('-u', '--unit-file', action='append', metavar='unit_file', type=str, default=[], - help='Unit file to be used for generation. Argument -u may be used multiple times.') - parser.add_argument('-q', '--quantity-file', action='append', metavar='quantity_file', type=str, default=[], - help='Quantity file to be used for generation. Argument -uqmay be used multiple times.') - parser.add_argument('vspec_file', metavar='', - help='The vehicle specification file to convert.') - parser.add_argument('output_file', metavar='', - help='The file to write output to.') - - type_group = parser.add_argument_group( - 'VSS Data Type Tree arguments', - 'Arguments related to struct/type support') - type_group.add_argument('-vt', '--vspec-types-file', action='append', metavar='vspec_types_file', type=str, - default=[], - help='Data types file in vspec format.') - type_group.add_argument('-ot', '--types-output-file', metavar='', - help='Output file for writing data types from vspec file. ' + - 'If not specified, a single file is used where applicable. ' + - 'In case of JSON and YAML, the data is exported under a ' + - 'special key - "ComplexDataTypes"') - - for entry in Exporter: - entry.value.add_arguments(parser.add_argument_group( - f"{entry.name.upper()} arguments", "")) - - args = parser.parse_args(arguments) - - # Figure out output format - if args.format is not None: # User has given format parameter - logging.info("Output to " + str(args.format.name) + " format") - else: # Else try to figure from output file suffix - try: - suffix = args.output_file[args.output_file.rindex(".") + 1:] - except BaseException: - logging.error( - "Can not determine output format. Try setting --format parameter") - sys.exit(-1) - try: - args.format = Exporter.from_string(suffix) - except BaseException: - logging.error( - "Can not determine output format. Try setting --format parameter") - sys.exit(-1) - - logging.info("Output to " + str(args.format.name) + " format") - - include_dirs = ["."] - include_dirs.extend(args.include_dir) - - abort_on_unknown_attribute = False - abort_on_namestyle = False - - if args.abort_on_unknown_attribute or args.strict: - abort_on_unknown_attribute = True - if args.abort_on_name_style or args.strict: - abort_on_namestyle = True - - known_extended_attributes_list = args.extended_attributes.split(",") - if len(known_extended_attributes_list) > 0: - vspec.model.vsstree.VSSNode.whitelisted_extended_attributes = known_extended_attributes_list - logging.info( - f"Known extended attributes: {', '.join(known_extended_attributes_list)}") - - exporter = args.format.value - - print_uuid = False - if args.uuid: - print_uuid = True - - vspec.load_quantities(args.vspec_file, args.quantity_file) - vspec.load_units(args.vspec_file, args.unit_file) - - # Warn if unsupported feature is used - if args.no_expand and not exporter.feature_supported("no_expand"): - logging.warning("--no_expand not supported by exporter") - - # process data type tree - if args.types_output_file is not None and not args.vspec_types_file: - parser.error("An output file for data types was provided. Please also provide " - "the input vspec file for data types") - data_type_tree = None - if args.vspec_types_file: - data_type_tree = processDataTypeTree( - parser, args, include_dirs, abort_on_namestyle) - vspec.verify_mandatory_attributes(data_type_tree, abort_on_unknown_attribute) - - try: - logging.info(f"Loading vspec from {args.vspec_file}...") - tree = vspec.load_tree( - args.vspec_file, include_dirs, VSSTreeType.SIGNAL_TREE, - break_on_name_style_violation=abort_on_namestyle, - expand_inst=False, data_type_tree=data_type_tree) - - for overlay in args.overlays: - logging.info(f"Applying VSS overlay from {overlay}...") - othertree = vspec.load_tree(overlay, include_dirs, VSSTreeType.SIGNAL_TREE, - break_on_name_style_violation=abort_on_namestyle, expand_inst=False, - data_type_tree=data_type_tree) - vspec.merge_tree(tree, othertree) - - vspec.check_type_usage(tree, VSSTreeType.SIGNAL_TREE, data_type_tree) - if not args.no_expand: - vspec.expand_tree_instances(tree) - - vspec.clean_metadata(tree) - vspec.verify_mandatory_attributes(tree, abort_on_unknown_attribute) - logging.info("Calling exporter...") - - # temporary until all exporters support data type tree - if args.format.name in SUPPORTED_STRUCT_EXPORT_FORMATS: - exporter.export(args, tree, print_uuid, data_type_tree) - else: - if data_type_tree is not None: - parser.error( - f"{args.format.name} format is not yet supported in vspec struct/data type support feature") - exporter.export(args, tree, print_uuid) - logging.info("All done.") - except vspec.VSpecError as e: - logging.error(f"Error: {e}") - sys.exit(255) - - -def processDataTypeTree(parser: argparse.ArgumentParser, args, include_dirs, - abort_on_namestyle: bool) -> VSSNode: - """ - Helper function to process command line arguments and invoke logic for processing data - type information provided in vspec format - """ - if args.types_output_file is None: - logging.info("Sensors and custom data types will be consolidated into one file.") - if args.format == Exporter.protobuf: - logging.info("Proto files will be written to the current working directory") - - logging.warning("All exports do not yet support structs. Please check documentation for your exporter!") - - first_tree = True - for type_file in args.vspec_types_file: - logging.info(f"Loading and processing struct/data type tree from {type_file}") - new_tree = vspec.load_tree(type_file, include_dirs, VSSTreeType.DATA_TYPE_TREE, - break_on_name_style_violation=abort_on_namestyle, expand_inst=False) - if first_tree: - tree = new_tree - first_tree = False - else: - vspec.merge_tree(tree, new_tree) - vspec.check_type_usage(tree, VSSTreeType.DATA_TYPE_TREE) - return tree - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/vspec2yaml.py b/vspec2yaml.py index 4fcf7de7..a30ca753 100755 --- a/vspec2yaml.py +++ b/vspec2yaml.py @@ -13,7 +13,12 @@ # import sys -import vspec2x +from vspec.vspec2x import Vspec2X +from vspec.vspec2vss_config import Vspec2VssConfig +from vspec.vssexporters.vss2yaml import Vss2Yaml if __name__ == "__main__": - vspec2x.main(["--format", "yaml"]+sys.argv[1:]) + vspec2vss_config = Vspec2VssConfig() + vss2yaml = Vss2Yaml() + vspec2x = Vspec2X(vss2yaml, vspec2vss_config) + vspec2x.main(sys.argv[1:])