From b936ebdcb076438d443ebb5b9529f201b21a9841 Mon Sep 17 00:00:00 2001 From: Rajeev Rao Date: Thu, 3 Feb 2022 10:57:18 -0800 Subject: [PATCH] Update Polygraphy to v0.35.1 Signed-off-by: Rajeev Rao --- tools/Polygraphy/.gitignore | 3 + tools/Polygraphy/CHANGELOG.md | 56 ++++ tools/Polygraphy/Makefile | 1 + tools/Polygraphy/README.md | 2 +- .../api/02_validating_on_a_dataset/README.md | 2 +- .../03_interoperating_with_tensorrt/README.md | 2 +- .../05_using_tensorrt_network_api/README.md | 2 +- .../02_reducing_failing_onnx_models/README.md | 8 +- .../README.md | 4 +- .../02_inspecting_a_tensorrt_engine/README.md | 21 +- .../03_inspecting_an_onnx_model/README.md | 4 +- .../cli/run/01_comparing_frameworks/README.md | 2 +- .../05_comparing_with_custom_data/README.md | 28 -- .../data_loader.py | 30 -- .../README.md | 56 ++++ .../data_loader.py | 43 +++ .../dynamic_identity.onnx | 0 .../README.md | 38 +++ .../generate_data.py | 63 +++++ .../identity.onnx | 15 + .../surgeon/01_isolating_subgraphs/README.md | 4 +- .../surgeon/02_folding_constants/README.md | 2 +- .../03_modifying_input_shapes/README.md | 2 +- tools/Polygraphy/polygraphy/__init__.py | 2 +- .../polygraphy/backend/onnx/util.py | 35 ++- .../polygraphy/backend/onnxrt/loader.py | 30 +- .../polygraphy/backend/onnxrt/runner.py | 2 +- .../backend/pluginref/references.py | 16 ++ .../Polygraphy/polygraphy/backend/tf/util.py | 16 +- .../polygraphy/backend/trt/calibrator.py | 4 +- .../polygraphy/backend/trt/loader.py | 67 ++++- .../Polygraphy/polygraphy/backend/trt/util.py | 101 ++++++- .../Polygraphy/polygraphy/common/constants.py | 19 -- tools/Polygraphy/polygraphy/common/cuda.py | 19 -- .../Polygraphy/polygraphy/common/exception.py | 19 -- tools/Polygraphy/polygraphy/common/func.py | 19 -- .../polygraphy/comparator/compare.py | 38 +-- .../polygraphy/comparator/data_loader.py | 8 +- .../polygraphy/comparator/struct.py | 56 +++- tools/Polygraphy/polygraphy/mod/exporter.py | 15 +- tools/Polygraphy/polygraphy/tools/README.md | 4 +- .../polygraphy/tools/args/comparator.py | 7 +- .../polygraphy/tools/args/data_loader.py | 2 +- .../polygraphy/tools/args/onnx/loader.py | 6 +- .../polygraphy/tools/args/onnxrt/__init__.py | 1 + .../polygraphy/tools/args/onnxrt/loader.py | 70 +++++ .../polygraphy/tools/args/onnxrt/runner.py | 22 +- .../polygraphy/tools/args/trt/config.py | 53 ++-- .../polygraphy/tools/args/util/util.py | 23 +- .../polygraphy/tools/data/subtool/to_input.py | 4 +- .../tools/debug/subtool/artifact_sorter.py | 3 +- .../polygraphy/tools/debug/subtool/reduce.py | 6 +- .../polygraphy/tools/inspect/subtool/model.py | 104 ++++--- tools/Polygraphy/polygraphy/tools/run/run.py | 2 + tools/Polygraphy/polygraphy/tools/script.py | 3 +- .../polygraphy/tools/surgeon/subtool/base.py | 4 +- tools/Polygraphy/polygraphy/util/__init__.py | 1 - tools/Polygraphy/polygraphy/util/serde.py | 19 -- tools/Polygraphy/polygraphy/util/util.py | 27 +- .../tests/backend/onnx/test_loader.py | 9 +- .../tests/backend/onnxrt/test_loader.py | 54 ++++ .../tests/backend/pluginref/test_runner.py | 2 +- .../tests/backend/trt/test_loader.py | 42 ++- .../tests/comparator/test_compare.py | 4 +- .../tests/comparator/test_data_loader.py | 6 +- .../tests/comparator/test_struct.py | 27 +- tools/Polygraphy/tests/helper.py | 15 + tools/Polygraphy/tests/pytest.ini | 3 + tools/Polygraphy/tests/requirements.txt | 4 +- tools/Polygraphy/tests/test_dependencies.py | 70 ++--- .../tests/test_deprecated_aliases.py | 39 ++- tools/Polygraphy/tests/test_examples.py | 9 +- .../tests/tools/args/onnxrt/__init__.py | 0 .../tests/tools/args/onnxrt/test_loader.py | 31 ++ .../tests/tools/args/trt/test_config.py | 65 ++++- tools/Polygraphy/tests/tools/test_convert.py | 2 +- tools/Polygraphy/tests/tools/test_data.py | 3 +- tools/Polygraphy/tests/tools/test_inspect.py | 265 +++++++++++++----- tools/Polygraphy/tests/tools/test_run.py | 7 + tools/Polygraphy/tests/util/test_util.py | 33 +-- 80 files changed, 1358 insertions(+), 547 deletions(-) delete mode 100644 tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/README.md delete mode 100644 tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/data_loader.py create mode 100644 tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/README.md create mode 100644 tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/data_loader.py rename tools/Polygraphy/examples/cli/run/{05_comparing_with_custom_data => 05_comparing_with_custom_input_data}/dynamic_identity.onnx (100%) create mode 100644 tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/README.md create mode 100644 tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/generate_data.py create mode 100644 tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/identity.onnx delete mode 100644 tools/Polygraphy/polygraphy/common/constants.py delete mode 100644 tools/Polygraphy/polygraphy/common/cuda.py delete mode 100644 tools/Polygraphy/polygraphy/common/exception.py delete mode 100644 tools/Polygraphy/polygraphy/common/func.py create mode 100644 tools/Polygraphy/polygraphy/tools/args/onnxrt/loader.py delete mode 100644 tools/Polygraphy/polygraphy/util/serde.py create mode 100644 tools/Polygraphy/tests/backend/onnxrt/test_loader.py create mode 100644 tools/Polygraphy/tests/pytest.ini create mode 100644 tools/Polygraphy/tests/tools/args/onnxrt/__init__.py create mode 100644 tools/Polygraphy/tests/tools/args/onnxrt/test_loader.py diff --git a/tools/Polygraphy/.gitignore b/tools/Polygraphy/.gitignore index 6d6ca5b2..247974bc 100644 --- a/tools/Polygraphy/.gitignore +++ b/tools/Polygraphy/.gitignore @@ -67,3 +67,6 @@ venv/ ENV/ env.bak/ venv.bak/ + +# Crash dumps +core diff --git a/tools/Polygraphy/CHANGELOG.md b/tools/Polygraphy/CHANGELOG.md index 84d3e422..e98ea13b 100644 --- a/tools/Polygraphy/CHANGELOG.md +++ b/tools/Polygraphy/CHANGELOG.md @@ -3,6 +3,62 @@ Dates are in YYYY-MM-DD format. +## v0.35.1 (2022-01-14) +### Added +- Added a `providers` parameter to `SessionFromOnnx` to specify execution providers for ONNX-Runtime and a + corresponding `--providers` argument to CLI tools. + +### Changed +- `CompareFunc.simple()` will now correctly display the minimum required tolerances when using `elemwise` mode. + Note that in elementwise comparison mode, each element of the output is compared against both tolerances, and + only counted as a mismatch if both are exceeded. Hence, the minimum required tolerances apply if only one type + of tolerance is being used. When both absolute/relative tolerance are set, the requirements may be lower. + + +## v0.35.0 (2022-01-06) +### Added +- Added a `memory_pool_limits` parameter to `CreateConfig`. +- Added a `--pool-limit`/`--memory-pool-limit` argument to command-line tools. + +### Changed +- Changed the default base calibrator class to `IInt8EntropyCalibrator2` since it works across both GPU and DLA. + To preserve the old behavior, specify `--calibration-base-class=IInt8MinMaxCalibrator` on the command-line + or specify the `BaseClass` argument in `Calibrator` in the Python API. +- Deprecated `--workspace` command-line option and `max_workspace_size` parameter in `CreateConfig`. + Use `--pool-limit` and `memory_pool_limits` respectively instead. + +### Removed +- Removed deprecated module `polygraphy.util.serde`. Use `polygraphy.json` instead. +- Removed `--tactic-replay` command-line option. Use `--load-tactics`/`--save-tactics` instead. + + +## v0.34.2 (2021-11-29) +### Added +- Added support for `MeanVarianceNormalization` to `PluginRefRunner`. + + +## v0.34.1 (2021-11-24) +### Added +- Added a `profiling_verbosity` parameter to `CreateConfig()`. +- Added support for displaying layer-level engine information in `inspect model` for newer versions of TensorRT. + + +## v0.34.0 (2021-11-22) +### Added +- Added a new `add()` API to `RunResults` to make it easier to create custom output data. + Added [a new example](./examples/cli/run/06_comparing_with_custom_output_data/) to demonstrate how to use this API. + +### Changed +- Deprecated `--mode` option in `inspect model`; a new `--show` option has been introduced + which can be used to individually control what is displayed. +- Command-line tools will now use `IInt8EntropyCalibrator2` for calbration if DLA and int8 mode are enabled + since the default does not work with DLA. + +### Removed +- Removed several deprecated submodules of `polygraphy.common`: `constants`, `cuda`, `exception`, `func`. + These can now be found under the top-level `polygraphy` module instead. + + ## v0.33.2 (2021-10-21) ### Changed - Improved the help messages of various subtools, including `run`, `debug build`, and `debug reduce`. diff --git a/tools/Polygraphy/Makefile b/tools/Polygraphy/Makefile index 9898c6e1..ac4b819a 100644 --- a/tools/Polygraphy/Makefile +++ b/tools/Polygraphy/Makefile @@ -19,6 +19,7 @@ leak_check: export PYTHONPATH=$(CURDIR):$${PYTHONPATH} && \ export PATH=$(CURDIR)/bin:$${PATH} && \ export POLYGRAPHY_INTERNAL_CORRECTNESS_CHECKS=1 && \ + export PYTHONMALLOC=malloc && \ valgrind --leak-check=full python3 -m pytest tests/ -v --durations=5 2>&1 | tee leak-check.log clean: diff --git a/tools/Polygraphy/README.md b/tools/Polygraphy/README.md index 79278e18..c97712ba 100644 --- a/tools/Polygraphy/README.md +++ b/tools/Polygraphy/README.md @@ -117,7 +117,7 @@ To enable this behavior, set the `POLYGRAPHY_AUTOINSTALL_DEPS` environment varia **NOTE**: *By default, dependencies will be installed using the current interpreter, and may overwrite existing* *packages. The default installation command, which is `python -m pip install`, can be overriden by setting* - *the `POLYGRAPHY_INSTALL_CMD` environment variable, or setting `polygraphy.config.INSTALL_CMD = "..."` using the Python API.* + *the `POLYGRAPHY_INSTALL_CMD` environment variable, or setting `polygraphy.config.INSTALL_CMD` using the Python API.* #### Installing Manually diff --git a/tools/Polygraphy/examples/api/02_validating_on_a_dataset/README.md b/tools/Polygraphy/examples/api/02_validating_on_a_dataset/README.md index ac181dfc..59aa9adb 100644 --- a/tools/Polygraphy/examples/api/02_validating_on_a_dataset/README.md +++ b/tools/Polygraphy/examples/api/02_validating_on_a_dataset/README.md @@ -10,7 +10,7 @@ with a real dataset that includes labels or golden values - especially if the da In such cases, it is recommended to use a runner directly instead. *NOTE: It is possible to provide custom input data to `Comparator.run()` using the `data_loader`* - *parameter. This is may be a viable option when using a smaller dataset.* + *parameter. This may be a viable option when using a smaller dataset.* In this example, we use a `TrtRunner` directly to validate an identity model on a trivial dataset. Unlike using the `Comparator`, using a runner gives you complete diff --git a/tools/Polygraphy/examples/api/03_interoperating_with_tensorrt/README.md b/tools/Polygraphy/examples/api/03_interoperating_with_tensorrt/README.md index 8d4c71d9..70056929 100644 --- a/tools/Polygraphy/examples/api/03_interoperating_with_tensorrt/README.md +++ b/tools/Polygraphy/examples/api/03_interoperating_with_tensorrt/README.md @@ -31,7 +31,7 @@ we will focus on cases where you may want to: generated TensorRT network, which should be named `"MyIdentity"`: ```bash - polygraphy inspect model example.py --trt-network-func load_network --mode=full + polygraphy inspect model example.py --trt-network-func load_network --show layers attrs weights ``` 3. Run the example: diff --git a/tools/Polygraphy/examples/api/05_using_tensorrt_network_api/README.md b/tools/Polygraphy/examples/api/05_using_tensorrt_network_api/README.md index 7bf8949c..b14cb044 100644 --- a/tools/Polygraphy/examples/api/05_using_tensorrt_network_api/README.md +++ b/tools/Polygraphy/examples/api/05_using_tensorrt_network_api/README.md @@ -21,7 +21,7 @@ loader to seamlessly integrate a network defined using TensorRT APIs with Polygr This will invoke `create_network()` from within the script and display the generated TensorRT network: ```bash - polygraphy inspect model example.py --trt-network-func create_network --mode=full + polygraphy inspect model example.py --trt-network-func create_network --show layers attrs weights ``` 3. Run the example: diff --git a/tools/Polygraphy/examples/cli/debug/02_reducing_failing_onnx_models/README.md b/tools/Polygraphy/examples/cli/debug/02_reducing_failing_onnx_models/README.md index 37671d1d..a9c01377 100644 --- a/tools/Polygraphy/examples/cli/debug/02_reducing_failing_onnx_models/README.md +++ b/tools/Polygraphy/examples/cli/debug/02_reducing_failing_onnx_models/README.md @@ -114,13 +114,13 @@ Hence, the final reduced model should contain just the `Mul` node (since the oth ```bash polygraphy debug reduce folded.onnx -o initial_reduced.onnx --mode=bisect \ --fail-regex "Op: Mul" \ - --check polygraphy inspect model polygraphy_debug.onnx --mode=basic + --check polygraphy inspect model polygraphy_debug.onnx --show layers ``` 4. **[Optional]** As a sanity check, we can inspect our reduced model to ensure that it does contain the `Mul` node: ```bash - polygraphy inspect model initial_reduced.onnx --mode=basic + polygraphy inspect model initial_reduced.onnx --show layers ``` 5. Since we used `bisect` mode in the previous step, the model may not be as minimal as it could be. @@ -137,12 +137,12 @@ Hence, the final reduced model should contain just the `Mul` node (since the oth ```bash polygraphy debug reduce initial_reduced.onnx -o final_reduced.onnx --mode=linear \ --fail-regex "Op: Mul" \ - --check polygraphy inspect model polygraphy_debug.onnx --mode=basic + --check polygraphy inspect model polygraphy_debug.onnx --show layers ``` 6. **[Optional]** At this stage, `final_reduced.onnx` should contain just the failing node - the `Mul`. We can verify this with `inspect model`: ```bash - polygraphy inspect model final_reduced.onnx --mode=basic + polygraphy inspect model final_reduced.onnx --show layers ``` diff --git a/tools/Polygraphy/examples/cli/inspect/01_inspecting_a_tensorrt_network/README.md b/tools/Polygraphy/examples/cli/inspect/01_inspecting_a_tensorrt_network/README.md index 4eca80f4..68bf6cec 100644 --- a/tools/Polygraphy/examples/cli/inspect/01_inspecting_a_tensorrt_network/README.md +++ b/tools/Polygraphy/examples/cli/inspect/01_inspecting_a_tensorrt_network/README.md @@ -13,7 +13,7 @@ into TensorRT networks, and then display them. ```bash polygraphy inspect model identity.onnx \ - --mode=basic --display-as=trt + --show layers --display-as=trt ``` This will display something like: @@ -34,4 +34,4 @@ into TensorRT networks, and then display them. -> {y [dtype=float32, shape=(1, 1, 2, 2)]} ``` - It is also possible to show detailed layer information, including layer attributes, using `--mode=full`. + It is also possible to show detailed layer information, including layer attributes, using `--show layers attrs weights`. diff --git a/tools/Polygraphy/examples/cli/inspect/02_inspecting_a_tensorrt_engine/README.md b/tools/Polygraphy/examples/cli/inspect/02_inspecting_a_tensorrt_engine/README.md index d043457b..08787f1f 100644 --- a/tools/Polygraphy/examples/cli/inspect/02_inspecting_a_tensorrt_engine/README.md +++ b/tools/Polygraphy/examples/cli/inspect/02_inspecting_a_tensorrt_engine/README.md @@ -21,14 +21,18 @@ about TensorRT engines, i.e. plan files: 2. Inspect the engine: ```bash - polygraphy inspect model dynamic_identity.engine + polygraphy inspect model dynamic_identity.engine \ + --show layers ``` + NOTE: `--show layers` only works if the engine was built with a `profiling_verbosity` other than `NONE`. + Higher verbosities make more per-layer information available. + This will display something like: ``` [I] ==== TensorRT Engine ==== - Name: Unnamed Network 0 | Explicit Batch Engine (2 layers) + Name: Unnamed Network 0 | Explicit Batch Engine ---- 1 Engine Input(s) ---- {X [dtype=float32, shape=(1, 2, -1, -1)]} @@ -47,4 +51,17 @@ about TensorRT engines, i.e. plan files: - Profile: 1 Binding Index: 2 (Input) [Name: X [profile 1]] | Shapes: min=(1, 2, 2, 2), opt=(1, 2, 4, 4), max=(1, 2, 6, 6) Binding Index: 3 (Output) [Name: Y [profile 1]] | Shape: (1, 2, -1, -1) + + ---- 1 Layer(s) Per Profile ---- + - Profile: 0 + Layer 0 | node_of_Y [Op: Reformat] + {X [shape=(1, 2, -1, -1)]} + -> {Y [shape=(1, 2, -1, -1)]} + + - Profile: 1 + Layer 0 | node_of_Y [profile 1] [Op: Reformat] + {X [profile 1] [shape=(1, 2, -1, -1)]} + -> {Y [profile 1] [shape=(1, 2, -1, -1)]} ``` + + It is also possible to show more detailed layer information using `--show layers attrs`. diff --git a/tools/Polygraphy/examples/cli/inspect/03_inspecting_an_onnx_model/README.md b/tools/Polygraphy/examples/cli/inspect/03_inspecting_an_onnx_model/README.md index 9c928712..5b71d338 100644 --- a/tools/Polygraphy/examples/cli/inspect/03_inspecting_an_onnx_model/README.md +++ b/tools/Polygraphy/examples/cli/inspect/03_inspecting_an_onnx_model/README.md @@ -11,7 +11,7 @@ The `inspect model` subtool can display ONNX models. 1. Inspect the ONNX model: ```bash - polygraphy inspect model identity.onnx --mode=basic + polygraphy inspect model identity.onnx --show layers ``` This will display something like: @@ -35,4 +35,4 @@ The `inspect model` subtool can display ONNX models. -> {y [dtype=float32, shape=(1, 1, 2, 2)]} ``` - It is also possible to show detailed layer information, including layer attributes, using `--mode=full`. + It is also possible to show detailed layer information, including layer attributes, using `--show layers attrs weights`. diff --git a/tools/Polygraphy/examples/cli/run/01_comparing_frameworks/README.md b/tools/Polygraphy/examples/cli/run/01_comparing_frameworks/README.md index 99ca8cb0..7e88a295 100644 --- a/tools/Polygraphy/examples/cli/run/01_comparing_frameworks/README.md +++ b/tools/Polygraphy/examples/cli/run/01_comparing_frameworks/README.md @@ -52,4 +52,4 @@ For more details on working with dynamic shapes in TensorRT, refer to [`convert` example 03](../../convert/03_dynamic_shapes_in_tensorrt/) or [API example 07](../../../api/07_tensorrt_and_dynamic_shapes/). -For details on how to supply real input data, see [`run` example 05](../05_comparing_with_custom_data/). +For details on how to supply real input data, see [`run` example 05](../05_comparing_with_custom_input_data/). diff --git a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/README.md b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/README.md deleted file mode 100644 index a13e4892..00000000 --- a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Comparing With Custom Input Data - -## Introduction - -In some cases, we may want to run comparisons using custom input data. -Polygraphy provides multiple ways to do so, which are detailed [here](../../../../polygraphy/tools/README.md#using-custom-input-data). - -In this example, we'll use a data loader script by defining a `load_data` function in a Python -script called `data_loader.py` and then use `polygraphy run` to compare TensorRT and ONNX-Runtime. - -Since our model has dynamic shapes, we'll need to set up a TensorRT Optimization Profile. -For details on how we can do this via the command-line, -see [`convert` example 03](../../convert/03_dynamic_shapes_in_tensorrt). -For simplicitly, we'll create a profile where `min` == `opt` == `max`. - -*NOTE: It is important that our optimization profile works with the shapes provided by our* - *custom data loader. In our very simple case, the data loader always generates inputs of* - *shape (1, 2, 28, 28), so we just need to ensure this falls within [`min`, `max`].* - -## Running The Example - -1. Run the model with TensorRT and ONNX-Runtime using custom input data: - - ```bash - polygraphy run dynamic_identity.onnx --trt --onnxrt \ - --data-loader-script data_loader.py \ - --trt-min-shapes X:[1,2,28,28] --trt-opt-shapes X:[1,2,28,28] --trt-max-shapes X:[1,2,28,28] - ``` diff --git a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/data_loader.py b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/data_loader.py deleted file mode 100644 index fc2ee4f5..00000000 --- a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/data_loader.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -Defines a `load_data` function that returns a generator yielding -feed_dicts so that this script can be used as the argument for -the --data-loader-script command-line parameter. -""" -import numpy as np - -INPUT_SHAPE = (1, 2, 28, 28) - - -def load_data(): - for _ in range(5): - yield {"x": np.ones(shape=INPUT_SHAPE, dtype=np.float32)} # Still totally real data diff --git a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/README.md b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/README.md new file mode 100644 index 00000000..6d1dddb0 --- /dev/null +++ b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/README.md @@ -0,0 +1,56 @@ +# Comparing With Custom Input Data + +## Introduction + +In some cases, we may want to run comparisons using custom input data. +Polygraphy provides multiple ways to do so, which are detailed [here](../../../../polygraphy/tools/README.md#using-custom-input-data). + +In this example, we'll demonstrate 2 different approaches: + +1. Using a data loader script by defining a `load_data()` function in a Python script (`data_loader.py`). + Polygraphy will use `load_data()` to generate inputs at runtime. + +2. Using a JSON file containing pre-generated inputs. + For convenience, we'll use our script from above (`data_loader.py`) to save the inputs + generated by `load_data()` to a file called `custom_inputs.json`. + +*TIP: Generally, a data loader script is preferrable when working with large amounts of input data* + *as it avoids the need to write to the disk.* + *On the other hand, JSON files may be more portable and can help ensure reproducibility.* + +Finally, we'll supply our custom input data to `polygraphy run` and compare outputs between +ONNX-Runtime and TensorRT. + +Since our model has dynamic shapes, we'll need to set up a TensorRT Optimization Profile. +For details on how we can do this via the command-line, +see [`convert` example 03](../../convert/03_dynamic_shapes_in_tensorrt). +For simplicitly, we'll create a profile where `min` == `opt` == `max`. + +*NOTE: It is important that our optimization profile works with the shapes provided by our* + *custom data loader. In our very simple case, the data loader always generates inputs of* + *shape (1, 2, 28, 28), so we just need to ensure this falls within [`min`, `max`].* + +## Running The Example + +1. Run the script to save input data to the disk. + *NOTE: This is only necessary for option 2.* + ```bash + python3 data_loader.py + ``` + +2. Run the model with TensorRT and ONNX-Runtime using custom input data: + - Option 1: Using the data loader script: + + ```bash + polygraphy run dynamic_identity.onnx --trt --onnxrt \ + --trt-min-shapes X:[1,2,28,28] --trt-opt-shapes X:[1,2,28,28] --trt-max-shapes X:[1,2,28,28] \ + --data-loader-script data_loader.py + ``` + + - Option 2: Using the JSON file containing the saved inputs: + + ```bash + polygraphy run dynamic_identity.onnx --trt --onnxrt \ + --trt-min-shapes X:[1,2,28,28] --trt-opt-shapes X:[1,2,28,28] --trt-max-shapes X:[1,2,28,28] \ + --load-inputs custom_inputs.json + ``` diff --git a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/data_loader.py b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/data_loader.py new file mode 100644 index 00000000..4284ddc1 --- /dev/null +++ b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/data_loader.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Demonstrates two methods of loading custom input data in Polygraphy: + +Option 1: Defines a `load_data` function that returns a generator yielding + feed_dicts so that this script can be used as the argument for + the --data-loader-script command-line parameter. + +Option 2: Writes input data to a JSON file that can be used as the argument for + the --load-inputs command-line parameter. +""" +import numpy as np +from polygraphy.json import save_json + +INPUT_SHAPE = (1, 2, 28, 28) + +# Option 1: Define a function that will yield feed_dicts (i.e. Dict[str, np.ndarray]) +def load_data(): + for _ in range(5): + yield {"x": np.ones(shape=INPUT_SHAPE, dtype=np.float32)} # Still totally real data + + +# Option 2: Create a JSON file containing the input data using the `save_json()` helper. +# The input to `save_json()` should have type: List[Dict[str, np.ndarray]]. +# For convenience, we'll reuse our `load_data()` implementation to generate the list. +input_data = list(load_data()) +save_json(input_data, "custom_inputs.json", description="custom input data") diff --git a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/dynamic_identity.onnx b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/dynamic_identity.onnx similarity index 100% rename from tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/dynamic_identity.onnx rename to tools/Polygraphy/examples/cli/run/05_comparing_with_custom_input_data/dynamic_identity.onnx diff --git a/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/README.md b/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/README.md new file mode 100644 index 00000000..dc4ff9ca --- /dev/null +++ b/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/README.md @@ -0,0 +1,38 @@ +# Comparing With Custom Output Data + +## Introduction + +In some cases, it may be useful to compare against output values generated outside Polygraphy. +The simplest way to do so is to create a `RunResults` object and save it to a file. + +This example illustrates how you can generate custom input and output data outside of Polygraphy +and seamlessly load it into Polygraphy for comparison. + +## Running The Example + +1. Generate the input and output data: + + ```bash + python3 generate_data.py + ``` + +2. **[Optional]** Inspect the data. + For inputs: + + ```bash + polygraphy inspect data custom_inputs.json + ``` + + For outputs: + + ```bash + polygraphy inspect data custom_outputs.json + ``` + +3. Run inference with the generated input data and then compare outputs against the custom outputs: + + ```bash + polygraphy run identity.onnx --trt \ + --load-inputs custom_inputs.json \ + --load-outputs custom_outputs.json + ``` diff --git a/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/generate_data.py b/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/generate_data.py new file mode 100644 index 00000000..6155e413 --- /dev/null +++ b/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/generate_data.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Generates input and output data for an identity model and saves it to disk. +""" +import numpy as np +from polygraphy.comparator import RunResults +from polygraphy.json import save_json + +INPUT_SHAPE = (1, 1, 2, 2) + + +# We'll generate arbitrary input data and then "compute" the expected output data before saving both to disk. +# In order for Polygraphy to load the input and output data, they must be in the following format: +# - Input Data: List[Dict[str, np.ndarray]] (A list of feed_dicts) +# - Output Data: RunResults + + +# Generate arbitrary input data compatible with the model. +# +# TIP: We could have alternatively used a generator as in `run` example 05 (05_comparing_with_custom_input_data). +# In that case, we would simply provide this script to `--data-loader-script` instead of saving the inputs here +# and then using `--load-inputs`. +input_data = {"x": np.ones(shape=INPUT_SHAPE, dtype=np.float32)} + +# NOTE: Input data must be in a list (to support multiple inputs), so we create one before saving it. +# The `description` argument is optional: +save_json([input_data], "custom_inputs.json", description="custom input data") + + +# "Compute" the outputs based on the input data. Since this is an identity model, we can just copy the inputs. +output_data = {"y": input_data["x"]} + +# To save output data, we can create a RunResults object: +custom_outputs = RunResults() + +# The `add()` helper function allows us to easily add entries. +# +# NOTE: As with input data, output data must be in a list, so we create one before saving it. +# +# TIP: Alternatively, we can manually add entries using an approach like: +# runner_name = "custom_runner" +# custom_outputs[runner_name] = [IterationResult(output_data, runner_name=runner_name), ...] +# +# TIP: To store outputs from multiple different implementations, you can specify different `runner_name`s to `add()`. +# If `runner_name` is omitted, a default is used. +custom_outputs.add([output_data], runner_name="custom_runner") +custom_outputs.save("custom_outputs.json") diff --git a/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/identity.onnx b/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/identity.onnx new file mode 100644 index 00000000..b7302044 --- /dev/null +++ b/tools/Polygraphy/examples/cli/run/06_comparing_with_custom_output_data/identity.onnx @@ -0,0 +1,15 @@ + backend-test:[ + +xy"Identity test_identityZ +x + + + + +b +y + + + + +B \ No newline at end of file diff --git a/tools/Polygraphy/examples/cli/surgeon/01_isolating_subgraphs/README.md b/tools/Polygraphy/examples/cli/surgeon/01_isolating_subgraphs/README.md index 466f527a..a710fdec 100644 --- a/tools/Polygraphy/examples/cli/surgeon/01_isolating_subgraphs/README.md +++ b/tools/Polygraphy/examples/cli/surgeon/01_isolating_subgraphs/README.md @@ -10,7 +10,7 @@ In this example, we'll extract a subgraph from a model that computes `Y = x0 + ( ![./model.png](./model.png) Let's assume that we want to isolate the subgraph that computes `(a * x1 + b)`, and that we've -used `polygraphy inspect model model.onnx --mode=basic` to determine the names of the input/output tensors +used `polygraphy inspect model model.onnx --show layers` to determine the names of the input/output tensors of this subgraph, but that we don't know the shapes or data types of any of the tensors involved. When shapes and data types are unknown, you can use `auto` to indicate that Polygraphy should @@ -47,7 +47,7 @@ type - hence `--inputs` requires 2 `auto`s and `--outputs` requires only 1. to confirm whether it looks correct: ```bash - polygraphy inspect model subgraph.onnx --mode=basic + polygraphy inspect model subgraph.onnx --show layers ``` ## A Note On `auto` diff --git a/tools/Polygraphy/examples/cli/surgeon/02_folding_constants/README.md b/tools/Polygraphy/examples/cli/surgeon/02_folding_constants/README.md index f1400728..c4e1c0b1 100644 --- a/tools/Polygraphy/examples/cli/surgeon/02_folding_constants/README.md +++ b/tools/Polygraphy/examples/cli/surgeon/02_folding_constants/README.md @@ -31,5 +31,5 @@ where `a`, `b`, and `d` are constants: 2. **[Optional]** You can use `inspect model` to confirm whether it looks correct: ```bash - polygraphy inspect model folded.onnx --mode=basic + polygraphy inspect model folded.onnx --show layers ``` diff --git a/tools/Polygraphy/examples/cli/surgeon/03_modifying_input_shapes/README.md b/tools/Polygraphy/examples/cli/surgeon/03_modifying_input_shapes/README.md index 8c5a41e0..62772bf2 100644 --- a/tools/Polygraphy/examples/cli/surgeon/03_modifying_input_shapes/README.md +++ b/tools/Polygraphy/examples/cli/surgeon/03_modifying_input_shapes/README.md @@ -27,5 +27,5 @@ Output shapes can be inferred and so these are not modified (nor do they need to 2. **[Optional]** You can use `inspect model` to confirm whether it looks correct: ```bash - polygraphy inspect model dynamic_identity.onnx --mode=basic + polygraphy inspect model dynamic_identity.onnx --show layers ``` diff --git a/tools/Polygraphy/polygraphy/__init__.py b/tools/Polygraphy/polygraphy/__init__.py index ad406a02..0286d5f0 100644 --- a/tools/Polygraphy/polygraphy/__init__.py +++ b/tools/Polygraphy/polygraphy/__init__.py @@ -1,3 +1,3 @@ import polygraphy.config -__version__ = "0.33.2" +__version__ = "0.35.1" diff --git a/tools/Polygraphy/polygraphy/backend/onnx/util.py b/tools/Polygraphy/polygraphy/backend/onnx/util.py index a256c314..042d499f 100644 --- a/tools/Polygraphy/polygraphy/backend/onnx/util.py +++ b/tools/Polygraphy/polygraphy/backend/onnx/util.py @@ -149,17 +149,22 @@ def get_output_metadata(graph): return get_tensor_metadata(graph.output) -def str_from_onnx(model, mode="full"): +def str_from_onnx(model, show_layers=None, show_attrs=None, show_weights=None): """ Converts an ONNX Graph to a human-readable representation Args: graph (onnx.GraphProto): The onnx graph. - mode (str): Controls what is displayed. Choices: ["none", "basic", "attrs", "full"] + show_layers (bool): Whether to display per-layer information. + show_attrs (bool): Whether to display per-layer attributes. + show_weights (bool): Whether to display the value of weights. Returns: str """ + show_layers = util.default(show_layers, False) + show_attrs = util.default(show_attrs, False) + show_weights = util.default(show_weights, False) def get_opset(): try: @@ -172,11 +177,14 @@ def get_opset(): onnx_str += "Name: {:} | Opset: {:}\n".format(model.graph.name, get_opset()) onnx_str += "\n" - onnx_str += str_from_onnx_graph(model.graph, mode=mode, tensors={}) + onnx_str += str_from_onnx_graph( + model.graph, tensors={}, show_layers=show_layers, show_attrs=show_attrs, show_weights=show_weights + ) return onnx_str -def str_from_onnx_graph(graph, mode, tensors, indent_level=0): +def str_from_onnx_graph(graph, tensors, show_layers, show_attrs, show_weights, indent_level=0): + input_metadata = get_input_metadata(graph) output_metadata = get_output_metadata(graph) initializer_metadata = get_tensor_metadata(graph.initializer) @@ -195,14 +203,14 @@ def str_from_onnx_graph(graph, mode, tensors, indent_level=0): onnx_str += "---- {:} {:} Output(s) ----\n{:}\n\n".format(len(output_metadata), graph_type, output_metadata) onnx_str += "---- {:} Initializer(s) ----\n".format(len(initializer_metadata)) - if mode == "full": + if show_weights: for init in graph.initializer: onnx_str += "Initializer | {:} [dtype={:}, shape={:}] | Values:\n{:}\n\n".format( init.name, get_dtype(init), get_shape(init), util.indent_block(str(get_values(init))) ) if not graph.initializer: onnx_str += "{}\n\n" - elif mode != "none": + elif show_layers: onnx_str += str(initializer_metadata) onnx_str += "\n\n" else: @@ -242,11 +250,18 @@ def process_attr(attr_str: str): processed = processed.decode() elif attr_str == "TENSOR": tensor_str = "Tensor: [dtype={:}, shape={:}]".format(get_dtype(processed), get_shape(processed)) - if mode == "full": + if show_weights: tensor_str += " | Values:\n" + util.indent_block(str(get_values(processed))) processed = tensor_str elif attr_str == "GRAPH": - processed = "\n" + str_from_onnx_graph(processed, mode, tensors, indent_level=indent_level + 2) + processed = "\n" + str_from_onnx_graph( + processed, + tensors, + indent_level=indent_level + 2, + show_layers=show_layers, + show_attrs=show_attrs, + show_weights=show_weights, + ) elif attr_str == "FLOATS" or attr_str == "INTS": # Proto hacky list to normal Python list processed = [p for p in processed] @@ -270,14 +285,14 @@ def process_attr(attr_str: str): return attr_dict onnx_str += "---- {:} Node(s) ----\n".format(len(graph.node)) - if mode != "none": + if show_layers: for index, node in enumerate(graph.node): input_info = metadata_from_names(node.input) output_info = metadata_from_names(node.output) onnx_str += util.str_from_layer("Node", index, node.name, node.op_type, input_info, output_info) - if mode in ["attrs", "full"]: + if show_attrs: attrs = attrs_to_dict(node.attribute) if attrs: onnx_str += util.indent_block("---- Attributes ----") + "\n" diff --git a/tools/Polygraphy/polygraphy/backend/onnxrt/loader.py b/tools/Polygraphy/polygraphy/backend/onnxrt/loader.py index 18c64043..959ecb7c 100644 --- a/tools/Polygraphy/polygraphy/backend/onnxrt/loader.py +++ b/tools/Polygraphy/polygraphy/backend/onnxrt/loader.py @@ -15,8 +15,9 @@ # from polygraphy import mod, util from polygraphy.backend.base import BaseLoader +from polygraphy.logger import G_LOGGER -onnxruntime = mod.lazy_import("onnxruntime") +onnxrt = mod.lazy_import("onnxruntime") @mod.export(funcify=True) @@ -25,15 +26,24 @@ class SessionFromOnnx(BaseLoader): Functor that builds an ONNX-Runtime inference session. """ - def __init__(self, model_bytes): + def __init__(self, model_bytes, providers=None): """ Builds an ONNX-Runtime inference session. Args: model_bytes (Union[Union[bytes, str], Callable() -> Union[bytes, str]]): A serialized ONNX model or a path to a model or a callable that returns one of those. + + providers (Sequence[str]): + A sequence of execution providers to use in order of priority. + Each element of the sequence may be either an exact match or a case-insensitive partial match + for the execution providers available in ONNX-Runtime. For example, a value of "cpu" would + match the "CPUExecutionProvider". + Defaults to ``["cpu"]``. + """ self._model_bytes_or_path = model_bytes + self.providers = util.default(providers, ["cpu"]) def call_impl(self): """ @@ -41,4 +51,18 @@ def call_impl(self): onnxruntime.InferenceSession: The inference session. """ model_bytes, _ = util.invoke_if_callable(self._model_bytes_or_path) - return onnxruntime.InferenceSession(model_bytes) + + available_providers = onnxrt.get_available_providers() + providers = [] + for prov in self.providers: + matched_prov = util.find_str_in_iterable(prov, available_providers) + if matched_prov is None: + G_LOGGER.critical( + "Could not find specified ONNX-Runtime execution provider.\n" + "Note: Requested provider was: {:}, but available providers are: {:}".format( + prov, available_providers + ) + ) + providers.append(matched_prov) + + return onnxrt.InferenceSession(model_bytes, providers=providers) diff --git a/tools/Polygraphy/polygraphy/backend/onnxrt/runner.py b/tools/Polygraphy/polygraphy/backend/onnxrt/runner.py index 796a302f..03470f86 100644 --- a/tools/Polygraphy/polygraphy/backend/onnxrt/runner.py +++ b/tools/Polygraphy/polygraphy/backend/onnxrt/runner.py @@ -54,7 +54,7 @@ def get_input_metadata_impl(self): "tensor(uint32)": np.uint32, "tensor(uint64)": np.uint64, "tensor(uint8)": np.uint8, - "tensor(bool)": np.bool, + "tensor(bool)": bool, "tensor(string)": np.unicode, } diff --git a/tools/Polygraphy/polygraphy/backend/pluginref/references.py b/tools/Polygraphy/polygraphy/backend/pluginref/references.py index fb71311e..63ec1b56 100644 --- a/tools/Polygraphy/polygraphy/backend/pluginref/references.py +++ b/tools/Polygraphy/polygraphy/backend/pluginref/references.py @@ -83,3 +83,19 @@ def run_instancenorm(attrs, x, weights, bias): res = weights * (x - mean) / np.sqrt(var + epsilon) + bias return [res] + + +@register("MeanVarianceNormalization") +def run_meanvarnorm(attrs, x): + epsilon = 1.0e-9 + axes = attrs.get("axes", [0, 2, 3]) + axes = tuple(axes) + + data_mean = np.mean(x, axis=axes, keepdims=True) + data_mean_squared = np.power(data_mean, 2) + data_squared = np.power(x, 2) + data_squared_mean = np.mean(data_squared, axis=axes, keepdims=True) + std = np.sqrt(data_squared_mean - data_mean_squared) + res = (x - data_mean) / (std + epsilon) + + return [res] diff --git a/tools/Polygraphy/polygraphy/backend/tf/util.py b/tools/Polygraphy/polygraphy/backend/tf/util.py index 60f481d0..71537b3c 100644 --- a/tools/Polygraphy/polygraphy/backend/tf/util.py +++ b/tools/Polygraphy/polygraphy/backend/tf/util.py @@ -161,9 +161,7 @@ def is_output_node(node): "logging verbosity to EXTRA_VERBOSE to view them.".format(len(output_nodes) - len(output_tensors)) ) - G_LOGGER.extra_verbose( - "Found output op types in graph: {:}".format({tensor.op.type for tensor in output_tensors}) - ) + G_LOGGER.extra_verbose("Found output op types in graph: {:}".format({tensor.op.type for tensor in output_tensors})) G_LOGGER.verbose("Retrieved TensorFlow output_tensors: {:}".format(output_tensors)) return get_tensor_metadata(output_tensors) @@ -172,7 +170,11 @@ def get_graph_output_names(graph): return list(get_output_metadata(graph).keys()) -def str_from_graph(graph, mode): +def str_from_graph(graph, show_layers=None, show_attrs=None, show_weights=None): + show_layers = util.default(show_layers, False) + show_attrs = util.default(show_attrs, False) + show_weights = util.default(show_weights, False) + graph_str = "" input_metadata = get_input_metadata(graph) output_metadata = get_output_metadata(graph) @@ -180,12 +182,12 @@ def str_from_graph(graph, mode): graph_str += "---- {:} Graph Inputs ----\n{:}\n\n".format(len(input_metadata), input_metadata) graph_str += "---- {:} Graph Outputs ----\n{:}\n\n".format(len(output_metadata), output_metadata) graph_str += "---- {:} Nodes ----\n".format(len(graph.as_graph_def().node)) - if mode == "basic": + if show_layers: G_LOGGER.warning( "Displaying layer information is unsupported for TensorFlow graphs. " - "Please use --mode=full if you would like to see the raw nodes" + "Please use --show layers attrs weights if you would like to see the raw nodes" ) - if mode == "full": + if show_attrs or show_weights: for node in graph.as_graph_def().node: graph_str += str(node) + "\n" graph_str += "\n" diff --git a/tools/Polygraphy/polygraphy/backend/trt/calibrator.py b/tools/Polygraphy/polygraphy/backend/trt/calibrator.py index e879e44b..6c71db0c 100644 --- a/tools/Polygraphy/polygraphy/backend/trt/calibrator.py +++ b/tools/Polygraphy/polygraphy/backend/trt/calibrator.py @@ -48,7 +48,7 @@ def Calibrator( By default, the calibration cache is not saved. BaseClass (type): The type of calibrator to inherit from. - Defaults to ``trt.IInt8MinMaxCalibrator``. + Defaults to ``trt.IInt8EntropyCalibrator2``. batch_size (int): [DEPRECATED] The size of each batch provided by the data loader. quantile (float): @@ -64,7 +64,7 @@ def Calibrator( Has no effect for other calibrator types. Defaults to ``trt.CalibrationAlgoType.MINMAX_CALIBRATION``. """ - BaseClass = util.default(BaseClass, trt.IInt8MinMaxCalibrator) + BaseClass = util.default(BaseClass, trt.IInt8EntropyCalibrator2) class CalibratorClass(BaseClass): """ diff --git a/tools/Polygraphy/polygraphy/backend/trt/loader.py b/tools/Polygraphy/polygraphy/backend/trt/loader.py index 89ab6dc6..bf4271cc 100644 --- a/tools/Polygraphy/polygraphy/backend/trt/loader.py +++ b/tools/Polygraphy/polygraphy/backend/trt/loader.py @@ -269,14 +269,13 @@ def __init__( restricted=None, use_dla=None, allow_gpu_fallback=None, + profiling_verbosity=None, + memory_pool_limits=None, ): """ Creates a TensorRT IBuilderConfig that can be used by EngineFromNetwork. Args: - max_workspace_size (int): - The maximum workspace size, in bytes, when building the engine. - Defaults to 16 MiB. tf32 (bool): Whether to build the engine with TF32 precision enabled. Defaults to False. @@ -332,8 +331,23 @@ def __init__( [EXPERIMENTAL] When DLA is enabled, whether to allow layers to fall back to GPU if they cannot be run on DLA. Has no effect if DLA is not enabled. Defaults to False. + profiling_verbosity (trt.ProfilingVerbosity): + The verbosity of NVTX annotations in the generated engine. + Higher verbosity allows you to determine more information about the engine. + Defaults to ``trt.ProfilingVerbosity.VERBOSE``. + memory_pool_limits (Dict[trt.MemoryPoolType, int]): + Limits for different memory pools. + This should be a mapping of pool types to their respective limits in bytes. + max_workspace_size (int): + [DEPRECATED - use memory_pool_limits] + The maximum workspace size, in bytes, when building the engine. + Defaults to 16 MiB. + """ self.max_workspace_size = util.default(max_workspace_size, 1 << 24) + if max_workspace_size is not None: + mod.warn_deprecated("max_workspace_size", use_instead="memory_pool_limits", remove_in="0.45.0") + self.tf32 = util.default(tf32, False) self.fp16 = util.default(fp16, False) self.int8 = util.default(int8, False) @@ -348,6 +362,8 @@ def __init__( self.tactic_sources = tactic_sources self.use_dla = util.default(use_dla, False) self.allow_gpu_fallback = util.default(allow_gpu_fallback, False) + self.profiling_verbosity = profiling_verbosity + self.memory_pool_limits = memory_pool_limits if self.calibrator is not None and not self.int8: G_LOGGER.warning( @@ -436,6 +452,22 @@ def try_set_flag(flag_name): if self.allow_gpu_fallback: try_set_flag("GPU_FALLBACK") + if self.profiling_verbosity is not None: + + def set_profiling_verbosity(): + config.profiling_verbosity = self.profiling_verbosity + + try_run(set_profiling_verbosity, name="profiling_verbosity") + else: + try: + config.profiling_verbosity = trt.ProfilingVerbosity.VERBOSE + except AttributeError: + pass + + if self.memory_pool_limits is not None: + for pool_type, pool_size in self.memory_pool_limits.items(): + try_run(lambda: config.set_memory_pool_limit(pool_type, pool_size), name="memory_pool_limits") + if self.tactic_sources is not None: tactic_sources_flag = 0 for source in self.tactic_sources: @@ -461,7 +493,7 @@ def try_set_flag(flag_name): def set_algo_selector(): config.algorithm_selector = self.algorithm_selector - try_run(set_algo_selector, "algorithm_selector") + try_run(set_algo_selector, name="algorithm_selector") return config @@ -538,9 +570,16 @@ def call_impl(self): else: stack.enter_context(config.int8_calibrator) - network_log_mode = "full" if G_LOGGER.severity <= G_LOGGER.ULTRA_VERBOSE else "attrs" G_LOGGER.super_verbose( - lambda: ("Displaying TensorRT Network:\n" + trt_util.str_from_network(network, mode=network_log_mode)) + lambda: ( + "Displaying TensorRT Network:\n" + + trt_util.str_from_network( + network, + show_layers=True, + show_attrs=True, + show_weights=G_LOGGER.severity <= G_LOGGER.ULTRA_VERBOSE, + ) + ) ) G_LOGGER.start("Building engine with configuration:\n{:}".format(trt_util.str_from_config(config))) @@ -561,15 +600,15 @@ def call_impl(self): G_LOGGER.finish("Finished engine building in {:.3f} seconds".format(end_time - start_time)) - try: - timing_cache = config.get_timing_cache() - except AttributeError: - if self.timing_cache_path: + if self.timing_cache_path: + try: + timing_cache = config.get_timing_cache() + except AttributeError: trt_util.fail_unavailable("save_timing_cache in EngineBytesFromNetwork") - else: - if timing_cache and self.timing_cache_path: - with timing_cache.serialize() as buffer: - util.save_file(buffer, self.timing_cache_path, description="tactic timing cache") + else: + if timing_cache: + with timing_cache.serialize() as buffer: + util.save_file(buffer, self.timing_cache_path, description="tactic timing cache") return engine_bytes diff --git a/tools/Polygraphy/polygraphy/backend/trt/util.py b/tools/Polygraphy/polygraphy/backend/trt/util.py index c7b985f4..8a1cfb2a 100644 --- a/tools/Polygraphy/polygraphy/backend/trt/util.py +++ b/tools/Polygraphy/polygraphy/backend/trt/util.py @@ -14,6 +14,7 @@ # limitations under the License. # import contextlib +import json from polygraphy import config, mod, util from polygraphy.common import TensorMetadata @@ -188,17 +189,23 @@ def is_valid_attribute(attr, layer): ] -def str_from_network(network, mode="full"): +def str_from_network(network, show_layers=None, show_attrs=None, show_weights=None): """ Converts a TensorRT network to a human-readable representation Args: network (trt.INetworkDefinition): The network. - mode (str): Controls what is displayed for each layer. Choices: ["none", "basic", "attrs", "full"] + show_layers (bool): Whether to display per-layer information. + show_attrs (bool): Whether to display per-layer attributes. + show_weights (bool): Whether to display the value of weights. Returns: str """ + show_layers = util.default(show_layers, False) + show_attrs = util.default(show_attrs, False) + show_weights = util.default(show_weights, False) + LAYER_TYPE_CLASS_MAPPING = get_layer_class_mapping() network_str = "Name: {:} | {:} Batch Network{:}\n".format( @@ -217,14 +224,14 @@ def str_from_network(network, mode="full"): output_metadata = get_network_output_metadata(network) network_str += "---- {:} Network Output(s) ----\n{:}\n\n".format(len(output_metadata), output_metadata) network_str += "---- {:} Layer(s) ----\n".format(network.num_layers) - if mode != "none": + if show_layers: for index, layer in enumerate(network): if layer.type in LAYER_TYPE_CLASS_MAPPING: layer.__class__ = LAYER_TYPE_CLASS_MAPPING[layer.type] network_str += str_from_layer(layer, index) - if mode in ["attrs", "full"]: + if show_attrs: # Exclude special attributes, as well as any attributes of the base layer class (those can be displayed above). attrs = get_layer_attribute_names(layer) if attrs: @@ -232,7 +239,7 @@ def str_from_network(network, mode="full"): for attr in attrs: with G_LOGGER.verbosity(): val = getattr(layer, attr) - if mode == "full" or not isinstance(val, np.ndarray): + if show_weights or not isinstance(val, np.ndarray): attr_str = "" if layer.name: attr_str += "{:}.".format(layer.name) @@ -436,15 +443,17 @@ def get_output_metadata_from_engine(engine, start_binding, end_binding): return outputs -def str_from_engine(engine): +def str_from_engine(engine, show_layers=None, show_attrs=None): + show_layers = util.default(show_layers, False) + show_attrs = util.default(show_attrs, False) + bindings_per_profile = get_bindings_per_profile(engine) - engine_str = "Name: {:} | {:}{:} Batch Engine ({:} layers)\n".format( + engine_str = "Name: {:} | {:}{:} Batch Engine\n".format( engine.name, "Refittable " if engine.refittable else "", "Implicit" if hasattr(engine, "has_implicit_batch_dimension") and engine.has_implicit_batch_dimension else "Explicit", - engine.num_layers, ) engine_str += "\n" @@ -481,6 +490,82 @@ def str_from_engine(engine): else: engine_str += " | Shape: {:}\n".format(engine.get_binding_shape(binding)) engine_str += "\n" + + layers_per_profile = engine.num_layers // engine.num_optimization_profiles + engine_str += "---- {:} Layer(s){:} ----\n".format( + layers_per_profile, " Per Profile" if engine.num_optimization_profiles > 1 else "" + ) + if show_layers: + try: + inspector = engine.create_engine_inspector() + except AttributeError: + G_LOGGER.warning( + "Cannot show layer information because IEngineInspector is not available in this version of TensorRT ({:})".format( + trt.__version__ + ) + ) + else: + for profile_idx in range(engine.num_optimization_profiles): + indent_level = 0 + if engine.num_optimization_profiles >= 1: + indent_level = 1 + engine_str += "- Profile: {:}\n".format(profile_idx) + + offset = profile_idx * layers_per_profile + for index in range(layers_per_profile): + layer_info = json.loads( + inspector.get_layer_information(offset + index, trt.LayerInformationFormat.JSON) + ) + + op = "Unknown" + input_info = TensorMetadata() + output_info = TensorMetadata() + origin = "Unknown" + tactic = "Unknown" + if engine.profiling_verbosity == trt.ProfilingVerbosity.DETAILED: + name = layer_info.get("Name", "Unknown") + op = layer_info.get("LayerType", "Unknown") + + def meta_from_inspector(key): + meta = TensorMetadata() + info = layer_info.get(key) + if info is None: + return meta + for elem in info: + meta.add(name=elem["Name"], dtype=None, shape=elem["Dimensions"]) + return meta + + input_info = meta_from_inspector("Inputs") + output_info = meta_from_inspector("Outputs") + origin = layer_info.get("Origin", "Unknown") + tactic = layer_info.get("TacticValue", "Unknown") + else: + G_LOGGER.warning( + "This engine was created with a profiling verbosity of: {:}. Some layer information may be missing. " + "Try setting a higher profiling verbosity to see more detailed layer information. ".format( + engine.profiling_verbosity + ), + mode=LogMode.ONCE, + ) + name = layer_info + + engine_str += ( + util.indent_block( + util.str_from_layer( + "Layer", index, name, op, input_info=input_info, output_info=output_info + ), + indent_level, + ) + + "\n" + ) + + if show_attrs: + engine_str += util.indent_block("---- Attributes ----", indent_level + 1) + "\n" + engine_str += util.indent_block("Origin = {:}".format(origin), indent_level + 1) + "\n" + engine_str += util.indent_block("Tactic = {:}".format(tactic), indent_level + 1) + "\n" + + engine_str += "\n" + return util.indent_block(engine_str, level=0) diff --git a/tools/Polygraphy/polygraphy/common/constants.py b/tools/Polygraphy/polygraphy/common/constants.py deleted file mode 100644 index 56efabd2..00000000 --- a/tools/Polygraphy/polygraphy/common/constants.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from polygraphy import mod -from polygraphy.constants import * - -mod.warn_deprecated("polygraphy.common.constants", "polygraphy.constants", remove_in="0.34.0") diff --git a/tools/Polygraphy/polygraphy/common/cuda.py b/tools/Polygraphy/polygraphy/common/cuda.py deleted file mode 100644 index 460d23a9..00000000 --- a/tools/Polygraphy/polygraphy/common/cuda.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from polygraphy import mod -from polygraphy.cuda import * - -mod.warn_deprecated("polygraphy.common.cuda", "polygraphy.cuda", remove_in="0.34.0") diff --git a/tools/Polygraphy/polygraphy/common/exception.py b/tools/Polygraphy/polygraphy/common/exception.py deleted file mode 100644 index f34f1100..00000000 --- a/tools/Polygraphy/polygraphy/common/exception.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from polygraphy import mod -from polygraphy.exception import * - -mod.warn_deprecated("polygraphy.common.exception", "polygraphy.exception", remove_in="0.34.0") diff --git a/tools/Polygraphy/polygraphy/common/func.py b/tools/Polygraphy/polygraphy/common/func.py deleted file mode 100644 index d48310f8..00000000 --- a/tools/Polygraphy/polygraphy/common/func.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from polygraphy import mod -from polygraphy.func import * - -mod.warn_deprecated("polygraphy.common.func", "polygraphy.func", remove_in="0.34.0") diff --git a/tools/Polygraphy/polygraphy/comparator/compare.py b/tools/Polygraphy/polygraphy/comparator/compare.py index c689fbac..b2dc1784 100644 --- a/tools/Polygraphy/polygraphy/comparator/compare.py +++ b/tools/Polygraphy/polygraphy/comparator/compare.py @@ -126,9 +126,6 @@ def check_outputs_match( mean_absdiff = comp_util.compute_mean(absdiff) median_absdiff = comp_util.compute_median(absdiff) - max_elemwiseabs = "Unknown" - max_elemwiserel = "Unknown" - if per_out_err_stat == "mean": failed = mean_absdiff > per_out_atol and (np.isnan(mean_reldiff) or mean_reldiff > per_out_rtol) elif per_out_err_stat == "median": @@ -143,12 +140,6 @@ def check_outputs_match( failed = np.any(mismatches) try: - with np.testing.suppress_warnings() as sup: - sup.filter(RuntimeWarning) - # Special because we need to account for tolerances too. - max_elemwiseabs = comp_util.compute_max(absdiff[mismatches]) - max_elemwiserel = comp_util.compute_max(reldiff[mismatches]) - with G_LOGGER.indent(): G_LOGGER.super_verbose("Mismatched indices:\n{:}".format(np.argwhere(mismatches))) G_LOGGER.extra_verbose("{:35} | Mismatched values:\n{:}".format(runner0_name, out0[mismatches])) @@ -167,21 +158,23 @@ def check_outputs_match( G_LOGGER.info("Error Metrics: {:}".format(out0_name)) with G_LOGGER.indent(): - def req_tol(mean_diff, median_diff, max_diff, elemwise_diff): + def req_tol(mean_diff, median_diff, max_diff): return { "mean": mean_diff, "median": median_diff, "max": max_diff, - "elemwise": elemwise_diff, + "elemwise": max_diff, }[per_out_err_stat] - G_LOGGER.info( - "Minimum Required Tolerance: {:} error | [abs={:.5g}] OR [rel={:.5g}]".format( - per_out_err_stat, - req_tol(mean_absdiff, median_absdiff, max_absdiff, max_elemwiseabs), - req_tol(mean_reldiff, median_reldiff, max_reldiff, max_elemwiserel), - ) + msg = "Minimum Required Tolerance: {:} error | [abs={:.5g}] OR [rel={:.5g}]".format( + per_out_err_stat, + req_tol(mean_absdiff, median_absdiff, max_absdiff), + req_tol(mean_reldiff, median_reldiff, max_reldiff), ) + if per_out_err_stat == "elemwise": + msg += " (requirements may be lower if both abs/rel tolerances are set)" + G_LOGGER.info(msg) + comp_util.log_output_stats(absdiff, failed, "Absolute Difference") with np.testing.suppress_warnings() as sup: sup.filter(RuntimeWarning) @@ -235,6 +228,9 @@ def simple(check_shapes=None, rtol=None, atol=None, fail_fast=None, find_output_ Defaults to True. rtol (Union[float, Dict[str, float]]): The relative tolerance to use when checking accuracy. + This is expressed as a percentage of the second set of output values. + For example, a value of 0.01 would check that the first set of outputs is within 1% of the second. + This can be provided on a per-output basis using a dictionary. In that case, use an empty string ("") as the key to specify default tolerance for outputs not explicitly listed. Defaults to 1e-5. @@ -256,6 +252,10 @@ def simple(check_shapes=None, rtol=None, atol=None, fail_fast=None, find_output_ The error statistic to check. Possible values are: - "elemwise": Checks each element in the output to determine if it exceeds both tolerances specified. + The minimum required tolerances displayed in this mode are only applicable when just one type of tolerance + is set. Because of the nature of the check, when both absolute/relative tolerance are specified, the required + minimum tolerances may be lower. + - "max": Checks the maximum absolute/relative errors against the respective tolerances. This is the strictest possible check. - "mean" Checks the mean absolute/relative errors against the respective tolerances. - "median": Checks the median absolute/relative errors against the respective tolerances. @@ -320,11 +320,11 @@ def check_dict(dct, dict_name): G_LOGGER.info("Strict shape checking disabled. Will attempt to match output shapes before comparisons") def default_find_output_func(output_name, index, iter_result): - found_name = util.find_in_dict(output_name, iter_result, index) + found_name = util.find_str_in_iterable(output_name, iter_result.keys(), index) if found_name is None: return None elif found_name != output_name: - exact_match = util.find_in_dict(found_name, iter_result0) + exact_match = util.find_str_in_iterable(found_name, iter_result0.keys()) if exact_match == found_name: G_LOGGER.verbose( "Will not compare {:} with {:}, since the former already has an exact match: {:}".format( diff --git a/tools/Polygraphy/polygraphy/comparator/data_loader.py b/tools/Polygraphy/polygraphy/comparator/data_loader.py index 46061925..d338ed8a 100644 --- a/tools/Polygraphy/polygraphy/comparator/data_loader.py +++ b/tools/Polygraphy/polygraphy/comparator/data_loader.py @@ -88,12 +88,12 @@ def default_tuple(tup, default): self.int_range_set = int_range is not None if self.int_range_set: - mod.warn_deprecated("The int_range parameter in DataLoader", "val_range", remove_in="0.35.0") + mod.warn_deprecated("The int_range parameter in DataLoader", "val_range", remove_in="0.50.0") self.int_range = default_tuple(int_range, (1, 25)) self.float_range_set = float_range is not None if self.float_range_set: - mod.warn_deprecated("The float_range parameter in DataLoader", "val_range", remove_in="0.35.0") + mod.warn_deprecated("The float_range parameter in DataLoader", "val_range", remove_in="0.50.0") self.float_range = default_tuple(float_range, (-1.0, 1.0)) self.input_metadata = None @@ -247,7 +247,7 @@ def generate_buffer(name, dtype, shape): msg = "Input tensor: {:} | Metadata was provided, but the input does not exist in one or more runners.".format( name ) - close_match = util.find_in_dict(name, self.input_metadata) + close_match = util.find_str_in_iterable(name, self.input_metadata.keys()) if close_match: msg += "\nMaybe you meant to set: {:}?".format(close_match) G_LOGGER.warning(msg) @@ -282,7 +282,7 @@ def __getitem__(self, iteration): # Attempts to match existing input buffers to the requested input_metadata def coerce_cached_input(index, name, dtype, shape): cached_feed_dict = self.cache[iteration] - cached_name = util.find_in_dict(name, cached_feed_dict, index) + cached_name = util.find_str_in_iterable(name, cached_feed_dict.keys(), index) util.check(cached_name is not None) if cached_name != name: diff --git a/tools/Polygraphy/polygraphy/comparator/struct.py b/tools/Polygraphy/polygraphy/comparator/struct.py index 19d31fc5..a8c47781 100644 --- a/tools/Polygraphy/polygraphy/comparator/struct.py +++ b/tools/Polygraphy/polygraphy/comparator/struct.py @@ -14,9 +14,10 @@ # limitations under the License. # +import time from collections import OrderedDict -from polygraphy import mod, util, config +from polygraphy import config, mod, util from polygraphy.common.interface import TypedDict, TypedList from polygraphy.json import Decoder, Encoder, add_json_methods, load_json, save_json from polygraphy.logger import G_LOGGER @@ -111,9 +112,12 @@ def __init__(self, outputs=None, runtime=None, runner_name=None): Args: outputs (Dict[str, np.array]): The outputs of this iteration, mapped to their names. - - runtime (float): The time required for this iteration, in seconds. - runner_name (str): The name of the runner that produced this output. + runtime (float): + The time required for this iteration, in seconds. + Only used for logging purposes. + runner_name (str): + The name of the runner that produced this output. + If this is omitted, a default name is generated. """ if outputs and config.ARRAY_SWAP_THRESHOLD_MB < 0: total_size_gb = sum(arr.nbytes for arr in outputs.values() if isinstance(arr, np.ndarray)) / (1024.0 ** 3) @@ -126,7 +130,7 @@ def __init__(self, outputs=None, runtime=None, runner_name=None): super().__init__(IterationResult._to_lazy_dict(outputs)) self.runtime = runtime - self.runner_name = util.default(runner_name, "") + self.runner_name = util.default(runner_name, "custom_runner") # Convenience methods to preserve np.ndarray in the interface. def update(self, other): @@ -178,7 +182,18 @@ def decode(dct): @add_json_methods("inference results") class RunResults(TypedList(lambda: tuple)): """ - Maps runner names to zero or more IterationResults. + Maps runners to per-iteration outputs (in the form of a ``List[IterationResult]``). + + For example, if ``results`` is an instance of ``RunResults()``, then + to access the outputs of the first iteration from a specified runner, do: + :: + + iteration = 0 + runner_name = "trt-runner" + outputs = results[runner_name][iteration] + + # `outputs` is a `Dict[str, np.ndarray]` + Note: Technically, this is a ``List[Tuple[str, List[IterationResult]]]``, but includes helpers that make it behave like an OrderedDict that can contain duplicates. @@ -218,6 +233,35 @@ def update(self, other): self.lst[name] = iteration_results return self + def add(self, out_list, runtime=None, runner_name=None): + """ + A helper to create a ``List[IterationResult]`` and map it to the specified runner_name. + + This method cannot be used to modify an existing entry. + + Calling this method is equivalent to: + :: + + results[runner_name] = [] + for out in out_list: + results[runner_name].append(IterationResult(out, runtime, runner_name)) + + Args: + out_list (List[Dict[str, np.array]]): + One or more set of outputs where each output is a dictionary + of output names mapped to NumPy arrays. + + runtime (float): + The time required for this iteration, in seconds. + Only used for logging purposes. + runner_name (str): + The name of the runner that produced this output. + If this is omitted, a default name is generated. + """ + runner_name = util.default(runner_name, "custom_runner") + iter_results = [IterationResult(out, runtime, runner_name) for out in out_list] + self[runner_name] = iter_results + def __getitem__(self, key): if isinstance(key, int): return self.lst[key] diff --git a/tools/Polygraphy/polygraphy/mod/exporter.py b/tools/Polygraphy/polygraphy/mod/exporter.py index 7e757126..a79a3d19 100644 --- a/tools/Polygraphy/polygraphy/mod/exporter.py +++ b/tools/Polygraphy/polygraphy/mod/exporter.py @@ -205,18 +205,20 @@ def pascal_to_snake(name): return export_impl -def warn_deprecated(name, use_instead, remove_in, module_name=None): +def warn_deprecated(name, use_instead, remove_in, module_name=None, always_show_warning=False): + if config.INTERNAL_CORRECTNESS_CHECKS and version(polygraphy.__version__) >= version(remove_in): G_LOGGER.internal_error("{:} should have been removed in version: {:}".format(name, remove_in)) full_obj_name = "{:}.{:}".format(module_name, name) if module_name else name - warnings.warn( - "{:} is deprecated and will be removed in Polygraphy {:}. " - "Use {:} instead.".format(full_obj_name, remove_in, use_instead), - DeprecationWarning, - stacklevel=3, + msg = "{:} is deprecated and will be removed in Polygraphy {:}. Use {:} instead.".format( + full_obj_name, remove_in, use_instead ) + warnings.warn(msg, DeprecationWarning, stacklevel=3) + if always_show_warning: + G_LOGGER.warning(msg) + def deprecate(remove_in, use_instead, module_name=None, name=None): """ @@ -314,7 +316,6 @@ def export_deprecated_alias_impl(obj): obj ) _define_in_module(name, new_obj, module) - _add_to_all(name, module) return obj return export_deprecated_alias_impl diff --git a/tools/Polygraphy/polygraphy/tools/README.md b/tools/Polygraphy/polygraphy/tools/README.md index 98c0ede1..9e64bd18 100644 --- a/tools/Polygraphy/polygraphy/tools/README.md +++ b/tools/Polygraphy/polygraphy/tools/README.md @@ -98,7 +98,7 @@ You can find the complete listing of `run` examples [here](../../examples/cli/ru For any tools that use inference input data, such as `run` or `convert`, Polygraphy provides 2 ways to supply custom input data: -1. `--load-input-data`, which takes a path to a JSON file containing a `List[Dict[str, np.ndarray]]`. +1. `--load-inputs`/`--load-input-data`, which takes a path to a JSON file containing a `List[Dict[str, np.ndarray]]`. This will cause Polygraphy to load the entire object into memory. *NOTE: This may be impractical or impossible if the data is very large.* @@ -111,7 +111,7 @@ provides 2 ways to supply custom input data: *a separate script just for the sake of `--data-loader-script`. You can simply use the existing script* *and optionally use the `--data-loader-func-name` argument to specify the name of the function if it's not `load_data`* -For more information, refer to [`run` example 05](../../examples/cli/run/05_comparing_with_custom_data/). +For more information, refer to [`run` example 05](../../examples/cli/run/05_comparing_with_custom_input_data/). ### Modifying Input Shapes In An ONNX Model diff --git a/tools/Polygraphy/polygraphy/tools/args/comparator.py b/tools/Polygraphy/polygraphy/tools/args/comparator.py index 748d84f6..d7aa0795 100644 --- a/tools/Polygraphy/polygraphy/tools/args/comparator.py +++ b/tools/Polygraphy/polygraphy/tools/args/comparator.py @@ -120,11 +120,12 @@ def add_to_parser(self, parser): "--rtol", "--rel-tol", dest="rtol", - help="Relative tolerance for output comparison. " + help="Relative tolerance for output comparison. This is expressed as a percentage of the second set of output values. " + "For example, a value of 0.01 would check that the first set of outputs is within 1%% of the second. " "To specify per-output tolerances, use the format: --rtol [:]. If no output name is provided, " "the tolerance is used for any outputs not explicitly specified. For example: " "--rtol 1e-5 out0:1e-4 out1:1e-3. " - "Note that the default tolerance typically works well for FP32, but may be too strict for lower precisions like FP16 or INT8.", + "Note that the default tolerance typically works well for FP32 but may be too strict for lower precisions like FP16 or INT8.", nargs="+", default=None, ) @@ -136,7 +137,7 @@ def add_to_parser(self, parser): "To specify per-output tolerances, use the format: --atol [:]. If no output name is provided, " "the tolerance is used for any outputs not explicitly specified. For example: " "--atol 1e-5 out0:1e-4 out1:1e-3. " - "Note that the default tolerance typically works well for FP32, but may be too strict for lower precisions like FP16 or INT8.", + "Note that the default tolerance typically works well for FP32 but may be too strict for lower precisions like FP16 or INT8.", nargs="+", default=None, ) diff --git a/tools/Polygraphy/polygraphy/tools/args/data_loader.py b/tools/Polygraphy/polygraphy/tools/args/data_loader.py index 0b19e9a6..35c52f1a 100644 --- a/tools/Polygraphy/polygraphy/tools/args/data_loader.py +++ b/tools/Polygraphy/polygraphy/tools/args/data_loader.py @@ -124,7 +124,7 @@ def omit_none_tuple(tup): "Please use `--val-range` instead, which allows you to specify per-input data ranges." ) - self.val_range = args_util.parse_dict_with_default(args_util.get(args, "val_range"), cast_to=tuple) + self.val_range = args_util.parse_dict_with_default(args_util.get(args, "val_range"), cast_to=lambda x: tuple(args_util.cast(x))) if self.val_range is not None: for name, vals in self.val_range.items(): if len(vals) != 2: diff --git a/tools/Polygraphy/polygraphy/tools/args/onnx/loader.py b/tools/Polygraphy/polygraphy/tools/args/onnx/loader.py index 7ebfc70e..ba02573d 100644 --- a/tools/Polygraphy/polygraphy/tools/args/onnx/loader.py +++ b/tools/Polygraphy/polygraphy/tools/args/onnx/loader.py @@ -91,9 +91,9 @@ def add_to_parser(self, parser): "--external-data-size-threshold", help="The size threshold, in bytes, above which tensor data will be stored in the external file. " "Tensors smaller that this threshold will remain in the ONNX file. " - "Optionally, use a `K`, `M`, or `G` suffix to indicate KiB, MiB, or GiB respectively." - "For example, `--external-data-size-threshold=16M` is equivalent to `--external-data-size-threshold=16777216`" - "Has no effect if `--save-external-data` is not set", + "Optionally, use a `K`, `M`, or `G` suffix to indicate KiB, MiB, or GiB respectively. " + "For example, `--external-data-size-threshold=16M` is equivalent to `--external-data-size-threshold=16777216`. " + "Has no effect if `--save-external-data` is not set. ", default=None, ) self.group.add_argument( diff --git a/tools/Polygraphy/polygraphy/tools/args/onnxrt/__init__.py b/tools/Polygraphy/polygraphy/tools/args/onnxrt/__init__.py index 6ea77703..bd11ef91 100644 --- a/tools/Polygraphy/polygraphy/tools/args/onnxrt/__init__.py +++ b/tools/Polygraphy/polygraphy/tools/args/onnxrt/__init__.py @@ -14,3 +14,4 @@ # limitations under the License. # from polygraphy.tools.args.onnxrt.runner import * +from polygraphy.tools.args.onnxrt.loader import * diff --git a/tools/Polygraphy/polygraphy/tools/args/onnxrt/loader.py b/tools/Polygraphy/polygraphy/tools/args/onnxrt/loader.py new file mode 100644 index 00000000..f338a82d --- /dev/null +++ b/tools/Polygraphy/polygraphy/tools/args/onnxrt/loader.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from polygraphy import mod +from polygraphy.tools.args import util as args_util +from polygraphy.tools.args.base import BaseArgs +from polygraphy.tools.script import make_invocable + + +@mod.export() +class OnnxrtSessionArgs(BaseArgs): + def register(self, maker): + from polygraphy.tools.args.model import ModelArgs + from polygraphy.tools.args.onnx.loader import OnnxLoaderArgs + + if isinstance(maker, OnnxLoaderArgs): + self.onnx_loader_args = maker + if isinstance(maker, ModelArgs): + self.model_args = maker + + def add_to_parser(self, parser): + self.group = parser.add_argument_group( + "ONNX-Runtime Session Loader", "Options for the ONNX-Runtime Session Loader" + ) + self.group.add_argument( + "--providers", + "--execution-providers", + dest="providers", + help="A list of execution providers to use in order of priority. " + "Each provider may be either an exact match or a case-insensitive partial match " + "for the execution providers available in ONNX-Runtime. For example, a value of 'cpu' would " + "match the 'CPUExecutionProvider'", + nargs="+", + default=None, + ) + + def check_registered(self): + assert self.onnx_loader_args is not None, "OnnxLoaderArgs is required!" + assert self.model_args is not None, "ModelArgs is required!" + + def parse(self, args): + self.providers = args_util.get(args, "providers") + + def add_onnxrt_session(self, script): + if self.onnx_loader_args.should_use_onnx_loader(): + onnx_name = self.onnx_loader_args.add_serialized_onnx_loader(script) + else: + onnx_name = self.model_args.model_file + + script.add_import(imports=["SessionFromOnnx"], frm="polygraphy.backend.onnxrt") + loader_name = script.add_loader( + make_invocable("SessionFromOnnx", onnx_name, providers=self.providers), "build_onnxrt_session" + ) + return loader_name + + def load_onnxrt_session(self): + loader = args_util.run_script(self.add_onnxrt_session) + return loader() diff --git a/tools/Polygraphy/polygraphy/tools/args/onnxrt/runner.py b/tools/Polygraphy/polygraphy/tools/args/onnxrt/runner.py index 5477f2a5..128c77d3 100644 --- a/tools/Polygraphy/polygraphy/tools/args/onnxrt/runner.py +++ b/tools/Polygraphy/polygraphy/tools/args/onnxrt/runner.py @@ -21,26 +21,14 @@ @mod.export() class OnnxrtRunnerArgs(BaseArgs): def register(self, maker): - from polygraphy.tools.args.model import ModelArgs - from polygraphy.tools.args.onnx.loader import OnnxLoaderArgs + from polygraphy.tools.args.onnxrt.loader import OnnxrtSessionArgs - if isinstance(maker, OnnxLoaderArgs): - self.onnx_loader_args = maker - if isinstance(maker, ModelArgs): - self.model_args = maker + if isinstance(maker, OnnxrtSessionArgs): + self.session_args = maker def check_registered(self): - assert self.onnx_loader_args is not None, "OnnxLoaderArgs is required!" - assert self.model_args is not None, "ModelArgs is required!" + assert self.session_args is not None, "OnnxrtSessionArgs is required!" def add_to_script(self, script): script.add_import(imports=["OnnxrtRunner"], frm="polygraphy.backend.onnxrt") - if self.onnx_loader_args.should_use_onnx_loader(): - onnx_name = self.onnx_loader_args.add_serialized_onnx_loader(script) - else: - onnx_name = self.model_args.model_file - - script.add_import(imports=["SessionFromOnnx"], frm="polygraphy.backend.onnxrt") - loader_name = script.add_loader(make_invocable("SessionFromOnnx", onnx_name), "build_onnxrt_session") - - script.add_runner(make_invocable("OnnxrtRunner", loader_name)) + script.add_runner(make_invocable("OnnxrtRunner", self.session_args.add_onnxrt_session(script))) diff --git a/tools/Polygraphy/polygraphy/tools/args/trt/config.py b/tools/Polygraphy/polygraphy/tools/args/trt/config.py index b44e0409..cb4c2791 100644 --- a/tools/Polygraphy/polygraphy/tools/args/trt/config.py +++ b/tools/Polygraphy/polygraphy/tools/args/trt/config.py @@ -184,9 +184,9 @@ def add_to_parser(self, parser): trt_config_args.add_argument( "--workspace", metavar="BYTES", - help="Amount of memory, in bytes, to allocate for the TensorRT builder's workspace. " - "Optionally, use a `K`, `M`, or `G` suffix to indicate KiB, MiB, or GiB respectively." - "For example, `--workspace=16M` is equivalent to `--workspace=16777216`", + help="[DEPRECATED - use --pool-limit] Amount of memory, in bytes, to allocate for the TensorRT builder's workspace. " + "Optionally, use a `K`, `M`, or `G` suffix to indicate KiB, MiB, or GiB respectively. " + "For example, `--workspace=16M` is equivalent to `--workspace=16777216`. ", default=None, ) trt_config_args.add_argument( @@ -226,14 +226,6 @@ def add_to_parser(self, parser): ) replay = trt_config_args.add_mutually_exclusive_group() - replay.add_argument( - "--tactic-replay", - help="[DEPRECATED - use --load/save-tactics] Path to load/save a tactic replay file. " - "Used to record and replay tactics selected by TensorRT to provide deterministic engine builds. " - "If the provided path does not yet exist, tactics will be recorded and written to it. " - "If the provided path does exist, it will be read and used to replay previously recorded tactics. ", - default=None, - ) replay.add_argument( "--save-tactics", help="Path to save a Polygraphy tactic replay file. " @@ -251,7 +243,7 @@ def add_to_parser(self, parser): "--tactic-sources", help="Tactic sources to enable. This controls which libraries " "(e.g. cudnn, cublas, etc.) TensorRT is allowed to load tactics from. " - "Values come from the names of the values in the trt.TacticSource enum, and are case-insensitive. " + "Values come from the names of the values in the trt.TacticSource enum and are case-insensitive. " "If no arguments are provided, e.g. '--tactic-sources', then all tactic sources are disabled.", nargs="*", default=None, @@ -289,6 +281,17 @@ def add_to_parser(self, parser): action="store_true", default=None, ) + trt_config_args.add_argument( + "--pool-limit", + "--memory-pool-limit", + dest="memory_pool_limit", + help="Set memory pool limits. Memory pool names come from the names of values in the trt.MemoryPoolType enum and are case-insensitive" + "Format: `--pool-limit : ...`. For example, `--pool-limit dla_local_dram:1e9 workspace:16777216`. " + "Optionally, use a `K`, `M`, or `G` suffix to indicate KiB, MiB, or GiB respectively. " + "For example, `--pool-limit workspace:16M` is equivalent to `--pool-limit workspace:16777216`. ", + nargs="*", + default=None, + ) def register(self, maker): from polygraphy.tools.args.data_loader import DataLoaderArgs @@ -312,6 +315,10 @@ def parse(self, args): self.profile_dicts = parse_profile_shapes(default_shapes, trt_min_shapes, trt_opt_shapes, trt_max_shapes) self.workspace = args_util.parse_num_bytes(args_util.get(args, "workspace")) + if self.workspace is not None: + mod.warn_deprecated( + "--workspace", use_instead="--pool-limit workspace:", remove_in="0.45.0" + ) self.tf32 = args_util.get(args, "tf32") self.fp16 = args_util.get(args, "fp16") @@ -333,16 +340,8 @@ def parse(self, args): self.sparse_weights = args_util.get(args, "sparse_weights") self.timing_cache = args_util.get(args, "timing_cache") - tactic_replay = args_util.get(args, "tactic_replay") self.load_tactics = args_util.get(args, "load_tactics") self.save_tactics = args_util.get(args, "save_tactics") - if tactic_replay is not None: - mod.warn_deprecated("--tactic-replay", "--save-tactics or --load-tactics", remove_in="0.35.0") - G_LOGGER.warning("--tactic-replay is deprecated. Use either --save-tactics or --load-tactics instead.") - if os.path.exists(tactic_replay) and util.get_file_size(tactic_replay) > 0: - self.load_tactics = tactic_replay - else: - self.save_tactics = tactic_replay tactic_sources = args_util.get(args, "tactic_sources") self.tactic_sources = None @@ -359,6 +358,17 @@ def parse(self, args): self.use_dla = args_util.get(args, "use_dla") self.allow_gpu_fallback = args_util.get(args, "allow_gpu_fallback") + memory_pool_limits = args_util.parse_dict_with_default( + args_util.get(args, "memory_pool_limit"), cast_to=args_util.parse_num_bytes, allow_empty_key=False + ) + self.memory_pool_limits = None + if memory_pool_limits is not None: + self.memory_pool_limits = {} + for pool_type, pool_size in memory_pool_limits.items(): + pool_type = safe(assert_identifier(pool_type.upper())) + pool_type_str = safe("trt.MemoryPoolType.{:}", inline(pool_type)) + self.memory_pool_limits[inline(pool_type_str)] = pool_size + def add_trt_config_loader(self, script): profiles = [] for (min_shape, opt_shape, max_shape) in self.profile_dicts: @@ -418,7 +428,7 @@ def add_trt_config_loader(self, script): script.add_import(imports=["TacticRecorder"], frm="polygraphy.backend.trt") algo_selector = make_invocable("TacticRecorder", record=self.save_tactics) - if self.tactic_sources is not None: + if self.tactic_sources is not None or self.memory_pool_limits is not None: script.add_import(imports=["tensorrt as trt"]) if self.trt_config_script is not None: @@ -446,6 +456,7 @@ def add_trt_config_loader(self, script): tactic_sources=self.tactic_sources, use_dla=self.use_dla, allow_gpu_fallback=self.allow_gpu_fallback, + memory_pool_limits=self.memory_pool_limits, ) if config_loader_str is not None: script.add_import(imports=["CreateConfig as CreateTrtConfig"], frm="polygraphy.backend.trt") diff --git a/tools/Polygraphy/polygraphy/tools/args/util/util.py b/tools/Polygraphy/polygraphy/tools/args/util/util.py index 861921b0..c1845d3e 100644 --- a/tools/Polygraphy/polygraphy/tools/args/util/util.py +++ b/tools/Polygraphy/polygraphy/tools/args/util/util.py @@ -155,7 +155,7 @@ def np_type_from_str(dt_str): @mod.export() -def parse_dict_with_default(arg_lst, cast_to=None, sep=None): +def parse_dict_with_default(arg_lst, cast_to=None, sep=None, allow_empty_key=None): """ Generate a dictionary from a list of arguments of the form: ``:``. If ```` is empty, the value will be assigned @@ -165,15 +165,19 @@ def parse_dict_with_default(arg_lst, cast_to=None, sep=None): arg_lst (List[str]): The arguments to map. - cast_to (type): - The type to cast the values in the map. By default, - uses the type returned by ``cast``. + cast_to (Callable): + A callable to cast types before adding them to the map. + Defaults to `cast()`. sep (str): The separator between the key and value strings. + allow_empty_key (bool): + Whether empty keys should be allowed. Returns: Dict[str, obj]: The mapping. """ sep = util.default(sep, ":") + cast_to = util.default(cast_to, cast) + allow_empty_key = util.default(allow_empty_key, True) if arg_lst is None: return @@ -181,15 +185,16 @@ def parse_dict_with_default(arg_lst, cast_to=None, sep=None): arg_map = {} for arg in arg_lst: key, _, val = arg.rpartition(sep) - val = cast(val) - if cast_to: - val = cast_to(val) - arg_map[key] = val + if not key and not allow_empty_key: + G_LOGGER.critical( + "Could not parse argument: {:}. Expected an argument in the format: `key{:}value`.\n".format(arg, sep) + ) + arg_map[key] = cast_to(val) return arg_map @mod.deprecate( - remove_in="0.35.0", + remove_in="0.42.0", use_instead=": as a separator and write shapes in the form [dim0,...,dimN]", name="Using , as a separator", ) diff --git a/tools/Polygraphy/polygraphy/tools/data/subtool/to_input.py b/tools/Polygraphy/polygraphy/tools/data/subtool/to_input.py index ff21bd0f..1f1fb203 100644 --- a/tools/Polygraphy/polygraphy/tools/data/subtool/to_input.py +++ b/tools/Polygraphy/polygraphy/tools/data/subtool/to_input.py @@ -17,7 +17,7 @@ from polygraphy import util from polygraphy.comparator import RunResults -from polygraphy.json import load_json +from polygraphy.json import load_json, save_json from polygraphy.logger import G_LOGGER from polygraphy.tools.base import Tool @@ -68,4 +68,4 @@ def update_inputs(new_inputs, path): data = [data] update_inputs(data, path) - util.save_json(inputs, args.output, description="input file containing {:} iteration(s)".format(len(inputs))) + save_json(inputs, args.output, description="input file containing {:} iteration(s)".format(len(inputs))) diff --git a/tools/Polygraphy/polygraphy/tools/debug/subtool/artifact_sorter.py b/tools/Polygraphy/polygraphy/tools/debug/subtool/artifact_sorter.py index a86bd3a5..3b9889f1 100644 --- a/tools/Polygraphy/polygraphy/tools/debug/subtool/artifact_sorter.py +++ b/tools/Polygraphy/polygraphy/tools/debug/subtool/artifact_sorter.py @@ -22,6 +22,7 @@ import time from polygraphy import util +from polygraphy.json import save_json from polygraphy.logger import G_LOGGER from polygraphy.tools.args import util as args_util from polygraphy.tools.args.base import BaseArgs @@ -268,7 +269,7 @@ def is_success(status): stack.callback(try_remove(self.iter_artifact)) if self.iteration_info: - util.save_json({"iteration": iteration}, self.iteration_info) + save_json({"iteration": iteration}, self.iteration_info) stack.callback(try_remove(self.iteration_info)) G_LOGGER.info("Running check command: {:}".format(" ".join(self.check))) diff --git a/tools/Polygraphy/polygraphy/tools/debug/subtool/reduce.py b/tools/Polygraphy/polygraphy/tools/debug/subtool/reduce.py index 4718e955..05f0eaeb 100644 --- a/tools/Polygraphy/polygraphy/tools/debug/subtool/reduce.py +++ b/tools/Polygraphy/polygraphy/tools/debug/subtool/reduce.py @@ -413,7 +413,7 @@ def get_io(index): "You may want to reduce {:} again using --mode=linear. ".format(self.arg_groups[OnnxSaveArgs].path) ) - G_LOGGER.info("Minimum Bad Model:\n{:}\n\n".format(onnx_util.str_from_onnx(reduced_model, mode="none"))) + G_LOGGER.info("Minimum Bad Model:\n{:}\n\n".format(onnx_util.str_from_onnx(reduced_model))) self.arg_groups[OnnxSaveArgs].save_onnx(reduced_model) # == Write Good Model == @@ -425,7 +425,5 @@ def get_io(index): "Could not find a minimal model close in size to the reduced model that does not cause a failure." ) else: - G_LOGGER.info( - "Minimum Good Model:\n{:}\n\n".format(onnx_util.str_from_onnx(min_good_model, mode="none")) - ) + G_LOGGER.info("Minimum Good Model:\n{:}\n\n".format(onnx_util.str_from_onnx(min_good_model))) self.arg_groups[OnnxSaveArgs].save_onnx(min_good_model, args.min_good) diff --git a/tools/Polygraphy/polygraphy/tools/inspect/subtool/model.py b/tools/Polygraphy/polygraphy/tools/inspect/subtool/model.py index f3a81178..d0fce26b 100644 --- a/tools/Polygraphy/polygraphy/tools/inspect/subtool/model.py +++ b/tools/Polygraphy/polygraphy/tools/inspect/subtool/model.py @@ -56,10 +56,23 @@ def add_parser_args(self, parser): choices=["trt"], dest="display_as", ) - parser.add_argument( + + dispopts = parser.add_mutually_exclusive_group() + dispopts.add_argument( + "--show", + help="Controls what is displayed: {{" + "'layers': Display basic layer information like name, op, inputs, and outputs, " + "'attrs': Display all available per-layer attributes; has no effect if 'layers' is not enabled, " + "'weights': Display all weights in the model; if 'layers' is enabled, also shows per-layer constants" + "}}. More than one option may be specified", + choices=["layers", "attrs", "weights"], + nargs="*", + default=[], + ) + dispopts.add_argument( "--mode", "--layer-info", - help="Display layers: {{" + help="[DEPRECATED - use --show instead] Display layers: {{" "'none': Display no layer information, " "'basic': Display layer inputs and outputs, " "'attrs': Display layer inputs, outputs and attributes, " @@ -67,50 +80,61 @@ def add_parser_args(self, parser): "}}", choices=["none", "basic", "attrs", "full"], dest="mode", - default="none", + default=None, ) def run(self, args): - func = None + if args.mode: + mod.warn_deprecated("--mode", "--show", remove_in="0.40.0", always_show_warning=True) + args.show = { + "none": [], + "basic": ["layers"], + "attrs": ["layers", "attrs"], + "full": ["layers", "attrs", "weights"], + }[args.mode] + args.mode = None + + def show(aspect): + return aspect in args.show + + def inspect_trt(): + if self.arg_groups[ModelArgs].model_type == "engine": + with self.arg_groups[TrtEngineLoaderArgs].load_serialized_engine() as engine: + engine_str = trt_util.str_from_engine(engine, show_layers=show("layers"), show_attrs=show("attrs")) + G_LOGGER.info("==== TensorRT Engine ====\n{:}".format(engine_str)) + else: + builder, network, parser = util.unpack_args(self.arg_groups[TrtNetworkLoaderArgs].load_network(), 3) + with contextlib.ExitStack() as stack: + stack.enter_context(builder) + stack.enter_context(network) + if parser: + stack.enter_context(parser) + network_str = trt_util.str_from_network( + network, show_layers=show("layers"), show_attrs=show("attrs"), show_weights=show("weights") + ).strip() + G_LOGGER.info("==== TensorRT Network ====\n{:}".format(network_str)) + + def inspect_onnx(): + onnx_model = self.arg_groups[OnnxLoaderArgs].load_onnx() + model_str = onnx_util.str_from_onnx( + onnx_model, show_layers=show("layers"), show_attrs=show("attrs"), show_weights=show("weights") + ).strip() + G_LOGGER.info("==== ONNX Model ====\n{:}".format(model_str)) + + def inspect_tf(): + tf_graph, _ = self.arg_groups[TfLoaderArgs].load_graph() + graph_str = tf_util.str_from_graph( + tf_graph, show_layers=show("layers"), show_attrs=show("attrs"), show_weights=show("weights") + ).strip() + G_LOGGER.info("==== TensorFlow Graph ====\n{:}".format(graph_str)) + func = None if self.arg_groups[ModelArgs].model_type.is_tf(): - func = self.inspect_tf - + func = inspect_tf if self.arg_groups[ModelArgs].model_type.is_onnx(): - func = self.inspect_onnx - + func = inspect_onnx if self.arg_groups[ModelArgs].model_type.is_trt() or args.display_as == "trt": - func = self.inspect_trt - + func = inspect_trt if func is None: G_LOGGER.critical("Could not determine how to display this model. Maybe you need to specify --display-as?") - - func(args) - - def inspect_trt(self, args): - if self.arg_groups[ModelArgs].model_type == "engine": - if args.mode != "none": - G_LOGGER.warning("Displaying layer information for TensorRT engines is not currently supported") - - with self.arg_groups[TrtEngineLoaderArgs].load_serialized_engine() as engine: - engine_str = trt_util.str_from_engine(engine) - G_LOGGER.info("==== TensorRT Engine ====\n{:}".format(engine_str)) - else: - builder, network, parser = util.unpack_args(self.arg_groups[TrtNetworkLoaderArgs].load_network(), 3) - with contextlib.ExitStack() as stack: - stack.enter_context(builder) - stack.enter_context(network) - if parser: - stack.enter_context(parser) - network_str = trt_util.str_from_network(network, mode=args.mode).strip() - G_LOGGER.info("==== TensorRT Network ====\n{:}".format(network_str)) - - def inspect_onnx(self, args): - onnx_model = self.arg_groups[OnnxLoaderArgs].load_onnx() - model_str = onnx_util.str_from_onnx(onnx_model, mode=args.mode).strip() - G_LOGGER.info("==== ONNX Model ====\n{:}".format(model_str)) - - def inspect_tf(self, args): - tf_graph, _ = self.arg_groups[TfLoaderArgs].load_graph() - graph_str = tf_util.str_from_graph(tf_graph, mode=args.mode).strip() - G_LOGGER.info("==== TensorFlow Graph ====\n{:}".format(graph_str)) + func() diff --git a/tools/Polygraphy/polygraphy/tools/run/run.py b/tools/Polygraphy/polygraphy/tools/run/run.py index 1023a393..092bf860 100644 --- a/tools/Polygraphy/polygraphy/tools/run/run.py +++ b/tools/Polygraphy/polygraphy/tools/run/run.py @@ -25,6 +25,7 @@ ModelArgs, OnnxLoaderArgs, OnnxrtRunnerArgs, + OnnxrtSessionArgs, OnnxSaveArgs, OnnxShapeInferenceArgs, PluginRefArgs, @@ -136,6 +137,7 @@ def __init__(self): self.subscribe_args(OnnxSaveArgs(output="save-onnx", short_opt=None)) self.subscribe_args(OnnxShapeInferenceArgs()) self.subscribe_args(OnnxLoaderArgs(save=True)) + self.subscribe_args(OnnxrtSessionArgs()) self.subscribe_args(OnnxrtRunnerArgs()) self.subscribe_args(PluginRefArgs()) self.subscribe_args( diff --git a/tools/Polygraphy/polygraphy/tools/script.py b/tools/Polygraphy/polygraphy/tools/script.py index 2e1e6475..05b502d4 100644 --- a/tools/Polygraphy/polygraphy/tools/script.py +++ b/tools/Polygraphy/polygraphy/tools/script.py @@ -39,9 +39,10 @@ def assert_identifier(inp): def safe(base_str, *args, **kwargs): """ - Marks a string as being safe. + Indicates a string is safe to use and will not compromise security. NOTE: The caller is reponsible for checking that the string is actually safe. + This function serves only to mark the string as such. Can work with format strings as well. For example: :: diff --git a/tools/Polygraphy/polygraphy/tools/surgeon/subtool/base.py b/tools/Polygraphy/polygraphy/tools/surgeon/subtool/base.py index ff32a173..0bedb8a6 100644 --- a/tools/Polygraphy/polygraphy/tools/surgeon/subtool/base.py +++ b/tools/Polygraphy/polygraphy/tools/surgeon/subtool/base.py @@ -30,7 +30,7 @@ def __init__(self, name): def load_model(self, log_model=True): model = self.arg_groups[OnnxLoaderArgs].load_onnx() if log_model: - G_LOGGER.info("Original Model:\n{:}\n\n".format(onnx_util.str_from_onnx(model, mode="none"))) + G_LOGGER.info("Original Model:\n{:}\n\n".format(onnx_util.str_from_onnx(model))) return model # Since new graph outputs may be added, and we don't know the types, @@ -41,7 +41,7 @@ def export_graph(self, graph, do_type_check=False): def save_model(self, model, log_model=True): model = self.arg_groups[OnnxSaveArgs].save_onnx(model) if log_model: - G_LOGGER.info("New Model:\n{:}\n\n".format(onnx_util.str_from_onnx(model, mode="none"))) + G_LOGGER.info("New Model:\n{:}\n\n".format(onnx_util.str_from_onnx(model))) def run_impl(self, args): raise NotImplementedError("Subclasses must implement run_impl!") diff --git a/tools/Polygraphy/polygraphy/util/__init__.py b/tools/Polygraphy/polygraphy/util/__init__.py index 1b3ae738..f7f3b069 100644 --- a/tools/Polygraphy/polygraphy/util/__init__.py +++ b/tools/Polygraphy/polygraphy/util/__init__.py @@ -1,2 +1 @@ from polygraphy.util.util import * -from polygraphy.util.serde import * diff --git a/tools/Polygraphy/polygraphy/util/serde.py b/tools/Polygraphy/polygraphy/util/serde.py deleted file mode 100644 index 7b0a481e..00000000 --- a/tools/Polygraphy/polygraphy/util/serde.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from polygraphy import mod -from polygraphy.json import * - -mod.warn_deprecated("JSON utilities in polygraphy.util", "polygraphy.json", remove_in="0.35.0") diff --git a/tools/Polygraphy/polygraphy/util/util.py b/tools/Polygraphy/polygraphy/util/util.py index fe9c8952..d7f0b8ff 100644 --- a/tools/Polygraphy/polygraphy/util/util.py +++ b/tools/Polygraphy/polygraphy/util/util.py @@ -44,27 +44,28 @@ def check(cond, msg=None): @mod.export() -def find_in_dict(name, mapping, index=None): +def find_str_in_iterable(name, seq, index=None): """ - Attempts to partially match keys in a dictionary. Checks for exact matches and - substring matches, falling back to index based matching. + Attempts to find matching strings in a sequence. Checks for exact matches, then + case-insensitive substring matches, finally falling back to index based matching. Args: name (str): The key to search for. - mapping (dict): The dictionary to search in. - index (int): An index to fall back to if the key could not be found by name. + seq (Sequence[str]): The dictionary to search in. + index (int): An index to fall back to if the string could not be found. Returns: - str: The key found in the dict, or None if it could not be found. + str: The element found in the sequence, or None if it could not be found. """ - G_LOGGER.ultra_verbose("Searching for key: {:}. Fallback index is set to {:}".format(name, index)) - if name in mapping: + if name in seq: return name - for key in mapping.keys(): - if name.lower() in key.lower() or key.lower() in name.lower(): - return key - if index is not None and index >= 0 and index < len(mapping.keys()): - return list(mapping.keys())[index] + + for elem in seq: + if name.lower() in elem.lower() or elem.lower() in name.lower(): + return elem + + if index is not None and index < len(seq): + return list(seq)[index] return None diff --git a/tools/Polygraphy/tests/backend/onnx/test_loader.py b/tools/Polygraphy/tests/backend/onnx/test_loader.py index 0ec15070..07fa101a 100644 --- a/tools/Polygraphy/tests/backend/onnx/test_loader.py +++ b/tools/Polygraphy/tests/backend/onnx/test_loader.py @@ -96,11 +96,12 @@ def test_layerwise(self, copy): assert len(original_model.graph.output) == 1 or not copy assert len(model.graph.output) == 2 - def test_custom_outputs(self): - loader = ModifyOutputs(OnnxFromPath(ONNX_MODELS["identity_identity"].path), outputs=["identity_out_0"]) + @pytest.mark.parametrize("output", ["identity_out_0", "identity_out_2"]) + def test_custom_outputs(self, output): + loader = ModifyOutputs(OnnxFromPath(ONNX_MODELS["identity_identity"].path), outputs=[output]) model = loader() assert len(model.graph.output) == 1 - assert model.graph.output[0].name == "identity_out_0" + assert model.graph.output[0].name == output def test_exclude_outputs_with_layerwise(self): loader = ModifyOutputs( @@ -154,7 +155,7 @@ def test_basic(self, copy): model = loader() assert original_model.graph.input[0].type.tensor_type.elem_type == 1 or not copy - assert model.graph.value_info[0].type.tensor_type.elem_type == 10 + assert model.graph.input[0].type.tensor_type.elem_type == 10 class TestFoldConstants: diff --git a/tools/Polygraphy/tests/backend/onnxrt/test_loader.py b/tools/Polygraphy/tests/backend/onnxrt/test_loader.py new file mode 100644 index 00000000..c695870a --- /dev/null +++ b/tools/Polygraphy/tests/backend/onnxrt/test_loader.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from polygraphy.backend.onnxrt import SessionFromOnnx +from polygraphy.exception import PolygraphyException +from tests.models.meta import ONNX_MODELS + +import onnxruntime as onnxrt +import pytest + + +class TestSessionFromOnnx(object): + def test_defaults(self): + model = ONNX_MODELS["identity"] + loader = SessionFromOnnx(model.loader) + sess = loader() + + assert sess + assert isinstance(sess, onnxrt.InferenceSession) + assert sess.get_providers() == ["CPUExecutionProvider"] + + @pytest.mark.parametrize( + "providers,expected", + [ + (["cpu"], ["CPUExecutionProvider"]), + (["CPU"], ["CPUExecutionProvider"]), + ], + ) + def test_provider_matching(self, providers, expected): + model = ONNX_MODELS["identity"] + loader = SessionFromOnnx(model.loader, providers=providers) + sess = loader() + + assert sess + assert isinstance(sess, onnxrt.InferenceSession) + assert sess.get_providers() == expected + + def test_invalid_providers_raise_errors(self): + model = ONNX_MODELS["identity"] + loader = SessionFromOnnx(model.loader, providers=["cpu", "not_a_real_provider"]) + with pytest.raises(PolygraphyException, match="Could not find specified ONNX-Runtime execution provider"): + loader() diff --git a/tools/Polygraphy/tests/backend/pluginref/test_runner.py b/tools/Polygraphy/tests/backend/pluginref/test_runner.py index 2e69bcdf..d19099a4 100644 --- a/tools/Polygraphy/tests/backend/pluginref/test_runner.py +++ b/tools/Polygraphy/tests/backend/pluginref/test_runner.py @@ -51,7 +51,7 @@ def test_fail_on_unsupported_node(self): model = ONNX_MODELS["and"] with PluginRefRunner(GsFromOnnx(OnnxFromPath(model.path))) as runner: with pytest.raises(PolygraphyException, match="does not have a reference implementation registered!"): - runner.infer({"x": np.ones(shape=(3, 4), dtype=np.bool), "y": np.ones(shape=(3, 4), dtype=np.bool)}) + runner.infer({"x": np.ones(shape=(3, 4), dtype=bool), "y": np.ones(shape=(3, 4), dtype=bool)}) @pytest.mark.parametrize( "names, err", diff --git a/tools/Polygraphy/tests/backend/trt/test_loader.py b/tools/Polygraphy/tests/backend/trt/test_loader.py index b8ee0064..dd738f4e 100644 --- a/tools/Polygraphy/tests/backend/trt/test_loader.py +++ b/tools/Polygraphy/tests/backend/trt/test_loader.py @@ -38,7 +38,7 @@ onnx_like_from_network, ) from polygraphy.comparator import DataLoader -from tests.helper import get_file_size, is_file_non_empty +from tests.helper import get_file_size, has_dla, is_file_non_empty from tests.models.meta import ONNX_MODELS ## @@ -201,7 +201,7 @@ def test_unmark_shape_outputs(self, modifiable_reshape_network): assert network.num_outputs == 1 -class TestConfigLoader(object): +class TestCreateConfig(object): def test_defaults(self, identity_builder_network): builder, network = identity_builder_network loader = CreateConfig() @@ -221,7 +221,7 @@ def test_defaults(self, identity_builder_network): if mod.version(trt.__version__) < mod.version("8.0"): assert config.get_tactic_sources() == 3 else: - assert config.get_tactic_sources() == 7 + assert config.get_tactic_sources() == 7 with contextlib.suppress(AttributeError): assert not config.get_flag(trt.BuilderFlag.OBEY_PRECISION_CONSTRAINTS) @@ -370,6 +370,42 @@ def test_empty_timing_cache_when_default(self, identity_builder_network): new_cache_size = len(bytes(buffer)) assert cache_size == new_cache_size + @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.0"), reason="Unsupported for TRT 7.2 and older") + def test_profiling_verbosity(self, identity_builder_network): + builder, network = identity_builder_network + expected = trt.ProfilingVerbosity.NONE + loader = CreateConfig(profiling_verbosity=expected) + with loader(builder, network) as config: + assert config.profiling_verbosity == expected + + with contextlib.suppress(AttributeError): + POOL_LIMITS = [ + {trt.MemoryPoolType.WORKSPACE: 25}, + {trt.MemoryPoolType.DLA_MANAGED_SRAM: 25}, + {trt.MemoryPoolType.DLA_LOCAL_DRAM: 25}, + {trt.MemoryPoolType.DLA_GLOBAL_DRAM: 25}, + # Multiple limits + { + trt.MemoryPoolType.DLA_LOCAL_DRAM: 20, + trt.MemoryPoolType.DLA_GLOBAL_DRAM: 25, + trt.MemoryPoolType.WORKSPACE: 39, + }, + ] + + @pytest.mark.skipif( + mod.version(trt.__version__) < mod.version("8.3"), reason="Unsupported for TRT versions prior to 8.3" + ) + @pytest.mark.parametrize("pool_limits", POOL_LIMITS) + def test_memory_pool_limits(self, pool_limits, identity_builder_network): + if any("dla" in key.name.lower() for key in pool_limits) and not has_dla(): + pytest.skip("DLA is not available on this system") + + builder, network = identity_builder_network + loader = CreateConfig(memory_pool_limits=pool_limits) + with loader(builder, network) as config: + for pool_type, pool_size in pool_limits.items(): + assert config.get_memory_pool_limit(pool_type) == pool_size + class TestEngineBytesFromNetwork(object): def test_can_build(self, identity_network): diff --git a/tools/Polygraphy/tests/comparator/test_compare.py b/tools/Polygraphy/tests/comparator/test_compare.py index 3555e3aa..8ebe7632 100644 --- a/tools/Polygraphy/tests/comparator/test_compare.py +++ b/tools/Polygraphy/tests/comparator/test_compare.py @@ -23,8 +23,8 @@ class TestBasicCompareFunc(object): def test_can_compare_bool(self): - iter_result0 = IterationResult(outputs={"output": np.zeros((4, 4), dtype=np.bool)}) - iter_result1 = IterationResult(outputs={"output": np.ones((4, 4), dtype=np.bool)}) + iter_result0 = IterationResult(outputs={"output": np.zeros((4, 4), dtype=bool)}) + iter_result1 = IterationResult(outputs={"output": np.ones((4, 4), dtype=bool)}) compare_func = CompareFunc.simple() acc = compare_func(iter_result0, iter_result1) diff --git a/tools/Polygraphy/tests/comparator/test_data_loader.py b/tools/Polygraphy/tests/comparator/test_data_loader.py index 3c756d19..7870132a 100644 --- a/tools/Polygraphy/tests/comparator/test_data_loader.py +++ b/tools/Polygraphy/tests/comparator/test_data_loader.py @@ -28,7 +28,7 @@ def meta(dtype): class TestDataLoader(object): - @pytest.mark.parametrize("dtype", [np.int32, np.bool, np.float32, np.int64]) + @pytest.mark.parametrize("dtype", [np.int32, bool, np.float32, np.int64]) def test_default_ranges(self, dtype): data_loader = DataLoader(input_metadata=meta(dtype)) x, y = data_loader[0].values() @@ -47,7 +47,7 @@ def test_can_override_shape(self): feed_dict = data_loader[0] assert tuple(feed_dict["X"].shape) == shape - @pytest.mark.parametrize("dtype", [np.int32, np.bool, np.float32, np.int64]) + @pytest.mark.parametrize("dtype", [np.int32, bool, np.float32, np.int64]) @pytest.mark.parametrize("range_val", [0, 1]) def test_range_min_max_equal(self, dtype, range_val): data_loader = DataLoader(input_metadata=meta(dtype), val_range=(range_val, range_val)) @@ -60,7 +60,7 @@ def test_range_min_max_equal(self, dtype, range_val): [ (0, 1, np.int32), (5.0, 5.5, np.float32), - (0, 1, np.bool), + (0, 1, bool), ], ) def test_val_ranges(self, range): diff --git a/tools/Polygraphy/tests/comparator/test_struct.py b/tools/Polygraphy/tests/comparator/test_struct.py index 1ad03897..3824e219 100644 --- a/tools/Polygraphy/tests/comparator/test_struct.py +++ b/tools/Polygraphy/tests/comparator/test_struct.py @@ -7,11 +7,15 @@ from polygraphy.exception import PolygraphyException +def make_outputs(): + return {"dummy_out": np.zeros((4, 4))} + + def make_iter_results(runner_name): - return [IterationResult(outputs={"dummy_out": np.zeros((4, 4))}, runner_name=runner_name)] * 2 + return [IterationResult(outputs=make_outputs(), runner_name=runner_name)] * 2 -@pytest.fixture(scope="session") +@pytest.fixture() def run_results(): results = RunResults() results.append(("runner0", make_iter_results("runner0"))) @@ -53,7 +57,7 @@ def check_results(results, is_none=False): for iter_res in results["runner1"]: if is_none: assert not iter_res - assert iter_res.runner_name == "" + assert iter_res.runner_name == "custom_runner" else: assert iter_res assert iter_res.runner_name @@ -77,6 +81,23 @@ def test_contains(self, run_results): assert "runner1" in run_results assert "runner3" not in run_results + def test_add_new(self): + results = RunResults() + results.add([make_outputs()], runner_name="custom") + + iter_results = results["custom"] + assert len(iter_results) == 1 + assert all(isinstance(iter_result, IterationResult) for iter_result in iter_results) + + def test_add_new_default_name(self): + results = RunResults() + results.add([make_outputs()]) + + name = results[0][0] + iter_results = results[name] + assert len(iter_results) == 1 + assert all(isinstance(iter_result, IterationResult) for iter_result in iter_results) + class TestLazyNumpyArray(object): @pytest.mark.parametrize("set_threshold", [True, False]) diff --git a/tools/Polygraphy/tests/helper.py b/tools/Polygraphy/tests/helper.py index a7fa900c..62e712c6 100644 --- a/tools/Polygraphy/tests/helper.py +++ b/tools/Polygraphy/tests/helper.py @@ -16,6 +16,9 @@ import os import time +import tensorrt as trt +from polygraphy.backend.trt import get_trt_logger + ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.pardir)) @@ -54,3 +57,15 @@ def time_func(func, warm_up=50, iters=100): end = time.time() total += end - start return total / float(iters) + + +HAS_DLA = None + + +def has_dla(): + global HAS_DLA + if HAS_DLA is None: + builder = trt.Builder(get_trt_logger()) + HAS_DLA = builder.num_DLA_cores > 0 + + return HAS_DLA diff --git a/tools/Polygraphy/tests/pytest.ini b/tools/Polygraphy/tests/pytest.ini new file mode 100644 index 00000000..b849e10e --- /dev/null +++ b/tools/Polygraphy/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + serial: Indicates tests do not support parallel execution diff --git a/tools/Polygraphy/tests/requirements.txt b/tools/Polygraphy/tests/requirements.txt index 1530ed63..4608f86f 100644 --- a/tools/Polygraphy/tests/requirements.txt +++ b/tools/Polygraphy/tests/requirements.txt @@ -1,9 +1,9 @@ colored numpy onnx_graphsurgeon>=0.3.4 -onnx==1.9.0 +onnx==1.10.0 onnxmltools -onnxruntime==1.8.1 +onnxruntime==1.10.0 pytest pytest-virtualenv pytest-xdist diff --git a/tools/Polygraphy/tests/test_dependencies.py b/tools/Polygraphy/tests/test_dependencies.py index e2e0bea9..9bfdbbcb 100644 --- a/tools/Polygraphy/tests/test_dependencies.py +++ b/tools/Polygraphy/tests/test_dependencies.py @@ -32,8 +32,9 @@ @pytest.fixture() -def virtualenv_with_poly(virtualenv): +def polygraphy_venv(virtualenv): virtualenv.env["PYTHONPATH"] = ROOT_DIR + virtualenv.env["LD_LIBRARY_PATH"] = "" yield virtualenv @@ -52,31 +53,35 @@ def is_submodule(path): class TestPublicImports(object): - def test_no_extra_submodule_dependencies_required(self, virtualenv_with_poly): + def test_no_extra_submodule_dependencies_required(self, polygraphy_venv): # Submodules should not require any extra dependencies to import. for submodule_path in SUBMODULE_PATHS: submodule_name = ".".join(submodule_path.split(os.path.sep)) - cmd = ["python3", "-c", "from {:} import *".format(submodule_name)] + cmd = [polygraphy_venv.python, "-c", "from {:} import *".format(submodule_name)] print(" ".join(cmd)) - output = virtualenv_with_poly.run(cmd, capture=True) + output = polygraphy_venv.run(cmd, capture=True) print(output) - def test_can_json_without_numpy(self, virtualenv_with_poly): - cmd = ["python3", "-c", "from polygraphy.json import to_json, from_json; x = to_json(1); x = from_json(x)"] + def test_can_json_without_numpy(self, polygraphy_venv): + cmd = [ + polygraphy_venv.python, + "-c", + "from polygraphy.json import to_json, from_json; x = to_json(1); x = from_json(x)", + ] print(" ".join(cmd)) - output = virtualenv_with_poly.run(cmd, capture=True) + output = polygraphy_venv.run(cmd, capture=True) print(output) class TestToolImports(object): # We should be able to at least launch tools with no dependencies installed. @pytest.mark.parametrize("tool, subtools", ALL_TOOLS.items()) - def test_can_run_tool_without_deps(self, virtualenv_with_poly, tool, subtools): + def test_can_run_tool_without_deps(self, polygraphy_venv, tool, subtools): POLYGRAPHY_BIN = os.path.join(ROOT_DIR, "bin", "polygraphy") - BASE_TOOL_CMD = ["python3", POLYGRAPHY_BIN, tool, "-h"] + BASE_TOOL_CMD = [polygraphy_venv.python, POLYGRAPHY_BIN, tool, "-h"] def check_tool(tool): - output = virtualenv_with_poly.run(tool, capture=True) + output = polygraphy_venv.run(tool, capture=True) assert "This tool could not be loaded due to an error:" not in output assert "error:" not in output assert "could not be loaded" not in output @@ -103,18 +108,15 @@ class TestAutoinstallDeps(object): ], ], ) - def test_can_automatically_install_deps(self, virtualenv_with_poly, cmd): + def test_can_automatically_install_deps(self, polygraphy_venv, cmd): if "--trt" in cmd and mod.version(trt.__version__) < mod.version("7.0"): pytest.skip("TRT 6 container has an old version of CUDA") - if "--trt" in cmd: - pytest.xfail("TensorRT 8.0.1.6 wheels are currently broken") - - virtualenv_with_poly.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" + polygraphy_venv.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" POLYGRAPHY_BIN = os.path.join(ROOT_DIR, "bin", "polygraphy") - cmd = ["python3", POLYGRAPHY_BIN] + cmd + cmd = [polygraphy_venv.python, POLYGRAPHY_BIN] + cmd print("Running: {:}".format(" ".join(cmd))) - output = virtualenv_with_poly.run(cmd, capture=True) + output = polygraphy_venv.run(cmd, capture=True) print(output) assert "is required, but not installed. Attempting to install now" in output @@ -125,19 +127,19 @@ def test_can_automatically_install_deps(self, virtualenv_with_poly, cmd): (mod.LATEST_VERSION, ">=1.4.2"), ], ) - def test_can_automatically_upgrade_deps(self, virtualenv_with_poly, new_ver, expected): - virtualenv_with_poly.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" + def test_can_automatically_upgrade_deps(self, polygraphy_venv, new_ver, expected): + polygraphy_venv.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" def get_colored_version(): - return virtualenv_with_poly.installed_packages()["colored"].version + return polygraphy_venv.installed_packages()["colored"].version - virtualenv_with_poly.run(["python3", "-m", "pip", "install", "colored==1.4.0"]) + polygraphy_venv.run([polygraphy_venv.python, "-m", "pip", "install", "colored==1.4.0"]) assert get_colored_version() == "1.4.0" # Insert our own preferred version to make sure it upgrades. - virtualenv_with_poly.run( + polygraphy_venv.run( [ - "python3", + polygraphy_venv.python, "-c", "from polygraphy import mod; " "colored = mod.lazy_import('colored', version='{:}'); " @@ -146,27 +148,27 @@ def get_colored_version(): ) assert _version_ok(get_colored_version(), expected) - def test_autoinstall(self, virtualenv_with_poly): - virtualenv_with_poly.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" - assert "colored" not in virtualenv_with_poly.installed_packages() + def test_autoinstall(self, polygraphy_venv): + polygraphy_venv.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" + assert "colored" not in polygraphy_venv.installed_packages() - virtualenv_with_poly.run( + polygraphy_venv.run( [ - "python3", + polygraphy_venv.python, "-c", "from polygraphy import mod; " "colored = mod.lazy_import('colored'); " "mod.autoinstall(colored)", ] ) - assert "colored" in virtualenv_with_poly.installed_packages() + assert "colored" in polygraphy_venv.installed_packages() # We can import inner modules, and Polygraphy should still autoinstall the outermost one. - def test_can_install_for_nested_import(self, virtualenv_with_poly): - virtualenv_with_poly.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" + def test_can_install_for_nested_import(self, polygraphy_venv): + polygraphy_venv.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" - virtualenv_with_poly.run( + polygraphy_venv.run( [ - "python3", + polygraphy_venv.python, "-c", "from polygraphy import mod; " "shape_inference = mod.lazy_import('onnx.shape_inference'); " @@ -174,7 +176,7 @@ def test_can_install_for_nested_import(self, virtualenv_with_poly): ] ) - assert "onnx" in virtualenv_with_poly.installed_packages() + assert "onnx" in polygraphy_venv.installed_packages() def test_all_lazy_imports(self): # NOTE: If this test fails, it means a new lazy dependency has been diff --git a/tools/Polygraphy/tests/test_deprecated_aliases.py b/tools/Polygraphy/tests/test_deprecated_aliases.py index 2ca1b34f..baebaafb 100644 --- a/tools/Polygraphy/tests/test_deprecated_aliases.py +++ b/tools/Polygraphy/tests/test_deprecated_aliases.py @@ -13,46 +13,39 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import pytest +from tests.models.meta import ONNX_MODELS, TF_MODELS +from tests.tools.common import run_polygraphy_inspect -class TestCuda(object): - def test_cuda(self): - from polygraphy.common import cuda - assert cuda.DeviceArray +@pytest.fixture(scope="session", params=["none", "basic", "attrs", "full"]) +def run_inspect_model_deprecated(request): + yield lambda additional_opts: run_polygraphy_inspect( + ["model"] + ["--mode={:}".format(request.param)] + additional_opts + ) -class TestFunc(object): - def test_func(self): - from polygraphy.common import func +class TestInspectModelDeprecated(object): + def test_model_onnx(self, run_inspect_model_deprecated): + run_inspect_model_deprecated([ONNX_MODELS["identity"].path]) - assert hasattr(func, "extend") + def test_model_trt_sanity(self, run_inspect_model_deprecated): + run_inspect_model_deprecated([ONNX_MODELS["identity"].path, "--display-as=trt"]) + def test_model_tf_sanity(self, run_inspect_model_deprecated): + pytest.importorskip("tensorflow") -class TestException(object): - def test_exception(self): - from polygraphy.common import exception - - assert hasattr(exception, "PolygraphyException") + run_inspect_model_deprecated([TF_MODELS["identity"].path, "--model-type=frozen"]) class TestConstants(object): - def test_constants(self): - from polygraphy.common import constants - - assert constants.MARK_ALL - def test_config(self): from polygraphy import constants assert (constants.INTERNAL_CORRECTNESS_CHECKS, constants.AUTOINSTALL_DEPS) -class TestUtilJson(object): - def test_json(self): - from polygraphy.util import Decoder, Encoder, from_json, load_json, save_json, to_json - - class TestCompareFunc(object): def test_basic_compare_func(self): from polygraphy.comparator import CompareFunc diff --git a/tools/Polygraphy/tests/test_examples.py b/tools/Polygraphy/tests/test_examples.py index 8c99d82d..1bf0f624 100644 --- a/tools/Polygraphy/tests/test_examples.py +++ b/tools/Polygraphy/tests/test_examples.py @@ -139,7 +139,11 @@ def test_api_examples(example): ["cli", "run", "04_defining_a_tensorrt_network_or_config_manually"], artifact_names=["my_define_network.py", "my_create_config.py"], ), - Example(["cli", "run", "05_comparing_with_custom_data"]), + Example(["cli", "run", "05_comparing_with_custom_input_data"], artifact_names=["custom_inputs.json"]), + Example( + ["cli", "run", "06_comparing_with_custom_output_data"], + artifact_names=["custom_inputs.json", "custom_outputs.json"], + ), # Convert Example( ["cli", "convert", "01_int8_calibration_in_tensorrt"], @@ -204,6 +208,9 @@ def test_cli_examples(example): @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("7.0"), reason="Unsupported for TRT 6") @pytest.mark.parametrize("example", CLI_INSPECT_EXAMPLES, ids=lambda case: str(case)) def test_cli_inspect_examples(example): + if mod.version(trt.__version__) < mod.version("8.2") and example.path.endswith("02_inspecting_a_tensorrt_engine"): + pytest.skip("Engine layer inspection is not supported on older versions of TRT") + # Last block should be the expected output, and last command should generate it. with example as blocks: commands, expected_output = blocks[:-1], blocks[-1] diff --git a/tools/Polygraphy/tests/tools/args/onnxrt/__init__.py b/tools/Polygraphy/tests/tools/args/onnxrt/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/Polygraphy/tests/tools/args/onnxrt/test_loader.py b/tools/Polygraphy/tests/tools/args/onnxrt/test_loader.py new file mode 100644 index 00000000..339ed86c --- /dev/null +++ b/tools/Polygraphy/tests/tools/args/onnxrt/test_loader.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import onnxruntime as onnxrt +from polygraphy.tools.args import ModelArgs, OnnxLoaderArgs, OnnxrtSessionArgs +from tests.models.meta import ONNX_MODELS +from tests.tools.args.helper import ArgGroupTestHelper + + +class TestOnnxrtSessionArgs(object): + def test_execution_providers(self): + arg_group = ArgGroupTestHelper(OnnxrtSessionArgs(), deps=[ModelArgs(), OnnxLoaderArgs()]) + arg_group.parse_args([ONNX_MODELS["identity_identity"].path, "--providers", "cpu"]) + sess = arg_group.load_onnxrt_session() + + assert sess + assert isinstance(sess, onnxrt.InferenceSession) + assert sess.get_providers() == ["CPUExecutionProvider"] diff --git a/tools/Polygraphy/tests/tools/args/trt/test_config.py b/tools/Polygraphy/tests/tools/args/trt/test_config.py index 7859d152..17c0659f 100644 --- a/tools/Polygraphy/tests/tools/args/trt/test_config.py +++ b/tools/Polygraphy/tests/tools/args/trt/test_config.py @@ -14,14 +14,17 @@ # limitations under the License. # +import contextlib from textwrap import dedent +import polygraphy.tools.args.util as args_util import pytest import tensorrt as trt from polygraphy import mod, util from polygraphy.backend.trt import TacticRecorder, TacticReplayData, TacticReplayer, create_network from polygraphy.exception import PolygraphyException from polygraphy.tools.args import DataLoaderArgs, ModelArgs, TrtConfigArgs +from tests.helper import has_dla from tests.tools.args.helper import ArgGroupTestHelper @@ -90,6 +93,13 @@ def test_dla(self, trt_config_args): assert config.default_device_type == trt.DeviceType.DLA assert config.DLA_core == 0 + def test_calibrator_when_dla(self, trt_config_args): + trt_config_args.parse_args(["--use-dla", "--int8"]) + + builder, network = create_network() + with builder, network, trt_config_args.create_config(builder, network=network) as config: + assert isinstance(config.int8_calibrator, trt.IInt8EntropyCalibrator2) + @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.0"), reason="SAFETY_SCOPE was added in TRT 8") def test_restricted_flags(self, trt_config_args): trt_config_args.parse_args(["--trt-safety-restricted"]) @@ -98,17 +108,6 @@ def test_restricted_flags(self, trt_config_args): with builder, network, trt_config_args.create_config(builder, network=network) as config: assert config.get_flag(getattr(trt.BuilderFlag, "SAFETY_SCOPE")) - @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.0"), reason="Bugged before TRT 8") - def test_tactic_replay(self, trt_config_args): - with util.NamedTemporaryFile(suffix=".json") as f: - trt_config_args.parse_args(["--tactic-replay", f.name]) - builder, network = create_network() - - with builder, network, trt_config_args.create_config(builder, network=network) as config: - recorder = config.algorithm_selector - assert recorder.make_func == TacticRecorder - assert recorder.path == f.name - @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.0"), reason="Bugged before TRT 8") @pytest.mark.parametrize( "opt, cls", @@ -259,3 +258,47 @@ def my_load_config(config): def test_code_injection_checks(self, trt_config_args, args): with pytest.raises(PolygraphyException): trt_config_args.parse_args(args) + + with contextlib.suppress(AttributeError): + + @pytest.mark.skipif( + mod.version(trt.__version__) < mod.version("8.3"), reason="Unsupported for TRT versions prior to 8.3" + ) + @pytest.mark.parametrize( + "args,expected", + [ + (["--pool-limit", "workspace:250"], {trt.MemoryPoolType.WORKSPACE: 250}), + (["--pool-limit", "dla_managed_sram:250"], {trt.MemoryPoolType.DLA_MANAGED_SRAM: 250}), + (["--pool-limit", "dla_local_dram:250"], {trt.MemoryPoolType.DLA_LOCAL_DRAM: 250}), + (["--pool-limit", "dla_global_dram:250"], {trt.MemoryPoolType.DLA_GLOBAL_DRAM: 250}), + # Test case insensitivity + (["--pool-limit", "wOrkSpaCE:250"], {trt.MemoryPoolType.WORKSPACE: 250}), + # Test works with K/M/G suffixes + (["--pool-limit", "workspace:2M"], {trt.MemoryPoolType.WORKSPACE: 2 << 20}), + # Test works with scientific notation + (["--pool-limit", "workspace:2e3"], {trt.MemoryPoolType.WORKSPACE: 2e3}), + ], + ) + def test_memory_pool_limits(self, args, expected, trt_config_args): + trt_config_args.parse_args(args) + builder, network = create_network() + loader = args_util.run_script(trt_config_args.add_trt_config_loader) + assert loader.memory_pool_limits == expected + with builder, network, loader(builder, network=network) as config: + for pool_type, pool_size in expected.items(): + if "dla" in pool_type.name.lower() and not has_dla(): + pytest.skip("DLA is not available on this system") + config.get_memory_pool_limit(pool_type) == pool_size + + @pytest.mark.skipif( + mod.version(trt.__version__) < mod.version("8.3"), reason="Unsupported for TRT versions prior to 8.3" + ) + @pytest.mark.parametrize( + "args", + [ + ["--pool-limit", "250"], + ], + ) + def test_memory_pool_limits_empty_key_not_allowed(self, args, trt_config_args): + with pytest.raises(PolygraphyException, match="Could not parse argument"): + trt_config_args.parse_args(args) diff --git a/tools/Polygraphy/tests/tools/test_convert.py b/tools/Polygraphy/tests/tools/test_convert.py index 6168824d..5321dea4 100644 --- a/tools/Polygraphy/tests/tools/test_convert.py +++ b/tools/Polygraphy/tests/tools/test_convert.py @@ -38,7 +38,7 @@ def test_fp_to_fp16(self): run_polygraphy_convert( [ONNX_MODELS["identity_identity"].path, "--convert-to=onnx", "--fp-to-fp16", "-o", outmodel.name] ) - assert onnx.load(outmodel.name).graph.value_info[0].type.tensor_type.elem_type == 10 + assert onnx.load(outmodel.name).graph.input[0].type.tensor_type.elem_type == 10 class TestConvertToTrt(object): diff --git a/tools/Polygraphy/tests/tools/test_data.py b/tools/Polygraphy/tests/tools/test_data.py index fbb5604d..1ff630e6 100644 --- a/tools/Polygraphy/tests/tools/test_data.py +++ b/tools/Polygraphy/tests/tools/test_data.py @@ -15,6 +15,7 @@ # import numpy as np from polygraphy import util +from polygraphy.json import load_json from tests.models.meta import ONNX_MODELS from tests.tools.common import run_polygraphy_data, run_polygraphy_run @@ -29,7 +30,7 @@ def test_merge_inputs_outputs(self): run_polygraphy_data(["to-input", inps.name, outs.name, "-o", merged.name]) - merged_data = util.load_json(merged.name) + merged_data = load_json(merged.name) assert len(merged_data) == 1 assert list(merged_data[0].keys()) == ["x", "y"] assert all(isinstance(val, np.ndarray) for val in merged_data[0].values()) diff --git a/tools/Polygraphy/tests/tools/test_inspect.py b/tools/Polygraphy/tests/tools/test_inspect.py index 85abcef4..eb3f3cd5 100644 --- a/tools/Polygraphy/tests/tools/test_inspect.py +++ b/tools/Polygraphy/tests/tools/test_inspect.py @@ -25,17 +25,38 @@ from tests.tools.common import run_polygraphy_inspect, run_polygraphy_run -@pytest.fixture(scope="session", params=["none", "basic", "attrs", "full"]) +@pytest.fixture( + scope="session", + params=[ + [], + ["layers"], + ["layers", "attrs"], + ["layers", "attrs", "weights"], + ["weights"], + ], +) def run_inspect_model(request): - yield lambda additional_opts: run_polygraphy_inspect( - ["model"] + ["--mode={:}".format(request.param)] + additional_opts - ) + show_args = (["--show"] if request.param else []) + request.param + yield lambda additional_opts: run_polygraphy_inspect(["model"] + additional_opts + show_args) @pytest.fixture(scope="session") -def identity_engine(): - with util.NamedTemporaryFile() as outpath: - run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--save-engine", outpath.name]) +def dynamic_identity_engine(): + with util.NamedTemporaryFile(suffix=".engine") as outpath: + run_polygraphy_run( + [ + ONNX_MODELS["dynamic_identity"].path, + "--trt", + "--trt-min-shapes=X:[1,2,1,1]", + "--trt-opt-shapes=X:[1,2,3,3]", + "--trt-max-shapes=X:[1,2,5,5]", + "--trt-min-shapes=X:[1,2,2,2]", + "--trt-opt-shapes=X:[1,2,4,4]", + "--trt-max-shapes=X:[1,2,6,6]", + "--save-engine", + outpath.name, + ] + ) yield outpath.name @@ -55,11 +76,10 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): assert acline == exline -# ONNX cases ONNX_CASES = [ [ "identity", - "none", + [], r""" [I] ==== ONNX Model ==== Name: test_identity | Opset: 8 @@ -77,7 +97,7 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], [ "identity", - "basic", + ["layers"], r""" [I] ==== ONNX Model ==== Name: test_identity | Opset: 8 @@ -99,7 +119,7 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], [ "identity_with_initializer", - "basic", + ["layers"], r""" [I] ==== ONNX Model ==== Name: onnx_graphsurgeon | Opset: 11 @@ -121,7 +141,7 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], [ "identity_with_initializer", - "full", + ["layers", "attrs", "weights"], r""" [I] ==== ONNX Model ==== Name: onnx_graphsurgeon | Opset: 11 @@ -145,7 +165,7 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], [ "tensor_attr", - "basic", + ["layers"], r""" [I] ==== ONNX Model ==== Name: onnx_graphsurgeon | Opset: 11 @@ -166,7 +186,7 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], [ "tensor_attr", - "attrs", + ["layers", "attrs"], r""" [I] ==== ONNX Model ==== Name: onnx_graphsurgeon | Opset: 11 @@ -189,7 +209,7 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], [ "tensor_attr", - "full", + ["layers", "attrs", "weights"], r""" [I] ==== ONNX Model ==== Name: onnx_graphsurgeon | Opset: 11 @@ -226,7 +246,7 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], [ "scan", - "full", + ["layers", "attrs", "weights"], r""" [I] ==== ONNX Model ==== Name: graph | Opset: 10 @@ -276,7 +296,7 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], [ "dim_param", - "basic", + ["layers"], r""" [I] ==== ONNX Model ==== Name: tf2onnx | Opset: 10 @@ -298,61 +318,121 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ], ] +# Format: List[Tuple[show_opts, expected]] +ENGINE_CASES = [ + ( + [], + r""" + [I] ==== TensorRT Engine ==== + Name: Unnamed Network 0 | Explicit Batch Engine -# List[model, expected_files, expected_output] -TEST_CAPABILITY_CASES = [ + ---- 1 Engine Input(s) ---- + {X [dtype=float32, shape=(1, 2, -1, -1)]} + + ---- 1 Engine Output(s) ---- + {Y [dtype=float32, shape=(1, 2, -1, -1)]} + + ---- Memory ---- + Device Memory: 0 bytes + + ---- 2 Profile(s) (2 Binding(s) Each) ---- + - Profile: 0 + Binding Index: 0 (Input) [Name: X] | Shapes: min=(1, 2, 1, 1), opt=(1, 2, 3, 3), max=(1, 2, 5, 5) + Binding Index: 1 (Output) [Name: Y] | Shape: (1, 2, -1, -1) + + - Profile: 1 + Binding Index: 2 (Input) [Name: X [profile 1]] | Shapes: min=(1, 2, 2, 2), opt=(1, 2, 4, 4), max=(1, 2, 6, 6) + Binding Index: 3 (Output) [Name: Y [profile 1]] | Shape: (1, 2, -1, -1) + + ---- 1 Layer(s) Per Profile ---- + """, + ), ( - "capability", - [ - "results.txt", - "supported_subgraph-nodes-1-1.onnx", - "supported_subgraph-nodes-3-3.onnx", - "unsupported_subgraph-nodes-0-0.onnx", - "unsupported_subgraph-nodes-2-2.onnx", - "unsupported_subgraph-nodes-4-4.onnx", - ], - """ - [I] ===== Summary ===== - Operator | Count | Reason | Nodes - ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - FAKE! | 2 | In node 0 (importFallbackPluginImporter): UNSUPPORTED_NODE: Assertion failed: creator && "Plugin not found, are the plugin name, version, and namespace correct?" | [[0, 1], [2, 3]] - FAKER! | 1 | In node 0 (importFallbackPluginImporter): UNSUPPORTED_NODE: Assertion failed: creator && "Plugin not found, are the plugin name, version, and namespace correct?" | [[4, 5]] + ["layers"], + r""" + [I] ==== TensorRT Engine ==== + Name: Unnamed Network 0 | Explicit Batch Engine + + ---- 1 Engine Input(s) ---- + {X [dtype=float32, shape=(1, 2, -1, -1)]} + + ---- 1 Engine Output(s) ---- + {Y [dtype=float32, shape=(1, 2, -1, -1)]} + + ---- Memory ---- + Device Memory: 0 bytes + + ---- 2 Profile(s) (2 Binding(s) Each) ---- + - Profile: 0 + Binding Index: 0 (Input) [Name: X] | Shapes: min=(1, 2, 1, 1), opt=(1, 2, 3, 3), max=(1, 2, 5, 5) + Binding Index: 1 (Output) [Name: Y] | Shape: (1, 2, -1, -1) + + - Profile: 1 + Binding Index: 2 (Input) [Name: X [profile 1]] | Shapes: min=(1, 2, 2, 2), opt=(1, 2, 4, 4), max=(1, 2, 6, 6) + Binding Index: 3 (Output) [Name: Y [profile 1]] | Shape: (1, 2, -1, -1) + + ---- 1 Layer(s) Per Profile ---- + - Profile: 0 + Layer 0 | node_of_Y [Op: Reformat] + {X [shape=(1, 2, -1, -1)]} + -> {Y [shape=(1, 2, -1, -1)]} + + - Profile: 1 + Layer 0 | node_of_Y [profile 1] [Op: Reformat] + {X [profile 1] [shape=(1, 2, -1, -1)]} + -> {Y [profile 1] [shape=(1, 2, -1, -1)]} """, ), ( - "identity_identity", - [], - """ - Graph is fully supported by TensorRT; Will not generate subgraphs. + ["layers", "attrs"], + r""" + [I] ==== TensorRT Engine ==== + Name: Unnamed Network 0 | Explicit Batch Engine + + ---- 1 Engine Input(s) ---- + {X [dtype=float32, shape=(1, 2, -1, -1)]} + + ---- 1 Engine Output(s) ---- + {Y [dtype=float32, shape=(1, 2, -1, -1)]} + + ---- Memory ---- + Device Memory: 0 bytes + + ---- 2 Profile(s) (2 Binding(s) Each) ---- + - Profile: 0 + Binding Index: 0 (Input) [Name: X] | Shapes: min=(1, 2, 1, 1), opt=(1, 2, 3, 3), max=(1, 2, 5, 5) + Binding Index: 1 (Output) [Name: Y] | Shape: (1, 2, -1, -1) + + - Profile: 1 + Binding Index: 2 (Input) [Name: X [profile 1]] | Shapes: min=(1, 2, 2, 2), opt=(1, 2, 4, 4), max=(1, 2, 6, 6) + Binding Index: 3 (Output) [Name: Y [profile 1]] | Shape: (1, 2, -1, -1) + + ---- 1 Layer(s) Per Profile ---- + - Profile: 0 + Layer 0 | node_of_Y [Op: Reformat] + {X [shape=(1, 2, -1, -1)]} + -> {Y [shape=(1, 2, -1, -1)]} + ---- Attributes ---- + Origin = IDENTITY + Tactic = 0x0 + + - Profile: 1 + Layer 0 | node_of_Y [profile 1] [Op: Reformat] + {X [profile 1] [shape=(1, 2, -1, -1)]} + -> {Y [profile 1] [shape=(1, 2, -1, -1)]} + ---- Attributes ---- + Origin = IDENTITY + Tactic = 0x0 """, ), ] -class TestCapability(object): - @pytest.mark.skipif( - mod.version(trt.__version__) < mod.version("8.0"), reason="supports_model API not available before TRT 8.0" - ) - @pytest.mark.parametrize("case", TEST_CAPABILITY_CASES, ids=lambda case: case[0]) - def test_capability(self, case): - model, expected_files, expected_summary = case - with tempfile.TemporaryDirectory() as outdir: - status = run_polygraphy_inspect( - ["capability", ONNX_MODELS[model].path, "-o", os.path.join(outdir, "subgraphs")], - ) - assert sorted(map(os.path.basename, glob.glob(os.path.join(outdir, "subgraphs", "**")))) == sorted( - expected_files - ) - assert dedent(expected_summary).strip() in status.stdout - - class TestInspectModel(object): @pytest.mark.parametrize("case", ONNX_CASES, ids=lambda case: "{:}-{:}".format(case[0], case[1])) - def test_model_onnx(self, case): - model, mode, expected = case - status = run_polygraphy_inspect( - ["model", ONNX_MODELS[model].path, "--mode={:}".format(mode)], disable_verbose=True - ) + def test_onnx(self, case): + model, show, expected = case + status = run_polygraphy_inspect(["model", ONNX_MODELS[model].path, "--show"] + show, disable_verbose=True) expected = dedent(expected).strip() actual = "\n".join(status.stdout.splitlines()[1:]) # Ignore loading message @@ -360,7 +440,7 @@ def test_model_onnx(self, case): check_lines_match(actual, expected) @pytest.mark.parametrize("model", ["identity", "scan", "tensor_attr"]) - def test_model_trt_sanity(self, run_inspect_model, model): + def test_trt_sanity(self, run_inspect_model, model): import tensorrt as trt if model == "tensor_attr" and mod.version(trt.__version__) < mod.version("7.2"): @@ -371,7 +451,7 @@ def test_model_trt_sanity(self, run_inspect_model, model): run_inspect_model([ONNX_MODELS[model].path, "--display-as=trt"]) - def test_model_trt_network_script(self): + def test_trt_network_script(self): script = dedent( """ from polygraphy.backend.trt import CreateNetwork @@ -392,10 +472,18 @@ def load_network(builder, network): run_polygraphy_inspect(["model", f.name]) - def test_model_trt_engine_sanity(self, run_inspect_model, identity_engine): - run_inspect_model([identity_engine, "--model-type=engine"]) + @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.2"), reason="Unsupported for TRT 8.0 and older") + @pytest.mark.parametrize("case", ENGINE_CASES, ids=lambda case: "{:}".format(case[0])) + def test_trt_engine(self, case, dynamic_identity_engine): + show, expected = case + status = run_polygraphy_inspect(["model", dynamic_identity_engine, "--show"] + show, disable_verbose=True) + + expected = dedent(expected).strip() + actual = "\n".join(status.stdout.splitlines()[1:]) # Ignore loading message - def test_model_tf_sanity(self, run_inspect_model): + check_lines_match(actual, expected) + + def test_tf_sanity(self, run_inspect_model): pytest.importorskip("tensorflow") run_inspect_model([TF_MODELS["identity"].path, "--model-type=frozen"]) @@ -440,3 +528,50 @@ def test_show_tactics(self, case): actual = status.stdout check_lines_match(actual, expected, should_check_line=lambda line: "Algorithm: " not in line) + + +# List[model, expected_files, expected_output] +TEST_CAPABILITY_CASES = [ + ( + "capability", + [ + "results.txt", + "supported_subgraph-nodes-1-1.onnx", + "supported_subgraph-nodes-3-3.onnx", + "unsupported_subgraph-nodes-0-0.onnx", + "unsupported_subgraph-nodes-2-2.onnx", + "unsupported_subgraph-nodes-4-4.onnx", + ], + """ + [I] ===== Summary ===== + Operator | Count | Reason | Nodes + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + FAKE! | 2 | In node 0 (importFallbackPluginImporter): UNSUPPORTED_NODE: Assertion failed: creator && "Plugin not found, are the plugin name, version, and namespace correct?" | [[0, 1], [2, 3]] + FAKER! | 1 | In node 0 (importFallbackPluginImporter): UNSUPPORTED_NODE: Assertion failed: creator && "Plugin not found, are the plugin name, version, and namespace correct?" | [[4, 5]] + """, + ), + ( + "identity_identity", + [], + """ + Graph is fully supported by TensorRT; Will not generate subgraphs. + """, + ), +] + + +class TestCapability(object): + @pytest.mark.skipif( + mod.version(trt.__version__) < mod.version("8.0"), reason="supports_model API not available before TRT 8.0" + ) + @pytest.mark.parametrize("case", TEST_CAPABILITY_CASES, ids=lambda case: case[0]) + def test_capability(self, case): + model, expected_files, expected_summary = case + with tempfile.TemporaryDirectory() as outdir: + status = run_polygraphy_inspect( + ["capability", ONNX_MODELS[model].path, "-o", os.path.join(outdir, "subgraphs")], + ) + assert sorted(map(os.path.basename, glob.glob(os.path.join(outdir, "subgraphs", "**")))) == sorted( + expected_files + ) + assert dedent(expected_summary).strip() in status.stdout diff --git a/tools/Polygraphy/tests/tools/test_run.py b/tools/Polygraphy/tests/tools/test_run.py index b9a4bb62..8e2f90ec 100644 --- a/tools/Polygraphy/tests/tools/test_run.py +++ b/tools/Polygraphy/tests/tools/test_run.py @@ -275,6 +275,10 @@ def test_tactic_replay(self): def test_tactic_sources(self): run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--tactic-sources", "CUBLAS", "CUBLAS_LT"]) + @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.3"), reason="Unsupported before TRT 8.3") + def test_pool_limits(self): + run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--pool-limit", "workspace:32M"]) + def test_data_loader_script_calibration(self): with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write( @@ -396,6 +400,9 @@ def test_external_data(self): model = ONNX_MODELS["ext_weights"] assert run_polygraphy_run([model.path, "--onnxrt", "--external-data-dir", model.ext_data]) + def test_providers(self): + run_polygraphy_run([ONNX_MODELS["identity"].path, "--onnxrt", "--providers", "cpu"]) + class TestOther(object): def test_0_iterations(self): diff --git a/tools/Polygraphy/tests/util/test_util.py b/tools/Polygraphy/tests/util/test_util.py index 61d202b7..603cfe59 100644 --- a/tools/Polygraphy/tests/util/test_util.py +++ b/tools/Polygraphy/tests/util/test_util.py @@ -35,33 +35,28 @@ def test_volume(case): assert util.volume(it) == vol -class FindInDictCase(object): - def __init__(self, name, map, index, expected): +class FindStrInIterableCase(object): + def __init__(self, name, seq, index, expected): self.name = name - self.map = map + self.seq = seq self.index = index self.expected = expected -FIND_IN_DICT_CASES = [ - FindInDictCase( - "resnet50_v1.5/output/Softmax:0", - map={"resnet50_v1.5/output/Softmax:0": "x"}, - index=None, - expected="resnet50_v1.5/output/Softmax:0", - ), - FindInDictCase( - "resnet50_v1.5/output/Softmax:0", - map={"resnet50_v1.5/output/softmax:0": "x"}, - index=None, - expected="resnet50_v1.5/output/softmax:0", - ), +FIND_STR_IN_ITERABLE_CASES = [ + # Case insensitve, plus function should return element from sequence, not name. + FindStrInIterableCase("Softmax:0", seq=["Softmax:0"], index=None, expected="Softmax:0"), + FindStrInIterableCase("Softmax:0", seq=["softmax:0"], index=None, expected="softmax:0"), + # Exact matches should take priority + FindStrInIterableCase("exact_name", seq=["exact_name_plus", "exact_name"], index=0, expected="exact_name"), + # Index should come into play when no matches are found + FindStrInIterableCase("non-existent", seq=["test", "test2"], index=1, expected="test2"), ] -@pytest.mark.parametrize("case", FIND_IN_DICT_CASES) -def test_find_in_dict(case): - actual = util.find_in_dict(case.name, case.map, case.index) +@pytest.mark.parametrize("case", FIND_STR_IN_ITERABLE_CASES) +def test_find_str_in_iterable(case): + actual = util.find_str_in_iterable(case.name, case.seq, case.index) assert actual == case.expected