diff --git a/tools/Polygraphy/CHANGELOG.md b/tools/Polygraphy/CHANGELOG.md index 222c4330..43c297f6 100644 --- a/tools/Polygraphy/CHANGELOG.md +++ b/tools/Polygraphy/CHANGELOG.md @@ -3,6 +3,24 @@ Dates are in YYYY-MM-DD format. +## v0.31.1 (2021-07-16) +### Changed +- Improved the quality of several examples and added information on how to load serialized TensorRT engines + as well as how to use custom input data. + + +## v0.31.0 (2021-07-02) +### Added +- Added `inspect capability` subtool that will partition a ONNX graph into supported and unsupported subgraphs for usage within TensorRT. +- Added Windows support to the CUDA wrapper in `polygraphy/cuda/cuda.py`. + +### Changed +- `SaveOnnx` will now create parent directories if they do not already exist. + +### Fixed +- Fixed a bug where `ExtractSubgraph` would modify the original graph instead of creating a new graph. + + ## v0.30.3 (2021-06-25) ### Fixed - Fixed various typos, added more details to some tool READMEs. diff --git a/tools/Polygraphy/CONTRIBUTING.md b/tools/Polygraphy/CONTRIBUTING.md index cbf06089..73b0653f 100644 --- a/tools/Polygraphy/CONTRIBUTING.md +++ b/tools/Polygraphy/CONTRIBUTING.md @@ -64,6 +64,7 @@ In those cases, we can export the old name as a deprecated alias to preserve bac - For a class or function, annotate the replacement with the `export_deprecated_alias` decorator. For example: + ```python @mod.export_deprecated_alias("Old", remove_in="0.25.0") class New(object): @@ -72,6 +73,7 @@ In those cases, we can export the old name as a deprecated alias to preserve bac - For modules, invoke the decorator manually within the module file. For example: + ```python mod.export_deprecated_alias("old_mod_name", remove_in="0.25.0")(sys.modules[__name__]) ``` diff --git a/tools/Polygraphy/Makefile b/tools/Polygraphy/Makefile index 6596ea92..06e5c919 100644 --- a/tools/Polygraphy/Makefile +++ b/tools/Polygraphy/Makefile @@ -22,7 +22,7 @@ build: clean python3 setup.py bdist_wheel install_deps: build - -python3 -m pip install colored + -python3 -m pip install colored wheel install: install_deps python3 -m pip install --force-reinstall $(CURDIR)/dist/*.whl diff --git a/tools/Polygraphy/README.md b/tools/Polygraphy/README.md index 723d6ee5..2786744c 100644 --- a/tools/Polygraphy/README.md +++ b/tools/Polygraphy/README.md @@ -9,7 +9,6 @@ - [Examples](#examples) - [Advanced](#advanced) - [Using The Python API](#using-the-python-api) - - [Building API Docs](#building-api-docs) - [Enabling Internal Correctness Checks](#enabling-internal-correctness-checks) - [Contributing](#contributing) @@ -37,45 +36,71 @@ Among other things, Polygraphy lets you: ## Installation +**IMPORTANT**: **Polygraphy does *not* support Python 2.X.** + **All the instructions here assume you are using Python 3 or later.** + ### Installing Prebuilt Wheels ```bash -python3 -m pip install colored polygraphy --index-url https://pypi.ngc.nvidia.com +python -m pip install colored polygraphy --index-url https://pypi.ngc.nvidia.com ``` -**NOTE:** When using this method, the command-line toolkit will be installed into `${HOME}/.local/bin` by default. -Make sure to add this directory to your `PATH` environment variable. +**NOTE:** *When using this method, the command-line toolkit will be installed into `${HOME}/.local/bin` by default.* + *Make sure to add this directory to your `PATH` environment variable.* ### Building From Source -#### Using Make Targets +#### Using Make Targets (Linux) ```bash make install ``` +#### Using Powershell Script (Windows) + +Make sure you are allowed to execute scripts on your system then run: +```ps +.\install.ps1 +``` + #### Building Manually -1. Build a wheel: +1. Install prerequisites: -```bash -python3 setup.py bdist_wheel +``` +python -m pip install wheel ``` -2. Install the wheel manually from **outside** the repository: +2. Build a wheel: -```bash -python3 -m pip install Polygraphy/dist/polygraphy-*-py2.py3-none-any.whl ``` - -**NOTE:** It is strongly recommended to install the `colored` module for colored output -from Polygraphy, as this can greatly improve readability: -```bash -python3 -m pip install colored +python setup.py bdist_wheel ``` +3. Install the wheel manually from **outside** the repository: + + On Linux, run: + + ``` + python -m pip install Polygraphy/dist/polygraphy-*-py2.py3-none-any.whl + ``` + + On Windows, using Powershell, run: + + ```ps + $wheel_path = gci -Name Polygraphy\dist + python -m pip install Polygraphy\dist\$wheel_path + ``` + + + **NOTE:** *It is strongly recommended to install the `colored` module for colored output* + *from Polygraphy, as this can greatly improve readability:* + ``` + python -m pip install colored + ``` + ### Installing Dependencies @@ -91,9 +116,9 @@ To make this easier, Polygraphy can optionally automatically install or upgrade To enable this behavior, set the `POLYGRAPHY_AUTOINSTALL_DEPS` environment variable to `1` or `polygraphy.config.AUTOINSTALL_DEPS = True` using the Python API. -NOTE: By default, dependencies will be installed using the current interpreter, and may overwrite existing -packages. The default installation command, which is `python3 -m pip install`, can be overriden by setting -the `POLYGRAPHY_INSTALL_CMD` environment variable, or setting `polygraphy.config.INSTALL_CMD = "..."` using the Python API. +**NOTE**: *By default, dependencies will be installed using the current interpreter, and may overwrite existing* + *packages. The default installation command, which is `python -m pip install`, can be overriden by setting* + *the `POLYGRAPHY_INSTALL_CMD` environment variable, or setting `polygraphy.config.INSTALL_CMD = "..."` using the Python API.* #### Installing Manually @@ -103,13 +128,13 @@ by the backend, but does serve as a good starting point. You can install the requirements for whichever backends you're interested in with: ```bash -python3 -m pip install -r polygraphy/backend//requirements.txt +python -m pip install -r polygraphy/backend//requirements.txt ``` If additional packages are required, warnings or errors will be logged. You can install the additional packages manually with: ```bash -python3 -m pip install +python -m pip install ``` ## Usage @@ -130,26 +155,9 @@ For examples of both the CLI and Python API, see the [examples directory](./exam ### Using The Python API -For a high-level overview of the Polygraphy Python API, see the [API directory](./polygraphy). -For more details, see the [Polygraphy API documentation](https://docs.nvidia.com/deeplearning/tensorrt/polygraphy/docs/index.html). - - -### Building API Docs - -To build the API documentation, first install required packages: - -```bash -python3 -m pip install -r docs/requirements.txt -``` - -and then use the `make` target to build docs: - -```bash -make docs -``` +For more information on the Polygraphy Python API, including a high-level overview and the +Python API reference documentation, see the [API directory](./polygraphy). -The HTML documentation will be generated under `build/docs` -To view the docs, open `build/docs/index.html` in a browser or HTML viewer. ### Enabling Internal Correctness Checks @@ -162,8 +170,8 @@ When the checks are enabled, Polygraphy will ensure, for example, that loaders d modify their state when they are called, and that runners will reset their state correctly in `deactivate()`. -Note that `POLYGRAPHY_INTERNAL_CORRECTNESS_CHECKS` only relates to checks that validate Polygraphy's internal -APIs. User input validation and public API checks are always enabled and cannot be disabled. +**NOTE:** *`POLYGRAPHY_INTERNAL_CORRECTNESS_CHECKS` only relates to checks that validate Polygraphy's* + *internal APIs. User input validation and public API checks are always enabled and cannot be disabled.* ## Contributing diff --git a/tools/Polygraphy/docs/_templates/layout.html b/tools/Polygraphy/docs/_templates/layout.html new file mode 100644 index 00000000..729c3290 --- /dev/null +++ b/tools/Polygraphy/docs/_templates/layout.html @@ -0,0 +1,9 @@ +{% extends '!layout.html' %} + +{% block extrahead %} + +{% endblock %} + +{% block footer %} + +{% endblock %} diff --git a/tools/Polygraphy/docs/conf.py b/tools/Polygraphy/docs/conf.py index 568010f0..b3cbe4f7 100644 --- a/tools/Polygraphy/docs/conf.py +++ b/tools/Polygraphy/docs/conf.py @@ -76,7 +76,11 @@ html_show_sourcelink = False # Output file base name for HTML help builder. -htmlhelp_basename = "TensorRTdoc" +htmlhelp_basename = "PolygraphyDoc" + +# Template files to extend default Sphinx templates. +# See https://www.sphinx-doc.org/en/master/templating.html for details. +templates_path = ["_templates"] # For constructor arguments to show up in Sphinx generated doc autoclass_content = "both" diff --git a/tools/Polygraphy/examples/api/00_inference_with_tensorrt/README.md b/tools/Polygraphy/examples/api/00_inference_with_tensorrt/README.md index 3fc0acb9..550d4e5d 100644 --- a/tools/Polygraphy/examples/api/00_inference_with_tensorrt/README.md +++ b/tools/Polygraphy/examples/api/00_inference_with_tensorrt/README.md @@ -8,30 +8,40 @@ and run inference with various backends. For an overview of the Polygraphy Python API, see [here](../../../polygraphy/). In this example, we'll look at how you can leverage the API to easily convert an ONNX -model to TensorRT and run inference with FP16 precision enabled. +model to TensorRT and run inference with FP16 precision enabled. We'll then save the +engine to a file and see how you can load it again and run inference. -## Running the Example +## Running The Example 1. Install prerequisites * Ensure that TensorRT is installed * Install other dependencies with `python3 -m pip install -r requirements.txt` 2. **[Optional]** Inspect the model before running the example: + ```bash polygraphy inspect model identity.onnx ``` -3. Run the example: +3. Run the script that builds and runs the engine: + ```bash - python3 example.py + python3 build_and_run.py ``` 4. **[Optional]** Inspect the TensorRT engine built by the example: + ```bash polygraphy inspect model identity.engine ``` +5. Run the script that loads the previously built engine, then runs it: + + ```bash + python3 load_and_run.py + ``` + ## Further Reading For more details on the Polygraphy Python API, see the diff --git a/tools/Polygraphy/examples/api/00_inference_with_tensorrt/example.py b/tools/Polygraphy/examples/api/00_inference_with_tensorrt/build_and_run.py similarity index 83% rename from tools/Polygraphy/examples/api/00_inference_with_tensorrt/example.py rename to tools/Polygraphy/examples/api/00_inference_with_tensorrt/build_and_run.py index 48783a9e..166c3505 100644 --- a/tools/Polygraphy/examples/api/00_inference_with_tensorrt/example.py +++ b/tools/Polygraphy/examples/api/00_inference_with_tensorrt/build_and_run.py @@ -16,7 +16,8 @@ # """ -This script runs an identity model with TensorRT with FP16 precision enabled. +This script builds and runs a TensorRT engine with FP16 precision enabled +starting from an ONNX identity model. """ import numpy as np from polygraphy.backend.trt import CreateConfig, EngineFromNetwork, NetworkFromOnnxPath, SaveEngine, TrtRunner @@ -25,13 +26,17 @@ def main(): # We can compose multiple lazy loaders together to get the desired conversion. # In this case, we want ONNX -> TensorRT Network -> TensorRT engine (w/ fp16). + # + # NOTE: `build_engine` is a *callable* that returns an engine, not the engine itself. + # To get the engine directly, you can use the immediately evaluated functional API. + # See eexamples/api/06_immediate_eval_api for details. build_engine = EngineFromNetwork( NetworkFromOnnxPath("identity.onnx"), config=CreateConfig(fp16=True) ) # Note that config is an optional argument. # To reuse the engine elsewhere, we can serialize and save it to a file. - # The `SaveEngine` lazy loader will return the TensorRT engine, which allows us to chain - # it together with other loaders or runners. + # The `SaveEngine` lazy loader will return the TensorRT engine when called, + # which allows us to chain it together with other loaders. build_engine = SaveEngine(build_engine, path="identity.engine") # Once our loader is ready, inference is simply a matter of constructing a runner, diff --git a/tools/Polygraphy/examples/api/00_inference_with_tensorrt/load_and_run.py b/tools/Polygraphy/examples/api/00_inference_with_tensorrt/load_and_run.py new file mode 100644 index 00000000..b51b623f --- /dev/null +++ b/tools/Polygraphy/examples/api/00_inference_with_tensorrt/load_and_run.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +This script loads the TensorRT engine built by `build_and_run.py` and then runs it. +""" +import numpy as np +from polygraphy.backend.common import BytesFromPath +from polygraphy.backend.trt import EngineFromBytes, TrtRunner + + +def main(): + # Just as we did when building, we can compose multiple loaders together + # to achieve the behavior we want. Specifically, we want to load a serialized + # engine from a file, then deserialize it into a TensorRT engine. + load_engine = EngineFromBytes(BytesFromPath("identity.engine")) + + # Inference remains virtually exactly the same as before: + with TrtRunner(load_engine) as runner: + inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32) + + # NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls. + # Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`. + outputs = runner.infer(feed_dict={"x": inp_data}) + + assert np.array_equal(outputs["y"], inp_data) # It's an identity model! + + print("Inference succeeded!") + + +if __name__ == "__main__": + main() diff --git a/tools/Polygraphy/examples/api/01_comparing_frameworks/README.md b/tools/Polygraphy/examples/api/01_comparing_frameworks/README.md index c589cb14..443b7a4d 100644 --- a/tools/Polygraphy/examples/api/01_comparing_frameworks/README.md +++ b/tools/Polygraphy/examples/api/01_comparing_frameworks/README.md @@ -11,7 +11,7 @@ In this example, we'll look at how you can use the Polygraphy API to run inferen on a model using ONNX Runtime and TensorRT, and then compare the results. -## Running the Example +## Running The Example 1. Install prerequisites * Ensure that TensorRT is installed @@ -23,6 +23,7 @@ on a model using ONNX Runtime and TensorRT, and then compare the results. ``` 3. **[Optional]** Inspect the inference outputs from the example: + ```bash polygraphy inspect data inference_results.json ``` diff --git a/tools/Polygraphy/examples/api/01_comparing_frameworks/example.py b/tools/Polygraphy/examples/api/01_comparing_frameworks/example.py index c4451e93..fff7eb6c 100644 --- a/tools/Polygraphy/examples/api/01_comparing_frameworks/example.py +++ b/tools/Polygraphy/examples/api/01_comparing_frameworks/example.py @@ -38,8 +38,11 @@ def main(): OnnxrtRunner(build_onnxrt_session), ] - # `Comparator.run()` will run each runner separately using synthetic input data and return a `RunResults` instance. - # See `polygraphy/comparator/struct.py` for details. + # `Comparator.run()` will run each runner separately using synthetic input data and + # return a `RunResults` instance. See `polygraphy/comparator/struct.py` for details. + # + # TIP: To use custom input data, you can set the `data_loader` parameter in `Comparator.run()`` + # to a generator or iterable that yields `Dict[str, np.ndarray]`. run_results = Comparator.run(runners) # `Comparator.compare_accuracy()` checks that outputs match between runners. diff --git a/tools/Polygraphy/examples/api/02_using_real_data/README.md b/tools/Polygraphy/examples/api/02_validating_on_a_dataset/README.md similarity index 78% rename from tools/Polygraphy/examples/api/02_using_real_data/README.md rename to tools/Polygraphy/examples/api/02_validating_on_a_dataset/README.md index 3f90e790..ac181dfc 100644 --- a/tools/Polygraphy/examples/api/02_using_real_data/README.md +++ b/tools/Polygraphy/examples/api/02_validating_on_a_dataset/README.md @@ -1,4 +1,4 @@ -# Checking Accuracy Using Real Data +# Validating Accuracy On A Real Dataset ## Introduction @@ -9,17 +9,20 @@ with a real dataset that includes labels or golden values - especially if the da In such cases, it is recommended to use a runner directly instead. +*NOTE: It is possible to provide custom input data to `Comparator.run()` using the `data_loader`* + *parameter. This is may be a viable option when using a smaller dataset.* + In this example, we use a `TrtRunner` directly to validate an identity model on a trivial dataset. Unlike using the `Comparator`, using a runner gives you complete freedom as to how you load your input data, as well as how you validate the results. Since all runners provide the same interface, you can freely drop-in other runners without touching the rest of your validation code. For example, in this case, validating -the model using ONNX-Runtime would require changing just 2 lines (this is left as an -exercise for the reader). +the model using ONNX-Runtime would require changing just 2 lines; this is left as an +exercise for the reader. -## Running the Example +## Running The Example 1. Install prerequisites * Ensure that TensorRT is installed diff --git a/tools/Polygraphy/examples/api/02_using_real_data/example.py b/tools/Polygraphy/examples/api/02_validating_on_a_dataset/example.py similarity index 82% rename from tools/Polygraphy/examples/api/02_using_real_data/example.py rename to tools/Polygraphy/examples/api/02_validating_on_a_dataset/example.py index cb640cf3..93174780 100644 --- a/tools/Polygraphy/examples/api/02_using_real_data/example.py +++ b/tools/Polygraphy/examples/api/02_validating_on_a_dataset/example.py @@ -22,6 +22,7 @@ import numpy as np from polygraphy.backend.trt import EngineFromNetwork, NetworkFromOnnxPath, TrtRunner +# Pretend that this is a very large dataset. REAL_DATASET = [ np.ones((1, 1, 2, 2), dtype=np.float32), np.zeros((1, 1, 2, 2), dtype=np.float32), @@ -29,8 +30,8 @@ np.zeros((1, 1, 2, 2), dtype=np.float32), ] # Definitely real data -# For our identity network, the golden output values are the same as the input values. -# Though this network appears to do nothing, it can be incredibly useful in some cases (like here!). +# For an identity network, the golden output values are the same as the input values. +# Though such a network appears useless at first glance, it can be very useful in some cases (like here!). EXPECTED_OUTPUTS = REAL_DATASET @@ -40,7 +41,7 @@ def main(): with TrtRunner(build_engine) as runner: for (data, golden) in zip(REAL_DATASET, EXPECTED_OUTPUTS): # NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls. - # Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`. + # Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`. outputs = runner.infer(feed_dict={"x": data}) assert np.array_equal(outputs["y"], golden) diff --git a/tools/Polygraphy/examples/api/02_using_real_data/identity.onnx b/tools/Polygraphy/examples/api/02_validating_on_a_dataset/identity.onnx similarity index 100% rename from tools/Polygraphy/examples/api/02_using_real_data/identity.onnx rename to tools/Polygraphy/examples/api/02_validating_on_a_dataset/identity.onnx diff --git a/tools/Polygraphy/examples/api/02_using_real_data/requirements.txt b/tools/Polygraphy/examples/api/02_validating_on_a_dataset/requirements.txt similarity index 100% rename from tools/Polygraphy/examples/api/02_using_real_data/requirements.txt rename to tools/Polygraphy/examples/api/02_validating_on_a_dataset/requirements.txt diff --git a/tools/Polygraphy/examples/api/03_interoperating_with_tensorrt/README.md b/tools/Polygraphy/examples/api/03_interoperating_with_tensorrt/README.md index d5592a86..8d4c71d9 100644 --- a/tools/Polygraphy/examples/api/03_interoperating_with_tensorrt/README.md +++ b/tools/Polygraphy/examples/api/03_interoperating_with_tensorrt/README.md @@ -19,7 +19,7 @@ we will focus on cases where you may want to: - Use a TensorRT builder flag not currently supported by Polygraphy -## Running the Example +## Running The Example 1. Install prerequisites * Ensure that TensorRT is installed @@ -29,11 +29,13 @@ we will focus on cases where you may want to: 2. **[Optional]** Inspect the TensorRT network generated by `load_network()`. This will invoke `load_network()` from within the script and display the generated TensorRT network, which should be named `"MyIdentity"`: + ```bash polygraphy inspect model example.py --trt-network-func load_network --mode=full ``` 3. Run the example: + ```bash python3 example.py ``` diff --git a/tools/Polygraphy/examples/api/04_int8_calibration_in_tensorrt/README.md b/tools/Polygraphy/examples/api/04_int8_calibration_in_tensorrt/README.md index 1e50e215..c2ab9224 100644 --- a/tools/Polygraphy/examples/api/04_int8_calibration_in_tensorrt/README.md +++ b/tools/Polygraphy/examples/api/04_int8_calibration_in_tensorrt/README.md @@ -19,13 +19,14 @@ with (fake) calibration data, and how you can manage the calibration cache with parameter. -## Running the Example +## Running The Example 1. Install prerequisites * Ensure that TensorRT is installed * Install other dependencies with `python3 -m pip install -r requirements.txt` 2. Run the example: + ```bash python3 example.py ``` @@ -33,6 +34,7 @@ parameter. 3. The first time you run the example, it will create a calibration cache called `identity-calib.cache`. If you run the example again, you should see that it now uses the cache instead of running calibration again: + ```bash python3 example.py ``` diff --git a/tools/Polygraphy/examples/api/05_using_tensorrt_network_api/README.md b/tools/Polygraphy/examples/api/05_using_tensorrt_network_api/README.md index 1d912be6..7bf8949c 100644 --- a/tools/Polygraphy/examples/api/05_using_tensorrt_network_api/README.md +++ b/tools/Polygraphy/examples/api/05_using_tensorrt_network_api/README.md @@ -11,7 +11,7 @@ In this example, we'll look at how you can use Polygraphy's `extend` decorator, loader to seamlessly integrate a network defined using TensorRT APIs with Polygraphy. -## Running the Example +## Running The Example 1. Install prerequisites * Ensure that TensorRT is installed @@ -19,11 +19,13 @@ loader to seamlessly integrate a network defined using TensorRT APIs with Polygr 2. **[Optional]** Inspect the TensorRT network generated by `create_network()`. This will invoke `create_network()` from within the script and display the generated TensorRT network: + ```bash polygraphy inspect model example.py --trt-network-func create_network --mode=full ``` 3. Run the example: + ```bash python3 example.py ``` diff --git a/tools/Polygraphy/examples/api/06_immediate_eval_api/README.md b/tools/Polygraphy/examples/api/06_immediate_eval_api/README.md index fdfc86a2..59473059 100644 --- a/tools/Polygraphy/examples/api/06_immediate_eval_api/README.md +++ b/tools/Polygraphy/examples/api/06_immediate_eval_api/README.md @@ -12,6 +12,7 @@ Most of the time, the lazy loaders included with Polygraphy have several advanta - They allow us to define a sequence of operations in advance by chaining loaders together, which provides an easy way to build reusable functions. For example, we could create a loader that imports a model from ONNX and generates a serialized TensorRT Engine: + ```python build_engine = EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx")) ``` @@ -52,13 +53,14 @@ engine = engine_from_network((builder, network), config) `example.py` showcases basic usage of the immediately evaluated functional API. -## Running the Example +## Running The Example 1. Install prerequisites * Ensure that TensorRT is installed * Install other dependencies with `python3 -m pip install -r requirements.txt` 2. Run the example: + ```bash python3 example.py ``` diff --git a/tools/Polygraphy/examples/api/07_tensorrt_and_dynamic_shapes/README.md b/tools/Polygraphy/examples/api/07_tensorrt_and_dynamic_shapes/README.md index 593a283e..c9d9ab25 100644 --- a/tools/Polygraphy/examples/api/07_tensorrt_and_dynamic_shapes/README.md +++ b/tools/Polygraphy/examples/api/07_tensorrt_and_dynamic_shapes/README.md @@ -24,6 +24,7 @@ Using the TensorRT API, the process involves two steps: the entire output(s). For a single-input, single-output model, this would look roughly as follows: + ```python context.set_binding_shape(0, inp.shape) @@ -108,18 +109,20 @@ the cost of activating a runner should be small - it just needs to allocate input and output buffers. Hence, we'll be able to activate runners on-demand quickly. -## Running the Example +## Running The Example 1. Install prerequisites * Ensure that TensorRT is installed * Install other dependencies with `python3 -m pip install -r requirements.txt` 2. Run the example: + ```bash python3 example.py ``` -3. [Optional] Inspect the generated engine: +3. **[Optional]** Inspect the generated engine: + ```bash polygraphy inspect model dynamic_identity.engine ``` diff --git a/tools/Polygraphy/examples/api/README.md b/tools/Polygraphy/examples/api/README.md index 393d1db9..f97c9a3f 100644 --- a/tools/Polygraphy/examples/api/README.md +++ b/tools/Polygraphy/examples/api/README.md @@ -2,3 +2,61 @@ This directory includes examples that use the Polygraphy Python API. For examples of the command-line tools, see the [cli](../cli/) directory instead. + + +## Generating Your Own Examples + +In the event that the examples here do not cover a particular use-case, you can typically +use `polygraphy run` to fill the gap; `polygraphy run` is capable of dynamically generating +Python scripts that use the Polygraphy API that do exactly what the tool would otherwise do. + +Thus, if `polygraphy run` includes functionality you need, but you cannot find a +corresponding API example, try running: +```bash +polygraphy run --gen - +``` +The argument to `--gen` should be the name of the file in which to write the generated script. +The special value `-` corresponds to `stdout`. + +For example, running: +```bash +polygraphy run --gen - model.onnx --trt --onnxrt +``` +will display something like this on `stdout`: +```py +#!/usr/bin/env python3 +# Template auto-generated by polygraphy [v0.31.0] on 01/01/20 at 10:10:10 +# Generation Command: /infrastructure/Polygraphy/bin/polygraphy run --gen - model.onnx --trt --onnxrt +# This script compares model.onnx between TensorRT and ONNX Runtime + +from polygraphy.logger import G_LOGGER + +from polygraphy.backend.onnxrt import OnnxrtRunner, SessionFromOnnx +from polygraphy.backend.trt import EngineFromNetwork, NetworkFromOnnxPath, TrtRunner +from polygraphy.comparator import Comparator +import sys + +# Loaders +parse_network_from_onnx = NetworkFromOnnxPath('model.onnx') +build_engine = EngineFromNetwork(parse_network_from_onnx) +build_onnxrt_session = SessionFromOnnx('model.onnx') + +# Runners +runners = [ + TrtRunner(build_engine), + OnnxrtRunner(build_onnxrt_session), +] + +# Runner Execution +results = Comparator.run(runners) + +success = True +# Accuracy Comparison +success &= bool(Comparator.compare_accuracy(results)) + +# Report Results +cmd_run = ' '.join(sys.argv) +if not success: + G_LOGGER.critical("FAILED | Command: {}".format(cmd_run)) +G_LOGGER.finish("PASSED | Command: {}".format(cmd_run)) +``` diff --git a/tools/Polygraphy/examples/cli/README.md b/tools/Polygraphy/examples/cli/README.md index 56c62738..204bc0f9 100644 --- a/tools/Polygraphy/examples/cli/README.md +++ b/tools/Polygraphy/examples/cli/README.md @@ -2,3 +2,65 @@ This directory includes examples that use the Polygraphy CLI. For examples of the Python API, see the [api](../api/) directory instead. + +## Common Topics + +This section covers some concepts that are common to multiple tools. + +### Using Custom Input Data + +For tools that require input data, such as `run`, Polygraphy currently +provides 2 ways to use custom input data: + +1. `--load-input-data`, which takes a path to a JSON file containing a `List[Dict[str, np.ndarray]]`. + This will cause Polygraphy to load the entire object into memory. + *NOTE: This may be impractical or even impossible if the data is very large.* + +2. `--data-loader-script`, which takes a path to a Python script that defines a `load_data` function + that returns a data loader. The data loader can be any iterable or generator that yields + `Dict[str, np.ndarray]`. By using a generator, we can avoid loading all the data + at once, and instead limit it to just a single input at a time. + + *TIP: If you have an existing script that already defines such a function, you do **not** need to create* + *a separate script just for the sake of `--data-loader-script`. You can simply use the existing script* + *and optionally use the `--data-loader-func-name` argument to specify the name of the function (if it's not `load_data`)* + +See [`run` example 05](run/05_comparing_with_custom_data) for details. + +### Defining A Custom TensorRT Network + +Many of the command-line tools involve creating TensorRT networks. In most cases, these +are created by parsing a model from a framework (generally in ONNX format). However, it +is also possible to define the TensorRT network manually using a Python script. + +1. To get started, generate a template script with: + + ```bash + polygraphy template trt-network -o define_network.py + ``` + + If you want to start from a model and modify the resulting TensorRT network instead + of creating one from scratch, simply provide the model as an argument to `template trt-network`. + For example, for an ONNX model: + + ```bash + polygraphy template trt-network -o define_network.py + ``` + +2. Once you've filled out the body of the `load_network` function in `define_network.py`, + you can use it in the tools by providing the script in place of a model argument. + For example: + + ```bash + polygraphy run define_network.py --model-type=trt-network-script --trt + ``` + +See [`run` example 04](run/04_defining_a_tensorrt_network_manually) for details. + + +### Defining a Custom TensorRT Builder Configuration + +Similar to defining custom TensorRT networks, it is possible to provide custom +TensorRT builder configurations on the command-line using a Python script. + +See [`run` example 04](run/04_defining_a_tensorrt_network_manually) for details. diff --git a/tools/Polygraphy/examples/cli/convert/01_int8_calibration_in_tensorrt/README.md b/tools/Polygraphy/examples/cli/convert/01_int8_calibration_in_tensorrt/README.md index f660a134..7d6ac06a 100644 --- a/tools/Polygraphy/examples/cli/convert/01_int8_calibration_in_tensorrt/README.md +++ b/tools/Polygraphy/examples/cli/convert/01_int8_calibration_in_tensorrt/README.md @@ -1,40 +1,38 @@ # Int8 Calibration In TensorRT + +## Introduction + In [API example 04](../../../api/04_int8_calibration_in_tensorrt/), we saw how we can leverage Polygraphy's included calibrator to easily run int8 calibration with TensorRT. But what if we wanted to do the same thing on the command-line? To do this, we need a way to supply custom input data to our command-line tools. -Polygraphy currently provides 2 ways to do so: +Polygraphy provides multiple ways to do so, which are detailed [here](../../). -1. Using `--load-input-data`, which takes a path to a JSON file containing a `List[Dict[str, np.ndarray]]`. - This will cause Polygraphy to load the entire object into memory, which would - be impractical or even impossible for a large calibration set. +In this example, we'll use a data loader script by defining a `load_data` function in a Python +script called `data_loader.py` and then use `polygraphy convert` to build the TensorRT engine. -2. Using `--data-loader-script` which takes a path to a Python script that defines a `load_data` function - that returns a data loader. The data loader can be any iterable or generator that yields - `Dict[str, np.ndarray]`. By using a generator, we can avoid loading the entire calibration set - at once, and instead limit it to just a single batch at a time. +*TIP: We can use the same approach with `polygraphy run` to build and run the engine.* +## Running The Example -We'll define a `load_data` function in a Python script called `data_loader.py` and -then use `polygraphy convert` to build the TensorRT engine. -Note that we could have just as easily used `polygraphy run` to build *and run* the engine. +1. Convert the model, using the custom data loader script to supply calibration data: -```bash -polygraphy convert identity.onnx --int8 \ - --data-loader-script ./data_loader.py \ - -o identity.engine -``` + ```bash + polygraphy convert identity.onnx --int8 \ + --data-loader-script ./data_loader.py \ + -o identity.engine + ``` -In fact, this system is so flexible, we can even use the data loader we defined in the API example! -We just need to set `--data-loader-func-name` since the example does not use `load_data` as the function name. + In fact, this system is so flexible, we can even use the data loader we defined in the API example! + We just need to set `--data-loader-func-name` since the example does not use `load_data` as the function name. -```bash -polygraphy convert identity.onnx --int8 \ - --data-loader-script ../../../api/04_int8_calibration_in_tensorrt/example.py \ - --data-loader-func-name calib_data \ - -o identity.engine -``` + ```bash + polygraphy convert identity.onnx --int8 \ + --data-loader-script ../../../api/04_int8_calibration_in_tensorrt/example.py \ + --data-loader-func-name calib_data \ + -o identity.engine + ``` diff --git a/tools/Polygraphy/examples/cli/convert/03_dynamic_shapes_in_tensorrt/README.md b/tools/Polygraphy/examples/cli/convert/03_dynamic_shapes_in_tensorrt/README.md index 0f547d81..644a750b 100644 --- a/tools/Polygraphy/examples/cli/convert/03_dynamic_shapes_in_tensorrt/README.md +++ b/tools/Polygraphy/examples/cli/convert/03_dynamic_shapes_in_tensorrt/README.md @@ -7,13 +7,14 @@ In order to use dynamic input shapes with TensorRT, we have to specify a range For details on how this works, refer to [API example 07](../../../api/07_tensorrt_and_dynamic_shapes/). -When using the CLI, we can specify the minimum, optimum, and maximum -shapes for each input one or more times. If shapes are specified more than +When using the CLI, we can specify the per-input minimum, optimum, and maximum +shapes one or more times. If shapes are specified more than once per input, multiple optimization profiles are created. ## Running The Example 1. Build an engine with 3 separate profiles: + ```bash polygraphy convert dynamic_identity.onnx -o dynamic_identity.engine \ --trt-min-shapes X:[1,3,28,28] --trt-opt-shapes X:[1,3,28,28] --trt-max-shapes X:[1,3,28,28] \ @@ -21,11 +22,15 @@ once per input, multiple optimization profiles are created. --trt-min-shapes X:[128,3,28,28] --trt-opt-shapes X:[128,3,28,28] --trt-max-shapes X:[128,3,28,28] ``` - For models with multiple inputs, simply provide more arguments to each `--trt-*-shapes` parameter. - For example: `--trt-min-shapes input0:[10,10] input1:[10,10] input2:[10,10] --trt-opt-shapes ...` + For models with multiple inputs, simply provide multiple arguments to each `--trt-*-shapes` parameter. + For example: `--trt-min-shapes input0:[10,10] input1:[10,10] input2:[10,10] ...` + + *TIP: If we want to use only a single profile where min == opt == max, we can leverage the runtime input* + *shapes option: `--input-shapes` as a conveneint shorthand instead of setting min/opt/max separately.* + +2. **[Optional]** Inspect the resulting engine: -2. [Optional] Inspect the resulting engine: ```bash polygraphy inspect model dynamic_identity.engine ``` diff --git a/tools/Polygraphy/examples/cli/debug/01_debugging_flaky_trt_tactics/README.md b/tools/Polygraphy/examples/cli/debug/01_debugging_flaky_trt_tactics/README.md index 2e375adf..19f6bbad 100644 --- a/tools/Polygraphy/examples/cli/debug/01_debugging_flaky_trt_tactics/README.md +++ b/tools/Polygraphy/examples/cli/debug/01_debugging_flaky_trt_tactics/README.md @@ -1,5 +1,8 @@ # Debugging Flaky TensorRT Tactics + +## Introduction + Sometimes, a tactic in TensorRT may produce incorrect results, or have otherwise buggy behavior. Since the TensorRT builder relies on timing tactics, engine builds are non-deterministic, which can make tactic bugs @@ -12,6 +15,9 @@ is likely to be the source of error. The `debug` subtool allows us to automate this process. + +## Running The Example + For this example, we'll break the process down into 3 steps: 1. Generate golden outputs from ONNX-Runtime: diff --git a/tools/Polygraphy/examples/cli/inspect/01_inspecting_a_tensorrt_network/README.md b/tools/Polygraphy/examples/cli/inspect/01_inspecting_a_tensorrt_network/README.md index d8c3fb3f..4eca80f4 100644 --- a/tools/Polygraphy/examples/cli/inspect/01_inspecting_a_tensorrt_network/README.md +++ b/tools/Polygraphy/examples/cli/inspect/01_inspecting_a_tensorrt_network/README.md @@ -1,31 +1,37 @@ # Inspecting A TensorRT Network + +## Introduction + The `inspect model` subtool can automatically convert supported formats into TensorRT networks, and then display them. -For example: -```bash -polygraphy inspect model identity.onnx \ - --mode=basic --display-as=trt -``` +## Running The Example -This will display something like: +1. Display the TensorRT network after parsing an ONNX model: -``` -[I] ==== TensorRT Network ==== - Name: Unnamed Network 0 | Explicit Batch Network + ```bash + polygraphy inspect model identity.onnx \ + --mode=basic --display-as=trt + ``` - ---- 1 Network Input(s) ---- - {x [dtype=float32, shape=(1, 1, 2, 2)]} + This will display something like: - ---- 1 Network Output(s) ---- - {y [dtype=float32, shape=(1, 1, 2, 2)]} + ``` + [I] ==== TensorRT Network ==== + Name: Unnamed Network 0 | Explicit Batch Network - ---- 1 Layer(s) ---- - Layer 0 | node_of_y [Op: LayerType.IDENTITY] + ---- 1 Network Input(s) ---- {x [dtype=float32, shape=(1, 1, 2, 2)]} - -> {y [dtype=float32, shape=(1, 1, 2, 2)]} -``` -It is also possible to show detailed layer information, including layer attributes, using `--mode=full`. + ---- 1 Network Output(s) ---- + {y [dtype=float32, shape=(1, 1, 2, 2)]} + + ---- 1 Layer(s) ---- + Layer 0 | node_of_y [Op: LayerType.IDENTITY] + {x [dtype=float32, shape=(1, 1, 2, 2)]} + -> {y [dtype=float32, shape=(1, 1, 2, 2)]} + ``` + + It is also possible to show detailed layer information, including layer attributes, using `--mode=full`. diff --git a/tools/Polygraphy/examples/cli/inspect/02_inspecting_a_tensorrt_engine/README.md b/tools/Polygraphy/examples/cli/inspect/02_inspecting_a_tensorrt_engine/README.md index f3462db4..d043457b 100644 --- a/tools/Polygraphy/examples/cli/inspect/02_inspecting_a_tensorrt_engine/README.md +++ b/tools/Polygraphy/examples/cli/inspect/02_inspecting_a_tensorrt_engine/README.md @@ -1,45 +1,50 @@ # Inspecting A TensorRT Engine + +## Introduction + The `inspect model` subtool can load and display information about TensorRT engines, i.e. plan files: -For example, first we'll generate an engine with dynamic shapes -and 2 profiles: -```bash -polygraphy run dynamic_identity.onnx --trt \ - --trt-min-shapes X:[1,2,1,1] --trt-opt-shapes X:[1,2,3,3] --trt-max-shapes X:[1,2,5,5] \ - --trt-min-shapes X:[1,2,2,2] --trt-opt-shapes X:[1,2,4,4] --trt-max-shapes X:[1,2,6,6] \ - --save-engine dynamic_identity.engine -``` +## Running The Example + +1. Generate an engine with dynamic shapes and 2 profiles: + + ```bash + polygraphy run dynamic_identity.onnx --trt \ + --trt-min-shapes X:[1,2,1,1] --trt-opt-shapes X:[1,2,3,3] --trt-max-shapes X:[1,2,5,5] \ + --trt-min-shapes X:[1,2,2,2] --trt-opt-shapes X:[1,2,4,4] --trt-max-shapes X:[1,2,6,6] \ + --save-engine dynamic_identity.engine + ``` -Next, we can inspect it: +2. Inspect the engine: -```bash -polygraphy inspect model dynamic_identity.engine -``` + ```bash + polygraphy inspect model dynamic_identity.engine + ``` -This will display something like: + This will display something like: -``` -[I] ==== TensorRT Engine ==== - Name: Unnamed Network 0 | Explicit Batch Engine (2 layers) + ``` + [I] ==== TensorRT Engine ==== + Name: Unnamed Network 0 | Explicit Batch Engine (2 layers) - ---- 1 Engine Input(s) ---- - {X [dtype=float32, shape=(1, 2, -1, -1)]} + ---- 1 Engine Input(s) ---- + {X [dtype=float32, shape=(1, 2, -1, -1)]} - ---- 1 Engine Output(s) ---- - {Y [dtype=float32, shape=(1, 2, -1, -1)]} + ---- 1 Engine Output(s) ---- + {Y [dtype=float32, shape=(1, 2, -1, -1)]} - ---- Memory ---- - Device Memory: 0 bytes + ---- Memory ---- + Device Memory: 0 bytes - ---- 2 Profile(s) (2 Binding(s) Each) ---- - - Profile: 0 - Binding Index: 0 (Input) [Name: X] | Shapes: min=(1, 2, 1, 1), opt=(1, 2, 3, 3), max=(1, 2, 5, 5) - Binding Index: 1 (Output) [Name: Y] | Shape: (1, 2, -1, -1) + ---- 2 Profile(s) (2 Binding(s) Each) ---- + - Profile: 0 + Binding Index: 0 (Input) [Name: X] | Shapes: min=(1, 2, 1, 1), opt=(1, 2, 3, 3), max=(1, 2, 5, 5) + Binding Index: 1 (Output) [Name: Y] | Shape: (1, 2, -1, -1) - - Profile: 1 - Binding Index: 2 (Input) [Name: X [profile 1]] | Shapes: min=(1, 2, 2, 2), opt=(1, 2, 4, 4), max=(1, 2, 6, 6) - Binding Index: 3 (Output) [Name: Y [profile 1]] | Shape: (1, 2, -1, -1) -``` + - Profile: 1 + Binding Index: 2 (Input) [Name: X [profile 1]] | Shapes: min=(1, 2, 2, 2), opt=(1, 2, 4, 4), max=(1, 2, 6, 6) + Binding Index: 3 (Output) [Name: Y [profile 1]] | Shape: (1, 2, -1, -1) + ``` diff --git a/tools/Polygraphy/examples/cli/inspect/03_inspecting_an_onnx_model/README.md b/tools/Polygraphy/examples/cli/inspect/03_inspecting_an_onnx_model/README.md index 7fb967a1..9c928712 100644 --- a/tools/Polygraphy/examples/cli/inspect/03_inspecting_an_onnx_model/README.md +++ b/tools/Polygraphy/examples/cli/inspect/03_inspecting_an_onnx_model/README.md @@ -1,32 +1,38 @@ # Inspecting An ONNX Model -The `inspect model` subtool can display ONNX models. -For example: +## Introduction + +The `inspect model` subtool can display ONNX models. -```bash -polygraphy inspect model identity.onnx --mode=basic -``` -This will display something like: +## Running The Example -``` -[I] ==== ONNX Model ==== - Name: test_identity | Opset: 8 +1. Inspect the ONNX model: - ---- 1 Graph Input(s) ---- - {x [dtype=float32, shape=(1, 1, 2, 2)]} + ```bash + polygraphy inspect model identity.onnx --mode=basic + ``` - ---- 1 Graph Output(s) ---- - {y [dtype=float32, shape=(1, 1, 2, 2)]} + This will display something like: - ---- 0 Initializer(s) ---- - {} + ``` + [I] ==== ONNX Model ==== + Name: test_identity | Opset: 8 - ---- 1 Node(s) ---- - Node 0 | [Op: Identity] + ---- 1 Graph Input(s) ---- {x [dtype=float32, shape=(1, 1, 2, 2)]} - -> {y [dtype=float32, shape=(1, 1, 2, 2)]} -``` -It is also possible to show detailed layer information, including layer attributes, using `--mode=full`. + ---- 1 Graph Output(s) ---- + {y [dtype=float32, shape=(1, 1, 2, 2)]} + + ---- 0 Initializer(s) ---- + {} + + ---- 1 Node(s) ---- + Node 0 | [Op: Identity] + {x [dtype=float32, shape=(1, 1, 2, 2)]} + -> {y [dtype=float32, shape=(1, 1, 2, 2)]} + ``` + + It is also possible to show detailed layer information, including layer attributes, using `--mode=full`. diff --git a/tools/Polygraphy/examples/cli/inspect/04_inspecting_a_tensorflow_graph/README.md b/tools/Polygraphy/examples/cli/inspect/04_inspecting_a_tensorflow_graph/README.md index 8974e7ba..a705c461 100644 --- a/tools/Polygraphy/examples/cli/inspect/04_inspecting_a_tensorflow_graph/README.md +++ b/tools/Polygraphy/examples/cli/inspect/04_inspecting_a_tensorflow_graph/README.md @@ -1,22 +1,28 @@ # Inspecting A TensorFlow Graph + +## Introduction + The `inspect model` subtool can display TensorFlow graphs. -For example: -```bash -polygraphy inspect model identity.pb --model-type=frozen -``` +## Running The Example + +1. Inspect a TensorFlow frozen model: + + ```bash + polygraphy inspect model identity.pb --model-type=frozen + ``` -This will display something like: + This will display something like: -``` -[I] ==== TensorFlow Graph ==== - ---- 1 Graph Inputs ---- - {Input:0 [dtype=float32, shape=(1, 15, 25, 30)]} + ``` + [I] ==== TensorFlow Graph ==== + ---- 1 Graph Inputs ---- + {Input:0 [dtype=float32, shape=(1, 15, 25, 30)]} - ---- 1 Graph Outputs ---- - {Identity_2:0 [dtype=float32, shape=(1, 15, 25, 30)]} + ---- 1 Graph Outputs ---- + {Identity_2:0 [dtype=float32, shape=(1, 15, 25, 30)]} - ---- 4 Nodes ---- -``` + ---- 4 Nodes ---- + ``` diff --git a/tools/Polygraphy/examples/cli/inspect/05_inspecting_inference_outputs/README.md b/tools/Polygraphy/examples/cli/inspect/05_inspecting_inference_outputs/README.md index 103401b2..6ebfab58 100644 --- a/tools/Polygraphy/examples/cli/inspect/05_inspecting_inference_outputs/README.md +++ b/tools/Polygraphy/examples/cli/inspect/05_inspecting_inference_outputs/README.md @@ -1,29 +1,34 @@ # Inspecting Inference Outputs + +## Introduction + The `inspect data` subtool can display information about the `RunResults` object generated by `Comparator.run()`, which represents inference outputs. -For example, first we'll generate some inference outputs using ONNX-Runtime: -```bash -polygraphy run identity.onnx --onnxrt --save-outputs outputs.json -``` +## Running The Example + +1. Generate some inference outputs using ONNX-Runtime: + + ```bash + polygraphy run identity.onnx --onnxrt --save-outputs outputs.json + ``` -Next, we can inspect them: +2. Inspect the results: -```bash -polygraphy inspect data outputs.json --show-values -``` + ```bash + polygraphy inspect data outputs.json --show-values + ``` -This will display something like: + This will display something like: -``` -[I] ==== Run Results (1 runners) ==== + ``` + [I] ==== Run Results (1 runners) ==== - ---- onnxrt-runner-N0-05/24/21-12:44:30 (1 iterations) ---- + ---- onnxrt-runner-N0-07/15/21-10:46:07 (1 iterations) ---- - y [dtype=float32, shape=(1, 1, 2, 2)] | Stats - mean=0.35995, std-dev=0.25784, var=0.066482, median=0.35968, min=0.00011437 at (0, 0, 1, 0), max=0.72032 at (0, 0, 0, 1) - [[[[4.17021990e-01 7.20324516e-01] - [1.14374816e-04 3.02332580e-01]]]] -``` + y [dtype=float32, shape=(1, 1, 2, 2)] | Stats: mean=0.35995, std-dev=0.25784, var=0.066482, median=0.35968, min=0.00011437 at (0, 0, 1, 0), max=0.72032 at (0, 0, 0, 1) + [[[[4.17021990e-01 7.20324516e-01] + [1.14374816e-04 3.02332580e-01]]]] + ``` diff --git a/tools/Polygraphy/examples/cli/inspect/06_inspecting_input_data/README.md b/tools/Polygraphy/examples/cli/inspect/06_inspecting_input_data/README.md index 4ca228c6..6bc37afc 100644 --- a/tools/Polygraphy/examples/cli/inspect/06_inspecting_input_data/README.md +++ b/tools/Polygraphy/examples/cli/inspect/06_inspecting_input_data/README.md @@ -1,27 +1,31 @@ # Inspecting Input Data + +## Introduction + The `inspect data` subtool can display information about input data generated by a data loader. -For example, first we'll generate some input data by running inference: -```bash -polygraphy run identity.onnx --onnxrt --save-inputs inputs.json -``` +## Running The Example +1. Generate some input data by running inference: + + ```bash + polygraphy run identity.onnx --onnxrt --save-inputs inputs.json + ``` -Next, we can inspect them: +2. Inspect the input data: -```bash -polygraphy inspect data inputs.json --show-values -``` + ```bash + polygraphy inspect data inputs.json --show-values + ``` -This will display something like: + This will display something like: -``` -[I] ==== Data (1 iterations) ==== + ``` + [I] ==== Data (1 iterations) ==== - x [dtype=float32, shape=(1, 1, 2, 2)] | Stats - mean=0.35995, std-dev=0.25784, var=0.066482, median=0.35968, min=0.00011437 at (0, 0, 1, 0), max=0.72032 at (0, 0, 0, 1) - [[[[4.17021990e-01 7.20324516e-01] - [1.14374816e-04 3.02332580e-01]]]] -``` + x [dtype=float32, shape=(1, 1, 2, 2)] | Stats: mean=0.35995, std-dev=0.25784, var=0.066482, median=0.35968, min=0.00011437 at (0, 0, 1, 0), max=0.72032 at (0, 0, 0, 1) + [[[[4.17021990e-01 7.20324516e-01] + [1.14374816e-04 3.02332580e-01]]]] + ``` diff --git a/tools/Polygraphy/examples/cli/inspect/07_inspecting_tactic_replays/README.md b/tools/Polygraphy/examples/cli/inspect/07_inspecting_tactic_replays/README.md index ecb2d4b3..ed65a060 100644 --- a/tools/Polygraphy/examples/cli/inspect/07_inspecting_tactic_replays/README.md +++ b/tools/Polygraphy/examples/cli/inspect/07_inspecting_tactic_replays/README.md @@ -1,23 +1,29 @@ # Inspecting Tactic Replay Files + +## Introduction + The `inspect tactics` subtool can display information about TensorRT tactic replay files generated by Polygraphy. -For example, first we'll generate a tactic replay file: -```bash -polygraphy run identity.onnx --trt --save-tactics replay.json -``` +## Running The Example + +1. Generate a tactic replay file: + + ```bash + polygraphy run identity.onnx --trt --save-tactics replay.json + ``` -Next, we can inspect it: +2. Inspect the tactic replay: -```bash -polygraphy inspect tactics replay.json -``` + ```bash + polygraphy inspect tactics replay.json + ``` -This will display something like: + This will display something like: -``` -[I] Layer: node_of_y - Algorithm: (Implementation: -2147483642, Tactic: 0) | Inputs: (('TensorFormat.LINEAR', 'DataType.FLOAT'),) | Outputs: (('TensorFormat.LINEAR', 'DataType.FLOAT'),) -``` + ``` + [I] Layer: node_of_y + Algorithm: (Implementation: -2147483642, Tactic: 0) | Inputs: (('TensorFormat.LINEAR', 'DataType.FLOAT'),) | Outputs: (('TensorFormat.LINEAR', 'DataType.FLOAT'),) + ``` diff --git a/tools/Polygraphy/examples/cli/run/01_comparing_frameworks/README.md b/tools/Polygraphy/examples/cli/run/01_comparing_frameworks/README.md index a5b49744..c38a9a04 100644 --- a/tools/Polygraphy/examples/cli/run/01_comparing_frameworks/README.md +++ b/tools/Polygraphy/examples/cli/run/01_comparing_frameworks/README.md @@ -1,21 +1,29 @@ # Comparing Frameworks + +## Introduction + You can use the `run` subtool to compare a model between different frameworks. In the simplest case, you can supply a model, and one or more framework flags. -For example, to compare an ONNX model between TensorRT and ONNX Runtime: -```bash -polygraphy run dynamic_identity.onnx --trt --onnxrt -``` +## Running The Example + +1. Compare an ONNX model between TensorRT and ONNX Runtime: + + ```bash + polygraphy run dynamic_identity.onnx --trt --onnxrt + ``` + + If our model uses dynamic input shapes, we can specify the shapes to use at + runtime with the `--input-shapes` option: -If our model uses dynamic input shapes, we can specify the shapes to use at -runtime with the `--input-shapes` option: + ```bash + polygraphy run dynamic_identity.onnx --trt --onnxrt \ + --input-shapes X:[1,2,4,4] + ``` -```bash -polygraphy run dynamic_identity.onnx --trt --onnxrt \ - --input-shapes X:[1,2,4,4] -``` +## Further Reading For more details on working with dynamic shapes in TensorRT, refer to [`convert` example 03](../../convert/03_dynamic_shapes_in_tensorrt/) or diff --git a/tools/Polygraphy/examples/cli/run/02_comparing_across_runs/README.md b/tools/Polygraphy/examples/cli/run/02_comparing_across_runs/README.md index 3a556309..f58d0390 100644 --- a/tools/Polygraphy/examples/cli/run/02_comparing_across_runs/README.md +++ b/tools/Polygraphy/examples/cli/run/02_comparing_across_runs/README.md @@ -1,22 +1,28 @@ # Comparing Across Runs + +## Introduction + In some cases, it may be be useful to compare results across different runs of `polygraphy run` - for example, comparing between different machines, or comparing between multiple different versions of the same library. -For example, if you need to compare results between two different systems (let's call them -System A and System B), you can first save the results from System A: +In this example, we'll compare results generated on two different (hypothetical) systems: System A and System B. + + +## Running The Example +1. First save the results from System A: -```bash -polygraphy run identity.onnx --onnxrt \ - --save-outputs system_a_results.json -``` + ```bash + polygraphy run identity.onnx --onnxrt \ + --save-outputs system_a_results.json + ``` -Next, you can run the model on System B, and load the results saved from -System A to compare against: +2. Next, run the model on System B, and load the results saved from + System A to compare against: -```bash -polygraphy run identity.onnx --onnxrt \ - --load-outputs system_a_results.json -``` + ```bash + polygraphy run identity.onnx --onnxrt \ + --load-outputs system_a_results.json + ``` diff --git a/tools/Polygraphy/examples/cli/run/03_generating_a_comparison_script/README.md b/tools/Polygraphy/examples/cli/run/03_generating_a_comparison_script/README.md index ca7602cd..916e6a90 100644 --- a/tools/Polygraphy/examples/cli/run/03_generating_a_comparison_script/README.md +++ b/tools/Polygraphy/examples/cli/run/03_generating_a_comparison_script/README.md @@ -1,20 +1,27 @@ # Generating A Script For Advanced Comparisons + +## Introduction + For more advanced requirements, you may want to use the [API](../../../../polygraphy). Instead of writing a script from scratch, you can use `run`'s `--gen-script` option to create a Python script that you can use as a starting point. -First, generate a comparison script: -```bash -polygraphy run identity.onnx --trt --onnxrt \ - --gen-script=compare_trt_onnxrt.py -``` +## Running The Example + +1. Generate a comparison script: + + ```bash + polygraphy run identity.onnx --trt --onnxrt \ + --gen-script=compare_trt_onnxrt.py + ``` + + The generated script will do exactly what the `run` command would otherwise do. -The generated script will do exactly what the `run` command would otherwise do. -You are free to modify this script, and finally run it: +2. Run the comparison script, optionally after modifying it: -```bash -python3 compare_trt_onnxrt.py -``` + ```bash + python3 compare_trt_onnxrt.py + ``` diff --git a/tools/Polygraphy/examples/cli/run/04_defining_a_tensorrt_network_manually/README.md b/tools/Polygraphy/examples/cli/run/04_defining_a_tensorrt_network_manually/README.md index 072dfce3..6c51a9e9 100644 --- a/tools/Polygraphy/examples/cli/run/04_defining_a_tensorrt_network_manually/README.md +++ b/tools/Polygraphy/examples/cli/run/04_defining_a_tensorrt_network_manually/README.md @@ -1,5 +1,9 @@ # Defining A TensorRT Network Manually + +## Introduction + + In some cases, it can be useful to define a TensorRT network from scratch using the Python API, or modify a network created by other means (e.g. a parser). Normally, this would restrict you from using CLI tools, at least until you build an engine, since the network cannot be serialized @@ -14,18 +18,25 @@ and optionally parser, then you can provide your Python script in place of a mod In this example, the included `define_network.py` script parses an ONNX model and appends an identity layer to it. Since it returns the builder, network, and parser in a function called `load_network`, -we can build and run a TensorRT engine from it using just a single command: +we can build and run a TensorRT engine from it using just a single command. + + +## Running The Example + +1. Run the network defined in `define_network.py`: + + ```bash + polygraphy run --trt define_network.py --model-type=trt-network-script + ``` -```bash -polygraphy run --trt define_network.py --model-type=trt-network-script -``` +2. **[Optional]** Similarly, we can define a TensorRT builder configuration using a script, provided that we define + a function called `load_config` that accepts a builder and network and returns a builder configuration: -Similarly, we can define a TensorRT builder configuration using a script, provided that we define -a function called `load_config` that accepts a builder and network and returns a builder configuration: + ```bash + polygraphy run --trt define_network.py --model-type=trt-network-script --trt-config-script=create_config.py + ``` -```bash -polygraphy run --trt define_network.py --model-type=trt-network-script --trt-config-script=create_config.py -``` + Note that we could have defined both `load_network` and `load_config` in the same script. + In fact, we could have retrieved these functions from arbitrary scripts, or even modules. -Note that we could have defined both `load_network` and `load_config` in the same script. -In fact, we could have retrieved these functions from arbitrary scripts, or even modules. +*TIP: We can use the same approach with `polygraphy convert` to build, but not run, the engine.* diff --git a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/README.md b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/README.md new file mode 100644 index 00000000..faa400e5 --- /dev/null +++ b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/README.md @@ -0,0 +1,28 @@ +# Comparing With Custom Input Data + +## Introduction + +In some cases, we may want to run comparisons using custom input data. +Polygraphy provides multiple ways to do so, which are detailed [here](../../). + +In this example, we'll use a data loader script by defining a `load_data` function in a Python +script called `data_loader.py` and then use `polygraphy convert` to build the TensorRT engine. + +Since our model has dynamic shapes, we'll need to set up a TensorRT Optimization Profile. +For details on how we can do this via the command-line, +see [`convert` example 03](../..//convert/03_dynamic_shapes_in_tensorrt). +For simplicitly, we'll create a profile where `min` == `opt` == `max`. + +*NOTE: It is important that our optimization profile works with the shapes provided by our* + *custom data loader. In our very simple case, the data loader always generates inputs of* + *shape (1, 2, 28, 28), so we just need to ensure this falls within [`min`, `max`].* + +## Running The Example + +1. Run the model with TensorRT and ONNX-Runtime using custom input data: + + ```bash + polygraphy run dynamic_identity.onnx --trt --onnxrt \ + --data-loader-script data_loader.py \ + --trt-min-shapes X:[1,2,28,28] --trt-opt-shapes X:[1,2,28,28] --trt-max-shapes X:[1,2,28,28] + ``` diff --git a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/data_loader.py b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/data_loader.py new file mode 100644 index 00000000..fc2ee4f5 --- /dev/null +++ b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/data_loader.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Defines a `load_data` function that returns a generator yielding +feed_dicts so that this script can be used as the argument for +the --data-loader-script command-line parameter. +""" +import numpy as np + +INPUT_SHAPE = (1, 2, 28, 28) + + +def load_data(): + for _ in range(5): + yield {"x": np.ones(shape=INPUT_SHAPE, dtype=np.float32)} # Still totally real data diff --git a/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/dynamic_identity.onnx b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/dynamic_identity.onnx new file mode 100644 index 00000000..59f843a7 --- /dev/null +++ b/tools/Polygraphy/examples/cli/run/05_comparing_with_custom_data/dynamic_identity.onnx @@ -0,0 +1,15 @@ + backend_test:y + +XY"Identityonnx_dynamic_identityZ& +X! + + + +height +widthb& +Y! + + + +height +widthB diff --git a/tools/Polygraphy/examples/cli/surgeon/01_isolating_subgraphs/README.md b/tools/Polygraphy/examples/cli/surgeon/01_isolating_subgraphs/README.md index dfdb4161..466f527a 100644 --- a/tools/Polygraphy/examples/cli/surgeon/01_isolating_subgraphs/README.md +++ b/tools/Polygraphy/examples/cli/surgeon/01_isolating_subgraphs/README.md @@ -1,5 +1,8 @@ # Using Extract To Isolate A Subgraph + +## Introduction + The `surgeon extract` subtool can be used to extract a subgraph from a model with a single command. In this example, we'll extract a subgraph from a model that computes `Y = x0 + (a * x1 + b)`: @@ -15,34 +18,37 @@ attempt to automatically determine these. For inputs, we must specify both shape and data type, whereas outputs only require the data type - hence `--inputs` requires 2 `auto`s and `--outputs` requires only 1. -In our case, we can run: -```bash -polygraphy surgeon extract model.onnx \ - --inputs x1:auto:auto \ - --outputs add_out:auto \ - -o subgraph.onnx -``` +## Running The Example + +1. Extract the subgraph: + + ```bash + polygraphy surgeon extract model.onnx \ + --inputs x1:auto:auto \ + --outputs add_out:auto \ + -o subgraph.onnx + ``` -If we knew the shapes and/or data types, we could instead write, for example: + If we knew the shapes and/or data types, we could instead write, for example: -```bash -polygraphy surgeon extract model.onnx \ - --inputs x1:[1,3,224,224]:float32 \ - --outputs add_out:float32 \ - -o subgraph.onnx -``` + ```bash + polygraphy surgeon extract model.onnx \ + --inputs x1:[1,3,224,224]:float32 \ + --outputs add_out:float32 \ + -o subgraph.onnx + ``` -The resulting subgraph will look like this: + The resulting subgraph will look like this: -![./subgraph.png](./subgraph.png) + ![./subgraph.png](./subgraph.png) -At this point, the model is ready for use. You can use `inspect model` to confirm -whether it looks correct: +2. **[Optional]** At this point, the model is ready for use. You can use `inspect model` + to confirm whether it looks correct: -```bash -polygraphy inspect model subgraph.onnx --mode=basic -``` + ```bash + polygraphy inspect model subgraph.onnx --mode=basic + ``` ## A Note On `auto` diff --git a/tools/Polygraphy/examples/cli/surgeon/02_folding_constants/README.md b/tools/Polygraphy/examples/cli/surgeon/02_folding_constants/README.md index 457f619e..f1400728 100644 --- a/tools/Polygraphy/examples/cli/surgeon/02_folding_constants/README.md +++ b/tools/Polygraphy/examples/cli/surgeon/02_folding_constants/README.md @@ -1,5 +1,8 @@ # Using Sanitize To Fold Constants + +## Introduction + The `surgeon sanitize` subtool can be used to fold constants in graphs, remove unused nodes, and topologically sort nodes. In cases where shapes are statically known, it can also simplify subgraphs involving shape operations. @@ -9,21 +12,24 @@ where `a`, `b`, and `d` are constants: ![./model.png](./model.png) -To fold constants, we can run: -```bash -polygraphy surgeon sanitize model.onnx \ - --fold-constants \ - -o folded.onnx -``` +## Running The Example + +1. Fold constants with: + + ```bash + polygraphy surgeon sanitize model.onnx \ + --fold-constants \ + -o folded.onnx + ``` -This collapses `a`, `b`, and `d` into a constant tensor, and the resulting graph -computes `output = input + e`: + This collapses `a`, `b`, and `d` into a constant tensor, and the resulting graph + computes `output = input + e`: -![./folded.png](./folded.png) + ![./folded.png](./folded.png) -You can use `inspect model` to confirm whether it looks correct: +2. **[Optional]** You can use `inspect model` to confirm whether it looks correct: -```bash -polygraphy inspect model folded.onnx --mode=basic -``` + ```bash + polygraphy inspect model folded.onnx --mode=basic + ``` diff --git a/tools/Polygraphy/examples/dev/01_writing_cli_tools/README.md b/tools/Polygraphy/examples/dev/01_writing_cli_tools/README.md index ef180f7e..5f5a2543 100644 --- a/tools/Polygraphy/examples/dev/01_writing_cli_tools/README.md +++ b/tools/Polygraphy/examples/dev/01_writing_cli_tools/README.md @@ -17,11 +17,13 @@ group provided by Polygraphy. ## Running The Example 1. You can run the example tool from this directory. For example: + ```bash ./gen-data -o data.json --num-values 25 ``` 2. We can even inspect the generated data with `inspect data`: + ```bash polygraphy inspect data data.json -s ``` diff --git a/tools/Polygraphy/examples/dev/01_writing_cli_tools/gen-data b/tools/Polygraphy/examples/dev/01_writing_cli_tools/gen-data index 40fbed8f..66405e7e 100755 --- a/tools/Polygraphy/examples/dev/01_writing_cli_tools/gen-data +++ b/tools/Polygraphy/examples/dev/01_writing_cli_tools/gen-data @@ -39,14 +39,12 @@ class GenData(Tool): super().__init__() self.subscribe_args(DataLoaderArgs()) - # Next, we'll add custom arguments, beyond those provided by our subscribed # argument groups, by defining `add_parser_args`. def add_parser_args(self, parser): parser.add_argument("-o", "--output", help="Path at which to write generated data.", required=True) parser.add_argument("--num-values", help="The number of random values to generate.", default=1, type=int) - # Lastly, we implement `run`, which will implement the functionality of our tool. def run(self, args): # The DataLoaderArgs argument group provides a helper called `get_data_loader`, which @@ -54,11 +52,11 @@ class GenData(Tool): # See `polygraphy/tools/args/data_loader.py` for implementation details. # # To get data of the shape we want, we'll set the `input_metadata` parameter based on --num-values. - meta = TensorMetadata().add(name="data", dtype=np.float32, shape=(args.num_values, )) + meta = TensorMetadata().add(name="data", dtype=np.float32, shape=(args.num_values,)) data_loader = self.arg_groups[DataLoaderArgs].get_data_loader(meta) - # Generate the data using `list()` so that we respect the --iterations argument, - # then save it as JSON. + # data_loader behaves like a generator/iterable, so we can cast it to a `list` to + # generate all the data at once. save_json(list(data_loader), dest=args.output, description="randomly generated numbers") diff --git a/tools/Polygraphy/install.ps1 b/tools/Polygraphy/install.ps1 new file mode 100644 index 00000000..b439b014 --- /dev/null +++ b/tools/Polygraphy/install.ps1 @@ -0,0 +1,4 @@ +python.exe -m pip install wheel colored +python.exe setup.py bdist_wheel +$wheel_path = gci -Name dist +python.exe -m pip install --force-reinstall dist\$wheel_path diff --git a/tools/Polygraphy/polygraphy/README.md b/tools/Polygraphy/polygraphy/README.md index c3d3baf4..1583db90 100644 --- a/tools/Polygraphy/polygraphy/README.md +++ b/tools/Polygraphy/polygraphy/README.md @@ -12,21 +12,18 @@ - [Putting It All Together](#putting-it-all-together) - [Enabling PyTorch](#enabling-pytorch) - [Examples](#examples) +- [Python API Reference Documentation](#python-api-reference-documentation) + - [Building Python API Documentation Locally](#building-python-api-documentation-locally) ## Introduction -**IMPORTANT:** The Python API is still not completely stable, and minor but breaking changes -may be made in future versions. - The Polygraphy API consists broadly of two major components: [`Backend`s](#backends) and the [`Comparator`](#comparator). **NOTE:** To help you get started with the API, you can use the [`run`](./tools/run/) tool with the `--gen-script` option to auto-generate template scripts that use the Polygraphy API. -Also see the [Polygraphy API documentation](https://docs.nvidia.com/deeplearning/tensorrt/polygraphy/docs/index.html). - ## Backends @@ -191,3 +188,25 @@ In order to enable PyTorch, you need to provide three things to the `PytRunner`: ## Examples You can find complete code examples that use the Polygraphy Python API [here](../examples/api). + + +## Python API Reference Documentation + +For more details, see the [Polygraphy Python API reference documentation](https://docs.nvidia.com/deeplearning/tensorrt/polygraphy/docs/index.html). + +### Building Python API Documentation Locally + +To build the API documentation, first install required packages: + +```bash +python -m pip install -r docs/requirements.txt +``` + +and then use the `make` target to build docs: + +```bash +make docs +``` + +The HTML documentation will be generated under `build/docs` +To view the docs, open `build/docs/index.html` in a browser or HTML viewer. diff --git a/tools/Polygraphy/polygraphy/__init__.py b/tools/Polygraphy/polygraphy/__init__.py index e87340db..03d9ea19 100644 --- a/tools/Polygraphy/polygraphy/__init__.py +++ b/tools/Polygraphy/polygraphy/__init__.py @@ -1,3 +1,3 @@ import polygraphy.config -__version__ = "0.30.3" +__version__ = "0.31.1" diff --git a/tools/Polygraphy/polygraphy/backend/onnx/loader.py b/tools/Polygraphy/polygraphy/backend/onnx/loader.py index c6389558..70af98b4 100644 --- a/tools/Polygraphy/polygraphy/backend/onnx/loader.py +++ b/tools/Polygraphy/polygraphy/backend/onnx/loader.py @@ -75,7 +75,7 @@ def __enter__(self): model, _ = util.invoke_if_callable(self._model) self.USE_GS_GRAPH = isinstance(model, gs.Graph) if self.USE_GS_GRAPH: - self.graph = model + self.graph = model.copy() else: self.graph = gs.import_onnx(model) return self @@ -355,7 +355,7 @@ def run_const_fold_pass(model): model = infer_shapes(model) return model - if not mod.has_mod(onnxrt, "__version__"): + if not mod.has_mod(onnxrt): G_LOGGER.error( "ONNX-Runtime is not installed, so constant folding may be suboptimal or not work at all.\n" "Consider installing ONNX-Runtime: {:} -m pip install onnxruntime".format(sys.executable) @@ -377,7 +377,7 @@ def run_const_fold_pass(model): if not self.error_ok: raise G_LOGGER.warning( - "Constant folding pass failed. Skipping subsequent passes.\n" "Note: Error was:\n{:}".format(err) + "Constant folding pass failed. Skipping subsequent passes.\nNote: Error was:\n{:}".format(err) ) break else: @@ -462,13 +462,13 @@ def call_impl(self): if isinstance(model, onnx.ModelProto): model = shape_inference.infer_shapes(model) else: - with tempfile.NamedTemporaryFile(prefix="tmp_polygraphy_", suffix=".onnx") as f: - G_LOGGER.verbose("Writing shape-inferred model to: {:}".format(f.name)) - shape_inference.infer_shapes_path(model, f.name) - # When external_data_dir is unset, use the model's current directory - model = onnx_from_path( - f.name, external_data_dir=util.default(external_data_dir, os.path.dirname(model) or None) - ) + tmp_path = util.NamedTemporaryFile(prefix="tmp_polygraphy_", suffix=".onnx").name + G_LOGGER.verbose("Writing shape-inferred model to: {:}".format(tmp_path)) + shape_inference.infer_shapes_path(model, tmp_path) + # When external_data_dir is unset, use the model's current directory + model = onnx_from_path( + tmp_path, external_data_dir=util.default(external_data_dir, os.path.dirname(model) or None) + ) G_LOGGER.verbose("ONNX Shape Inference completed successfully") except Exception as err: if not self.error_ok: @@ -530,7 +530,9 @@ def get_tensor(name): def update_tensor(name, dtype, shape): tensor = get_tensor(name) - tensor.dtype, tensor.shape = dtype or tensor.dtype, shape or tensor.shape + # No need to update constants + if isinstance(tensor, gs.Variable): + tensor.dtype, tensor.shape = dtype or tensor.dtype, shape or tensor.shape return tensor def check_meta(name, dtype, shape, meta_type, needs_shape=True): @@ -642,6 +644,7 @@ def call_impl(self): "No external data will be written." ) + util.makedirs(self.path) onnx.save(model, self.path) return model diff --git a/tools/Polygraphy/polygraphy/backend/tf/__init__.py b/tools/Polygraphy/polygraphy/backend/tf/__init__.py index c48a6e5a..f6513e05 100644 --- a/tools/Polygraphy/polygraphy/backend/tf/__init__.py +++ b/tools/Polygraphy/polygraphy/backend/tf/__init__.py @@ -10,7 +10,7 @@ def set_tf_logging_level(sev): from polygraphy import mod tf = mod.lazy_import("tensorflow", version="<2.0") - if not mod.has_mod(tf, with_attr="__version__"): + if not mod.has_mod(tf): return if sev > G_LOGGER.WARNING: @@ -29,7 +29,7 @@ def set_tf_logging_level(sev): tf.compat.v1.logging.set_verbosity(tf_sev) os.environ["TF_CPP_MIN_LOG_LEVEL"] = tf_logging_level - G_LOGGER.register_callback(set_tf_logging_level) # Will be registered when this runner is imported. + G_LOGGER.register_callback(set_tf_logging_level) # Will be registered when this backend is imported. register_logger_callback() diff --git a/tools/Polygraphy/polygraphy/backend/trt/__init__.py b/tools/Polygraphy/polygraphy/backend/trt/__init__.py index f16e1624..fdfd15ad 100644 --- a/tools/Polygraphy/polygraphy/backend/trt/__init__.py +++ b/tools/Polygraphy/polygraphy/backend/trt/__init__.py @@ -13,7 +13,7 @@ def set_trt_logging_level(sev): from polygraphy import mod trt = mod.lazy_import("tensorrt") - if not mod.has_mod(trt, with_attr="__version__"): + if not mod.has_mod(trt): return if sev >= G_LOGGER.CRITICAL: @@ -27,7 +27,7 @@ def set_trt_logging_level(sev): else: get_trt_logger().min_severity = trt.Logger.VERBOSE - G_LOGGER.register_callback(set_trt_logging_level) # Will be registered when this runner is imported. + G_LOGGER.register_callback(set_trt_logging_level) # Will be registered when this backend is imported. register_logger_callback() diff --git a/tools/Polygraphy/polygraphy/backend/trt/algorithm_selector.py b/tools/Polygraphy/polygraphy/backend/trt/algorithm_selector.py index 77e65efe..ef0a274f 100644 --- a/tools/Polygraphy/polygraphy/backend/trt/algorithm_selector.py +++ b/tools/Polygraphy/polygraphy/backend/trt/algorithm_selector.py @@ -19,7 +19,6 @@ from polygraphy.common.interface import TypedDict from polygraphy.json import Decoder, Encoder, add_json_methods from polygraphy.logger import G_LOGGER, LogMode -from polygraphy.exception import PolygraphyException trt = mod.lazy_import("tensorrt") diff --git a/tools/Polygraphy/polygraphy/backend/trt/util.py b/tools/Polygraphy/polygraphy/backend/trt/util.py index 388eac81..f7fe3e4a 100644 --- a/tools/Polygraphy/polygraphy/backend/trt/util.py +++ b/tools/Polygraphy/polygraphy/backend/trt/util.py @@ -106,11 +106,15 @@ def try_add(layer_type, layer_cls): try_add("FILL", "IFillLayer") try_add("QUANTIZE", "IQuantizeLayer") try_add("DEQUANTIZE", "IDequantizeLayer") + try_add("CONDITION", "IConditionLayer") + try_add("CONDITIONAL_INPUT", "IIfConditionalInputLayer") + try_add("CONDITIONAL_OUTPUT", "IIfConditionalOutputLayer") return layer_class_mapping def np_dtype_from_trt(trt_dtype): + _ = mod.has_mod(np) # Force numpy to be imported return np.dtype(trt.nptype(trt_dtype)) @@ -341,7 +345,7 @@ def str_from_config(config): config_str += "{:20} | {:}\n".format("Tactic Sources", source_vals) with contextlib.suppress(AttributeError): - config_str += "{:20}: {:}\n".format("Safety Restricted", config.get_flag(trt.BuilderFlag.SAFETY_SCOPE)) + config_str += "{:20} | {:}\n".format("Safety Restricted", config.get_flag(trt.BuilderFlag.SAFETY_SCOPE)) if config.int8_calibrator: config_str += "{:20} | {:}\n".format("Calibrator", config.int8_calibrator) @@ -395,7 +399,7 @@ def get_input_metadata_from_profile(profile, network): "will use fixed input shapes (this is not necessarily an issue)." ) # Always use opt shape - input_metadata.add(name=tensor.name, dtype=trt.nptype(tensor.dtype), shape=shapes[1]) + input_metadata.add(name=tensor.name, dtype=np_dtype_from_trt(tensor.dtype), shape=shapes[1]) return input_metadata @@ -404,7 +408,7 @@ def add_binding_to_metadata(engine, binding, metadata, name_binding): # get all binding names in the runner metadata.add( name=engine[name_binding], - dtype=trt.nptype(engine.get_binding_dtype(binding)), + dtype=np_dtype_from_trt(engine.get_binding_dtype(binding)), shape=list(engine.get_binding_shape(binding)), ) diff --git a/tools/Polygraphy/polygraphy/comparator/compare.py b/tools/Polygraphy/polygraphy/comparator/compare.py index d6cdb2d0..f0f3ee59 100644 --- a/tools/Polygraphy/polygraphy/comparator/compare.py +++ b/tools/Polygraphy/polygraphy/comparator/compare.py @@ -170,7 +170,7 @@ def check_dict(dct, dict_name): check_dict(rtol, "the rtol dictionary") check_dict(atol, "the atol dictionary") - check_dict(check_error_stat, "the chcek_error_stat dictionary") + check_dict(check_error_stat, "the check_error_stat dictionary") # Returns whether the outputs match def check_outputs_match(out0, out0_name, out1, out1_name, per_out_rtol, per_out_atol, per_out_err_stat): diff --git a/tools/Polygraphy/polygraphy/comparator/struct.py b/tools/Polygraphy/polygraphy/comparator/struct.py index c1d77d03..19d31fc5 100644 --- a/tools/Polygraphy/polygraphy/comparator/struct.py +++ b/tools/Polygraphy/polygraphy/comparator/struct.py @@ -14,7 +14,6 @@ # limitations under the License. # -import tempfile from collections import OrderedDict from polygraphy import mod, util, config @@ -40,13 +39,13 @@ def __init__(self, arr): self.arr = None self.tmpfile = None if config.ARRAY_SWAP_THRESHOLD_MB >= 0 and arr.nbytes > (config.ARRAY_SWAP_THRESHOLD_MB << 20): - self.tmpfile = tempfile.NamedTemporaryFile(mode="w+", suffix=".json") + self.tmpfile = util.NamedTemporaryFile(suffix=".json") G_LOGGER.extra_verbose( "Evicting large array ({:.3f} MiB) from memory and saving to {:}".format( arr.nbytes / (1024.0 ** 2), self.tmpfile.name ) ) - save_json(arr, self.tmpfile) + save_json(arr, self.tmpfile.name) else: self.arr = arr @@ -61,7 +60,7 @@ def numpy(self): return self.arr assert self.tmpfile is not None, "Path and NumPy array cannot both be None!" - return load_json(self.tmpfile) + return load_json(self.tmpfile.name) @Encoder.register(LazyNumpyArray) diff --git a/tools/Polygraphy/polygraphy/config.py b/tools/Polygraphy/polygraphy/config.py index f720915b..fd1b2cac 100644 --- a/tools/Polygraphy/polygraphy/config.py +++ b/tools/Polygraphy/polygraphy/config.py @@ -18,27 +18,27 @@ INTERNAL_CORRECTNESS_CHECKS = bool(os.environ.get("POLYGRAPHY_INTERNAL_CORRECTNESS_CHECKS", "0") != "0") """ -Whether internal correctness checks are enabled. +bool: Whether internal correctness checks are enabled. This can be configured by setting the 'POLYGRAPHY_INTERNAL_CORRECTNESS_CHECKS' environment variable. """ AUTOINSTALL_DEPS = bool(os.environ.get("POLYGRAPHY_AUTOINSTALL_DEPS", "0") != "0") """ -Whether Polygraphy will automatically install required Python packages at runtime. +bool: Whether Polygraphy will automatically install required Python packages at runtime. This can be configured by setting the 'POLYGRAPHY_AUTOINSTALL_DEPS' environment variable. """ INSTALL_CMD = os.environ.get("POLYGRAPHY_INSTALL_CMD", "{:} -m pip install".format(sys.executable)).split() """ -The command to use to automatically install dependencies. Only relevant when AUTOINSTALL_DEPS -is enabled. Defaults to ``["python3", "-m", "pip", "install"]``. +List[str]: The command to use to automatically install dependencies. Only relevant when +AUTOINSTALL_DEPS is enabled. Defaults to ``["python", "-m", "pip", "install"]``. This can be configured by setting the 'POLYGRAPHY_INSTALL_CMD' environment variable to a string containing the command; for example: ``python3 -m pip install``. """ ARRAY_SWAP_THRESHOLD_MB = int(os.environ.get("POLYGRAPHY_ARRAY_SWAP_THRESHOLD_MB", "-1")) """ -The threshold, in megabytes, above which Polygraphy will evict a NumPy array from memory and swap it to disk. +int: The threshold, in megabytes, above which Polygraphy will evict a NumPy array from memory and swap it to disk. A negative value disables swapping and a value of 0 causes all arrays to be saved to disk. Disabled by default. This can be configured by setting the 'POLYGRAPHY_ARRAY_SWAP_THRESHOLD_MB' environment variable. diff --git a/tools/Polygraphy/polygraphy/cuda/cuda.py b/tools/Polygraphy/polygraphy/cuda/cuda.py index ecf555c4..1f71597b 100644 --- a/tools/Polygraphy/polygraphy/cuda/cuda.py +++ b/tools/Polygraphy/polygraphy/cuda/cuda.py @@ -14,6 +14,9 @@ # limitations under the License. # import ctypes +import glob +import os +import sys from polygraphy import func, mod, util from polygraphy.logger import G_LOGGER @@ -52,7 +55,22 @@ class Cuda(object): """ def __init__(self): - self.handle = ctypes.CDLL("libcudart.so") + self.handle = None + if sys.platform.startswith("win"): + cuda_paths = [os.environ.get("CUDA_PATH", "")] + cuda_paths += os.environ.get("PATH", "").split(os.path.pathsep) + cuda_paths = list(filter(lambda x: x, cuda_paths)) # Filter out empty paths (i.e. "") + + candidates = util.find_in_dirs("cudart64_*.dll", cuda_paths) + if not candidates: + G_LOGGER.critical( + "Could not find the CUDA runtime library.\nNote: Paths searched were:\n{:}".format(cuda_paths) + ) + + self.handle = ctypes.CDLL(candidates[0]) + else: + self.handle = ctypes.CDLL("libcudart.so") + if not self.handle: G_LOGGER.critical("Could not load the CUDA runtime library. Is it on your loader path?") diff --git a/tools/Polygraphy/polygraphy/json/serde.py b/tools/Polygraphy/polygraphy/json/serde.py index f781897d..ee5c8145 100644 --- a/tools/Polygraphy/polygraphy/json/serde.py +++ b/tools/Polygraphy/polygraphy/json/serde.py @@ -202,7 +202,7 @@ def try_register_numpy_json(func): @functools.wraps(func) def wrapped(*args, **kwargs): global NUMPY_REGISTRATION_SUCCESS - if not NUMPY_REGISTRATION_SUCCESS and mod.has_mod(np, "__version__"): + if not NUMPY_REGISTRATION_SUCCESS and mod.has_mod(np): # We define this alongside load_json/save_json so that it is guaranteed to be # imported before we need to encode/decode NumPy arrays. @Encoder.register(np.ndarray) @@ -273,7 +273,7 @@ def from_json(src): @mod.export_deprecated_alias( "pickle_save", - remove_in="0.31.0", + remove_in="0.32.0", use_instead="JSON serialization. " "This function has been migrated to use JSON and will NOT pickle the input object. " "Use save_json", @@ -293,7 +293,7 @@ def save_json(obj, dest, description=None): util.save_file(to_json(obj), dest, mode="w", description=description) -@mod.export_deprecated_alias("pickle_load", remove_in="0.31.0", use_instead="load_json") +@mod.export_deprecated_alias("pickle_load", remove_in="0.32.0", use_instead="load_json") @mod.export() @try_register_numpy_json def load_json(src, description=None): @@ -312,7 +312,7 @@ def load_json(src, description=None): return from_json(util.load_file(src, mode="r", description=description)) except UnicodeDecodeError: # This is a pickle file from Polygraphy 0.26.1 or older. - mod.warn_deprecated("pickle", use_instead="JSON", remove_in="0.31.0") + mod.warn_deprecated("pickle", use_instead="JSON", remove_in="0.32.0") G_LOGGER.critical( "It looks like you're trying to load a Pickle file.\nPolygraphy migrated to using JSON " "instead of Pickle in version 0.27.0 for security reasons.\nYou can convert your existing " diff --git a/tools/Polygraphy/polygraphy/logger/logger.py b/tools/Polygraphy/polygraphy/logger/logger.py index a5365d7d..6f0b1034 100644 --- a/tools/Polygraphy/polygraphy/logger/logger.py +++ b/tools/Polygraphy/polygraphy/logger/logger.py @@ -111,7 +111,7 @@ class Logger(object): EXTRA_VERBOSE: "medium_purple", VERBOSE: "light_magenta", INFO: None, - START: "light_blue", + START: "light_cyan", FINISH: "light_green", WARNING: "light_yellow", ERROR: "light_red", diff --git a/tools/Polygraphy/polygraphy/mod/importer.py b/tools/Polygraphy/polygraphy/mod/importer.py index 14744027..cb7e01ba 100644 --- a/tools/Polygraphy/polygraphy/mod/importer.py +++ b/tools/Polygraphy/polygraphy/mod/importer.py @@ -108,9 +108,9 @@ def install_mod(raise_error=True): ) status = sp.run(cmd) if status.returncode != 0: - G_LOGGER.log( - "Could not automatically install required package: {:}. Please install it manually.".format(pkg), - severity=G_LOGGER.CRITICAL if raise_error else G_LOGGER.WARNING, + log_func = G_LOGGER.critical if raise_error else G_LOGGER.warning + log_func( + "Could not automatically install required package: {:}. Please install it manually.".format(pkg) ) mod = importlib.import_module(name) @@ -123,14 +123,13 @@ def install_mod(raise_error=True): if config.AUTOINSTALL_DEPS: mod = install_mod() else: - G_LOGGER.error( + G_LOGGER.critical( "Module: {:} is required but could not be imported.\n" "You can try setting POLYGRAPHY_AUTOINSTALL_DEPS=1 in your environment variables " "to allow Polygraphy to automatically install missing packages.\n" "Note that this may cause existing packages to be overwritten - hence, it may be " "desirable to use a Python virtual environment or container. ".format(name) ) - raise # Auto-upgrade if necessary if version is not None and hasattr(mod, "__version__") and not _version_ok(mod.__version__, version): @@ -167,7 +166,7 @@ def __setattr__(self, name, value): return LazyModule() -def has_mod(lazy_mod, with_attr): +def has_mod(lazy_mod, with_attr="__version__"): """ Checks whether a module is available. @@ -210,8 +209,8 @@ def reset_sys_path(): stack.callback(reset_sys_path) - mod = importlib.import_module(modname) try: + mod = importlib.import_module(modname) return getattr(mod, name) except Exception as err: ext = os.path.splitext(path)[1] diff --git a/tools/Polygraphy/polygraphy/tools/args/model.py b/tools/Polygraphy/polygraphy/tools/args/model.py index 52cc2e1c..faf4e00c 100644 --- a/tools/Polygraphy/polygraphy/tools/args/model.py +++ b/tools/Polygraphy/polygraphy/tools/args/model.py @@ -141,6 +141,6 @@ def use_ext(ext_mapping): if self.model_type == "trt-network-script" and (not self.model_file or not self.model_file.endswith(".py")): G_LOGGER.critical( - "TensorRT network scripts must exist and have '.py' extensions. " + "TensorRT network scripts must exist and have '.py' extensions.\n" "Note: Provided network script path was: {:}".format(self.model_file) ) diff --git a/tools/Polygraphy/polygraphy/tools/args/onnx/loader.py b/tools/Polygraphy/polygraphy/tools/args/onnx/loader.py index 9eaed0f5..f5add269 100644 --- a/tools/Polygraphy/polygraphy/tools/args/onnx/loader.py +++ b/tools/Polygraphy/polygraphy/tools/args/onnx/loader.py @@ -26,12 +26,24 @@ @mod.export() class OnnxSaveArgs(BaseArgs): - def __init__(self, infer_shapes=False, output="output", short_opt="-o", required=False): + def __init__( + self, + infer_shapes=False, + output="output", + short_opt="-o", + required=False, + allow_ext_data_path=True, + custom_help=None, + default_output_path=None, + ): super().__init__() self._infer_shapes = infer_shapes self._output = output self._short_opt = short_opt self._required = required + self._allow_ext_data_path = allow_ext_data_path + self._custom_help = custom_help + self._default_output_path = default_output_path self.onnx_shape_inference_args = None def register(self, maker): @@ -44,17 +56,36 @@ def add_to_parser(self, parser): flag = "--{:}".format(self._output) short = self._short_opt or flag self.group.add_argument( - short, flag, help="Path to save the ONNX model", dest="save_onnx", default=None, required=self._required + short, + flag, + help=self._custom_help or "Path to save the ONNX model", + dest="save_onnx", + default=self._default_output_path, + required=self._required, ) + if self._allow_ext_data_path: + ext_data_params = { + "action": "append", + "nargs": "?", + } + else: + ext_data_params = { + "action": "append_const", + "const": "", + } + self.group.add_argument( "--save-external-data", help="Whether to save weight data in external file(s). " - "To use a non-default path, supply the desired path as an argument. This is always a relative path; " - "external data is always written to the same directory as the model. ", + + ( + "To use a non-default path, supply the desired path as an argument. This is always a relative path; " + "external data is always written to the same directory as the model. " + if self._allow_ext_data_path + else "" + ), default=None, - action="append", - nargs="?", + **ext_data_params, ) self.group.add_argument( "--external-data-size-threshold", @@ -248,7 +279,8 @@ def add_to_parser(self, parser): "--load-external-data", "--ext", dest="load_external_data", - help="Path to a directory containing external data for the model. ", + help="Path to a directory containing external data for the model. " + "Generally, this is only required if the external data is not stored in the model directory.", ) if self._output_prefix is not None: diff --git a/tools/Polygraphy/polygraphy/tools/base/tool.py b/tools/Polygraphy/polygraphy/tools/base/tool.py index 78826f00..1887eadd 100644 --- a/tools/Polygraphy/polygraphy/tools/base/tool.py +++ b/tools/Polygraphy/polygraphy/tools/base/tool.py @@ -96,7 +96,7 @@ def setup_parser(self, subparsers=None): try: self.add_parser_args(parser) except Exception as err: - G_LOGGER.warning( + G_LOGGER.internal_error( "Could not register tool argument parser for: {:}\nNote: Error was: {:}".format(self.name, err) ) return parser diff --git a/tools/Polygraphy/polygraphy/tools/debug/README.md b/tools/Polygraphy/polygraphy/tools/debug/README.md index 90c4a75d..bd4978ad 100644 --- a/tools/Polygraphy/polygraphy/tools/debug/README.md +++ b/tools/Polygraphy/polygraphy/tools/debug/README.md @@ -65,6 +65,7 @@ All the `debug` tools work on the same general principles: You can invoke it with the model and a command that can check intermediate models. The intermediate models will be written to `polygraphy_debug.onnx` by default. For example, to reduce a model with accuracy errors: + ```bash polygraphy debug reduce model.onnx -o reduced.onnx \ --check polygraphy run polygraphy_debug.onnx --onnxrt --trt diff --git a/tools/Polygraphy/polygraphy/tools/inspect/README.md b/tools/Polygraphy/polygraphy/tools/inspect/README.md index f15d4e4e..a86bc891 100644 --- a/tools/Polygraphy/polygraphy/tools/inspect/README.md +++ b/tools/Polygraphy/polygraphy/tools/inspect/README.md @@ -22,6 +22,7 @@ The `inspect` tool can be used to display information about supported types of f `polygraphy run --save-outputs/--save-inputs`. - [EXPERIMENTAL] `tactics` displays information about the contents of tactic replay files, like those written by `polygraphy run --save-tactics` +- `capability` partitions an ONNX model into one or more subgraphs that are supported or unsupported by TensorRT. ## Usage diff --git a/tools/Polygraphy/polygraphy/tools/inspect/inspect.py b/tools/Polygraphy/polygraphy/tools/inspect/inspect.py index b3497868..3ee61a37 100644 --- a/tools/Polygraphy/polygraphy/tools/inspect/inspect.py +++ b/tools/Polygraphy/polygraphy/tools/inspect/inspect.py @@ -14,7 +14,7 @@ # limitations under the License. # from polygraphy.tools.base import Tool -from polygraphy.tools.inspect.subtool import Data, Model, Tactics +from polygraphy.tools.inspect.subtool import Data, Model, Tactics, Capability class Inspect(Tool): @@ -33,6 +33,7 @@ def add_parser_args(self, parser): Model(), Data(), Tactics(), + Capability() ] for subtool in SUBTOOLS: diff --git a/tools/Polygraphy/polygraphy/tools/inspect/subtool/__init__.py b/tools/Polygraphy/polygraphy/tools/inspect/subtool/__init__.py index 7add71ef..8beea6b9 100644 --- a/tools/Polygraphy/polygraphy/tools/inspect/subtool/__init__.py +++ b/tools/Polygraphy/polygraphy/tools/inspect/subtool/__init__.py @@ -1,3 +1,4 @@ from polygraphy.tools.inspect.subtool.model import Model from polygraphy.tools.inspect.subtool.data import Data from polygraphy.tools.inspect.subtool.tactics import Tactics +from polygraphy.tools.inspect.subtool.capability import Capability diff --git a/tools/Polygraphy/polygraphy/tools/inspect/subtool/capability.py b/tools/Polygraphy/polygraphy/tools/inspect/subtool/capability.py new file mode 100644 index 00000000..02d5a4bd --- /dev/null +++ b/tools/Polygraphy/polygraphy/tools/inspect/subtool/capability.py @@ -0,0 +1,261 @@ +# +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from polygraphy import mod +from polygraphy.common.interface import TypedDict +from polygraphy.logger import G_LOGGER +from polygraphy.tools.args import ModelArgs, OnnxLoaderArgs, OnnxSaveArgs, OnnxShapeInferenceArgs +from polygraphy.tools.base import Tool + +common_backend = mod.lazy_import("polygraphy.backend.common") +gs = mod.lazy_import("onnx_graphsurgeon") +onnx_backend = mod.lazy_import("polygraphy.backend.onnx") +tools_util = mod.lazy_import("polygraphy.tools.util") +trt = mod.lazy_import("tensorrt") +trt_backend = mod.lazy_import("polygraphy.backend.trt") +trt_util = mod.lazy_import("polygraphy.backend.trt.util") +util = mod.lazy_import("polygraphy.util") + + +class UnsupportedNodeDict(TypedDict(lambda: str, lambda: dict)): + """ + An ordered dictionary that maps ops to error(s) encountered by TensorRT + while trying to parse them, and the range of node indices for the subgraphs + where these errors were encountered. + + More specifically, it is an `OrderedDict[str, Dict[str, List[Tuple[int]]]]`. + """ + + def add(self, op, err_string, node_range): + """ + Add a single entry for a single error in a subgraph. + + Multiple node ranges may apply to a single op/error combination. + + Args: + op (str): The name of the op that was unsupported. + err_string (str): The error encountered. + node_range (Union[Tuple[int], int]): + The start (inclusive) and end (exclusive) node indices of the subgraph + """ + if op not in self: + self[op] = {} + + if err_string not in self[op]: + self[op][err_string] = [] + + self[op][err_string].append(node_range) + + +def supports_model(path): + """ + Invokes the ONNX parser's `supports_model` on the specified model. + + Args: + path (str): The path to the ONNX model. + + Returns: + Tuple[bool, SubgraphCollection, parser]: + (1) Whether the model is supported. + (2) A List[Tuple[List[int], bool]] mapping groups of node indices to a boolean + indicating whether they are supported. + (3) The TensorRT ONNX parser instance. + """ + _, network = trt_backend.create_network() + parser = trt.OnnxParser(network, trt_backend.get_trt_logger()) + + try: + parser.supports_model + except AttributeError: + trt_util.fail_unavailable("supports_model in tensorrt.OnnxParser") + + supported, nodelists = parser.supports_model(common_backend.bytes_from_path(path), path) + return supported, nodelists, parser + + +def save_subgraph(onnx_save_args, graph, start, end, prefix="", use_tmp_file=False): + """ + Extracts a subgraph from the main graph and saves it to disk. + + Args: + graph (onnx_graphsurgeon.Graph): The parent/main graph. + start (int): The (inclusive) index of the start node. + end (int): The (exclusive) index of the end node. + prefix (str): The prefix for the model file name. + use_tmp_file (bool): + Whether the subgraph should be written to a temporary file instead of the output directory. + + Returns: + str: The full path to the ONNX model of the subgraph. + """ + subgraph_nodes = graph.nodes[start:end] + out_dict = {out.name: out for node in subgraph_nodes for out in node.outputs} + in_dict = {inp.name: inp for node in subgraph_nodes for inp in node.inputs} + + # Guess graph inputs/outputs by checking all output tensor names against all input tensor names, and vice-versa. + subgraph_inputs = tools_util.meta_from_gs_tensors([in_dict[k] for k in in_dict if k not in out_dict]) + subgraph_outputs = tools_util.meta_from_gs_tensors([out_dict[k] for k in out_dict if k not in in_dict]) + + subgraph = gs.export_onnx(onnx_backend.extract_subgraph(graph, subgraph_inputs, subgraph_outputs)) + + if use_tmp_file: + path = util.NamedTemporaryFile(prefix=prefix, suffix=".onnx").name + else: + # end is exclusive, so subtract one to make the model names friendlier. + path = os.path.join(onnx_save_args.path, "{:}_subgraph-nodes-{:}-{:}.onnx".format(prefix, start, end - 1)) + onnx_save_args.save_onnx(subgraph, path) + return path + + +def gen_results_summary(final_unsupported): + """ + Generates a results summary given the final unsupported nodes dictionary. + + Args: + final_unsupported (UnsupportedNodeDict): + The unsupported ops and corresponding errors and node index ranges. + + Returns: + str: A summary of all the unsupported ops in model, along with reasons and node index ranges. + """ + op_width = max(map(len, list(final_unsupported.keys()) + ["Operator "])) + reason_width = max(len(reason) for node_index_map in final_unsupported.values() for reason in node_index_map.keys()) + + summary = "===== Summary =====\n" + + header = "{:{op_width}}| {:7} | {:{reason_width}} | {:}\n".format( + "Operator", "Count", "Reason", "Nodes", op_width=op_width, reason_width=reason_width + ) + summary += header + "-" * len(header) + "\n" + + for op, node_index_map in final_unsupported.items(): + for reason, node_indices in node_index_map.items(): + summary += "{:{op_width}}| {:7} | {:{reason_width}} | {:}\n".format( + op, len(node_indices), reason, node_indices, op_width=op_width, reason_width=reason_width + ) + return summary + + +class Capability(Tool): + """ + Determine the capability of TensorRT to run an ONNX graph. Graph will be paritioned into supported and unsupported subgraphs. + """ + + def __init__(self): + super().__init__("capability") + self.subscribe_args(ModelArgs(model_required=True, inputs=None, model_type="onnx")) + self.subscribe_args(OnnxShapeInferenceArgs(default=True)) + self.subscribe_args(OnnxLoaderArgs(output_prefix=None)) + # Disallow ext data path since we're writing multiple models - otherwise, it'll be clobbered each time. + self.subscribe_args( + OnnxSaveArgs( + allow_ext_data_path=False, + custom_help="Directory to write out supported and unsupported subgraphs. " + "Defaults to 'polygraphy_capability_dumps' in the current directory", + default_output_path="polygraphy_capability_dumps", + ) + ) + + def run(self, args): + supported, nodelists, _ = supports_model(self.arg_groups[ModelArgs].model_file) + if supported: + G_LOGGER.info("Graph is fully supported by TensorRT; Will not generate subgraphs.") + return + + parent_graph = gs.import_onnx(self.arg_groups[OnnxLoaderArgs].load_onnx()) + + def partition(nodelists, offset): + """ + Partitions a set of subgraphs into supported and unsupported subgraphs. + + Args: + nodelists (List[Tuple[List[int], bool]]): + A list that maps node indices to a boolean indicating whether they + are supported by TensorRT. + + Returns: + List[List[int]]: + A list of subgraphs supported by TensorRT, each described by a list of node indices. + """ + supported_subgraphs = [] + for (node_indices, supported) in nodelists: + if supported: + supported_subgraphs.append([index + offset for index in node_indices]) + continue + + start = node_indices[0] + offset + end = node_indices[-1] + offset + 1 + subgraph_path = save_subgraph( + self.arg_groups[OnnxSaveArgs], + parent_graph, + start, + end, + prefix="intermediate_", + use_tmp_file=True, + ) + _, new_nodelists, _ = supports_model(subgraph_path) + # Recursively partition each unsupported subgraph. + supported_subgraphs += partition(new_nodelists, start) + + return supported_subgraphs + + supported_subgraphs = partition(nodelists, offset=0) + unsupported_node_dict = UnsupportedNodeDict() + + def save_unsupported_graph(start, end): + """ + Saves an unsupported subgraph, determines the error reason and adds it + to unsupported_node_dict + + Args: + start (int): The (inclusive) index of the start node. + end (int): The (exclusive) index of the end node. + """ + subgraph_path = save_subgraph(self.arg_groups[OnnxSaveArgs], parent_graph, start, end, "unsupported") + _, _, parser = supports_model(subgraph_path) + + err_string = ( + " | ".join([str(parser.get_error(err_idx)) for err_idx in range(parser.num_errors)]) or "UNKNOWN ERROR" + ) + unsupported_node_dict.add(parent_graph.nodes[start].op, err_string, [start, end]) + + # Log errors for all the unsupported graphs between supported subgraphs. + for index, subg_node_idxs in enumerate(supported_subgraphs): + save_subgraph( + self.arg_groups[OnnxSaveArgs], + parent_graph, + subg_node_idxs[0], + subg_node_idxs[-1] + 1, + "supported", + ) + + if index == 0 and subg_node_idxs[0] != 0: + save_unsupported_graph(0, subg_node_idxs[0]) + + if index == len(supported_subgraphs) - 1 and supported_subgraphs[-1][-1] != len(parent_graph.nodes) - 1: + save_unsupported_graph(subg_node_idxs[-1] + 1, len(parent_graph.nodes)) + + if index < len(supported_subgraphs) - 1: + next_subg_node_idxs = supported_subgraphs[index + 1] + save_unsupported_graph(subg_node_idxs[-1] + 1, next_subg_node_idxs[0]) + + summary = gen_results_summary(unsupported_node_dict) + + G_LOGGER.finish(summary) + util.save_file( + summary, os.path.join(self.arg_groups[OnnxSaveArgs].path, "results.txt"), "w", description="results" + ) diff --git a/tools/Polygraphy/polygraphy/tools/inspect/subtool/data.py b/tools/Polygraphy/polygraphy/tools/inspect/subtool/data.py index 9eba0cb9..fdc0da39 100644 --- a/tools/Polygraphy/polygraphy/tools/inspect/subtool/data.py +++ b/tools/Polygraphy/polygraphy/tools/inspect/subtool/data.py @@ -59,17 +59,23 @@ def str_from_iters(iters): out_str = "" for index, iter_result in enumerate(iters): iter_meta = meta_from_iter_result(iter_result) + indent = 1 if len(iters) > 1 and args.all: - out_str += util.indent_block("Iteration: {:} | ".format(index)) + out_str += util.indent_block("\n-- Iteration: {:}\n".format(index), indent - 1) + indent = 2 for name, arr in iter_result.items(): - out_str += "\n{:} {:} | Stats\n".format(name, iter_meta[name]) - out_str += util.indent_block(comp_util.str_output_stats(arr)) + "\n" + out_str += util.indent_block( + "\n{:} {:} | Stats: {:}".format(name, iter_meta[name], comp_util.str_output_stats(arr)), + indent - 1, + ) if args.histogram: - out_str += util.indent_block(comp_util.str_histogram(arr)) + "\n" + out_str += "\n{:}".format(util.indent_block(comp_util.str_histogram(arr), indent)) if args.show_values: - out_str += "{:}\n".format(util.indent_block(str(arr))) + out_str += "\n{:}".format(util.indent_block(str(arr), indent)) + if indent == 2: + out_str += "\n" if not args.all: break return out_str diff --git a/tools/Polygraphy/polygraphy/tools/run/run.py b/tools/Polygraphy/polygraphy/tools/run/run.py index 7c672d90..a15bb57c 100644 --- a/tools/Polygraphy/polygraphy/tools/run/run.py +++ b/tools/Polygraphy/polygraphy/tools/run/run.py @@ -73,7 +73,7 @@ def join_list(lst): new_list = copy.copy(lst) if len(new_list) > 1: new_list[-1] = "and {:}".format(new_list[-1]) - return ", ".join(new_list) + return ", ".join(new_list) if len(new_list) > 2 else " ".join(new_list) summary = "" @@ -94,7 +94,7 @@ def join_list(lst): } runners = [runner_names[runner] for runner in runners] summary += "between " if len(runners) > 1 else "using " - summary += join_list(runners) + summary += join_list(runners) + "." if load_results: summary += "\nIt will check against outputs stored in {:}\n".format(join_list(load_results)) diff --git a/tools/Polygraphy/polygraphy/tools/to_json/to_json.py b/tools/Polygraphy/polygraphy/tools/to_json/to_json.py index ca530288..02727233 100644 --- a/tools/Polygraphy/polygraphy/tools/to_json/to_json.py +++ b/tools/Polygraphy/polygraphy/tools/to_json/to_json.py @@ -21,12 +21,12 @@ class ToJSON(Tool): """ [TEMPORARY] Converts pickled data to JSON. - This tool will be removed in 0.31.0 since all future versions of Polygraphy + This tool will be removed in 0.32.0 since all future versions of Polygraphy will not use Pickle for serialization. """ def __init__(self): - mod.warn_deprecated("to-json", use_instead="JSON serialization", remove_in="0.31.0") + mod.warn_deprecated("to-json", use_instead="JSON serialization", remove_in="0.32.0") super().__init__(name="to-json") def add_parser_args(self, parser): diff --git a/tools/Polygraphy/polygraphy/util/util.py b/tools/Polygraphy/polygraphy/util/util.py index b577f00b..7ea5956e 100644 --- a/tools/Polygraphy/polygraphy/util/util.py +++ b/tools/Polygraphy/polygraphy/util/util.py @@ -14,8 +14,10 @@ # limitations under the License. # import contextlib +import glob import os import sys +import tempfile import zlib from collections import OrderedDict @@ -207,6 +209,80 @@ def unpack_args(args, num): ## +@mod.export() +class NamedTemporaryFile(object): + """ + Cross-platform temporary file implementation. Unlike tempfile.NamedTemporaryFile, + it can be opened multiple times without error on Windows. + """ + + def __init__(self, mode=None, prefix=None, suffix=None): + """ + Args: + mode (str): The mode to use when opening the file. + prefix (str): The prefix to use for the file path. + suffix (str): The suffix to use for the file path. + """ + self.mode = default(mode, "wb+") + prefix = default(prefix, "") + suffix = default(suffix, "") + + def rand_path(): + return os.path.join(tempfile.gettempdir(), "{:}{:}{:}".format(prefix, os.urandom(24).hex(), suffix)) + + # In the unlikely event the path exists, generate a new one. Only try 100 times so + # we don't end up in an infinite loop. + path = rand_path() + for _ in range(100): + if not os.path.exists(path): + break + path = rand_path() + else: + G_LOGGER.critical("Could not create a temporary file under: {:}".format(tempfile.gettempdir())) + + self.name = path # Use 'name' to be compatible with tempfile.NamedTemporaryFile + open(self.name, "x").close() + self._fhandle = None + + def __enter__(self): + """ + Opens the temporary file using the mode specified in the constructor. + + Returns: + file-like: The open file object. + """ + self._fhandle = open(self.name, self.mode) + return self._fhandle + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Closes the file handle. + """ + self._fhandle.close() + + +@mod.export() +def find_in_dirs(name_glob, dirs): + """ + Finds a file, optionally including a glob expression, in the specified directories. + + Args: + name_glob (str): + The name of the file, optionally including a glob expression. + Only the first match will be returned. + dirs (Sequence[str]): + The directories in which to search. + + Returns: + List[str]: The paths found, or an empty list if it could not be found. + """ + for dir_name in dirs: + paths = glob.glob(os.path.join(dir_name, name_glob)) + if paths: + return paths + return [] + + @mod.export() def get_file_size(src): """ @@ -263,6 +339,16 @@ def is_file_like(obj): return True +@mod.export() +def makedirs(path): + dir_path = os.path.dirname(path) + if dir_path: + dir_path = os.path.realpath(dir_path) + if not os.path.exists(dir_path): + G_LOGGER.verbose("{:} does not exist, creating now.".format(dir_path)) + os.makedirs(dir_path, exist_ok=True) + + @mod.export() def load_file(src, mode="rb", description=None): """ @@ -338,13 +424,7 @@ def save_file(contents, dest, mode="wb", description=None): "{:} bytes were written".format(content_bytes, bytes_written) ) else: - dir_path = os.path.dirname(dest) - if dir_path: - dir_path = os.path.realpath(dir_path) - if not os.path.exists(dir_path): - G_LOGGER.verbose("{:} does not exist, creating now.".format(dir_path)) - os.makedirs(dir_path, exist_ok=True) - + makedirs(dest) with open(dest, mode) as f: f.write(contents) return dest diff --git a/tools/Polygraphy/setup.py b/tools/Polygraphy/setup.py index 5bce5a16..3bc9aa84 100644 --- a/tools/Polygraphy/setup.py +++ b/tools/Polygraphy/setup.py @@ -15,8 +15,10 @@ # import os import sys + +from setuptools import find_packages, setup + import polygraphy -from setuptools import setup, find_packages ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) BIN_DIR = os.path.join(ROOT_DIR, "bin") diff --git a/tools/Polygraphy/tests/backend/common/test_loader.py b/tools/Polygraphy/tests/backend/common/test_loader.py index fd6ed71b..a07a2627 100644 --- a/tools/Polygraphy/tests/backend/common/test_loader.py +++ b/tools/Polygraphy/tests/backend/common/test_loader.py @@ -14,11 +14,11 @@ # limitations under the License. # -import tempfile from textwrap import dedent import pytest import tensorrt as trt +from polygraphy import util from polygraphy.backend.common import InvokeFromScript, invoke_from_script from polygraphy.exception import PolygraphyException @@ -40,7 +40,7 @@ def load_network(builder, network): """ ) - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f: + with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write(script) f.flush() @@ -63,7 +63,7 @@ def example(): """ ) - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f: + with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write(script) f.flush() diff --git a/tools/Polygraphy/tests/backend/onnx/test_loader.py b/tools/Polygraphy/tests/backend/onnx/test_loader.py index 29029de2..7e4cc1bf 100644 --- a/tools/Polygraphy/tests/backend/onnx/test_loader.py +++ b/tools/Polygraphy/tests/backend/onnx/test_loader.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os import tempfile import numpy as np import onnx_graphsurgeon as gs import pytest -from polygraphy import constants +from polygraphy import constants, util from polygraphy.backend.onnx import ( ConvertToFp16, FoldConstants, @@ -31,7 +32,6 @@ onnx_from_path, ) from polygraphy.common import TensorMetadata -from polygraphy.exception import PolygraphyException from polygraphy.logger import G_LOGGER from tests.helper import is_file_non_empty from tests.models.meta import ONNX_MODELS, TF_MODELS @@ -143,7 +143,9 @@ class TestFoldConstants: @pytest.mark.parametrize("copy", [True, False]) def test_basic(self, partitioning, fold_shapes, copy): original_model = onnx_from_path(ONNX_MODELS["const_foldable"].path) - loader = FoldConstants(original_model, partitioning=partitioning, fold_shapes=fold_shapes, copy=copy) + loader = FoldConstants( + original_model, partitioning=partitioning, fold_shapes=fold_shapes, copy=copy, error_ok=False + ) model = loader() assert len(original_model.graph.node) != 1 or not copy assert len(model.graph.node) == 1 @@ -151,13 +153,14 @@ def test_basic(self, partitioning, fold_shapes, copy): class TestSaveOnnx(object): def test_save_onnx(self): - with tempfile.NamedTemporaryFile() as outpath: - loader = SaveOnnx(OnnxFromPath(ONNX_MODELS["identity"].path), path=outpath.name) + with tempfile.TemporaryDirectory() as outdir: + outpath = os.path.join(outdir, "test", "nested") + loader = SaveOnnx(OnnxFromPath(ONNX_MODELS["identity"].path), path=outpath) loader() - assert is_file_non_empty(outpath.name) + assert is_file_non_empty(outpath) def test_external_data(self): - with tempfile.NamedTemporaryFile() as path, tempfile.NamedTemporaryFile() as data: + with util.NamedTemporaryFile() as path, util.NamedTemporaryFile() as data: model = OnnxFromPath(ONNX_MODELS["const_foldable"].path) loader = SaveOnnx(model, path.name, external_data_path=data.name, size_threshold=0) loader() @@ -206,15 +209,18 @@ def test_extract_onnx_model_no_output_meta(self, extract_model): def test_extract_onnx_gs_graph(self, extract_model): model, input_meta, output_meta = extract_model graph = gs.import_onnx(model) - graph = extract_subgraph(graph, input_meta, output_meta) - assert isinstance(graph, gs.Graph) - assert len(graph.nodes) == 1 + subgraph = extract_subgraph(graph, input_meta, output_meta) + # Make sure original graph isn't modified. + assert len(graph.nodes) == 2 - assert len(graph.inputs) == 1 - assert graph.inputs[0].name == "X" + assert isinstance(subgraph, gs.Graph) + assert len(subgraph.nodes) == 1 - assert len(graph.outputs) == 1 - assert graph.outputs[0].name == "identity_out_0" + assert len(subgraph.inputs) == 1 + assert subgraph.inputs[0].name == "X" + + assert len(subgraph.outputs) == 1 + assert subgraph.outputs[0].name == "identity_out_0" def test_extract_passes_no_input_shape(self, extract_model): model, input_meta, output_meta = extract_model diff --git a/tools/Polygraphy/tests/backend/tf/test_loader.py b/tools/Polygraphy/tests/backend/tf/test_loader.py index eac16af0..0fa55f5b 100644 --- a/tools/Polygraphy/tests/backend/tf/test_loader.py +++ b/tools/Polygraphy/tests/backend/tf/test_loader.py @@ -18,7 +18,7 @@ import pytest import tensorflow as tf -from polygraphy import constants +from polygraphy import constants, util from polygraphy.backend.tf import GraphFromFrozen, ModifyGraphOutputs, SaveGraph, graph_from_frozen from polygraphy.logger import G_LOGGER from tests.helper import is_file_non_empty @@ -58,7 +58,7 @@ def test_layerwise(self): class TestSaveGraph(object): def test_save_pb(self): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: tf_loader = SaveGraph(GraphFromFrozen(TF_MODELS["identity"].path), path=outpath.name) tf_loader() assert is_file_non_empty(outpath.name) diff --git a/tools/Polygraphy/tests/backend/tf/test_runner.py b/tools/Polygraphy/tests/backend/tf/test_runner.py index b1175531..0f5267bc 100644 --- a/tools/Polygraphy/tests/backend/tf/test_runner.py +++ b/tools/Polygraphy/tests/backend/tf/test_runner.py @@ -13,10 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import tempfile - import numpy as np import pytest +from polygraphy import util from polygraphy.backend.tf import SessionFromGraph, TfRunner from polygraphy.exception import PolygraphyException from tests.helper import is_file_non_empty @@ -39,7 +38,7 @@ def test_basic(self): @pytest.mark.skip(reason="Non-trivial to set up - requires CUPTI") def test_save_timeline(self): model = TF_MODELS["identity"] - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: with TfRunner(SessionFromGraph(model.loader), allow_growth=True, save_timeline=outpath.name) as runner: model.check_runner(runner) assert is_file_non_empty(outpath.name) diff --git a/tools/Polygraphy/tests/backend/trt/test_algorithm_selector.py b/tools/Polygraphy/tests/backend/trt/test_algorithm_selector.py index 1d0052bc..f5f63a29 100644 --- a/tools/Polygraphy/tests/backend/trt/test_algorithm_selector.py +++ b/tools/Polygraphy/tests/backend/trt/test_algorithm_selector.py @@ -14,12 +14,11 @@ # limitations under the License. # -import tempfile from collections import namedtuple import pytest import tensorrt as trt -from polygraphy import mod +from polygraphy import mod, util from polygraphy.backend.trt import Algorithm, TacticRecorder, TacticReplayData, TacticReplayer from polygraphy.exception import PolygraphyException @@ -160,11 +159,11 @@ def replay(request): in_replay_data = TacticReplayData().add(name, poly_algo) out_replay_data = TacticReplayData() if jsonify: - inpath = tempfile.NamedTemporaryFile("w") - in_replay_data.save(inpath) + inpath = util.NamedTemporaryFile("w") + in_replay_data.save(inpath.name) in_replay_data = inpath.name - outpath = tempfile.NamedTemporaryFile("r") + outpath = util.NamedTemporaryFile("r") out_replay_data = outpath.name yield context, poly_algo, trt_algo, in_replay_data, out_replay_data diff --git a/tools/Polygraphy/tests/backend/trt/test_calibrator.py b/tools/Polygraphy/tests/backend/trt/test_calibrator.py index 409d3b67..e88bbd96 100644 --- a/tools/Polygraphy/tests/backend/trt/test_calibrator.py +++ b/tools/Polygraphy/tests/backend/trt/test_calibrator.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import tempfile - import numpy as np import pytest import tensorrt as trt -from polygraphy import cuda, mod +from polygraphy import cuda, mod, util from polygraphy.backend.trt import ( Calibrator, CreateConfig, @@ -170,7 +168,7 @@ def test_calibrator_with_path_name_cache(self, identity_builder_network): builder, network = identity_builder_network data = [{"x": np.ones((1, 1, 2, 2), dtype=np.float32)}] - with tempfile.NamedTemporaryFile() as cache: + with util.NamedTemporaryFile() as cache: calibrator = Calibrator(data, cache=cache.name) create_config = CreateConfig(int8=True, calibrator=calibrator) with engine_from_network((builder, network), create_config): @@ -182,7 +180,7 @@ def test_calibrator_with_file_object_cache(self, identity_builder_network, mode) builder, network = identity_builder_network data = [{"x": np.ones((1, 1, 2, 2), dtype=np.float32)}] - with tempfile.NamedTemporaryFile(mode=mode) as cache: + with util.NamedTemporaryFile(mode=mode) as cache: calibrator = Calibrator(data, cache=cache) create_config = CreateConfig(int8=True, calibrator=calibrator) with engine_from_network((builder, network), create_config): @@ -210,7 +208,7 @@ def test_calibrator_rechecks_cache_on_reset(self, identity_builder_network): builder, network = identity_builder_network data = [{"x": np.ones((1, 1, 2, 2), dtype=np.float32)}] - with tempfile.NamedTemporaryFile(mode="wb+") as cache: + with util.NamedTemporaryFile(mode="wb+") as cache: calibrator = Calibrator(data, cache=cache.name) # First, populate the cache create_config = CreateConfig(int8=True, calibrator=calibrator) diff --git a/tools/Polygraphy/tests/backend/trt/test_loader.py b/tools/Polygraphy/tests/backend/trt/test_loader.py index d4456a0d..b928a171 100644 --- a/tools/Polygraphy/tests/backend/trt/test_loader.py +++ b/tools/Polygraphy/tests/backend/trt/test_loader.py @@ -14,11 +14,11 @@ # limitations under the License. # import contextlib -import tempfile +import sys import pytest import tensorrt as trt -from polygraphy import constants, mod +from polygraphy import constants, mod, util from polygraphy.backend.trt import ( Calibrator, CreateConfig, @@ -38,7 +38,7 @@ onnx_like_from_network, ) from polygraphy.comparator import DataLoader -from tests.helper import is_file_non_empty, get_file_size +from tests.helper import get_file_size, is_file_non_empty from tests.models.meta import ONNX_MODELS ## @@ -104,14 +104,16 @@ def test_can_load_libnvinfer_plugins(self): def get_plugin_names(): return [pc.name for pc in trt.get_plugin_registry().plugin_creator_list] - loader = LoadPlugins(plugins=["libnvinfer_plugin.so"]) + loader = LoadPlugins( + plugins=["nvinfer_plugin.dll" if sys.platform.startswith("win") else "libnvinfer_plugin.so"] + ) loader() assert get_plugin_names() class TestSerializedEngineLoader(object): def test_serialized_engine_loader_from_lambda(self, identity_engine): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: with open(outpath.name, "wb") as f, identity_engine.serialize() as buffer: f.write(buffer) @@ -325,7 +327,7 @@ def test_multiple_profiles(self, identity_builder_network): @pytest.mark.parametrize("path_mode", [True, False], ids=["path", "file-like"]) def test_timing_cache(self, identity_builder_network, path_mode): builder, network = identity_builder_network - with tempfile.NamedTemporaryFile() as cache: + with util.NamedTemporaryFile() as cache: loader = CreateConfig(load_timing_cache=cache.name if path_mode else cache) with loader(builder, network) as config: assert config.get_timing_cache() @@ -381,7 +383,7 @@ def test_can_build_with_calibrator(self, identity_builder_network): @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.0"), reason="Unsupported for TRT 7.2 and older") @pytest.mark.parametrize("path_mode", [True, False], ids=["path", "file-like"]) def test_timing_cache_generate_and_append(self, path_mode): - with tempfile.NamedTemporaryFile() as total_cache, tempfile.NamedTemporaryFile() as identity_cache: + with util.NamedTemporaryFile() as total_cache, util.NamedTemporaryFile() as identity_cache: def build_engine(model, cache): if not path_mode: @@ -429,7 +431,7 @@ def test_serialize_engine(self, identity_network): class TestSaveEngine(object): def test_save_engine(self, identity_network): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: engine_loader = SaveEngine(EngineFromNetwork(identity_network), path=outpath.name) with engine_loader(): assert is_file_non_empty(outpath.name) diff --git a/tools/Polygraphy/tests/logger/test_logger.py b/tools/Polygraphy/tests/logger/test_logger.py index 078390e7..55b32f97 100644 --- a/tools/Polygraphy/tests/logger/test_logger.py +++ b/tools/Polygraphy/tests/logger/test_logger.py @@ -14,8 +14,7 @@ # limitations under the License. # -import tempfile - +from polygraphy import util from polygraphy.logger.logger import Logger @@ -23,7 +22,7 @@ class TestLogger(object): def test_log_file(self): logger = Logger() - with tempfile.NamedTemporaryFile("w+") as log_file: + with util.NamedTemporaryFile("w+") as log_file: logger.log_file = log_file.name assert logger.log_file == log_file.name logger.info("Hello") diff --git a/tools/Polygraphy/tests/mod/test_importer.py b/tools/Polygraphy/tests/mod/test_importer.py index 90368dbd..c0df3fa8 100644 --- a/tools/Polygraphy/tests/mod/test_importer.py +++ b/tools/Polygraphy/tests/mod/test_importer.py @@ -16,12 +16,11 @@ import copy import sys -import tempfile from textwrap import dedent import pytest import tensorrt as trt -from polygraphy import mod +from polygraphy import mod, util from polygraphy.exception import PolygraphyException from polygraphy.mod.importer import _version_ok @@ -42,7 +41,7 @@ def load_network(builder, network): """ ) - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f: + with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write(script) f.flush() @@ -65,7 +64,7 @@ def example(): """ ) - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f: + with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write(script) f.flush() diff --git a/tools/Polygraphy/tests/models/capability.onnx b/tools/Polygraphy/tests/models/capability.onnx new file mode 100644 index 00000000..2f56089d Binary files /dev/null and b/tools/Polygraphy/tests/models/capability.onnx differ diff --git a/tools/Polygraphy/tests/models/meta.py b/tools/Polygraphy/tests/models/meta.py index cbf03081..6a268b37 100644 --- a/tools/Polygraphy/tests/models/meta.py +++ b/tools/Polygraphy/tests/models/meta.py @@ -140,4 +140,9 @@ def no_check_implemented(runner): check_runner=no_check_implemented, ext_data=model_path("ext_weights_same_dir"), ), + "capability": Model( + path=model_path("capability.onnx"), + LoaderType=BytesFromPath, + check_runner=no_check_implemented + ) } diff --git a/tools/Polygraphy/tests/requirements.txt b/tools/Polygraphy/tests/requirements.txt index bdb64bfd..7cb0f26b 100644 --- a/tools/Polygraphy/tests/requirements.txt +++ b/tools/Polygraphy/tests/requirements.txt @@ -1,12 +1,11 @@ -onnx==1.8.1 -tensorflow < 2.0 -onnx-tf < 1.5.0 -pytest -tf2onnx -onnxruntime==1.6.0 colored numpy -pytest-xdist -pytest-virtualenv onnx_graphsurgeon>=0.3.4 +onnx==1.9.0 onnxmltools +onnxruntime==1.8.1 +pytest +pytest-virtualenv +pytest-xdist +tensorflow<2.0 +tf2onnx diff --git a/tools/Polygraphy/tests/test_deps.py b/tools/Polygraphy/tests/test_deps.py index 4634c20b..1de3bc0a 100644 --- a/tools/Polygraphy/tests/test_deps.py +++ b/tools/Polygraphy/tests/test_deps.py @@ -16,11 +16,10 @@ import glob import os -import tempfile import pytest import tensorrt as trt -from polygraphy import mod +from polygraphy import mod, util from polygraphy.mod.importer import _version_ok from tests.models.meta import ONNX_MODELS @@ -71,7 +70,7 @@ def test_can_json_without_numpy(self, virtualenv_with_poly): TOOLS = { "run": [], "convert": [], - "inspect": ["data", "model", "tactics"], + "inspect": ["data", "model", "tactics", "capability"], "surgeon": ["extract", "insert", "sanitize"], "template": ["trt-network"], "debug": ["build", "precision", "diff-tactics", "reduce", "repeat"], @@ -109,7 +108,7 @@ class TestAutoinstallDeps(object): "--fold-constants", ONNX_MODELS["const_foldable"].path, "-o", - tempfile.NamedTemporaryFile().name, + util.NamedTemporaryFile().name, ], ], ) @@ -117,6 +116,9 @@ def test_can_automatically_install_deps(self, virtualenv_with_poly, cmd): if "--trt" in cmd and mod.version(trt.__version__) < mod.version("7.0"): pytest.skip("TRT 6 container has an old version of CUDA") + if "--trt" in cmd: + pytest.xfail("TensorRT 8.0.1.6 wheels are currently broken") + virtualenv_with_poly.env["POLYGRAPHY_AUTOINSTALL_DEPS"] = "1" POLYGRAPHY_BIN = os.path.join(ROOT_DIR, "bin", "polygraphy") output = virtualenv_with_poly.run(["python3", POLYGRAPHY_BIN] + cmd, capture=True) diff --git a/tools/Polygraphy/tests/test_examples.py b/tools/Polygraphy/tests/test_examples.py index 76333abb..2fe79516 100644 --- a/tools/Polygraphy/tests/test_examples.py +++ b/tools/Polygraphy/tests/test_examples.py @@ -17,6 +17,7 @@ import os import shutil import subprocess as sp +from textwrap import dedent import pytest import tensorrt as trt @@ -29,6 +30,12 @@ # Extract any ``` blocks from the README # Each block is stored as a separate string in the returned list def load_code_blocks_from_readme(readme, ignore_block): + with open(readme, "r") as f: + contents = f.read() + # Check that the README has all the expected sections. + assert "## Introduction" in contents, "All example READMEs should have an 'Introduction' section!" + assert "## Running The Example" in contents, "All example READMEs should have a 'Running The Example' section!" + def ignore_command(cmd): return "pip" in cmd @@ -49,7 +56,7 @@ def ignore_command(cmd): block.append(line.rstrip()) # commands is List[List[str]] - flatten and remove start/end markers: - commands = ["\n".join(block[1:-1]) for block in commands] + commands = [dedent("\n".join(block[1:-1])) for block in commands] return commands @@ -96,7 +103,7 @@ def __str__(self): API_EXAMPLES = [ Example(["api", "00_inference_with_tensorrt"], artifact_names=["identity.engine"]), Example(["api", "01_comparing_frameworks"], artifact_names=["inference_results.json"]), - Example(["api", "02_using_real_data"]), + Example(["api", "02_validating_on_a_dataset"]), Example(["api", "03_interoperating_with_tensorrt"]), Example(["api", "04_int8_calibration_in_tensorrt"], artifact_names=["identity-calib.cache"]), Example(["api", "05_using_tensorrt_network_api"]), @@ -126,6 +133,7 @@ def test_api_examples(example): Example(["cli", "run", "02_comparing_across_runs"], artifact_names=["system_a_results.json"]), Example(["cli", "run", "03_generating_a_comparison_script"], artifact_names=["compare_trt_onnxrt.py"]), Example(["cli", "run", "04_defining_a_tensorrt_network_manually"]), + Example(["cli", "run", "05_comparing_with_custom_data"]), # Convert Example(["cli", "convert", "01_int8_calibration_in_tensorrt"], artifact_names=["identity.engine"]), Example( diff --git a/tools/Polygraphy/tests/tools/args/onnx/test_loader.py b/tools/Polygraphy/tests/tools/args/onnx/test_loader.py index df1e124e..ed7f0475 100644 --- a/tools/Polygraphy/tests/tools/args/onnx/test_loader.py +++ b/tools/Polygraphy/tests/tools/args/onnx/test_loader.py @@ -18,6 +18,7 @@ import os import tempfile +from polygraphy import util from polygraphy.backend.onnx import onnx_from_path from polygraphy.tools.args import DataLoaderArgs, ModelArgs, OnnxLoaderArgs, OnnxSaveArgs, OnnxShapeInferenceArgs from polygraphy.tools.script import Script @@ -83,7 +84,7 @@ class TestOnnxSaveArgs(object): def test_external_data(self): model = onnx_from_path(ONNX_MODELS["const_foldable"].path) arg_group = ArgGroupTestHelper(OnnxSaveArgs(), deps=[ModelArgs(), OnnxLoaderArgs()]) - with tempfile.NamedTemporaryFile() as path, tempfile.NamedTemporaryFile() as data: + with util.NamedTemporaryFile() as path, util.NamedTemporaryFile() as data: arg_group.parse_args( ["-o", path.name, "--save-external-data", data.name, "--external-data-size-threshold=0"] ) @@ -95,7 +96,7 @@ def test_external_data(self): def test_size_threshold(self): model = onnx_from_path(ONNX_MODELS["const_foldable"].path) arg_group = ArgGroupTestHelper(OnnxSaveArgs(), deps=[ModelArgs(), OnnxLoaderArgs()]) - with tempfile.NamedTemporaryFile() as path, tempfile.NamedTemporaryFile() as data: + with util.NamedTemporaryFile() as path, util.NamedTemporaryFile() as data: arg_group.parse_args( ["-o", path.name, "--save-external-data", data.name, "--external-data-size-threshold=1024"] ) diff --git a/tools/Polygraphy/tests/tools/args/test_data_loader.py b/tools/Polygraphy/tests/tools/args/test_data_loader.py index d043f1e6..8333f3c8 100644 --- a/tools/Polygraphy/tests/tools/args/test_data_loader.py +++ b/tools/Polygraphy/tests/tools/args/test_data_loader.py @@ -14,7 +14,6 @@ # limitations under the License. # -import tempfile from textwrap import dedent import numpy as np @@ -79,7 +78,7 @@ def test_override_input_metadata(self): def test_data_loader_script(self): arg_group = ArgGroupTestHelper(DataLoaderArgs()) - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f: + with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write( dedent( """ diff --git a/tools/Polygraphy/tests/tools/args/test_logger.py b/tools/Polygraphy/tests/tools/args/test_logger.py index 4e98a615..572f9b30 100644 --- a/tools/Polygraphy/tests/tools/args/test_logger.py +++ b/tools/Polygraphy/tests/tools/args/test_logger.py @@ -18,7 +18,7 @@ import tempfile import pytest -from polygraphy.logger.logger import G_LOGGER +from polygraphy.logger import G_LOGGER from polygraphy.tools.args import LoggerArgs from tests.tools.args.helper import ArgGroupTestHelper diff --git a/tools/Polygraphy/tests/tools/args/trt/test_config.py b/tools/Polygraphy/tests/tools/args/trt/test_config.py index 15f6d59e..c1555776 100644 --- a/tools/Polygraphy/tests/tools/args/trt/test_config.py +++ b/tools/Polygraphy/tests/tools/args/trt/test_config.py @@ -14,14 +14,12 @@ # limitations under the License. # -from polygraphy.backend.trt.algorithm_selector import TacticReplayData -import tempfile from textwrap import dedent import pytest import tensorrt as trt -from polygraphy import mod -from polygraphy.backend.trt import TacticRecorder, TacticReplayer, create_network +from polygraphy import mod, util +from polygraphy.backend.trt import TacticRecorder, TacticReplayData, TacticReplayer, create_network from polygraphy.exception import PolygraphyException from polygraphy.tools.args import DataLoaderArgs, ModelArgs, TrtConfigArgs from tests.tools.args.helper import ArgGroupTestHelper @@ -68,7 +66,7 @@ def test_restricted_flags(self, trt_config_args): @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.0"), reason="Bugged before TRT 8") def test_tactic_replay(self, trt_config_args): - with tempfile.NamedTemporaryFile(suffix=".json") as f: + with util.NamedTemporaryFile(suffix=".json") as f: trt_config_args.parse_args(["--tactic-replay", f.name]) builder, network = create_network() @@ -86,7 +84,7 @@ def test_tactic_replay(self, trt_config_args): ], ) def test_tactics(self, trt_config_args, opt, cls): - with tempfile.NamedTemporaryFile("w+", suffix=".json") as f: + with util.NamedTemporaryFile("w+", suffix=".json") as f: if opt == "--load-tactics": TacticReplayData().save(f) @@ -185,7 +183,7 @@ def test_no_deps_profiles_int8(self): def test_config_script(self): arg_group = ArgGroupTestHelper(TrtConfigArgs()) - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f: + with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write( dedent( """ diff --git a/tools/Polygraphy/tests/tools/args/trt/test_loader.py b/tools/Polygraphy/tests/tools/args/trt/test_loader.py index eef2b4ad..9076aba6 100644 --- a/tools/Polygraphy/tests/tools/args/trt/test_loader.py +++ b/tools/Polygraphy/tests/tools/args/trt/test_loader.py @@ -14,11 +14,10 @@ # limitations under the License. # -import tempfile - import pytest import tensorrt as trt -from polygraphy.backend.trt import engine_bytes_from_network, network_from_onnx_path, create_network +from polygraphy import util +from polygraphy.backend.trt import create_network, engine_bytes_from_network, network_from_onnx_path from polygraphy.tools.args import ( ModelArgs, OnnxLoaderArgs, @@ -77,7 +76,7 @@ def test_build_engine_custom_network(self, engine_loader_args): assert engine[1] == "output" def test_load_serialized_engine(self, engine_loader_args): - with tempfile.NamedTemporaryFile() as f, engine_bytes_from_network( + with util.NamedTemporaryFile() as f, engine_bytes_from_network( network_from_onnx_path(ONNX_MODELS["identity"].path) ) as engine_bytes: f.write(engine_bytes) diff --git a/tools/Polygraphy/tests/tools/test_convert.py b/tools/Polygraphy/tests/tools/test_convert.py index 93253408..3e996e44 100644 --- a/tools/Polygraphy/tests/tools/test_convert.py +++ b/tools/Polygraphy/tests/tools/test_convert.py @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import tempfile from textwrap import dedent import onnx import pytest import tensorrt as trt -from polygraphy import mod +from polygraphy import mod, util from polygraphy.backend.common.loader import BytesFromPath from polygraphy.backend.trt.loader import EngineFromBytes from tests.models.meta import ONNX_MODELS, TF_MODELS @@ -28,12 +27,12 @@ class TestConvertToOnnx(object): def test_tf2onnx(self): - with tempfile.NamedTemporaryFile(suffix=".onnx") as outmodel: + with util.NamedTemporaryFile(suffix=".onnx") as outmodel: run_polygraphy_convert([TF_MODELS["identity"].path, "--model-type=frozen", "-o", outmodel.name]) assert onnx.load(outmodel.name) def test_fp_to_fp16(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_convert( [ONNX_MODELS["identity_identity"].path, "--convert-to=onnx", "--fp-to-fp16", "-o", outmodel.name] ) @@ -47,7 +46,7 @@ def check_engine(self, path): assert isinstance(engine, trt.ICudaEngine) def test_onnx_to_trt(self): - with tempfile.NamedTemporaryFile(suffix=".engine") as outmodel: + with util.NamedTemporaryFile(suffix=".engine") as outmodel: run_polygraphy_convert([ONNX_MODELS["identity"].path, "--model-type=onnx", "-o", outmodel.name]) self.check_engine(outmodel.name) @@ -55,7 +54,7 @@ def test_onnx_to_trt(self): mod.version(trt.__version__) < mod.version("8.0"), reason="Bug in older versions of TRT breaks this test" ) def test_tf_to_onnx_to_trt(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_convert( [TF_MODELS["identity"].path, "--model-type=frozen", "--convert-to=trt", "-o", outmodel.name] ) @@ -80,7 +79,7 @@ def load_config(config): """ ) - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f, tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile("w+", suffix=".py") as f, util.NamedTemporaryFile() as outmodel: f.write(script) f.flush() @@ -99,7 +98,7 @@ def load_config(config): self.check_engine(outmodel.name) def test_modify_onnx_outputs(self): - with tempfile.NamedTemporaryFile(suffix=".onnx") as outmodel: + with util.NamedTemporaryFile(suffix=".onnx") as outmodel: run_polygraphy_convert( [ONNX_MODELS["identity_identity"].path, "-o", outmodel.name, "--onnx-outputs", "mark", "all"] ) @@ -114,7 +113,7 @@ class TestConvertToOnnxLikeTrt(object): "model_name", ["identity", "empty_tensor_expand", "const_foldable", "and", "scan", "dim_param", "tensor_attr"] ) def test_onnx_to_trt_to_onnx_like(self, model_name): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_convert( [ONNX_MODELS[model_name].path, "--convert-to=onnx-like-trt-network", "-o", outmodel.name] ) diff --git a/tools/Polygraphy/tests/tools/test_inspect.py b/tools/Polygraphy/tests/tools/test_inspect.py index 4f3399d9..522b6a42 100644 --- a/tools/Polygraphy/tests/tools/test_inspect.py +++ b/tools/Polygraphy/tests/tools/test_inspect.py @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import glob +import os import tempfile from textwrap import dedent import pytest import tensorrt as trt -from polygraphy import mod +from polygraphy import mod, util from tests.models.meta import ONNX_MODELS, TF_MODELS from tests.tools.common import run_polygraphy_inspect, run_polygraphy_run @@ -32,7 +34,7 @@ def run_inspect_model(request): @pytest.fixture(scope="session") def identity_engine(): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--save-engine", outpath.name]) yield outpath.name @@ -297,6 +299,53 @@ def check_lines_match(actual, expected, should_check_line=lambda x: True): ] +# List[model, expected_files, expected_output] +TEST_CAPABILITY_CASES = [ + ( + "capability", + [ + "results.txt", + "supported_subgraph-nodes-1-1.onnx", + "supported_subgraph-nodes-3-3.onnx", + "unsupported_subgraph-nodes-0-0.onnx", + "unsupported_subgraph-nodes-2-2.onnx", + "unsupported_subgraph-nodes-4-4.onnx", + ], + """ + [I] ===== Summary ===== + Operator | Count | Reason | Nodes + ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + FAKE! | 2 | In node 0 (importFallbackPluginImporter): UNSUPPORTED_NODE: Assertion failed: creator && "Plugin not found, are the plugin name, version, and namespace correct?" | [[0, 1], [2, 3]] + FAKER! | 1 | In node 0 (importFallbackPluginImporter): UNSUPPORTED_NODE: Assertion failed: creator && "Plugin not found, are the plugin name, version, and namespace correct?" | [[4, 5]] + """, + ), + ( + "identity_identity", + [], + """ + Graph is fully supported by TensorRT; Will not generate subgraphs. + """, + ), +] + + +class TestCapability(object): + @pytest.mark.skipif( + mod.version(trt.__version__) < mod.version("8.0"), reason="supports_model API not available before TRT 8.0" + ) + @pytest.mark.parametrize("case", TEST_CAPABILITY_CASES, ids=lambda case: case[0]) + def test_capability(self, case): + model, expected_files, expected_summary = case + with tempfile.TemporaryDirectory() as outdir: + status = run_polygraphy_inspect( + ["capability", ONNX_MODELS[model].path, "-o", os.path.join(outdir, "subgraphs")], + ) + assert sorted(map(os.path.basename, glob.glob(os.path.join(outdir, "subgraphs", "**")))) == sorted( + expected_files + ) + assert dedent(expected_summary).strip() in status.stdout + + class TestInspectModel(object): @pytest.mark.parametrize("case", ONNX_CASES, ids=lambda case: "{:}-{:}".format(case[0], case[1])) def test_model_onnx(self, case): @@ -337,7 +386,7 @@ def load_network(builder, network): """ ) - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f: + with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write(script) f.flush() @@ -353,13 +402,13 @@ def test_model_tf_sanity(self, run_inspect_model): class TestInspectData(object): @pytest.mark.parametrize("opts", [[], ["--show-values"]]) def test_outputs(self, opts): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: run_polygraphy_run([ONNX_MODELS["identity"].path, "--onnxrt", "--save-outputs", outpath.name]) run_polygraphy_inspect(["data", outpath.name] + opts) @pytest.mark.parametrize("opts", [[], ["--show-values"]]) def test_inputs(self, opts): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: run_polygraphy_run([ONNX_MODELS["identity"].path, "--onnxrt", "--save-inputs", outpath.name]) run_polygraphy_inspect(["data", outpath.name] + opts) @@ -379,7 +428,7 @@ def test_inputs(self, opts): class TestInspectTactics(object): @pytest.mark.parametrize("case", TACTIC_REPLAY_CASES, ids=lambda case: case[0]) def test_show_tactics(self, case): - with tempfile.NamedTemporaryFile() as replay: + with util.NamedTemporaryFile() as replay: model_name, expected = case run_polygraphy_run([ONNX_MODELS[model_name].path, "--trt", "--save-tactics", replay.name]) diff --git a/tools/Polygraphy/tests/tools/test_run.py b/tools/Polygraphy/tests/tools/test_run.py index 85da5194..d8e727ef 100644 --- a/tools/Polygraphy/tests/tools/test_run.py +++ b/tools/Polygraphy/tests/tools/test_run.py @@ -24,7 +24,7 @@ import onnx import pytest import tensorrt as trt -from polygraphy import mod +from polygraphy import mod, util from polygraphy.json import load_json from tests.helper import get_file_size, is_file_non_empty from tests.models.meta import ONNX_MODELS, TF_MODELS @@ -33,7 +33,7 @@ class TestGen(object): def test_polygraphy_run_gen_script(self): - with tempfile.NamedTemporaryFile(mode="w") as f: + with util.NamedTemporaryFile(mode="w") as f: run_polygraphy_run(["--gen-script={:}".format(f.name), ONNX_MODELS["identity"].path]) with open(f.name, "r") as script: print(script.read()) @@ -73,13 +73,20 @@ def test_basic(self): run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt"]) def test_plugins(self): - run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--plugins", "libnvinfer_plugin.so"]) + run_polygraphy_run( + [ + ONNX_MODELS["identity"].path, + "--trt", + "--plugins", + "nvinfer_plugin.dll" if sys.platform.startswith("win") else "libnvinfer_plugin.so", + ] + ) def test_custom_outputs(self): run_polygraphy_run([ONNX_MODELS["identity_identity"].path, "--trt", "--trt-outputs", "identity_out_0"]) def test_layerwise_outputs(self): - with tempfile.NamedTemporaryFile() as outfile0: + with util.NamedTemporaryFile() as outfile0: run_polygraphy_run( [ ONNX_MODELS["identity_identity"].path, @@ -98,7 +105,7 @@ def test_layerwise_outputs(self): assert "identity_out_2" in result def test_exclude_outputs_with_layerwise(self): - with tempfile.NamedTemporaryFile() as outfile0: + with util.NamedTemporaryFile() as outfile0: run_polygraphy_run( [ ONNX_MODELS["identity_identity"].path, @@ -213,7 +220,7 @@ def test_multiple_profiles(self): ) def test_int8_calibration_cache(self): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: cmd = [ONNX_MODELS["identity"].path, "--trt", "--int8", "--calibration-cache", outpath.name] if mod.version(trt.__version__) >= mod.version("7.0"): cmd += ["--onnxrt"] @@ -252,14 +259,14 @@ def test_timing_cache(self): assert total_cache_size <= (const_foldable_cache_size + identity_cache_size) def test_save_load_engine(self): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--save-engine", outpath.name]) assert is_file_non_empty(outpath.name) run_polygraphy_run(["--trt", outpath.name, "--model-type=engine"]) @pytest.mark.skipif(mod.version(trt.__version__) < mod.version("8.0"), reason="Unsupported for TRT 7.2 and older") def test_tactic_replay(self): - with tempfile.NamedTemporaryFile() as tactic_replay: + with util.NamedTemporaryFile() as tactic_replay: run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--save-tactics", tactic_replay.name]) assert is_file_non_empty(tactic_replay.name) run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--load-tactics", tactic_replay.name]) @@ -269,7 +276,7 @@ def test_tactic_sources(self): run_polygraphy_run([ONNX_MODELS["identity"].path, "--trt", "--tactic-sources", "CUBLAS", "CUBLAS_LT"]) def test_data_loader_script_calibration(self): - with tempfile.NamedTemporaryFile("w+", suffix=".py") as f: + with util.NamedTemporaryFile("w+", suffix=".py") as f: f.write( dedent( """ @@ -291,7 +298,7 @@ def test_tf(self): run_polygraphy_run([TF_MODELS["identity"].path, "--tf", "--gpu-memory-fraction=0.5"]) def test_tf_save_pb(self): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: run_polygraphy_run( [TF_MODELS["identity"].path, "--tf", "--gpu-memory-fraction=0.5", "--save-pb", outpath.name] ) @@ -307,7 +314,7 @@ def test_tf_save_tensorboard(self): @pytest.mark.skip(reason="Non-trivial to set up - requires CUPTI") def test_tf_save_timeline(self): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: run_polygraphy_run( [TF_MODELS["identity"].path, "--tf", "--gpu-memory-fraction=0.5", "--save-timeline", outpath.name] ) @@ -325,7 +332,7 @@ def test_tf2onnxrt(self): run_polygraphy_run([TF_MODELS["identity"].path, "--onnxrt", "--model-type=frozen"]) def test_tf2onnx_save_onnx(self): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: run_polygraphy_run( [TF_MODELS["identity"].path, "--onnxrt", "--model-type=frozen", "--save-onnx", outpath.name] ) @@ -336,7 +343,7 @@ def test_onnx_rt(self): run_polygraphy_run([ONNX_MODELS["identity"].path, "--onnxrt"]) def test_onnx_rt_save_onnx(self): - with tempfile.NamedTemporaryFile() as outpath: + with util.NamedTemporaryFile() as outpath: run_polygraphy_run([ONNX_MODELS["identity"].path, "--onnxrt", "--save-onnx", outpath.name]) assert is_file_non_empty(outpath.name) assert onnx.load(outpath.name) @@ -345,7 +352,7 @@ def test_onnx_rt_custom_outputs(self): run_polygraphy_run([ONNX_MODELS["identity_identity"].path, "--onnxrt", "--onnx-outputs", "identity_out_0"]) def test_onnx_rt_layerwise_outputs(self): - with tempfile.NamedTemporaryFile() as outfile0: + with util.NamedTemporaryFile() as outfile0: run_polygraphy_run( [ ONNX_MODELS["identity_identity"].path, @@ -364,7 +371,7 @@ def test_onnx_rt_layerwise_outputs(self): assert "identity_out_2" in result def test_onnx_rt_exclude_outputs_with_layerwise(self): - with tempfile.NamedTemporaryFile() as outfile0: + with util.NamedTemporaryFile() as outfile0: run_polygraphy_run( [ ONNX_MODELS["identity_identity"].path, @@ -464,7 +471,7 @@ def test_save_load_outputs(self, tmp_path): ) # Make sure it actually compared stuff. def test_save_load_inputs(self): - with tempfile.NamedTemporaryFile() as infile0, tempfile.NamedTemporaryFile() as infile1: + with util.NamedTemporaryFile() as infile0, util.NamedTemporaryFile() as infile1: run_polygraphy_run([ONNX_MODELS["identity"].path, "--onnxrt", "--save-input-data", infile0.name]) run_polygraphy_run( [ diff --git a/tools/Polygraphy/tests/tools/test_surgeon.py b/tools/Polygraphy/tests/tools/test_surgeon.py index bf17cead..4ef4004d 100644 --- a/tools/Polygraphy/tests/tools/test_surgeon.py +++ b/tools/Polygraphy/tests/tools/test_surgeon.py @@ -19,6 +19,7 @@ import onnx import onnx_graphsurgeon as gs import pytest +from polygraphy import util from tests.helper import is_file_non_empty from tests.models.meta import ONNX_MODELS from tests.tools.common import run_polygraphy_run, run_polygraphy_surgeon @@ -34,7 +35,7 @@ def was_shape_inference_run(status): class TestSurgeonExtract(object): def test_no_shape_inference_if_has_metadata(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: status = run_polygraphy_surgeon( ["extract", ONNX_MODELS["identity_identity"].path, "-o", outmodel.name, "--inputs", "X:auto:auto"] ) @@ -42,7 +43,7 @@ def test_no_shape_inference_if_has_metadata(self): assert not was_shape_inference_run(status) def test_onnx_shape_inference_if_no_metadata(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: status = run_polygraphy_surgeon( [ "extract", @@ -57,7 +58,7 @@ def test_onnx_shape_inference_if_no_metadata(self): assert was_shape_inference_run(status) def test_fallback_shape_inference_no_onnx_shape_inference(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: status = run_polygraphy_surgeon( [ "extract", @@ -75,7 +76,7 @@ def test_fallback_shape_inference_no_onnx_shape_inference(self): assert not was_shape_inference_run(status) def test_force_fallback_shape_inference_will_override_model_shapes(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon( [ "extract", @@ -96,7 +97,7 @@ def test_force_fallback_shape_inference_will_override_model_shapes(self): assert tuple(graph.outputs[0].shape) == (1, 2, 1, 1) def test_sanity_dim_param(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon(["extract", ONNX_MODELS["dim_param"].path, "-o", outmodel.name]) onnx_model_sanity_check(outmodel.name) @@ -115,7 +116,7 @@ def check_insert_model(self, path, expected_node_ops, expected_graph_input_names def test_insert_at_tensor(self): # Insert a new node in between existing nodes without replacing any existing nodes. - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon( [ "insert", @@ -131,7 +132,7 @@ def test_insert_at_tensor(self): def test_graph_output(self): # FakeOp output tensor should be marked as a graph output. Name should be preserved - identity_out_2 - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon( [ "insert", @@ -146,7 +147,7 @@ def test_graph_output(self): self.check_insert_model(outmodel.name, ["Identity", "Identity", "FakeOp"], ["X"], ["identity_out_2"]) def test_at_graph_input(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon( [ "insert", @@ -163,7 +164,7 @@ def test_at_graph_input(self): # When a specified input tensor is used by multiple other nodes, it should not be # disconnected from other nodes. def test_multi_use_input(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon( [ "insert", @@ -186,7 +187,7 @@ def test_multi_use_input(self): assert other_branch_node.input == ["add_out_4"] def test_with_attributes(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: # str_attr='0' should be interpreted as a string, not an int # float_attr=0.0 should be interpreted as a float, not an int # int_attr=0 should be interpreted as an int @@ -242,7 +243,7 @@ class TestSurgeonSanitize(object): @pytest.mark.parametrize("fold_shapes", [None, "--no-fold-shapes"]) @pytest.mark.parametrize("partitioning", [None, "basic", "recursive"]) def test_fold_constants(self, no_per_pass_shape_inf, partitioning, fold_shapes): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: cmd = ["sanitize", ONNX_MODELS["const_foldable"].path, "-o", outmodel.name, "--fold-constants"] if fold_shapes: cmd += [fold_shapes] @@ -257,7 +258,7 @@ def test_fold_constants(self, no_per_pass_shape_inf, partitioning, fold_shapes): assert len(model.graph.node) == 1 def test_fold_constants_single_pass(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: status = run_polygraphy_surgeon( [ "sanitize", @@ -278,7 +279,7 @@ def test_fold_constants_single_pass(self): @pytest.mark.parametrize("new_dim", [1, 2, 3]) def test_override_shapes(self, new_dim): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: cmd = [ "sanitize", ONNX_MODELS["dynamic_identity"].path, @@ -299,7 +300,7 @@ def test_override_shapes(self, new_dim): assert shape == [1, 2, new_dim, new_dim] def test_override_shapes_no_clear_const_tensors_meta(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon( [ "sanitize", @@ -311,7 +312,7 @@ def test_override_shapes_no_clear_const_tensors_meta(self): ) def test_override_shapes_partial_inputs(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon( [ "sanitize", @@ -326,7 +327,7 @@ def test_override_shapes_partial_inputs(self): assert model.graph.input[0].type.tensor_type.shape.dim[3].dim_param == "width" def test_override_shapes_no_reorder(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: run_polygraphy_surgeon( [ "sanitize", @@ -343,7 +344,7 @@ def test_override_shapes_no_reorder(self): assert model.graph.input[1].name == "Y0" def test_modify_onnx_outputs(self): - with tempfile.NamedTemporaryFile(suffix=".onnx") as outmodel: + with util.NamedTemporaryFile(suffix=".onnx") as outmodel: run_polygraphy_surgeon( ["sanitize", ONNX_MODELS["identity_identity"].path, "-o", outmodel.name, "--outputs", "mark", "all"] ) @@ -352,7 +353,7 @@ def test_modify_onnx_outputs(self): assert len(model.graph.output) == 2 def test_cleanup(self): - with tempfile.NamedTemporaryFile(suffix=".onnx") as outmodel: + with util.NamedTemporaryFile(suffix=".onnx") as outmodel: run_polygraphy_surgeon( [ "sanitize", @@ -394,7 +395,7 @@ def test_external_data(self): assert run_polygraphy_run([outmodel, "--onnxrt", "--external-data-dir", outdir]) def test_force_fallback_shape_inference_will_override_model_shapes(self): - with tempfile.NamedTemporaryFile() as outmodel: + with util.NamedTemporaryFile() as outmodel: status = run_polygraphy_surgeon( [ "sanitize", diff --git a/tools/Polygraphy/tests/tools/test_template.py b/tools/Polygraphy/tests/tools/test_template.py index dd3cd520..37f373b2 100644 --- a/tools/Polygraphy/tests/tools/test_template.py +++ b/tools/Polygraphy/tests/tools/test_template.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import tempfile import tensorrt as trt +from polygraphy import util from polygraphy.backend.common.loader import InvokeFromScript from tests.models.meta import ONNX_MODELS from tests.tools.common import run_polygraphy_template @@ -23,7 +23,7 @@ class TestTrtNetwork(object): def test_no_model_file(self): - with tempfile.NamedTemporaryFile("w+", suffix=".py") as template: + with util.NamedTemporaryFile("w+", suffix=".py") as template: run_polygraphy_template(["trt-network", "-o", template.name]) load_network = InvokeFromScript(template.name, "load_network") @@ -33,7 +33,7 @@ def test_no_model_file(self): assert isinstance(network, trt.INetworkDefinition) def test_with_model_file(self): - with tempfile.NamedTemporaryFile("w+", suffix=".py") as template: + with util.NamedTemporaryFile("w+", suffix=".py") as template: run_polygraphy_template(["trt-network", ONNX_MODELS["identity"].path, "-o", template.name]) load_network = InvokeFromScript(template.name, "load_network") diff --git a/tools/Polygraphy/tests/util/test_serde.py b/tools/Polygraphy/tests/util/test_serde.py index 42f197ea..5516f806 100644 --- a/tools/Polygraphy/tests/util/test_serde.py +++ b/tools/Polygraphy/tests/util/test_serde.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import tempfile import numpy as np import pytest import tensorrt as trt -from polygraphy import constants +from polygraphy import constants, util from polygraphy.backend.trt import Algorithm, TacticReplayData from polygraphy.comparator import IterationResult, RunResults from polygraphy.exception import PolygraphyException @@ -126,7 +125,7 @@ def test_to_from_json(self, obj): @pytest.mark.parametrize("obj", JSONABLE_CASES) def test_save_load(self, obj): - with tempfile.NamedTemporaryFile("w+") as f: + with util.NamedTemporaryFile("w+") as f: obj.save(f) decoded = type(obj).load(f) assert decoded == obj diff --git a/tools/Polygraphy/tests/util/test_util.py b/tools/Polygraphy/tests/util/test_util.py index 79e01b4f..899b7e03 100644 --- a/tools/Polygraphy/tests/util/test_util.py +++ b/tools/Polygraphy/tests/util/test_util.py @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os +import tempfile +import random + import numpy as np import pytest from polygraphy import util @@ -122,3 +126,18 @@ def test_unpack_args(case): def test_unique_list(case): lst, expected = case assert util.unique_list(lst) == expected + + +def test_find_in_dirs(): + with tempfile.TemporaryDirectory() as topdir: + dirs = list(map(lambda x: os.path.join(topdir, x), ["test0", "test1", "test2", "test3", "test4"])) + for subdir in dirs: + os.makedirs(subdir) + + path_dir = random.choice(dirs) + path = os.path.join(path_dir, "cudart64_11.dll") + + with open(path, "w") as f: + f.write("This file should be found by find_in_dirs") + + assert util.find_in_dirs("cudart64_*.dll", dirs) == [path]