diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index b5bf5ff6e7f..ad3237a7fce 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -44,9 +44,8 @@ jobs: cache: pip - name: cpuinfo run: cat /proc/cpuinfo - - name: Install NNCF and test requirements + - name: Install test requirements run: | - pip install -e . pip install -r tests/cross_fw/examples/requirements.txt - name: Print installed modules run: pip list diff --git a/.gitignore b/.gitignore index 1328d5a0416..9ca07758d16 100644 --- a/.gitignore +++ b/.gitignore @@ -119,23 +119,19 @@ nncf_debug/ # NNCF examples examples/torch/object_detection/eval/ -examples/post_training_quantization/onnx/mobilenet_v2/mobilenet_v2_* -examples/post_training_quantization/openvino/mobilenet_v2/mobilenet_v2_* -examples/post_training_quantization/tensorflow/mobilenet_v2/mobilenet_v2_* -examples/post_training_quantization/torch/mobilenet_v2/mobilenet_v2_* -examples/post_training_quantization/torch/ssd300_vgg16/ssd300_vgg16_* -examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/stfpm_* -examples/post_training_quantization/openvino/yolov8/yolov8n* -examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/yolov8n* -examples/**/runs/** -examples/**/results/** -examples/llm_compression/openvino/tiny_llama_find_hyperparams/statistics -compressed_graph.dot -original_graph.dot -datasets/** +examples/**/*.xml +examples/**/*.bin +examples/**/*.pt +examples/**/*.onnx +examples/**/statistics +examples/**/runs +examples/**/results +examples/**/metrics.json # Tests tests/**/runs/** tests/**/tmp*/** open_model_zoo/ nncf-tests.xml +compressed_graph.dot +original_graph.dot diff --git a/examples/post_training_quantization/onnx/mobilenet_v2/main.py b/examples/post_training_quantization/onnx/mobilenet_v2/main.py index 39a8e8c8465..c6516bfd19f 100755 --- a/examples/post_training_quantization/onnx/mobilenet_v2/main.py +++ b/examples/post_training_quantization/onnx/mobilenet_v2/main.py @@ -12,7 +12,7 @@ import re import subprocess from pathlib import Path -from typing import List, Optional +from typing import List import numpy as np import onnx @@ -20,28 +20,29 @@ import torch from fastdownload import FastDownload from fastdownload import download_url +from rich.progress import track from sklearn.metrics import accuracy_score from torchvision import datasets from torchvision import transforms -from tqdm import tqdm import nncf ROOT = Path(__file__).parent.resolve() MODEL_URL = "https://huggingface.co/alexsu52/mobilenet_v2_imagenette/resolve/main/mobilenet_v2_imagenette.onnx" DATASET_URL = "https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-320.tgz" -DATASET_PATH = "~/.cache/nncf/datasets" -MODEL_PATH = "~/.cache/nncf/models" +DATASET_PATH = Path().home() / ".cache" / "nncf" / "datasets" +MODEL_PATH = Path().home() / ".cache" / "nncf" / "models" DATASET_CLASSES = 10 def download_dataset() -> Path: - downloader = FastDownload(base=DATASET_PATH, archive="downloaded", data="extracted") + downloader = FastDownload(base=DATASET_PATH.as_posix(), archive="downloaded", data="extracted") return downloader.get(DATASET_URL) def download_model() -> Path: - return download_url(MODEL_URL, Path(MODEL_PATH).resolve()) + MODEL_PATH.mkdir(exist_ok=True, parents=True) + return download_url(MODEL_URL, MODEL_PATH.resolve()) def validate(path_to_model: Path, validation_loader: torch.utils.data.DataLoader) -> float: @@ -51,7 +52,7 @@ def validate(path_to_model: Path, validation_loader: torch.utils.data.DataLoader compiled_model = ov.compile_model(path_to_model, device_name="CPU") output = compiled_model.outputs[0] - for images, target in tqdm(validation_loader): + for images, target in track(validation_loader, description="Validating"): pred = compiled_model(images)[output] predictions.append(np.argmax(pred, axis=1)) references.append(target) @@ -61,13 +62,17 @@ def validate(path_to_model: Path, validation_loader: torch.utils.data.DataLoader return accuracy_score(predictions, references) -def run_benchmark(path_to_model: Path, shape: Optional[List[int]] = None, verbose: bool = True) -> float: - command = f"benchmark_app -m {path_to_model} -d CPU -api async -t 15" - if shape is not None: - command += f' -shape [{",".join(str(x) for x in shape)}]' - cmd_output = subprocess.check_output(command, shell=True) # nosec - if verbose: - print(*str(cmd_output).split("\\n")[-9:-1], sep="\n") +def run_benchmark(path_to_model: Path, shape: List[int]) -> float: + command = [ + "benchmark_app", + "-m", path_to_model.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "15", + "-shape", str(shape), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + print(*cmd_output.splitlines()[-8:], sep="\n") match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) return float(match.group(1)) @@ -136,9 +141,9 @@ def transform_fn(data_item): print(f"[2/7] Save INT8 model: {int8_model_path}") print("[3/7] Benchmark FP32 model:") -fp32_fps = run_benchmark(fp32_model_path, shape=[1, 3, 224, 224], verbose=True) +fp32_fps = run_benchmark(fp32_model_path, shape=[1, 3, 224, 224]) print("[4/7] Benchmark INT8 model:") -int8_fps = run_benchmark(int8_model_path, shape=[1, 3, 224, 224], verbose=True) +int8_fps = run_benchmark(int8_model_path, shape=[1, 3, 224, 224]) print("[5/7] Validate ONNX FP32 model in OpenVINO:") fp32_top1 = validate(fp32_model_path, val_loader) diff --git a/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/deploy.py b/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/deploy.py index b1b5a7a8c9f..d2898f40e17 100644 --- a/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/deploy.py +++ b/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/deploy.py @@ -16,10 +16,10 @@ import openvino as ov import torch -from tqdm import tqdm +from rich.progress import track from ultralytics.cfg import get_cfg -from ultralytics.engine.validator import BaseValidator as Validator from ultralytics.models.yolo import YOLO +from ultralytics.models.yolo.segment.val import SegmentationValidator from ultralytics.utils import DEFAULT_CFG from ultralytics.utils.metrics import ConfusionMatrix @@ -37,7 +37,7 @@ def validate_ov_model( ov_model: ov.Model, data_loader: torch.utils.data.DataLoader, - validator: Validator, + validator: SegmentationValidator, num_samples: Optional[int] = None, ) -> Tuple[Dict, int, int]: validator.seen = 0 @@ -47,7 +47,7 @@ def validate_ov_model( validator.confusion_matrix = ConfusionMatrix(nc=validator.nc) compiled_model = ov.compile_model(ov_model, device_name="CPU") num_outputs = len(compiled_model.outputs) - for batch_i, batch in enumerate(data_loader): + for batch_i, batch in enumerate(track(data_loader, description="Validating")): if num_samples is not None and batch_i == num_samples: break batch = validator.preprocess(batch) @@ -65,12 +65,17 @@ def validate_ov_model( return stats, validator.seen, validator.nt_per_class.sum() -def run_benchmark(model_path: str, config) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 30" - command += f' -shape "[1,3,{config.imgsz},{config.imgsz}]"' - cmd_output = subprocess.check_output(command, shell=True) # nosec - - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) +def run_benchmark(model_path: Path, config) -> float: + command = [ + "benchmark_app", + "-m", model_path.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "30", + "-shape", str([1, 3, config.imgsz, config.imgsz]), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) @@ -96,11 +101,11 @@ def run_benchmark(model_path: str, config) -> float: validator, data_loader = prepare_validation(YOLO(ROOT / f"{MODEL_NAME}.pt"), args) print("[5/7] Validate OpenVINO FP32 model:") -fp32_stats, total_images, total_objects = validate_ov_model(fp32_ov_model, tqdm(data_loader), validator) +fp32_stats, total_images, total_objects = validate_ov_model(fp32_ov_model, data_loader, validator) print_statistics(fp32_stats, total_images, total_objects) print("[6/7] Validate OpenVINO INT8 model:") -int8_stats, total_images, total_objects = validate_ov_model(int8_ov_model, tqdm(data_loader), validator) +int8_stats, total_images, total_objects = validate_ov_model(int8_ov_model, data_loader, validator) print_statistics(int8_stats, total_images, total_objects) print("[7/7] Report:") diff --git a/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/main.py b/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/main.py index db385d520a5..cc7307bfd4c 100644 --- a/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/main.py +++ b/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/main.py @@ -13,16 +13,15 @@ from pathlib import Path from typing import Any, Dict, Tuple -import numpy as np import onnx import onnxruntime import torch -from tqdm import tqdm +from rich.progress import track from ultralytics.cfg import get_cfg from ultralytics.data.converter import coco80_to_coco91_class from ultralytics.data.utils import check_det_dataset -from ultralytics.engine.validator import BaseValidator as Validator from ultralytics.models.yolo import YOLO +from ultralytics.models.yolo.segment.val import SegmentationValidator from ultralytics.utils import DATASETS_DIR from ultralytics.utils import DEFAULT_CFG from ultralytics.utils import ops @@ -30,11 +29,16 @@ import nncf +MODEL_NAME = "yolov8n-seg" + ROOT = Path(__file__).parent.resolve() def validate( - model: onnx.ModelProto, data_loader: torch.utils.data.DataLoader, validator: Validator, num_samples: int = None + model: onnx.ModelProto, + data_loader: torch.utils.data.DataLoader, + validator: SegmentationValidator, + num_samples: int = None, ) -> Tuple[Dict, int, int]: validator.seen = 0 validator.jdict = [] @@ -49,7 +53,7 @@ def validate( output_names = [output.name for output in session.get_outputs()] num_outputs = len(output_names) - for batch_i, batch in enumerate(data_loader): + for batch_i, batch in enumerate(track(data_loader, description="Validating")): if num_samples is not None and batch_i == num_samples: break batch = validator.preprocess(batch) @@ -71,7 +75,7 @@ def validate( return stats, validator.seen, validator.nt_per_class.sum() -def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) -> None: +def print_statistics(stats: Dict[str, float], total_images: int, total_objects: int) -> None: print("Metrics(Box):") mp, mr, map50, mean_ap = ( stats["metrics/precision(B)"], @@ -84,38 +88,35 @@ def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) - pf = "%20s" + "%12i" * 2 + "%12.3g" * 4 # print format print(pf % ("all", total_images, total_objects, mp, mr, map50, mean_ap)) - # print the mask metrics for segmentation - if "metrics/precision(M)" in stats: - print("Metrics(Mask):") - s_mp, s_mr, s_map50, s_mean_ap = ( - stats["metrics/precision(M)"], - stats["metrics/recall(M)"], - stats["metrics/mAP50(M)"], - stats["metrics/mAP50-95(M)"], - ) - # Print results - s = ("%20s" + "%12s" * 6) % ("Class", "Images", "Labels", "Precision", "Recall", "mAP@.5", "mAP@.5:.95") - print(s) - pf = "%20s" + "%12i" * 2 + "%12.3g" * 4 # print format - print(pf % ("all", total_images, total_objects, s_mp, s_mr, s_map50, s_mean_ap)) - - -def prepare_validation(model: YOLO, args: Any) -> Tuple[Validator, torch.utils.data.DataLoader]: - validator = model.task_map[model.task]["validator"](args=args) - validator.data = check_det_dataset(args.data) - validator.stride = 32 + print("Metrics(Mask):") + s_mp, s_mr, s_map50, s_mean_ap = ( + stats["metrics/precision(M)"], + stats["metrics/recall(M)"], + stats["metrics/mAP50(M)"], + stats["metrics/mAP50-95(M)"], + ) + # Print results + s = ("%20s" + "%12s" * 6) % ("Class", "Images", "Labels", "Precision", "Recall", "mAP@.5", "mAP@.5:.95") + print(s) + pf = "%20s" + "%12i" * 2 + "%12.3g" * 4 # print format + print(pf % ("all", total_images, total_objects, s_mp, s_mr, s_map50, s_mean_ap)) - data_loader = validator.get_dataloader(f"{DATASETS_DIR}/coco128-seg", 1) +def prepare_validation(model: YOLO, args: Any) -> Tuple[SegmentationValidator, torch.utils.data.DataLoader]: + validator: SegmentationValidator = model.task_map[model.task]["validator"](args=args) + validator.data = check_det_dataset(args.data) + validator.stride = 32 validator.is_coco = True validator.class_map = coco80_to_coco91_class() validator.names = model.model.names validator.metrics.names = validator.names validator.nc = model.model.model[-1].nc - validator.nm = 32 validator.process = ops.process_mask validator.plot_masks = [] + coco_data_path = DATASETS_DIR / "coco128-seg" + data_loader = validator.get_dataloader(coco_data_path.as_posix(), 1) + return validator, data_loader @@ -129,7 +130,7 @@ def prepare_onnx_model(model: YOLO, model_name: str) -> Tuple[onnx.ModelProto, P def quantize_ac( - model: onnx.ModelProto, data_loader: torch.utils.data.DataLoader, validator_ac: Validator + model: onnx.ModelProto, data_loader: torch.utils.data.DataLoader, validator_ac: SegmentationValidator ) -> onnx.ModelProto: input_name = model.graph.input[0].name @@ -140,7 +141,7 @@ def transform_fn(data_item: Dict): def validation_ac( val_model: onnx.ModelProto, validation_loader: torch.utils.data.DataLoader, - validator: Validator, + validator: SegmentationValidator, num_samples: int = None, ) -> float: validator.seen = 0 @@ -155,7 +156,6 @@ def validation_ac( output_names = [output.name for output in session.get_outputs()] num_outputs = len(output_names) - counter = 0 for batch_i, batch in enumerate(validation_loader): if num_samples is not None and batch_i == num_samples: break @@ -172,13 +172,12 @@ def validation_ac( ] preds = validator.postprocess(preds) validator.update_metrics(preds, batch) - counter += 1 + stats = validator.get_stats() if num_outputs == 1: stats_metrics = stats["metrics/mAP50-95(B)"] else: stats_metrics = stats["metrics/mAP50-95(M)"] - print(f"Validate: dataset length = {counter}, metric value = {stats_metrics:.3f}") return stats_metrics, None quantization_dataset = nncf.Dataset(data_loader, transform_fn) @@ -213,8 +212,6 @@ def validation_ac( def run_example(): - MODEL_NAME = "yolov8n-seg" - model = YOLO(ROOT / f"{MODEL_NAME}.pt") args = get_cfg(cfg=DEFAULT_CFG) args.data = "coco128-seg.yaml" @@ -231,11 +228,11 @@ def run_example(): print(f"[2/5] Save INT8 model: {int8_model_path}") print("[3/5] Validate ONNX FP32 model:") - fp_stats, total_images, total_objects = validate(fp32_model, tqdm(data_loader), validator) + fp_stats, total_images, total_objects = validate(fp32_model, data_loader, validator) print_statistics(fp_stats, total_images, total_objects) print("[4/5] Validate ONNX INT8 model:") - q_stats, total_images, total_objects = validate(int8_model, tqdm(data_loader), validator) + q_stats, total_images, total_objects = validate(int8_model, data_loader, validator) print_statistics(q_stats, total_images, total_objects) print("[5/5] Report:") diff --git a/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/requirements.txt b/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/requirements.txt index 796f9bacef9..5f9b85873f1 100644 --- a/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/requirements.txt +++ b/examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/requirements.txt @@ -1,3 +1,4 @@ ultralytics==8.3.22 onnx==1.17.0 +onnxruntime==1.19.2 openvino==2024.5 diff --git a/examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/main.py b/examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/main.py index 659666962fe..fbe4ba57a7c 100644 --- a/examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/main.py +++ b/examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/main.py @@ -10,13 +10,12 @@ # limitations under the License. import json -import os import re import subprocess import sys from functools import partial from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Tuple import numpy as np import openvino as ov @@ -69,7 +68,6 @@ def validate( output = model.outputs[0] - counter = 0 for batch in val_loader: anomaly_maps = model(batch["image"])[output] pred_scores = np.max(anomaly_maps, axis=(1, 2, 3)) @@ -79,37 +77,38 @@ def validate( per_sample_metric = 1.0 if pred_label == groundtruth_label else 0.0 per_sample_metric_values.append(per_sample_metric) metric.update(torch.from_numpy(pred_scores), groundtruth_label) - counter += 1 metric_value = metric.compute() - print(f"Validate: dataset length = {counter}, metric value = {metric_value:.3f}") return metric_value, per_sample_metric_values -def run_benchmark(model_path: str, shape: Optional[List[int]] = None, verbose: bool = True) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 15" - if shape is not None: - command += f' -shape [{",".join(str(x) for x in shape)}]' - cmd_output = subprocess.check_output(command, shell=True) # nosec - if verbose: - print(*str(cmd_output).split("\\n")[-9:-1], sep="\n") - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) +def run_benchmark(model_path: Path, shape: List[int]) -> float: + command = [ + "benchmark_app", + "-m", model_path.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "15", + "-shape", str(shape), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + print(*cmd_output.splitlines()[-8:], sep="\n") + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) -def get_model_size(ir_path: str, m_type: str = "Mb", verbose: bool = True) -> float: - xml_size = os.path.getsize(ir_path) - bin_size = os.path.getsize(ir_path.replace("xml", "bin")) +def get_model_size(ir_path: Path, m_type: str = "Mb") -> float: + xml_size = ir_path.stat().st_size + bin_size = ir_path.with_suffix(".bin").stat().st_size for t in ["bytes", "Kb", "Mb"]: if m_type == t: break xml_size /= 1024 bin_size /= 1024 model_size = xml_size + bin_size - if verbose: - print(f"Model graph (xml): {xml_size:.3f} Mb") - print(f"Model weights (bin): {bin_size:.3f} Mb") - print(f"Model size: {model_size:.3f} Mb") + print(f"Model graph (xml): {xml_size:.3f} Mb") + print(f"Model weights (bin): {bin_size:.3f} Mb") + print(f"Model size: {model_size:.3f} Mb") return model_size @@ -168,24 +167,24 @@ def transform_fn(data_item): ############################################################################### # Benchmark performance, calculate compression rate and validate accuracy - fp32_ir_path = f"{ROOT}/stfpm_fp32.xml" + fp32_ir_path = ROOT / "stfpm_fp32.xml" ov.save_model(ov_model, fp32_ir_path, compress_to_fp16=False) print(f"[1/7] Save FP32 model: {fp32_ir_path}") - fp32_size = get_model_size(fp32_ir_path, verbose=True) + fp32_size = get_model_size(fp32_ir_path) # To avoid an accuracy drop when saving a model due to compression of unquantized # weights to FP16, compress_to_fp16=False should be used. This is necessary because # nncf.quantize_with_accuracy_control(...) keeps the most impactful operations within # the model in the original precision to achieve the specified model accuracy. - int8_ir_path = f"{ROOT}/stfpm_int8.xml" + int8_ir_path = ROOT / "stfpm_int8.xml" ov.save_model(ov_quantized_model, int8_ir_path) print(f"[2/7] Save INT8 model: {int8_ir_path}") - int8_size = get_model_size(int8_ir_path, verbose=True) + int8_size = get_model_size(int8_ir_path) print("[3/7] Benchmark FP32 model:") - fp32_fps = run_benchmark(fp32_ir_path, shape=[1, 3, 256, 256], verbose=True) + fp32_fps = run_benchmark(fp32_ir_path, shape=[1, 3, 256, 256]) print("[4/7] Benchmark INT8 model:") - int8_fps = run_benchmark(int8_ir_path, shape=[1, 3, 256, 256], verbose=True) + int8_fps = run_benchmark(int8_ir_path, shape=[1, 3, 256, 256]) print("[5/7] Validate OpenVINO FP32 model:") compiled_model = ov.compile_model(ov_model, device_name="CPU") diff --git a/examples/post_training_quantization/openvino/mobilenet_v2/main.py b/examples/post_training_quantization/openvino/mobilenet_v2/main.py index 80e9d2eaf8b..931beb0e6a2 100644 --- a/examples/post_training_quantization/openvino/mobilenet_v2/main.py +++ b/examples/post_training_quantization/openvino/mobilenet_v2/main.py @@ -9,33 +9,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re import subprocess from pathlib import Path -from typing import List, Optional +from typing import List import numpy as np import openvino as ov import torch from fastdownload import FastDownload +from rich.progress import track from sklearn.metrics import accuracy_score from torchvision import datasets from torchvision import transforms -from tqdm import tqdm import nncf ROOT = Path(__file__).parent.resolve() +DATASET_PATH = Path().home() / ".cache" / "nncf" / "datasets" +MODEL_PATH = Path().home() / ".cache" / "nncf" / "models" MODEL_URL = "https://huggingface.co/alexsu52/mobilenet_v2_imagenette/resolve/main/openvino_model.tgz" -MODEL_PATH = "~/.cache/nncf/models" DATASET_URL = "https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-320.tgz" -DATASET_PATH = "~/.cache/nncf/datasets" DATASET_CLASSES = 10 -def download(url: str, path: str) -> Path: - downloader = FastDownload(base=path, archive="downloaded", data="extracted") +def download(url: str, path: Path) -> Path: + downloader = FastDownload(base=path.resolve(), archive="downloaded", data="extracted") return downloader.get(url) @@ -46,7 +45,7 @@ def validate(model: ov.Model, val_loader: torch.utils.data.DataLoader) -> float: compiled_model = ov.compile_model(model, device_name="CPU") output = compiled_model.outputs[0] - for images, target in tqdm(val_loader): + for images, target in track(val_loader, description="Validating"): pred = compiled_model(images)[output] predictions.append(np.argmax(pred, axis=1)) references.append(target) @@ -56,30 +55,26 @@ def validate(model: ov.Model, val_loader: torch.utils.data.DataLoader) -> float: return accuracy_score(predictions, references) -def run_benchmark(model_path: Path, shape: Optional[List[int]] = None, verbose: bool = True) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 15" - if shape is not None: - command += f' -shape [{",".join(str(x) for x in shape)}]' - cmd_output = subprocess.check_output(command, shell=True) # nosec - if verbose: - print(*str(cmd_output).split("\\n")[-9:-1], sep="\n") - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) +def run_benchmark(model_path: Path, shape: List[int]) -> float: + cmd = ["benchmark_app", "-m", model_path.as_posix(), "-d", "CPU", "-api", "async", "-t", "15", "-shape", str(shape)] + cmd_output = subprocess.check_output(cmd, text=True) + print(*cmd_output.splitlines()[-8:], sep="\n") + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) -def get_model_size(ir_path: Path, m_type: str = "Mb", verbose: bool = True) -> float: - xml_size = os.path.getsize(ir_path) - bin_size = os.path.getsize(os.path.splitext(ir_path)[0] + ".bin") +def get_model_size(ir_path: Path, m_type: str = "Mb") -> float: + xml_size = ir_path.stat().st_size + bin_size = ir_path.with_suffix(".bin").stat().st_size for t in ["bytes", "Kb", "Mb"]: if m_type == t: break xml_size /= 1024 bin_size /= 1024 model_size = xml_size + bin_size - if verbose: - print(f"Model graph (xml): {xml_size:.3f} Mb") - print(f"Model weights (bin): {bin_size:.3f} Mb") - print(f"Model size: {model_size:.3f} Mb") + print(f"Model graph (xml): {xml_size:.3f} Mb") + print(f"Model weights (bin): {bin_size:.3f} Mb") + print(f"Model size: {model_size:.3f} Mb") return model_size @@ -139,17 +134,17 @@ def transform_fn(data_item): fp32_ir_path = ROOT / "mobilenet_v2_fp32.xml" ov.save_model(ov_model, fp32_ir_path, compress_to_fp16=False) print(f"[1/7] Save FP32 model: {fp32_ir_path}") -fp32_model_size = get_model_size(fp32_ir_path, verbose=True) +fp32_model_size = get_model_size(fp32_ir_path) int8_ir_path = ROOT / "mobilenet_v2_int8.xml" ov.save_model(ov_quantized_model, int8_ir_path) print(f"[2/7] Save INT8 model: {int8_ir_path}") -int8_model_size = get_model_size(int8_ir_path, verbose=True) +int8_model_size = get_model_size(int8_ir_path) print("[3/7] Benchmark FP32 model:") -fp32_fps = run_benchmark(fp32_ir_path, shape=[1, 3, 224, 224], verbose=True) +fp32_fps = run_benchmark(fp32_ir_path, shape=[1, 3, 224, 224]) print("[4/7] Benchmark INT8 model:") -int8_fps = run_benchmark(int8_ir_path, shape=[1, 3, 224, 224], verbose=True) +int8_fps = run_benchmark(int8_ir_path, shape=[1, 3, 224, 224]) print("[5/7] Validate OpenVINO FP32 model:") fp32_top1 = validate(ov_model, val_data_loader) diff --git a/examples/post_training_quantization/openvino/yolov8/main.py b/examples/post_training_quantization/openvino/yolov8/main.py index 1ec7b6b4a4d..b75e7577f17 100644 --- a/examples/post_training_quantization/openvino/yolov8/main.py +++ b/examples/post_training_quantization/openvino/yolov8/main.py @@ -13,26 +13,27 @@ from pathlib import Path from typing import Any, Dict, Tuple -import numpy as np import openvino as ov import torch -from tqdm import tqdm +from rich.progress import track from ultralytics.cfg import get_cfg from ultralytics.data.converter import coco80_to_coco91_class from ultralytics.data.utils import check_det_dataset -from ultralytics.engine.validator import BaseValidator as Validator from ultralytics.models.yolo import YOLO +from ultralytics.models.yolo.detect.val import DetectionValidator from ultralytics.utils import DATASETS_DIR from ultralytics.utils import DEFAULT_CFG from ultralytics.utils.metrics import ConfusionMatrix import nncf +MODEL_NAME = "yolov8n" + ROOT = Path(__file__).parent.resolve() def validate( - model: ov.Model, data_loader: torch.utils.data.DataLoader, validator: Validator, num_samples: int = None + model: ov.Model, data_loader: torch.utils.data.DataLoader, validator: DetectionValidator, num_samples: int = None ) -> Tuple[Dict, int, int]: validator.seen = 0 validator.jdict = [] @@ -41,7 +42,7 @@ def validate( model.reshape({0: [1, 3, -1, -1]}) compiled_model = ov.compile_model(model, device_name="CPU") output_layer = compiled_model.output(0) - for batch_i, batch in enumerate(data_loader): + for batch_i, batch in enumerate(track(data_loader, description="Validating")): if num_samples is not None and batch_i == num_samples: break batch = validator.preprocess(batch) @@ -52,7 +53,7 @@ def validate( return stats, validator.seen, validator.nt_per_class.sum() -def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) -> None: +def print_statistics(stats: Dict[str, float], total_images: int, total_objects: int) -> None: mp, mr, map50, mean_ap = ( stats["metrics/precision(B)"], stats["metrics/recall(B)"], @@ -65,37 +66,40 @@ def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) - print(pf % ("all", total_images, total_objects, mp, mr, map50, mean_ap)) -def prepare_validation(model: YOLO, args: Any) -> Tuple[Validator, torch.utils.data.DataLoader]: - validator = model.task_map[model.task]["validator"](args=args) +def prepare_validation(model: YOLO, args: Any) -> Tuple[DetectionValidator, torch.utils.data.DataLoader]: + validator: DetectionValidator = model.task_map[model.task]["validator"](args=args) validator.data = check_det_dataset(args.data) validator.stride = 32 - dataset = validator.data["val"] - print(f"{dataset}") - - data_loader = validator.get_dataloader(f"{DATASETS_DIR}/coco128", 1) - validator.is_coco = True validator.class_map = coco80_to_coco91_class() validator.names = model.model.names validator.metrics.names = validator.names validator.nc = model.model.model[-1].nc - return validator, data_loader + coco_data_path = DATASETS_DIR / "coco128" + data_loader = validator.get_dataloader(coco_data_path.as_posix(), 1) + return validator, data_loader -def benchmark_performance(model_path, config) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 30" - command += f' -shape "[1,3,{config.imgsz},{config.imgsz}]"' - cmd_output = subprocess.check_output(command, shell=True) # nosec - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) +def benchmark_performance(model_path: Path, config) -> float: + command = [ + "benchmark_app", + "-m", model_path.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "30", + "-shape", str([1, 3, config.imgsz, config.imgsz]), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) def prepare_openvino_model(model: YOLO, model_name: str) -> Tuple[ov.Model, Path]: - ir_model_path = Path(f"{ROOT}/{model_name}_openvino_model/{model_name}.xml") + ir_model_path = ROOT / f"{model_name}_openvino_model" / f"{model_name}.xml" if not ir_model_path.exists(): - onnx_model_path = Path(f"{ROOT}/{model_name}.onnx") + onnx_model_path = ROOT / f"{model_name}.onnx" if not onnx_model_path.exists(): model.export(format="onnx", dynamic=True, half=False) @@ -103,7 +107,7 @@ def prepare_openvino_model(model: YOLO, model_name: str) -> Tuple[ov.Model, Path return ov.Core().read_model(ir_model_path), ir_model_path -def quantize(model: ov.Model, data_loader: torch.utils.data.DataLoader, validator: Validator) -> ov.Model: +def quantize(model: ov.Model, data_loader: torch.utils.data.DataLoader, validator: DetectionValidator) -> ov.Model: def transform_fn(data_item: Dict): """ Quantization transform function. Extracts and preprocess input data from dataloader @@ -137,9 +141,7 @@ def transform_fn(data_item: Dict): def main(): - MODEL_NAME = "yolov8n" - - model = YOLO(f"{ROOT}/{MODEL_NAME}.pt") + model = YOLO(ROOT / f"{MODEL_NAME}.pt") args = get_cfg(cfg=DEFAULT_CFG) args.data = "coco128.yaml" @@ -151,16 +153,16 @@ def main(): # Quantize mode in OpenVINO representation quantized_model = quantize(ov_model, data_loader, validator) - quantized_model_path = Path(f"{ROOT}/{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml") + quantized_model_path = ov_model_path.with_name(ov_model_path.stem + "_quantized" + ov_model_path.suffix) ov.save_model(quantized_model, str(quantized_model_path)) # Validate FP32 model - fp_stats, total_images, total_objects = validate(ov_model, tqdm(data_loader), validator) + fp_stats, total_images, total_objects = validate(ov_model, data_loader, validator) print("Floating-point model validation results:") print_statistics(fp_stats, total_images, total_objects) # Validate quantized model - q_stats, total_images, total_objects = validate(quantized_model, tqdm(data_loader), validator) + q_stats, total_images, total_objects = validate(quantized_model, data_loader, validator) print("Quantized model validation results:") print_statistics(q_stats, total_images, total_objects) diff --git a/examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/main.py b/examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/main.py index abdcb982d12..56b38149f58 100644 --- a/examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/main.py +++ b/examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/main.py @@ -14,15 +14,14 @@ from pathlib import Path from typing import Any, Dict, Tuple -import numpy as np import openvino as ov import torch -from tqdm import tqdm +from rich.progress import track from ultralytics.cfg import get_cfg from ultralytics.data.converter import coco80_to_coco91_class from ultralytics.data.utils import check_det_dataset -from ultralytics.engine.validator import BaseValidator as Validator from ultralytics.models.yolo import YOLO +from ultralytics.models.yolo.segment.val import SegmentationValidator from ultralytics.utils import DATASETS_DIR from ultralytics.utils import DEFAULT_CFG from ultralytics.utils import ops @@ -30,11 +29,13 @@ import nncf +MODEL_NAME = "yolov8n-seg" + ROOT = Path(__file__).parent.resolve() def validate( - model: ov.Model, data_loader: torch.utils.data.DataLoader, validator: Validator, num_samples: int = None + model: ov.Model, data_loader: torch.utils.data.DataLoader, validator: SegmentationValidator, num_samples: int = None ) -> Tuple[Dict, int, int]: validator.seen = 0 validator.jdict = [] @@ -44,7 +45,7 @@ def validate( model.reshape({0: [1, 3, -1, -1]}) compiled_model = ov.compile_model(model, device_name="CPU") num_outputs = len(model.outputs) - for batch_i, batch in enumerate(data_loader): + for batch_i, batch in enumerate(track(data_loader, description="Validating")): if num_samples is not None and batch_i == num_samples: break batch = validator.preprocess(batch) @@ -62,7 +63,7 @@ def validate( return stats, validator.seen, validator.nt_per_class.sum() -def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) -> None: +def print_statistics(stats: Dict[str, float], total_images: int, total_objects: int) -> None: print("Metrics(Box):") mp, mr, map50, mean_ap = ( stats["metrics/precision(B)"], @@ -75,56 +76,56 @@ def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) - pf = "%20s" + "%12i" * 2 + "%12.3g" * 4 # print format print(pf % ("all", total_images, total_objects, mp, mr, map50, mean_ap)) - # print the mask metrics for segmentation - if "metrics/precision(M)" in stats: - print("Metrics(Mask):") - s_mp, s_mr, s_map50, s_mean_ap = ( - stats["metrics/precision(M)"], - stats["metrics/recall(M)"], - stats["metrics/mAP50(M)"], - stats["metrics/mAP50-95(M)"], - ) - # Print results - s = ("%20s" + "%12s" * 6) % ("Class", "Images", "Labels", "Precision", "Recall", "mAP@.5", "mAP@.5:.95") - print(s) - pf = "%20s" + "%12i" * 2 + "%12.3g" * 4 # print format - print(pf % ("all", total_images, total_objects, s_mp, s_mr, s_map50, s_mean_ap)) - - -def prepare_validation(model: YOLO, args: Any) -> Tuple[Validator, torch.utils.data.DataLoader]: - validator = model.task_map[model.task]["validator"](args=args) - validator.data = check_det_dataset(args.data) - validator.stride = 32 - dataset = validator.data["val"] - print(f"{dataset}") + print("Metrics(Mask):") + s_mp, s_mr, s_map50, s_mean_ap = ( + stats["metrics/precision(M)"], + stats["metrics/recall(M)"], + stats["metrics/mAP50(M)"], + stats["metrics/mAP50-95(M)"], + ) + # Print results + s = ("%20s" + "%12s" * 6) % ("Class", "Images", "Labels", "Precision", "Recall", "mAP@.5", "mAP@.5:.95") + print(s) + pf = "%20s" + "%12i" * 2 + "%12.3g" * 4 # print format + print(pf % ("all", total_images, total_objects, s_mp, s_mr, s_map50, s_mean_ap)) - data_loader = validator.get_dataloader(f"{DATASETS_DIR}/coco128-seg", 1) +def prepare_validation(model: YOLO, args: Any) -> Tuple[SegmentationValidator, torch.utils.data.DataLoader]: + validator: SegmentationValidator = model.task_map[model.task]["validator"](args=args) + validator.data = check_det_dataset(args.data) + validator.stride = 32 validator.is_coco = True validator.class_map = coco80_to_coco91_class() validator.names = model.model.names validator.metrics.names = validator.names validator.nc = model.model.model[-1].nc - validator.nm = 32 validator.process = ops.process_mask validator.plot_masks = [] + coco_data_path = DATASETS_DIR / "coco128-seg" + data_loader = validator.get_dataloader(coco_data_path.as_posix(), 1) + return validator, data_loader def benchmark_performance(model_path, config) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 30" - command += f' -shape "[1,3,{config.imgsz},{config.imgsz}]"' - cmd_output = subprocess.check_output(command, shell=True) # nosec - - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) + command = [ + "benchmark_app", + "-m", model_path.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "30", + "-shape", str([1, 3, config.imgsz, config.imgsz]), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) def prepare_openvino_model(model: YOLO, model_name: str) -> Tuple[ov.Model, Path]: - ir_model_path = Path(f"{ROOT}/{model_name}_openvino_model/{model_name}.xml") + ir_model_path = ROOT / f"{model_name}_openvino_model" / f"{model_name}.xml" if not ir_model_path.exists(): - onnx_model_path = Path(f"{ROOT}/{model_name}.onnx") + onnx_model_path = ROOT / f"{model_name}.onnx" if not onnx_model_path.exists(): model.export(format="onnx", dynamic=True, half=False) @@ -132,7 +133,9 @@ def prepare_openvino_model(model: YOLO, model_name: str) -> Tuple[ov.Model, Path return ov.Core().read_model(ir_model_path), ir_model_path -def quantize_ac(model: ov.Model, data_loader: torch.utils.data.DataLoader, validator_ac: Validator) -> ov.Model: +def quantize_ac( + model: ov.Model, data_loader: torch.utils.data.DataLoader, validator_ac: SegmentationValidator +) -> ov.Model: def transform_fn(data_item: Dict): input_tensor = validator_ac.preprocess(data_item)["img"].numpy() return input_tensor @@ -140,7 +143,7 @@ def transform_fn(data_item: Dict): def validation_ac( compiled_model: ov.CompiledModel, validation_loader: torch.utils.data.DataLoader, - validator: Validator, + validator: SegmentationValidator, num_samples: int = None, ) -> float: validator.seen = 0 @@ -150,7 +153,6 @@ def validation_ac( validator.confusion_matrix = ConfusionMatrix(nc=validator.nc) num_outputs = len(compiled_model.outputs) - counter = 0 for batch_i, batch in enumerate(validation_loader): if num_samples is not None and batch_i == num_samples: break @@ -165,13 +167,11 @@ def validation_ac( ] preds = validator.postprocess(preds) validator.update_metrics(preds, batch) - counter += 1 stats = validator.get_stats() if num_outputs == 1: stats_metrics = stats["metrics/mAP50-95(B)"] else: stats_metrics = stats["metrics/mAP50-95(M)"] - print(f"Validate: dataset length = {counter}, metric value = {stats_metrics:.3f}") return stats_metrics quantization_dataset = nncf.Dataset(data_loader, transform_fn) @@ -205,9 +205,7 @@ def validation_ac( def main(): - MODEL_NAME = "yolov8n-seg" - - model = YOLO(f"{ROOT}/{MODEL_NAME}.pt") + model = YOLO(ROOT / f"{MODEL_NAME}.pt") args = get_cfg(cfg=DEFAULT_CFG) args.data = "coco128-seg.yaml" args.workers = 0 @@ -221,17 +219,17 @@ def main(): # Quantize mode in OpenVINO representation quantized_model = quantize_ac(ov_model, data_loader, validator) - quantized_model_path = Path(f"{ROOT}/{MODEL_NAME}_openvino_model/{MODEL_NAME}_quantized.xml") + quantized_model_path = ov_model_path.with_name(ov_model_path.stem + "_quantized" + ov_model_path.suffix) ov.save_model(quantized_model, str(quantized_model_path)) # Validate FP32 model - fp_stats, total_images, total_objects = validate(ov_model, tqdm(data_loader), validator) print("Floating-point model validation results:") + fp_stats, total_images, total_objects = validate(ov_model, data_loader, validator) print_statistics(fp_stats, total_images, total_objects) # Validate quantized model - q_stats, total_images, total_objects = validate(quantized_model, tqdm(data_loader), validator) print("Quantized model validation results:") + q_stats, total_images, total_objects = validate(quantized_model, data_loader, validator) print_statistics(q_stats, total_images, total_objects) # Benchmark performance of FP32 model diff --git a/examples/post_training_quantization/tensorflow/mobilenet_v2/main.py b/examples/post_training_quantization/tensorflow/mobilenet_v2/main.py index bf4707e6dc6..ec4c081edaa 100644 --- a/examples/post_training_quantization/tensorflow/mobilenet_v2/main.py +++ b/examples/post_training_quantization/tensorflow/mobilenet_v2/main.py @@ -9,16 +9,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re import subprocess from pathlib import Path -from typing import List, Optional +from typing import List import openvino as ov import tensorflow as tf import tensorflow_datasets as tfds -from tqdm import tqdm +from rich.progress import track import nncf @@ -32,37 +31,40 @@ def validate(model: ov.Model, val_loader: tf.data.Dataset) -> tf.Tensor: output = compiled_model.outputs[0] metric = tf.keras.metrics.CategoricalAccuracy(name="acc@1") - for images, labels in tqdm(val_loader): + for images, labels in track(val_loader): pred = compiled_model(images.numpy())[output] metric.update_state(labels, pred) return metric.result() -def run_benchmark(model_path: str, shape: Optional[List[int]] = None, verbose: bool = True) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 15" - if shape is not None: - command += f' -shape [{",".join(str(x) for x in shape)}]' - cmd_output = subprocess.check_output(command, shell=True) # nosec - if verbose: - print(*str(cmd_output).split("\\n")[-9:-1], sep="\n") - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) +def run_benchmark(model_path: Path, shape: List[int]) -> float: + command = [ + "benchmark_app", + "-m", model_path.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "15", + "-shape", str(shape), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + print(*cmd_output.splitlines()[-8:], sep="\n") + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) -def get_model_size(ir_path: str, m_type: str = "Mb", verbose: bool = True) -> float: - xml_size = os.path.getsize(ir_path) - bin_size = os.path.getsize(os.path.splitext(ir_path)[0] + ".bin") +def get_model_size(ir_path: Path, m_type: str = "Mb") -> float: + xml_size = ir_path.stat().st_size + bin_size = ir_path.with_suffix(".bin").stat().st_size for t in ["bytes", "Kb", "Mb"]: if m_type == t: break xml_size /= 1024 bin_size /= 1024 model_size = xml_size + bin_size - if verbose: - print(f"Model graph (xml): {xml_size:.3f} Mb") - print(f"Model weights (bin): {bin_size:.3f} Mb") - print(f"Model size: {model_size:.3f} Mb") + print(f"Model graph (xml): {xml_size:.3f} Mb") + print(f"Model weights (bin): {bin_size:.3f} Mb") + print(f"Model size: {model_size:.3f} Mb") return model_size @@ -148,20 +150,20 @@ def transform_fn(data_item): ov_model = ov.convert_model(tf_model, share_weights=False) ov_quantized_model = ov.convert_model(tf_quantized_model, share_weights=False) -fp32_ir_path = f"{ROOT}/mobilenet_v2_fp32.xml" +fp32_ir_path = ROOT / "mobilenet_v2_fp32.xml" ov.save_model(ov_model, fp32_ir_path, compress_to_fp16=False) print(f"[1/7] Save FP32 model: {fp32_ir_path}") -fp32_model_size = get_model_size(fp32_ir_path, verbose=True) +fp32_model_size = get_model_size(fp32_ir_path) -int8_ir_path = f"{ROOT}/mobilenet_v2_int8.xml" +int8_ir_path = ROOT / "mobilenet_v2_int8.xml" ov.save_model(ov_quantized_model, int8_ir_path) print(f"[2/7] Save INT8 model: {int8_ir_path}") -int8_model_size = get_model_size(int8_ir_path, verbose=True) +int8_model_size = get_model_size(int8_ir_path) print("[3/7] Benchmark FP32 model:") -fp32_fps = run_benchmark(fp32_ir_path, shape=[1, 224, 224, 3], verbose=True) +fp32_fps = run_benchmark(fp32_ir_path, shape=[1, 224, 224, 3]) print("[4/7] Benchmark INT8 model:") -int8_fps = run_benchmark(int8_ir_path, shape=[1, 224, 224, 3], verbose=True) +int8_fps = run_benchmark(int8_ir_path, shape=[1, 224, 224, 3]) print("[5/7] Validate OpenVINO FP32 model:") fp32_top1 = validate(ov_model, val_dataset) diff --git a/examples/post_training_quantization/torch/mobilenet_v2/main.py b/examples/post_training_quantization/torch/mobilenet_v2/main.py index de309234445..a381a562b43 100644 --- a/examples/post_training_quantization/torch/mobilenet_v2/main.py +++ b/examples/post_training_quantization/torch/mobilenet_v2/main.py @@ -9,34 +9,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re import subprocess from functools import partial from pathlib import Path -from typing import List, Optional, Tuple +from typing import List, Tuple import numpy as np import openvino as ov import torch from fastdownload import FastDownload +from rich.progress import track from sklearn.metrics import accuracy_score from torchvision import datasets from torchvision import models from torchvision import transforms import nncf -from nncf.common.logging.track_progress import track ROOT = Path(__file__).parent.resolve() CHECKPOINT_URL = "https://huggingface.co/alexsu52/mobilenet_v2_imagenette/resolve/main/pytorch_model.bin" DATASET_URL = "https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-320.tgz" -DATASET_PATH = "~/.cache/nncf/datasets" +DATASET_PATH = Path().home() / ".cache" / "nncf" / "datasets" DATASET_CLASSES = 10 def download_dataset() -> Path: - downloader = FastDownload(base=DATASET_PATH, archive="downloaded", data="extracted") + downloader = FastDownload(base=DATASET_PATH.resolve(), archive="downloaded", data="extracted") return downloader.get(DATASET_URL) @@ -63,30 +62,33 @@ def validate(model: ov.Model, val_loader: torch.utils.data.DataLoader) -> float: return accuracy_score(predictions, references) -def run_benchmark(model_path: Path, shape: Optional[List[int]] = None, verbose: bool = True) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 15" - if shape is not None: - command += f' -shape [{",".join(str(x) for x in shape)}]' - cmd_output = subprocess.check_output(command, shell=True) # nosec - if verbose: - print(*str(cmd_output).split("\\n")[-9:-1], sep="\n") - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) +def run_benchmark(model_path: Path, shape: List[int]) -> float: + command = [ + "benchmark_app", + "-m", model_path.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "15", + "-shape", str(shape), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + print(*cmd_output.splitlines()[-8:], sep="\n") + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) -def get_model_size(ir_path: Path, m_type: str = "Mb", verbose: bool = True) -> float: - xml_size = os.path.getsize(ir_path) - bin_size = os.path.getsize(os.path.splitext(ir_path)[0] + ".bin") +def get_model_size(ir_path: Path, m_type: str = "Mb") -> float: + xml_size = ir_path.stat().st_size + bin_size = ir_path.with_suffix(".bin").stat().st_size for t in ["bytes", "Kb", "Mb"]: if m_type == t: break xml_size /= 1024 bin_size /= 1024 model_size = xml_size + bin_size - if verbose: - print(f"Model graph (xml): {xml_size:.3f} {m_type}") - print(f"Model weights (bin): {bin_size:.3f} {m_type}") - print(f"Model size: {model_size:.3f} {m_type}") + print(f"Model graph (xml): {xml_size:.3f} {m_type}") + print(f"Model weights (bin): {bin_size:.3f} {m_type}") + print(f"Model size: {model_size:.3f} {m_type}") return model_size @@ -156,17 +158,17 @@ def transform_fn(data_item: Tuple[torch.Tensor, int], device: torch.device) -> t fp32_ir_path = ROOT / "mobilenet_v2_fp32.xml" ov.save_model(ov_model, fp32_ir_path, compress_to_fp16=False) print(f"[1/7] Save FP32 model: {fp32_ir_path}") -fp32_model_size = get_model_size(fp32_ir_path, verbose=True) +fp32_model_size = get_model_size(fp32_ir_path) int8_ir_path = ROOT / "mobilenet_v2_int8.xml" ov.save_model(ov_quantized_model, int8_ir_path) print(f"[2/7] Save INT8 model: {int8_ir_path}") -int8_model_size = get_model_size(int8_ir_path, verbose=True) +int8_model_size = get_model_size(int8_ir_path) print("[3/7] Benchmark FP32 model:") -fp32_fps = run_benchmark(fp32_ir_path, shape=[1, 3, 224, 224], verbose=True) +fp32_fps = run_benchmark(fp32_ir_path, shape=[1, 3, 224, 224]) print("[4/7] Benchmark INT8 model:") -int8_fps = run_benchmark(int8_ir_path, shape=[1, 3, 224, 224], verbose=True) +int8_fps = run_benchmark(int8_ir_path, shape=[1, 3, 224, 224]) print("[5/7] Validate OpenVINO FP32 model:") fp32_top1 = validate(ov_model, val_data_loader) diff --git a/examples/post_training_quantization/torch/ssd300_vgg16/main.py b/examples/post_training_quantization/torch/ssd300_vgg16/main.py index 3e28861743c..c29e64bac97 100644 --- a/examples/post_training_quantization/torch/ssd300_vgg16/main.py +++ b/examples/post_training_quantization/torch/ssd300_vgg16/main.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import re import subprocess from pathlib import Path @@ -28,43 +27,39 @@ from torchvision.models.detection.ssd import SSD from torchvision.models.detection.ssd import GeneralizedRCNNTransform from torchvision.models.detection.anchor_utils import DefaultBoxGenerator -from nncf.common.logging.track_progress import track +from rich.progress import track from functools import partial ROOT = Path(__file__).parent.resolve() DATASET_URL = "https://ultralytics.com/assets/coco128.zip" -DATASET_PATH = "~/.cache/nncf/datasets" +DATASET_PATH = Path().home() / ".cache" / "nncf" / "datasets" def download_dataset() -> Path: - downloader = FastDownload(base=DATASET_PATH, archive="downloaded", data="extracted") + downloader = FastDownload(base=DATASET_PATH.resolve(), archive="downloaded", data="extracted") return downloader.get(DATASET_URL) -def get_model_size(ir_path: str, m_type: str = "Mb", verbose: bool = True) -> float: - xml_size = os.path.getsize(ir_path) - bin_size = os.path.getsize(os.path.splitext(ir_path)[0] + ".bin") +def get_model_size(ir_path: Path, m_type: str = "Mb") -> float: + xml_size = ir_path.stat().st_size + bin_size = ir_path.with_suffix(".bin").stat().st_size for t in ["bytes", "Kb", "Mb"]: if m_type == t: break xml_size /= 1024 bin_size /= 1024 model_size = xml_size + bin_size - if verbose: - print(f"Model graph (xml): {xml_size:.3f} {m_type}") - print(f"Model weights (bin): {bin_size:.3f} {m_type}") - print(f"Model size: {model_size:.3f} {m_type}") + print(f"Model graph (xml): {xml_size:.3f} {m_type}") + print(f"Model weights (bin): {bin_size:.3f} {m_type}") + print(f"Model size: {model_size:.3f} {m_type}") return model_size -def run_benchmark(model_path: str, shape=None, verbose: bool = True) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 15" - if shape is not None: - command += f' -shape [{",".join(str(x) for x in shape)}]' - cmd_output = subprocess.check_output(command, shell=True) # nosec - if verbose: - print(*str(cmd_output).split("\\n")[-9:-1], sep="\n") - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) +def run_benchmark(model_path: Path) -> float: + command = ["benchmark_app", "-m", model_path.as_posix(), "-d", "CPU", "-api", "async", "-t", "15"] + cmd_output = subprocess.check_output(command, text=True) + print(*cmd_output.splitlines()[-8:], sep="\n") + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) @@ -157,28 +152,28 @@ def main(): # Convert to OpenVINO dummy_input = torch.randn(1, 3, 480, 480) - fp32_onnx_path = f"{ROOT}/ssd300_vgg16_fp32.onnx" + fp32_onnx_path = ROOT / "ssd300_vgg16_fp32.onnx" torch.onnx.export(model.cpu(), dummy_input, fp32_onnx_path) ov_model = ov.convert_model(fp32_onnx_path) - int8_onnx_path = f"{ROOT}/ssd300_vgg16_int8.onnx" + int8_onnx_path = ROOT / "ssd300_vgg16_int8.onnx" torch.onnx.export(quantized_model.cpu(), dummy_input, int8_onnx_path) ov_quantized_model = ov.convert_model(int8_onnx_path) - fp32_ir_path = f"{ROOT}/ssd300_vgg16_fp32.xml" + fp32_ir_path = ROOT / "ssd300_vgg16_fp32.xml" ov.save_model(ov_model, fp32_ir_path, compress_to_fp16=False) print(f"[1/7] Save FP32 model: {fp32_ir_path}") - fp32_model_size = get_model_size(fp32_ir_path, verbose=True) + fp32_model_size = get_model_size(fp32_ir_path) - int8_ir_path = f"{ROOT}/ssd300_vgg16_int8.xml" + int8_ir_path = ROOT / "ssd300_vgg16_int8.xml" ov.save_model(ov_quantized_model, int8_ir_path) print(f"[2/7] Save INT8 model: {int8_ir_path}") - int8_model_size = get_model_size(int8_ir_path, verbose=True) + int8_model_size = get_model_size(int8_ir_path) print("[3/7] Benchmark FP32 model:") - fp32_fps = run_benchmark(fp32_ir_path, verbose=True) + fp32_fps = run_benchmark(fp32_ir_path) print("[4/7] Benchmark INT8 model:") - int8_fps = run_benchmark(int8_ir_path, verbose=True) + int8_fps = run_benchmark(int8_ir_path) print("[5/7] Validate FP32 model:") torch.backends.cudnn.deterministic = True diff --git a/examples/post_training_quantization/torch_fx/resnet18/main.py b/examples/post_training_quantization/torch_fx/resnet18/main.py index eb348e9bfd7..ed6a9f20e7d 100644 --- a/examples/post_training_quantization/torch_fx/resnet18/main.py +++ b/examples/post_training_quantization/torch_fx/resnet18/main.py @@ -26,11 +26,11 @@ import torchvision.models as models import torchvision.transforms as transforms from fastdownload import FastDownload +from rich.progress import track from torch._dynamo.exc import BackendCompilerFailed import nncf import nncf.torch -from nncf.common.logging.track_progress import track from nncf.common.utils.helpers import create_table from nncf.common.utils.os import is_windows from nncf.torch import disable_patching @@ -44,11 +44,11 @@ "https://storage.openvinotoolkit.org/repositories/nncf/openvino_notebook_ckpts/302_resnet18_fp32_v1.pth" ) DATASET_URL = "http://cs231n.stanford.edu/tiny-imagenet-200.zip" -DATASET_PATH = "~/.cache/nncf/datasets" +DATASET_PATH = Path().home() / ".cache" / "nncf" / "datasets" def download_dataset() -> Path: - downloader = FastDownload(base=DATASET_PATH, archive="downloaded", data="extracted") + downloader = FastDownload(base=DATASET_PATH.resolve(), archive="downloaded", data="extracted") return downloader.get(DATASET_URL) diff --git a/examples/quantization_aware_training/torch/anomalib/main.py b/examples/quantization_aware_training/torch/anomalib/main.py index f90ca45cd3b..627c47c2952 100644 --- a/examples/quantization_aware_training/torch/anomalib/main.py +++ b/examples/quantization_aware_training/torch/anomalib/main.py @@ -30,11 +30,11 @@ import nncf HOME_PATH = Path.home() -DATASET_PATH = HOME_PATH / ".cache/nncf/datasets/mvtec" -CHECKPOINT_PATH = HOME_PATH / ".cache/nncf/models/anomalib" +DATASET_PATH = HOME_PATH / ".cache" / "nncf" / "datasets" / "mvtec" +CHECKPOINT_PATH = HOME_PATH / ".cache" / "nncf" / "models" / "anomalib" ROOT = Path(__file__).parent.resolve() -FP32_RESULTS_ROOT = ROOT / "fp32" -INT8_RESULTS_ROOT = ROOT / "int8" +FP32_RESULTS_ROOT = ROOT / "results" / "fp32" +INT8_RESULTS_ROOT = ROOT / "results" / "int8" CHECKPOINT_URL = "https://storage.openvinotoolkit.org/repositories/nncf/examples/torch/anomalib/stfpm_mvtec.ckpt" USE_PRETRAINED = True @@ -61,11 +61,17 @@ def create_dataset(root: Path) -> MVTec: def run_benchmark(model_path: Path, shape: List[int]) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 15" - command += f' -shape "[{",".join(str(x) for x in shape)}]"' - cmd_output = subprocess.check_output(command, shell=True) # nosec - print(*str(cmd_output).split("\\n")[-9:-1], sep="\n") - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) + command = [ + "benchmark_app", + "-m", model_path.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "15", + "-shape", str(shape), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + print(*cmd_output.splitlines()[-8:], sep="\n") + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) diff --git a/examples/quantization_aware_training/torch/resnet18/main.py b/examples/quantization_aware_training/torch/resnet18/main.py index 7ab3b7af14a..d7909db7329 100644 --- a/examples/quantization_aware_training/torch/resnet18/main.py +++ b/examples/quantization_aware_training/torch/resnet18/main.py @@ -15,7 +15,7 @@ import warnings from copy import deepcopy from pathlib import Path -from typing import List, Tuple +from typing import Tuple import openvino as ov import torch @@ -28,11 +28,11 @@ import torchvision.models as models import torchvision.transforms as transforms from fastdownload import FastDownload +from rich.progress import track from torch.jit import TracerWarning import nncf import nncf.torch -from nncf.common.logging.track_progress import track from nncf.common.utils.helpers import create_table warnings.filterwarnings("ignore", category=TracerWarning) @@ -51,11 +51,11 @@ "https://storage.openvinotoolkit.org/repositories/nncf/openvino_notebook_ckpts/302_resnet18_fp32_v1.pth" ) DATASET_URL = "http://cs231n.stanford.edu/tiny-imagenet-200.zip" -DATASET_PATH = "~/.cache/nncf/datasets" +DATASET_PATH = Path().home() / ".cache" / "nncf" / "datasets" def download_dataset() -> Path: - downloader = FastDownload(base=DATASET_PATH, archive="downloaded", data="extracted") + downloader = FastDownload(base=DATASET_PATH.resolve(), archive="downloaded", data="extracted") return downloader.get(DATASET_URL) @@ -208,17 +208,23 @@ def prepare_tiny_imagenet_200(dataset_dir: Path): val_images_dir.rmdir() -def run_benchmark(model_path: str, shape: List[int]) -> float: - command = f"benchmark_app -m {model_path} -d CPU -api async -t 15" - command += f' -shape "[{",".join(str(x) for x in shape)}]"' - cmd_output = subprocess.check_output(command, shell=True) # nosec - match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output)) +def run_benchmark(model_path: Path, shape: Tuple[int, ...]) -> float: + command = [ + "benchmark_app", + "-m", model_path.as_posix(), + "-d", "CPU", + "-api", "async", + "-t", "15", + "-shape", str(list(shape)), + ] # fmt: skip + cmd_output = subprocess.check_output(command, text=True) + match = re.search(r"Throughput\: (.+?) FPS", cmd_output) return float(match.group(1)) -def get_model_size(ir_path: str, m_type: str = "Mb") -> float: - xml_size = os.path.getsize(ir_path) - bin_size = os.path.getsize(os.path.splitext(ir_path)[0] + ".bin") +def get_model_size(ir_path: Path, m_type: str = "Mb") -> float: + xml_size = ir_path.stat().st_size + bin_size = ir_path.with_suffix(".bin").stat().st_size for t in ["bytes", "Kb", "Mb"]: if m_type == t: break @@ -305,13 +311,13 @@ def transform_fn(data_item): example_input = torch.randn(*input_shape).cpu() # Export FP32 model to OpenVINO™ IR - fp32_ir_path = f"{ROOT}/{BASE_MODEL_NAME}_fp32.xml" + fp32_ir_path = ROOT / f"{BASE_MODEL_NAME}_fp32.xml" ov_model = ov.convert_model(model.cpu(), example_input=example_input, input=input_shape) ov.save_model(ov_model, fp32_ir_path, compress_to_fp16=False) print(f"Original model path: {fp32_ir_path}") # Export INT8 model to OpenVINO™ IR - int8_ir_path = f"{ROOT}/{BASE_MODEL_NAME}_int8.xml" + int8_ir_path = ROOT / f"{BASE_MODEL_NAME}_int8.xml" ov_model = ov.convert_model(quantized_model.cpu(), example_input=example_input, input=input_shape) ov.save_model(ov_model, int8_ir_path, compress_to_fp16=False) print(f"Quantized model path: {int8_ir_path}") diff --git a/nncf/quantization/algorithms/accuracy_control/evaluator.py b/nncf/quantization/algorithms/accuracy_control/evaluator.py index 5dad46e23f9..1c06ed28bb8 100644 --- a/nncf/quantization/algorithms/accuracy_control/evaluator.py +++ b/nncf/quantization/algorithms/accuracy_control/evaluator.py @@ -14,6 +14,7 @@ import nncf from nncf.common.logging import nncf_logger +from nncf.common.logging.track_progress import track from nncf.common.utils.backend import BackendType from nncf.common.utils.backend import get_backend from nncf.common.utils.timer import timer @@ -276,20 +277,20 @@ def collect_values_for_each_item_using_prepared_model( :param prepared_model: Model to infer. :param dataset: Dataset to collect values. - :param indices: The zero-based indices of data items that should be selected from - the dataset. + :param indices: The zero-based indices of data items that should be selected from the dataset. :return: Collected values. """ + total = len(indices) if indices is not None else dataset.get_length() if self._metric_mode: # Collect metrics for each item values_for_each_item = [ self._validation_fn(prepared_model.model_for_inference, [data_item])[0] - for data_item in dataset.get_data(indices) + for data_item in track(dataset.get_data(indices), total=total, description="Collecting metrics") ] else: # Collect outputs for each item values_for_each_item = [] - for data_item in dataset.get_inference_data(indices): + for data_item in track(dataset.get_inference_data(indices), total=total, description="Collecting outputs"): logits = prepared_model(data_item) values_for_each_item.append(list(logits.values())) @@ -307,8 +308,7 @@ def collect_values_for_each_item( :param model: A target model. :param dataset: Dataset to collect values. - :param indices: The zero-based indices of data items that should be selected from - the dataset. + :param indices: The zero-based indices of data items that should be selected from the dataset. :return: Collected values. """ prepared_model = self.prepare_model(model) diff --git a/tests/cross_fw/examples/conftest.py b/tests/cross_fw/examples/conftest.py index 778db6abe3c..e6ab4ea3d4c 100644 --- a/tests/cross_fw/examples/conftest.py +++ b/tests/cross_fw/examples/conftest.py @@ -32,6 +32,7 @@ def pytest_addoption(parser): "--ov_version_override", default=None, help="Parameter to set OpenVINO into the env with the version from PyPI" ) parser.addoption("--data", type=str, default=None, help="Path to test datasets") + parser.addoption("--reuse-venv", action="store_true", help="Use venv from example directory") @pytest.fixture(scope="module") @@ -52,3 +53,8 @@ def ov_version_override(request): @pytest.fixture(scope="module") def data(request): return request.config.getoption("--data") + + +@pytest.fixture(scope="module") +def reuse_venv(request) -> bool: + return request.config.getoption("--reuse-venv") diff --git a/tests/cross_fw/examples/run_example.py b/tests/cross_fw/examples/run_example.py index 78570c3251a..17aff2996b1 100644 --- a/tests/cross_fw/examples/run_example.py +++ b/tests/cross_fw/examples/run_example.py @@ -278,6 +278,14 @@ def main(argv): parser.add_argument("-o", "--output", help="Path to the json file to save example metrics", required=True) args = parser.parse_args(args=argv) + # Disable progress bar for fastdownload module + try: + import fastprogress.fastprogress + + fastprogress.fastprogress.NO_BAR = True + except ImportError: + pass + if args.name == "quantization_aware_training_torch_anomalib": metrics = globals()[args.name](args.data) else: diff --git a/tests/cross_fw/examples/test_examples.py b/tests/cross_fw/examples/test_examples.py index 486a20aa45f..02d1b93e675 100644 --- a/tests/cross_fw/examples/test_examples.py +++ b/tests/cross_fw/examples/test_examples.py @@ -54,6 +54,7 @@ def test_examples( is_check_performance: bool, ov_version_override: str, data: str, + reuse_venv: bool, ): print("\n" + "-" * 64) print(f"Example name: {example_name}") @@ -64,6 +65,9 @@ def test_examples( backend = example_params["backend"] skip_if_backend_not_selected(backend, backends_list) + if reuse_venv: + # Use example directory as tmp_path + tmp_path = Path(example_params["requirements"]).parent venv_path = create_venv_with_nncf(tmp_path, "pip_e_local", "venv", {backend}) pip_with_venv = get_pip_executable_with_venv(venv_path) if "requirements" in example_params: diff --git a/tests/cross_fw/shared/command.py b/tests/cross_fw/shared/command.py index bad708bdea5..8f36017a6f3 100644 --- a/tests/cross_fw/shared/command.py +++ b/tests/cross_fw/shared/command.py @@ -17,7 +17,9 @@ from pathlib import Path from typing import Any, Dict, List -from nncf.common.utils.os import is_windows + +def is_windows() -> bool: + return "win32" in sys.platform class Command: diff --git a/tests/cross_fw/shared/helpers.py b/tests/cross_fw/shared/helpers.py index 1b9b9ef2d66..768afdda406 100644 --- a/tests/cross_fw/shared/helpers.py +++ b/tests/cross_fw/shared/helpers.py @@ -19,15 +19,20 @@ import numpy as np -import nncf -from nncf.common.utils.os import is_linux -from nncf.common.utils.os import is_windows from tests.cross_fw.shared.paths import GITHUB_REPO_URL from tests.cross_fw.shared.paths import PROJECT_ROOT TensorType = TypeVar("TensorType") +def is_windows() -> bool: + return "win32" in sys.platform + + +def is_linux() -> bool: + return "linux" in sys.platform + + def get_cli_dict_args(args): cli_args = {} for key, val in args.items(): @@ -48,7 +53,7 @@ def get_cli_dict_args(args): def create_venv_with_nncf(tmp_path: Path, package_type: str, venv_type: str, backends: Set[str] = None): venv_path = tmp_path / "venv" - venv_path.mkdir() + venv_path.mkdir(exist_ok=True) python_executable_with_venv = get_python_executable_with_venv(venv_path) pip_with_venv = get_pip_executable_with_venv(venv_path) @@ -68,7 +73,7 @@ def create_venv_with_nncf(tmp_path: Path, package_type: str, venv_type: str, bac subprocess.check_call(f"{pip_with_venv} install build", shell=True) run_path = tmp_path / "run" - run_path.mkdir() + run_path.mkdir(exist_ok=True) if package_type == "pip_pypi": run_cmd_line = f"{pip_with_venv} install nncf" @@ -83,7 +88,7 @@ def create_venv_with_nncf(tmp_path: Path, package_type: str, venv_type: str, bac elif package_type == "build_w": run_cmd_line = f"{python_executable_with_venv} -m build -w" else: - raise nncf.ValidationError(f"Invalid package type: {package_type}") + raise ValueError(f"Invalid package type: {package_type}") subprocess.run(run_cmd_line, check=True, shell=True, cwd=PROJECT_ROOT) if backends: