From e6f67f7e63e4d31bea59b30661b8881170ccd1b0 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Mon, 17 Apr 2023 13:18:12 -0700 Subject: [PATCH 1/9] Rework search tactic predictors Signed-off-by: Maciej Szankin --- dynast/search/search_tactic.py | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/dynast/search/search_tactic.py b/dynast/search/search_tactic.py index 7972140..f912280 100644 --- a/dynast/search/search_tactic.py +++ b/dynast/search/search_tactic.py @@ -288,7 +288,7 @@ def train_predictors(self, results_path: str = None): """ # Store predictor objects by objective name in a dictionary - self.predictor_dict = dict() + self.predictors = dict() # Create/train a predictor for each objective for objective in SUPERNET_METRICS[self.supernet]: @@ -305,10 +305,10 @@ def train_predictors(self, results_path: str = None): ) log.info(f'Training {objective} predictor.') predictor = objective_predictor.train_predictor() - log.info(f'Updated self.predictor_dict[{objective}].') - self.predictor_dict[objective] = predictor + log.info(f'Updated self.predictors[{objective}].') + self.predictors[objective] = predictor else: - self.predictor_dict[objective] = None + self.predictors[objective] = None def search(self): """Runs the LINAS search""" @@ -339,10 +339,7 @@ def search(self): ]: runner_predict = OFARunner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_top1'], + predictors=self.predictors, dataset_path=self.dataset_path, device=self.device, dataloader_workers=self.dataloader_workers, @@ -351,10 +348,7 @@ def search(self): elif self.supernet == 'transformer_lt_wmt_en_de': runner_predict = TransformerLTRunner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['bleu'], + predictors=self.predictors, dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, ) @@ -362,10 +356,7 @@ def search(self): elif self.supernet == 'bert_base_sst2': runner_predict = BertSST2Runner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_sst2'], + predictors=self.predictors, dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, device=self.device, @@ -775,10 +766,7 @@ def search(self): ]: runner_predict = OFARunner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_top1'], + predictors=self.predictors, dataset_path=self.dataset_path, device=self.device, dataloader_workers=self.dataloader_workers, @@ -787,10 +775,7 @@ def search(self): elif self.supernet == 'transformer_lt_wmt_en_de': runner_predict = TransformerLTRunner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['bleu'], + predictors=self.predictors, dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, ) From 3c370aa9216cf9fb21eef68ba869e9d5461247e8 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Mon, 17 Apr 2023 13:18:40 -0700 Subject: [PATCH 2/9] Rework OFA for simplified predictors Signed-off-by: Maciej Szankin --- .../image_classification/ofa/ofa_interface.py | 68 ++++--------------- 1 file changed, 14 insertions(+), 54 deletions(-) diff --git a/dynast/supernetwork/image_classification/ofa/ofa_interface.py b/dynast/supernetwork/image_classification/ofa/ofa_interface.py index b517503..0590a85 100644 --- a/dynast/supernetwork/image_classification/ofa/ofa_interface.py +++ b/dynast/supernetwork/image_classification/ofa/ofa_interface.py @@ -16,7 +16,7 @@ import csv import uuid from datetime import datetime -from typing import Tuple +from typing import Dict, Tuple import torch @@ -46,20 +46,14 @@ def __init__( self, supernet: str, dataset_path: str, - acc_predictor: Predictor = None, - macs_predictor: Predictor = None, - latency_predictor: Predictor = None, - params_predictor: Predictor = None, + predictors: Dict[str, Predictor] = None, batch_size: int = 1, dataloader_workers: int = 4, device: str = 'cpu', test_size: int = None, ): self.supernet = supernet - self.acc_predictor = acc_predictor - self.macs_predictor = macs_predictor - self.latency_predictor = latency_predictor - self.params_predictor = params_predictor + self.predictors = predictors self.batch_size = batch_size self.device = device self.test_size = test_size @@ -82,21 +76,9 @@ def _init_data(self): num_workers=self.dataloader_workers, ) - def estimate_accuracy_top1(self, subnet_cfg) -> float: - top1 = self.acc_predictor.predict(subnet_cfg) - return top1 - - def estimate_macs(self, subnet_cfg) -> int: - macs = self.macs_predictor.predict(subnet_cfg) - return macs - - def estimate_latency(self, subnet_cfg) -> float: - latency = self.latency_predictor.predict(subnet_cfg) - return latency - - def estimate_parameters(self, subnet_cfg) -> int: - parameters = self.params_predictor.predict(subnet_cfg) - return parameters + def estimate_metric(self, metric: str, subnet_cfg) -> float: + predicted_val = self.predictors.get(metric).predict(subnet_cfg) + return predicted_val def validate_top1(self, subnet_cfg, device=None) -> float: device = self.device if not device else device @@ -219,21 +201,10 @@ def eval_subnet(self, x): # Predictor Mode if self.predictor_mode == True: - if 'params' in self.optimization_metrics: - individual_results['params'] = self.evaluator.estimate_parameters( - self.manager.onehot_generic(x).reshape(1, -1) - )[0] - if 'latency' in self.optimization_metrics: - individual_results['latency'] = self.evaluator.estimate_latency( - self.manager.onehot_generic(x).reshape(1, -1) - )[0] - if 'macs' in self.optimization_metrics: - individual_results['macs'] = self.evaluator.estimate_macs( - self.manager.onehot_generic(x).reshape(1, -1) - )[0] - if 'accuracy_top1' in self.optimization_metrics: - individual_results['accuracy_top1'] = self.evaluator.estimate_accuracy_top1( - self.manager.onehot_generic(x).reshape(1, -1) + for metric in self.optimization_metrics: + # TODO(macsz) Maybe move [0] to the `estimate_metric`. + individual_results[metric] = self.evaluator.estimate_metric( + metric, self.manager.onehot_generic(x).reshape(1, -1) )[0] # Validation Mode @@ -318,21 +289,10 @@ def eval_subnet(self, x): # Predictor Mode if self.predictor_mode == True: - if 'params' in self.optimization_metrics: - individual_results['params'] = self.evaluator.estimate_parameters( - self.manager.onehot_generic(x).reshape(1, -1) - )[0] - if 'latency' in self.optimization_metrics: - individual_results['latency'] = self.evaluator.estimate_latency( - self.manager.onehot_generic(x).reshape(1, -1) - )[0] - if 'macs' in self.optimization_metrics: - individual_results['macs'] = self.evaluator.estimate_macs( - self.manager.onehot_generic(x).reshape(1, -1) - )[0] - if 'accuracy_top1' in self.optimization_metrics: - individual_results['accuracy_top1'] = self.evaluator.estimate_accuracy_top1( - self.manager.onehot_generic(x).reshape(1, -1) + for metric in self.optimization_metrics: + # TODO(macsz) Maybe move [0] to the `estimate_metric`. + individual_results[metric] = self.evaluator.estimate_metric( + metric, self.manager.onehot_generic(x).reshape(1, -1) )[0] # Validation Mode From 5f0eae773ca178a4c62aee7e7037d8b6f228179a Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Fri, 29 Sep 2023 14:34:53 -0700 Subject: [PATCH 3/9] Revert TransformerLTRunner Signed-off-by: Maciej Szankin --- dynast/search/search_tactic.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dynast/search/search_tactic.py b/dynast/search/search_tactic.py index e2596d6..b4cb7b1 100644 --- a/dynast/search/search_tactic.py +++ b/dynast/search/search_tactic.py @@ -382,7 +382,10 @@ def search(self): elif self.supernet == 'transformer_lt_wmt_en_de': runner_predict = TransformerLTRunner( supernet=self.supernet, - predictors=self.predictors, + latency_predictor=self.predictors['latency'], + macs_predictor=self.predictors['macs'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['bleu'], dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, ) @@ -885,7 +888,10 @@ def search(self): elif self.supernet == 'transformer_lt_wmt_en_de': runner_predict = TransformerLTRunner( supernet=self.supernet, - predictors=self.predictors, + latency_predictor=self.predictors['latency'], + macs_predictor=self.predictors['macs'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['bleu'], dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, ) From 921337cecdb15fe4d3daab4a0fe7bc7e2c8aa42b Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Fri, 29 Sep 2023 14:35:56 -0700 Subject: [PATCH 4/9] Revert BERT-SST2 Runner Signed-off-by: Maciej Szankin --- dynast/search/search_tactic.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dynast/search/search_tactic.py b/dynast/search/search_tactic.py index b4cb7b1..e69f325 100644 --- a/dynast/search/search_tactic.py +++ b/dynast/search/search_tactic.py @@ -393,7 +393,10 @@ def search(self): elif self.supernet == 'bert_base_sst2': runner_predict = BertSST2Runner( supernet=self.supernet, - predictors=self.predictors, + latency_predictor=self.predictors['latency'], + macs_predictor=self.predictors['macs'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['accuracy_sst2'], dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, device=self.device, @@ -899,10 +902,10 @@ def search(self): elif self.supernet == 'bert_base_sst2': runner_predict = BertSST2Runner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_sst2'], + latency_predictor=self.predictors['latency'], + macs_predictor=self.predictors['macs'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['accuracy_sst2'], dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, device=self.device, From d17e47cc933c4cb80e459be01cb454b5d7200468 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Fri, 29 Sep 2023 14:38:36 -0700 Subject: [PATCH 5/9] Update predictors to predictor_dict for yet unsupported supernets Signed-off-by: Maciej Szankin --- dynast/search/search_tactic.py | 48 +++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/dynast/search/search_tactic.py b/dynast/search/search_tactic.py index e69f325..1233a98 100644 --- a/dynast/search/search_tactic.py +++ b/dynast/search/search_tactic.py @@ -404,10 +404,10 @@ def search(self): elif self.supernet == 'vit_base_imagenet': runner_predict = ViTRunner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_top1'], + latency_predictor=self.predictors['latency'], + macs_predictor=self.predictors['macs'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['accuracy_top1'], dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, batch_size=self.batch_size, @@ -417,10 +417,10 @@ def search(self): elif self.supernet == 'inc_quantization_ofa_resnet50': runner_predict = QuantizedOFARunner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - model_size_predictor=self.predictor_dict['model_size'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_top1'], + latency_predictor=self.predictors['latency'], + model_size_predictor=self.predictors['model_size'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['accuracy_top1'], dataset_path=self.dataset_path, device=self.device, dataloader_workers=self.dataloader_workers, @@ -431,10 +431,10 @@ def search(self): runner_predict = BootstrapNASRunner( bootstrapnas_supernetwork=self.bootstrapnas_supernetwork, supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_top1'], + latency_predictor=self.predictors['latency'], + macs_predictor=self.predictors['macs'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['accuracy_top1'], dataset_path=self.dataset_path, batch_size=self.batch_size, device=self.device, @@ -913,10 +913,10 @@ def search(self): elif self.supernet == 'vit_base_imagenet': runner_predict = ViTRunner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_top1'], + latency_predictor=self.predictors['latency'], + macs_predictor=self.predictors['macs'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['accuracy_top1'], dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, batch_size=self.batch_size, @@ -926,10 +926,10 @@ def search(self): elif self.supernet == 'inc_quantization_ofa_resnet50': runner_predict = QuantizedOFARunner( supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - model_size_predictor=self.predictor_dict['model_size'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_top1'], + latency_predictor=self.predictors['latency'], + model_size_predictor=self.predictors['model_size'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['accuracy_top1'], dataset_path=self.dataset_path, device=self.device, dataloader_workers=self.dataloader_workers, @@ -940,10 +940,10 @@ def search(self): runner_predict = BootstrapNASRunner( bootstrapnas_supernetwork=self.bootstrapnas_supernetwork, supernet=self.supernet, - latency_predictor=self.predictor_dict['latency'], - macs_predictor=self.predictor_dict['macs'], - params_predictor=self.predictor_dict['params'], - acc_predictor=self.predictor_dict['accuracy_top1'], + latency_predictor=self.predictors['latency'], + macs_predictor=self.predictors['macs'], + params_predictor=self.predictors['params'], + acc_predictor=self.predictors['accuracy_top1'], dataset_path=self.dataset_path, batch_size=self.batch_size, device=self.device, From c701c28e1c76acc2fdc32e053965d2ca20c41b41 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Fri, 29 Sep 2023 15:32:18 -0700 Subject: [PATCH 6/9] Add runner base class Signed-off-by: Maciej Szankin --- .../image_classification/ofa/ofa_interface.py | 32 ++++++------ dynast/supernetwork/runner.py | 51 +++++++++++++++++++ tests/supernetwork/test_runner.py | 27 ++++++++++ 3 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 dynast/supernetwork/runner.py create mode 100644 tests/supernetwork/test_runner.py diff --git a/dynast/supernetwork/image_classification/ofa/ofa_interface.py b/dynast/supernetwork/image_classification/ofa/ofa_interface.py index a47804d..10052e6 100644 --- a/dynast/supernetwork/image_classification/ofa/ofa_interface.py +++ b/dynast/supernetwork/image_classification/ofa/ofa_interface.py @@ -31,12 +31,13 @@ ImagenetRunConfig, RunManager, ) +from dynast.supernetwork.runner import Runner from dynast.utils import log from dynast.utils.datasets import ImageNet from dynast.utils.nn import get_macs, get_parameters, measure_latency, validate_classification -class OFARunner: +class OFARunner(Runner): """The OFARunner class manages the sub-network selection from the OFA super-network and the validation measurements of the sub-networks. ResNet50, MobileNetV3 w1.0, and MobileNetV3 w1.2 are currently supported. Imagenet is required for these super-networks `imagenet-ilsvrc2012`. @@ -46,23 +47,26 @@ def __init__( self, supernet: str, dataset_path: str, - predictors: Dict[str, Predictor] = None, + predictors: Dict[str, Predictor] = {}, batch_size: int = 128, eval_batch_size: int = 128, dataloader_workers: int = 4, device: str = 'cpu', test_fraction: float = 1.0, verbose: bool = False, - ): - self.supernet = supernet - self.predictors = predictors - self.batch_size = batch_size - self.eval_batch_size = eval_batch_size - self.device = device - self.test_fraction = test_fraction - self.dataset_path = dataset_path - self.dataloader_workers = dataloader_workers - self.verbose = verbose + ) -> None: + super().__init__( + supernet=supernet, + dataset_path=dataset_path, + predictors=predictors, + batch_size=batch_size, + eval_batch_size=eval_batch_size, + dataloader_workers=dataloader_workers, + device=device, + test_fraction=test_fraction, + verbose=verbose, + ) + ImagenetDataProvider.DEFAULT_PATH = dataset_path self.ofa_network = ofa_model_zoo.ofa_net(supernet, pretrained=True) @@ -84,10 +88,6 @@ def _init_data(self): self.dataloader = None log.warning('No dataset path provided. Cannot validate sub-networks.') - def estimate_metric(self, metric: str, subnet_cfg) -> float: - predicted_val = self.predictors.get(metric).predict(subnet_cfg) - return predicted_val - def validate_top1(self, subnet_cfg, device=None) -> float: device = self.device if not device else device diff --git a/dynast/supernetwork/runner.py b/dynast/supernetwork/runner.py new file mode 100644 index 0000000..6ff8de5 --- /dev/null +++ b/dynast/supernetwork/runner.py @@ -0,0 +1,51 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Dict, Union + +from dynast.predictors.dynamic_predictor import Predictor + + +class Runner(object): + def __init__( + self, + supernet: str, + dataset_path: Union[None, str] = None, + predictors: Dict[str, Predictor] = {}, + batch_size: int = 128, + eval_batch_size: int = 128, + dataloader_workers: int = 4, + device: str = 'cpu', + test_fraction: float = 1.0, + verbose: bool = False, + ) -> None: + self.supernet = supernet + self.predictors = predictors + self.batch_size = batch_size + self.eval_batch_size = eval_batch_size + self.device = device + self.test_fraction = test_fraction + self.dataset_path = dataset_path + self.dataloader_workers = dataloader_workers + self.verbose = verbose + + def estimate_metric(self, metric: str, subnet_cfg) -> float: + predictor: Union[Predictor, None] = self.predictors.get(metric) + if predictor is None: + raise Exception(f'No predictor for metric {metric} was found.') + + predicted_val = predictor.predict(subnet_cfg) + + return predicted_val diff --git a/tests/supernetwork/test_runner.py b/tests/supernetwork/test_runner.py new file mode 100644 index 0000000..ffa6b07 --- /dev/null +++ b/tests/supernetwork/test_runner.py @@ -0,0 +1,27 @@ +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from dynast.supernetwork.runner import Runner + + +def test_estimate_metric_invalid_predictor(): + runner = Runner( + supernet='some_supernet', + predictors={}, + ) + + with pytest.raises(Exception): + runner.estimate_metric(metric='some_metric', subnet_cfg={}) From 762fa7eee4e542e7465628ec73949245c302006e Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Fri, 29 Sep 2023 15:32:38 -0700 Subject: [PATCH 7/9] Update gitignore Signed-off-by: Maciej Szankin --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 8c7e23d..6ba2afa 100644 --- a/.gitignore +++ b/.gitignore @@ -128,3 +128,7 @@ runs/ # Neural Compressor nc_workspace/ + +mod/ +models/ +datasets/ From 5743d3cb5456dca910e08d93c8f675f6deb4293d Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Fri, 29 Sep 2023 22:14:35 -0700 Subject: [PATCH 8/9] Update ViT Runner Signed-off-by: Maciej Szankin --- dynast/search/search_tactic.py | 10 +- .../image_classification/vit/vit_interface.py | 108 ++++++------------ 2 files changed, 38 insertions(+), 80 deletions(-) diff --git a/dynast/search/search_tactic.py b/dynast/search/search_tactic.py index 1233a98..fc18917 100644 --- a/dynast/search/search_tactic.py +++ b/dynast/search/search_tactic.py @@ -404,10 +404,7 @@ def search(self): elif self.supernet == 'vit_base_imagenet': runner_predict = ViTRunner( supernet=self.supernet, - latency_predictor=self.predictors['latency'], - macs_predictor=self.predictors['macs'], - params_predictor=self.predictors['params'], - acc_predictor=self.predictors['accuracy_top1'], + predictors=self.predictors, dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, batch_size=self.batch_size, @@ -913,10 +910,7 @@ def search(self): elif self.supernet == 'vit_base_imagenet': runner_predict = ViTRunner( supernet=self.supernet, - latency_predictor=self.predictors['latency'], - macs_predictor=self.predictors['macs'], - params_predictor=self.predictors['params'], - acc_predictor=self.predictors['accuracy_top1'], + predictors=self.predictors, dataset_path=self.dataset_path, checkpoint_path=self.supernet_ckpt_path, batch_size=self.batch_size, diff --git a/dynast/supernetwork/image_classification/vit/vit_interface.py b/dynast/supernetwork/image_classification/vit/vit_interface.py index 7ce40da..504380b 100644 --- a/dynast/supernetwork/image_classification/vit/vit_interface.py +++ b/dynast/supernetwork/image_classification/vit/vit_interface.py @@ -19,12 +19,15 @@ import time import warnings from datetime import datetime +from typing import Dict, Tuple, Union import numpy as np import torch import torchprofile +from dynast.predictors.dynamic_predictor import Predictor from dynast.search.evaluation_interface import EvaluationInterface +from dynast.supernetwork.runner import Runner from dynast.utils import log from dynast.utils.datasets import ImageNet from dynast.utils.nn import validate_classification @@ -89,7 +92,7 @@ def compute_latency( batch_size=128, device: str = 'cpu', warmup_steps: int = 10, - measure_steps: int = 100, + measure_steps: int = 50, ): """Measure latency of the ViT-based model.""" @@ -154,7 +157,7 @@ def compute_macs(config, model, device: str = 'cpu'): return macs, params -class ViTRunner: +class ViTRunner(Runner): """The ViTRunner class manages the sub-network selection from the BERT super-network and the validation measurements of the sub-networks. Bert-Base network finetuned on SST-2 dataset is currently supported. @@ -163,33 +166,29 @@ class ViTRunner: def __init__( self, supernet, - dataset_path, - acc_predictor=None, - macs_predictor=None, - latency_predictor=None, - params_predictor=None, + dataset_path: Union[None, str] = None, + predictors: Dict[str, Predictor] = {}, batch_size: int = 16, eval_batch_size: int = 128, - checkpoint_path=None, + dataloader_workers: int = 4, device: str = 'cpu', test_fraction: float = 1.0, - warmup_steps: int = 10, - measure_steps: int = 100, - ): - self.supernet = supernet - self.acc_predictor = acc_predictor - self.macs_predictor = macs_predictor - self.latency_predictor = latency_predictor - self.params_predictor = params_predictor - self.batch_size = batch_size - self.eval_batch_size = eval_batch_size - self.dataset_path = dataset_path - self.checkpoint_path = checkpoint_path - self.device = device - self.test_fraction = test_fraction - self.warmup_steps = warmup_steps - self.measure_steps = measure_steps + verbose: bool = False, + checkpoint_path=None, + ) -> None: + super().__init__( + supernet=supernet, + dataset_path=dataset_path, + predictors=predictors, + batch_size=batch_size, + eval_batch_size=eval_batch_size, + dataloader_workers=dataloader_workers, + device=device, + test_fraction=test_fraction, + verbose=verbose, + ) + self.checkpoint_path = checkpoint_path self.supernet_model, self.max_layers = load_supernet(self.checkpoint_path) self._init_data() @@ -204,34 +203,6 @@ def _init_data(self): self.dataloader = None log.warning('No dataset path provided. Cannot validate sub-networks.') - def estimate_accuracy_imagenet( - self, - subnet_cfg: dict, - ) -> float: - top1 = self.acc_predictor.predict(subnet_cfg) - return top1 - - def estimate_macs( - self, - subnet_cfg: dict, - ) -> int: - macs = self.macs_predictor.predict(subnet_cfg) - return macs - - def estimate_parameters( - self, - subnet_cfg: dict, - ) -> int: - parameters = self.params_predictor.predict(subnet_cfg) - return parameters - - def estimate_latency( - self, - subnet_cfg: dict, - ) -> float: - latency = self.latency_predictor.predict(subnet_cfg) - return latency - def validate_accuracy_imagenet( self, subnet_cfg: dict, @@ -247,7 +218,7 @@ def validate_accuracy_imagenet( def validate_macs( self, subnet_cfg: dict, - ) -> float: + ) -> Tuple[float, float]: """Measure Torch model's FLOPs/MACs as per FVCore calculation Args: subnet_cfg: sub-network Torch model @@ -262,6 +233,9 @@ def validate_macs( def measure_latency( self, subnet_cfg: dict, + warmup_steps: int = 10, + measure_steps: int = 50, + device: str = None, ): """Measure Torch model's latency. Args: @@ -270,9 +244,11 @@ def measure_latency( mean latency; std latency """ + device = self.device if not device else device + logging.info( - f'Performing Latency measurements. Warmup = {self.warmup_steps},\ - Measure steps = {self.measure_steps}' + f'Performing Latency measurements. Warmup = {warmup_steps},\ + Measure steps = {measure_steps}' ) lat_mean, lat_std = compute_latency( @@ -280,8 +256,8 @@ def measure_latency( model=self.supernet_model, batch_size=self.batch_size, device=self.device, - warmup_steps=self.warmup_steps, - measure_steps=self.measure_steps, + warmup_steps=warmup_steps, + measure_steps=measure_steps, ) logging.info('Model\'s latency: {} +/- {}'.format(lat_mean, lat_std)) @@ -319,21 +295,9 @@ def eval_subnet(self, x): # Predictor Mode if self.predictor_mode == True: - if 'params' in self.optimization_metrics: - individual_results['params'] = self.evaluator.estimate_parameters( - self.manager.onehot_custom(param_dict, max_layers=self.evaluator.max_layers).reshape(1, -1) - )[0] - if 'latency' in self.optimization_metrics: - individual_results['latency'] = self.evaluator.estimate_latency( - self.manager.onehot_custom(param_dict, max_layers=self.evaluator.max_layers).reshape(1, -1) - )[0] - if 'macs' in self.optimization_metrics: - individual_results['macs'] = self.evaluator.estimate_macs( - self.manager.onehot_custom(param_dict, max_layers=self.evaluator.max_layers).reshape(1, -1) - )[0] - if 'accuracy_top1' in self.optimization_metrics: - individual_results['accuracy_top1'] = self.evaluator.estimate_accuracy_imagenet( - self.manager.onehot_custom(param_dict, max_layers=self.evaluator.max_layers).reshape(1, -1) + for metric in self.optimization_metrics: + individual_results[metric] = self.evaluator.estimate_metric( + metric, self.manager.onehot_custom(param_dict, max_layers=self.evaluator.max_layers).reshape(1, -1) )[0] # Validation Mode From fe05f541a95e95431b7ccf81a4e30ea3bbdc455c Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Fri, 29 Sep 2023 22:19:30 -0700 Subject: [PATCH 9/9] Update OFA Signed-off-by: Maciej Szankin --- .../supernetwork/image_classification/ofa/ofa_interface.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dynast/supernetwork/image_classification/ofa/ofa_interface.py b/dynast/supernetwork/image_classification/ofa/ofa_interface.py index 10052e6..0ecb620 100644 --- a/dynast/supernetwork/image_classification/ofa/ofa_interface.py +++ b/dynast/supernetwork/image_classification/ofa/ofa_interface.py @@ -16,7 +16,7 @@ import csv import uuid from datetime import datetime -from typing import Dict, Tuple +from typing import Dict, Tuple, Union import torch @@ -46,7 +46,7 @@ class OFARunner(Runner): def __init__( self, supernet: str, - dataset_path: str, + dataset_path: Union[None, str] = None, predictors: Dict[str, Predictor] = {}, batch_size: int = 128, eval_batch_size: int = 128, @@ -301,7 +301,6 @@ def eval_subnet(self, x): # Predictor Mode if self.predictor_mode == True: for metric in self.optimization_metrics: - # TODO(macsz) Maybe move [0] to the `estimate_metric`. individual_results[metric] = self.evaluator.estimate_metric( metric, self.manager.onehot_generic(x).reshape(1, -1) )[0]