From fb3d11f06518f4fb038fe0962c52cbbd7dacf4e7 Mon Sep 17 00:00:00 2001 From: Ignas Baranauskas Date: Wed, 14 Aug 2024 16:42:25 +0100 Subject: [PATCH] Fixes flake8 linting errors Signed-off-by: Ignas Baranauskas --- .../kubeflow-pipelines/mpi-job-horovod.py | 10 ++-- .../enas-cnn-cifar10/ModelConstructor.py | 25 +++----- .../enas-cnn-cifar10/op_library.py | 6 +- .../trial-images/pytorch-mnist/mnist.py | 12 ++-- hack/gen-python-sdk/post_gen.py | 20 +++---- .../kubeflow/katib/api/katib_client.py | 58 ++++++++++++------- 6 files changed, 68 insertions(+), 63 deletions(-) diff --git a/examples/v1beta1/kubeflow-pipelines/mpi-job-horovod.py b/examples/v1beta1/kubeflow-pipelines/mpi-job-horovod.py index 764c0227bc3..ef48d916bb8 100644 --- a/examples/v1beta1/kubeflow-pipelines/mpi-job-horovod.py +++ b/examples/v1beta1/kubeflow-pipelines/mpi-job-horovod.py @@ -21,10 +21,12 @@ # This Experiment is similar to this: # https://github.com/kubeflow/katib/blob/master/examples/v1beta1/kubeflow-training-operator/mpijob-horovod.yaml -# Check the training container source code here: https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod. +# Check the training container source code here: +# https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod. # Note: To run this example, your Kubernetes cluster should run MPIJob operator. -# Follow this guide to install MPIJob on your cluster: https://www.kubeflow.org/docs/components/training/mpi/ +# Follow this guide to install MPIJob on your cluster: +# https://www.kubeflow.org/docs/components/training/mpi/ import kfp from kfp import components @@ -48,7 +50,6 @@ def horovod_mnist_hpo( experiment_name: str = "mpi-horovod-mnist", experiment_namespace: str = "kubeflow-user-example-com", ): - # Trial count specification. max_trial_count = 6 max_failed_trial_count = 3 @@ -193,7 +194,8 @@ def horovod_mnist_hpo( # Get the Katib launcher. # Load component from the URL or from the file. katib_experiment_launcher_op = components.load_component_from_url( - "https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kubeflow/katib-launcher/component.yaml" + "https://raw.githubusercontent.com/kubeflow/pipelines/master/" + "components/kubeflow/katib-launcher/component.yaml" ) # katib_experiment_launcher_op = components.load_component_from_file( # "../../../components/kubeflow/katib-launcher/component.yaml" diff --git a/examples/v1beta1/trial-images/enas-cnn-cifar10/ModelConstructor.py b/examples/v1beta1/trial-images/enas-cnn-cifar10/ModelConstructor.py index 461e9a4fdef..8d482dc70fb 100644 --- a/examples/v1beta1/trial-images/enas-cnn-cifar10/ModelConstructor.py +++ b/examples/v1beta1/trial-images/enas-cnn-cifar10/ModelConstructor.py @@ -14,20 +14,11 @@ import json -from keras import backend as K -from keras.layers import Activation -from keras.layers import AveragePooling2D -from keras.layers import BatchNormalization -from keras.layers import concatenate -from keras.layers import Conv2D from keras.layers import Dense from keras.layers import Dropout from keras.layers import GlobalAveragePooling2D from keras.layers import Input -from keras.layers import MaxPooling2D -from keras.layers import ZeroPadding2D from keras.models import Model -import numpy as np from op_library import concat from op_library import conv from op_library import dw_conv @@ -47,8 +38,6 @@ def __init__(self, arc_json, nn_json): def build_model(self): # a list of the data all layers all_layers = [0 for _ in range(self.num_layers + 1)] - # a list of all the dimensions of all layers - all_dims = [0 for _ in range(self.num_layers + 1)] # ================= Stacking layers ================= # Input Layer. Layer 0 @@ -56,18 +45,18 @@ def build_model(self): all_layers[0] = input_layer # Intermediate Layers. Starting from layer 1. - for l in range(1, self.num_layers + 1): + for l_index in range(1, self.num_layers + 1): input_layers = list() - opt = self.arch[l - 1][0] + opt = self.arch[l_index - 1][0] opt_config = self.embedding[str(opt)] - skip = self.arch[l - 1][1 : l + 1] + skip = self.arch[l_index - 1][1 : l_index + 1] # set up the connection to the previous layer first - input_layers.append(all_layers[l - 1]) + input_layers.append(all_layers[l_index - 1]) # then add skip connections - for i in range(l - 1): - if l > 1 and skip[i] == 1: + for i in range(l_index - 1): + if l_index > 1 and skip[i] == 1: input_layers.append(all_layers[i]) layer_input = concat(input_layers) @@ -80,7 +69,7 @@ def build_model(self): elif opt_config["opt_type"] == "reduction": layer_output = reduction(layer_input, opt_config) - all_layers[l] = layer_output + all_layers[l_index] = layer_output # Final Layer # Global Average Pooling, then Fully connected with softmax. diff --git a/examples/v1beta1/trial-images/enas-cnn-cifar10/op_library.py b/examples/v1beta1/trial-images/enas-cnn-cifar10/op_library.py index 4e968fe8f63..919f92fb5b9 100644 --- a/examples/v1beta1/trial-images/enas-cnn-cifar10/op_library.py +++ b/examples/v1beta1/trial-images/enas-cnn-cifar10/op_library.py @@ -18,10 +18,7 @@ from keras.layers import BatchNormalization from keras.layers import concatenate from keras.layers import Conv2D -from keras.layers import Dense from keras.layers import DepthwiseConv2D -from keras.layers import GlobalAveragePooling2D -from keras.layers import Input from keras.layers import MaxPooling2D from keras.layers import SeparableConv2D from keras.layers import ZeroPadding2D @@ -142,7 +139,8 @@ def reduction(x, config): dim = K.int_shape(x) if dim[1] == 1 or dim[2] == 1: print( - "WARNING: One or more dimensions of the input of the reduction layer is 1. It cannot be further reduced. A identity layer will be used instead." + "WARNING: One or more dimensions of the input of the reduction layer is 1. " + "It cannot be further reduced. A identity layer will be used instead." ) return x diff --git a/examples/v1beta1/trial-images/pytorch-mnist/mnist.py b/examples/v1beta1/trial-images/pytorch-mnist/mnist.py index 596ca27028f..fdc87925dea 100644 --- a/examples/v1beta1/trial-images/pytorch-mnist/mnist.py +++ b/examples/v1beta1/trial-images/pytorch-mnist/mnist.py @@ -67,7 +67,7 @@ def train(args, model, device, train_loader, optimizer, epoch): loss.item(), ) logging.info(msg) - niter = epoch * len(train_loader) + batch_idx + niter = epoch * len(train_loader) + batch_idx # noqa: F841 def test(args, model, device, test_loader, epoch, hpt): @@ -88,10 +88,10 @@ def test(args, model, device, test_loader, epoch, hpt): test_loss /= len(test_loader.dataset) test_accuracy = float(correct) / len(test_loader.dataset) + logging.info( - "{{metricName: accuracy, metricValue: {:.4f}}};{{metricName: loss, metricValue: {:.4f}}}\n".format( - test_accuracy, test_loss - ) + "{{metricName: accuracy, metricValue: {:.4f}}};" + "{{metricName: loss, metricValue: {:.4f}}}\n".format(test_accuracy, test_loss) ) if args.logger == "hypertune": @@ -239,7 +239,7 @@ def main(): ), batch_size=args.batch_size, shuffle=True, - **kwargs + **kwargs, ) test_loader = torch.utils.data.DataLoader( @@ -248,7 +248,7 @@ def main(): ), batch_size=args.test_batch_size, shuffle=False, - **kwargs + **kwargs, ) model = Net().to(device) diff --git a/hack/gen-python-sdk/post_gen.py b/hack/gen-python-sdk/post_gen.py index db8daad3898..b56d259b3ac 100644 --- a/hack/gen-python-sdk/post_gen.py +++ b/hack/gen-python-sdk/post_gen.py @@ -34,7 +34,7 @@ def _rewrite_helper(input_file, output_file, rewrite_rules): for rule in rules: line = rule(line) # Remove ignored lines. - if not any(l in line for l in IGNORE_LINES): + if not any(li in line for li in IGNORE_LINES): lines.append(line) # Add Katib APIs to the init file. @@ -73,34 +73,34 @@ def update_python_sdk(src, dest, versions=("v1beta1")): # tiny transformers to refine generated codes rewrite_rules = [ # Models rules. - lambda l: l.replace("import katib", "import kubeflow.katib"), - lambda l: l.replace("from katib", "from kubeflow.katib"), + lambda line: line.replace("import katib", "import kubeflow.katib"), + lambda line: line.replace("from katib", "from kubeflow.katib"), # For the api_client.py. - lambda l: l.replace( + lambda line: line.replace( "klass = getattr(katib.models, klass)", "klass = getattr(kubeflow.katib.models, klass)", ), # Doc rules. - lambda l: l.replace("[**datetime**](V1Time.md)", "**datetime**"), - lambda l: l.replace( + lambda line: line.replace("[**datetime**](V1Time.md)", "**datetime**"), + lambda line: line.replace( "[**object**](V1UnstructuredUnstructured.md)", "**object**" ), - lambda l: l.replace( + lambda line: line.replace( "[**V1Container**](V1Container.md)", "[**V1Container**](https://github.com/kubernetes-client/" "python/blob/master/kubernetes/docs/V1Container.md)", ), - lambda l: l.replace( + lambda line: line.replace( "[**V1ObjectMeta**](V1ObjectMeta.md)", "[**V1ObjectMeta**](https://github.com/kubernetes-client/" "python/blob/master/kubernetes/docs/V1ObjectMeta.md)", ), - lambda l: l.replace( + lambda line: line.replace( "[**V1ListMeta**](V1ListMeta.md)", "[**V1ListMeta**](https://github.com/kubernetes-client/" "python/blob/master/kubernetes/docs/V1ListMeta.md)", ), - lambda l: l.replace( + lambda line: line.replace( "[**V1HTTPGetAction**](V1HTTPGetAction.md)", "[**V1HTTPGetAction**](https://github.com/kubernetes-client/" "python/blob/master/kubernetes/docs/V1HTTPGetAction.md)", diff --git a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py index 1677e79ea10..8f69b50eb81 100644 --- a/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py +++ b/sdk/python/v1beta1/kubeflow/katib/api/katib_client.py @@ -106,7 +106,7 @@ def create_experiment( namespace = namespace or self.namespace experiment_name = None - if type(experiment) == models.V1beta1Experiment: + if type(experiment) is models.V1beta1Experiment: if experiment.metadata.name is not None: experiment_name = experiment.metadata.name elif experiment.metadata.generate_name is not None: @@ -137,7 +137,8 @@ def create_experiment( except Exception as e: if hasattr(e, "status") and e.status == 409: raise Exception( - f"A Katib Experiment with the name {namespace}/{experiment_name} already exists." + f"A Katib Experiment with the name " + f"{namespace}/{experiment_name} already exists." ) raise RuntimeError( f"Failed to create Katib Experiment: {namespace}/{experiment_name}" @@ -149,16 +150,17 @@ def create_experiment( if self.in_cluster: import IPython - IPython.display.display( - IPython.display.HTML( - "Katib Experiment {} " - 'link here'.format( - experiment_name, - namespace, - experiment_name, - ) + IPython.display.display( + IPython.display.HTML( + "Katib Experiment {} " + 'link here'.format( + experiment_name, + namespace, + experiment_name, ) ) + ) def tune( self, @@ -215,7 +217,8 @@ def tune( https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1EnvFromSource.md) algorithm_name: Search algorithm for the HyperParameter tuning. algorithm_settings: Settings for the search algorithm given. - For available fields, check this doc: https://www.kubeflow.org/docs/components/katib/experiment/#search-algorithms-in-detail. + For available fields, check this doc: + https://www.kubeflow.org/docs/components/katib/experiment/#search-algorithms-in-detail. objective_metric_name: Objective metric that Katib optimizes. additional_metric_names: List of metrics that Katib collects from the objective function in addition to objective metric. @@ -223,7 +226,8 @@ def tune( Must be one of `minimize` or `maximize`. objective_goal: Objective goal that Experiment should reach to be Succeeded. max_trial_count: Maximum number of Trials to run. For the default - values check this doc: https://www.kubeflow.org/docs/components/katib/experiment/#configuration-spec. + values check this doc: + https://www.kubeflow.org/docs/components/katib/experiment/#configuration-spec. parallel_trial_count: Number of Trials that Experiment runs in parallel. max_failed_trial_count: Maximum number of Trials allowed to fail. resources_per_trial: A parameter that lets you specify how much @@ -330,10 +334,14 @@ def tune( # Otherwise, add value to the function input. input_params[p_name] = p_value - # Wrap objective function to execute it from the file. For example + # Wrap objective function to execute it from the file. For example: # def objective(parameters): # print(f'Parameters are {parameters}') - # objective({'lr': '${trialParameters.lr}', 'epochs': '${trialParameters.epochs}', 'is_dist': False}) + # objective({ + # 'lr': '${trialParameters.lr}', + # 'epochs': '${trialParameters.epochs}', + # 'is_dist': False + # }) objective_code = f"{objective_code}\n{objective.__name__}({input_params})\n" # Prepare execute script template. @@ -385,7 +393,8 @@ def tune( ) # Add metrics collector to the Katib Experiment. - # Up to now, We only support parameter `kind`, of which default value is `StdOut`, to specify the kind of metrics collector. + # Up to now, we only support parameter `kind`, of which default value + # is `StdOut`, to specify the kind of metrics collector. experiment.spec.metrics_collector_spec = models.V1beta1MetricsCollectorSpec( collector=models.V1beta1CollectorSpec(kind=metrics_collector_config["kind"]) ) @@ -825,20 +834,24 @@ def wait_for_experiment_condition( ) ): utils.print_experiment_status(experiment) + logger.debug( - f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n" + f"waiting for experiment: {namespace}/{name} " + f"to reach {expected_condition} condition\n\n\n" ) return experiment # Otherwise, print the current Experiment results and sleep for the pooling interval. utils.print_experiment_status(experiment) logger.debug( - f"Waiting for Experiment: {namespace}/{name} to reach {expected_condition} condition\n\n\n" + f"waiting for experiment: {namespace}/{name} " + f"to reach {expected_condition} condition\n\n\n" ) time.sleep(polling_interval) raise TimeoutError( - f"Timeout waiting for Experiment: {namespace}/{name} to reach {expected_condition} state" + f"Timeout waiting for Experiment: {namespace}/{name} " + f"to reach {expected_condition} state" ) def edit_experiment_budget( @@ -854,7 +867,8 @@ def edit_experiment_budget( budget to resume Succeeded Experiments with `LongRunning` and `FromVolume` resume policies. - Learn about resuming Experiments here: https://www.kubeflow.org/docs/components/katib/resume-experiment/ + Learn about resuming Experiments here: + https://www.kubeflow.org/docs/components/katib/resume-experiment/ Args: name: Name for the Experiment. @@ -1267,10 +1281,12 @@ def get_trial_metrics( use the default Katib DB Manager address: `katib-db-manager.kubeflow:6789`. If you run this API outside the cluster, you have to port-forward the - Katib DB Manager before getting the Trial metrics: `kubectl port-forward svc/katib-db-manager -n kubeflow 6789`. + Katib DB Manager before getting the Trial metrics: + `kubectl port-forward svc/katib-db-manager -n kubeflow 6789`. In that case, you can use this Katib DB Manager address: `localhost:6789`. - You can use `curl` to verify that Katib DB Manager is reachable: `curl `. + You can use `curl` to verify that Katib DB Manager is reachable: + `curl `. Args: name: Name for the Trial.