Fixes flake8 linting errors

Signed-off-by: Ignas Baranauskas <[email protected]>

examples/v1beta1/kubeflow-pipelines/mpi-job-horovod.py

@@ -21,10 +21,12 @@
 
 # This Experiment is similar to this:
 # https://github.com/kubeflow/katib/blob/master/examples/v1beta1/kubeflow-training-operator/mpijob-horovod.yaml
-# Check the training container source code here: https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod.
+# Check the training container source code here:
+# https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod.
 
 # Note: To run this example, your Kubernetes cluster should run MPIJob operator.
-# Follow this guide to install MPIJob on your cluster: https://www.kubeflow.org/docs/components/training/mpi/
+# Follow this guide to install MPIJob on your cluster:
+# https://www.kubeflow.org/docs/components/training/mpi/
 
 import kfp
 from kfp import components
@@ -48,7 +50,6 @@ def horovod_mnist_hpo(
     experiment_name: str = "mpi-horovod-mnist",
     experiment_namespace: str = "kubeflow-user-example-com",
 ):
-
     # Trial count specification.
     max_trial_count = 6
     max_failed_trial_count = 3
@@ -193,7 +194,8 @@ def horovod_mnist_hpo(
     # Get the Katib launcher.
     # Load component from the URL or from the file.
     katib_experiment_launcher_op = components.load_component_from_url(
-        "https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kubeflow/katib-launcher/component.yaml"
+        "https://raw.githubusercontent.com/kubeflow/pipelines/master/"
+        "components/kubeflow/katib-launcher/component.yaml"
     )
     # katib_experiment_launcher_op = components.load_component_from_file(
     #     "../../../components/kubeflow/katib-launcher/component.yaml"

examples/v1beta1/trial-images/enas-cnn-cifar10/ModelConstructor.py

@@ -14,20 +14,11 @@
 
 import json
 
-from keras import backend as K
-from keras.layers import Activation
-from keras.layers import AveragePooling2D
-from keras.layers import BatchNormalization
-from keras.layers import concatenate
-from keras.layers import Conv2D
 from keras.layers import Dense
 from keras.layers import Dropout
 from keras.layers import GlobalAveragePooling2D
 from keras.layers import Input
-from keras.layers import MaxPooling2D
-from keras.layers import ZeroPadding2D
 from keras.models import Model
-import numpy as np
 from op_library import concat
 from op_library import conv
 from op_library import dw_conv
@@ -47,27 +38,25 @@ def __init__(self, arc_json, nn_json):
     def build_model(self):
         # a list of the data all layers
         all_layers = [0 for _ in range(self.num_layers + 1)]
-        # a list of all the dimensions of all layers
-        all_dims = [0 for _ in range(self.num_layers + 1)]
 
         # ================= Stacking layers =================
         # Input Layer. Layer 0
         input_layer = Input(shape=self.input_sizes)
         all_layers[0] = input_layer
 
         # Intermediate Layers. Starting from layer 1.
-        for l in range(1, self.num_layers + 1):
+        for l_index in range(1, self.num_layers + 1):
             input_layers = list()
-            opt = self.arch[l - 1][0]
+            opt = self.arch[l_index - 1][0]
             opt_config = self.embedding[str(opt)]
-            skip = self.arch[l - 1][1 : l + 1]
+            skip = self.arch[l_index - 1][1 : l_index + 1]
 
             # set up the connection to the previous layer first
-            input_layers.append(all_layers[l - 1])
+            input_layers.append(all_layers[l_index - 1])
 
             # then add skip connections
-            for i in range(l - 1):
-                if l > 1 and skip[i] == 1:
+            for i in range(l_index - 1):
+                if l_index > 1 and skip[i] == 1:
                     input_layers.append(all_layers[i])
 
             layer_input = concat(input_layers)
@@ -80,7 +69,7 @@ def build_model(self):
             elif opt_config["opt_type"] == "reduction":
                 layer_output = reduction(layer_input, opt_config)
 
-            all_layers[l] = layer_output
+            all_layers[l_index] = layer_output
 
         # Final Layer
         # Global Average Pooling, then Fully connected with softmax.

examples/v1beta1/trial-images/enas-cnn-cifar10/op_library.py

@@ -18,10 +18,7 @@
 from keras.layers import BatchNormalization
 from keras.layers import concatenate
 from keras.layers import Conv2D
-from keras.layers import Dense
 from keras.layers import DepthwiseConv2D
-from keras.layers import GlobalAveragePooling2D
-from keras.layers import Input
 from keras.layers import MaxPooling2D
 from keras.layers import SeparableConv2D
 from keras.layers import ZeroPadding2D
@@ -142,7 +139,8 @@ def reduction(x, config):
     dim = K.int_shape(x)
     if dim[1] == 1 or dim[2] == 1:
         print(
-            "WARNING: One or more dimensions of the input of the reduction layer is 1. It cannot be further reduced. A identity layer will be used instead."
+            "WARNING: One or more dimensions of the input of the reduction layer is 1. "
+            "It cannot be further reduced. A identity layer will be used instead."
         )
         return x
 

examples/v1beta1/trial-images/pytorch-mnist/mnist.py

@@ -67,7 +67,7 @@ def train(args, model, device, train_loader, optimizer, epoch):
                 loss.item(),
             )
             logging.info(msg)
-            niter = epoch * len(train_loader) + batch_idx
+            niter = epoch * len(train_loader) + batch_idx  # noqa: F841
 
 
 def test(args, model, device, test_loader, epoch, hpt):
@@ -88,10 +88,10 @@ def test(args, model, device, test_loader, epoch, hpt):
 
     test_loss /= len(test_loader.dataset)
     test_accuracy = float(correct) / len(test_loader.dataset)
+
     logging.info(
-        "{{metricName: accuracy, metricValue: {:.4f}}};{{metricName: loss, metricValue: {:.4f}}}\n".format(
-            test_accuracy, test_loss
-        )
+        "{{metricName: accuracy, metricValue: {:.4f}}};"
+        "{{metricName: loss, metricValue: {:.4f}}}\n".format(test_accuracy, test_loss)
     )
 
     if args.logger == "hypertune":
@@ -239,7 +239,7 @@ def main():
         ),
         batch_size=args.batch_size,
         shuffle=True,
-        **kwargs
+        **kwargs,
     )
 
     test_loader = torch.utils.data.DataLoader(
@@ -248,7 +248,7 @@ def main():
         ),
         batch_size=args.test_batch_size,
         shuffle=False,
-        **kwargs
+        **kwargs,
     )
 
     model = Net().to(device)

hack/gen-python-sdk/post_gen.py

@@ -34,7 +34,7 @@ def _rewrite_helper(input_file, output_file, rewrite_rules):
             for rule in rules:
                 line = rule(line)
             # Remove ignored lines.
-            if not any(l in line for l in IGNORE_LINES):
+            if not any(li in line for li in IGNORE_LINES):
                 lines.append(line)
 
     # Add Katib APIs to the init file.
@@ -73,34 +73,34 @@ def update_python_sdk(src, dest, versions=("v1beta1")):
     # tiny transformers to refine generated codes
     rewrite_rules = [
         # Models rules.
-        lambda l: l.replace("import katib", "import kubeflow.katib"),
-        lambda l: l.replace("from katib", "from kubeflow.katib"),
+        lambda line: line.replace("import katib", "import kubeflow.katib"),
+        lambda line: line.replace("from katib", "from kubeflow.katib"),
         # For the api_client.py.
-        lambda l: l.replace(
+        lambda line: line.replace(
             "klass = getattr(katib.models, klass)",
             "klass = getattr(kubeflow.katib.models, klass)",
         ),
         # Doc rules.
-        lambda l: l.replace("[**datetime**](V1Time.md)", "**datetime**"),
-        lambda l: l.replace(
+        lambda line: line.replace("[**datetime**](V1Time.md)", "**datetime**"),
+        lambda line: line.replace(
             "[**object**](V1UnstructuredUnstructured.md)", "**object**"
         ),
-        lambda l: l.replace(
+        lambda line: line.replace(
             "[**V1Container**](V1Container.md)",
             "[**V1Container**](https://github.com/kubernetes-client/"
             "python/blob/master/kubernetes/docs/V1Container.md)",
         ),
-        lambda l: l.replace(
+        lambda line: line.replace(
             "[**V1ObjectMeta**](V1ObjectMeta.md)",
             "[**V1ObjectMeta**](https://github.com/kubernetes-client/"
             "python/blob/master/kubernetes/docs/V1ObjectMeta.md)",
         ),
-        lambda l: l.replace(
+        lambda line: line.replace(
             "[**V1ListMeta**](V1ListMeta.md)",
             "[**V1ListMeta**](https://github.com/kubernetes-client/"
             "python/blob/master/kubernetes/docs/V1ListMeta.md)",
         ),
-        lambda l: l.replace(
+        lambda line: line.replace(
             "[**V1HTTPGetAction**](V1HTTPGetAction.md)",
             "[**V1HTTPGetAction**](https://github.com/kubernetes-client/"
             "python/blob/master/kubernetes/docs/V1HTTPGetAction.md)",

sdk/python/v1beta1/kubeflow/katib/api/katib_client.py

@@ -106,7 +106,7 @@ def create_experiment(
         namespace = namespace or self.namespace
 
         experiment_name = None
-        if type(experiment) == models.V1beta1Experiment:
+        if type(experiment) is models.V1beta1Experiment:
             if experiment.metadata.name is not None:
                 experiment_name = experiment.metadata.name
             elif experiment.metadata.generate_name is not None:
@@ -137,7 +137,8 @@ def create_experiment(
         except Exception as e:
             if hasattr(e, "status") and e.status == 409:
                 raise Exception(
-                    f"A Katib Experiment with the name {namespace}/{experiment_name} already exists."
+                    f"A Katib Experiment with the name "
+                    f"{namespace}/{experiment_name} already exists."
                 )
             raise RuntimeError(
                 f"Failed to create Katib Experiment: {namespace}/{experiment_name}"
@@ -149,16 +150,17 @@ def create_experiment(
             if self.in_cluster:
                 import IPython
 
-                IPython.display.display(
-                    IPython.display.HTML(
-                        "Katib Experiment {} "
-                        'link <a href="/_/katib/#/katib/hp_monitor/{}/{}" target="_blank">here</a>'.format(
-                            experiment_name,
-                            namespace,
-                            experiment_name,
-                        )
+            IPython.display.display(
+                IPython.display.HTML(
+                    "Katib Experiment {} "
+                    'link <a href="/_/katib/#/katib/hp_monitor/{}/{}" '
+                    'target="_blank">here</a>'.format(
+                        experiment_name,
+                        namespace,
+                        experiment_name,
                     )
                 )
+            )
 
     def tune(
         self,
@@ -215,15 +217,17 @@ def tune(
                 https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1EnvFromSource.md)
             algorithm_name: Search algorithm for the HyperParameter tuning.
             algorithm_settings: Settings for the search algorithm given.
-                For available fields, check this doc: https://www.kubeflow.org/docs/components/katib/experiment/#search-algorithms-in-detail.
+                For available fields, check this doc:
+                https://www.kubeflow.org/docs/components/katib/experiment/#search-algorithms-in-detail.
             objective_metric_name: Objective metric that Katib optimizes.
             additional_metric_names: List of metrics that Katib collects from the
                 objective function in addition to objective metric.
             objective_type: Type for the Experiment optimization for the objective metric.
                 Must be one of `minimize` or `maximize`.
             objective_goal: Objective goal that Experiment should reach to be Succeeded.
             max_trial_count: Maximum number of Trials to run. For the default
-                values check this doc: https://www.kubeflow.org/docs/components/katib/experiment/#configuration-spec.
+                values check this doc:
+                https://www.kubeflow.org/docs/components/katib/experiment/#configuration-spec.
             parallel_trial_count: Number of Trials that Experiment runs in parallel.
             max_failed_trial_count: Maximum number of Trials allowed to fail.
             resources_per_trial: A parameter that lets you specify how much
@@ -330,10 +334,14 @@ def tune(
                 # Otherwise, add value to the function input.
                 input_params[p_name] = p_value
 
-        # Wrap objective function to execute it from the file. For example
+        # Wrap objective function to execute it from the file. For example:
         # def objective(parameters):
         #     print(f'Parameters are {parameters}')
-        # objective({'lr': '${trialParameters.lr}', 'epochs': '${trialParameters.epochs}', 'is_dist': False})
+        # objective({
+        #     'lr': '${trialParameters.lr}',
+        #     'epochs': '${trialParameters.epochs}',
+        #     'is_dist': False
+        # })
         objective_code = f"{objective_code}\n{objective.__name__}({input_params})\n"
 
         # Prepare execute script template.
@@ -385,7 +393,8 @@ def tune(
                     )
 
         # Add metrics collector to the Katib Experiment.
-        # Up to now, We only support parameter `kind`, of which default value is `StdOut`, to specify the kind of metrics collector.
+        # Up to now, we only support parameter `kind`, of which default value
+        # is `StdOut`, to specify the kind of metrics collector.
         experiment.spec.metrics_collector_spec = models.V1beta1MetricsCollectorSpec(
             collector=models.V1beta1CollectorSpec(kind=metrics_collector_config["kind"])
         )
@@ -825,20 +834,24 @@ def wait_for_experiment_condition(
                 )
             ):
                 utils.print_experiment_status(experiment)
+
                 logger.debug(
-                    f"Experiment: {namespace}/{name} is {expected_condition}\n\n\n"
+                    f"waiting for experiment: {namespace}/{name} "
+                    f"to reach {expected_condition} condition\n\n\n"
                 )
                 return experiment
 
             # Otherwise, print the current Experiment results and sleep for the pooling interval.
             utils.print_experiment_status(experiment)
             logger.debug(
-                f"Waiting for Experiment: {namespace}/{name} to reach {expected_condition} condition\n\n\n"
+                f"waiting for experiment: {namespace}/{name} "
+                f"to reach {expected_condition} condition\n\n\n"
             )
             time.sleep(polling_interval)
 
         raise TimeoutError(
-            f"Timeout waiting for Experiment: {namespace}/{name} to reach {expected_condition} state"
+            f"Timeout waiting for Experiment: {namespace}/{name} "
+            f"to reach {expected_condition} state"
         )
 
     def edit_experiment_budget(
@@ -854,7 +867,8 @@ def edit_experiment_budget(
         budget to resume Succeeded Experiments with `LongRunning` and `FromVolume`
         resume policies.
 
-        Learn about resuming Experiments here: https://www.kubeflow.org/docs/components/katib/resume-experiment/
+        Learn about resuming Experiments here:
+        https://www.kubeflow.org/docs/components/katib/resume-experiment/
 
         Args:
             name: Name for the Experiment.
@@ -1267,10 +1281,12 @@ def get_trial_metrics(
         use the default Katib DB Manager address: `katib-db-manager.kubeflow:6789`.
 
         If you run this API outside the cluster, you have to port-forward the
-        Katib DB Manager before getting the Trial metrics: `kubectl port-forward svc/katib-db-manager -n kubeflow 6789`.
+        Katib DB Manager before getting the Trial metrics:
+        `kubectl port-forward svc/katib-db-manager -n kubeflow 6789`.
         In that case, you can use this Katib DB Manager address: `localhost:6789`.
 
-        You can use `curl` to verify that Katib DB Manager is reachable: `curl <db-manager-address>`.
+        You can use `curl` to verify that Katib DB Manager is reachable:
+        `curl <db-manager-address>`.
 
         Args:
             name: Name for the Trial.