Begin enabling pre-commit hooks (#2242)

* Begin enabling pre-commit hooks

Signed-off-by: droctothorpe <[email protected]>

* Address PR feedback

Signed-off-by: droctothorpe <[email protected]>

---------

Signed-off-by: droctothorpe <[email protected]>

.github/workflows/test-lint.yaml

@@ -23,8 +23,8 @@ jobs:
         with:
           python-version: 3.9
 
-      - name: Check YAML files
-        run: make yamllint
-
       - name: Check shell scripts
         run: make shellcheck
+
+      - name: Run pre-commit
+        uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+        args: [--allow-multiple-documents]
+      - id: check-json
+  - repo: https://github.com/pycqa/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        name: isort
+        entry: isort --profile google
+exclude: |
+  (?x)^(
+    .*zz_generated.deepcopy.*|
+    .*pb.go|
+    pkg/apis/manager/.*pb2(?:_grpc)?.py(?:i)?|
+    pkg/apis/v1beta1/openapi_generated.go|
+    pkg/mock/.*|
+    pkg/client/controller/.*|
+    sdk/python/v1beta1/kubeflow/katib/configuration.py|
+    sdk/python/v1beta1/kubeflow/katib/rest.py|
+    sdk/python/v1beta1/kubeflow/katib/__init__.py|
+    sdk/python/v1beta1/kubeflow/katib/exceptions.py|
+    sdk/python/v1beta1/kubeflow/katib/api_client.py|
+    sdk/python/v1beta1/kubeflow/katib/models/.*
+  )$

cmd/earlystopping/medianstop/v1beta1/main.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
-import time
+from concurrent import futures
 import logging
+import time
+
+import grpc
+
 from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 from pkg.earlystopping.v1beta1.medianstop.service import MedianStopService
-from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 DEFAULT_PORT = "0.0.0.0:6788"

cmd/metricscollector/v1beta1/tfevent-metricscollector/main.py

@@ -12,14 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
 import argparse
+from logging import getLogger
+from logging import INFO
+from logging import StreamHandler
+
 import api_pb2
 import api_pb2_grpc
-from pns import WaitMainProcesses
 import const
+import grpc
+from pns import WaitMainProcesses
 from tfevent_loader import MetricsCollector
-from logging import getLogger, StreamHandler, INFO
 
 timeout_in_seconds = 60
 

cmd/suggestion/hyperband/v1beta1/main.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
+from concurrent import futures
 import time
-from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+
+import grpc
+
 from pkg.apis.manager.health.python import health_pb2_grpc
+from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 from pkg.suggestion.v1beta1.hyperband.service import HyperbandService
-from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 DEFAULT_PORT = "0.0.0.0:6789"

cmd/suggestion/hyperopt/v1beta1/main.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
+from concurrent import futures
 import time
-from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+
+import grpc
+
 from pkg.apis.manager.health.python import health_pb2_grpc
+from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 from pkg.suggestion.v1beta1.hyperopt.service import HyperoptService
-from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 DEFAULT_PORT = "0.0.0.0:6789"

cmd/suggestion/nas/darts/v1beta1/main.py

@@ -12,14 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
 from concurrent import futures
 import time
-from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+
+import grpc
+
 from pkg.apis.manager.health.python import health_pb2_grpc
+from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 from pkg.suggestion.v1beta1.nas.darts.service import DartsService
 
-
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 DEFAULT_PORT = "0.0.0.0:6789"
 

cmd/suggestion/nas/enas/v1beta1/main.py

@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
 from concurrent import futures
 import time
 
-from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+import grpc
+
 from pkg.apis.manager.health.python import health_pb2_grpc
+from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 from pkg.suggestion.v1beta1.nas.enas.service import EnasService
 
-
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 DEFAULT_PORT = "0.0.0.0:6789"
 

cmd/suggestion/optuna/v1beta1/main.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
+from concurrent import futures
 import time
-from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+
+import grpc
+
 from pkg.apis.manager.health.python import health_pb2_grpc
+from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 from pkg.suggestion.v1beta1.optuna.service import OptunaService
-from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 DEFAULT_PORT = "0.0.0.0:6789"

cmd/suggestion/pbt/v1beta1/main.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
+from concurrent import futures
 import time
-from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+
+import grpc
+
 from pkg.apis.manager.health.python import health_pb2_grpc
+from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 from pkg.suggestion.v1beta1.pbt.service import PbtService
-from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 DEFAULT_PORT = "0.0.0.0:6789"

cmd/suggestion/skopt/v1beta1/main.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import grpc
+from concurrent import futures
 import time
-from pkg.apis.manager.v1beta1.python import api_pb2_grpc
+
+import grpc
+
 from pkg.apis.manager.health.python import health_pb2_grpc
+from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 from pkg.suggestion.v1beta1.skopt.service import SkoptService
-from concurrent import futures
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 DEFAULT_PORT = "0.0.0.0:6789"

docs/developer-guide.md

@@ -18,6 +18,7 @@ see the following user guides:
 - [Java](https://docs.oracle.com/javase/8/docs/technotes/guides/install/install_overview.html) (8 or later)
 - [Python](https://www.python.org/) (3.11 or later)
 - [kustomize](https://kustomize.io/) (4.0.5 or later)
+- [pre-commit](https://pre-commit.com/)
 
 ## Build from source code
 
@@ -150,3 +151,21 @@ Please see [Katib UI README](../pkg/ui/v1beta1).
 ## Design proposals
 
 Please see [proposals](./proposals).
+
+## Code Style
+
+### pre-commit
+
+Make sure to install [pre-commit](https://pre-commit.com/) (`pip install
+pre-commit`) and run `pre-commit install` from the root of the repository at
+least once before creating git commits.
+
+The pre-commit [hooks](../.pre-commit-config.yaml) ensure code quality and
+consistency. They are executed in CI. PRs that fail to comply with the hooks
+will not be able to pass the corresponding CI gate. The hooks are only executed
+against staged files unless you run `pre-commit run --all`, in which case,
+they'll be executed against every file in the repository.
+
+Specific programmatically generated files listed in the `exclude` field in
+[.pre-commit-config.yaml](../.pre-commit-config.yaml) are deliberately excluded
+from the hooks. 

docs/release/changelog.py

@@ -1,6 +1,7 @@
-from github import Github
 import argparse
 
+from github import Github
+
 REPO_NAME = "kubeflow/katib"
 CHANGELOG_FILE = "CHANGELOG.md"
 

examples/v1beta1/kubeflow-pipelines/mpi-job-horovod.py

@@ -27,18 +27,17 @@
 # Follow this guide to install MPIJob on your cluster: https://www.kubeflow.org/docs/components/training/mpi/
 
 import kfp
-import kfp.dsl as dsl
 from kfp import components
-
+import kfp.dsl as dsl
 from kubeflow.katib import ApiClient
-from kubeflow.katib import V1beta1ExperimentSpec
-from kubeflow.katib import V1beta1AlgorithmSpec
 from kubeflow.katib import V1beta1AlgorithmSetting
+from kubeflow.katib import V1beta1AlgorithmSpec
+from kubeflow.katib import V1beta1ExperimentSpec
+from kubeflow.katib import V1beta1FeasibleSpace
 from kubeflow.katib import V1beta1ObjectiveSpec
 from kubeflow.katib import V1beta1ParameterSpec
-from kubeflow.katib import V1beta1FeasibleSpace
-from kubeflow.katib import V1beta1TrialTemplate
 from kubeflow.katib import V1beta1TrialParameterSpec
+from kubeflow.katib import V1beta1TrialTemplate
 
 
 @dsl.pipeline(

examples/v1beta1/trial-images/darts-cnn-cifar10/architect.py

@@ -12,9 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import torch
 import copy
 
+import torch
+
 
 class Architect():
     """" Architect controls architecture of cell by computing gradients of alphas

examples/v1beta1/trial-images/darts-cnn-cifar10/model.py

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from operations import FactorizedReduce
+from operations import MixedOp
+from operations import StdConv
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from operations import FactorizedReduce, StdConv, MixedOp
 
 
 class Cell(nn.Module):

examples/v1beta1/trial-images/darts-cnn-cifar10/operations.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import torch.nn as nn
 import torch
+import torch.nn as nn
 
 OPS = {
     'none': lambda channels, stride: Zero(stride),

examples/v1beta1/trial-images/darts-cnn-cifar10/run_trial.py

@@ -13,17 +13,16 @@
 # limitations under the License.
 
 
-import torch.nn as nn
-
-import torch
 import argparse
 import json
-import numpy as np
 
-from model import NetworkCNN
 from architect import Architect
-import utils
+from model import NetworkCNN
+import numpy as np
 from search_space import SearchSpace
+import torch
+import torch.nn as nn
+import utils
 
 
 def main():

examples/v1beta1/trial-images/darts-cnn-cifar10/search_space.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from collections import namedtuple
+
 import torch
 
 

examples/v1beta1/trial-images/enas-cnn-cifar10/ModelConstructor.py

@@ -12,13 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import numpy as np
-from keras.models import Model
-from keras import backend as K
 import json
-from keras.layers import Input, Conv2D, ZeroPadding2D, concatenate, MaxPooling2D, \
-    AveragePooling2D, Dense, Activation, BatchNormalization, GlobalAveragePooling2D, Dropout
-from op_library import concat, conv, sp_conv, dw_conv, reduction
+
+from keras import backend as K
+from keras.layers import Activation
+from keras.layers import AveragePooling2D
+from keras.layers import BatchNormalization
+from keras.layers import concatenate
+from keras.layers import Conv2D
+from keras.layers import Dense
+from keras.layers import Dropout
+from keras.layers import GlobalAveragePooling2D
+from keras.layers import Input
+from keras.layers import MaxPooling2D
+from keras.layers import ZeroPadding2D
+from keras.models import Model
+import numpy as np
+from op_library import concat
+from op_library import conv
+from op_library import dw_conv
+from op_library import reduction
+from op_library import sp_conv
 
 
 class ModelConstructor(object):

examples/v1beta1/trial-images/enas-cnn-cifar10/RunTrial.py

@@ -12,13 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from tensorflow import keras
+import argparse
+
 from keras.datasets import cifar10
 from ModelConstructor import ModelConstructor
-from tensorflow.keras.utils import to_categorical
-from tensorflow.keras.layers import RandomFlip, RandomTranslation, Rescaling
+from tensorflow import keras
 import tensorflow as tf
-import argparse
+from tensorflow.keras.layers import RandomFlip
+from tensorflow.keras.layers import RandomTranslation
+from tensorflow.keras.layers import Rescaling
+from tensorflow.keras.utils import to_categorical
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='TrainingContainer')