Import the JobSet models

Use kube-openapi generator to remove unnecessary models from apimachinery Signed-off-by: Andrey Velichkevich <[email protected]>
kubeflow · Nov 28, 2024 · e33549e · e33549e
1 parent dd3a7a6
commit e33549e
Show file tree

Hide file tree

Showing 66 changed files with 77 additions and 16,784 deletions.
diff --git a/api.v2/openapi-spec/swagger.json b/api.v2/openapi-spec/swagger.json
@@ -199,7 +199,7 @@
         "spec": {
           "description": "Specification of the desired JobSet which will be created from TrainJob.",
           "default": {},
-          "$ref": "#/definitions/sigs.k8s.io.jobset.api.jobset.v1alpha2.JobSetSpec"
+          "$ref": "#/definitions/jobset.v1alpha2.JobSetSpec"
         }
       }
     },

diff --git a/hack/python-sdk-v2/gen-sdk.sh b/hack/python-sdk-v2/gen-sdk.sh
@@ -66,5 +66,5 @@ else
 fi
 
 # Kubeflow models must have Kubernetes models to perform serialization.
-printf "\n# Import Kubernetes models for the serialization\n" >>${SDK_OUTPUT_PATH}/kubeflow/training/models/__init__.py
-printf "from kubernetes.client import *\n" >>${SDK_OUTPUT_PATH}/kubeflow/training/models/__init__.py
+printf "\n# Import JobSet models for the serialization. It imports the Kubernetes models.\n" >>${SDK_OUTPUT_PATH}/kubeflow/training/models/__init__.py
+printf "from jobset.models import *\n" >>${SDK_OUTPUT_PATH}/kubeflow/training/models/__init__.py
diff --git a/hack/python-sdk/swagger.json b/hack/python-sdk/swagger.json
diff --git a/hack/swagger-v2/main.go b/hack/swagger-v2/main.go
@@ -66,9 +66,9 @@ func main() {
 
 func swaggify(name string) string {
 	name = strings.Replace(name, "github.com/kubeflow/training-operator/pkg/apis/", "", -1)
+	name = strings.Replace(name, "sigs.k8s.io/jobset/api/", "", -1)
 	name = strings.Replace(name, "k8s.io/api/core/", "", -1)
 	name = strings.Replace(name, "k8s.io/apimachinery/pkg/apis/meta/", "", -1)
-	name = strings.Replace(name, "k8s.io/apimachinery/pkg/api/resource", "", -1)
 	name = strings.Replace(name, "/", ".", -1)
 	return name
 }
diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
@@ -7,44 +7,75 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-GO_CMD=${1:-go}
 CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}")
 TRAINING_OPERATOR_ROOT=$(realpath "${CURRENT_DIR}/..")
 TRAINING_OPERATOR_PKG="github.com/kubeflow/training-operator"
-CODEGEN_PKG=$(go list -m -mod=readonly -f "{{.Dir}}" k8s.io/code-generator)
 
 cd "$CURRENT_DIR/.."
 
-# shellcheck source=/dev/null
+# Get the code-generator binary.
+CODEGEN_PKG=$(go list -m -mod=readonly -f "{{.Dir}}" k8s.io/code-generator)
 source "${CODEGEN_PKG}/kube_codegen.sh"
+echo ">> Using ${CODEGEN_PKG}"
 
-# Generating conversion and defaults functions
+# Generating deepcopy and defaults.
+echo "Generating deepcopy and defaults for kubeflow.org/v1 and kubeflow.org/v2alpha1"
 kube::codegen::gen_helpers \
   --boilerplate "${TRAINING_OPERATOR_ROOT}/hack/boilerplate/boilerplate.go.txt" \
   "${TRAINING_OPERATOR_ROOT}/pkg/apis"
 
-# Generating OpenAPI for Kueue API extensions for v1
-kube::codegen::gen_openapi \
+# Generate clients for Training Operator V1 and V2
+echo "Generating clients for kubeflow.org/v1 and kubeflow.org/v2alpha1"
+kube::codegen::gen_client \
   --boilerplate "${TRAINING_OPERATOR_ROOT}/hack/boilerplate/boilerplate.go.txt" \
+  --output-dir "${TRAINING_OPERATOR_ROOT}/pkg/client" \
+  --output-pkg "${TRAINING_OPERATOR_PKG}/pkg/client" \
+  --with-watch \
+  --with-applyconfig \
+  "${TRAINING_OPERATOR_ROOT}/pkg/apis"
+
+# Get the kube-openapi binary.
+OPENAPI_PKG=$(go list -m -mod=readonly -f "{{.Dir}}" k8s.io/kube-openapi)
+echo ">> Using ${OPENAPI_PKG}"
+
+echo "Generating OpenAPI specification for kubeflow.org/v1"
+go run ${OPENAPI_PKG}/cmd/openapi-gen \
+  --go-header-file "${TRAINING_OPERATOR_ROOT}/hack/boilerplate/boilerplate.go.txt" \
   --output-pkg "${TRAINING_OPERATOR_PKG}/pkg/apis/kubeflow.org/v1" \
   --output-dir "${TRAINING_OPERATOR_ROOT}/pkg/apis/kubeflow.org/v1" \
+  --output-file "zz_generated.openapi.go" \
   --report-filename "${TRAINING_OPERATOR_ROOT}/hack/violation_exception_v1.list" \
-  --update-report \
   "${TRAINING_OPERATOR_ROOT}/pkg/apis/kubeflow.org/v1"
 
-# Generating OpenAPI for Kueue API extensions for v2alpha1
-kube::codegen::gen_openapi \
-  --boilerplate "${TRAINING_OPERATOR_ROOT}/hack/boilerplate/boilerplate.go.txt" \
+echo "Generating OpenAPI specification for kubeflow.org/v2alpha1"
+go run ${OPENAPI_PKG}/cmd/openapi-gen \
+  --go-header-file "${TRAINING_OPERATOR_ROOT}/hack/boilerplate/boilerplate.go.txt" \
   --output-pkg "${TRAINING_OPERATOR_PKG}/pkg/apis/kubeflow.org/v2alpha1" \
   --output-dir "${TRAINING_OPERATOR_ROOT}/pkg/apis/kubeflow.org/v2alpha1" \
+  --output-file "zz_generated.openapi.go" \
   --report-filename "${TRAINING_OPERATOR_ROOT}/hack/violation_exception_v2alpha1.list" \
-  --update-report \
   "${TRAINING_OPERATOR_ROOT}/pkg/apis/kubeflow.org/v2alpha1"
 
-kube::codegen::gen_client \
-  --boilerplate "${TRAINING_OPERATOR_ROOT}/hack/boilerplate/boilerplate.go.txt" \
-  --output-dir "${TRAINING_OPERATOR_ROOT}/pkg/client" \
-  --output-pkg "${TRAINING_OPERATOR_PKG}/pkg/client" \
-  --with-watch \
-  --with-applyconfig \
-  "${TRAINING_OPERATOR_ROOT}/pkg/apis"
+# # Generating OpenAPI spec for Training Operator V1.
+# echo "Generating OpenAPI specification for kubeflow.org/v1"
+# kube::codegen::gen_openapi \
+#   --boilerplate "${TRAINING_OPERATOR_ROOT}/hack/boilerplate/boilerplate.go.txt" \
+#   --output-pkg "${TRAINING_OPERATOR_PKG}/pkg/apis/kubeflow.org/v1" \
+#   --output-dir "${TRAINING_OPERATOR_ROOT}/pkg/apis/kubeflow.org/v1" \
+#   --report-filename "${TRAINING_OPERATOR_ROOT}/hack/violation_exception_v1.list" \
+#   --update-report \
+#   "${TRAINING_OPERATOR_ROOT}/pkg/apis/kubeflow.org/v1"
+
+# # Generating OpenAPI spec for Training Operator V2.
+# echo "Generating OpenAPI specification for kubeflow.org/v2alpha1"
+# kube::codegen::gen_openapi \
+#   --boilerplate "${TRAINING_OPERATOR_ROOT}/hack/boilerplate/boilerplate.go.txt" \
+#   --output-pkg "${TRAINING_OPERATOR_PKG}/pkg/apis/kubeflow.org/v2alpha1" \
+#   --output-dir "${TRAINING_OPERATOR_ROOT}/pkg/apis/kubeflow.org/v2alpha1" \
+#   --report-filename "${TRAINING_OPERATOR_ROOT}/hack/violation_exception_v2alpha1.list" \
+#   --update-report \
+#   "${TRAINING_OPERATOR_ROOT}/pkg/apis/kubeflow.org/v2alpha1"
+
+# Generating OpenAPI Swagger for Training Operator V2.
+echo "Generate OpenAPI Swagger for kubeflow.org/v2alpha1"
+go run hack/swagger-v2/main.go >api.v2/openapi-spec/swagger.json
diff --git a/hack/violation_exception_v1.list b/hack/violation_exception_v1.list
@@ -4,12 +4,3 @@ API rule violation: list_type_missing,github.com/kubeflow/training-operator/pkg/
 API rule violation: list_type_missing,github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v1,PaddleElasticPolicy,Metrics
 API rule violation: names_match,github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v1,ElasticPolicy,RDZVID
 API rule violation: names_match,github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v1,PyTorchJobSpec,PyTorchReplicaSpecs
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Duration,Duration
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Object
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Type
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time
-API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentEncoding
-API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentType
diff --git a/hack/violation_exception_v2alpha1.list b/hack/violation_exception_v2alpha1.list
@@ -1,9 +0,0 @@
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Duration,Duration
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Object
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Type
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type
-API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time
-API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentEncoding
-API rule violation: names_match,k8s.io/apimachinery/pkg/runtime,Unknown,ContentType