From 50b319d469dd013227684dcf1550438e59290ae7 Mon Sep 17 00:00:00 2001 From: Roman Wozniak Date: Wed, 1 Nov 2023 11:55:31 +0800 Subject: [PATCH] Add support of memoryOverhead and coreLimit on both driver and executor spec --- .../java/dev/caraml/store/sparkjob/crd/SparkDriverSpec.java | 2 ++ .../dev/caraml/store/sparkjob/crd/SparkExecutorSpec.java | 2 ++ .../test/java/dev/caraml/store/sparkjob/JobServiceTest.java | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/caraml-store-registry/src/main/java/dev/caraml/store/sparkjob/crd/SparkDriverSpec.java b/caraml-store-registry/src/main/java/dev/caraml/store/sparkjob/crd/SparkDriverSpec.java index 6cdc5a7..2833c47 100644 --- a/caraml-store-registry/src/main/java/dev/caraml/store/sparkjob/crd/SparkDriverSpec.java +++ b/caraml-store-registry/src/main/java/dev/caraml/store/sparkjob/crd/SparkDriverSpec.java @@ -14,9 +14,11 @@ public class SparkDriverSpec { private List tolerations; private Integer cores; private String coreRequest; + private String coreLimit; private String mainApplicationFile; private String mainClass; private String memory; + private String memoryOverhead; private String javaOptions; private Map labels; private String serviceAccount; diff --git a/caraml-store-registry/src/main/java/dev/caraml/store/sparkjob/crd/SparkExecutorSpec.java b/caraml-store-registry/src/main/java/dev/caraml/store/sparkjob/crd/SparkExecutorSpec.java index ba2cec0..ce65f9e 100644 --- a/caraml-store-registry/src/main/java/dev/caraml/store/sparkjob/crd/SparkExecutorSpec.java +++ b/caraml-store-registry/src/main/java/dev/caraml/store/sparkjob/crd/SparkExecutorSpec.java @@ -14,8 +14,10 @@ public class SparkExecutorSpec { private List tolerations; private Integer cores; private String coreRequest; + private String coreLimit; private Integer instances; private String memory; + private String memoryOverhead; private String javaOptions; private Map labels; private Map annotations; diff --git a/caraml-store-registry/src/test/java/dev/caraml/store/sparkjob/JobServiceTest.java b/caraml-store-registry/src/test/java/dev/caraml/store/sparkjob/JobServiceTest.java index e73b8ed..525befa 100644 --- a/caraml-store-registry/src/test/java/dev/caraml/store/sparkjob/JobServiceTest.java +++ b/caraml-store-registry/src/test/java/dev/caraml/store/sparkjob/JobServiceTest.java @@ -161,8 +161,11 @@ public void shouldCreateStreamingJob() throws IOException, SparkOperatorApiExcep SparkExecutorSpec templateExecutorSpec = new SparkExecutorSpec(); templateDriverSpec.setCores(1); templateDriverSpec.setMemory("1g"); + templateDriverSpec.setMemoryOverhead("250m"); templateDriverSpec.setLabels(Map.of("team", "${team}")); templateExecutorSpec.setCores(2); + templateExecutorSpec.setCoreRequest("100m"); + templateExecutorSpec.setCoreLimit("200m"); templateExecutorSpec.setMemory("2g"); templateSparkApplicationSpec.setDriver(templateDriverSpec); templateSparkApplicationSpec.setExecutor(templateExecutorSpec); @@ -205,10 +208,13 @@ public void shouldCreateStreamingJob() throws IOException, SparkOperatorApiExcep SparkDriverSpec expectedDriverSpec = new SparkDriverSpec(); expectedDriverSpec.setCores(2); expectedDriverSpec.setMemory("1g"); + expectedDriverSpec.setMemoryOverhead("250m"); expectedDriverSpec.setLabels(Map.of("team", "some-team")); expectedSparkApplicationSpec.setDriver(expectedDriverSpec); SparkExecutorSpec expectedExecutorSpec = new SparkExecutorSpec(); expectedExecutorSpec.setCores(2); + expectedExecutorSpec.setCoreRequest("100m"); + expectedExecutorSpec.setCoreLimit("200m"); expectedExecutorSpec.setMemory("3g"); expectedSparkApplicationSpec.setExecutor(expectedExecutorSpec); expectedSparkApplicationSpec.setImage("sparkImage:latest");