diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index 295a5080f40..225cd817bee 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -23,6 +23,11 @@ Additional requirements for the Livy interpreter are: * Livy server. ### Configuration +We added some common configurations for spark, and you can set any configuration you want. +This link contains all spark configurations: http://spark.apache.org/docs/latest/configuration.html#available-properties. +And instead of starting property with `spark.` it should be replaced with `livy.spark.`. +Example: `spark.master` to `livy.spark.master` + @@ -30,7 +35,7 @@ Additional requirements for the Livy interpreter are: - + @@ -44,6 +49,56 @@ Additional requirements for the Livy interpreter are: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyDescription
zeppelin.livy.masterlivy.spark.master local[*] Spark master uri. ex) spark://masterhost:7077
1000 Max number of SparkSQL result to display.
livy.spark.driver.coresDriver cores. ex) 1, 2.
livy.spark.driver.memoryDriver memory. ex) 512m, 32g.
livy.spark.executor.instancesExecutor instances. ex) 1, 4.
livy.spark.executor.coresNum cores per executor. ex) 1, 4.
livy.spark.executor.memoryExecutor memory per worker instance. ex) 512m, 32g.
livy.spark.dynamicAllocation.enabledUse dynamic resource allocation. ex) True, False.
livy.spark.dynamicAllocation.cachedExecutorIdleTimeoutRemove an executor which has cached data blocks.
livy.spark.dynamicAllocation.minExecutorsLower bound for the number of executors.
livy.spark.dynamicAllocation.initialExecutorsInitial number of executors to run.
livy.spark.dynamicAllocation.maxExecutorsUpper bound for the number of executors.
@@ -105,3 +160,6 @@ The session would have timed out, you may need to restart the interpreter. > Blacklisted configuration values in session config: spark.master edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line. + +if you choose to work on livy in `apps/spark/java` directory in https://github.com/cloudera/hue , +copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master` diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java index 27fc422948d..7f3517eea5a 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java @@ -20,6 +20,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; + import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; @@ -40,7 +41,9 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; @@ -60,27 +63,28 @@ public class LivyHelper { public Integer createSession(InterpreterContext context, String kind) throws Exception { try { + Map conf = new HashMap(); + + Iterator> it = property.entrySet().iterator(); + while (it.hasNext()) { + Entry pair = it.next(); + if (pair.getKey().toString().startsWith("livy.spark.") && + !pair.getValue().toString().isEmpty()) + conf.put(pair.getKey().toString().substring(5), pair.getValue().toString()); + } + + String confData = gson.toJson(conf); + String json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions", - "POST", + "POST", "{" + "\"kind\": \"" + kind + "\", " + - "\"master\": \"" + property.getProperty("zeppelin.livy.master") + "\", " + - "\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + "\"" + + "\"conf\": " + confData + ", " + + "\"proxyUser\": " + context.getAuthenticationInfo().getUser() + "}", context.getParagraphId() ); - if (json.contains("CreateInteractiveRequest[\\\"master\\\"]")) { - json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions", - "POST", - "{" + - "\"kind\": \"" + kind + "\", " + - "\"conf\":{\"spark.master\": \"" - + property.getProperty("zeppelin.livy.master") + "\"}," + - "\"proxyUser\": \"" + context.getAuthenticationInfo().getUser() + "\"" + - "}", - context.getParagraphId() - ); - } + Map jsonMap = (Map) gson.fromJson(json, new TypeToken>() { }.getType()); diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java index 23a6379bebf..0a29d795f6d 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java @@ -45,7 +45,22 @@ public class LivySparkInterpreter extends Interpreter { LivySparkInterpreter.class.getName(), new InterpreterPropertyBuilder() .add("zeppelin.livy.url", DEFAULT_URL, "The URL for Livy Server.") - .add("zeppelin.livy.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077") + .add("livy.spark.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077") + .add("livy.spark.driver.cores", "", "Driver cores. ex) 1, 2") + .add("livy.spark.driver.memory", "", "Driver memory. ex) 512m, 32g") + .add("livy.spark.executor.instances", "", "Executor instances. ex) 1, 4") + .add("livy.spark.executor.cores", "", "Num cores per executor. ex) 1, 4") + .add("livy.spark.executor.memory", "", + "Executor memory per worker instance. ex) 512m, 32g") + .add("livy.spark.dynamicAllocation.enabled", "", "Use dynamic resource allocation") + .add("livy.spark.dynamicAllocation.cachedExecutorIdleTimeout", "", + "Remove an executor which has cached data blocks") + .add("livy.spark.dynamicAllocation.minExecutors", "", + "Lower bound for the number of executors if dynamic allocation is enabled. ") + .add("livy.spark.dynamicAllocation.initialExecutors", "", + "Initial number of executors to run if dynamic allocation is enabled. ") + .add("livy.spark.dynamicAllocation.maxExecutors", "", + "Upper bound for the number of executors if dynamic allocation is enabled. ") .build() ); }