From 98090abc075b97e3896ae6ea0c6e48f57e8d6434 Mon Sep 17 00:00:00 2001 From: mahmoudelgamal Date: Wed, 20 Jul 2016 00:12:54 +0200 Subject: [PATCH 1/3] Adding external libraries to livy --- docs/interpreter/livy.md | 35 ++++++++++++++++++- .../main/resources/interpreter-setting.json | 10 ++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index 2f47364aae8..ea467ce7b6f 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -30,7 +30,7 @@ We added some common configurations for spark, and you can set any configuration This link contains all spark configurations: http://spark.apache.org/docs/latest/configuration.html#available-properties. And instead of starting property with `spark.` it should be replaced with `livy.spark.`. Example: `spark.master` to `livy.spark.master` - + @@ -102,8 +102,41 @@ Example: `spark.master` to `livy.spark.master` + + + + + + + + + +
Property Upper bound for the number of executors.
livy.spark.executor.extraClassPathAdding extra libraries to the executors
livy.spark.driver.extraClassPathAdding extra libraries to the driver
+## Adding External libraries +You should put the libraries manually on the livy server and then set the paths of the libraries to those properties `livy.spark.executor.extraClassPath` and `livy.spark.driver.extraClassPath` + +Example + + + + + + + + + + + + + + + + + +
PropertyDefaultDescription
livy.spark.executor.extraClassPathpath/to/jar1.jar:path/to/jar2.jarAdding extra libraries to the executors
livy.spark.driver.extraClassPathpath/to/jar1.jar:path/to/jar2.jarAdding extra libraries to the driver
+ ## How to use Basically, you can use diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json index 2c1a0beceba..1304fd69797 100644 --- a/livy/src/main/resources/interpreter-setting.json +++ b/livy/src/main/resources/interpreter-setting.json @@ -81,6 +81,16 @@ "propertyName": "zeppelin.livy.keytab", "defaultValue": "", "description": "Kerberos keytab to authenticate livy" + }, + "livy.spark.executor.extraClassPath": { + "propertyName": "livy.spark.executor.extraClassPath", + "defaultValue": "", + "description": "Adding extra libraries to the executors" + }, + "livy.spark.driver.extraClassPath": { + "propertyName": "livy.spark.driver.extraClassPath", + "defaultValue": "", + "description": "Adding extra libraries to the driver" } } }, From d7f68187affc0898800a438f746a5fcf53c5647b Mon Sep 17 00:00:00 2001 From: mahmoudelgamal Date: Wed, 3 Aug 2016 17:09:37 +0200 Subject: [PATCH 2/3] Adding livy.spark.jars.packages property --- docs/interpreter/livy.md | 22 +++++-------------- .../main/resources/interpreter-setting.json | 11 +++------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index ea467ce7b6f..76e061b0f17 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -103,19 +103,14 @@ Example: `spark.master` to `livy.spark.master` Upper bound for the number of executors. - livy.spark.executor.extraClassPath + livy.spark.jars.packages - Adding extra libraries to the executors + Adding extra libraries to livy interpreter - - livy.spark.driver.extraClassPath - - Adding extra libraries to the driver - ## Adding External libraries -You should put the libraries manually on the livy server and then set the paths of the libraries to those properties `livy.spark.executor.extraClassPath` and `livy.spark.driver.extraClassPath` +You can load dynamic library to livy interpreter by set `livy.spark.jars.packages` property to comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. The format for the coordinates should be groupId:artifactId:version. Example @@ -126,15 +121,10 @@ Example Description - livy.spark.executor.extraClassPath - path/to/jar1.jar:path/to/jar2.jar - Adding extra libraries to the executors + livy.spark.jars.packages + io.spray:spray-json_2.10:1.3.1 + Adding extra libraries to livy interpreter - - livy.spark.driver.extraClassPath - path/to/jar1.jar:path/to/jar2.jar - Adding extra libraries to the driver - ## How to use diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json index 1304fd69797..1643ced050b 100644 --- a/livy/src/main/resources/interpreter-setting.json +++ b/livy/src/main/resources/interpreter-setting.json @@ -82,15 +82,10 @@ "defaultValue": "", "description": "Kerberos keytab to authenticate livy" }, - "livy.spark.executor.extraClassPath": { - "propertyName": "livy.spark.executor.extraClassPath", + "livy.spark.jars.packages": { + "propertyName": "livy.spark.jars.packages", "defaultValue": "", - "description": "Adding extra libraries to the executors" - }, - "livy.spark.driver.extraClassPath": { - "propertyName": "livy.spark.driver.extraClassPath", - "defaultValue": "", - "description": "Adding extra libraries to the driver" + "description": "Adding extra libraries to livy interpreter" } } }, From 5411a2f085f37dd448798542399472ce4e9cbe6d Mon Sep 17 00:00:00 2001 From: mahmoudelgamal Date: Mon, 8 Aug 2016 10:42:31 +0200 Subject: [PATCH 3/3] changeing default to example --- docs/interpreter/livy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index 76e061b0f17..8a9aaf4b14c 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -117,7 +117,7 @@ Example - +
PropertyDefaultExample Description