diff --git a/CHANGELOG.md b/CHANGELOG.md
index bdc7ff1fe..de1d0c4d1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).
+## 1.11.1 - 2020-09-29
+
+### Added
+- Added healtz endpoint that is cheap and quick to return, useful for kubernetes live/ready checks.
+
+### Fixed
+- Fixed health check script when using custom path prefix.
+- Proxy will no correctly handle paths that end with a / at the end.
+- Submitting an extraction will always return a 500 error, see [#84](https://github.com/clowder-framework/clowder/issues/84)
+- Added MongoDB index for `folders.files`.
+
+### Changed
+- Updated update-clowder script to work with migration to github. Has the ability now to push a message to MSTEAMS as well as influxdb.
+
## 1.11.0 - 2020-08-31
### Added
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 1b5c1e124..ae8fd60b7 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -1,5 +1,6 @@
Following is a list of contributors in alphabetical order:
+- Aaraj Habib
- Ashwini Vaidya
- Avinash Kumar
- Ben Galewsky
diff --git a/app/api/Extractions.scala b/app/api/Extractions.scala
index c1759b68d..021b20a2e 100644
--- a/app/api/Extractions.scala
+++ b/app/api/Extractions.scala
@@ -530,22 +530,24 @@ class Extractions @Inject()(
datasetId = datasetslists.head.id
}
// if extractor_id is not specified default to execution of all extractors matching mime type
- val key = (request.body \ "extractor").asOpt[String] match {
+ (request.body \ "extractor").asOpt[String] match {
case Some(extractorId) =>
val job_id = p.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra,
datasetId, newFlags, request.apiKey, request.user)
Ok(Json.obj("status" -> "OK", "job_id" -> job_id))
- case None =>
+ case None => {
p.fileCreated(file, None, Utils.baseUrl(request), request.apiKey) match {
case Some(job_id) => {
Ok(Json.obj("status" -> "OK", "job_id" -> job_id))
}
+ case None => {
+ val message = "No jobId found for Extraction on fileid=" + file_id.stringify
+ Logger.error(message)
+ InternalServerError(toJson(Map("status" -> "KO", "msg" -> message)))
+ }
}
+ }
}
-
- val message = "No jobId found for Extraction on fileid=" + file_id.stringify
- Logger.error(message)
- InternalServerError(toJson(Map("status" -> "KO", "msg" -> message)))
} else {
Conflict(toJson(Map("status" -> "error", "msg" -> "File is not ready. Please wait and try again.")))
}
diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala
index 07bc5ddd0..fa70ce3fb 100644
--- a/app/controllers/Application.scala
+++ b/app/controllers/Application.scala
@@ -31,8 +31,8 @@ class Application @Inject() (files: FileService, collections: CollectionService,
* @param path the path minus the slash
* @return moved permanently to path without /
*/
- def untrail(path: String) = Action {
- MovedPermanently("/" + path)
+ def untrail(path: String) = Action { implicit request =>
+ MovedPermanently(s"${Utils.baseUrl(request, false)}/${path}")
}
def swaggerUI = Action { implicit request =>
@@ -263,6 +263,10 @@ class Application @Inject() (files: FileService, collections: CollectionService,
Ok("")
}
+ def healthz() = Action { implicit request =>
+ Ok("healthy")
+ }
+
/**
* Bookmarklet
*/
diff --git a/app/controllers/Utils.scala b/app/controllers/Utils.scala
index f759e97d0..8eda878fb 100644
--- a/app/controllers/Utils.scala
+++ b/app/controllers/Utils.scala
@@ -14,8 +14,12 @@ object Utils {
* Return base url given a request. This will add http or https to the front, for example
* https://localhost:9443 will be returned if it is using https.
*/
- def baseUrl(request: Request[Any]) = {
- routes.Files.list().absoluteURL(https(request))(request).replace("/files", "")
+ def baseUrl(request: Request[Any], absolute: Boolean = true) = {
+ if (absolute) {
+ routes.Files.list().absoluteURL(https(request))(request).replace("/files", "")
+ } else {
+ routes.Files.list().url.replace("/files", "")
+ }
}
/**
diff --git a/app/services/mongodb/MongoSalatPlugin.scala b/app/services/mongodb/MongoSalatPlugin.scala
index 5705bf6d7..2389e400b 100644
--- a/app/services/mongodb/MongoSalatPlugin.scala
+++ b/app/services/mongodb/MongoSalatPlugin.scala
@@ -141,6 +141,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
collection("extractions").ensureIndex(MongoDBObject("file_id" -> 1))
collection("folders").ensureIndex(MongoDBObject("parentDatasetId" -> 1))
+ collection("folders").ensureIndex(MongoDBObject("files" -> 1))
collection("uploads").ensureIndex(MongoDBObject("uploadDate" -> -1))
collection("uploads").ensureIndex(MongoDBObject("author.email" -> 1))
diff --git a/conf/routes b/conf/routes
index 64530b9e2..b8d703c87 100644
--- a/conf/routes
+++ b/conf/routes
@@ -7,6 +7,13 @@
#OPTIONS /*path @controllers.Application.options(path)
OPTIONS /*path api.ApiHelp.options(path)
+
+# operations are applied in this order, to prevent untrail from removing
+# the trailing / at the end of the /api/proxy calls it is placed before
+# the untrail call
+GET /api/proxy/:endpoint_key @api.Proxy.get(endpoint_key: String, pathSuffix: String = null)
+GET /api/proxy/:endpoint_key/ @api.Proxy.get(endpoint_key: String, pathSuffix: String = "")
+GET /api/proxy/:endpoint_key/*pathSuffix @api.Proxy.get(endpoint_key: String, pathSuffix: String)
GET /*path/ @controllers.Application.untrail(path: String)
# ----------------------------------------------------------------------
@@ -17,6 +24,7 @@ GET /
GET /about @controllers.Application.about
GET /tos @controllers.Application.tos(redirect: Option[String] ?= None)
GET /email @controllers.Application.email(subject: String ?= "", body: String ?= "")
+GET /healthz @controllers.Application.healthz()
# ----------------------------------------------------------------------
# Map static resources from the /public folder to the /assets URL path
@@ -859,17 +867,14 @@ DELETE /api/standardvocab/:id
# PROXY API
# ----------------------------------------------------------------------
-GET /api/proxy/:endpoint_key @api.Proxy.get(endpoint_key: String, pathSuffix: String = null)
-GET /api/proxy/:endpoint_key/ @api.Proxy.get(endpoint_key: String, pathSuffix: String = null)
-GET /api/proxy/:endpoint_key/*pathSuffix @api.Proxy.get(endpoint_key: String, pathSuffix: String)
POST /api/proxy/:endpoint_key @api.Proxy.post(endpoint_key: String, pathSuffix: String = null)
-POST /api/proxy/:endpoint_key/ @api.Proxy.post(endpoint_key: String, pathSuffix: String = null)
+POST /api/proxy/:endpoint_key/ @api.Proxy.post(endpoint_key: String, pathSuffix: String = "")
POST /api/proxy/:endpoint_key/*pathSuffix @api.Proxy.post(endpoint_key: String, pathSuffix: String)
PUT /api/proxy/:endpoint_key @api.Proxy.put(endpoint_key: String, pathSuffix: String = null)
-PUT /api/proxy/:endpoint_key/ @api.Proxy.put(endpoint_key: String, pathSuffix: String = null)
+PUT /api/proxy/:endpoint_key/ @api.Proxy.put(endpoint_key: String, pathSuffix: String = "")
PUT /api/proxy/:endpoint_key/*pathSuffix @api.Proxy.put(endpoint_key: String, pathSuffix: String)
DELETE /api/proxy/:endpoint_key @api.Proxy.delete(endpoint_key: String, pathSuffix: String = null)
-DELETE /api/proxy/:endpoint_key/ @api.Proxy.delete(endpoint_key: String, pathSuffix: String = null)
+DELETE /api/proxy/:endpoint_key/ @api.Proxy.delete(endpoint_key: String, pathSuffix: String = "")
DELETE /api/proxy/:endpoint_key/*pathSuffix @api.Proxy.delete(endpoint_key: String, pathSuffix: String)
# ----------------------------------------------------------------------
diff --git a/doc/src/sphinx/conf.py b/doc/src/sphinx/conf.py
index 6a2bc7e79..bff310b81 100644
--- a/doc/src/sphinx/conf.py
+++ b/doc/src/sphinx/conf.py
@@ -22,7 +22,7 @@
author = 'Luigi Marini'
# The full version, including alpha/beta/rc tags
-release = '1.11.0'
+release = '1.11.1'
# -- General configuration ---------------------------------------------------
diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh
index c5a7564b6..80e64d498 100755
--- a/docker/healthcheck.sh
+++ b/docker/healthcheck.sh
@@ -1,3 +1,6 @@
#!/bin/bash
-curl -s --fail http://localhost:9000/api/status || exit 1
+### Add trailing backslash if not in context
+[[ "${CLOWDER_CONTEXT}" != */ ]] && CLOWDER_CONTEXT="${CLOWDER_CONTEXT}/"
+
+curl -s --fail http://localhost:8000${CLOWDER_CONTEXT:-/}api/status || exit 1
diff --git a/project/Build.scala b/project/Build.scala
index 2f45e4312..c17a04d60 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -13,7 +13,7 @@ import NativePackagerKeys._
object ApplicationBuild extends Build {
val appName = "clowder"
- val version = "1.11.0"
+ val version = "1.11.1"
val jvm = "1.7"
def appVersion: String = {
diff --git a/public/jsonld/contexts/extractors.jsonld b/public/jsonld/contexts/extractors.jsonld
index 7ad8f6335..4250cfd60 100644
--- a/public/jsonld/contexts/extractors.jsonld
+++ b/public/jsonld/contexts/extractors.jsonld
@@ -3,7 +3,7 @@
"extractor": "https://clowder.ncsa.illinois.edu/contexts/extractor#",
"id":"extractor/internal_id",
"name": {"@id": "extractor:extractor/id", "@type": "@id"},
- "version": "extractor:version"
+ "version": "extractor:version",
"created_at": {
"@id": "extractor:updated",
"@type": "http://www.w3.org/2001/XMLSchema#dateTime"
@@ -19,4 +19,4 @@
"libraries": "extractor:libraries",
"bibtex": "extractor:bibtex",
"dependencies":"extractor:dependencies"
-}
\ No newline at end of file
+}
diff --git a/public/jsonld/contexts/metadata.jsonld b/public/jsonld/contexts/metadata.jsonld
index 802ef30ea..573cfb40f 100644
--- a/public/jsonld/contexts/metadata.jsonld
+++ b/public/jsonld/contexts/metadata.jsonld
@@ -20,11 +20,9 @@
"extractor": "cat:extractor",
"content": {
"@id": "https://clowder.ncsa.illinois.edu/metadata#content"
- }
+ },
"attached_to": "https://clowder.ncsa.illinois.edu/metadata/attachedTo",
"resource_type": "https://clowder.ncsa.illinois.edu/resource/type",
"url": "https://clowder.ncsa.illinois.edu/resource"
}
}
-
-
diff --git a/public/swagger.yml b/public/swagger.yml
index c31a641ce..7fe11a73b 100644
--- a/public/swagger.yml
+++ b/public/swagger.yml
@@ -9,7 +9,7 @@ info:
Clowder is a customizable and scalable data management system to support any
data format and multiple research domains. It is under active development
and deployed for a variety of research projects.
- version: 1.11.0
+ version: 1.11.1
termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos
contact:
name: Clowder
diff --git a/scripts/ubuntu/update-clowder.sh b/scripts/ubuntu/update-clowder.sh
index e7a2f7312..c96a778d5 100644
--- a/scripts/ubuntu/update-clowder.sh
+++ b/scripts/ubuntu/update-clowder.sh
@@ -1,48 +1,83 @@
#!/bin/bash
-# CATS-CORE is the main branch for this server
-CLOWDER_BRANCH=${CLOWDER_BRANCH:-"CATS-CORE"}
-CLOWDER_BUILD=${CLOWDER_BUILD:-"latestSuccessful"}
+# what version ("" == latest release)
+CLOWDER_BRANCH=${CLOWDER_BRANCH:-}
+#CLOWDER_BRANCH="develop"
+
+# compute some other variables
+if [ "${CLOWDER_BRANCH}" = "" ]; then
+ CLOWDER_VERSION=$(curl -s https://opensource.ncsa.illinois.edu/projects/artifacts.php?key=CATS | grep '#\1#')
+ CLOWDER_ZIPFILE=clowder-${CLOWDER_VERSION}.zip
+else
+ CLOWDER_VERSION="${CLOWDER_BRANCH}"
+ CLOWDER_ZIPFILE=clowder-${CLOWDER_VERSION}-${CLOWDER_BRANCH}.zip
+fi
+
+# Should text go to stdout, empty is not
+STDOUT="YES"
+
+# HipChat token for notifications
+HIPCHAT_TOKEN=""
+HIPCHAT_ROOM=""
+
+# Slack token for notifications
+SLACK_TOKEN=""
+SLACK_CHANNEL="#simpl-ops"
+
+# MSTeams Webhook
+MSTEAMS_URL=""
+
+# INFLUXDB
+INFLUXDB_URL=""
+INFLUXDB_DATABASE=""
+INFLUXDB_USERNAME=""
+INFLUXDB_PASSWORD=""
# change to folder where script is installed
cd /home/clowder
# fetch software
-if [[ ${CLOWDER_BUILD} == latest* ]]; then
- BB="${CLOWDER_BRANCH}/${CLOWDER_BUILD}"
+if [ -e ]; then
+ curl -f -s -z "${CLOWDER_ZIPFILE}" -o "${CLOWDER_ZIPFILE}" https://opensource.ncsa.illinois.edu/projects/artifacts/CATS/${CLOWDER_VERSION}/files/${CLOWDER_ZIPFILE} || exit 1
else
- BB="${CLOWDER_BRANCH}-${CLOWDER_BUILD}"
+ curl -f -s -o "${CLOWDER_ZIPFILE}" https://opensource.ncsa.illinois.edu/projects/artifacts/CATS/${CLOWDER_VERSION}/files/${CLOWDER_ZIPFILE} || exit 1
fi
-URL="https://opensource.ncsa.illinois.edu/bamboo/browse/${BB}/artifact/shared/dist/"
-/usr/bin/wget -q -e robots=off -A "clowder-*.zip" -nd -r -N -l1 ${URL}
-LATEST=$( /bin/ls -1rt clowder-*.zip | tail -1 )
-if [ -s ${LATEST} ]; then
- if [ "$1" == "--force" -o ${LATEST} -nt clowder ]; then
+if [ -s ${CLOWDER_ZIPFILE} ]; then
+ if [ "$1" == "--force" -o ${CLOWDER_ZIPFILE} -nt clowder ]; then
+ exec 3>&1
+ exec &> "/tmp/$$.txt"
+
echo "UPDATING CLOWDER ON ${HOSTNAME}"
echo " bamboo branch = ${CLOWDER_BRANCH}"
# stop clowder
- /sbin/stop clowder
+ /bin/systemctl stop clowder
# save local modifications
if [ -d clowder/custom ]; then
mv clowder/custom clowder.custom
fi
+ if [ -d clowder/logs ]; then
+ mv clowder/logs clowder.logs
+ fi
# install new version
- /bin/rm -rf clowder $( basename ${LATEST} .zip )
- /usr/bin/unzip -q ${LATEST}
- /bin/mv $( basename ${LATEST} .zip ) clowder
+ /bin/rm -rf clowder $( basename ${CLOWDER_ZIPFILE} .zip )
+ /usr/bin/unzip -q ${CLOWDER_ZIPFILE}
+ /bin/mv $( basename ${CLOWDER_ZIPFILE} .zip ) clowder
/usr/bin/touch clowder
# get some nice values from the build
- echo " bamboo build = $( grep '\-Dbuild.bamboo' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
+ echo " github build = $( grep '\-Dbuild.bamboo' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
echo " clowder version = $( grep '\-Dbuild.version' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
echo " clowder branch = $( grep '\-Dbuild.branch' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
echo " clowder gitsha1 = $( grep '\-Dbuild.gitsha1' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
# restore local modifications
+ if [ -d clowder.logs ]; then
+ mv clowder.logs clowder/logs
+ fi
if [ -d clowder.custom ]; then
mv clowder.custom clowder/custom
fi
@@ -56,6 +91,41 @@ if [ -s ${LATEST} ]; then
fi
# start clowder again
- /sbin/start clowder
+ /bin/systemctl start clowder
+
+ # send message by hipchat
+ if [ "${HIPCHAT_TOKEN}" != "" -a "${HIPCHAT_ROOM}" != "" ]; then
+ url="https://hipchat.ncsa.illinois.edu/v1/rooms/message?format=json&auth_token=${HIPCHAT_TOKEN}"
+ txt=$(cat /tmp/$$.txt | sed 's/ /%20/g;s/!/%21/g;s/"/%22/g;s/#/%23/g;s/\$/%24/g;s/\&/%26/g;s/'\''/%27/g;s/(/%28/g;s/)/%29/g;s/:/%3A/g;s/$//g')
+ room=$(echo ${HIPCHAT_ROOM} | sed 's/ /%20/g;s/!/%21/g;s/"/%22/g;s/#/%23/g;s/\$/%24/g;s/\&/%26/g;s/'\''/%27/g;s/(/%28/g;s/)/%29/g;s/:/%3A/g')
+ body="room_id=${room}&from=clowder&message=${txt}"
+ result=$(curl -s -X POST -d "${body}" $url)
+ fi
+ if [ "${SLACK_TOKEN}" != "" -a "${SLACK_CHANNEL}" != "" ]; then
+ url="https://hooks.slack.com/services/${SLACK_TOKEN}"
+ txt=$(cat /tmp/$$.txt | sed 's/"/\\"/g;s/$/\\/g' | tr '\n' 'n' )
+ payload="payload={\"channel\": \"${SLACK_CHANNEL}\", \"username\": \"clowder\", \"text\": \"${txt}\", \"icon_url\": \"https://opensource.ncsa.illinois.edu/projects/artifacts/CATS/logo.png\"}"
+ result=$(curl -s -X POST --data-urlencode "${payload}" $url)
+ fi
+ if [ "${MSTEAMS_URL}" != "" ]; then
+ txt=$(cat /tmp/$$.txt | sed 's/"/\\"/g;s/$/\\/g' | tr '\n' 'n' )
+ payload="{\"title\": \"UPDATE CLOWDER\", \"text\": \"${txt}\" }"
+ result=$(curl -s -X POST -H "Content-Type: application/json" -d "${payload}" $MSTEAMS_URL)
+ fi
+ if [ "${INFLUXDB_URL}" != "" -a "${INFLUXDB_DATABASE}" != "" -a "${INFLUXDB_USERNAME}" != "" -a "${INFLUXDB_PASSWORD}" != "" ]; then
+ url="${INFLUXDB_URL}/api/v2/write?bucket=${INFLUXDB_DATABASE}&precision=ns"
+ tags="host=${HOSTNAME}"
+ timestamp="$(date -u +"%s%N")"
+ values="version=\"$( grep '\-Dbuild.version' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )\""
+ values="${values},branch=\"$( grep '\-Dbuild.branch' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )\""
+ values="${values},build=\"$( grep '\-Dbuild.bamboo' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )\""
+ values="${values},gitsha1=\"$( grep '\-Dbuild.gitsha1' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )\""
+ auth="Authorization: Token ${INFLUXDB_USERNAME}:${INFLUXDB_PASSWORD}"
+ result=$(curl -s -i -XPOST "$url" --header "$auth" --data "clowder_update,${tags} ${values} ${timestamp}")
+ fi
+ if [ "${STDOUT}" != "" ]; then
+ cat /tmp/$$.txt >&3
+ fi
+ rm /tmp/$$.txt
fi
fi