Merge pull request #87 from clowder-framework/release/1.11.1

Release/1.11.1

CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.11.1 - 2020-09-29
+
+### Added
+- Added healtz endpoint that is cheap and quick to return, useful for kubernetes live/ready checks.
+
+### Fixed
+- Fixed health check script when using custom path prefix.
+- Proxy will no correctly handle paths that end with a / at the end.
+- Submitting an extraction will always return a 500 error, see [#84](https://github.com/clowder-framework/clowder/issues/84)
+- Added MongoDB index for `folders.files`.
+
+### Changed
+- Updated update-clowder script to work with migration to github. Has the ability now to push a message to MSTEAMS as well as influxdb.
+
 ## 1.11.0 - 2020-08-31
 
 ### Added

CONTRIBUTORS.md

@@ -1,5 +1,6 @@
 Following is a list of contributors in alphabetical order:
 
+- Aaraj Habib
 - Ashwini Vaidya
 - Avinash Kumar
 - Ben Galewsky

app/api/Extractions.scala

@@ -530,22 +530,24 @@ class Extractions @Inject()(
                 datasetId = datasetslists.head.id
               }
               // if extractor_id is not specified default to execution of all extractors matching mime type
-              val key = (request.body \ "extractor").asOpt[String] match {
+              (request.body \ "extractor").asOpt[String] match {
                 case Some(extractorId) =>
                   val job_id = p.submitFileManually(new UUID(originalId), file, Utils.baseUrl(request), extractorId, extra,
                     datasetId, newFlags, request.apiKey, request.user)
                   Ok(Json.obj("status" -> "OK", "job_id" -> job_id))
-                case None =>
+                case None => {
                   p.fileCreated(file, None, Utils.baseUrl(request), request.apiKey) match {
                     case Some(job_id) => {
                       Ok(Json.obj("status" -> "OK", "job_id" -> job_id))
                     }
+                    case None => {
+                      val message = "No jobId found for Extraction on fileid=" + file_id.stringify
+                      Logger.error(message)
+                      InternalServerError(toJson(Map("status" -> "KO", "msg" -> message)))
+                    }
                   }
+                }
               }
-
-              val message = "No jobId found for Extraction on fileid=" + file_id.stringify
-              Logger.error(message)
-              InternalServerError(toJson(Map("status" -> "KO", "msg" -> message)))
             } else {
               Conflict(toJson(Map("status" -> "error", "msg" -> "File is not ready. Please wait and try again.")))
             }

app/controllers/Application.scala

@@ -31,8 +31,8 @@ class Application @Inject() (files: FileService, collections: CollectionService,
    * @param path the path minus the slash
    * @return moved permanently to path without /
    */
-  def untrail(path: String) = Action {
-    MovedPermanently("/" + path)
+  def untrail(path: String) = Action { implicit request =>
+    MovedPermanently(s"${Utils.baseUrl(request, false)}/${path}")
   }
 
   def swaggerUI = Action { implicit request =>
@@ -263,6 +263,10 @@ class Application @Inject() (files: FileService, collections: CollectionService,
     Ok("")
    }
 
+  def healthz() = Action { implicit request =>
+    Ok("healthy")
+  }
+
   /**
    * Bookmarklet
    */

app/controllers/Utils.scala

@@ -14,8 +14,12 @@ object Utils {
    * Return base url given a request. This will add http or https to the front, for example
    * https://localhost:9443 will be returned if it is using https.
    */
-  def baseUrl(request: Request[Any]) = {
-    routes.Files.list().absoluteURL(https(request))(request).replace("/files", "")
+  def baseUrl(request: Request[Any], absolute: Boolean = true) = {
+    if (absolute) {
+      routes.Files.list().absoluteURL(https(request))(request).replace("/files", "")
+    } else {
+      routes.Files.list().url.replace("/files", "")
+    }
   }
 
   /**

app/services/mongodb/MongoSalatPlugin.scala

@@ -141,6 +141,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
     collection("extractions").ensureIndex(MongoDBObject("file_id" -> 1))
 
     collection("folders").ensureIndex(MongoDBObject("parentDatasetId" -> 1))
+    collection("folders").ensureIndex(MongoDBObject("files" -> 1))
 
     collection("uploads").ensureIndex(MongoDBObject("uploadDate" -> -1))
     collection("uploads").ensureIndex(MongoDBObject("author.email" -> 1))

conf/routes

@@ -7,6 +7,13 @@
 
 #OPTIONS       /*path                                                                   @controllers.Application.options(path)
 OPTIONS        /*path                                                                   api.ApiHelp.options(path)
+
+# operations are applied in this order, to prevent untrail from removing
+# the trailing / at the end of the /api/proxy calls it is placed before
+# the untrail call
+GET            /api/proxy/:endpoint_key                                                 @api.Proxy.get(endpoint_key: String, pathSuffix: String = null)
+GET            /api/proxy/:endpoint_key/                                                @api.Proxy.get(endpoint_key: String, pathSuffix: String = "")
+GET            /api/proxy/:endpoint_key/*pathSuffix                                     @api.Proxy.get(endpoint_key: String, pathSuffix: String)
 GET            /*path/                                                                  @controllers.Application.untrail(path: String)
 
 # ----------------------------------------------------------------------
@@ -17,6 +24,7 @@ GET            /
 GET            /about                                                                   @controllers.Application.about
 GET            /tos                                                                     @controllers.Application.tos(redirect: Option[String] ?= None)
 GET            /email                                                                   @controllers.Application.email(subject: String ?= "", body: String ?= "")
+GET            /healthz                                                                 @controllers.Application.healthz()
 
 # ----------------------------------------------------------------------
 # Map static resources from the /public folder to the /assets URL path
@@ -859,17 +867,14 @@ DELETE       /api/standardvocab/:id
 # PROXY API
 # ----------------------------------------------------------------------
 
-GET         /api/proxy/:endpoint_key                                                        @api.Proxy.get(endpoint_key: String, pathSuffix: String = null)
-GET         /api/proxy/:endpoint_key/                                                       @api.Proxy.get(endpoint_key: String, pathSuffix: String = null)
-GET         /api/proxy/:endpoint_key/*pathSuffix                                            @api.Proxy.get(endpoint_key: String, pathSuffix: String)
 POST        /api/proxy/:endpoint_key                                                        @api.Proxy.post(endpoint_key: String, pathSuffix: String = null)
-POST        /api/proxy/:endpoint_key/                                                       @api.Proxy.post(endpoint_key: String, pathSuffix: String = null)
+POST        /api/proxy/:endpoint_key/                                                       @api.Proxy.post(endpoint_key: String, pathSuffix: String = "")
 POST        /api/proxy/:endpoint_key/*pathSuffix                                            @api.Proxy.post(endpoint_key: String, pathSuffix: String)
 PUT         /api/proxy/:endpoint_key                                                        @api.Proxy.put(endpoint_key: String, pathSuffix: String = null)
-PUT         /api/proxy/:endpoint_key/                                                       @api.Proxy.put(endpoint_key: String, pathSuffix: String = null)
+PUT         /api/proxy/:endpoint_key/                                                       @api.Proxy.put(endpoint_key: String, pathSuffix: String = "")
 PUT         /api/proxy/:endpoint_key/*pathSuffix                                            @api.Proxy.put(endpoint_key: String, pathSuffix: String)
 DELETE      /api/proxy/:endpoint_key                                                        @api.Proxy.delete(endpoint_key: String, pathSuffix: String = null)
-DELETE      /api/proxy/:endpoint_key/                                                       @api.Proxy.delete(endpoint_key: String, pathSuffix: String = null)
+DELETE      /api/proxy/:endpoint_key/                                                       @api.Proxy.delete(endpoint_key: String, pathSuffix: String = "")
 DELETE      /api/proxy/:endpoint_key/*pathSuffix                                            @api.Proxy.delete(endpoint_key: String, pathSuffix: String)
 
 # ----------------------------------------------------------------------

doc/src/sphinx/conf.py

@@ -22,7 +22,7 @@
 author = 'Luigi Marini'
 
 # The full version, including alpha/beta/rc tags
-release = '1.11.0'
+release = '1.11.1'
 
 
 # -- General configuration ---------------------------------------------------

docker/healthcheck.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
 
-curl -s --fail http://localhost:9000/api/status || exit 1
+### Add trailing backslash if not in context
+[[ "${CLOWDER_CONTEXT}" != */ ]] && CLOWDER_CONTEXT="${CLOWDER_CONTEXT}/"
+
+curl -s --fail http://localhost:8000${CLOWDER_CONTEXT:-/}api/status || exit 1

project/Build.scala

@@ -13,7 +13,7 @@ import NativePackagerKeys._
 object ApplicationBuild extends Build {
 
   val appName = "clowder"
-  val version = "1.11.0"
+  val version = "1.11.1"
   val jvm = "1.7"
 
   def appVersion: String = {

public/jsonld/contexts/extractors.jsonld

@@ -3,7 +3,7 @@
 	"extractor": "https://clowder.ncsa.illinois.edu/contexts/extractor#",
 	"id":"extractor/internal_id",
 	"name": {"@id": "extractor:extractor/id", "@type": "@id"},
-	"version": "extractor:version"
+	"version": "extractor:version",
 	"created_at": {
 	      "@id": "extractor:updated",
 	      "@type": "http://www.w3.org/2001/XMLSchema#dateTime"
@@ -19,4 +19,4 @@
 	"libraries": "extractor:libraries",
 	"bibtex": "extractor:bibtex",
 	"dependencies":"extractor:dependencies"
-}
+}

public/jsonld/contexts/metadata.jsonld

@@ -20,11 +20,9 @@
     "extractor": "cat:extractor",
     "content": {
       "@id": "https://clowder.ncsa.illinois.edu/metadata#content"
-    }
+    },
     "attached_to": "https://clowder.ncsa.illinois.edu/metadata/attachedTo",
     "resource_type": "https://clowder.ncsa.illinois.edu/resource/type",
     "url": "https://clowder.ncsa.illinois.edu/resource"
   }
 }
-
-

public/swagger.yml

@@ -9,7 +9,7 @@ info:
     Clowder is a customizable and scalable data management system to support any
     data format and multiple research domains. It is under active development
     and deployed for a variety of research projects.
-  version: 1.11.0
+  version: 1.11.1
   termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos
   contact:
     name: Clowder

scripts/ubuntu/update-clowder.sh

@@ -1,48 +1,83 @@
 #!/bin/bash
 
-# CATS-CORE is the main branch for this server
-CLOWDER_BRANCH=${CLOWDER_BRANCH:-"CATS-CORE"}
-CLOWDER_BUILD=${CLOWDER_BUILD:-"latestSuccessful"}
+# what version ("" == latest release)
+CLOWDER_BRANCH=${CLOWDER_BRANCH:-}
+#CLOWDER_BRANCH="develop"
+
+# compute some other variables
+if [ "${CLOWDER_BRANCH}" = "" ]; then
+  CLOWDER_VERSION=$(curl -s  https://opensource.ncsa.illinois.edu/projects/artifacts.php?key=CATS | grep '<option selected>' | sed 's#.*<option.*>\([0-9\.]*\)</option>#\1#')
+  CLOWDER_ZIPFILE=clowder-${CLOWDER_VERSION}.zip
+else
+  CLOWDER_VERSION="${CLOWDER_BRANCH}"
+  CLOWDER_ZIPFILE=clowder-${CLOWDER_VERSION}-${CLOWDER_BRANCH}.zip
+fi
+
+# Should text go to stdout, empty is not
+STDOUT="YES"
+
+# HipChat token for notifications
+HIPCHAT_TOKEN=""
+HIPCHAT_ROOM=""
+
+# Slack token for notifications
+SLACK_TOKEN=""
+SLACK_CHANNEL="#simpl-ops"
+
+# MSTeams Webhook
+MSTEAMS_URL=""
+
+# INFLUXDB
+INFLUXDB_URL=""
+INFLUXDB_DATABASE=""
+INFLUXDB_USERNAME=""
+INFLUXDB_PASSWORD=""
 
 # change to folder where script is installed
 cd /home/clowder
 
 # fetch software
-if [[ ${CLOWDER_BUILD} == latest* ]]; then
-  BB="${CLOWDER_BRANCH}/${CLOWDER_BUILD}"
+if [ -e ]; then
+  curl -f -s -z "${CLOWDER_ZIPFILE}" -o "${CLOWDER_ZIPFILE}" https://opensource.ncsa.illinois.edu/projects/artifacts/CATS/${CLOWDER_VERSION}/files/${CLOWDER_ZIPFILE} || exit 1
 else
-  BB="${CLOWDER_BRANCH}-${CLOWDER_BUILD}"
+  curl -f -s -o "${CLOWDER_ZIPFILE}" https://opensource.ncsa.illinois.edu/projects/artifacts/CATS/${CLOWDER_VERSION}/files/${CLOWDER_ZIPFILE} || exit 1
 fi
-URL="https://opensource.ncsa.illinois.edu/bamboo/browse/${BB}/artifact/shared/dist/"
-/usr/bin/wget -q -e robots=off -A "clowder-*.zip" -nd -r -N -l1 ${URL}
-LATEST=$( /bin/ls -1rt clowder-*.zip | tail -1 )
 
-if [ -s ${LATEST} ]; then
-  if [ "$1" == "--force" -o ${LATEST} -nt clowder ]; then
+if [ -s ${CLOWDER_ZIPFILE} ]; then
+  if [ "$1" == "--force" -o ${CLOWDER_ZIPFILE} -nt clowder ]; then
+    exec 3>&1
+    exec &> "/tmp/$$.txt"
+
     echo "UPDATING CLOWDER ON ${HOSTNAME}"
     echo " bamboo  branch  = ${CLOWDER_BRANCH}"
 
     # stop clowder
-    /sbin/stop clowder
+    /bin/systemctl stop clowder
 
     # save local modifications
     if [ -d clowder/custom ]; then
       mv clowder/custom clowder.custom
     fi
+    if [ -d clowder/logs ]; then
+      mv clowder/logs clowder.logs
+    fi
 
     # install new version
-    /bin/rm -rf clowder $( basename ${LATEST} .zip )
-    /usr/bin/unzip -q ${LATEST}
-    /bin/mv $( basename ${LATEST} .zip ) clowder
+    /bin/rm -rf clowder $( basename ${CLOWDER_ZIPFILE} .zip )
+    /usr/bin/unzip -q ${CLOWDER_ZIPFILE}
+    /bin/mv $( basename ${CLOWDER_ZIPFILE} .zip ) clowder
     /usr/bin/touch clowder
 
     # get some nice values from the build
-    echo " bamboo  build   = $( grep '\-Dbuild.bamboo' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
+    echo " github  build   = $( grep '\-Dbuild.bamboo' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
     echo " clowder version = $( grep '\-Dbuild.version' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
     echo " clowder branch  = $( grep '\-Dbuild.branch' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
     echo " clowder gitsha1 = $( grep '\-Dbuild.gitsha1' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )"
 
     # restore local modifications
+    if [ -d clowder.logs ]; then
+      mv clowder.logs clowder/logs
+    fi
     if [ -d clowder.custom ]; then
       mv clowder.custom clowder/custom
     fi
@@ -56,6 +91,41 @@ if [ -s ${LATEST} ]; then
     fi
 
     # start clowder again
-    /sbin/start clowder
+    /bin/systemctl start clowder
+
+    # send message by hipchat
+    if [ "${HIPCHAT_TOKEN}" != "" -a "${HIPCHAT_ROOM}" != "" ]; then
+      url="https://hipchat.ncsa.illinois.edu/v1/rooms/message?format=json&auth_token=${HIPCHAT_TOKEN}"
+      txt=$(cat /tmp/$$.txt | sed 's/ /%20/g;s/!/%21/g;s/"/%22/g;s/#/%23/g;s/\$/%24/g;s/\&/%26/g;s/'\''/%27/g;s/(/%28/g;s/)/%29/g;s/:/%3A/g;s/$/<br\/>/g')
+      room=$(echo ${HIPCHAT_ROOM} | sed 's/ /%20/g;s/!/%21/g;s/"/%22/g;s/#/%23/g;s/\$/%24/g;s/\&/%26/g;s/'\''/%27/g;s/(/%28/g;s/)/%29/g;s/:/%3A/g')
+      body="room_id=${room}&from=clowder&message=${txt}"
+      result=$(curl -s -X POST -d "${body}" $url)
+    fi
+    if [ "${SLACK_TOKEN}" != "" -a "${SLACK_CHANNEL}" != "" ]; then
+      url="https://hooks.slack.com/services/${SLACK_TOKEN}"
+      txt=$(cat /tmp/$$.txt | sed 's/"/\\"/g;s/$/\\/g' | tr '\n' 'n' )
+      payload="payload={\"channel\": \"${SLACK_CHANNEL}\", \"username\": \"clowder\", \"text\": \"${txt}\", \"icon_url\": \"https://opensource.ncsa.illinois.edu/projects/artifacts/CATS/logo.png\"}"
+      result=$(curl -s -X POST --data-urlencode "${payload}" $url)
+    fi
+    if [ "${MSTEAMS_URL}" != "" ]; then
+      txt=$(cat /tmp/$$.txt | sed 's/"/\\"/g;s/$/\\/g' | tr '\n' 'n' )
+      payload="{\"title\": \"UPDATE CLOWDER\", \"text\": \"${txt}\" }"
+      result=$(curl -s -X POST -H "Content-Type: application/json" -d "${payload}" $MSTEAMS_URL)
+    fi
+    if [ "${INFLUXDB_URL}" != "" -a "${INFLUXDB_DATABASE}" != "" -a "${INFLUXDB_USERNAME}" != "" -a "${INFLUXDB_PASSWORD}" != "" ]; then
+      url="${INFLUXDB_URL}/api/v2/write?bucket=${INFLUXDB_DATABASE}&precision=ns"
+      tags="host=${HOSTNAME}"
+      timestamp="$(date -u +"%s%N")"
+      values="version=\"$( grep '\-Dbuild.version' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )\""
+      values="${values},branch=\"$( grep '\-Dbuild.branch' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )\""
+      values="${values},build=\"$( grep '\-Dbuild.bamboo' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )\""
+      values="${values},gitsha1=\"$( grep '\-Dbuild.gitsha1' clowder/bin/clowder | sed 's/.*=\(.*\)"$/\1/' )\""
+      auth="Authorization: Token ${INFLUXDB_USERNAME}:${INFLUXDB_PASSWORD}"
+      result=$(curl -s -i -XPOST "$url" --header "$auth" --data "clowder_update,${tags} ${values} ${timestamp}")
+    fi
+    if [ "${STDOUT}" != "" ]; then
+      cat /tmp/$$.txt >&3
+    fi
+    rm /tmp/$$.txt
   fi
 fi