Merge pull request #379 from clowder-framework/release/1.21.0

Release/1.21.0

.github/workflows/docker.yml

@@ -29,6 +29,8 @@ env:
 jobs:
   docker:
     runs-on: ubuntu-latest
+    env:
+      dockerhub: ${{ secrets.DOCKERHUB_USERNAME }}
     permissions:
       packages: write
     strategy:
@@ -95,7 +97,9 @@ jobs:
 
           push_tags=""
           for tag in ${tags}; do
-            push_tags="${push_tags}${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}:${tag},"
+            if [ "${{ secrets.DOCKERHUB_USERNAME }}" == "" ]; then
+              push_tags="${push_tags}${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}:${tag},"
+            fi
             push_tags="${push_tags}ghcr.io/${{ github.repository_owner }}/${{ matrix.IMAGE }}:${tag},"
           done
           push_tags="${push_tags%,*}"
@@ -127,6 +131,7 @@ jobs:
 
       # login to registries
       - name: Login to DockerHub
+        if: env.dockerhub != ''
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -193,7 +198,7 @@ jobs:
 
       # update README at DockerHub
       - name: Docker Hub Description
-        if: matrix.README != '' && github.event_name == 'push' && github.repository == env.MASTER_REPO && env.BRANCH == 'master'
+        if: env.dockerhub != '' && matrix.README != '' && github.event_name == 'push' && github.repository == env.MASTER_REPO && env.BRANCH == 'master'
         uses: peter-evans/dockerhub-description@v2
         env:
           DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}

CHANGELOG.md

@@ -5,6 +5,36 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+
+## 1.21.0 - 2022-08-23
+
+**_Important:_** This update requires a MongoDB update schema due to a bug in the original migration  of showing summary statistics at the
+space level. Make sure to start the application with -DMONGOUPDATE=1. You can also run the [fixCounts.js](https://github.com/clowder-framework/clowder/blob/develop/scripts/updates/fix-counts.js)
+script prior to upgrading to minimize the downtime.
+
+### Added
+- api.Files jsonfile, adds two fields "downloads" and "views" [#228](https://github.com/clowder-framework/clowder/issues/228)
+- Dataset and file scala.html pages incl schema.org jsonld metadata for (google)datasetsearch [#335](https://github.com/clowder-framework/clowder/issues/335)
+- MiniUser and LicenseData now have to_jsonld methods to return string part of [#335](https://github.com/clowder-framework/clowder/issues/335) metadata
+- LicenseData has urlViaAttributes used by it's to_jsonld to guess url when empty, for [#335](https://github.com/clowder-framework/clowder/issues/335)
+- MRI previewer for NIFTY (.nii) files.
+- Dataset page usually defaults to Files tab, but if no files will now show Metadata first
+- HEIC (.heic) and HEIF (.heif) mimetypes to support new Apple iPhone image file format.
+- In the docker container the folder /home/clowder/data is now whitelisted by default for uploading by reference. 
+  This can be changed using the environment variable CLOWDER_SOURCEPATH.
+- The current CLA for developers of clowder.
+
+### Fixed
+- Send email to all admins in a single email when a user submits 'Request access' for a space
+- Send email to all admins and request user in a single email when any admin accepts/rejects 'Request access' for a space [#330](https://github.com/clowder-framework/clowder/issues/330)
+- script/code to count space in files was not correct [#366](https://github.com/clowder-framework/clowder/issues/336)
+- github actions would fail for docker builds due to secrets not existing
+- Fix to remove dataset from a space [#349](https://github.com/clowder-framework/clowder/issues/349)
+
+### Changed
+- Utils.baseURL now on RequestHeader instead of Request[Any]
+- MongoDB Service log error:'Not all dataset IDs found for Dataset|Folder bulk get request', now incl all the IDs notFound
+
 ## 1.20.3 - 2022-06-10
 
 ### Fixed
@@ -13,6 +43,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Changed
 - docker builds images for amd64 and arm64 [#322](https://github.com/clowder-framework/clowder/issues/322)
 
+
 ## 1.20.2 - 2022-04-30
 
 ### Fixed
@@ -37,15 +68,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Documentation: Added "How to contribute documentation" page
 - Documentation: New Sphinx plugins for dropdowns and menus.
 
+
 ## 1.20.0 - 2022-02-07
 
+### Added 
+
+- An IFC previewer
+
 ### Fixed
 - Conf file and code had incosistent spelling of BagIt. Now all have capital B and I.
 - When event stream is disabled don't show for logged in user [#280](https://github.com/clowder-framework/clowder/issues/280)
 - three.js is no longer associated with application/octet-stream, now with models [#305](https://github.com/clowder-framework/clowder/issues/305)
 
 ### Changed
 - Download of dataset/collection now has optional parameter bagit (default false) to download items in bagit format.
+- The FBX previewer can also load GLTF files
 
 ## 1.19.5 - 2022-01-21
 

CLA.md

@@ -0,0 +1,28 @@
+# CLOWDER PROJECT Contributor License Agreement 
+
+Thank you for your interest in contributing to the Clowder Project. In order to contribute, you will need to provide your name and contact information and sign this Clowder Project Contributor License Agreement, which sets for the terms and conditions of the intellectual property license granted with your contributions.
+
+This Clowder Project Contributor License Agreement (“Agreement”) is by and between you (any person or entity “You” or “Your”) and The Board of Trustees of the University of Illinois, through its National Center for Supercomputing Applications (“Illinois”). Please read this document carefully before signing and keep a copy for your records . By signing this Agreement or making a “Contribution” to the “Clowder Project” as defined below, You agree to the following: 
+
+1. “Clowder Project” is an open-source project that aims to simplify the management of research data. Clowder provides tools to manage the full lifecycle of research data; scalable with respect to data size and extensible to the needs of different resea rch domains.
+2. “Contribution” means all of Your contributions of object code, source code, and documentation and any modifications thereof to the Clowder Project.
+3. “Licensed Patents” mean patent claims licensable by Contributor which are necessarily infringed by the making, using, selling, offering for sale, having made, import, or transfer of either its Contribution alone or when combined with the Clowder Project.
+4. You represent that to the best of your knowledge the following:
+   1. You are at least 18 years of age and have full power and authority to enter into this Agreement and to grant the rights in and to the Contribution as set forth herein (individuals who are under 18 years of age and who wish to contribute to the Clowder project may not enter into this Agreement, but may contact Clowder at [email protected] to explore alternatives);
+   2. If your employer has rights to intellectual property that You create as part of the Contribution, You represent that you have obtained permission from Your employer to make Contributions on behalf of that employer or Your employer waived any rights in and to Your Contributions, or your employer authorizes the Contribution and agrees to be bound by the terms herein by signing as an entity below;
+   3. That either:
+      1. all documentation and code in the Contribution is Your original work and includes complete details of any thirdparty license and any other restriction (including, but not limited to related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions; or
+      2. any part of the Contribution that is not Your original creation is submitted to Clowder separately from any original Contribution, includes the complete details of its source and any corresponding license and any other restriction (including, but not limited to related patents, trademarks, and license agreements) of which you are personally aware, and is conspicuously marked as "Submitted on behalf of a third-party: [named here]".
+   4. That Your Contribution does not include any viruses, worms, Trojan horses, malicious code or other harmful or destructive content;
+   5. That You are not debarred from receiving services or other exports under U.S. law, including, without limitation, the Foreign Assets Control Regulations, 31 C.F.R. 500 et seq.; the Export Administration Regulations, 15 C.F.R. 730 et seq.; and the International Traffic in Arms Regulations, 22 C.F.R. 120 et seq. Persons who may not be eligible to receive services or exports under U.S. law include citizens and residents of countries subject to U.S. embargoes, and individuals specifically identified on the Specially Designated Nationals List, the Denied Persons List , the Arms Export Control Act Debarred Parties List, or any other list or General Order issued by the U.S. Department of the Treasury, Office of Foreign Assets Control; the U.S. Department of Commerce, Bureau of Industry and Security; the U.S. Department of State, Directorate of Defense Trade Controls; or any other agency with jurisdiction to issue debarment orders; and
+   6. Your Contribution does not include any encryption technology and no government license or permission is required for the export, import, transfer or use of the Contribution.
+5. You represent that the representations made herein are accurate and agree to notify Illinois of any facts or circumstances of which You become aware that would make any of Your representations inaccurate in any respect.
+6. You hereby grant to Illinois and to recipients of the Clowder software distributed by Illinois (collectively, “Recipient”), a perpetual, irrevocable, non-exclusive, worldwide, royalty-free unrestricted license to use, reproduce, prepare derivative works of, publically display, publically perform, distribute, and sublicense the Contribution, and such derivative works, in source code and object code form.
+7. You hereby grant to Recipient a perpetual, non-exclusive, worldwide, royalty-free patent license under Licensed Patents, if any, to make, have made, use, offer to sell, sell, import, and otherwise transfer Your Contribution in source code and object code form. This patent license shall apply to the combination of Your Contribution as part of the Clowder Project if, at the time the Contribution is added by You, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. If any Recipient institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that Your Contribution or Clowder Project (excluding combinations of the Clowder Project with other software or hardware) infringes Your Licensed Patent, then the patent license granted to such Recipient under this Agreement shall terminate.
+8. Except for the rights granted to Recipients above, You reserve all right, title and interest in and to Your Contribution. You are not expected to provide support for your Contributions.
+9. Subject to Your representations above, Your Contributions are provided on an “AS-IS” basis WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND (express or implied), including, without limitation, any implied warranty of merchantability and fitness for a particular purpose and any warranty of non-infringement.
+10. Illinois, its trustees, directors, officers, employees, students and agents assume no liability in respect of any infringement of any copyright, patent or other right of third parties in connection with any Contributions, the Clowder Project, or Clowder software, and are not liable for any direct, indirect, punitive, special, incidental, consequential or exemplary damages arising in connection with any Contribution, the Clowder Project, or Clowder software.
+11. INDEMNITY. You agree to indemnify and hold Clowder and its subsidiaries, affiliates, officers, agents, employees, partners and licensors harmless from any claim or demand, including but not limited to reasonable attorneys' fees, made by any third party due to or arising out of Contributions and Content you submit, post, transmit or otherwise make available through the Clowder Project, your violation of the Clowder Code of Conduct at http://clowderframework.org/, or your violation of any rights of another.
+12. You agree that Illinois may assign this Agreement and Clowder Project to any third party.
+13. The Clowder Project is under no obligation to accept and include every Contribution.
+14. This Agreement is governed by the laws of the State of Illinois, excluding its conflict of laws provisions. 

CONTRIBUTORS.md

@@ -31,6 +31,7 @@ Following is a list of contributors in alphabetical order:
 - Mike Lambert
 - Nicholas Tenczar
 - Nishant Nayudu
+- Peter Groves
 - Rob Kooper
 - Rui Liu
 - Sandeep Puthanveetil Satheesan

app/api/Files.scala

@@ -716,6 +716,12 @@ class Files @Inject()(
   }
 
   def jsonFile(file: File, serverAdmin: Boolean = false): JsValue = {
+    val foldersContainingFile = folders.findByFileId(file.id)
+    val allPaths: List[List[String]] =  (for (folder <- foldersContainingFile) yield (folderPath(folder, List()).tail))
+    var path_str = allPaths.map(xl => "/" + xl.map(x => x.toString()).mkString("/")).mkString("")
+    if(path_str == "") {
+       path_str = "/" 
+    }
     val defaultMap = Map(
       "id" -> file.id.toString,
       "filename" -> file.filename,
@@ -725,7 +731,11 @@ class Files @Inject()(
       "size" -> file.length.toString,
       "thumbnail" -> file.thumbnail_id.orNull,
       "authorId" -> file.author.id.stringify,
-      "status" -> file.status)
+      "status" -> file.status,
+      "views" -> file.stats.views.toString(),
+      "downloads" -> file.stats.downloads.toString(),
+      "path" -> path_str
+      )
 
     // Only include filepath if using DiskByte storage and user is serverAdmin
     val jsonMap = file.loader match {

app/api/Spaces.scala

@@ -13,6 +13,7 @@ import util.Mail
 
 import java.util.Date
 import javax.inject.Inject
+import scala.collection.mutable.ListBuffer
 
 /**
  * Spaces allow users to partition the data into realms only accessible to users with the right permissions.
@@ -687,9 +688,8 @@ class Spaces @Inject()(spaces: SpaceService,
             }
             if(requestUser.email.isDefined) {
               val subject: String = "Authorization Request from " + AppConfiguration.getDisplayName + " Accepted"
-              val recipient: String = requestUser.email.get.toString
               val body = views.html.spaces.requestresponseemail(user.get, id.toString, s.name, "accepted your request and assigned you as " + role + " to")
-              Mail.sendEmail(subject, request.user, recipient, body)
+              Mail.sendEmail(subject, request.user, getRecipientsList(s, requestUser), body)
             }
             Ok(Json.obj("status" -> "success"))
           }
@@ -700,6 +700,22 @@ class Spaces @Inject()(spaces: SpaceService,
     }
   }
 
+  def getRecipientsList(s: ProjectSpace, requestUser: User): List[String] = {
+    val recipients = new ListBuffer[String]()
+    recipients += requestUser.email.get.toString
+
+    for (requestReceiver <- spaces.getUsersInSpace(s.id, None)) {
+      spaces.getRoleForUserInSpace(s.id, requestReceiver.id) match {
+        case Some(aRole) => {
+          if (aRole.permissions.contains(Permission.EditSpace.toString) && requestReceiver.id != requestUser.id) {
+            recipients += requestReceiver.toString
+          }
+        }
+      }
+    }
+    return recipients.toList
+  }
+
   def rejectRequest(id:UUID, requestuser:String) = PermissionAction(Permission.EditSpace, Some(ResourceRef(ResourceRef.space, id))) { implicit request =>
     implicit val user = request.user
     spaces.get(id) match {
@@ -711,9 +727,8 @@ class Spaces @Inject()(spaces: SpaceService,
             spaces.removeRequest(id, requestUser.id)
             if(requestUser.email.isDefined) {
               val subject: String = "Authorization Request from " + AppConfiguration.getDisplayName + " Rejected"
-              val recipient: String = requestUser.email.get.toString
               val body = views.html.spaces.requestresponseemail(user.get, id.toString, s.name, "rejected your request to")
-              Mail.sendEmail(subject, request.user, recipient, body)
+              Mail.sendEmail(subject, request.user, getRecipientsList(s, requestUser), body)
             }
             Ok(Json.obj("status" -> "success"))
           }

app/controllers/Datasets.scala

@@ -836,4 +836,4 @@ class Datasets @Inject() (
     implicit val user = request.user
     Ok(views.html.generalMetadataSearch())
   }
-}
+}

app/controllers/Spaces.scala

@@ -354,7 +354,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
   def addRequest(id: UUID) = AuthenticatedAction { implicit request =>
     implicit val requestuser = request.user
     requestuser match {
-      case Some(user) => {
+      case Some(user) =>
         spaces.get(id) match {
           case Some(s) => {
             // when permission is public, user can reach the authorization request button, so we check if the request is
@@ -367,24 +367,25 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
               Logger.debug("Request submitted in controller.Space.addRequest  ")
               val subject: String = "Request for access from " + AppConfiguration.getDisplayName
               val body = views.html.spaces.requestemail(user, id.toString, s.name)
+              val spaceAdminRecipients = new ListBuffer[String]
 
               for (requestReceiver <- spaces.getUsersInSpace(s.id, None)) {
                 spaces.getRoleForUserInSpace(s.id, requestReceiver.id) match {
                   case Some(aRole) => {
                     if (aRole.permissions.contains(Permission.EditSpace.toString)) {
                       events.addRequestEvent(Some(user), requestReceiver, id, s.name, "postrequest_space")
-                      Mail.sendEmail(subject, request.user, requestReceiver, body)
+                      spaceAdminRecipients += requestReceiver.toString
                     }
                   }
                 }
               }
+              Mail.sendEmail(subject, request.user, spaceAdminRecipients.toList, body)
               spaces.addRequest(id, user.id, user.fullName)
               Ok(views.html.authorizationMessage("Request submitted for " + spaceTitle + " " + s.name, s))
             }
           }
           case None => InternalServerError(spaceTitle + " not found")
         }
-      }
 
       case None => InternalServerError("User not found")
     }

app/controllers/Utils.scala

@@ -13,12 +13,13 @@ object Utils {
   /**
    * Return base url given a request. This will add http or https to the front, for example
    * https://localhost:9443 will be returned if it is using https.
+   *
    */
-  def baseUrl(request: Request[Any], absolute: Boolean = true) = {
+  def baseUrl(request: RequestHeader, absolute: Boolean = true) = {  
     if (absolute) {
-      routes.Files.list().absoluteURL(https(request))(request).replace("/files", "")
+       routes.Files.list().absoluteURL(https(request))(request).replace("/files", "")
     } else {
-      routes.Files.list().url.replace("/files", "")
+       routes.Files.list().url.replace("/files", "")
     }
   }
 
@@ -171,4 +172,4 @@ object Utils {
       decodedReplies.toList
     }
   }
-}
+}