Merge pull request #302 from clowder-framework/release/1.19.3

Release v1.19.3

CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.19.3 - 2021-11-11
+
+See [fix-counts.js](scripts/updates/fix-count.js) for a script that can be run before this update (as well as the
+update for 1.19.0) to pre populate the migration. This will speed up the update and will not impact the running
+instance.
+
+### Fixed
+- If a space has a lot of datasets, rendering the space page is very slow. Files in a space is now cached.
+- Set permissions for folders to be 777 this fixes [clowder-helm#5](https://github.com/clowder-framework/clowder-helm/issues/5)
+
 ## 1.19.2 - 2021-10-20
 
 ### Fixed

Dockerfile

@@ -67,7 +67,7 @@ COPY docker/custom.conf docker/play.plugins /home/clowder/custom/
 # Containers should NOT run as root as a good practice
 # numeric id to be compatible with openshift, will run as random userid:0
 RUN mkdir -p /home/clowder/data && \
-    chmod g+w /home/clowder/logs /home/clowder/data /home/clowder/custom
+    chmod 777 /home/clowder/logs /home/clowder/data /home/clowder/custom
 USER 10001
 
 # command to run when starting docker

app/api/Files.scala

@@ -1678,6 +1678,7 @@ class Files @Inject()(
           val ds_spaces = ds.spaces
           for (ds_s <- ds_spaces) {
             spaces.decrementSpaceBytes(ds_s, file.length)
+            spaces.decrementFileCounter(ds_s, 1)
           }
         }
 

app/api/Spaces.scala

@@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService,
         val userId = request.user.get.id
         val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId,
           homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0,
-          datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
+          datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
         spaces.insert(c) match {
           case Some(id) => {
             appConfig.incrementCount('spaces, 1)

app/controllers/SecuredController.scala

@@ -153,7 +153,7 @@ trait SecuredController extends Controller {
         val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService])
         spaces.get(id) match {
           case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user)))
-          case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user)))
+          case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user)))
         }
       }
 

app/controllers/Spaces.scala

@@ -167,8 +167,6 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
         var creatorActual: User = null
         val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size))
         val datasetsInSpace = datasets.listSpace(size, id.toString(), user)
-        val spaceBytes : Long = s.spaceBytes
-        val spaceFiles : Integer = getFilesPerSpace(id, user)
         val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user)
         val usersInSpace = spaces.getUsersInSpace(id, None)
         var curationObjectsInSpace: List[CurationObject] = List()
@@ -217,7 +215,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
           case None => List.empty
         }
         sinkService.logSpaceViewEvent(s, user)
-        Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles))
+        Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections))
       }
       case None => BadRequest(views.html.notFound(spaceTitle + " does not exist."))
     }
@@ -414,7 +412,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
                       val newSpace = ProjectSpace(name = formData.name, description = formData.description,
                         created = new Date, creator = userId, homePage = formData.homePage,
                         logoURL = formData.logoURL, bannerURL = formData.bannerURL,
-                        collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty,
+                        collectionCount = 0, datasetCount = 0, fileCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty,
                         resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled,
                         status = formData.access,
                         affiliatedSpaces = formData.affSpace)
@@ -640,15 +638,4 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
         case None => BadRequest(views.html.notFound(spaceTitle + " does not exist."))
       }
   }
-
-  private def getFilesPerSpace(spaceId: UUID, user: Option[User]) : Integer = {
-    var spaceFiles: Integer = 0
-    val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), user)
-    for (ds <- allDatasetsInSpace) {
-      val files_in_ds = ds.files.length
-      spaceFiles += files_in_ds
-    }
-    spaceFiles
-  }
-
 }

app/models/Space.scala

@@ -22,6 +22,7 @@ case class ProjectSpace (
   bannerURL: Option[URL],
   collectionCount: Integer,
   datasetCount: Integer,
+  fileCount: Integer,
   userCount: Integer,
   spaceBytes: Long, 
   metadata: List[Metadata],

app/services/SpaceService.scala

@@ -94,6 +94,10 @@ trait SpaceService {
 
   def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int)
 
+  def incrementFileCounter(space: UUID, increment: Long)
+
+  def decrementFileCounter(space: UUID, decrement: Long)
+
   def incrementSpaceBytes(space: UUID, increment: Long)
 
   def decrementSpaceBytes(space: UUID, decrement: Long)

app/services/mongodb/MongoDBDatasetService.scala

@@ -1399,6 +1399,15 @@ class MongoDBDatasetService @Inject() (
         }
         for (f <- dataset.files) {
           val notTheDataset = for (currDataset <- findByFileIdDirectlyContain(f) if !dataset.id.toString.equals(currDataset.id.toString)) yield currDataset
+          files.get(f) match {
+            case Some(file) => {
+              for(space <- dataset.spaces) {
+                spaces.decrementFileCounter(space, 1)
+                spaces.decrementSpaceBytes(space, file.length)
+              }
+            }
+            case None => Logger.error(s"Error file with with id ${f} no longer exists")
+          }
           if (notTheDataset.size == 0)
             files.removeFile(f, host, apiKey, user)
         }

app/services/mongodb/MongoDBSpaceService.scala

@@ -375,15 +375,23 @@ class MongoDBSpaceService @Inject() (
   }
 
   def decrementCollectionCounter(collection: UUID, space: UUID, decrement: Int): Unit = {
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
+  }
+
+  def incrementFileCounter(space: UUID, increment: Long ): Unit = {
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> increment), upsert=false, multi=false, WriteConcern.Safe)
+  }
+
+  def decrementFileCounter(space: UUID, decrement: Long): Unit = {
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
   }
 
   def incrementSpaceBytes(space: UUID, increment: Long ): Unit = {
     ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> increment), upsert=false, multi=false, WriteConcern.Safe)
   }
 
   def decrementSpaceBytes(space: UUID, decrement: Long): Unit = {
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe)
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * decrement), upsert=false, multi=false, WriteConcern.Safe)
   }
 
   def removeCollection(collection:UUID, space:UUID): Unit = {
@@ -398,11 +406,15 @@ class MongoDBSpaceService @Inject() (
    */
   def addDataset(dataset: UUID, space: UUID): Unit = {
     log.debug(s"Space Service - Adding $dataset to $space")
-    val datasetBytes = datasets.getBytesForDataset(dataset)
-    datasets.addToSpace(dataset, space)
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe)
-
+    datasets.get(dataset) match {
+      case Some(x) => {
+        val datasetBytes = datasets.getBytesForDataset(dataset)
+        datasets.addToSpace(dataset, space)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> x.files.length), upsert=false, multi=false, WriteConcern.Safe)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe)
+      }
+    }
   }
 
   /**
@@ -414,9 +426,15 @@ class MongoDBSpaceService @Inject() (
   def removeDataset(dataset:UUID, space:UUID): Unit = {
     log.debug(s"Space Service - removing $dataset from $space")
     datasets.removeFromSpace(dataset, space)
-    val datasetBytes = datasets.getBytesForDataset(dataset)
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
-    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
+    datasets.get(dataset) match {
+      case Some(x) => {
+        val datasetBytes = datasets.getBytesForDataset(dataset)
+        datasets.addToSpace(dataset, space)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -1 * datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("fileCount" -> -1 * x.files.length), upsert=false, multi=false, WriteConcern.Safe)
+        ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
+      }
+    }
   }
 
   /**

app/services/mongodb/MongoSalatPlugin.scala

@@ -452,6 +452,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
 
     // Adds space bytes to space
     updateMongo(updateKey = "update-space-bytes", updateSpaceBytes)
+    updateMongo(updateKey = "update-space-files", updateSpaceFiles)
   }
 
   private def updateMongo(updateKey: String, block: () => Unit): Unit = {
@@ -513,7 +514,8 @@ class MongoSalatPlugin(app: Application) extends Plugin {
         val spacename = java.net.InetAddress.getLocalHost.getHostName
         val newspace = new ProjectSpace(name = spacename, description = "", created = new Date(), creator = UUID("000000000000000000000000"),
           homePage = List.empty[URL], logoURL = None, bannerURL = None, metadata = List.empty[Metadata],
-          collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, spaceBytes = 0)
+          collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, fileCount = 0,
+          spaceBytes = 0)
         ProjectSpaceDAO.save(newspace)
         val spaceId = new ObjectId(newspace.id.stringify)
 
@@ -1707,4 +1709,16 @@ class MongoSalatPlugin(app: Application) extends Plugin {
       collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("spaceBytes" -> currentSpaceBytes))
     }
   }
+
+  private def updateSpaceFiles(): Unit = {
+    collection("spaces.projects").find().toList.foreach{ space =>
+      var fileCount: Integer = 0
+      val spaceId = space.get("_id")
+      val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList
+      spaceDatasets.foreach{ spaceDataset =>
+        fileCount += spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty).length
+      }
+      collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("fileCount" -> fileCount))
+    }
+  }
 }

app/util/FileUtils.scala

@@ -595,6 +595,7 @@ object FileUtils {
           val datasetSpaces = dataset.get.spaces
           for (s <- datasetSpaces){
             spaceService.incrementSpaceBytes(s, file.length)
+            spaceService.incrementFileCounter(s, 1)
           }
         }
       }

app/views/spaces/space.scala.html

@@ -1,4 +1,4 @@
-@(space: ProjectSpace, collections: List[Collection], publicDatasets: List[Dataset], datasets: List[Dataset], publishedData: List[play.api.libs.json.JsValue], servicesUrl: String, userRoleMap: Map[User, String], userSelections: List[String], spaceBytes: Long, spaceFiles: Integer)(implicit user: Option[models.User])
+@(space: ProjectSpace, collections: List[Collection], publicDatasets: List[Dataset], datasets: List[Dataset], publishedData: List[play.api.libs.json.JsValue], servicesUrl: String, userRoleMap: Map[User, String], userSelections: List[String])(implicit user: Option[models.User])
 @import play.api.libs.json._
 
 @import play.api.Play.current
@@ -119,10 +119,10 @@ <h3>@Messages("collections.title")</h3>
         @* right column, space statistics, actions *@
         <div class="col-md-4 col-lg-4 col-sm-4">
             @if(user.isDefined) {
-                @spaces.statistics(space, spaceBytes, None, spaceFiles )
+                @spaces.statistics(space, None )
                 @spaces.otherActions(space)
             } else {
-                @spaces.statistics(space, spaceBytes, None, spaceFiles)
+                @spaces.statistics(space, None)
             }
             @spaces.externalLinks(space.homePage, space, "row ds-section-sm")
             @if(play.Play.application().configuration().getBoolean("enablePublic")) {

app/views/spaces/statistics.scala.html

@@ -1,4 +1,4 @@
-@(space: ProjectSpace, spaceBytes: Long, isPublic: Option[Boolean], spaceFiles: Integer)(implicit user: Option[models.User])
+@(space: ProjectSpace, isPublic: Option[Boolean])(implicit user: Option[models.User])
 
 @import play.api.i18n.Messages
 @import _root_.util.Formatters._
@@ -12,23 +12,23 @@ <h3>Statistics</h3>
             <tbody>
                 <tr>
                     <td><b><a href="@routes.Collections.sortedListInSpace(space.id.stringify, 0,12,isPublic.getOrElse(true))">@Messages("collections.title")</a></b></td>
-                    <td><span class="pull-right">@space.collectionCount</span></td>
+                    <td><span class="pull-right">@humanReadableNumber(space.collectionCount)</span></td>
                 </tr>
                 <tr>
                     <td><b><a href="@routes.Datasets.sortedListInSpace(space.id.stringify, 0,12,isPublic.getOrElse(true))">@Messages("datasets.title")</a></b></td>
-                    <td><span class="pull-right">@space.datasetCount</span></td>
+                    <td><span class="pull-right">@humanReadableNumber(space.datasetCount)</span></td>
                 </tr>
                 <tr>
                     <td><b>Files:</b></td>
-                    <td><span class="pull-right">@spaceFiles</span></td>
+                    <td><span class="pull-right">@humanReadableNumber(space.fileCount)</span></td>
                 </tr>
                 <tr>
                     <td><b>Bytes:</b></td>
-                    <td><span class="pull-right">@humanReadableByteCount(spaceBytes)</span></td>
+                    <td><span class="pull-right">@humanReadableByteCount(space.spaceBytes)</span></td>
                 </tr>
                 <tr>
                     <td><b>Users:</b></td>
-                    <td><span class="pull-right">@space.userCount</span></td>
+                    <td><span class="pull-right">@humanReadableNumber(space.userCount)</span></td>
                 </tr>
             </tbody>
         </table>

doc/src/sphinx/conf.py

@@ -22,7 +22,7 @@
 author = 'Luigi Marini'
 
 # The full version, including alpha/beta/rc tags
-release = '1.19.2'
+release = '1.19.3'
 
 
 # -- General configuration ---------------------------------------------------

project/Build.scala

@@ -13,7 +13,7 @@ import NativePackagerKeys._
 object ApplicationBuild extends Build {
 
   val appName = "clowder"
-  val version = "1.19.2"
+  val version = "1.19.3"
   val jvm = "1.7"
 
   def appVersion: String = {

public/swagger.yml

@@ -9,7 +9,7 @@ info:
     Clowder is a customizable and scalable data management system to support any
     data format and multiple research domains. It is under active development
     and deployed for a variety of research projects.
-  version: 1.19.2
+  version: 1.19.3
   termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos
   contact:
     name: Clowder

scripts/updates/fix-counts.js

@@ -1,7 +1,28 @@
+db.app.configuration.update({"key" : "mongodb.updates"}, { $addToSet: {"value": "update-space-bytes"}})
+db.spaces.projects.update({"spaceBytes": {$exists : false}}, {$set: {'spaceBytes': NumberLong(0)}})
+db.app.configuration.update({"key" : "mongodb.updates"}, { $addToSet: {"value": "update-space-files"}})
+db.spaces.projects.update({"fileCount": {$exists : false}}, {$set: {'fileCount': 0}})
+
 db.app.configuration.update({"key" : "countof.users"}, {$set: { "value": NumberLong(db.social.users.count())}})
 db.app.configuration.update({"key" : "countof.datasets"}, {$set: { "value": NumberLong(db.datasets.count())}})
 db.app.configuration.update({"key" : "countof.files"}, {$set: { "value": NumberLong(db.uploads.count())}})
 db.app.configuration.update({"key" : "countof.bytes"}, {$set: { "value": db.uploads.aggregate({$group: {_id: "bytes", "sum": {$sum : "$length"}}}).next().sum}})
 db.app.configuration.update({"key" : "countof.collections"}, {$set: { "value": NumberLong(db.collections.count())}})
 db.app.configuration.update({"key" : "countof.spaces"}, {$set: { "value": NumberLong(db.spaces.projects.count())}})
 
+db.spaces.projects.find().sort({"datasetCount": 1}).forEach(function(s) {
+	var spaceBytes = 0;
+	var filecount = 0;
+	db.datasets.find({"spaces": s._id}).forEach(function(d) {
+		d.files.forEach(function(fid) {
+			db.uploads.find({"_id": fid}).forEach(function(f) {
+				spaceBytes += f.length;
+				filecount += 1;
+			});
+		});
+	});
+	s.spaceBytes = NumberLong(spaceBytes);
+	s.fileCount = filecount;
+	db.spaces.projects.save(s);
+	print(s["_id"] + " " + s.name + " = " + s.spaceBytes + " in " + s.fileCount + " files");
+})

test/integration/spaces/SpaceMongoDBSpec.scala

@@ -24,6 +24,7 @@ class SpaceMongoDBSpec extends PlaySpec with OneServerPerSuite {
     bannerURL = None,
     collectionCount = 0,
     datasetCount = 0,
+    fileCount = 0,
     userCount = 0,
     spaceBytes = 0,
     metadata = List.empty