Merge pull request #265 from clowder-framework/release/v1.18.1

release 1.18.1
clowder-framework · Aug 16, 2021 · 3e27a54 · 3e27a54
2 parents 13a9427 + 0204459
commit 3e27a54
Show file tree

Hide file tree

Showing 12 changed files with 1,508 additions and 744 deletions.
diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml
@@ -0,0 +1,28 @@
+name: swagger
+
+# This will run when:
+# - when new code is pushed to master/develop to make sure the
+#   code does compile.
+# - when a pull request is created and updated to make sure the
+#   code does compile.
+on:
+  push:
+    branches:
+      - master
+      - develop
+
+  pull_request:
+
+# Check to make sure swagger document is valid
+jobs:
+
+  # downloads all the dependencies and compiles the scala code
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: openapi-lint 
+        uses: mhiew/redoc-lint-github-action@v2
+        with:
+          args: 'public/swagger.yml --skip-rule operation-operationId'
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.18.1 - 2021-08-16
+
+This release fixes a critical issue where invalid zip files could result in the files not being uploaded correctly. To check to see if you are affected, please use the following query:
+
+```
+db.uploads.find({"status": "CREATED", "contentType": "application/x-zip-compressed"}, {"author.fullName": 1, "author.email": 1, "filename": 1, "uploadDate": 1, "length": 1})
+```
+
+If any files are returned, you should check to see if these files affected and are missing from clowder.
+
+### Fixed
+- When zip file is uploaded, it will parse the file to check if it is a valid zip file, this couuld result in files not stored in final storage space [#264](https://github.com/clowder-framework/clowder/issues/264)
+- Updated swagger documentation
+- Return 404 not found when calling file/dataset/space api endpoints with an invalid ID [#251](https://github.com/clowder-framework/clowder/issues/251)
+- Line breaks in welcome message breaks swagger build [#187](https://github.com/clowder-framework/clowder/issues/187)
+
+### Changed
+- Added more information when writing files to make sure files are written correctly
+- Made cilogon group check debug message instead of error message
+
 ## 1.18.0 - 2021-07-08
 
 ### Added

diff --git a/app/api/Datasets.scala b/app/api/Datasets.scala
@@ -61,7 +61,7 @@ class  Datasets @Inject()(
   def get(id: UUID) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
     datasets.get(id) match {
       case Some(d) => Ok(toJson(d))
-      case None => BadRequest(toJson(s"Could not find dataset with id [${id.stringify}]"))
+      case None => NotFound(toJson(s"Could not find dataset with id [${id.stringify}]"))
     }
   }
 
@@ -203,11 +203,11 @@ class  Datasets @Inject()(
             case Some(spaceId) =>
               spaces.get(UUID(spaceId)) match {
                 case Some(s) => d = Dataset(name=name,description=description, created=new Date(), author=identity, licenseData = License.fromAppConfig(), spaces = List(UUID(spaceId)), stats = new Statistics())
-                case None => BadRequest(toJson("Bad space = " + spaceId))
+                case None => NotFound(toJson("Bad space = " + spaceId))
               }
           }
         }
-        case None => InternalServerError("User Not found")
+        case None => NotFound("User Not found")
       }
       appConfig.incrementCount('datasets, 1)
 
@@ -248,7 +248,7 @@ class  Datasets @Inject()(
                 }
                 case None => Ok(toJson(Map("status" -> "error")))
               }
-            case None => BadRequest(toJson("Bad file_id = " + file_id))
+            case None => NotFound(toJson("Bad file_id = " + file_id))
 
           }
         }
@@ -556,7 +556,7 @@ class  Datasets @Inject()(
 
             }
           }
-          case None => InternalServerError("User Not found")
+          case None => NotFound("User Not found")
         }
         events.addObjectEvent(request.user, d.id, d.name, EventType.CREATE_DATASET.toString)
 
@@ -599,7 +599,7 @@ class  Datasets @Inject()(
                 }
                 case None => {
                   Logger.error("Error getting dataset" + id)
-                  BadRequest(toJson(s"The given dataset id $id is not a valid ObjectId."))
+                  NotFound(toJson(s"The given dataset id $id is not a valid ObjectId."))
                 }
               }
 
@@ -639,7 +639,7 @@ class  Datasets @Inject()(
             }
             case None => {
               Logger.error("Error getting dataset" + dsId)
-              BadRequest(toJson(s"The given dataset id $dsId is not a valid ObjectId."))
+              NotFound(toJson(s"The given dataset id $dsId is not a valid ObjectId."))
             }
           }
 
@@ -661,7 +661,7 @@ class  Datasets @Inject()(
       }
       case None => {
         Logger.error("Error getting dataset" + id)
-        BadRequest(toJson(s"The given dataset id $id is not a valid ObjectId."))
+        NotFound(toJson(s"The given dataset id $id is not a valid ObjectId."))
       }
     }
   }
@@ -717,13 +717,13 @@ class  Datasets @Inject()(
           }
           case None => {
             Logger.error("Error getting file" + fileId)
-            BadRequest(toJson(s"The given dataset id $dsId is not a valid ObjectId."))
+            NotFound(toJson(s"The given dataset id $dsId is not a valid ObjectId."))
           }
         }
       }
       case None => {
         Logger.error("Error getting dataset" + dsId)
-        BadRequest(toJson(s"The given dataset id $dsId is not a valid ObjectId."))
+        NotFound(toJson(s"The given dataset id $dsId is not a valid ObjectId."))
       }
     }
   }
@@ -810,19 +810,19 @@ class  Datasets @Inject()(
               }
               case None => {
                 Logger.error ("Error getting file" + fileId)
-                BadRequest (toJson (s"The given file id $fileId is not a valid ObjectId.") )
+                NotFound(toJson (s"The given file id $fileId is not a valid ObjectId.") )
               }
             }
           }
           case None => {
             Logger.error ("Error getting dataset" + toDatasetId)
-            BadRequest (toJson (s"The given dataset id $toDatasetId is not a valid ObjectId.") )
+            NotFound(toJson (s"The given dataset id $toDatasetId is not a valid ObjectId.") )
           }
         }
       }
       case None => {
         Logger.error ("Error getting dataset" + datasetId)
-        BadRequest (toJson (s"The given dataset id $datasetId is not a valid ObjectId.") )
+        NotFound(toJson (s"The given dataset id $datasetId is not a valid ObjectId.") )
       }
     }
   }
@@ -870,7 +870,7 @@ class  Datasets @Inject()(
         datasets.index(id)
         Ok(toJson(Map("status" -> "success")))
       }
-      case None => Logger.error(s"Error getting dataset $id"); NotFound
+      case None => Logger.error(s"Error getting dataset $id"); NotFound(toJson(s"Error getting dataset $id"))
     }
   }
 
@@ -923,7 +923,7 @@ class  Datasets @Inject()(
               }
             }
           }
-          case None => Logger.error(s"Error getting dataset $id"); NotFound
+          case None => Logger.error(s"Error getting dataset $id"); NotFound(toJson(s"Error getting dataset $id"))
         }
      }
 
@@ -956,7 +956,7 @@ class  Datasets @Inject()(
         }
         Ok(toJson(metadataDefinitions.toList.sortWith( _.json.\("label").asOpt[String].getOrElse("") < _.json.\("label").asOpt[String].getOrElse("") )))
       }
-      case None => BadRequest(toJson("The requested dataset does not exist"))
+      case None => NotFound(toJson("The requested dataset does not exist"))
     }
   }
 
@@ -975,7 +975,7 @@ class  Datasets @Inject()(
       }
       case None => {
         Logger.error("Error getting dataset  " + id);
-        BadRequest(toJson("Error getting dataset  " + id))
+        NotFound(toJson("Error getting dataset  " + id))
       }
     }
   }
@@ -1000,7 +1000,7 @@ class  Datasets @Inject()(
       }
       case None => {
         Logger.error("Error getting dataset  " + id)
-        BadRequest(toJson("Error getting dataset  " + id))
+        NotFound(toJson("Error getting dataset  " + id))
       }
     }
   }
@@ -1053,7 +1053,7 @@ class  Datasets @Inject()(
           Ok(toJson(list))
         }
       }
-      case None => Logger.error("Error getting dataset" + id); InternalServerError
+      case None => Logger.error("Error getting dataset" + id); NotFound(toJson("Error getting dataset" + id))
     }
   }
 
@@ -1090,7 +1090,7 @@ class  Datasets @Inject()(
         }
         Ok(toJson(list))
       }
-      case None => Logger.error("Error getting dataset" + id); InternalServerError
+      case None => Logger.error("Error getting dataset" + id); NotFound(toJson("Error getting dataset" + id))
     }
   }
 
@@ -1105,7 +1105,7 @@ class  Datasets @Inject()(
         }
       }
       case None => {
-        BadRequest(s"Dataset with id=${dataset_id} does not exist")
+        NotFound(s"Dataset with id=${dataset_id} does not exist")
       }
     }
   }
@@ -1121,7 +1121,7 @@ class  Datasets @Inject()(
         }
       }
       case None => {
-        BadRequest(s"Dataset with id=${dataset_id} does not exist")
+        NotFound(s"Dataset with id=${dataset_id} does not exist")
       }
     }
   }
@@ -1837,16 +1837,19 @@ class  Datasets @Inject()(
               case Some(dataset) => {
                 events.addSourceEvent(request.user, comment.id, comment.text , dataset.id, dataset.name, EventType.ADD_COMMENT_DATASET.toString)
               }
+              case None => {
+                NotFound(s"The given id $id is not a valid ObjectId.")
+              }
             }
             Ok(comment.id.toString())
           }
           case None => {
             Logger.error("no text specified.")
-            BadRequest
+            BadRequest("no text specified.")
           }
         }
       }
-      case None => BadRequest
+      case None => NotFound(toJson("User not found!"))
     }
   }
 
@@ -1916,7 +1919,7 @@ class  Datasets @Inject()(
         Ok(toJson(Map("isBeingProcessed" -> isActivity)))
       }
       case None => {
-        Logger.error(s"Error getting dataset $id"); InternalServerError
+        Logger.error(s"Error getting dataset $id"); NotFound(toJson(s"Error getting dataset $id"))
       }
     }
   }
@@ -1977,7 +1980,7 @@ class  Datasets @Inject()(
         Ok(jsonPreviewsFiles(previewslist.asInstanceOf[List[(models.File, List[(java.lang.String, String, String, String, java.lang.String, String, Long)])]]))
       }
       case None => {
-        Logger.error("Error getting dataset" + id); InternalServerError
+        Logger.error("Error getting dataset" + id); NotFound(toJson("Error getting dataset\" + id"))
       }
     }
   }
@@ -2051,7 +2054,7 @@ class  Datasets @Inject()(
           Ok(toJson(Map("status"->"success")))
         }
       }
-      case None => BadRequest("No dataset found with id " + id)
+      case None => NotFound("No dataset found with id " + id)
     }
   }
 
@@ -2066,10 +2069,10 @@ class  Datasets @Inject()(
 
             Ok(toJson(Map("status" -> "success")))
           }
-          case None => InternalServerError("Update Access failed")
+          case None => NotFound(s"Update Access failed. Dataset id $id not found.")
         }
       }
-      case None => BadRequest("No user supplied")
+      case None => NotFound("User not found")
     }
   }
 
@@ -2095,7 +2098,7 @@ class  Datasets @Inject()(
         val trashDatasets = datasets.listUserTrash(user,limit)
         Ok(toJson(trashDatasets))
       }
-      case None => BadRequest("No user supplied")
+      case None => NotFound("User not found")
     }
   }
 
@@ -2124,7 +2127,7 @@ class  Datasets @Inject()(
               .withHeaders(CONTENT_TYPE -> "application/rdf+xml")
               .withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(resultFile.getName(),request.headers.get("user-agent").getOrElse(""))))
           }
-          case None => BadRequest(toJson("Dataset not found " + id))
+          case None => NotFound(toJson("Dataset not found " + id))
         }
       }
       case _ => Ok("RDF export plugin not enabled")
@@ -2165,7 +2168,7 @@ class  Datasets @Inject()(
           case Some(listJson) => {
             Ok(listJson)
           }
-          case None => Logger.error(s"Error getting dataset $id"); InternalServerError
+          case None => Logger.error(s"Error getting dataset $id");NotFound(s"Error getting dataset $id")
         }
       }
       case false => {
@@ -2182,7 +2185,7 @@ class  Datasets @Inject()(
           .map(JSONLD.jsonMetadataWithContext(_) \ "content")
         Ok(toJson(listOfMetadata))
       }
-      case None => Logger.error("Error finding dataset" + id); InternalServerError
+      case None => Logger.error("Error finding dataset" + id); NotFound("Error finding dataset" + id)
     }
   }
 
@@ -2191,7 +2194,7 @@ class  Datasets @Inject()(
       case Some(dataset) => {
         Ok(datasets.getXMLMetadataJSON(id))
       }
-      case None => {Logger.error("Error finding dataset" + id); InternalServerError}
+      case None => {Logger.error("Error finding dataset" + id); NotFound("Error finding dataset" + id)}
     }
   }
 
@@ -2201,8 +2204,7 @@ class  Datasets @Inject()(
         Ok(datasets.getUserMetadataJSON(id))
       }
       case None => {
-        Logger.error("Error finding dataset" + id);
-        InternalServerError
+        Logger.error("Error finding dataset" + id);NotFound("Error finding dataset" + id)
       }
 
     }
@@ -2983,13 +2985,13 @@ class  Datasets @Inject()(
                     case None => BadRequest(s"Unable to copy the dataset with id $datasetId to space with id: $spaceId")
                   }
                 }
-                case None => BadRequest(s"No space found with id: + $spaceId.")
+                case None => NotFound(s"No space found with id: + $spaceId.")
               }
             } else {
               BadRequest("You don't have permission to copy the dataset.")
             }
           }
-          case None => BadRequest(s"No dataset  found with id: $datasetId")
+          case None => NotFound(s"No dataset  found with id: $datasetId")
         }
       }
       case None => BadRequest("You need to be logged in to copy a dataset to a space.")
@@ -3126,7 +3128,7 @@ class  Datasets @Inject()(
       }
       case None => {
         Logger.error("Error getting dataset " + id)
-        NotFound
+        NotFound("Error getting dataset " + id)
       }
     }
   }
@@ -3149,7 +3151,7 @@ class  Datasets @Inject()(
       }
       case None => {
         Logger.error("Error getting dataset " + id)
-        NotFound
+        NotFound("Error getting dataset " + id)
       }
     }
   }