Update submodule during merge of branches/rudder/8.0 into master

datasources/README.adoc

@@ -172,4 +172,8 @@ You need to choose one behavior among:
 - Do not change the node property corresponding to that data source,
 - Set the node property corresponding to the data source to a configured value. You have access to a field to fill the value, where JSON is accepted. If the field is let empty, the node property is deleted (ie equivalent to first option).
 
+== Deleting a data source
 
+When a data source is deleted, the corresponding properties on nodes will also be deleted if and only if they were set by the data source (ie property
+name is data source `ID` and property `provider` is `datasources` plugin).
+If you don't want to delete properties linked to a data source, you should only disable it in place of deleting it. 

datasources/src/main/scala/com/normation/plugins/datasources/QueryService.scala

@@ -45,6 +45,8 @@ import com.normation.rudder.domain.nodes.NodeState
 import com.normation.rudder.domain.policies.GlobalPolicyMode
 import com.normation.rudder.domain.properties.CompareProperties
 import com.normation.rudder.domain.properties.GlobalParameter
+import com.normation.rudder.domain.properties.NodeProperty
+import com.normation.rudder.domain.properties.PropertyProvider
 import com.normation.rudder.repository.RoParameterRepository
 import com.normation.rudder.repository.WoNodeRepository
 import com.normation.rudder.services.nodes.NodeInfoService
@@ -83,6 +85,54 @@ trait QueryDataSourceService {
    * A version that only query one node - do not use if you want to query several nodes
    */
   def queryOne(datasource: DataSource, nodeId: NodeId, cause: UpdateCause): IOResult[NodeUpdateResult]
+
+  /*
+   * We can delete a datasource on all nodes
+   */
+  def deleteAll(datasource: DataSourceId, cause: UpdateCause): IOResult[Set[NodeUpdateResult]]
+}
+
+object QueryService {
+
+  /*
+   * Utility method that handle updating a node with given property value
+   * for given datasource
+   */
+  def updateNode(
+      nodeInfo:          NodeInfo,
+      repository:        WoNodeRepository,
+      optProperty:       Option[NodeProperty],
+      cause:             UpdateCause,
+      enforceSameOrigin: Boolean // for deletion, we want to be able to remove only prop set by datasource provider
+  ): IOResult[NodeUpdateResult] = {
+    optProperty match {
+      // on none, don't update anything, the life is wonderful (because 'none' means 'don't update')
+      case None           => NodeUpdateResult.Unchanged(nodeInfo.id).succeed
+      case Some(property) =>
+        // look for the property value in the node to know if an update is needed.
+        // If enforceSameOrigin is true, we only touch the property if it's default or datasource owned. 
+        nodeInfo.properties.find(_.name == property.name) match {
+          case Some(p) if (p.value == property.value) => NodeUpdateResult.Unchanged(nodeInfo.id).succeed
+          case Some(p)
+              if (enforceSameOrigin && p.provider.getOrElse(PropertyProvider.defaultPropertyProvider) != property.provider
+                .getOrElse(PropertyProvider.defaultPropertyProvider)) =>
+            NodeUpdateResult.Unchanged(nodeInfo.id).succeed
+          case _                                      =>
+            for {
+              newProps    <- CompareProperties.updateProperties(nodeInfo.properties, Some(property :: Nil)).toIO
+              newNode      = nodeInfo.node.copy(properties = newProps)
+              nodeUpdated <-
+                repository
+                  .updateNode(newNode, cause.modId, cause.actor, cause.reason)
+                  .chainError(
+                    s"Cannot save value for node '${nodeInfo.id.value}' for property '${property.name}'"
+                  )
+            } yield {
+              NodeUpdateResult.Updated(nodeUpdated.id)
+            }
+        }
+    }
+  }
 }
 
 /**
@@ -200,29 +250,7 @@ class HttpQueryDataSourceService(
                         datasource.requestTimeOut,
                         datasource.requestTimeOut
                       )
-      nodeResult   <- optProperty match {
-                        // on none, don't update anything, the life is wonderful (because 'none' means 'don't update')
-                        case None           => NodeUpdateResult.Unchanged(nodeInfo.id).succeed
-                        case Some(property) =>
-                          // look for the property value in the node to know if an update is needed.
-                          // we only care about value here (not provider or other meta-info)
-                          nodeInfo.properties.find(_.name == property.name).map(_.value) match {
-                            case Some(value) if (value == property.value) => NodeUpdateResult.Unchanged(nodeInfo.id).succeed
-                            case _                                        =>
-                              for {
-                                newProps    <- CompareProperties.updateProperties(nodeInfo.properties, Some(property :: Nil)).toIO
-                                newNode      = nodeInfo.node.copy(properties = newProps)
-                                nodeUpdated <-
-                                  nodeRepository
-                                    .updateNode(newNode, cause.modId, cause.actor, cause.reason)
-                                    .chainError(
-                                      s"Cannot save value for node '${nodeInfo.id.value}' for property '${property.name}'"
-                                    )
-                              } yield {
-                                NodeUpdateResult.Updated(nodeUpdated.id)
-                              }
-                          }
-                      }
+      nodeResult   <- QueryService.updateNode(nodeInfo, nodeRepository, optProperty, cause, enforceSameOrigin = false)
     } yield {
       nodeResult
     }).chainError(
@@ -321,6 +349,17 @@ class HttpQueryDataSourceService(
     }
   }
 
+  override def deleteAll(datasource: DataSourceId, cause: UpdateCause): IOResult[Set[NodeUpdateResult]] = {
+    import com.normation.rudder.domain.properties.GenericProperty._
+    val deleteProp = DataSource.nodeProperty(datasource.value, "".toConfigValue)
+    for {
+      nodes <- nodeInfo.getAll()
+      res   <- ZIO.foreach(nodes.values)(node =>
+                 QueryService.updateNode(node, nodeRepository, Some(deleteProp), cause, enforceSameOrigin = true)
+               )
+    } yield res.toSet
+  }
+
   def queryNodeByNode(
       datasourceId:     DataSourceId,
       datasource:       DataSourceType.HTTP,

datasources/src/main/scala/com/normation/plugins/datasources/Repository.scala

@@ -83,7 +83,7 @@ trait DataSourceRepository {
 
   def save(source: DataSource): IOResult[DataSource]
 
-  def delete(id: DataSourceId): IOResult[DataSourceId]
+  def delete(id: DataSourceId, cause: UpdateCause): IOResult[DataSourceId]
 }
 
 /*
@@ -141,7 +141,8 @@ class MemoryDataSourceRepository extends DataSourceRepository {
 
   def save(source: DataSource) = sourcesRef.update(sources => sources + ((source.id, source))) *> source.succeed
 
-  def delete(id: DataSourceId): IOResult[DataSourceId] = sourcesRef.update(sources => sources - (id)) *> id.succeed
+  def delete(id: DataSourceId, cause: UpdateCause): IOResult[DataSourceId] =
+    sourcesRef.update(sources => sources - (id)) *> id.succeed
 }
 
 /**
@@ -297,10 +298,16 @@ class DataSourceRepoImpl(
   /*
    * delete need to clean existing live resource
    */
-  override def delete(id: DataSourceId): IOResult[DataSourceId] = dataSourcesLock.withPermit {
+  override def delete(id: DataSourceId, cause: UpdateCause): IOResult[DataSourceId] = dataSourcesLock.withPermit {
     // start by cleaning
     datasources.delete(id) *>
-    backend.delete(id)
+    backend.delete(id, cause) *>
+    fetch.deleteAll(id, cause) *>
+    DataSourceLoggerPure
+      .info(
+        s"Datasource with id '${id.value}' was correctly deleted and relative node properties deleted. Cause: ${cause}"
+      )
+      .map(_ => id)
   }
 
   ///
@@ -484,7 +491,7 @@ class DataSourceJdbcRepository(
     }
   }
 
-  override def delete(sourceId: DataSourceId): IOResult[DataSourceId] = {
+  override def delete(sourceId: DataSourceId, cause: UpdateCause): IOResult[DataSourceId] = {
     val query = sql"""delete from datasources where id = ${sourceId}"""
     transactIOResult(s"Error when deleting datasource '${sourceId.value}'")(xa =>
       query.update.run.map(_ => sourceId).transact(xa)

datasources/src/main/scala/com/normation/plugins/datasources/api/DataSourceApi.scala

@@ -37,11 +37,10 @@
 
 package com.normation.plugins.datasources.api
 
+import com.normation.rudder.AuthorizationType
 import com.normation.rudder.api.HttpAction._
 import com.normation.rudder.rest._
 import com.normation.rudder.rest.EndpointSchema.syntax._
-import com.normation.rudder.AuthorizationType
-
 import sourcecode.Line
 
 sealed trait DataSourceApi extends EndpointSchema with GeneralApi with SortIndex
@@ -114,17 +113,17 @@ object DataSourceApi       extends ApiModuleProvider[DataSourceApi] {
     val description    = "Get the list of all defined datasources"
     val (action, path) = GET / "datasources"
 
-    override def dataContainer: Option[String] = None
-    override def authz: List[AuthorizationType] = List(AuthorizationType.Administration.Read)
+    override def dataContainer: Option[String]          = None
+    override def authz:         List[AuthorizationType] = List(AuthorizationType.Administration.Read)
   }
 
   final case object GetDataSource extends DataSourceApi with OneParam with StartsAtVersion9 with SortIndex {
     val z              = implicitly[Line].value
     val description    = "Get information about the given datasource"
     val (action, path) = GET / "datasources" / "{datasourceid}"
 
-    override def dataContainer: Option[String] = None
-    override def authz: List[AuthorizationType] = List(AuthorizationType.Administration.Read)
+    override def dataContainer: Option[String]          = None
+    override def authz:         List[AuthorizationType] = List(AuthorizationType.Administration.Read)
   }
 
   final case object DeleteDataSource extends DataSourceApi with OneParam with StartsAtVersion9 with SortIndex {

datasources/src/main/scala/com/normation/plugins/datasources/api/DataSourceApiImpl.scala

@@ -336,7 +336,14 @@ class DataSourceApiImpl(
     ): LiftResponse = {
 
       val res = for {
-        source <- dataSourceRepo.delete(DataSourceId(sourceId))
+        source <- dataSourceRepo.delete(
+                    DataSourceId(sourceId),
+                    UpdateCause(
+                      ModificationId(uuidGen.newUuid),
+                      authzToken.actor,
+                      Some(s"Deletion of datasource '${sourceId}' requested by API")
+                    )
+                  )
       } yield {
         JArray((("id" -> sourceId) ~ ("message" -> s"Data source ${sourceId} deleted")) :: Nil)
       }

datasources/src/test/scala/com/normation/plugins/datasources/UpdateHttpDatasetTest.scala

@@ -70,6 +70,7 @@ import com.normation.rudder.services.policies.NodeConfigData
 import com.normation.utils.StringUuidGeneratorImpl
 import com.normation.zio._
 import com.normation.zio.ZioRuntime
+import com.softwaremill.quicklens._
 import com.typesafe.config.ConfigFactory
 import com.typesafe.config.ConfigValue
 import net.liftweb.common._
@@ -277,17 +278,25 @@ object CmdbServer {
         Response.text("""{"foo":"bar"}""")
       }
 
-    case GET -> !! / "server"       =>
+    case GET -> !! / "server" =>
       counterSuccess.update(_ + 1) *>
       ZIO.succeed {
         Response.text("""{"hostname":"server.rudder.local"}""")
       }
+
     case GET -> !! / "hostnameJson" =>
       counterSuccess.update(_ + 1) *>
       ZIO.succeed {
         Response.text(hostnameJson)
       }
 
+    case GET -> !! / "lifecycle" / id =>
+      counterSuccess.update(_ + 1) *>
+      ZIO.succeed {
+        if (id == "node1") Response.status(Status.NotFound)
+        else Response.text("1")
+      }
+
     case GET -> !! / "404" =>
       ZIO.succeed(Response.status(Status.NotFound))
 
@@ -914,6 +923,63 @@ class UpdateHttpDatasetTest extends Specification with BoxSpecMatcher with Logga
     fiberRunning(r11) and (r12 must beEqualTo(Fiber.Status.Done)) and fiberRunning(r21)
   }
 
+  "a datasource creation, node update, deletion should create properties and then delete them" >> {
+    // set the variable by hand for node1 and node2. Node2 will have it overridden and then deleted, node1 kept (b/c 404 for datasources)
+    val id    = DataSourceId("test-ds-lifecycle")
+    val infos = new TestNodeRepoInfo(NodeConfigData.allNodesInfo.map {
+      case (NodeId("node1"), n) =>
+        (n.id, n.modify(_.node.properties).using(NodeProperty(id.value, "do not touch".toConfigValue, None, None) :: _))
+      case (NodeId("node2"), n) =>
+        (n.id, n.modify(_.node.properties).using(NodeProperty(id.value, "should be updated".toConfigValue, None, None) :: _))
+      case (k, v)               => (k, v)
+    })
+    val repos = new DataSourceRepoImpl(
+      new MemoryDataSourceRepository(),
+      new HttpQueryDataSourceService(
+        infos,
+        parameterRepo,
+        infos,
+        interpolation,
+        noPostHook,
+        () => alwaysEnforce.succeed
+      ),
+      MyDatasource.uuidGen,
+      AlwaysEnabledPluginStatus
+    )
+
+    def getProps(nodes: Map[NodeId, NodeInfo]) = {
+      nodes.collect {
+        case (k, n) => n.properties.find(_.name == id.value).map(p => (k.value, p.value.unwrapped()))
+      }.flatten.toMap
+    }
+
+    val datasource = NewDataSource(
+      name = id.value,
+      url = s"${REST_SERVER_URL}/lifecycle/$${rudder.node.id}", // this one does not set the prop for node1 else return 1
+      path = "$",
+      schedule = Scheduled(5.minute),
+      onMissing = MissingNodeBehavior.NoChange
+    )
+
+    val p0 = getProps(infos.getAll().runNow)
+
+    val (n1, n2) = (for {
+      _      <- repos.save(datasource) // init
+      _      <- repos.onUserAskUpdateAllNodesFor(actor, id)
+      // check nodes have the property now
+      nodes1 <- infos.getAll()
+      // now delete datasource
+      _      <- repos.delete(datasource.id, UpdateCause(ModificationId("test"), actor, None))
+      // property must be deleted now
+      nodes2 <- infos.getAll()
+    } yield (nodes1, nodes2))
+      .runTimeout(1.minute)
+
+    (p0 must containTheSameElementsAs(List("node1" -> "do not touch", "node2" -> "should be updated"))) and
+    (getProps(n1) must containTheSameElementsAs(List("root" -> "1", "node1" -> "do not touch", "node2" -> "1"))) and
+    (getProps(n2) must containTheSameElementsAs(List("node1" -> "do not touch")))
+  }
+
   "querying a lot of nodes" should {
 
     // test on 100 nodes. With 30s timeout, even on small hardware it will be ok.