feat(valiation): account for controls not evaluated by Lula

Makefile

@@ -21,9 +21,9 @@ PKG         := ./...
 UNIT_PKG    := $(shell go list ./... | grep -v 'e2e')
 TAGS        :=
 TESTS       := .
-TESTFLAGS   := -race -v
+TESTFLAGS   ?= -race -v
 LDFLAGS     := -w -s -X 'github.com/defenseunicorns/lula/src/config.CLIVersion=$(CLI_VERSION)'
-GOFLAGS     :=
+GOFLAGS     ?=
 CGO_ENABLED ?= 0
 FUZZTIME := 10s
 

src/pkg/common/requirement-store/requirement-store.go

@@ -96,19 +96,31 @@ func (r *RequirementStore) GenerateFindings(validationStore *validationstore.Val
 		}
 
 		// Using language from Assessment Results model for Target Objective Status State
-		var state string
+		var state, reason, remarks string
 		message.Debugf("Pass: %v / Fail: %v / Existing State: %s", pass, fail, finding.Target.Status.State)
 		if finding.Target.Status.State == "not-satisfied" {
 			state = "not-satisfied"
-		} else if pass > 0 && fail <= 0 {
+		} else if pass > 0 && fail == 0 {
 			state = "satisfied"
+			reason = "pass"
+		} else if pass == 0 && fail == 0 {
+			// If there is no result (pass or fail) it means that no validation was performed by Lula.
+			// When that happens we can explicitly add a note to the finding, to properly explain the
+			// reason for the control being not-satisfied
+			state = "not-satisfied"
+			reason = "other"
+			remarks = "No Lula validations were defined for this control"
+			finding.Remarks = remarks
 		} else {
 			state = "not-satisfied"
+			reason = "fail"
 		}
 
 		finding.Target = oscalTypes.FindingTarget{
 			Status: oscalTypes.ObjectiveStatus{
-				State: state,
+				State:   state,
+				Reason:  reason,
+				Remarks: remarks,
 			},
 			TargetId: requirement.ImplementedRequirement.ControlId,
 			Type:     "objective-id",

src/pkg/common/requirement-store/requirement-store_test.go

@@ -4,7 +4,17 @@ import (
 	"testing"
 
 	oscalTypes "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-3"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/defenseunicorns/lula/src/internal/testhelpers"
+	"github.com/defenseunicorns/lula/src/pkg/common/oscal"
 	requirementstore "github.com/defenseunicorns/lula/src/pkg/common/requirement-store"
+	validationstore "github.com/defenseunicorns/lula/src/pkg/common/validation-store"
+)
+
+const (
+	validCompDefMultiValidations = "../../../test/unit/common/oscal/valid-component-no-lula.yaml"
+	controlImplementationSource  = "https://github.com/defenseunicorns/lula"
 )
 
 func TestNewRequirementStore(t *testing.T) {
@@ -14,3 +24,24 @@ func TestNewRequirementStore(t *testing.T) {
 		t.Error("Expected a new RequirementStore, but got nil")
 	}
 }
+
+func TestGenerateFindings(t *testing.T) {
+	model := testhelpers.OscalFromPath(t, validCompDefMultiValidations)
+	vs := validationstore.NewValidationStoreFromBackMatter(*model.ComponentDefinition.BackMatter)
+	controlMap := oscal.FilterControlImplementations(model.ComponentDefinition)
+	impls := controlMap[controlImplementationSource]
+	rs := requirementstore.NewRequirementStore(&impls)
+
+	findings := rs.GenerateFindings(vs)
+
+	assert.Len(t, findings, 2)
+	assert.Empty(t, findings["ID-1"].Remarks)
+	assert.Equal(t, "not-satisfied", findings["ID-1"].Target.Status.State)
+	assert.Equal(t, "fail", findings["ID-1"].Target.Status.Reason)
+	assert.Empty(t, findings["ID-1"].Target.Status.Remarks)
+
+	assert.Equal(t, "No Lula validations were defined for this control", findings["ID-2"].Remarks)
+	assert.Equal(t, "not-satisfied", findings["ID-2"].Target.Status.State)
+	assert.Equal(t, "other", findings["ID-2"].Target.Status.Reason)
+	assert.Equal(t, "No Lula validations were defined for this control", findings["ID-2"].Target.Status.Remarks)
+}

src/test/unit/common/oscal/valid-component-no-lula.yaml

@@ -0,0 +1,115 @@
+# add the descriptions inline
+component-definition:
+  uuid: E6A291A4-2BC8-43A0-B4B2-FD67CAAE1F8F
+  metadata:
+    title: OSCAL Demo Tool
+    last-modified: "2022-09-13T12:00:00Z"
+    version: "20220913"
+    oscal-version: 1.1.1
+    parties:
+      # Should be consistent across all of the packages, but where is ground truth?
+      - uuid: C18F4A9F-A402-415B-8D13-B51739D689FF
+        type: organization
+        name: Defense Unicorns
+        links:
+          - href: https://github.com/defenseunicorns/lula
+            rel: website
+  components:
+    - uuid: A9D5204C-7E5B-4C43-BD49-34DF759B9F04
+      type: software
+      title: lula
+      description: |
+        Defense Unicorns lula
+      purpose: Validate compliance controls
+      responsible-roles:
+        - role-id: provider
+          party-uuids:
+            - C18F4A9F-A402-415B-8D13-B51739D689FF # matches parties entry for Defense Unicorns
+      control-implementations:
+        - uuid: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A
+          source: https://github.com/defenseunicorns/lula
+          description: Validate generic security requirements
+          implemented-requirements:
+            - uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD
+              control-id: ID-1
+              remarks: >-
+                Here are some remarks about this control.
+              description: >-
+                Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 
+                quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum 
+                dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+              links:
+                - href: "#88AB3470-B96B-4D7C-BC36-02BF9563C46C"
+                  rel: lula
+            - uuid: EB61471D-979F-4CA2-BAC4-DF10AB035405
+              control-id: ID-2
+              remarks: >-
+                Here are some remarks about this control.
+              description: >-
+                Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 
+                quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum 
+                dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+  back-matter:
+    resources:
+      - uuid: 88AB3470-B96B-4D7C-BC36-02BF9563C46C
+        remarks: >-
+          Get data for all resources fields specified
+        description: >-
+          metadata:
+            name: Validate pods with label foo=bar
+            uuid: 88AB3470-B96B-4D7C-BC36-02BF9563C46C
+          domain:
+            type: kubernetes
+            kubernetes-spec:
+              resources:
+                - name: jsoncm
+                  resource-rule:
+                    name: configmap-json
+                    version: v1
+                    resource: configmaps
+                    namespaces: [validation-test]
+                    field:
+                      jsonpath: .data.person.json
+                      type: yaml
+                - name: yamlcm
+                  resource-rule:
+                    name: configmap-yaml
+                    version: v1
+                    resource: configmaps
+                    namespaces: [validation-test]
+                    field:
+                      jsonpath: .data.app-config.yaml
+                      type: yaml
+                - name: secret
+                  resource-rule:
+                    name: example-secret
+                    version: v1
+                    resource: secrets
+                    namespaces: [validation-test]
+                    field:
+                      jsonpath: .data.auth
+                      type: yaml
+                      base64: true
+                - name: pod
+                  resource-rule:
+                    name: example-pod
+                    version: v1
+                    resource: pods
+                    namespaces: [validation-test]
+                    field:
+                      jsonpath: .metadata.annotations.annotation.io/simple
+                      type: json
+          provider:
+            type: opa
+            opa-spec:
+              rego: |
+                package validate
+
+                import future.keywords.every
+
+                validate {
+                  input.jsoncm.name == "bob"
+                  input.yamlcm.logging.level == "INFO"
+                  input.secret.username == "username"
+                  "item1" in input.pod.items
+                }