aws.securityhub_findings: Improve support for CDR (#11158)

* Add CSPM fields - 1

* reformat

* reformat

* Add more ECS fields

* Consider multiple resources

* Split single and multiple resource logic. Add multiple resources test.

* Add tags and update comments

* Add visualization to findings dashboard

* update typeMigrationVersion on kibana searches

* Address PR comments.

* Address PR comments-1

* Add PR comment-2
 - Add host.ip and host.name

* Address PR comments-3. Use constant_keyword

* Address PR comments-4. Separate res.Details != null condition block and field separation.

* fix HEAD

* ecs fields sorted.

* Address Pr comments-5. Remove unused fields from mapping.

* Add misconfiguration_latest transform

* Address PR comment. Update transform retention to 90d.

* Address PR comments. Updated rule fields to keyword.

* update readme

* Remove references from pipeline tests

* update fields to ecs

* address pr comments.

* fix static test

* update/fix readme

* address pr comments

* address pr comments. Remove unnecessary `ignore_empty_value` option

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.31.0"
+  changes:
+    - description: Improve support for Cloud Detection and Response (CDR) workflows in securityhub_findings data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11158
 - version: "2.30.1"
   changes:
     - description: Update the AWS dashboard panels.

packages/aws/data_stream/securityhub_findings/fields/base-fields.yml

@@ -1,16 +1,16 @@
 - name: data_stream.type
-  type: constant_keyword
-  description: Data stream type.
+  external: ecs
 - name: data_stream.dataset
-  type: constant_keyword
-  description: Data stream dataset.
+  external: ecs
 - name: data_stream.namespace
-  type: constant_keyword
-  description: Data stream namespace.
+  external: ecs
 - name: event.module
+  external: ecs
   type: constant_keyword
-  description: Event module.
   value: aws
+- name: event.dataset
+  external: ecs
+  type: constant_keyword
+  value: aws.securityhub_findings
 - name: '@timestamp'
-  type: date
-  description: Event timestamp.
+  external: ecs

packages/aws/data_stream/securityhub_findings/fields/ecs.yml

@@ -0,0 +1,10 @@
+# Define ECS constant fields as constant_keyword
+- name: cloud.provider
+  type: constant_keyword
+  value: aws
+- name: event.kind
+  type: constant_keyword
+  value: state
+- name: observer.vendor
+  type: constant_keyword
+  value: AWS Security Hub

packages/aws/data_stream/securityhub_findings/fields/fields.yml

@@ -271,6 +271,9 @@
     - name: compliance
       type: group
       fields:
+        - name: security_control_id
+          type: keyword
+          description: Unique identifier of a control across standards.
         - name: related_requirements
           type: keyword
           description: For a control, the industry or regulatory framework requirements that are related to the control.
@@ -289,6 +292,9 @@
     - name: confidence
       type: long
       description: A finding's confidence. Confidence is defined as the likelihood that a finding accurately identifies the behavior or issue that it was intended to identify.
+    - name: processed_at
+      type: date
+      description: Indicates when AWS Security Hub received a finding and begins to process it.
     - name: created_at
       type: date
       description: Indicates when the security-findings provider created the potential security issue that a finding captured.

packages/aws/data_stream/securityhub_findings/fields/resource.yml

@@ -0,0 +1,9 @@
+- name: resource
+  type: group
+  fields:
+    - name: id
+      type: keyword
+    - name: name
+      type: keyword
+    - name: type
+      type: keyword

packages/aws/data_stream/securityhub_findings/fields/result.yml

@@ -0,0 +1,5 @@
+- name: result
+  type: group
+  fields:
+    - name: evaluation
+      type: keyword

packages/aws/data_stream/securityhub_findings/fields/rule.yml

@@ -0,0 +1,5 @@
+- name: rule
+  type: group
+  fields:
+    - name: remediation
+      type: keyword

packages/aws/data_stream/securityhub_findings/sample_event.json

@@ -352,7 +352,7 @@
         "dataset": "aws.securityhub_findings",
         "id": "us-west-2/111111111111/98aebb2207407c87f51e89943f12b1ef",
         "ingested": "2022-07-27T12:47:45Z",
-        "kind": "event",
+        "kind": "state",
         "original": "{\"Action\":{\"ActionType\":\"PORT_PROBE\",\"PortProbeAction\":{\"Blocked\":false,\"PortProbeDetails\":[{\"LocalIpDetails\":{\"IpAddressV4\":\"1.128.0.0\"},\"LocalPortDetails\":{\"Port\":80,\"PortName\":\"HTTP\"},\"RemoteIpDetails\":{\"City\":{\"CityName\":\"Example City\"},\"Country\":{\"CountryName\":\"Example Country\"},\"GeoLocation\":{\"Lat\":0,\"Lon\":0},\"Organization\":{\"Asn\":64496,\"AsnOrg\":\"ExampleASO\",\"Isp\":\"ExampleISP\",\"Org\":\"ExampleOrg\"}}}]}},\"AwsAccountId\":\"111111111111\",\"CompanyName\":\"AWS\",\"Compliance\":{\"RelatedRequirements\":[\"Req1\",\"Req2\"],\"Status\":\"PASSED\",\"StatusReasons\":[{\"Description\":\"CloudWatch alarms do not exist in the account\",\"ReasonCode\":\"CLOUDWATCH_ALARMS_NOT_PRESENT\"}]},\"Confidence\":42,\"CreatedAt\":\"2017-03-22T13:22:13.933Z\",\"Criticality\":99,\"Description\":\"The version of openssl found on instance i-abcd1234 is known to contain a vulnerability.\",\"FindingProviderFields\":{\"Confidence\":42,\"Criticality\":99,\"RelatedFindings\":[{\"Id\":\"123e4567-e89b-12d3-a456-426655440000\",\"ProductArn\":\"arn:aws:securityhub:us-west-2::product/aws/guardduty\"}],\"Severity\":{\"Label\":\"MEDIUM\",\"Original\":\"MEDIUM\"},\"Types\":[\"Software and Configuration Checks/Vulnerabilities/CVE\"]},\"FirstObservedAt\":\"2017-03-22T13:22:13.933Z\",\"GeneratorId\":\"acme-vuln-9ab348\",\"Id\":\"us-west-2/111111111111/98aebb2207407c87f51e89943f12b1ef\",\"LastObservedAt\":\"2017-03-23T13:22:13.933Z\",\"Malware\":[{\"Name\":\"Stringler\",\"Path\":\"/usr/sbin/stringler\",\"State\":\"OBSERVED\",\"Type\":\"COIN_MINER\"}],\"Network\":{\"DestinationDomain\":\"example2.com\",\"DestinationIpV4\":\"1.128.0.0\",\"DestinationIpV6\":\"2a02:cf40::\",\"DestinationPort\":\"80\",\"Direction\":\"IN\",\"OpenPortRange\":{\"Begin\":443,\"End\":443},\"Protocol\":\"TCP\",\"SourceDomain\":\"example1.com\",\"SourceIpV4\":\"1.128.0.0\",\"SourceIpV6\":\"2a02:cf40::\",\"SourceMac\":\"00:0d:83:b1:c0:8e\",\"SourcePort\":\"42\"},\"NetworkPath\":[{\"ComponentId\":\"abc-01a234bc56d8901ee\",\"ComponentType\":\"AWS::EC2::InternetGateway\",\"Egress\":{\"Destination\":{\"Address\":[\"1.128.0.0/24\"],\"PortRanges\":[{\"Begin\":443,\"End\":443}]},\"Protocol\":\"TCP\",\"Source\":{\"Address\":[\"175.16.199.1/24\"]}},\"Ingress\":{\"Destination\":{\"Address\":[\"175.16.199.1/24\"],\"PortRanges\":[{\"Begin\":443,\"End\":443}]},\"Protocol\":\"TCP\",\"Source\":{\"Address\":[\"175.16.199.1/24\"]}}}],\"Note\":{\"Text\":\"Don't forget to check under the mat.\",\"UpdatedAt\":\"2018-08-31T00:15:09Z\",\"UpdatedBy\":\"jsmith\"},\"PatchSummary\":{\"FailedCount\":\"0\",\"Id\":\"pb-123456789098\",\"InstalledCount\":\"100\",\"InstalledOtherCount\":\"1023\",\"InstalledPendingReboot\":\"0\",\"InstalledRejectedCount\":\"0\",\"MissingCount\":\"100\",\"Operation\":\"Install\",\"OperationEndTime\":\"2018-09-27T23:39:31Z\",\"OperationStartTime\":\"2018-09-27T23:37:31Z\",\"RebootOption\":\"RebootIfNeeded\"},\"Process\":{\"LaunchedAt\":\"2018-09-27T22:37:31Z\",\"Name\":\"syslogd\",\"ParentPid\":56789,\"Path\":\"/usr/sbin/syslogd\",\"Pid\":12345,\"TerminatedAt\":\"2018-09-27T23:37:31Z\"},\"ProductArn\":\"arn:aws:securityhub:us-east-1:111111111111:product/111111111111/default\",\"ProductFields\":{\"Service_Name\":\"cloudtrail.amazonaws.com\",\"aws/inspector/AssessmentTargetName\":\"My prod env\",\"aws/inspector/AssessmentTemplateName\":\"My daily CVE assessment\",\"aws/inspector/RulesPackageName\":\"Common Vulnerabilities and Exposures\",\"generico/secure-pro/Count\":\"6\"},\"ProductName\":\"Security Hub\",\"RecordState\":\"ACTIVE\",\"Region\":\"us-east-1\",\"RelatedFindings\":[{\"Id\":\"123e4567-e89b-12d3-a456-426655440000\",\"ProductArn\":\"arn:aws:securityhub:us-west-2::product/aws/guardduty\"},{\"Id\":\"AcmeNerfHerder-111111111111-x189dx7824\",\"ProductArn\":\"arn:aws:securityhub:us-west-2::product/aws/guardduty\"}],\"Remediation\":{\"Recommendation\":{\"Text\":\"Run sudo yum update and cross your fingers and toes.\",\"Url\":\"http://myfp.com/recommendations/dangerous_things_and_how_to_fix_them.html\"}},\"Resources\":[{\"Details\":{\"IamInstanceProfileArn\":\"arn:aws:iam::123456789012:role/IamInstanceProfileArn\",\"ImageId\":\"ami-79fd7eee\",\"IpV4Addresses\":[\"175.16.199.1\"],\"IpV6Addresses\":[\"2a02:cf40::\"],\"KeyName\":\"testkey\",\"LaunchedAt\":\"2018-09-29T01:25:54Z\",\"MetadataOptions\":{\"HttpEndpoint\":\"enabled\",\"HttpProtocolIpv6\":\"enabled\",\"HttpPutResponseHopLimit\":1,\"HttpTokens\":\"optional\",\"InstanceMetadataTags\":\"disabled\"},\"NetworkInterfaces\":[{\"NetworkInterfaceId\":\"eni-e5aa89a3\"}],\"SubnetId\":\"PublicSubnet\",\"Type\":\"i3.xlarge\",\"VirtualizationType\":\"hvm\",\"VpcId\":\"TestVPCIpv6\"},\"Id\":\"i-cafebabe\",\"Partition\":\"aws\",\"Region\":\"us-west-2\",\"Tags\":{\"billingCode\":\"Lotus-1-2-3\",\"needsPatching\":\"true\"},\"Type\":\"AwsEc2Instance\"}],\"Sample\":true,\"SchemaVersion\":\"2018-10-08\",\"Severity\":{\"Label\":\"CRITICAL\",\"Original\":\"8.3\"},\"SourceUrl\":\"http://threatintelweekly.org/backdoors/8888\",\"ThreatIntelIndicators\":[{\"Category\":\"BACKDOOR\",\"LastObservedAt\":\"2018-09-27T23:37:31Z\",\"Source\":\"Threat Intel Weekly\",\"SourceUrl\":\"http://threatintelweekly.org/backdoors/8888\",\"Type\":\"IPV4_ADDRESS\",\"Value\":\"175.16.199.1\"}],\"Threats\":[{\"FilePaths\":[{\"FileName\":\"b.txt\",\"FilePath\":\"/tmp/b.txt\",\"Hash\":\"sha256\",\"ResourceId\":\"arn:aws:ec2:us-west-2:123456789012:volume/vol-032f3bdd89aee112f\"}],\"ItemCount\":3,\"Name\":\"Iot.linux.mirai.vwisi\",\"Severity\":\"HIGH\"}],\"Title\":\"EC2.20 Both VPN tunnels for an AWS Site-to-Site VPN connection should be up\",\"Types\":[\"Software and Configuration Checks/Vulnerabilities/CVE\"],\"UpdatedAt\":\"2018-08-31T00:15:09Z\",\"UserDefinedFields\":{\"comeBackToLater\":\"Check this again on Monday\",\"reviewedByCio\":\"true\"},\"VerificationState\":\"UNKNOWN\",\"Vulnerabilities\":[{\"Cvss\":[{\"BaseScore\":4.7,\"BaseVector\":\"AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N\",\"Version\":\"V3\"},{\"BaseScore\":4.7,\"BaseVector\":\"AV:L/AC:M/Au:N/C:C/I:N/A:N\",\"Version\":\"V2\"}],\"Id\":\"CVE-2020-12345\",\"ReferenceUrls\":[\"http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12418\",\"http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-17563\"],\"RelatedVulnerabilities\":[\"CVE-2020-12345\"],\"Vendor\":{\"Name\":\"Alas\",\"Url\":\"https://alas.aws.amazon.com/ALAS-2020-1337.html\",\"VendorCreatedAt\":\"2020-01-16T00:01:43Z\",\"VendorSeverity\":\"Medium\",\"VendorUpdatedAt\":\"2020-01-16T00:01:43Z\"},\"VulnerablePackages\":[{\"Architecture\":\"x86_64\",\"Epoch\":\"1\",\"Name\":\"openssl\",\"Release\":\"16.amzn2.0.3\",\"Version\":\"1.0.2k\"}]}],\"Workflow\":{\"Status\":\"NEW\"},\"WorkflowState\":\"NEW\"}",
         "type": [
             "info"