From 1c07c842435a04ad6022877bf679093314ee09b1 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 10 Oct 2024 07:32:57 +1030 Subject: [PATCH] proofpoint_tap: improve clarity of agent config and fix pagination logic (#11361) There was no check for the pagination termination state, which for this API is an empty array of events. Add that check after clarifying the HTTPJSON configuration, including ensuring that all time ranges are valid queries. Extend system test conditions to exercise pagination logic and termination, and fix configuration of request tracing in system tests. --- .../_dev/deploy/docker/files/config.yml | 511 +++++++++++++++++- packages/proofpoint_tap/changelog.yml | 8 + .../_dev/test/system/test-default-config.yml | 4 +- .../agent/stream/httpjson.yml.hbs | 47 +- .../_dev/test/system/test-default-config.yml | 4 +- .../agent/stream/httpjson.yml.hbs | 47 +- .../_dev/test/system/test-default-config.yml | 4 +- .../agent/stream/httpjson.yml.hbs | 47 +- .../_dev/test/system/test-default-config.yml | 4 +- .../agent/stream/httpjson.yml.hbs | 47 +- packages/proofpoint_tap/manifest.yml | 2 +- 11 files changed, 700 insertions(+), 25 deletions(-) diff --git a/packages/proofpoint_tap/_dev/deploy/docker/files/config.yml b/packages/proofpoint_tap/_dev/deploy/docker/files/config.yml index 7dca7d9b806..c870e58e086 100644 --- a/packages/proofpoint_tap/_dev/deploy/docker/files/config.yml +++ b/packages/proofpoint_tap/_dev/deploy/docker/files/config.yml @@ -1,25 +1,520 @@ rules: - path: /v2/siem/messages/blocked methods: [GET] + query_params: + interval: "{interval:20[2-6][0-9].*}" responses: - status_code: 200 - body: | - {"queryEndTime":"2022-03-30T13:00:00Z","messagesBlocked":[{"GUID":"x11xxxx1-12f9-111x-x12x-1x1x123456xx","QID":"x2XXxXXX111111","ccAddresses":["abc@example.com"],"clusterId":"pharmtech_hosted","completelyRewritten":"true","fromAddress":"abc@example.com","headerCC":"\"Example Abc\" ","headerFrom":"\"A. Bc\" ","headerReplyTo":null,"headerTo":"\"Aa Bb\" ; \"Hey Hello\" ","impostorScore":0,"malwareScore":100,"messageID":"12345678912345.12345.mail@example.com","messageParts":[{"contentType":"text/plain","disposition":"inline","filename":"text.txt","md5":"b10a8db164e0754105b7a99be72e3fe5","oContentType":"text/plain","sandboxStatus":"unsupported","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e"},{"contentType":"application/pdf","disposition":"attached","filename":"text.pdf","md5":"b10a8db164e0754105b7a99be72e3fe5","oContentType":"application/pdf","sandboxStatus":"threat","sha256":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e"}],"messageTime":"2021-11-25T09:10:00.050Z","modulesRun":["pdr","sandbox","spam","urldefense"],"phishScore":46,"policyRoutes":["default_inbound","executives"],"quarantineFolder":"Attachment Defense","quarantineRule":"module.sandbox.threat","recipient":["example.abc@example.com","hey.hello@example.com"],"replyToAddress":null,"sender":"x99x7x5580193x6x51x597xx2x0210@example.com","senderIP":"175.16.199.1","spamScore":4,"subject":"Please find a totally safe invoice attached.","threatsInfoMap":[{"campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","classification":"MALWARE","threat":"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e","threatId":"2xxx740f143fc1aa4c1cd0146d334x5593b1428x6x062b2c406e5efe8xxx95xx","threatStatus":"active","threatTime":"2021-11-25T09:10:00.050Z","threatType":"ATTACHMENT","threatUrl":"https://www.example.com/?name=john"},{"campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","classification":"MALWARE","threat":"example.com","threatId":"3xx97xx852c66a7xx761450xxxxxx9f4ffab74715b591294f78b5e37a76481xx","threatTime":"2021-07-20T05:00:00.050Z","threatType":"URL","threatUrl":"https://www.example.com/?name=john"}],"toAddresses":["example.abc@example.com","hey.hello@example.com"],"xmailer":"Spambot v2.5"}]} + body: |- + {{ minify_json ` + { + "queryEndTime": "2072-03-30T13:00:00Z", + "messagesBlocked": [ + { + "GUID": "x11xxxx1-12f9-111x-x12x-1x1x123456xx", + "QID": "x2XXxXXX111111", + "ccAddresses": [ + "abc@example.com" + ], + "clusterId": "pharmtech_hosted", + "completelyRewritten": "true", + "fromAddress": "abc@example.com", + "headerCC": "\"Example Abc\" ", + "headerFrom": "\"A. Bc\" ", + "headerReplyTo": null, + "headerTo": "\"Aa Bb\" ; \"Hey Hello\" ", + "impostorScore": 0, + "malwareScore": 100, + "messageID": "12345678912345.12345.mail@example.com", + "messageParts": [ + { + "contentType": "text/plain", + "disposition": "inline", + "filename": "text.txt", + "md5": "b10a8db164e0754105b7a99be72e3fe5", + "oContentType": "text/plain", + "sandboxStatus": "unsupported", + "sha256": "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e" + }, + { + "contentType": "application/pdf", + "disposition": "attached", + "filename": "text.pdf", + "md5": "b10a8db164e0754105b7a99be72e3fe5", + "oContentType": "application/pdf", + "sandboxStatus": "threat", + "sha256": "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e" + } + ], + "messageTime": "2021-11-25T09:10:00.050Z", + "modulesRun": [ + "pdr", + "sandbox", + "spam", + "urldefense" + ], + "phishScore": 46, + "policyRoutes": [ + "default_inbound", + "executives" + ], + "quarantineFolder": "Attachment Defense", + "quarantineRule": "module.sandbox.threat", + "recipient": [ + "example.abc@example.com", + "hey.hello@example.com" + ], + "replyToAddress": null, + "sender": "x99x7x5580193x6x51x597xx2x0210@example.com", + "senderIP": "175.16.199.1", + "spamScore": 4, + "subject": "Please find a totally safe invoice attached.", + "threatsInfoMap": [ + { + "campaignId": "46x01x8x-x899-404x-xxx9-111xx393d1x7", + "classification": "MALWARE", + "threat": "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e", + "threatId": "2xxx740f143fc1aa4c1cd0146d334x5593b1428x6x062b2c406e5efe8xxx95xx", + "threatStatus": "active", + "threatTime": "2021-11-25T09:10:00.050Z", + "threatType": "ATTACHMENT", + "threatUrl": "https://www.example.com/?name=john" + }, + { + "campaignId": "46x01x8x-x899-404x-xxx9-111xx393d1x7", + "classification": "MALWARE", + "threat": "example.com", + "threatId": "3xx97xx852c66a7xx761450xxxxxx9f4ffab74715b591294f78b5e37a76481xx", + "threatTime": "2021-07-20T05:00:00.050Z", + "threatType": "URL", + "threatUrl": "https://www.example.com/?name=john" + } + ], + "toAddresses": [ + "example.abc@example.com", + "hey.hello@example.com" + ], + "xmailer": "Spambot v2.5" + } + ] + } + `}} + - path: /v2/siem/messages/blocked + methods: [GET] + query_params: + interval: "{interval:2072.*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "queryEndTime": "2082-03-30T13:00:00Z", + "messagesBlocked": [ + { + "GUID": "x11xxxx1-12f9-111x-x12x-1x1x123456xx", + "QID": "x2XXxXXX111111", + "ccAddresses": [ + "abc@example.com" + ], + "clusterId": "pharmtech_hosted", + "completelyRewritten": "true", + "fromAddress": "abc@example.com", + "headerCC": "\"Example Abc\" ", + "headerFrom": "\"A. Bc\" ", + "headerReplyTo": null, + "headerTo": "\"Aa Bb\" ; \"Hey Hello\" ", + "impostorScore": 0, + "malwareScore": 100, + "messageID": "12345678912345.12345.mail@example.com", + "messageParts": [ + { + "contentType": "text/plain", + "disposition": "inline", + "filename": "text.txt", + "md5": "b10a8db164e0754105b7a99be72e3fe5", + "oContentType": "text/plain", + "sandboxStatus": "unsupported", + "sha256": "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e" + }, + { + "contentType": "application/pdf", + "disposition": "attached", + "filename": "text.pdf", + "md5": "b10a8db164e0754105b7a99be72e3fe5", + "oContentType": "application/pdf", + "sandboxStatus": "threat", + "sha256": "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e" + } + ], + "messageTime": "2071-11-25T09:10:00.050Z", + "modulesRun": [ + "pdr", + "sandbox", + "spam", + "urldefense" + ], + "phishScore": 46, + "policyRoutes": [ + "default_inbound", + "executives" + ], + "quarantineFolder": "Attachment Defense", + "quarantineRule": "module.sandbox.threat", + "recipient": [ + "example.abc@example.com", + "hey.hello@example.com" + ], + "replyToAddress": null, + "sender": "x99x7x5580193x6x51x597xx2x0210@example.com", + "senderIP": "175.16.199.1", + "spamScore": 4, + "subject": "Please find a totally safe invoice attached.", + "threatsInfoMap": [ + { + "campaignId": "46x01x8x-x899-404x-xxx9-111xx393d1x7", + "classification": "MALWARE", + "threat": "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e", + "threatId": "2xxx740f143fc1aa4c1cd0146d334x5593b1428x6x062b2c406e5efe8xxx95xx", + "threatStatus": "active", + "threatTime": "2021-11-25T09:10:00.050Z", + "threatType": "ATTACHMENT", + "threatUrl": "https://www.example.com/?name=john" + }, + { + "campaignId": "46x01x8x-x899-404x-xxx9-111xx393d1x7", + "classification": "MALWARE", + "threat": "example.com", + "threatId": "3xx97xx852c66a7xx761450xxxxxx9f4ffab74715b591294f78b5e37a76481xx", + "threatTime": "2021-07-20T05:00:00.050Z", + "threatType": "URL", + "threatUrl": "https://www.example.com/?name=john" + } + ], + "toAddresses": [ + "example.abc@example.com", + "hey.hello@example.com" + ], + "xmailer": "Spambot v2.5" + } + ] + } + `}} + - path: /v2/siem/messages/blocked + methods: [GET] + query_params: + interval: "{interval:2082.*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "queryEndTime": "2082-03-30T13:00:00Z", + "messagesBlocked": [] + } + `}} + + - path: /v2/siem/messages/delivered + methods: [GET] + query_params: + interval: "{interval:20[2-6][0-9].*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "queryEndTime": "2072-03-29T20:00:00Z", + "messagesDelivered": [ + { + "spamScore": 0, + "phishScore": 0, + "threatsInfoMap": [ + { + "threatID": "b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb", + "threatStatus": "active", + "classification": "spam", + "threatUrl": "https://threatinsight.proofpoint.com/aaabcdef-1234-b1abcdefghe/threat/email/b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb", + "threatTime": "2021-11-25T13:02:58.640Z", + "threat": "http://zbcd123456x0.example.com", + "campaignID": null, + "threatType": "url" + }, + { + "threatID": "aaabcdefg123456f009971a9c193abcdefg123456bf5abcdefg1234566", + "threatStatus": "active", + "classification": "phish", + "threatUrl": "https://threatinsight.proofpoint.com/aaabcdef-1234-b1abcdefghe/threat/email/b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb", + "threatTime": "2021-07-19T10:28:15.100Z", + "threat": "http://zbcd123456x0.example.com", + "campaignID": null, + "threatType": "url" + } + ], + "messageTime": "2022-01-01T00:00:00.000Z", + "impostorScore": 0, + "malwareScore": 0, + "cluster": "pharmtech_hosted", + "subject": null, + "quarantineFolder": null, + "quarantineRule": null, + "policyRoutes": null, + "modulesRun": null, + "messageSize": 0, + "headerFrom": null, + "headerReplyTo": null, + "fromAddress": null, + "ccAddresses": null, + "replyToAddress": null, + "toAddresses": null, + "xmailer": null, + "messageParts": null, + "completelyRewritten": true, + "id": "2hsvbU-i8abc123-12345-xxxxx12", + "QID": null, + "GUID": "NxxxsxvxbxUxixcx2xxxxx5x6xWxBxOxxxxxjxx", + "sender": "", + "recipient": [ + "fxxxxhxsxxvxbcx2xx5xxx6x3xx26@example.com" + ], + "senderIP": "89.160.20.112", + "messageID": "" + } + ] + } + `}} + - path: /v2/siem/messages/delivered + methods: [GET] + query_params: + interval: "{interval:2072.*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "queryEndTime": "2082-03-29T20:00:00Z", + "messagesDelivered": [ + { + "spamScore": 0, + "phishScore": 0, + "threatsInfoMap": [ + { + "threatID": "b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb", + "threatStatus": "active", + "classification": "spam", + "threatUrl": "https://threatinsight.proofpoint.com/aaabcdef-1234-b1abcdefghe/threat/email/b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb", + "threatTime": "2021-11-25T13:02:58.640Z", + "threat": "http://zbcd123456x0.example.com", + "campaignID": null, + "threatType": "url" + }, + { + "threatID": "aaabcdefg123456f009971a9c193abcdefg123456bf5abcdefg1234566", + "threatStatus": "active", + "classification": "phish", + "threatUrl": "https://threatinsight.proofpoint.com/aaabcdef-1234-b1abcdefghe/threat/email/b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb", + "threatTime": "2021-07-19T10:28:15.100Z", + "threat": "http://zbcd123456x0.example.com", + "campaignID": null, + "threatType": "url" + } + ], + "messageTime": "2072-01-01T00:00:00.000Z", + "impostorScore": 0, + "malwareScore": 0, + "cluster": "pharmtech_hosted", + "subject": null, + "quarantineFolder": null, + "quarantineRule": null, + "policyRoutes": null, + "modulesRun": null, + "messageSize": 0, + "headerFrom": null, + "headerReplyTo": null, + "fromAddress": null, + "ccAddresses": null, + "replyToAddress": null, + "toAddresses": null, + "xmailer": null, + "messageParts": null, + "completelyRewritten": true, + "id": "2hsvbU-i8abc123-12345-xxxxx12", + "QID": null, + "GUID": "NxxxsxvxbxUxixcx2xxxxx5x6xWxBxOxxxxxjxx", + "sender": "", + "recipient": [ + "fxxxxhxsxxvxbcx2xx5xxx6x3xx26@example.com" + ], + "senderIP": "89.160.20.112", + "messageID": "" + } + ] + } + `}} - path: /v2/siem/messages/delivered methods: [GET] + query_params: + interval: "{interval:2082.*}" responses: - status_code: 200 - body: | - {"queryEndTime":"2022-03-29T20:00:00Z","messagesDelivered":[{"spamScore":0,"phishScore":0,"threatsInfoMap":[{"threatID":"b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb","threatStatus":"active","classification":"spam","threatUrl":"https://threatinsight.proofpoint.com/aaabcdef-1234-b1abcdefghe/threat/email/b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb","threatTime":"2021-11-25T13:02:58.640Z","threat":"http://zbcd123456x0.example.com","campaignID":null,"threatType":"url"},{"threatID":"aaabcdefg123456f009971a9c193abcdefg123456bf5abcdefg1234566","threatStatus":"active","classification":"phish","threatUrl":"https://threatinsight.proofpoint.com/aaabcdef-1234-b1abcdefghe/threat/email/b7exxxxxxxx0d10xxxxxxe2xxxxxxxxxxxx81cxxxxxx034ac9cxxxxxxxxxxxxb","threatTime":"2021-07-19T10:28:15.100Z","threat":"http://zbcd123456x0.example.com","campaignID":null,"threatType":"url"}],"messageTime":"2022-01-01T00:00:00.000Z","impostorScore":0,"malwareScore":0,"cluster":"pharmtech_hosted","subject":null,"quarantineFolder":null,"quarantineRule":null,"policyRoutes":null,"modulesRun":null,"messageSize":0,"headerFrom":null,"headerReplyTo":null,"fromAddress":null,"ccAddresses":null,"replyToAddress":null,"toAddresses":null,"xmailer":null,"messageParts":null,"completelyRewritten":true,"id":"2hsvbU-i8abc123-12345-xxxxx12","QID":null,"GUID":"NxxxsxvxbxUxixcx2xxxxx5x6xWxBxOxxxxxjxx","sender":"","recipient":["fxxxxhxsxxvxbcx2xx5xxx6x3xx26@example.com"],"senderIP":"89.160.20.112","messageID":""}]} + body: |- + {{ minify_json ` + { + "queryEndTime": "2082-03-29T20:00:00Z", + "messagesDelivered": [] + } + `}} + - path: /v2/siem/clicks/permitted methods: [GET] + query_params: + interval: "{interval:20[2-6][0-9].*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "queryEndTime": "2072-03-30T13:00:00Z", + "clicksPermitted": [ + { + "url": "https://example.com/collab/?id=x4x3x6xsx1xxxx8xEdxexnxxxaxX", + "classification": "phish", + "clickTime": "2022-03-21T20:39:37.000Z", + "threatTime": "2022-03-30T10:05:57.000Z", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46", + "campaignId": "46x01x8x-x899-404x-xxx9-111xx393d1x7", + "id": "de7eef56-1234-1234-1234-5xxfx7xxxdxxxx", + "clickIP": "89.160.20.112", + "sender": "abc123@example.com", + "recipient": "abc@example.com", + "senderIP": "81.2.69.143", + "GUID": "cTxxxxxxzx7xxxxxxxxxx8x4xwxx", + "threatID": "92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx", + "threatURL": "https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx", + "threatStatus": "active", + "messageID": "12345678912345.12345.mail@example.com" + } + ] + } + `}} + - path: /v2/siem/clicks/permitted + methods: [GET] + query_params: + interval: "{interval:2072.*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "queryEndTime": "2082-03-30T13:00:00Z", + "clicksPermitted": [ + { + "url": "https://example.com/collab/?id=x4x3x6xsx1xxxx8xEdxexnxxxaxX", + "classification": "phish", + "clickTime": "2072-03-21T20:39:37.000Z", + "threatTime": "2072-03-30T10:05:57.000Z", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46", + "campaignId": "46x01x8x-x899-404x-xxx9-111xx393d1x7", + "id": "de7eef56-1234-1234-1234-5xxfx7xxxdxxxx", + "clickIP": "89.160.20.112", + "sender": "abc123@example.com", + "recipient": "abc@example.com", + "senderIP": "81.2.69.143", + "GUID": "cTxxxxxxzx7xxxxxxxxxx8x4xwxx", + "threatID": "92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx", + "threatURL": "https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx", + "threatStatus": "active", + "messageID": "12345678912345.12345.mail@example.com" + } + ] + } + `}} + - path: /v2/siem/clicks/permitted + methods: [GET] + query_params: + interval: "{interval:2082.*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "queryEndTime": "2082-03-30T13:00:00Z", + "clicksPermitted": [] + } + `}} + + - path: /v2/siem/clicks/blocked + methods: [GET] + query_params: + interval: "{interval:20[2-6][0-9].*}" + responses: + - status_code: 200 + body: |- + {{ minify_json ` + { + "queryEndTime": "2072-03-30T13:00:00Z", + "clicksBlocked": [ + { + "url": "https://www.example.com/abcdabcd123?query=0", + "classification": "malware", + "clickTime": "2022-03-30T10:11:12.000Z", + "threatTime": "2022-03-21T14:40:31.000Z", + "userAgent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/199.0.427504638 Mobile/15E148 Safari/604.1", + "campaignId": "46x01x8x-x899-404x-xxx9-111xx393d1x7", + "id": "a5c9f8bb-1234-1234-1234-dx9xxx2xx9xxx", + "clickIP": "89.160.20.112", + "sender": "abc123@example.com", + "recipient": "9c52aa64228824247c48df69b066e5a7@example.com", + "senderIP": "81.2.69.143", + "GUID": "ZcxxxxVxyxFxyxLxxxDxVxx4xxxxx", + "threatID": "502b7xxxx0x5x1x3xb6xcxexbxxxxxxxcxxexc6xbxxxxxxdx7fxcx6x9xxxx9xdxxxxxxxx5f", + "threatURL": "https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/502xxxxxxxxxcebxxxxxxxxxxa04277xxxxx5dxc6xxxxxxxxx5f", + "threatStatus": "active", + "messageID": "12345678912345.12345.mail@example.com" + } + ] + } + `}} + - path: /v2/siem/clicks/blocked + methods: [GET] + query_params: + interval: "{interval:2072.*}" responses: - status_code: 200 - body: | - {"queryEndTime":"2022-03-30T13:00:00Z","clicksPermitted":[{"url":"https://example.com/collab/?id=x4x3x6xsx1xxxx8xEdxexnxxxaxX","classification":"phish","clickTime":"2022-03-21T20:39:37.000Z","threatTime":"2022-03-30T10:05:57.000Z","userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"de7eef56-1234-1234-1234-5xxfx7xxxdxxxx","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"abc@example.com","senderIP":"81.2.69.143","GUID":"cTxxxxxxzx7xxxxxxxxxx8x4xwxx","threatID":"92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/92c17aaxxxxxxxxxx07xx7xxxx9xexcx3x3xxxxxx8xx3xxxx","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"}]} + body: |- + {{ minify_json ` + { + "queryEndTime": "2082-03-30T13:00:00Z", + "clicksBlocked": [ + { + "url": "https://www.example.com/abcdabcd123?query=0", + "classification": "malware", + "clickTime": "2072-03-30T10:11:12.000Z", + "threatTime": "2072-03-21T14:40:31.000Z", + "userAgent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/199.0.427504638 Mobile/15E148 Safari/604.1", + "campaignId": "46x01x8x-x899-404x-xxx9-111xx393d1x7", + "id": "a5c9f8bb-1234-1234-1234-dx9xxx2xx9xxx", + "clickIP": "89.160.20.112", + "sender": "abc123@example.com", + "recipient": "9c52aa64228824247c48df69b066e5a7@example.com", + "senderIP": "81.2.69.143", + "GUID": "ZcxxxxVxyxFxyxLxxxDxVxx4xxxxx", + "threatID": "502b7xxxx0x5x1x3xb6xcxexbxxxxxxxcxxexc6xbxxxxxxdx7fxcx6x9xxxx9xdxxxxxxxx5f", + "threatURL": "https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/502xxxxxxxxxcebxxxxxxxxxxa04277xxxxx5dxc6xxxxxxxxx5f", + "threatStatus": "active", + "messageID": "12345678912345.12345.mail@example.com" + } + ] + } + `}} - path: /v2/siem/clicks/blocked methods: [GET] + query_params: + interval: "{interval:2082.*}" responses: - status_code: 200 - body: | - {"queryEndTime":"2022-03-30T13:00:00Z","clicksBlocked":[{"url":"https://www.example.com/abcdabcd123?query=0","classification":"malware","clickTime":"2022-03-30T10:11:12.000Z","threatTime":"2022-03-21T14:40:31.000Z","userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/199.0.427504638 Mobile/15E148 Safari/604.1","campaignId":"46x01x8x-x899-404x-xxx9-111xx393d1x7","id":"a5c9f8bb-1234-1234-1234-dx9xxx2xx9xxx","clickIP":"89.160.20.112","sender":"abc123@example.com","recipient":"9c52aa64228824247c48df69b066e5a7@example.com","senderIP":"81.2.69.143","GUID":"ZcxxxxVxyxFxyxLxxxDxVxx4xxxxx","threatID":"502b7xxxx0x5x1x3xb6xcxexbxxxxxxxcxxexc6xbxxxxxxdx7fxcx6x9xxxx9xdxxxxxxxx5f","threatURL":"https://threatinsight.proofpoint.com/a2abc123-1234-1234-1234-babcded1234/threat/email/502xxxxxxxxxcebxxxxxxxxxxa04277xxxxx5dxc6xxxxxxxxx5f","threatStatus":"active","messageID":"12345678912345.12345.mail@example.com"}]} + body: |- + {{ minify_json ` + { + "queryEndTime": "2082-03-30T13:00:00Z", + "clicksBlocked": [] + } + `}} diff --git a/packages/proofpoint_tap/changelog.yml b/packages/proofpoint_tap/changelog.yml index 418611a6986..2e377f4d7a1 100644 --- a/packages/proofpoint_tap/changelog.yml +++ b/packages/proofpoint_tap/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "1.24.0" + changes: + - description: Improve clarity of agent behavior configuration. + type: enhancement + link: https://github.com/elastic/integrations/pull/11361 + - description: Fix pagination termination condition check. + type: bugfix + link: https://github.com/elastic/integrations/pull/11361 - version: "1.23.0" changes: - description: Set default search period to one day. diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/system/test-default-config.yml b/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/system/test-default-config.yml index 360ba65cd09..a12ff0fd355 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/system/test-default-config.yml +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/system/test-default-config.yml @@ -1,10 +1,12 @@ input: httpjson service: proofpoint_tap vars: + enable_request_tracer: true url: http://{{Hostname}}:{{Port}} principal: xxxx secret: xxxx data_stream: vars: preserve_original_event: true - enable_request_tracer: true +assert: + hit_count: 2 diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs b/packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs index 113b2902d6e..ba8e7037d31 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs @@ -20,12 +20,53 @@ request.transforms: value: json - set: target: url.params.interval - value: '[[if (le (formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))]][[else]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate now]][[end]]' - default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]/[[formatDate ((now (parseDuration "-{{initial_interval}}")).Add (parseDuration "1h"))]]' + value: |- + [[- $last := (parseDate .cursor.last_received_time "RFC3339") -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $last -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $last ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $last ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] + default: |- + [[- $start := (now (parseDuration "-{{initial_interval}}")) -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $start -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $start ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $start ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] response.pagination: - set: target: url.params.interval - value: '[[if (le (formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .last_response.body.queryEndTime "RFC3339")]]/[[formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))]][[end]]' + value: |- + [[- if ne (len .last_response.body.clicksBlocked) 0 -]] + [[- $last := (parseDate .last_response.body.queryEndTime "RFC3339") -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $last -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $last ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $last ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] + [[- end -]] fail_on_template_error: true cursor: last_received_time: diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/system/test-default-config.yml b/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/system/test-default-config.yml index 360ba65cd09..a12ff0fd355 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/system/test-default-config.yml +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/system/test-default-config.yml @@ -1,10 +1,12 @@ input: httpjson service: proofpoint_tap vars: + enable_request_tracer: true url: http://{{Hostname}}:{{Port}} principal: xxxx secret: xxxx data_stream: vars: preserve_original_event: true - enable_request_tracer: true +assert: + hit_count: 2 diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs b/packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs index 20a2347e23e..cf1359e6016 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs @@ -20,12 +20,53 @@ request.transforms: value: json - set: target: url.params.interval - value: '[[if (le (formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))]][[else]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate now]][[end]]' - default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]/[[formatDate ((now (parseDuration "-{{initial_interval}}")).Add (parseDuration "1h"))]]' + value: |- + [[- $last := (parseDate .cursor.last_received_time "RFC3339") -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $last -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $last ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $last ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] + default: |- + [[- $start := (now (parseDuration "-{{initial_interval}}")) -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $start -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $start ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $start ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] response.pagination: - set: target: url.params.interval - value: '[[if (le (formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .last_response.body.queryEndTime "RFC3339")]]/[[formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))]][[end]]' + value: |- + [[- if ne (len .last_response.body.clicksPermitted) 0 -]] + [[- $last := (parseDate .last_response.body.queryEndTime "RFC3339") -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $last -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $last ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $last ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] + [[- end -]] fail_on_template_error: true cursor: last_received_time: diff --git a/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/system/test-default-config.yml b/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/system/test-default-config.yml index 360ba65cd09..a12ff0fd355 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/system/test-default-config.yml +++ b/packages/proofpoint_tap/data_stream/message_blocked/_dev/test/system/test-default-config.yml @@ -1,10 +1,12 @@ input: httpjson service: proofpoint_tap vars: + enable_request_tracer: true url: http://{{Hostname}}:{{Port}} principal: xxxx secret: xxxx data_stream: vars: preserve_original_event: true - enable_request_tracer: true +assert: + hit_count: 2 diff --git a/packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs b/packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs index 03d10fa9cf5..acaa63a1e34 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs +++ b/packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs @@ -20,12 +20,53 @@ request.transforms: value: json - set: target: url.params.interval - value: '[[if (le (formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))]][[else]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate now]][[end]]' - default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]/[[formatDate ((now (parseDuration "-{{initial_interval}}")).Add (parseDuration "1h"))]]' + value: |- + [[- $last := (parseDate .cursor.last_received_time "RFC3339") -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $last -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $last ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $last ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] + default: |- + [[- $start := (now (parseDuration "-{{initial_interval}}")) -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $start -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $start ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $start ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] response.pagination: - set: target: url.params.interval - value: '[[if (le (formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .last_response.body.queryEndTime "RFC3339")]]/[[formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))]][[end]]' + value: |- + [[- if ne (len .last_response.body.messagesBlocked) 0 -]] + [[- $last := (parseDate .last_response.body.queryEndTime "RFC3339") -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $last -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $last ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $last ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] + [[- end -]] fail_on_template_error: true cursor: last_received_time: diff --git a/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/system/test-default-config.yml b/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/system/test-default-config.yml index 360ba65cd09..a12ff0fd355 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/system/test-default-config.yml +++ b/packages/proofpoint_tap/data_stream/message_delivered/_dev/test/system/test-default-config.yml @@ -1,10 +1,12 @@ input: httpjson service: proofpoint_tap vars: + enable_request_tracer: true url: http://{{Hostname}}:{{Port}} principal: xxxx secret: xxxx data_stream: vars: preserve_original_event: true - enable_request_tracer: true +assert: + hit_count: 2 diff --git a/packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs b/packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs index 647f33f46f0..ff220f97549 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs +++ b/packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs @@ -20,12 +20,53 @@ request.transforms: value: json - set: target: url.params.interval - value: '[[if (le (formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))]][[else]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate now]][[end]]' - default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]/[[formatDate ((now (parseDuration "-{{initial_interval}}")).Add (parseDuration "1h"))]]' + value: |- + [[- $last := (parseDate .cursor.last_received_time "RFC3339") -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $last -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $last ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $last ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] + default: |- + [[- $start := (now (parseDuration "-{{initial_interval}}")) -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $start -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $start ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $start ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] response.pagination: - set: target: url.params.interval - value: '[[if (le (formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .last_response.body.queryEndTime "RFC3339")]]/[[formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))]][[end]]' + value: |- + [[- if ne (len .last_response.body.messagesDelivered) 0 -]] + [[- $last := (parseDate .last_response.body.queryEndTime "RFC3339") -]] + [[- $hour := (parseDuration "1h") -]] + [[- $end := 0 -]][[- /* Predeclare $end. */ -]] + [[- with $last -]] + [[- $end = .Add $hour -]] + [[- end -]] + [[- with $end -]] + [[- if .Before now -]] + [[- formatDate $last ]]/[[ formatDate $end -]] + [[- else -]] + [[- formatDate $last ]]/[[ formatDate now -]] + [[- end -]] + [[- end -]] + [[- end -]] fail_on_template_error: true cursor: last_received_time: diff --git a/packages/proofpoint_tap/manifest.yml b/packages/proofpoint_tap/manifest.yml index c6aa3e00d72..8803be08048 100644 --- a/packages/proofpoint_tap/manifest.yml +++ b/packages/proofpoint_tap/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: proofpoint_tap title: Proofpoint TAP -version: "1.23.0" +version: "1.24.0" description: Collect logs from Proofpoint TAP with Elastic Agent. type: integration categories: