From 13b9f1457b85008fe1de12c400dc703594527e0e Mon Sep 17 00:00:00 2001 From: muleysteve <30406388+muleysteve@users.noreply.github.com> Date: Thu, 18 Aug 2022 16:43:25 +0100 Subject: [PATCH] Release 1.9.1 (#197) * Done with CoreServicesMetric - usersTotal pagination * Added SDLC Metrics for Azure DevOps * Removed extra logging and changed component names * externalized dataweave scripts * Update pom.xml * Simplifying Postman collection Simplified Postman collection to use only Environment vars rather than Global and Environment. Added section to README on Postman collection usage Removed toLowerCase() from pre-request scripts * Update README.md * corrected copy paste error * Update README.md * Update README.md * Changed links to be relative * Added secure properties list to mule-artifact.json * Updated HTTP ELK Request Config to use defined property * Fix for incorrect variable rawData The variable rawData was set in the same Transform Message component as the variable loaderDetails, on which the DataWeave for rawData depends. Thus vars.rawData was not set correctly. * Adding version to the Splunk forms according Splunk 9.0.0 specifications on https://docs.splunk.com/Documentation/Splunk/9.0.0/Viz/DashboardjQuery * #195 Excluded secure property files from index * Release 1.9.1 Co-authored-by: Andres Alleva Co-authored-by: Scott Ratliff Co-authored-by: richardmckinley <58299919+richardmckinley@users.noreply.github.com> Co-authored-by: Alberici <92322091+angelalbericisfdc@users.noreply.github.com> Co-authored-by: Rodrigo Fuentes Co-authored-by: Victor Moreno Co-authored-by: Christina Bauer Co-authored-by: George Kroon --- README.md | 12 ++- dashboards/splunk/access_management.xml | 2 +- .../api_manager_and_analytics__production.xml | 2 +- .../api_manager_and_analytics__sandbox.xml | 2 +- dashboards/splunk/current__consolidated.xml | 2 +- dashboards/splunk/design_center.xml | 2 +- dashboards/splunk/exchange.xml | 2 +- dashboards/splunk/mq__production.xml | 2 +- dashboards/splunk/mq__sandbox.xml | 2 +- dashboards/splunk/osv2__production.xml | 2 +- dashboards/splunk/osv2__sandbox.xml | 2 +- dashboards/splunk/platform_benefits.xml | 2 +- .../runtime_manager__cloudhub__networking.xml | 2 +- .../runtime_manager__cloudhub__production.xml | 2 +- .../runtime_manager__cloudhub__sandbox.xml | 2 +- .../splunk/runtime_manager__rtf__capacity.xml | 2 +- .../runtime_manager__rtf__production.xml | 2 +- .../splunk/runtime_manager__rtf__sandbox.xml | 2 +- ...untime_manager__standalone__production.xml | 2 +- .../runtime_manager__standalone__sandbox.xml | 2 +- mule-artifact.json | 31 ++++++- pom.xml | 2 +- postman/environment_template.json | 56 ++++++++++++ postman/postman_collection.json | 81 ++++++++++------- .../aggregators/aggregator-sdlc-metrics.xml | 3 + .../systems/collector-azuredevops.xml | 45 ++++++++++ src/main/mule/common.xml | 8 +- src/main/mule/loaders/loader-elk.xml | 2 +- .../platform/apis/api-call-coreservices.xml | 51 +++++++++-- .../systems/apis/api-call-azuredevops.xml | 90 +++++++++++++++++++ src/main/resources/application-types.xml | 1 + .../build-sdlc-metrics-aggregation.dwl | 15 ++++ .../build-sdlc-raw-data-response.dwl | 2 + .../save-aggregated-member-payload-var.dwl | 6 ++ .../dw/anypoint/save-members-obtained.dwl | 7 ++ .../set-sdlc-details-from-props-var.dwl | 4 + .../dw/sdlc/azure-aggregate-cicd-runs.dwl | 14 +++ .../resources/dw/sdlc/azure-backlog-query.dwl | 6 ++ .../dw/sdlc/azure-convert-repo-list.dwl | 7 ++ .../dw/sdlc/azure-convert-workitem-list.dwl | 4 + .../sdlc/azure-convert-workitem-payload.dwl | 9 ++ .../resources/dw/sdlc/azure-sprint-query.dwl | 6 ++ src/main/resources/properties/app-dev.yaml | 5 ++ .../resources/properties/secure/.gitignore | 2 + .../properties/secure/_template.yaml | 59 ++++++++++++ .../Input-Variables-errors.wev | 4 + .../Output-Attributes.wev | 4 + .../Output-Payload.wev | 39 ++++++++ .../Output-Variables-errors.wev | 4 + .../Output-Variables-queryDetails.wev | 4 + .../Output-Variables-workItemList.wev | 4 + 51 files changed, 565 insertions(+), 60 deletions(-) create mode 100644 postman/environment_template.json create mode 100644 src/main/mule/collectors/external/systems/collector-azuredevops.xml create mode 100644 src/main/mule/sources/external/systems/apis/api-call-azuredevops.xml create mode 100644 src/main/resources/dw/anypoint/save-aggregated-member-payload-var.dwl create mode 100644 src/main/resources/dw/anypoint/save-members-obtained.dwl create mode 100644 src/main/resources/dw/sdlc/azure-aggregate-cicd-runs.dwl create mode 100644 src/main/resources/dw/sdlc/azure-backlog-query.dwl create mode 100644 src/main/resources/dw/sdlc/azure-convert-repo-list.dwl create mode 100644 src/main/resources/dw/sdlc/azure-convert-workitem-list.dwl create mode 100644 src/main/resources/dw/sdlc/azure-convert-workitem-payload.dwl create mode 100644 src/main/resources/dw/sdlc/azure-sprint-query.dwl create mode 100644 src/main/resources/properties/secure/.gitignore create mode 100644 src/main/resources/properties/secure/_template.yaml create mode 100644 src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Input-Variables-errors.wev create mode 100644 src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Attributes.wev create mode 100644 src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Payload.wev create mode 100644 src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-errors.wev create mode 100644 src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-queryDetails.wev create mode 100644 src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-workItemList.wev diff --git a/README.md b/README.md index 9fe09243..a9388dcc 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Table of Contents * [Steps](#steps) * [Properties configurations](#properties-configurations) * [SDLC metrics collectors properties configuration](#sdlc-metrics-collectors-properties-configuration) + * [Postman Collection](#postman-collection) * [Splunk steps](#splunk-steps) * [Properties specific for Splunk](#properties-specific-for-splunk) * [ELK steps](#elk-steps) @@ -22,7 +23,7 @@ Table of Contents * [Tableau steps](#tableau-steps) * [MongoDB steps](#mongodb-steps) * [Properties specific for MongoDB](#properties-specific-for-mongodb) - * [SFDC-Einstein Analytics Steps](#einstein-analytics-dashboard-steps) + * [SFDC-Einstein Analytics Steps](#sfdc-steps) * [Considerations](#considerations) * [Some Theory around the Toolkit](#some-theory-around-the-toolkit) * [Business Needs](#business-needs) @@ -328,6 +329,8 @@ sdlc.splunk.password | Password to access REST APIs | **NOTE:** Please note that each external system collector should be self-contained, it means that all associated configuration must be part of the Mule configuration file itself and must not be externalized inside the `global.xml` +### Postman Collection +A Postman collection which contains sample API requests is included under /postman/postman_collection.json. To import this, open Postman and click File > Import, and select the .json file. The collection contains documentation describing prerequisites and setup. Clicking the collection folder opens the documentation window. A number of Environment variables are needed. A template environment is also included under /postman/environment_template.json. To import into Postman, click Environemts > Import and select the .json file. Populate the variables with accurate values. ### Splunk steps @@ -575,3 +578,10 @@ How to link business needs, measurements and data sources? ## Final Notes Enjoy and provide feedback / contribute :) + + +## Common Errors + +### Permission errors +- All `403` errors, specifically, for the endpoint related to RTF deployments /hybrid/api/v2/ are permissions issues, be sure the user/connected-app has the right permissions (Runtime Manager and Runtime Fabric specifically): [Requirements](#requirements) + diff --git a/dashboards/splunk/access_management.xml b/dashboards/splunk/access_management.xml index 43bdb44b..f4015dd8 100644 --- a/dashboards/splunk/access_management.xml +++ b/dashboards/splunk/access_management.xml @@ -1,4 +1,4 @@ -
+
diff --git a/dashboards/splunk/api_manager_and_analytics__production.xml b/dashboards/splunk/api_manager_and_analytics__production.xml index 703d4f85..f86ea453 100644 --- a/dashboards/splunk/api_manager_and_analytics__production.xml +++ b/dashboards/splunk/api_manager_and_analytics__production.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/api_manager_and_analytics__sandbox.xml b/dashboards/splunk/api_manager_and_analytics__sandbox.xml index 1032841c..f3ce949c 100644 --- a/dashboards/splunk/api_manager_and_analytics__sandbox.xml +++ b/dashboards/splunk/api_manager_and_analytics__sandbox.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/current__consolidated.xml b/dashboards/splunk/current__consolidated.xml index a4fe9459..7e9337a0 100644 --- a/dashboards/splunk/current__consolidated.xml +++ b/dashboards/splunk/current__consolidated.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/design_center.xml b/dashboards/splunk/design_center.xml index b0318ca2..dc1fe756 100644 --- a/dashboards/splunk/design_center.xml +++ b/dashboards/splunk/design_center.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/exchange.xml b/dashboards/splunk/exchange.xml index 0b07cf6b..414e31af 100644 --- a/dashboards/splunk/exchange.xml +++ b/dashboards/splunk/exchange.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/mq__production.xml b/dashboards/splunk/mq__production.xml index bb23a2d8..3459b44e 100644 --- a/dashboards/splunk/mq__production.xml +++ b/dashboards/splunk/mq__production.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/mq__sandbox.xml b/dashboards/splunk/mq__sandbox.xml index ea2f36b9..bf99d729 100644 --- a/dashboards/splunk/mq__sandbox.xml +++ b/dashboards/splunk/mq__sandbox.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/osv2__production.xml b/dashboards/splunk/osv2__production.xml index ca73e3e8..0e5ed9a1 100644 --- a/dashboards/splunk/osv2__production.xml +++ b/dashboards/splunk/osv2__production.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/osv2__sandbox.xml b/dashboards/splunk/osv2__sandbox.xml index fa1cd2d7..4d1451c6 100644 --- a/dashboards/splunk/osv2__sandbox.xml +++ b/dashboards/splunk/osv2__sandbox.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/platform_benefits.xml b/dashboards/splunk/platform_benefits.xml index ff274e72..1d5e6972 100644 --- a/dashboards/splunk/platform_benefits.xml +++ b/dashboards/splunk/platform_benefits.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/runtime_manager__cloudhub__networking.xml b/dashboards/splunk/runtime_manager__cloudhub__networking.xml index 9c98b56f..48c1502a 100644 --- a/dashboards/splunk/runtime_manager__cloudhub__networking.xml +++ b/dashboards/splunk/runtime_manager__cloudhub__networking.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/runtime_manager__cloudhub__production.xml b/dashboards/splunk/runtime_manager__cloudhub__production.xml index 4e748a4c..c1979438 100644 --- a/dashboards/splunk/runtime_manager__cloudhub__production.xml +++ b/dashboards/splunk/runtime_manager__cloudhub__production.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/runtime_manager__cloudhub__sandbox.xml b/dashboards/splunk/runtime_manager__cloudhub__sandbox.xml index c601360f..051663c1 100644 --- a/dashboards/splunk/runtime_manager__cloudhub__sandbox.xml +++ b/dashboards/splunk/runtime_manager__cloudhub__sandbox.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/runtime_manager__rtf__capacity.xml b/dashboards/splunk/runtime_manager__rtf__capacity.xml index 6d51bfcf..240c2329 100644 --- a/dashboards/splunk/runtime_manager__rtf__capacity.xml +++ b/dashboards/splunk/runtime_manager__rtf__capacity.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/runtime_manager__rtf__production.xml b/dashboards/splunk/runtime_manager__rtf__production.xml index ce8d83cf..c1a6ca8a 100644 --- a/dashboards/splunk/runtime_manager__rtf__production.xml +++ b/dashboards/splunk/runtime_manager__rtf__production.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/runtime_manager__rtf__sandbox.xml b/dashboards/splunk/runtime_manager__rtf__sandbox.xml index 84fe4745..15144ad8 100644 --- a/dashboards/splunk/runtime_manager__rtf__sandbox.xml +++ b/dashboards/splunk/runtime_manager__rtf__sandbox.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/runtime_manager__standalone__production.xml b/dashboards/splunk/runtime_manager__standalone__production.xml index 9ef65050..32e85371 100644 --- a/dashboards/splunk/runtime_manager__standalone__production.xml +++ b/dashboards/splunk/runtime_manager__standalone__production.xml @@ -1,4 +1,4 @@ - +
diff --git a/dashboards/splunk/runtime_manager__standalone__sandbox.xml b/dashboards/splunk/runtime_manager__standalone__sandbox.xml index 5f2a5eeb..7c3a7bae 100644 --- a/dashboards/splunk/runtime_manager__standalone__sandbox.xml +++ b/dashboards/splunk/runtime_manager__standalone__sandbox.xml @@ -1,4 +1,4 @@ - +
diff --git a/mule-artifact.json b/mule-artifact.json index 00e045ae..679a8089 100644 --- a/mule-artifact.json +++ b/mule-artifact.json @@ -1 +1,30 @@ -{"minMuleVersion":"4.4.0"} \ No newline at end of file +{ + "secureProperties": [ + "auth.orgId", + "auth.username", + "auth.password", + "auth.clientId", + "auth.clientSecret", + "ignoreLists.organizations", + "ignoreLists.environments", + "splunk.token", + "elk.user", + "elk.password", + "mongodb.username", + "mongodb.password", + "sdlc.confluence.user", + "sdlc.confluence.token", + "sdlc.bitbucket.user", + "sdlc.bitbucket.token", + "sdlc.jira.user", + "sdlc.jira.token", + "sdlc.jenkins.user", + "sdlc.jenkins.token", + "sdlc.splunk.user", + "sdlc.splunk.password", + "sfdc.username", + "sfdc.password", + "sfdc.securityToken" + ], + "minMuleVersion":"4.4.0" +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 8c96d824..16c00759 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.mulesoft metrics-toolkit - 1.9.0 + 1.9.1 mule-application metrics-toolkit diff --git a/postman/environment_template.json b/postman/environment_template.json new file mode 100644 index 00000000..1aea874c --- /dev/null +++ b/postman/environment_template.json @@ -0,0 +1,56 @@ +{ + "id": "895c16d3-7559-4c95-8cbb-727d7d104b23", + "name": "Metrics Toolkit - Template", + "values": [ + { + "key": "url", + "value": "http://localhost:8081", + "enabled": true + }, + { + "key": "anypointOrg", + "value": "", + "type": "default", + "enabled": true + }, + { + "key": "credentialType", + "value": "", + "type": "default", + "enabled": true + }, + { + "key": "anypointUsername", + "value": "", + "type": "default", + "enabled": true + }, + { + "key": "anypointPassword", + "value": "", + "type": "default", + "enabled": true + }, + { + "key": "isPlaintextSecret", + "value": "", + "type": "default", + "enabled": true + }, + { + "key": "connectedAppId", + "value": "", + "type": "default", + "enabled": true + }, + { + "key": "connectedAppSecret", + "value": "", + "type": "default", + "enabled": true + } + ], + "_postman_variable_scope": "environment", + "_postman_exported_at": "2022-01-28T15:48:03.890Z", + "_postman_exported_using": "Postman/9.10.1" +} \ No newline at end of file diff --git a/postman/postman_collection.json b/postman/postman_collection.json index aa4df37f..2888851e 100644 --- a/postman/postman_collection.json +++ b/postman/postman_collection.json @@ -1,8 +1,8 @@ { "info": { - "_postman_id": "302143d6-ebe1-4056-8a90-20ca73680da7", + "_postman_id": "706115af-7ce6-4b1e-9383-e47f0cbdd838", "name": "Metrics Toolkit", - "description": "# Metrics Toolkit\nThis provides endpoints for the Anypoint Metrics Toolkit API.\n\n## Features\nIt contains the categories of requests listed below, which are separated into folders.\n\n- Benefits\n- Platform Metrics\n\nThis also has prerequest scripts to handle headers. Each folder has a script that automatically sets Anypoint Platform headers for the folder's requests. This is in each folder that has Anypoint Platform requests, as some folders (SDLC) don't need those headers.\n\n# Setup\nThis requires variables to be set for the requests to work.\n\n## Environment Variables\n\n- `url`: The base url for the API. This contains the HTTP protocol, domain, and port. Example: `http://localhost:8081`\n\n## Global Variables\n\n- `anypointOrg`: The root Anypoint organization on which to run the metrics.\n- `credentialType`: Determines which Anypoint credential headers to generate. `user` for Anypoint Platform user credentials; `app` for connected app client credentials.\n- `anypointUsername`: Used for Anypoint Platform user credentials.\n- `anypointPassword`: Used for Anypoint Platform user credentials.\n- `connectedAppId`: Used for Connected App credentials.\n- `connectedAppSecret`: Used for Connected App credentials.\n- `isPlaintextSecret`: Indicates if the secret in the credentials is Base64 encoded or not. If `true`, it is not encoded.", + "description": "# Metrics Toolkit\n\nThis provides endpoints for the Anypoint Metrics Toolkit API.\n\n## Features\n\nIt contains the categories of requests listed below, which are separated into folders.\n\n* Benefits\n* Platform Metrics\n \n\nThis also has pre-request scripts to handle headers. Each folder has a script that automatically sets Anypoint Platform headers for the folder's requests. This is in each folder that has Anypoint Platform requests, as some folders (SDLC) don't need those headers.\n\n# Setup\n\nThis requires variables to be set for the requests to work.\n\n## Environment Variables\n\n* `url`: The base url for the API. This contains the HTTP protocol, domain, and port. Example: `http://localhost:8081`\n* `anypointOrg`: The root Anypoint organization on which to run the metrics.\n \n\n* `credentialType`: Determines which Anypoint credential headers to generate. `user` for Anypoint Platform user credentials; `app` for connected app client credentials.\n* `anypointUsername`: Used for Anypoint Platform user credentials.\n* `anypointPassword`: Used for Anypoint Platform user credentials.\n* `connectedAppId`: Used for Connected App credentials.\n* `connectedAppSecret`: Used for Connected App credentials.\n* `isPlaintextSecret`: Indicates if the secret in the credentials is Base64 encoded or not. If `true`, it is not encoded.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "item": [ @@ -11,9 +11,6 @@ "item": [ { "name": "Get Benefits", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, "request": { "method": "GET", "header": [ @@ -23,10 +20,6 @@ "type": "text" } ], - "body": { - "mode": "raw", - "raw": "" - }, "url": { "raw": "{{url}}/api/business-metrics?developAPIHistoricTime=20&developAPICurrentTime=10&maintainAPIHistoricTime=15&maintainAPICurrentTime=7&developerDayRate=200", "host": [ @@ -159,19 +152,19 @@ "/*", " Set Anypoint Platform headers from GLOBAL variables", "*/", - "var credentialType = pm.globals.get(\"credentialType\").toLowerCase();", + "var credentialType = pm.environment.get(\"credentialType\");", "var isUserCreds = credentialType == \"user\"; // user or app", "", "if (isUserCreds) {", - " pm.request.headers.upsert( {key: 'x-anypnt-username', value: pm.globals.get(\"anypointUsername\")} );", - " pm.request.headers.upsert( {key: 'x-anypnt-password', value: pm.globals.get(\"anypointPassword\")} );", + " pm.request.headers.upsert( {key: 'x-anypnt-username', value: pm.environment.get(\"anypointUsername\")} );", + " pm.request.headers.upsert( {key: 'x-anypnt-password', value: pm.environment.get(\"anypointPassword\")} );", "} else {", - " pm.request.headers.upsert( {key: 'x-anypnt-client-id', value: pm.globals.get(\"connectedAppId\")} );", - " pm.request.headers.upsert( {key: 'x-anypnt-client-secret', value: pm.globals.get(\"connectedAppSecret\")} );", + " pm.request.headers.upsert( {key: 'x-anypnt-client-id', value: pm.environment.get(\"connectedAppId\")} );", + " pm.request.headers.upsert( {key: 'x-anypnt-client-secret', value: pm.environment.get(\"connectedAppSecret\")} );", "}", - "var isPlaintextSecret = (typeof pm.globals.get(\"isPlaintextSecret\") !== 'undefined') ? pm.globals.get(\"isPlaintextSecret\") : \"true\"", + "var isPlaintextSecret = (typeof pm.environment.get(\"isPlaintextSecret\") !== 'undefined') ? pm.environment.get(\"isPlaintextSecret\") : \"true\"", "pm.request.headers.upsert( {key: 'x-plaintext-password', value: isPlaintextSecret} );", - "pm.request.headers.upsert( {key: 'x-anypnt-org-id', value: pm.globals.get(\"anypointOrg\")} );" + "pm.request.headers.upsert( {key: 'x-anypnt-org-id', value: pm.environment.get(\"anypointOrg\")} );" ] } }, @@ -191,9 +184,6 @@ "item": [ { "name": "Get Platform Metrics", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, "request": { "method": "GET", "header": [ @@ -203,10 +193,6 @@ "type": "text" } ], - "body": { - "mode": "raw", - "raw": "" - }, "url": { "raw": "{{url}}/api/platform-metrics?raw=false", "host": [ @@ -220,6 +206,11 @@ { "key": "raw", "value": "false" + }, + { + "key": "collectors", + "value": "", + "disabled": true } ] } @@ -342,6 +333,35 @@ }, "response": [] }, + { + "name": "Platform Metrics Load - AM", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"collectors\": [\"all\"],\n\t\"loaderDetails\": {\n\t\t\"strategy\": \"am\",\n\t\t\"rawData\": \"false\"\n\t}\n}" + }, + "url": { + "raw": "{{url}}/api/platform-metrics/load", + "host": [ + "{{url}}" + ], + "path": [ + "api", + "platform-metrics", + "load" + ] + } + }, + "response": [] + }, { "name": "Platform Metrics Load - Logger", "request": { @@ -410,19 +430,20 @@ "/*", " Set Anypoint Platform headers from GLOBAL variables", "*/", - "var credentialType = pm.globals.get(\"credentialType\").toLowerCase();", + "var credentialType = pm.environment.get(\"credentialType\");", "var isUserCreds = credentialType == \"user\"; // user or app", "", "if (isUserCreds) {", - " pm.request.headers.upsert( {key: 'x-anypnt-username', value: pm.globals.get(\"anypointUsername\")} );", - " pm.request.headers.upsert( {key: 'x-anypnt-password', value: pm.globals.get(\"anypointPassword\")} );", + " pm.request.headers.upsert( {key: 'x-anypnt-username', value: pm.environment.get(\"anypointUsername\")} );", + " pm.request.headers.upsert( {key: 'x-anypnt-password', value: pm.environment.get(\"anypointPassword\")} );", "} else {", - " pm.request.headers.upsert( {key: 'x-anypnt-client-id', value: pm.globals.get(\"connectedAppId\")} );", - " pm.request.headers.upsert( {key: 'x-anypnt-client-secret', value: pm.globals.get(\"connectedAppSecret\")} );", + " pm.request.headers.upsert( {key: 'x-anypnt-client-id', value: pm.environment.get(\"connectedAppId\")} );", + " pm.request.headers.upsert( {key: 'x-anypnt-client-secret', value: pm.environment.get(\"connectedAppSecret\")} );", "}", - "var isPlaintextSecret = (typeof pm.globals.get(\"isPlaintextSecret\") !== 'undefined') ? pm.globals.get(\"isPlaintextSecret\") : \"true\"", + "var isPlaintextSecret = (typeof pm.environment.get(\"isPlaintextSecret\") !== 'undefined') ? pm.environment.get(\"isPlaintextSecret\") : \"true\"", "pm.request.headers.upsert( {key: 'x-plaintext-password', value: isPlaintextSecret} );", - "pm.request.headers.upsert( {key: 'x-anypnt-org-id', value: pm.globals.get(\"anypointOrg\")} );" + "pm.request.headers.upsert( {key: 'x-anypnt-org-id', value: pm.environment.get(\"anypointOrg\")} );", + "" ] } }, diff --git a/src/main/mule/aggregators/aggregator-sdlc-metrics.xml b/src/main/mule/aggregators/aggregator-sdlc-metrics.xml index 016107ef..9d19c5e4 100644 --- a/src/main/mule/aggregators/aggregator-sdlc-metrics.xml +++ b/src/main/mule/aggregators/aggregator-sdlc-metrics.xml @@ -25,6 +25,9 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ + + + diff --git a/src/main/mule/collectors/external/systems/collector-azuredevops.xml b/src/main/mule/collectors/external/systems/collector-azuredevops.xml new file mode 100644 index 00000000..dadecdd1 --- /dev/null +++ b/src/main/mule/collectors/external/systems/collector-azuredevops.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/mule/common.xml b/src/main/mule/common.xml index 10b3b79c..4497fa04 100644 --- a/src/main/mule/common.xml +++ b/src/main/mule/common.xml @@ -71,11 +71,17 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ - + + + + + + + diff --git a/src/main/mule/loaders/loader-elk.xml b/src/main/mule/loaders/loader-elk.xml index a5fd1f23..aea116b5 100644 --- a/src/main/mule/loaders/loader-elk.xml +++ b/src/main/mule/loaders/loader-elk.xml @@ -7,7 +7,7 @@ http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ http://www.mulesoft.org/schema/mule/http http://www.mulesoft.org/schema/mule/http/current/mule-http.xsd"> - + diff --git a/src/main/mule/sources/anypoint/platform/apis/api-call-coreservices.xml b/src/main/mule/sources/anypoint/platform/apis/api-call-coreservices.xml index d81aac25..be98c594 100644 --- a/src/main/mule/sources/anypoint/platform/apis/api-call-coreservices.xml +++ b/src/main/mule/sources/anypoint/platform/apis/api-call-coreservices.xml @@ -82,9 +82,16 @@ else - - - + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/mule/sources/external/systems/apis/api-call-azuredevops.xml b/src/main/mule/sources/external/systems/apis/api-call-azuredevops.xml new file mode 100644 index 00000000..eac56dd3 --- /dev/null +++ b/src/main/mule/sources/external/systems/apis/api-call-azuredevops.xml @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/application-types.xml b/src/main/resources/application-types.xml index a78a3a3e..381e0f74 100644 --- a/src/main/resources/application-types.xml +++ b/src/main/resources/application-types.xml @@ -3,4 +3,5 @@ + \ No newline at end of file diff --git a/src/main/resources/dw/aggregation/build-sdlc-metrics-aggregation.dwl b/src/main/resources/dw/aggregation/build-sdlc-metrics-aggregation.dwl index fa8fa26c..945ef92f 100644 --- a/src/main/resources/dw/aggregation/build-sdlc-metrics-aggregation.dwl +++ b/src/main/resources/dw/aggregation/build-sdlc-metrics-aggregation.dwl @@ -6,6 +6,10 @@ var jenkinsData = payload[2].payload default [] var jiraData = payload[3].payload[0].payload default [] var jiraBacklogData = payload[3].payload[1].payload default [] var splunkData = payload[4].payload default [] +var azureDevOpsBacklogData = payload[5].payload[0].payload default [] +var azureDevOpsSprintData = payload[5].payload[1].payload default [] +var azureDevOpsBuildData = payload[5].payload[2].payload default [] +var azureDevOpsRepoData = payload[5].payload[3].payload default [] --- { date: vars.date, @@ -37,6 +41,17 @@ var splunkData = payload[4].payload default [] splunkMetrics: if(vars.sdlcDetails.splunk.enabled == "true") { totalDashboards: splunkData } else {}, + azuredevopsMetrics: if(vars.sdlcDetails.azuredevops.enabled == "true") { + totalTasksInBacklog: sizeOf(azureDevOpsBacklogData) default null, + totalTasksInSprint : sizeOf(azureDevOpsSprintData) default null, + totalTasksByType : (azureDevOpsSprintData groupBy $.taskType) mapObject {($$) : sizeOf($)} default null, + totalTasksByStatus : (azureDevOpsSprintData groupBy $.status) mapObject {($$) : sizeOf($)} default null, + totalRepositories: azureDevOpsRepoData.size, + totalJobs: sizeOf(azureDevOpsBuildData), + totalFailedJobs: sizeOf(azureDevOpsBuildData filter() -> $.result == "failed"), + totalSuccessJobs: sizeOf(azureDevOpsBuildData filter() -> $.result == "succeeded"), + totalUnexecutedJobs: sizeOf(azureDevOpsBuildData filter() -> $.result == "no runs") + } else {}, errors: vars.errors } filterObject (!isEmpty($)) } diff --git a/src/main/resources/dw/aggregation/build-sdlc-raw-data-response.dwl b/src/main/resources/dw/aggregation/build-sdlc-raw-data-response.dwl index 645efb06..e4e62d5d 100644 --- a/src/main/resources/dw/aggregation/build-sdlc-raw-data-response.dwl +++ b/src/main/resources/dw/aggregation/build-sdlc-raw-data-response.dwl @@ -5,6 +5,7 @@ var confluenceData = payload[1].payload var jenkinsData = payload[2].payload var jiraData = payload[3].payload var splunkData = payload[4].payload +var azureDevOpsData = payload[5].payload --- { date: vars.date, @@ -14,6 +15,7 @@ var splunkData = payload[4].payload jenkinsData: jenkinsData, jiraData: jiraData, splunkData: splunkData, + azureDevOpsData: azureDevOpsData, errors: vars.errors } filterObject (!isEmpty($)) } diff --git a/src/main/resources/dw/anypoint/save-aggregated-member-payload-var.dwl b/src/main/resources/dw/anypoint/save-aggregated-member-payload-var.dwl new file mode 100644 index 00000000..91cb0a67 --- /dev/null +++ b/src/main/resources/dw/anypoint/save-aggregated-member-payload-var.dwl @@ -0,0 +1,6 @@ +%dw 2.0 +import * from dw::util::Values +output application/java +--- +if (vars.aggregatedMemberPayload == null) payload +else (vars.aggregatedMemberPayload update "data" with (vars.aggregatedMemberPayload.data ++ payload.data)) diff --git a/src/main/resources/dw/anypoint/save-members-obtained.dwl b/src/main/resources/dw/anypoint/save-members-obtained.dwl new file mode 100644 index 00000000..78a4af2b --- /dev/null +++ b/src/main/resources/dw/anypoint/save-members-obtained.dwl @@ -0,0 +1,7 @@ +%dw 2.0 +output application/java +--- +if (vars.membersTotalObtained == null) +(sizeOf(payload.data)) +else +(vars.membersTotalObtained + (sizeOf(payload.data))) \ No newline at end of file diff --git a/src/main/resources/dw/common/set-sdlc-details-from-props-var.dwl b/src/main/resources/dw/common/set-sdlc-details-from-props-var.dwl index 48393135..153a356e 100644 --- a/src/main/resources/dw/common/set-sdlc-details-from-props-var.dwl +++ b/src/main/resources/dw/common/set-sdlc-details-from-props-var.dwl @@ -16,5 +16,9 @@ output application/java }, splunk: { enabled: p('sdlc.splunk.enabled') + }, + azuredevops: { + enabled: p('sdlc.azuredevops.enabled') } + } \ No newline at end of file diff --git a/src/main/resources/dw/sdlc/azure-aggregate-cicd-runs.dwl b/src/main/resources/dw/sdlc/azure-aggregate-cicd-runs.dwl new file mode 100644 index 00000000..d3b15e55 --- /dev/null +++ b/src/main/resources/dw/sdlc/azure-aggregate-cicd-runs.dwl @@ -0,0 +1,14 @@ +%dw 2.0 +output application/json +--- +vars.aggData default [] ++ + +if (payload.count == 0) + [{"result":"no runs"}] +else + [(((payload.value map() -> { + name: $.pipeline.name, + state: $.state, + result: $.result, + createdDate: $.createdDate as LocalDateTime, +}) orderBy $.createdDate)[-1 to 0])[0]] \ No newline at end of file diff --git a/src/main/resources/dw/sdlc/azure-backlog-query.dwl b/src/main/resources/dw/sdlc/azure-backlog-query.dwl new file mode 100644 index 00000000..df612398 --- /dev/null +++ b/src/main/resources/dw/sdlc/azure-backlog-query.dwl @@ -0,0 +1,6 @@ +%dw 2.0 +output application/json +--- +{ + "query": "Select [System.Id], [System.Title], [System.State], [System.WorkItemType] From WorkItems Where [System.IterationPath] = @project" +} \ No newline at end of file diff --git a/src/main/resources/dw/sdlc/azure-convert-repo-list.dwl b/src/main/resources/dw/sdlc/azure-convert-repo-list.dwl new file mode 100644 index 00000000..2ac274c2 --- /dev/null +++ b/src/main/resources/dw/sdlc/azure-convert-repo-list.dwl @@ -0,0 +1,7 @@ +%dw 2.0 +output application/json +--- +{ + size: payload.count, + repositories: (payload.value filter not $.isDisabled) map() -> {name: $.name} +} diff --git a/src/main/resources/dw/sdlc/azure-convert-workitem-list.dwl b/src/main/resources/dw/sdlc/azure-convert-workitem-list.dwl new file mode 100644 index 00000000..ee3e8439 --- /dev/null +++ b/src/main/resources/dw/sdlc/azure-convert-workitem-list.dwl @@ -0,0 +1,4 @@ +%dw 2.0 +output application/java +--- +payload.workItems.id joinBy "," \ No newline at end of file diff --git a/src/main/resources/dw/sdlc/azure-convert-workitem-payload.dwl b/src/main/resources/dw/sdlc/azure-convert-workitem-payload.dwl new file mode 100644 index 00000000..13808e12 --- /dev/null +++ b/src/main/resources/dw/sdlc/azure-convert-workitem-payload.dwl @@ -0,0 +1,9 @@ +%dw 2.0 +output application/json +--- +payload.value map() -> { + id: $.id, + name: $.fields."System.Title", + taskType: $.fields."System.WorkItemType", + status: $.fields."System.State" +} \ No newline at end of file diff --git a/src/main/resources/dw/sdlc/azure-sprint-query.dwl b/src/main/resources/dw/sdlc/azure-sprint-query.dwl new file mode 100644 index 00000000..1ba98cbb --- /dev/null +++ b/src/main/resources/dw/sdlc/azure-sprint-query.dwl @@ -0,0 +1,6 @@ +%dw 2.0 +output application/json +--- +{ + "query": "Select [System.Id], [System.Title], [System.State], [System.WorkItemType] From WorkItems Where [System.IterationPath] = @CurrentIteration" +} \ No newline at end of file diff --git a/src/main/resources/properties/app-dev.yaml b/src/main/resources/properties/app-dev.yaml index 2e4bafbc..41daf53b 100644 --- a/src/main/resources/properties/app-dev.yaml +++ b/src/main/resources/properties/app-dev.yaml @@ -125,3 +125,8 @@ sdlc: host: "test.splunk.com" port: "8089" path: "/servicesNS/nobody/mulesoft/data/ui/views?count=-0" + azuredevops: + enabled: "false" #Configure and set this to true if Azure DevOps metrics are required + host: "dev.azure.com" + apiPath: "/mycompany/mulesoft-projects/_apis" # Path should be set to /[organization name]/[project name]/_apis + projectName: "mulesoft-projects" diff --git a/src/main/resources/properties/secure/.gitignore b/src/main/resources/properties/secure/.gitignore new file mode 100644 index 00000000..f3d5391c --- /dev/null +++ b/src/main/resources/properties/secure/.gitignore @@ -0,0 +1,2 @@ +# Ignore all local secure property files +app-*.yaml diff --git a/src/main/resources/properties/secure/_template.yaml b/src/main/resources/properties/secure/_template.yaml new file mode 100644 index 00000000..074548c6 --- /dev/null +++ b/src/main/resources/properties/secure/_template.yaml @@ -0,0 +1,59 @@ +# +# IMPORTANT! +# Copy this file and rename it according to your environment: e.g. for the 'dev' environment copy this to `app-dev.yaml`. +# Provide values as secure properties eg: "![abcacbabcabc]" +# https://docs.mulesoft.com/mule-runtime/latest/secure-configuration-properties +# + + +auth: + orgId: "" + + # If auth mode = platform-credentials (ignored, otherwise): + username: "" + password: "" + + # If auth mode = connected-app-credentials (ignored, otherwise): + clientId: "" + clientSecret: "" + +ignoreLists: + organizations: "" #Array: comma-separated values 'orgId1,orgId2' + environments: "" #Array: comma-separated values 'envId1,envId2' + +splunk: + token: "" + +elk: + user: "" + password: "" + +mongodb: + username: "" + password: "" + +# SDLC Metrics +sdlc: + confluence: + user: "" + token: "" + bitbucket: + user: "" + token: "" + jira: + user: "" + token: "" + jenkins: + user: "" + token: "" + splunk: + user: "" + password: "" + azuredevops: + token: "" + +sfdc: + username : "" + password : "" + securityToken : "" + # authUrl : "" diff --git a/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Input-Variables-errors.wev b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Input-Variables-errors.wev new file mode 100644 index 00000000..04df1705 --- /dev/null +++ b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Input-Variables-errors.wev @@ -0,0 +1,4 @@ +%dw 2.0 + +type auto_b7ea5168_d0f5_4749_bdb6_928480016a89_Input_Variables_errors = Array + diff --git a/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Attributes.wev b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Attributes.wev new file mode 100644 index 00000000..2476cf6e --- /dev/null +++ b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Attributes.wev @@ -0,0 +1,4 @@ +%dw 2.0 + +type auto_b7ea5168_d0f5_4749_bdb6_928480016a89_Output_Attributes = Null + diff --git a/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Payload.wev b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Payload.wev new file mode 100644 index 00000000..f711c11b --- /dev/null +++ b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Payload.wev @@ -0,0 +1,39 @@ +%dw 2.0 + +type auto_b7ea5168_d0f5_4749_bdb6_928480016a89_Output_Payload = {| + "0": org_mule_runtime_api_message_Message, + "1": org_mule_runtime_api_message_Message_6, + "2": org_mule_runtime_api_message_Message_7, + "3": org_mule_runtime_api_message_Message_9 +|} | Null +type org_mule_runtime_api_message_Message_7 = {| + payload: Null, + attributes: org_mule_extension_http_api_HttpResponseAttributes +|} {"typeId": "org.mule.runtime.api.message.Message"} +type org_mule_extension_http_api_HttpResponseAttributes = org_mule_extension_http_api_HttpResponseAttributes + + + + +type org_mule_runtime_api_message_Message_9 = {| + payload: {| + size: Any, + repositories: Any + |}, + attributes: org_mule_extension_http_api_HttpResponseAttributes +|} {"typeId": "org.mule.runtime.api.message.Message"} + + +type org_mule_runtime_api_message_Message_6 = {| + payload: Any, + attributes: org_mule_extension_http_api_HttpResponseAttributes +|} {"typeId": "org.mule.runtime.api.message.Message"} + + +type org_mule_runtime_api_message_Message = {| + payload: Any, + attributes: org_mule_extension_http_api_HttpResponseAttributes +|} {"typeId": "org.mule.runtime.api.message.Message"} + + + diff --git a/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-errors.wev b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-errors.wev new file mode 100644 index 00000000..cd63f031 --- /dev/null +++ b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-errors.wev @@ -0,0 +1,4 @@ +%dw 2.0 + +type auto_b7ea5168_d0f5_4749_bdb6_928480016a89_Output_Variables_errors = Array + diff --git a/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-queryDetails.wev b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-queryDetails.wev new file mode 100644 index 00000000..1d656310 --- /dev/null +++ b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-queryDetails.wev @@ -0,0 +1,4 @@ +%dw 2.0 + +type auto_b7ea5168_d0f5_4749_bdb6_928480016a89_Output_Variables_queryDetails = {| query: String |} + diff --git a/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-workItemList.wev b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-workItemList.wev new file mode 100644 index 00000000..15a001e5 --- /dev/null +++ b/src/main/resources/weave/autogenerated/b7ea5168-d0f5-4749-bdb6-928480016a89/Output-Variables-workItemList.wev @@ -0,0 +1,4 @@ +%dw 2.0 + +type auto_b7ea5168_d0f5_4749_bdb6_928480016a89_Output_Variables_workItemList = Any +