Merge branch 'master' into juan-fernandez/SDTEST-1190/cucumber-di

packages/datadog-instrumentations/src/mysql2.js

@@ -8,7 +8,7 @@ const {
 const shimmer = require('../../datadog-shimmer')
 const semver = require('semver')
 
-addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, (Connection, version) => {
+function wrapConnection (Connection, version) {
   const startCh = channel('apm:mysql2:query:start')
   const finishCh = channel('apm:mysql2:query:finish')
   const errorCh = channel('apm:mysql2:query:error')
@@ -151,9 +151,8 @@ addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, (Conne
       }
     }, cmd))
   }
-})
-
-addHook({ name: 'mysql2', file: 'lib/pool.js', versions: ['>=1'] }, (Pool, version) => {
+}
+function wrapPool (Pool, version) {
   const startOuterQueryCh = channel('datadog:mysql2:outerquery:start')
   const shouldEmitEndAfterQueryAbort = semver.intersects(version, '>=1.3.3')
 
@@ -221,10 +220,9 @@ addHook({ name: 'mysql2', file: 'lib/pool.js', versions: ['>=1'] }, (Pool, versi
   })
 
   return Pool
-})
+}
 
-// PoolNamespace.prototype.query does not exist in mysql2<2.3.0
-addHook({ name: 'mysql2', file: 'lib/pool_cluster.js', versions: ['>=2.3.0'] }, PoolCluster => {
+function wrapPoolCluster (PoolCluster) {
   const startOuterQueryCh = channel('datadog:mysql2:outerquery:start')
   const wrappedPoolNamespaces = new WeakSet()
 
@@ -297,4 +295,11 @@ addHook({ name: 'mysql2', file: 'lib/pool_cluster.js', versions: ['>=2.3.0'] },
   })
 
   return PoolCluster
-})
+}
+
+addHook({ name: 'mysql2', file: 'lib/base/connection.js', versions: ['>=3.11.5'] }, wrapConnection)
+addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['1 - 3.11.4'] }, wrapConnection)
+addHook({ name: 'mysql2', file: 'lib/pool.js', versions: ['1 - 3.11.4'] }, wrapPool)
+
+// PoolNamespace.prototype.query does not exist in mysql2<2.3.0
+addHook({ name: 'mysql2', file: 'lib/pool_cluster.js', versions: ['2.3.0 - 3.11.4'] }, wrapPoolCluster)

packages/dd-trace/src/appsec/reporter.js

@@ -148,7 +148,9 @@ function reportAttack (attackData) {
     newTags['_dd.appsec.json'] = '{"triggers":' + attackData + '}'
   }
 
-  newTags['network.client.ip'] = req.socket.remoteAddress
+  if (req.socket) {
+    newTags['network.client.ip'] = req.socket.remoteAddress
+  }
 
   rootSpan.addTags(newTags)
 }

packages/dd-trace/test/appsec/reporter.spec.js

@@ -223,6 +223,22 @@ describe('reporter', () => {
       storage.disable()
     })
 
+    it('should add tags to request span when socket is not there', () => {
+      delete req.socket
+
+      const result = Reporter.reportAttack('[{"rule":{},"rule_matches":[{}]}]')
+
+      expect(result).to.not.be.false
+      expect(web.root).to.have.been.calledOnceWith(req)
+
+      expect(span.addTags).to.have.been.calledOnceWithExactly({
+        'appsec.event': 'true',
+        '_dd.origin': 'appsec',
+        '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}'
+      })
+      expect(prioritySampler.setPriority).to.have.been.calledOnceWithExactly(span, USER_KEEP, SAMPLING_MECHANISM_APPSEC)
+    })
+
     it('should add tags to request span', () => {
       const result = Reporter.reportAttack('[{"rule":{},"rule_matches":[{}]}]')
       expect(result).to.not.be.false

packages/dd-trace/test/config.spec.js

@@ -28,6 +28,14 @@ describe('Config', () => {
   const BLOCKED_TEMPLATE_GRAPHQL_PATH = require.resolve('./fixtures/config/appsec-blocked-graphql-template.json')
   const BLOCKED_TEMPLATE_GRAPHQL = readFileSync(BLOCKED_TEMPLATE_GRAPHQL_PATH, { encoding: 'utf8' })
   const DD_GIT_PROPERTIES_FILE = require.resolve('./fixtures/config/git.properties')
+  const CONFIG_NORM_RULES_PATH = require.resolve('./fixtures/telemetry/config_norm_rules.json')
+  const CONFIG_NORM_RULES = readFileSync(CONFIG_NORM_RULES_PATH, { encoding: 'utf8' })
+  const CONFIG_PREFIX_BLOCK_LIST_PATH = require.resolve('./fixtures/telemetry/config_prefix_block_list.json')
+  const CONFIG_PREFIX_BLOCK_LIST = readFileSync(CONFIG_PREFIX_BLOCK_LIST_PATH, { encoding: 'utf8' })
+  const CONFIG_AGGREGATION_LIST_PATH = require.resolve('./fixtures/telemetry/config_aggregation_list.json')
+  const CONFIG_AGGREGATION_LIST = readFileSync(CONFIG_AGGREGATION_LIST_PATH, { encoding: 'utf8' })
+  const NODEJS_CONFIG_RULES_PATH = require.resolve('./fixtures/telemetry/nodejs_config_rules.json')
+  const NODEJS_CONFIG_RULES = readFileSync(NODEJS_CONFIG_RULES_PATH, { encoding: 'utf8' })
 
   function reloadLoggerAndConfig () {
     log = proxyquire('../src/log', {})
@@ -2258,5 +2266,76 @@ describe('Config', () => {
       expect(taggingConfig).to.have.property('responsesEnabled', true)
       expect(taggingConfig).to.have.property('maxDepth', 7)
     })
+
+    it('config_norm_rules completeness', () => {
+      // ⚠️ Did this test just fail? Read here! ⚠️
+      //
+      // Some files are manually copied from dd-go from/to the following paths
+      // from: https://github.com/DataDog/dd-go/blob/prod/trace/apps/tracer-telemetry-intake/telemetry-payload/static/
+      // to: packages/dd-trace/test/fixtures/telemetry/
+      // files:
+      // - config_norm_rules.json
+      // - config_prefix_block_list.json
+      // - config_aggregation_list.json
+      // - nodejs_config_rules.json
+      //
+      // If this test fails, it means that a telemetry key was found in config.js that does not
+      // exist in any of the files listed above in dd-go
+      // The impact is that telemetry will not be reported to the Datadog backend won't be unusable
+      //
+      // To fix this, you must update dd-go to either
+      // 1) Add an exact config key to match config_norm_rules.json
+      // 2) Add a prefix that matches the config keys to config_prefix_block_list.json
+      // 3) Add a prefix rule that fits an existing prefix to config_aggregation_list.json
+      // 4) (Discouraged) Add a language-specific rule to nodejs_config_rules.json
+      //
+      // Once dd-go is updated, you can copy over the files to this repo and merge them in as part of your changes
+
+      function getKeysInDotNotation (obj, parentKey = '') {
+        const keys = []
+
+        for (const key in obj) {
+          if (Object.prototype.hasOwnProperty.call(obj, key)) {
+            const fullKey = parentKey ? `${parentKey}.${key}` : key
+
+            if (typeof obj[key] === 'object' && obj[key] !== null && !Array.isArray(obj[key])) {
+              keys.push(...getKeysInDotNotation(obj[key], fullKey))
+            } else {
+              keys.push(fullKey)
+            }
+          }
+        }
+
+        return keys
+      }
+
+      const config = new Config()
+
+      const libraryConfigKeys = getKeysInDotNotation(config).sort()
+
+      const nodejsConfigRules = JSON.parse(NODEJS_CONFIG_RULES)
+      const configNormRules = JSON.parse(CONFIG_NORM_RULES)
+      const configPrefixBlockList = JSON.parse(CONFIG_PREFIX_BLOCK_LIST)
+      const configAggregationList = JSON.parse(CONFIG_AGGREGATION_LIST)
+
+      const allowedConfigKeys = [
+        ...Object.keys(configNormRules),
+        ...Object.keys(nodejsConfigRules.normalization_rules)
+      ]
+      const blockedConfigKeyPrefixes = [...configPrefixBlockList, ...nodejsConfigRules.prefix_block_list]
+      const configAggregationPrefixes = [
+        ...Object.keys(configAggregationList),
+        ...Object.keys(nodejsConfigRules.reduce_rules)
+      ]
+
+      const missingConfigKeys = libraryConfigKeys.filter(key => {
+        const isAllowed = allowedConfigKeys.includes(key)
+        const isBlocked = blockedConfigKeyPrefixes.some(prefix => key.startsWith(prefix))
+        const isReduced = configAggregationPrefixes.some(prefix => key.startsWith(prefix))
+        return !isAllowed && !isBlocked && !isReduced
+      })
+
+      expect(missingConfigKeys).to.be.empty
+    })
   })
 })

packages/dd-trace/test/fixtures/telemetry/config_aggregation_list.json

@@ -0,0 +1,24 @@
+{
+  "tags": "tags",
+  "global_tag_": "global_tags",
+  "trace_global_tags": "trace_global_tags",
+  "DD_TAGS": "tags",
+  "trace_span_tags": "trace_span_tags",
+  "http_client_tag_headers": "http_client_tag_headers",
+  "DD_TRACE_HEADER_TAGS": "trace_header_tags",
+  "trace_header_tags": "trace_header_tags",
+  "_options.headertags": "trace_header_tags",
+  "trace_request_header_tags": "trace_request_header_tags",
+  "trace_response_header_tags": "trace_response_header_tags",
+  "trace_request_header_tags_comma_allowed": "trace_request_header_tags",
+  "trace.header_tags": "trace_header_tags",
+  "DD_TRACE_GRPC_TAGS": "trace_grpc_tags",
+  "DD_TRACE_SERVICE_MAPPING": "trace_service_mappings",
+  "service_mapping": "trace_service_mappings",
+  "serviceMapping.": "trace_service_mappings",
+  "logger.": "logger_configs",
+  "sampler.rules.": "sampler_rules",
+  "sampler.spansamplingrules.": "sampler_span_sampling_rules",
+  "appsec.rules.rules": "appsec_rules",
+  "installSignature": "install_signature"
+}