Merge branch 'master' into ugaitz/rasp-blocking

DataDog · Jun 25, 2024 · 4a29cc7 · 4a29cc7
2 parents 1c14eab + 22e4d55
commit 4a29cc7
Show file tree

Hide file tree

Showing 29 changed files with 210 additions and 151 deletions.
diff --git a/integration-tests/profiler/profiler.spec.js b/integration-tests/profiler/profiler.spec.js
@@ -110,7 +110,7 @@ async function gatherNetworkTimelineEvents (cwd, scriptFilePath, eventType, args
     }
   })
 
-  await processExitPromise(proc, 5000)
+  await processExitPromise(proc, 30000)
   const procEnd = BigInt(Date.now() * 1000000)
 
   const { profile, encoded } = await getLatestProfile(cwd, /^events_.+\.pprof$/)
@@ -171,7 +171,7 @@ describe('profiler', () => {
   let oomTestFile
   let oomEnv
   let oomExecArgv
-  const timeout = 5000
+  const timeout = 30000
 
   before(async () => {
     sandbox = await createSandbox()
@@ -201,7 +201,7 @@ describe('profiler', () => {
         }
       })
 
-      await processExitPromise(proc, 5000)
+      await processExitPromise(proc, 30000)
       const procEnd = BigInt(Date.now() * 1000000)
 
       const { profile, encoded } = await getLatestProfile(cwd, /^wall_.+\.pprof$/)

diff --git a/package.json b/package.json
@@ -88,7 +88,7 @@
     "koalas": "^1.0.2",
     "limiter": "1.1.5",
     "lodash.sortby": "^4.7.0",
-    "lru-cache": "^7.14.0",
+    "lru-cache": "^10.2.2",
     "module-details-from-path": "^1.0.3",
     "msgpack-lite": "^0.1.26",
     "opentracing": ">=0.12.1",

diff --git a/packages/datadog-instrumentations/src/aws-sdk.js b/packages/datadog-instrumentations/src/aws-sdk.js
@@ -20,7 +20,8 @@ function wrapRequest (send) {
 
     return innerAr.runInAsyncScope(() => {
       this.on('complete', innerAr.bind(response => {
-        channel(`apm:aws:request:complete:${channelSuffix}`).publish({ response })
+        const cbExists = typeof cb === 'function'
+        channel(`apm:aws:request:complete:${channelSuffix}`).publish({ response, cbExists })
       }))
 
       startCh.publish({

diff --git a/packages/datadog-plugin-aws-sdk/src/base.js b/packages/datadog-plugin-aws-sdk/src/base.js
@@ -64,11 +64,17 @@ class BaseAwsSdkPlugin extends ClientPlugin {
       span.setTag('region', region)
     })
 
-    this.addSub(`apm:aws:request:complete:${this.serviceIdentifier}`, ({ response }) => {
+    this.addSub(`apm:aws:request:complete:${this.serviceIdentifier}`, ({ response, cbExists = false }) => {
       const store = storage.getStore()
       if (!store) return
       const { span } = store
       if (!span) return
+      // try to extract DSM context from response if no callback exists as extraction normally happens in CB
+      if (!cbExists && this.serviceIdentifier === 'sqs') {
+        const params = response.request.params
+        const operation = response.request.operation
+        this.responseExtractDSMContext(operation, params, response.data, span)
+      }
       this.addResponseTags(span, response)
       this.finish(span, response, response.error)
     })

diff --git a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js
@@ -52,7 +52,7 @@ class Kinesis extends BaseAwsSdkPlugin {
 
         // extract DSM context after as we might not have a parent-child but may have a DSM context
         this.responseExtractDSMContext(
-          request.operation, response, span || null, streamName
+          request.operation, request.params, response, span || null, { streamName }
         )
       }
     })
@@ -100,7 +100,8 @@ class Kinesis extends BaseAwsSdkPlugin {
     }
   }
 
-  responseExtractDSMContext (operation, response, span, streamName) {
+  responseExtractDSMContext (operation, params, response, span, kwargs = {}) {
+    const { streamName } = kwargs
     if (!this.config.dsmEnabled) return
     if (operation !== 'getRecords') return
     if (!response || !response.Records || !response.Records[0]) return

diff --git a/packages/datadog-plugin-aws-sdk/src/services/sqs.js b/packages/datadog-plugin-aws-sdk/src/services/sqs.js
@@ -23,7 +23,7 @@ class Sqs extends BaseAwsSdkPlugin {
       const plugin = this
       const contextExtraction = this.responseExtract(request.params, request.operation, response)
       let span
-      let parsedMessageAttributes
+      let parsedMessageAttributes = null
       if (contextExtraction && contextExtraction.datadogContext) {
         obj.needsFinish = true
         const options = {
@@ -39,8 +39,9 @@ class Sqs extends BaseAwsSdkPlugin {
         this.enter(span, store)
       }
       // extract DSM context after as we might not have a parent-child but may have a DSM context
+
       this.responseExtractDSMContext(
-        request.operation, request.params, response, span || null, parsedMessageAttributes || null
+        request.operation, request.params, response, span || null, { parsedMessageAttributes }
       )
     })
 
@@ -165,7 +166,8 @@ class Sqs extends BaseAwsSdkPlugin {
     }
   }
 
-  responseExtractDSMContext (operation, params, response, span, parsedAttributes) {
+  responseExtractDSMContext (operation, params, response, span, kwargs = {}) {
+    let { parsedAttributes } = kwargs
     if (!this.config.dsmEnabled) return
     if (operation !== 'receiveMessage') return
     if (!response || !response.Messages || !response.Messages[0]) return
@@ -188,7 +190,7 @@ class Sqs extends BaseAwsSdkPlugin {
             // SQS to SQS
           }
         }
-        if (message.MessageAttributes && message.MessageAttributes._datadog) {
+        if (!parsedAttributes && message.MessageAttributes && message.MessageAttributes._datadog) {
           parsedAttributes = this.parseDatadogAttributes(message.MessageAttributes._datadog)
         }
       }
@@ -219,6 +221,17 @@ class Sqs extends BaseAwsSdkPlugin {
           this.injectToMessage(span, params.Entries[i], params.QueueUrl, i === 0)
         }
         break
+      case 'receiveMessage':
+        if (!params.MessageAttributeNames) {
+          params.MessageAttributeNames = ['_datadog']
+        } else if (
+          !params.MessageAttributeNames.includes('_datadog') &&
+          !params.MessageAttributeNames.includes('.*') &&
+          !params.MessageAttributeNames.includes('All')
+        ) {
+          params.MessageAttributeNames.push('_datadog')
+        }
+        break
     }
   }
 

diff --git a/packages/datadog-plugin-aws-sdk/test/sqs.spec.js b/packages/datadog-plugin-aws-sdk/test/sqs.spec.js
@@ -3,6 +3,7 @@
 const sinon = require('sinon')
 const agent = require('../../dd-trace/test/plugins/agent')
 const { setup } = require('./spec_helpers')
+const semver = require('semver')
 const { rawExpectedSchema } = require('./sqs-naming')
 
 const queueName = 'SQS_QUEUE_NAME'
@@ -408,6 +409,34 @@ describe('Plugin', () => {
           })
         })
 
+        if (sqsClientName === 'aws-sdk' && semver.intersects(version, '>=2.3')) {
+          it('Should set pathway hash tag on a span when consuming and promise() was used over a callback',
+            async () => {
+              await sqs.sendMessage({ MessageBody: 'test DSM', QueueUrl: QueueUrlDsm })
+              await sqs.receiveMessage({ QueueUrl: QueueUrlDsm }).promise()
+
+              let consumeSpanMeta = {}
+              return new Promise((resolve, reject) => {
+                agent.use(traces => {
+                  const span = traces[0][0]
+
+                  if (span.name === 'aws.request' && span.meta['aws.operation'] === 'receiveMessage') {
+                    consumeSpanMeta = span.meta
+                  }
+
+                  try {
+                    expect(consumeSpanMeta).to.include({
+                      'pathway.hash': expectedConsumerHash
+                    })
+                    resolve()
+                  } catch (error) {
+                    reject(error)
+                  }
+                })
+              })
+            })
+        }
+
         it('Should emit DSM stats to the agent when sending a message', done => {
           agent.expectPipelineStats(dsmStats => {
             let statsPointsReceived = 0

diff --git a/packages/datadog-plugin-dns/test/index.spec.js b/packages/datadog-plugin-dns/test/index.spec.js
@@ -142,19 +142,19 @@ describe('Plugin', () => {
             expect(traces[0][0]).to.deep.include({
               name: 'dns.resolve',
               service: 'test',
-              resource: 'ANY lvh.me'
+              resource: 'ANY localhost'
             })
             expect(traces[0][0].meta).to.deep.include({
               component: 'dns',
               'span.kind': 'client',
-              'dns.hostname': 'lvh.me',
+              'dns.hostname': 'localhost',
               'dns.rrtype': 'ANY'
             })
           })
           .then(done)
           .catch(done)
 
-        dns.resolveAny('lvh.me', err => err && done(err))
+        dns.resolveAny('localhost', () => done())
       })
 
       it('should instrument reverse', done => {

diff --git a/packages/datadog-plugin-express/test/index.spec.js b/packages/datadog-plugin-express/test/index.spec.js
@@ -114,6 +114,7 @@ describe('Plugin', () => {
                 expect(spans[0].meta).to.have.property('http.url', `http://localhost:${port}/user`)
                 expect(spans[0].meta).to.have.property('http.method', 'GET')
                 expect(spans[0].meta).to.have.property('http.status_code', '200')
+                expect(spans[0].meta).to.have.property('http.route', '/user')
               })
               .then(done)
               .catch(done)
@@ -1269,6 +1270,40 @@ describe('Plugin', () => {
           })
         })
 
+        it('should handle 404 errors', done => {
+          const app = express()
+
+          app.use((req, res, next) => {
+            next()
+          })
+
+          app.get('/does-exist', (req, res) => {
+            res.status(200).send('hi')
+          })
+
+          getPort().then(port => {
+            agent.use(traces => {
+              const spans = sort(traces[0])
+
+              expect(spans[0]).to.have.property('error', 0)
+              expect(spans[0]).to.have.property('resource', 'GET')
+              expect(spans[0].meta).to.have.property('http.status_code', '404')
+              expect(spans[0].meta).to.have.property('component', 'express')
+              expect(spans[0].meta).to.not.have.property('http.route')
+
+              done()
+            })
+
+            appListener = app.listen(port, 'localhost', () => {
+              axios
+                .get(`http://localhost:${port}/does-not-exist`, {
+                  validateStatus: status => status === 404
+                })
+                .catch(done)
+            })
+          })
+        })
+
         withVersions(plugin, 'loopback', loopbackVersion => {
           let loopback
 

diff --git a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js
@@ -1,14 +1,14 @@
 'use strict'
 
 const { MANUAL_KEEP } = require('../../../../../ext/tags')
-const LRU = require('lru-cache')
+const { LRUCache } = require('lru-cache')
 const vulnerabilitiesFormatter = require('./vulnerabilities-formatter')
 const { IAST_ENABLED_TAG_KEY, IAST_JSON_TAG_KEY } = require('./tags')
 const standalone = require('../standalone')
 
 const VULNERABILITIES_KEY = 'vulnerabilities'
 const VULNERABILITY_HASHES_MAX_SIZE = 1000
-const VULNERABILITY_HASHES = new LRU({ max: VULNERABILITY_HASHES_MAX_SIZE })
+const VULNERABILITY_HASHES = new LRUCache({ max: VULNERABILITY_HASHES_MAX_SIZE })
 const RESET_VULNERABILITY_CACHE_INTERVAL = 60 * 60 * 1000 // 1 hour
 
 let tracer

diff --git a/packages/dd-trace/src/datastreams/pathway.js b/packages/dd-trace/src/datastreams/pathway.js
@@ -3,7 +3,7 @@
 // this inconsistency is ok because hashes do not need to be consistent across services
 const crypto = require('crypto')
 const { encodeVarint, decodeVarint } = require('./encoding')
-const LRUCache = require('lru-cache')
+const { LRUCache } = require('lru-cache')
 
 const options = { max: 500 }
 const cache = new LRUCache(options)

diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server-express.plugin.spec.js
@@ -1,6 +1,5 @@
 'use strict'
 
-const getPort = require('get-port')
 const agent = require('../plugins/agent')
 const {
   schema,
@@ -41,10 +40,9 @@ withVersions('apollo-server-core', 'express', '>=4', expressVersion => {
 
       server.applyMiddleware({ app })
 
-      config.port = await getPort()
-
       return new Promise(resolve => {
         expressServer = app.listen({ port: config.port }, () => {
+          config.port = expressServer.address().port
           resolve()
         })
       })

diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server-fastify.plugin.spec.js
@@ -1,6 +1,5 @@
 'use strict'
 
-const getPort = require('get-port')
 const agent = require('../plugins/agent')
 const {
   schema,
@@ -41,10 +40,9 @@ withVersions('apollo-server-core', 'fastify', '3', fastifyVersion => {
 
       app.register(server.createHandler())
 
-      config.port = await getPort()
-
       return new Promise(resolve => {
         app.listen({ port: config.port }, (data) => {
+          config.port = app.server.address().port
           resolve()
         })
       })

diff --git a/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js b/packages/dd-trace/test/appsec/graphql.apollo-server.plugin.spec.js
@@ -1,6 +1,5 @@
 'use strict'
 
-const getPort = require('get-port')
 const path = require('path')
 const agent = require('../plugins/agent')
 const {
@@ -31,9 +30,9 @@ withVersions('apollo-server', '@apollo/server', apolloServerVersion => {
       resolvers
     })
 
-    config.port = await getPort()
+    const { url } = await startStandaloneServer(server, { listen: { port: 0 } })
 
-    await startStandaloneServer(server, { listen: { port: config.port } })
+    config.port = new URL(url).port
   })
 
   after(async () => {

diff --git a/...trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js b/...trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server-express.plugin.spec.js
@@ -1,6 +1,5 @@
 'use strict'
 
-const getPort = require('get-port')
 const agent = require('../../../../plugins/agent')
 const {
   schema,
@@ -41,10 +40,9 @@ withVersions('graphql', 'express', '>=4', expressVersion => {
 
       server.applyMiddleware({ app })
 
-      config.port = await getPort()
-
       return new Promise(resolve => {
         expressServer = app.listen({ port: config.port }, () => {
+          config.port = expressServer.address().port
           resolve()
         })
       })

diff --git a/...ages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js b/...ages/dd-trace/test/appsec/iast/taint-tracking/sources/plugin.apollo-server.plugin.spec.js
@@ -1,6 +1,5 @@
 'use strict'
 
-const getPort = require('get-port')
 const path = require('path')
 const agent = require('../../../../plugins/agent')
 const {
@@ -31,9 +30,9 @@ withVersions('apollo-server', '@apollo/server', apolloServerVersion => {
       resolvers
     })
 
-    config.port = await getPort()
+    const { url } = await startStandaloneServer(server, { listen: { port: config.port } })
 
-    await startStandaloneServer(server, { listen: { port: config.port } })
+    config.port = new URL(url).port
   })
 
   after(async () => {