Skip to content

Commit

Permalink
[mongo] Remove comment from obfuscate command and send comment as a s…
Browse files Browse the repository at this point in the history 
…eparate field in payload (DataDog#18404)
  • Loading branch information
@lu-zhengda
lu-zhengda authored Aug 27, 2024
1 parent 58d7d5e commit cebb45c
Show file tree
Hide file tree
Showing 14 changed files with 170 additions and 105 deletions.
1 change: 1 addition & 0 deletions mongo/changelog.d/18404.fixed
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Remove `comment` from obfuscate command and send it as a separate field in operation samples and slow operations payload.
8 changes: 6 additions & 2 deletions mongo/datadog_checks/mongo/dbm/operation_samples.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
get_command_collection,
get_command_truncation_state,
get_explain_plan,
obfuscate_command,
should_explain_operation,
)

Expand Down Expand Up @@ -115,7 +116,7 @@ def _get_operation_samples(self, now, databases_monitored: List[str]):
continue

command = operation.get("command")
obfuscated_command = datadog_agent.obfuscate_mongodb_string(json_util.dumps(command))
obfuscated_command = obfuscate_command(command)
query_signature = self._get_query_signature(obfuscated_command)
operation_metadata = self._get_operation_metadata(operation)

Expand Down Expand Up @@ -243,8 +244,10 @@ def _get_operation_cursor(self, operation: dict) -> Optional[OperationSampleOper
last_access_date = last_access_date.isoformat()

originating_command = cursor.get("originatingCommand")
originating_command_comment = None
if originating_command:
originating_command = datadog_agent.obfuscate_mongodb_string(json_util.dumps(originating_command))
originating_command_comment = originating_command.get("comment")
originating_command = obfuscate_command(originating_command)

return {
"cursor_id": cursor.get("cursorId"),
Expand All @@ -256,6 +259,7 @@ def _get_operation_cursor(self, operation: dict) -> Optional[OperationSampleOper
"tailable": cursor.get("tailable", False),
"await_data": cursor.get("awaitData", False),
"originating_command": originating_command,
"comment": originating_command_comment,
"plan_summary": cursor.get("planSummary"),
"operation_using_cursor_id": cursor.get("operationUsingCursorId"),
}
Expand Down
13 changes: 8 additions & 5 deletions mongo/datadog_checks/mongo/dbm/slow_operations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
get_command_collection,
get_command_truncation_state,
get_explain_plan,
obfuscate_command,
should_explain_operation,
)

Expand Down Expand Up @@ -190,16 +191,16 @@ def _collect_slow_operation_explain_plan(self, slow_operation, dbname):
self._check.log.error("Failed to collect explain plan for slow operation: %s", e)

def _obfuscate_slow_operation(self, slow_operation, db_name):
obfuscated_command = datadog_agent.obfuscate_mongodb_string(json_util.dumps(slow_operation["command"]))
obfuscated_command = obfuscate_command(slow_operation["command"])
query_signature = compute_exec_plan_signature(obfuscated_command)
slow_operation['dbname'] = db_name
slow_operation['obfuscated_command'] = obfuscated_command
slow_operation['query_signature'] = query_signature

if slow_operation.get('originatingCommand'):
slow_operation['originatingCommand'] = datadog_agent.obfuscate_mongodb_string(
json_util.dumps(slow_operation['originatingCommand'])
)
originating_command = slow_operation.get('originatingCommand')
if originating_command:
slow_operation['originatingCommandComment'] = originating_command.get('comment')
slow_operation['originatingCommand'] = obfuscate_command(originating_command)

return slow_operation

Expand Down Expand Up @@ -242,6 +243,7 @@ def _create_slow_operation_event(self, slow_operation):
"query_hash": slow_operation.get("queryHash"), # only available with profiling
"plan_cache_key": slow_operation.get("planCacheKey"), # only available with profiling
"query_framework": slow_operation.get("queryFramework"),
"comment": slow_operation["command"].get("comment"),
# metrics
# mills from profiler, durationMillis from logs
"mills": slow_operation.get("millis", slow_operation.get("durationMillis", 0)),
Expand Down Expand Up @@ -337,6 +339,7 @@ def _get_slow_operation_cursor(self, slow_operation):
return {
"cursor_id": cursor_id,
"originating_command": originating_command,
"comment": slow_operation.get("originatingCommandComment"),
}
return None

Expand Down
18 changes: 18 additions & 0 deletions mongo/datadog_checks/mongo/dbm/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@
from datadog_checks.base.utils.db.sql import compute_exec_plan_signature
from datadog_checks.base.utils.db.utils import RateLimitingTTLCache

try:
import datadog_agent
except ImportError:
from datadog_checks.base.stubs import datadog_agent


MONGODB_SYSTEM_DATABASES = frozenset(["admin", "config", "local"])

# exclude keys in sampled operation that cause issues with the explain command
Expand Down Expand Up @@ -150,3 +156,15 @@ def get_command_collection(command: dict, ns: str) -> Optional[str]:
if collection and isinstance(collection, str): # edge case like {"aggregate": 1}
return collection
return None


def obfuscate_command(command: dict):
# Obfuscate the command to remove sensitive information
# Remove the following keys from the command before obfuscating
# - comment: The comment field should not contribute to the query signature
# - lsid: The lsid field is a unique identifier for the session
# - $clusterTime: The $clusterTime field is a logical time used for ordering of operations
obfuscated_command = command.copy()
for key in ("comment", "lsid", "$clusterTime"):
obfuscated_command.pop(key, None)
return datadog_agent.obfuscate_mongodb_string(json_util.dumps(obfuscated_command))
1 change: 1 addition & 0 deletions mongo/tests/fixtures/$currentOp-mongos
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,7 @@
}
}
],
"comment": "get more movies",
"cursor": {
"batchSize": 0
},
Expand Down
1 change: 1 addition & 0 deletions mongo/tests/fixtures/$currentOp-standalone
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@
}
}
},
"comment": "oplogReplay",
"batchSize": 13981010,
"tailable": true,
"awaitData": true,
Expand Down
22 changes: 12 additions & 10 deletions mongo/tests/results/operation-activities-mongos.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
"mongodb_activity": [
{
"now": 1715911398.1112723,
"query_signature": "6324f030638a5dc0",
"statement": "{\"find\": \"users\", \"projection\": {\"$sortKey\": {\"$meta\": \"sortKey\"}}, \"sort\": {\"user_id\": 1}, \"needsMerge\": true, \"let\": {\"CLUSTER_TIME\": {\"$literal\": {\"t\": 1716215816, \"i\": 1}}, \"NOW\": {\"$literal\": \"2024-05-20T14:36:56.231+00:00\"}}, \"fromMongos\": true, \"singleBatch\": false, \"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"implicitDefault\"}, \"readConcern\": {\"level\": \"local\", \"provenance\": \"implicitDefault\"}, \"shardVersion\": {\"t\": {\"$timestamp\": {\"t\": 1716215708, \"i\": 12}}, \"e\": {\"$oid\": \"664b5f9d05fee29701ab2bfb\"}, \"v\": {\"$timestamp\": {\"t\": 1, \"i\": 7}}}, \"clientOperationKey\": {\"$binary\": {\"base64\": \"cZAjiXzbSZianZ+RrPdTaA==\", \"subType\": \"04\"}}, \"comment\": \"sort\", \"lsid\": {\"id\": {\"$binary\": {\"base64\": \"/5W615FQTDardJroxkDVTg==\", \"subType\": \"04\"}}, \"uid\": {\"$binary\": {\"base64\": \"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\", \"subType\": \"00\"}}}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1716215816, \"i\": 1}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"$configTime\": {\"$timestamp\": {\"t\": 1716215814, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716215696, \"i\": 24}}, \"$audit\": {\"$impersonatedUsers\": [{\"user\": \"test\", \"db\": \"admin\"}], \"$impersonatedRoles\": [{\"role\": \"root\", \"db\": \"admin\"}]}, \"$client\": {\"driver\": {\"name\": \"PyMongo\", \"version\": \"4.7.2\"}, \"os\": {\"type\": \"Darwin\", \"name\": \"Darwin\", \"architecture\": \"arm64\", \"version\": \"14.4.1\"}, \"platform\": \"CPython 3.9.16.final.0\", \"mongos\": {\"host\": \"7203110cbd0b:27017\", \"client\": \"192.168.65.1:39545\", \"version\": \"6.0.15\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"integration\"}",
"query_signature": "50114961ffa245f7",
"statement": "{\"find\": \"users\", \"projection\": {\"$sortKey\": {\"$meta\": \"sortKey\"}}, \"sort\": {\"user_id\": 1}, \"needsMerge\": true, \"let\": {\"CLUSTER_TIME\": {\"$literal\": {\"t\": 1716215816, \"i\": 1}}, \"NOW\": {\"$literal\": \"2024-05-20T14:36:56.231+00:00\"}}, \"fromMongos\": true, \"singleBatch\": false, \"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"implicitDefault\"}, \"readConcern\": {\"level\": \"local\", \"provenance\": \"implicitDefault\"}, \"shardVersion\": {\"t\": {\"$timestamp\": {\"t\": 1716215708, \"i\": 12}}, \"e\": {\"$oid\": \"664b5f9d05fee29701ab2bfb\"}, \"v\": {\"$timestamp\": {\"t\": 1, \"i\": 7}}}, \"clientOperationKey\": {\"$binary\": {\"base64\": \"cZAjiXzbSZianZ+RrPdTaA==\", \"subType\": \"04\"}}, \"$configTime\": {\"$timestamp\": {\"t\": 1716215814, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716215696, \"i\": 24}}, \"$audit\": {\"$impersonatedUsers\": [{\"user\": \"test\", \"db\": \"admin\"}], \"$impersonatedRoles\": [{\"role\": \"root\", \"db\": \"admin\"}]}, \"$client\": {\"driver\": {\"name\": \"PyMongo\", \"version\": \"4.7.2\"}, \"os\": {\"type\": \"Darwin\", \"name\": \"Darwin\", \"architecture\": \"arm64\", \"version\": \"14.4.1\"}, \"platform\": \"CPython 3.9.16.final.0\", \"mongos\": {\"host\": \"7203110cbd0b:27017\", \"client\": \"192.168.65.1:39545\", \"version\": \"6.0.15\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"integration\"}",
"active": true,
"desc": "conn143",
"opid": "shard04:265665",
Expand Down Expand Up @@ -84,8 +84,8 @@
},
{
"now": 1715911398.1112723,
"query_signature": "c9f4db9be9cc6b96",
"statement": "{\"getMore\": 7153547462305880513, \"collection\": \"movies\", \"lsid\": {\"id\": {\"$binary\": {\"base64\": \"zLd+DNpkQdqkWN1AZ828SA==\", \"subType\": \"04\"}}, \"uid\": {\"$binary\": {\"base64\": \"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\", \"subType\": \"00\"}}}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"$configTime\": {\"$timestamp\": {\"t\": 1718311809, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716428552, \"i\": 3}}, \"$audit\": {\"$impersonatedUser\": {\"user\": \"shopper_4\", \"db\": \"admin\"}, \"$impersonatedRoles\": [{\"role\": \"readWrite\", \"db\": \"dbmorders\"}, {\"role\": \"readWrite\", \"db\": \"integration\"}]}, \"$client\": {\"application\": {\"name\": \"orders-mongo\"}, \"driver\": {\"name\": \"mongo-go-driver\", \"version\": \"v1.15.0\"}, \"os\": {\"type\": \"linux\", \"architecture\": \"amd64\"}, \"platform\": \"go1.22.1\", \"env\": {\"container\": {\"runtime\": \"docker\"}}, \"mongos\": {\"host\": \"ip-172-29-203-167:27017\", \"client\": \"127.0.0.1:50388\", \"version\": \"7.0.9\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"integration\"}",
"query_signature": "dff032249380b09d",
"statement": "{\"getMore\": 7153547462305880513, \"collection\": \"movies\", \"$configTime\": {\"$timestamp\": {\"t\": 1718311809, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716428552, \"i\": 3}}, \"$audit\": {\"$impersonatedUser\": {\"user\": \"shopper_4\", \"db\": \"admin\"}, \"$impersonatedRoles\": [{\"role\": \"readWrite\", \"db\": \"dbmorders\"}, {\"role\": \"readWrite\", \"db\": \"integration\"}]}, \"$client\": {\"application\": {\"name\": \"orders-mongo\"}, \"driver\": {\"name\": \"mongo-go-driver\", \"version\": \"v1.15.0\"}, \"os\": {\"type\": \"linux\", \"architecture\": \"amd64\"}, \"platform\": \"go1.22.1\", \"env\": {\"container\": {\"runtime\": \"docker\"}}, \"mongos\": {\"host\": \"ip-172-29-203-167:27017\", \"client\": \"127.0.0.1:50388\", \"version\": \"7.0.9\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"integration\"}",
"active": true,
"desc": "conn703087",
"opid": "shard2:155914227",
Expand Down Expand Up @@ -132,9 +132,10 @@
"no_cursor_timeout": false,
"tailable": false,
"await_data": false,
"originating_command": "{\"aggregate\": \"movies\", \"pipeline\": [{\"$lookup\": {\"from\": \"comments\", \"as\": \"comments\", \"localField\": \"_id\", \"foreignField\": \"movie_id\"}}], \"cursor\": {\"batchSize\": 0}, \"needsMerge\": true, \"let\": {\"NOW\": {\"$literal\": {\"$date\": \"2024-06-13T20:50:10.805Z\"}}, \"CLUSTER_TIME\": {\"$literal\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}}}, \"fromMongos\": true, \"collation\": {\"locale\": \"simple\"}, \"readConcern\": {\"level\": \"local\", \"provenance\": \"implicitDefault\"}, \"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"implicitDefault\"}, \"shardVersion\": {\"e\": {\"$oid\": \"666b52b852fa5a96da6d59ae\"}, \"t\": {\"$timestamp\": {\"t\": 1718309560, \"i\": 188}}, \"v\": {\"$timestamp\": {\"t\": 1, \"i\": 5}}}, \"clientOperationKey\": {\"$binary\": {\"base64\": \"zc+70kUAQ5m1zVRdJjwYxw==\", \"subType\": \"04\"}}, \"lsid\": {\"id\": {\"$binary\": {\"base64\": \"zLd+DNpkQdqkWN1AZ828SA==\", \"subType\": \"04\"}}, \"uid\": {\"$binary\": {\"base64\": \"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=\", \"subType\": \"00\"}}}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"$configTime\": {\"$timestamp\": {\"t\": 1718311809, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716428552, \"i\": 3}}, \"$audit\": {\"$impersonatedUser\": {\"user\": \"shopper_4\", \"db\": \"admin\"}, \"$impersonatedRoles\": [{\"role\": \"readWrite\", \"db\": \"dbmorders\"}, {\"role\": \"readWrite\", \"db\": \"integration\"}]}, \"$client\": {\"application\": {\"name\": \"orders-mongo\"}, \"driver\": {\"name\": \"mongo-go-driver\", \"version\": \"v1.15.0\"}, \"os\": {\"type\": \"linux\", \"architecture\": \"amd64\"}, \"platform\": \"go1.22.1\", \"env\": {\"container\": {\"runtime\": \"docker\"}}, \"mongos\": {\"host\": \"ip-172-29-203-167:27017\", \"client\": \"127.0.0.1:50388\", \"version\": \"7.0.9\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"integration\"}",
"originating_command": "{\"aggregate\": \"movies\", \"pipeline\": [{\"$lookup\": {\"from\": \"comments\", \"as\": \"comments\", \"localField\": \"_id\", \"foreignField\": \"movie_id\"}}], \"cursor\": {\"batchSize\": 0}, \"needsMerge\": true, \"let\": {\"NOW\": {\"$literal\": {\"$date\": \"2024-06-13T20:50:10.805Z\"}}, \"CLUSTER_TIME\": {\"$literal\": {\"$timestamp\": {\"t\": 1718311810, \"i\": 6}}}}, \"fromMongos\": true, \"collation\": {\"locale\": \"simple\"}, \"readConcern\": {\"level\": \"local\", \"provenance\": \"implicitDefault\"}, \"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"implicitDefault\"}, \"shardVersion\": {\"e\": {\"$oid\": \"666b52b852fa5a96da6d59ae\"}, \"t\": {\"$timestamp\": {\"t\": 1718309560, \"i\": 188}}, \"v\": {\"$timestamp\": {\"t\": 1, \"i\": 5}}}, \"clientOperationKey\": {\"$binary\": {\"base64\": \"zc+70kUAQ5m1zVRdJjwYxw==\", \"subType\": \"04\"}}, \"$configTime\": {\"$timestamp\": {\"t\": 1718311809, \"i\": 1}}, \"$topologyTime\": {\"$timestamp\": {\"t\": 1716428552, \"i\": 3}}, \"$audit\": {\"$impersonatedUser\": {\"user\": \"shopper_4\", \"db\": \"admin\"}, \"$impersonatedRoles\": [{\"role\": \"readWrite\", \"db\": \"dbmorders\"}, {\"role\": \"readWrite\", \"db\": \"integration\"}]}, \"$client\": {\"application\": {\"name\": \"orders-mongo\"}, \"driver\": {\"name\": \"mongo-go-driver\", \"version\": \"v1.15.0\"}, \"os\": {\"type\": \"linux\", \"architecture\": \"amd64\"}, \"platform\": \"go1.22.1\", \"env\": {\"container\": {\"runtime\": \"docker\"}}, \"mongos\": {\"host\": \"ip-172-29-203-167:27017\", \"client\": \"127.0.0.1:50388\", \"version\": \"7.0.9\"}}, \"mayBypassWriteBlocking\": true, \"$db\": \"integration\"}",
"plan_summary": "COLLSCAN",
"operation_using_cursor_id": 155914227
"operation_using_cursor_id": 155914227,
"comment": "get more movies"
},
"type": "op",
"op": "getmore",
Expand All @@ -160,8 +161,8 @@
},
{
"now": 1715911398.1112723,
"query_signature": "8e567c5655c884d0",
"statement": "{\"getMore\": 7156629711810917986, \"collection\": \"oplog.rs\", \"batchSize\": 13981010, \"maxTimeMS\": 5000, \"term\": 2, \"lastKnownCommittedOpTime\": {\"ts\": {\"$timestamp\": {\"t\": 1720208938, \"i\": 17}}, \"t\": 2}, \"$replData\": 1, \"$oplogQueryData\": 1, \"$readPreference\": {\"mode\": \"secondaryPreferred\"}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1720208938, \"i\": 18}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"fAxz3pScinFgqdNkVCRZGX3goGY=\", \"subType\": \"00\"}}, \"keyId\": 7387155612372566021}}, \"$db\": \"local\"}",
"query_signature": "2098271ac4fceefe",
"statement": "{\"getMore\": 7156629711810917986, \"collection\": \"oplog.rs\", \"batchSize\": 13981010, \"maxTimeMS\": 5000, \"term\": 2, \"lastKnownCommittedOpTime\": {\"ts\": {\"$timestamp\": {\"t\": 1720208938, \"i\": 17}}, \"t\": 2}, \"$replData\": 1, \"$oplogQueryData\": 1, \"$readPreference\": {\"mode\": \"secondaryPreferred\"}, \"$db\": \"local\"}",
"active": true,
"desc": "conn1392",
"opid": 1399253104,
Expand Down Expand Up @@ -200,9 +201,10 @@
"no_cursor_timeout": false,
"tailable": true,
"await_data": true,
"originating_command": "{\"find\": \"oplog.rs\", \"filter\": {\"ts\": {\"$gte\": {\"$timestamp\": {\"t\": 1720208938, \"i\": 17}}}}, \"batchSize\": 13981010, \"tailable\": true, \"awaitData\": true, \"term\": 2, \"maxTimeMS\": 60000, \"readConcern\": {\"level\": \"local\", \"afterClusterTime\": {\"$timestamp\": {\"t\": 0, \"i\": 1}}}, \"$replData\": 1, \"$oplogQueryData\": 1, \"$readPreference\": {\"mode\": \"secondaryPreferred\"}, \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1720208938, \"i\": 18}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"fAxz3pScinFgqdNkVCRZGX3goGY=\", \"subType\": \"00\"}}, \"keyId\": 7387155612372566021}}, \"$db\": \"local\"}",
"originating_command": "{\"find\": \"oplog.rs\", \"filter\": {\"ts\": {\"$gte\": {\"$timestamp\": {\"t\": 1720208938, \"i\": 17}}}}, \"batchSize\": 13981010, \"tailable\": true, \"awaitData\": true, \"term\": 2, \"maxTimeMS\": 60000, \"readConcern\": {\"level\": \"local\", \"afterClusterTime\": {\"$timestamp\": {\"t\": 0, \"i\": 1}}}, \"$replData\": 1, \"$oplogQueryData\": 1, \"$readPreference\": {\"mode\": \"secondaryPreferred\"}, \"$db\": \"local\"}",
"plan_summary": null,
"operation_using_cursor_id": 1399253104
"operation_using_cursor_id": 1399253104,
"comment": null
},
"type": "op",
"op": "getmore",
Expand Down
Loading

0 comments on commit cebb45c

Please sign in to comment.