From feb7bcedc690a4c5993f35096759cb2e934f0987 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Tue, 23 Jun 2020 10:49:25 +0100 Subject: [PATCH 01/14] memory management how to guide --- .../img/memory-management-movies-graph.svg | 1 + .../img/query-plan-with-memory.svg | 1 + .../memory-management/memory-management.adoc | 153 ++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 in-production/memory-management/img/memory-management-movies-graph.svg create mode 100644 in-production/memory-management/img/query-plan-with-memory.svg create mode 100644 in-production/memory-management/memory-management.adoc diff --git a/in-production/memory-management/img/memory-management-movies-graph.svg b/in-production/memory-management/img/memory-management-movies-graph.svg new file mode 100644 index 00000000..d8c209a7 --- /dev/null +++ b/in-production/memory-management/img/memory-management-movies-graph.svg @@ -0,0 +1 @@ +Neo4j Graph VisualizationCreated using Neo4j (http://www.neo4j.com/)ACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INDIRECTEDDIRECTEDDIRECTEDDIRECTEDDIRECTEDDIRECTEDPRODUCEDPRODUCEDPRODUCEDACTED_INACTED_INACTED_INDIRECTEDACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_INACTED_IN The Matrix Keanu Reeves Carrie-Anne Moss Laurence Fishburne Hugo Weaving Lilly Wachowski Lana Wachowski Joel Silver Emil Eifrem The Matrix Reloaded The Matrix Revolutions The Devil's Advocate Charlize Theron Al Pacino Taylor Hackford A Few Good Men Tom Cruise Jack Nicholson Demi Moore Kevin Bacon Kiefer Sutherland Noah Wyle Cuba Gooding Jr. Kevin Pollak J.T. Walsh As Good as It Gets Hoffa One Flew Over the Cuckoo's Nest Something's Gotta Give \ No newline at end of file diff --git a/in-production/memory-management/img/query-plan-with-memory.svg b/in-production/memory-management/img/query-plan-with-memory.svg new file mode 100644 index 00000000..729a5cef --- /dev/null +++ b/in-production/memory-management/img/query-plan-with-memory.svg @@ -0,0 +1 @@ +Neo4j Graph VisualizationCreated using Neo4j (http://www.neo4j.com/)133 rows115,305 rows33,440 rows33,440 rows33,440 rows33,440 rows7,890 rows7,890 rows7,890 rows7,890 rowsResultProduceResults@neo4jpath, `p1.name`, anon_142, anon_134, p2, anon_138, `[rel in relationships(path) | type(rel)]`, `p2.name`, p1`p1.name`, `p2.name`, `[rel in relationships(path) | type(rel)]`Ordered by p1 ASC, p2 ASC17,062,912 total memory (bytes)2 estimated rows0 db hitsProjection@neo4j46,424 db hitspath, `p1.name`, anon_142, anon_134, p2, anon_138, `[rel in relationships(path) | type(rel)]`, `p2.name`, p1p1.name AS `p1.name`, p2.name AS `p2.name`, [rel IN relationships(path) | type(rel)] AS `[rel in relationships(path) | type(rel)]`Ordered by p1 ASC, p2 ASC2 estimated rowsProjection@neo4jpath, anon_142, anon_134, p2, anon_138, p1anon_134 AS p1, anon_138 AS p2, anon_142[$autoint_0] AS pathOrdered by p1 ASC, p2 ASC2 estimated rows0 db hitsOrderedAggregation@neo4janon_134, anon_138, anon_142p1 AS anon_134, p2 AS anon_138, collect(path) AS anon_142Ordered by anon_134 ASC, anon_138 ASC376,272 memory (bytes)2 estimated rows0 db hitsProjection@neo4janon_64, path, p2, `length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep))))`, p1(p1)-[anon_64*]-(p2) AS pathOrdered by p1 ASC, p2 ASC, length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep)))) DESC5 estimated rows0 db hitsSort@neo4jp2, anon_64, p1, `length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep))))`p1 ASC, p2 ASC, `length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep))))` DESCOrdered by p1 ASC, p2 ASC, length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep)))) DESC16,687,456 memory (bytes)5 estimated rows0 db hitsProjection@neo4j134,704 db hitsp2, anon_64, p1, `length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep))))`length((p1)-[anon_64*]-(p2)) AS `length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep))))`5 estimated rowsFilter@neo4j114,163 db hitsp2, anon_64, p1not p1 = p2 AND p1:Person5 estimated rowsVarLengthExpand(All)@neo4j188,936 db hitsp2, anon_64, p1(p2)-[anon_64*..5]-(p1)256 estimated rowsNodeByLabelScan@neo4j134 db hitsp2p2:Person64 memory (bytes)133 estimated rows \ No newline at end of file diff --git a/in-production/memory-management/memory-management.adoc b/in-production/memory-management/memory-management.adoc new file mode 100644 index 00000000..79e73059 --- /dev/null +++ b/in-production/memory-management/memory-management.adoc @@ -0,0 +1,153 @@ += How To: Manage Memory in Neo4j +:slug: memory-management +:level: Intermediate +:section: Neo4j Administration +:section-link: in-production +:sectanchors: +:toc: +:toc-title: Contents +:toclevels: 1 + +.Goals +[abstract] +In this guide, we will learn how to both measure and restrict memory usage in Neo4j. + +.Prerequisites +[abstract] +Please have link:/download[Neo4j^] (version 4.1 or later) downloaded and installed. +Familiarity with the link:/developer/cypher-query-language/[Cypher query language^] is required to understand the examples used in this guide. + +[role=expertise] +{level} + +In Neo4j (v4.1+), we can measure and restrict the amount of memory used by transactions running on the server. +This feature can be used to ensure that we don't encounter `OutOfMemory` exceptions when running our workloads. + +[#movies-dataset] +== Movie dataset + +The examples in this guide are based on the built-in movies dataset. +This can be loaded by running the `:play movies` command in the Neo4j Browser and following the instructions to import the dataset. +We can see a sample of the graph in the Neo4j Browser visualization below: + +image::{img}memory-management-movies-graph.svg[link="{img}movies-graph.svg",role="popup-link"] + +[#measure-memory-usage-tx] +== Measuring memory usage of one transaction + +We can measure the memory usage of individual Cypher queries when https://neo4j.com/docs/cypher-manual/current/query-tuning/how-do-i-profile-a-query/[profiling them^] using the `PROFILE` clause. + +The following query takes all pairs of people and returns the longest path between the pair of nodes up to a maximum distance of 5 hops. + +[source,cypher] +---- +PROFILE +MATCH (p1:Person), (p2:Person) +WHERE p1 <> p2 +MATCH path = (p1)-[*..5]-(p2) +WITH p1, p2, path +ORDER BY p1, p2, length(path) DESC +WITH p1, p2, collect(path)[0] AS path +RETURN p1.name, p2.name, [rel in relationships(path) | type(rel)]; +---- + +We can run this query in the https://neo4j.com/developer/neo4j-browser/[Neo4j Browser^], which returns a visual representation of the query plan that contains memory usage information. +We can see the output from running this query in the image below: + +image::{img}query-plan-with-memory.svg[link="{img}query-plan-with-memory",role="popup-link"] + +The total memory usage of 17,062,912 bytes is included on the `ProduceResults` operator. +That memory usage is broken down across the following operators: + +* `OrderedAggregation` - 376,272 bytes +* `Sort` - 16,687,456 bytes +* `NodeByLabelScan` - 64 bytes + +Alternatively, we can run the query in the https://neo4j.com/docs/operations-manual/current/tools/cypher-shell/[Cypher Shell^], which returns the following output (results excluded for brevity): + +[options="header"] +|=== +| Plan | Statement | Version | Planner | Runtime | Time | DbHits | Rows | Memory (Bytes) +| "PROFILE" | "READ_ONLY" | "CYPHER 4.1" | "COST" | "PIPELINED" | 68 | 484361 | 7890 | 17062912 +|=== + +[options="header", separator=¦] +|=== +¦ Operator ¦ Details ¦ Estimated Rows ¦ Rows ¦ DB Hits ¦ Time (ms) ¦ Memory (Bytes) ¦ Ordered by ¦ Other +¦ +ProduceResults@memorymanagement ¦ `p1.name`, `p2.name`, `[rel in relationships(path) | type(rel)]` ¦ 2 ¦ 7890 ¦ 0 ¦ 16.498 ¦ ¦ p1 ASC, p2 ASC ¦ 16498332; In Pipeline 2 +¦ +Projection@memorymanagement ¦ p1.name AS `p1.name`, p2.name AS `p2.name`, [rel IN relationships(path) | type(rel)] AS `[rel in relationships(path) | type(rel)]` ¦ 2 ¦ 7890 ¦ 46424 ¦ 48.497 ¦ ¦ p1 ASC, p2 ASC ¦ In Pipeline 2; 48497263 +¦ +Projection@memorymanagement ¦ anon_134 AS p1, anon_138 AS p2, anon_142[$autoint_0] AS path ¦ 2 ¦ 7890 ¦ 0 ¦ 5.987 ¦ ¦ p1 ASC, p2 ASC ¦ In Pipeline 2; 5986820 +¦ +OrderedAggregation@memorymanagement ¦ p1 AS anon_134, p2 AS anon_138, collect(path) AS anon_142 ¦ 2 ¦ 7890 ¦ 0 ¦ 26.009 ¦ 376272 ¦ anon_134 ASC, anon_138 ASC ¦ In Pipeline 2; 26009135 +¦ +Projection@memorymanagement ¦ (p1)-[anon_64*]-(p2) AS path ¦ 5 ¦ 33440 ¦ 0 ¦ 54.526 ¦ ¦ p1 ASC, p2 ASC, length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep)))) DESC ¦ In Pipeline 1; 54526010 +¦ +Sort@memorymanagement ¦ p1 ASC, p2 ASC, `length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep))))` DESC ¦ 5 ¦ 33440 ¦ 0 ¦ 96.382 ¦ 16687456 ¦ p1 ASC, p2 ASC, length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep)))) DESC ¦ In Pipeline 1; 96381994 +¦ +Projection@memorymanagement ¦ length((p1)-[anon_64*]-(p2)) AS `length(PathExpression(NodePathStep(Variable(p1),MultiRelationshipPathStep(Variable(anon_64),BOTH,Some(Variable(p2)),NilPathStep))))` ¦ 5 ¦ 33440 ¦ 134704 ¦ ¦ ¦ ¦ Fused in Pipeline 0 +¦ +Filter@memorymanagement ¦ not p1 = p2 AND p1:Person ¦ 5 ¦ 33440 ¦ 114163 ¦ ¦ ¦ ¦ Fused in Pipeline 0 +¦ +VarLengthExpand(All)@memorymanagement ¦ (p2)-[anon_64*..5]-(p1) ¦ 256 ¦ 115305 ¦ 188936 ¦ ¦ ¦ ¦ Fused in Pipeline 0 +|=== + +The output from Cypher Shell contains the total memory usage information in a separate summary table, rather than including it as part of the final operator. +The summary table contains a column indicating the memory usage of 17062184 bytes or 17MB. + +[#measure-memory-usage-server] +== Measuring memory usage on server + +We can also measure the memory usage on the Neo4j server using the following procedures: + +|=== +| Procedure | Description +|`CALL dbms.listPools()` | describes thread pool memory usage +|`CALL dbms.listTransactions()` | describes memory usage by running transactions +|`CALL dbms.listQueries()` | describes memory usage by running queries +|=== + +For example, we can see the memory usage when our all pairs of people query is running, by executing the following query: + +[source,cypher] +---- +CALL dbms.listQueries() +YIELD queryId, username, database, query, allocatedBytes +RETURN queryId, username, database, query, allocatedBytes; +---- + +[options="header"] +|=== +| queryId | username | database | query | allocatedBytes +| "query-32" | "neo4j" | "memorymanagement" | " PROFILE MATCH (p1:Person), (p2:Person) WHERE p1 <> p2 MATCH path = (p1)-[*..5]-(p2) WITH p1, p2, path ORDER BY p1, p2, length(path) DESC WITH p1, p2, collect(path)[0] AS path RETURN p1.name, p2.name, [rel in relationships(path) \| type(rel)];" | 3234176 +| "query-34" | "neo4j" | "neo4j" | "CALL dbms.listQueries() YIELD queryId, username, database, query, allocatedBytes RETURN queryId, username, database, query, allocatedBytes" | 64 +|=== + +At the time that we ran this query, our all pairs of people query was only using 3,234,176 bytes of memory out of the 17,062,912 that we know it uses in total. + + +[#restrict-memory-usage] +== Restricting memory usage + +We can restrict the amount of heap memory available to transactions by specifying the https://neo4j.com/docs/operations-manual/4.1/performance/memory-configuration/#memory-configuration-limit-transaction-memory[following config settings^] in `$NEO4J_HOME/neo4j.conf`. + +[options="header"] +|=== +| Setting | Description +|`dbms.memory.transaction.global_max_size` | configures the global maximum memory usage for all of the transactions running on the server. +|`dbms.memory.transaction.database_max_size` | limits the transaction memory usage per database +|`dbms.memory.transaction.max_size` | limits the memory usage per transaction +|=== + +If we want to restrict the amount of memory used by an individual transaction to 10MB, we can set the following config: + +.neo4j.conf +[source,properties] +---- +dbms.memory.transaction.max_size=10m +---- + +Our query from the link:#measure-memory-usage-tx[measuring memory usage^] of one transaction section uses more memory than this, so if we re-run that query we'll see the following error message: + +[source,text] +---- +The allocation of 64.3 KiB would use more than the limit 10.0 MiB. Currently using 9.9 MiB. dbms.memory.transaction.max_size threshold reached +---- + +[#resources] +== Resources + +* link:/docs/operations-manual/4.1/performance/memory-configuration/[Documentation: Memory configuration^] From 3428385ba367add54923d1d60d555068cfabb484 Mon Sep 17 00:00:00 2001 From: Mark Needham Date: Tue, 23 Jun 2020 10:51:18 +0100 Subject: [PATCH 02/14] add to menu --- _templates/menu_partial.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/_templates/menu_partial.erb b/_templates/menu_partial.erb index c060b588..e8c8f92a 100644 --- a/_templates/menu_partial.erb +++ b/_templates/menu_partial.erb @@ -151,6 +151,7 @@