Revert-store

.gitignore

@@ -4,4 +4,6 @@ target
 *.iml
 *.ipr
 .idea
+*.db*
+.shell_history
 

copy-store.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+EDITION=${1-community}
+shift
+SRC=$1
+DST=$2
+SKIP_RELS=$3
+SKIP_PROPS=$4
+SKIP_LABELS=$5
+DELETE_NODES=$6
+KEEP_NODE_IDS=$7
+HEAP=4G
+CACHE=2G
+CACHE_SRC=1G
+#$CACHE
+echo "Usage: copy-store.sh [community|enterprise] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip] [Labels,To,Delete,Nodes]"
+
+if [[ "$EDITION" != "enterprise" && "$EDITION" != "community" ]]
+then
+    echo "ATTENTION: The parameter '$EDITION' you passed in for the edition is neither 'community' nor 'enterprise'. Aborting."
+    exit
+fi
+if [[ "$SRC" = "" || "$DST" = "" ]]
+then
+    echo "ATTENTION: Source '$SRC' or target '$DST' directory not provided. Aborting."
+    exit
+fi
+
+if [[ ! -d $SRC ]]
+then
+    echo "ATTENTION: Source '$SRC' is not a directory. Aborting."
+    exit
+fi
+
+echo "Using: Heap $HEAP Pagecache $CACHE Edition '$EDITION' from '$SRC' to '$DST' skipping labels: '$SKIP_LABELS', removing nodes with labels: '$DELETE_NODES' rels: '$SKIP_RELS' props '$SKIP_PROPS' Keeping Node Ids: $KEEP_NODE_IDS"
+echo
+echo "Please note that you will need this memory ($CACHE + $CACHE_SRC + $HEAP) as it opens 2 databases one for reading and one for writing."
+# heap config
+export MAVEN_OPTS="-Xmx$HEAP -Xms$HEAP -XX:+UseG1GC"
+
+mvn clean compile exec:java -P${EDITION} -e -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Ddbms.pagecache.memory=$CACHE -Ddbms.pagecache.memory.source=$CACHE_SRC \
+      -Dexec.args="$SRC $DST $SKIP_RELS $SKIP_PROPS $SKIP_LABELS $DELETE_NODES $KEEP_NODE_IDS"
+
+#-Dneo4j.version=2.3.0

neo4j.properties

@@ -0,0 +1,18 @@
+dbms.pagecache.memory=2G
+dbms.pagecache.memory.source=2G
+
+cache_type=none
+allow_store_upgrade=true
+
+source_db_dir=
+target_db_dir=
+
+keep_node_ids=true
+
+properties_to_ignore=
+labels_to_ignore=
+labels_to_delete=
+rel_types_to_ignore=
+
+store_copy_log_dir=
+bad_entries_log_dir=

pom.xml

@@ -4,31 +4,52 @@
 
     <groupId>org.neo4j</groupId>
     <artifactId>store-util</artifactId>
-    <version>2.0.1</version>
+    <version>3.1.0</version>
     <packaging>jar</packaging>
 
     <name>store-util</name>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <neo4j.version>${project.version}</neo4j.version>
     </properties>
 
+    <profiles>
+        <profile>
+            <id>enterprise</id>
+            <dependencies>
+                <dependency>
+                    <groupId>org.neo4j</groupId>
+                    <artifactId>neo4j-enterprise</artifactId>
+                    <version>${neo4j.version}</version>
+                </dependency>
+            </dependencies>
+        </profile>
+        <profile>
+            <id>community</id>
+        </profile>
+    </profiles>
     <dependencies>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
             <version>4.11</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.neo4j</groupId>
+            <artifactId>neo4j-io</artifactId>
+            <version>${neo4j.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.neo4j</groupId>
             <artifactId>neo4j-kernel</artifactId>
-            <version>${project.version}</version>
+            <version>${neo4j.version}</version>
         </dependency>
         <dependency>
             <groupId>org.neo4j</groupId>
             <artifactId>neo4j-lucene-index</artifactId>
-            <version>${project.version}</version>
+            <version>${neo4j.version}</version>
         </dependency>
     </dependencies>
 
@@ -39,8 +60,8 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>2.3.2</version>
                 <configuration>
-                    <source>1.7</source>
-                    <target>1.7</target>
+                    <source>1.8</source>
+                    <target>1.8</target>
                     <compilerArgument>-Xlint:all</compilerArgument>
                     <showWarnings>true</showWarnings>
                     <showDeprecation>false</showDeprecation>

readme.md

@@ -1,22 +1,67 @@
-## Tools to copy and compare Neo4j Stores
+## Tool to copy Neo4j Stores
 
-Uses the GraphDatabaseService to read a store and the batch-inserter API to write the target store keeping the node-ids.
-Copies the index-files as is.
-Ignores broken nodes and relationships.
+Uses the BatchInserterImpl to read a store and write the target store keeping the node-ids.
+Copies the manual (legacy) index-files as is, please note it performs no index upgrade!
 
-Also useful to skip no longer wanted properties or relationships with a certain type. Good for store compaction as it
-rewrites the store file reclaiming space that is sitting empty.
+You will have to recreate any schema indexes too.
 
-Change the Neo4j version in pom.xml before running. (Currently 1.9.5)
+Ignores broken nodes and relationships and records them in `target/store-copy.log`
+
+Also useful to skip no longer wanted properties, relationships with a certain type.
+Or of certain labels and even nodes with certain labels.
+
+Good for store compaction and reorganization of relationships and properties as
+it rewrites the store file reclaiming space that is sitting empty.
+
+NOTE: With Neo4j 3.x there are two different store formats, so you have to provide "enterprise" or "community" as first argument of the call!
+
+You can now also decide if you want to compact the node-store, then you have to pass "false" as the parameter for keep-node-ids.
+
+Config is read from `neo4j.properties` file in current directory if it exists, but command line options override.
+
+neo4j.properties
+
+```
+source_db_dir=
+target_db_dir=
+
+keep_node_ids=true
+
+properties_to_ignore=
+labels_to_ignore=
+labels_to_delete=
+rel_types_to_ignore=
+
+store_copy_log_dir=
+bad_entries_log_dir=
+```
 
 ### Store Copy
 
-Usage:
+    copy-store.sh [enterprise|community] source.db target.db [RELS,TO,SKIP] [props,to,skip] [Labels,To,Skip] [Labels,To,Delete,Nodes] [keep-node-ids:true/false]
+
+
+The provided script contains these settings for page-cache (note you can configure a different, smaller setting for the source store than the target store).
+
+    dbms.pagecache.memory.source=2G
+    dbms.pagecache.memory=2G
+
+Heap config is in the shell-script, default is:
+
+    export MAVEN_OPTS="-Xmx4G -Xms4G -Xmn1G -XX:+UseG1GC"
+
+**Please adapt the settings as needed for your store.**
+
+**Please note that you will need the memory for (source-page-cache + target-page-cache + 1x heap) as it opens 2 databases one for reading and one for writing.**
+
+Change the Neo4j version in pom.xml before running as needed. (Currently 3.1.0)
+
+Optionally changeable from the outside with `-Dneo4j.version=3.1.0` on the `mvn` invocation.
+
+### Internally
 
-    mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreCopy" \
-      -Dexec.args="source-dir target-dir [rel,types,to,ignore] [properties,to,ignore] [labels,to,ignore]"
+Note: It calls under the hood:
 
-# Store Compare
+    mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreCopy" -Penterprise \
+      -Dexec.args="source-dir target-dir [REL,TYPES,TO,IGNORE] [properties,to,ignore] [Labels,To,Ignore] [Labels,To,Delete,Nodes] [keep-node-ids:true/false]"
 
-    mvn compile exec:java -Dexec.mainClass="org.neo4j.tool.StoreComparer" \
-      -Dexec.args="source-dir target-dir [rel,types,to,ignore] [properties,to,ignore]"

src/main/java/org/neo4j/tool/DomainAnalyzer.java

@@ -3,9 +3,9 @@
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.Node;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
-import org.neo4j.helpers.collection.IteratorUtil;
-import org.neo4j.kernel.EmbeddedGraphDatabase;
+import org.neo4j.helpers.collection.Iterables;
 
+import java.io.File;
 import java.util.*;
 
 /**
@@ -74,13 +74,13 @@ private String toString(Object value) {
         }
     }
 	public static void main(String[] args) {
-        graphDb = new GraphDatabaseFactory().newEmbeddedDatabase(args[0]);
+        graphDb = new GraphDatabaseFactory().newEmbeddedDatabase(new File(args[0]));
 
         long time = System.currentTimeMillis();
         Map<Set<String>,Sample> statistics = new HashMap<Set<String>, Sample>();
         int count = 0;
         for (Node node : graphDb.getAllNodes()) {
-            final HashSet<String> keys = IteratorUtil.addToCollection(node.getPropertyKeys(), new HashSet<String>());
+            final HashSet<String> keys = Iterables.addToCollection(node.getPropertyKeys(), new HashSet<String>());
             Sample sample = statistics.get(keys);
             if (sample==null) {
                 sample = new Sample(node);

src/main/java/org/neo4j/tool/GraphGenerator.java

@@ -5,8 +5,8 @@
 import org.neo4j.graphdb.Relationship;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
-import org.neo4j.kernel.EmbeddedGraphDatabase;
 
+import java.io.File;
 import java.util.Arrays;
 
 /**
@@ -17,7 +17,7 @@ public class GraphGenerator {
     public static final int MILLION = 1000 * 1000;
 
     public static void main(String[] args) {
-        final GraphDatabaseService gdb = new GraphDatabaseFactory().newEmbeddedDatabase("target/data");
+        final GraphDatabaseService gdb = new GraphDatabaseFactory().newEmbeddedDatabase(new File("target/data"));
         createDatabase(gdb);
         gdb.shutdown();
     }
@@ -39,14 +39,15 @@ public static void createDatabase(GraphDatabaseService graphdb) {
                     System.out.print(".");
                     if ((i % 10000) == 0) {
                         tx.success();
-                        tx.finish();
+                        tx.close();
                         System.out.println(" " + i);
                         tx = graphdb.beginTx();
                     }
                 }
             }
         } finally {
-            tx.finish();
+            tx.success();
+            tx.close();
         }
         System.out.println();
         long delta = (System.currentTimeMillis() - cpuTime);

src/main/java/org/neo4j/tool/PropertyAnalyzer.java

@@ -3,8 +3,8 @@
 import org.neo4j.graphdb.*;
 import org.neo4j.graphdb.factory.GraphDatabaseFactory;
 import org.neo4j.helpers.collection.MapUtil;
-import org.neo4j.kernel.EmbeddedGraphDatabase;
 
+import java.io.File;
 import java.lang.reflect.Array;
 import java.util.*;
 
@@ -96,7 +96,7 @@ public static Map<String,String> config() {
     }
 
     public static void main(String[] args) {
-        final GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(args[0]).setConfig(config()).newGraphDatabase();
+        final GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabaseBuilder(new File(args[0])).setConfig(config()).newGraphDatabase();
         int withoutProps=0, nodes = 0, rels = 0;
         Map<String,PropertyInfo> props=new HashMap<String, PropertyInfo>();
         for (Node node : db.getAllNodes()) {