parallelise file parsing

frankisawesome · Oct 31, 2021 · dd86242 · dd86242
1 parent 0ff9a2a
commit dd86242
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 10 deletions.
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/out/production/promoter/parallel/ParallelStreaming.class b/out/production/promoter/parallel/ParallelStreaming.class
diff --git a/out/production/promoter/qut/Sequential.class b/out/production/promoter/qut/Sequential.class
diff --git a/src/parallel/ParallelStreaming.java b/src/parallel/ParallelStreaming.java
@@ -51,10 +51,9 @@ public void run(String referenceFile, String dir) throws IOException {
         List<GenbankRecord> records = new LinkedList<>();
 
         // we might want to parallelise this for maxmimum performance
-        for (String filename : ListGenbankFiles(dir)) {
-            System.out.println(filename);
-            records.add(Parse(filename));
-        }
+        records = ListGenbankFiles(dir).parallelStream()
+                .map(Sequential::Parse)
+                .collect(Collectors.toList());
 
         // have all dependencies ready to run in parallel stream
         for (GenbankRecord record : records) {
@@ -68,8 +67,6 @@ public void run(String referenceFile, String dir) throws IOException {
 
         ForkJoinPool customThreadPool = new ForkJoinPool(11);
 
-        customThreadPool = new ForkJoinPool(11);
-
         try {
             customThreadPool.submit(
                     () ->  {

diff --git a/src/qut/Sequential.java b/src/qut/Sequential.java
@@ -81,11 +81,15 @@ protected static List<String> ListGenbankFiles(String dir) {
         return list;
     }
 
-    protected static GenbankRecord Parse(String file) throws IOException {
+    protected static GenbankRecord Parse(String file) {
         GenbankRecord record = new GenbankRecord();
-        BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
-        record.Parse(reader);
-        reader.close();
+        try {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
+            record.Parse(reader);
+            reader.close();
+        } catch (Exception e) {
+            //
+        }
         return record;
     }