From dd86242e41c4ded8ef66ac4d9c3dc0311a5f68ab Mon Sep 17 00:00:00 2001 From: Frank Li Date: Sun, 31 Oct 2021 22:37:49 +1000 Subject: [PATCH] parallelise file parsing --- .idea/vcs.xml | 6 ++++++ .../promoter/parallel/ParallelStreaming.class | Bin 6233 -> 6374 bytes out/production/promoter/qut/Sequential.class | Bin 7008 -> 7069 bytes src/parallel/ParallelStreaming.java | 9 +++------ src/qut/Sequential.java | 12 ++++++++---- 5 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 .idea/vcs.xml diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/out/production/promoter/parallel/ParallelStreaming.class b/out/production/promoter/parallel/ParallelStreaming.class index 32b5910fca2ef8f930454c30bd32412dfdb8d675..c2fdde725e704f792932cd283fa1b319eeb282fe 100644 GIT binary patch delta 2400 zcmaJ?3w&E;75|?+?#;dVnkE_Rxa+odrLUzeNsF@mvJKIn8^YR+J!t73x~#qHO53zF z3ERXiZUscBD14%zpiZ3ifsbvqYdd8VbOMft3Qh#ZHWgutsQ93prJnC5%J}{K+1%dm zpZ3i1 zo464l(Xm5dr(EeNt1*zoM+J6aHy5SDPTDC#@VrYjcBaw`b=*XV$om5kc9+C00w2Sz za+$ZrH-Os&ytrNN^lqY0$Wz`*YY?AQtWOa<)mP4I4uf!)fW)U|sn+Bh#AgIPi+f~7 zTT)=+bNIZD`vmUiOP%tTmaR#oy11AfcG|=sz9{e|XarwsOD5gkyEUof%krdlA$>(o zXeGV}@KqIQNS5lwW*9DfUEmw|rrfF5a#_D6@NIlYKBX^VnH&2A_TzvY_bt=DCoqD8 z(r-j-25^gk0gMYA!uL62RRMz8m@llo(1afk2^_`|X&Wut!vc@sNAh7~d1wGzP3ZWE zO8!woWwq>aRhTB~@wmWGaZEOw4ce0ePvL2KhgnfJfS(KeLPaU)w>R7UcCyXp-q|X` zvjV@warvydm>szAoWQT~yxiewI1u*NQKgPQa914cbtjS?3Ur;x4$gDQwl=#@AvUGs zPjZFWFGaz|f?!)FmG13XvxO1bE;9vp)1>qV%IOjr3zT~r2vNDqJ4-%M5Rx|sc6-T3 zD3W`OB26cgP%O^`isjP4y zvJSjZ=%M(dg&NJ`+t6%51$aKF9<9(za}}>l;4MrA5o4ta#i$hIClOT7JQ;M&2sg88 zjgr&~0#T4iMne~Q>IoHcS7?q|-q_pQmFd&@8aqRar@g?l?ZS)P0R}EKexHy}hUV71 z#C%mlpb#F+!AYhBOMxIx;WVEn-h;~-5hvu#@D6%O-V>gmP+FI` zXW!tF`wdg-uNwjO%ZmWl;J5f4$0#HCy~46rWo+67wP(&E zf%a@mKRFosFj2rvWXBoq|VFVwx^kHOk%BE%Jm1By^ zDQDw;%X0_=tr5#_8A|(R$MhgRK86|>T85^M4NcuQG566YWwZ!3lT&UA{1Lv?uouu=^#yK)bcS@!@ zyBQY0>tUb)VeU~eF2y{|$HiEHh5T8IMclu|Jkpn8FZcNo?(|W-7mwk6IEMG*IF_>O zEAcv7IM6!0iS-=rT1=vqqpYI<)>8-@9Mtq7>t`o()N0@v-NwHhJiZ)yEqd`6{FR|M z$zXZ|e`AU}mWP+36kHf`O#PiH7Ys7;APF=39}E`{r~Xg=>}G%+=fvN{1jA23sFbUa z(|;w)WX|C1e+|yD!Kto!oY`z0yf5uZ#p4ZPPvFuCxX!{g z#nt+sW`p1w>{+JcUq%k4S-!+;L6@3brWl^+;>)Pns;U{o2MFW%n!@LU@4#nd9N$&k zVHftXstc-dMb02C<2p9-AU0txSF@6vaMF=EF8@f3f>A=BZO6$ zj@2A;4Wp>pNhgwv!|7yCvm%Ei7r8kNfm-t5R!$^@D)N%X4pl4sph?WvN!Q5;nrf}( zb6&)%YV(9SnXzF!!Nj`_q{hala~RLJW~S1EHETysh`jg#Eq&Ui6Rh)S~YzBlikch7&%Irp6Z zocDJ1n^o2;AMbh&KpmBAw_;xPj;}b_h$evxB`(4OmFq7v8nIAf5f-a?{wY(J5bRa0 zYg;R0t?~BCg{iKNc)N)fTq1C(#AWz`y2)Q1v~UHk6lj%bQ}6nVEDJ#_mxyA8iUn&{ z3aljLBzsbo=SAaft?|_tN0%kKmYXQWDv8yI5q$C1HBmyi;eVaYHqnl_L;{@ze|t0@ z?dFg&u%1*$tg@k_J2hLNix5#wp@^mNDTyAeRjUFsj3lm-@Z*bWbD)i`zC)Bcy|_lN ze3_k2yI^{AnA7nUiS@Wv%@vJCFRquk0XM2U#k{!&ZYCJ#B-V7sqN%8bUfe42RRnmk zdu2~*d175$;5Nd9<5ADMYFV^1)sbMu*VI|Y7K_fF z-W)_Z?v%I--%u%UwL!RB;vRfU-DZ@BHes{C7KwXtA0bb@Z{(@xjd@uX9>9YF4@qpr zHr3#(H#Xs6iAV6L+TxpIh2g>D5>MbsWt;N@-;vmk9jeu=%t|6_F7lulD?JSAM-YV4JG2`{T{R!!g) ziC6KO+HaKP z)Q571X{_joMKu@H{aJ0MT#@KnU7zTPD?4--{X=aF6;q9RHB_wL%dSyVv)B7EM#xo_ zR<2hNsXf`{RHZ%$RnsNvG`l`zT4YgBkd%}~p`9CSAC(5O{glJ;P*~!h_?L~U>;Rpl z7jh+e$ZPA%QMM;9+|1Adt@(_^zwwcckJQ1OQ+!hiC2G{pQ%yO6PU1u&2WaM4biSe&j6IDug_5R!kv|Z7UHN!}2 zTb5&uqpP$ujt)A$5hPn8PS7zmy4&44h;>I}d9813Q{ULtR2vvUZ;NdV;!8TLh~wLZ zG`N?pTfz>*H)(Hyy(o181GuFRg|%V?x3@TA5MSS&u9aG6xH?Df-p-jlKqlp4gZnJv zMCDR3@slg4nGi<-->YGw24Svg0ZvC9X5w7T!UBAbr*s{segO zv$-jh)gV{5L(#fV#-G|yIM-q`TbN}O4U5hHIaImaCJBd?=W2i zn=I6mg8O$PnsjS(Fx@#^VIlk|rFJoTc|z^1x&-`_nGZ!ZZbc zWyyyE%ES9C`Po=K{)P{j!J36rI4|x7Xzpe3A0EU1o$;EFTOr5z10n>1%tiMqXKK1v zBXAfMM_>pvj$)EPyB5+~TguWQoPG$N!|;r8zkH&BUXZ|s`2q*W_~cCX|M$_$F@O(# z5E$bVeu^1nIz%^!o%ZN~!=fGo3Et!3XW5KlDB=nHO<)+`_FxG252Me+olzdyg~#;j zQyw1QdX{KToXlAQ=ZkPYhjsyMmZxGqr=s31U#5r4jur!DZnnwxAk8yu-1}BG=3_In zX7pnx|NDn9)L1@*XPPSZq9oD}ix>9{;Q2kMt{A}g^h$Wv@pm1L_k?g7m@|#Y!G+A5 zi`eA?Ch0;qoJhux4u|_hyTl33SkbWzaQV{#}D{1gdac7tC{5JQ43DW@v#|p-El*auSvtmPda}-1}3@20JDt@ n{%7+SDchAM&_v2}rF=S>CQ%`y^anM`UxLYOVG0#dF`WMbwp#V~ diff --git a/out/production/promoter/qut/Sequential.class b/out/production/promoter/qut/Sequential.class index f520b987c815ea82d4e78e670b73c27ba98f3035..e2c4e5a82e760cd9ddeaeb4991cfe8f0dcb2704c 100644 GIT binary patch delta 2317 zcmZ8id303e760AI%zJOLv@dVAue`vlpeu#L%$ zEs3>3?80svdlWjc*ZYlK6YRo18!3fu?_s;d>Q%TI_j{k)RTcXc9>f8GKM`9W7lfDo z9P5-IZotC|kKmxU$SJfQQ+OOtc)Q`04RPd{b3x z%ci7*0YRX?c|&W9HXKqI7Iv=VN1;WV$57Sm&y8MtXC5*nirv54dE9El<$M=Hivn9T1M$g zK4mpxAW<1--bh2bk9aqQua-P7oHMIkTb+aerVx%yK?Y95WK!S&{5)xu<0%1oSRv!3 zfM|tikO?@*KnkhX_(mfWshLDgE;0sDXv4NqGyvDez5OVc$vm-WQ6$AOlEzftCA<6( zhVxX;f;pq1SDDZw@bOZ$l;R2gYN?Ep7$*VvDi;V5$V3CMDEC_C)0bOg@|j0p4VWP1 zSk27lV75#{Gc#MvNGhn)nI)wX&lupOREaL7A8m31Pe<|y!#{u=nKFc2TTT?|M*;nj z?%E&k&0u)C`0gIGm3JYT!scCkE%V7}B6Y{~z?JF7pTh5P9eqT+onspr^WfMb6Z1sI zHvt7wBf1+voy=gB^`}l|QYT02ZO&vCU`i2ze+Xe)W{V79yp1jWaE~C6u5vA7=DyMX zh;$)y)J2W-BGR-E`AetuVoOuGw>^@1o{l<;F#u=4!E*NT%wyURk(q`E4zf&KqtO@* zqmi`7Q|OV)CtamGVH?b9&&?}q?|{Ey+AfyO;gnV252T`YJAp>P3Tju9Osi0gCZ3-| zi`A&a8WV;tPa#5vgd@DDNe47l=Ez*S)8xy6Z1cQ0tHSyYW}SRy?4kokH|`YXf`m_h_IC|A+=s=&7|TM~v;Loi@H+ zOUAgM*3rh!UOb@ft-W|?#5foYrj6UzXfbUaiiY;$QEmTd3jf@SdHTw~qoEX@t#zYr z3je89QS}%~_MkM4`tku(MO7DGrz$+F3rF8OhPQVNZ|E4F&x0ccB%%SkI|mhd#97SzOBL+=RoNwFov#KI>1l%L~&nKCAykZThw`A73n+Y#^S^`d2z>qU4M0)TkCWt> zm4=|y%P-PHb~%Sk4{&)3-!AAy83*l1_`C4;U1%SqsxV$ho=o~(j!2?MNM)9NbazM0 zRSj_k!nhLoxQe5&<>&|yR!)~ig!iLFekqHYbUwnegi?kHNiWo5II>j#516UnX(o`J zEzgd))6XI<1TqUci3$u=g Qx?}0GD;1sKa0sdKW^ zIi2%Pr!dXRipn+xaS$^z)6$|v%gPp8w0&PxMCiW9sKg&LcYg2Q`@8qP-@9xWaqLph zf$kl<0F0vG7S9U9<3S!C7HGse`Is>-uwGyT))C^TRIRALGE}oTSaZw7s_M`R8v$$- zcnptAr#Z;I2~V1MO5kZ(Vg{^Dct+q^JSP{Jne@C2o5}Wzc*(@e0$Z?EZZu2WE!bwF zRbab3%HeGS4`K%)M`ra)mKm1M*eUP|cFB=ermB&|P617Wfb!$+hn3#>WDm;8S_bU1EGD@HxJae$PP>t{NW1_|4c2B)sHv}8ZQ}%f5cm<# z5sVu{^^-#y`dQ!?1z8Nu3060Rygb<71b)XK@{p%u_KWyS;BWlH9yP(5<+G}mg$OaZ zd6U%8V=SCvwz|;LP+h2INr;CXT2WNZW4F*ra7|dUyt+y`b_twCHwV@=)bvg)QRXMu za)a$Ph{?zyx7oS2NtQ`&K_0T@_x4;nmf}q6Cn%m0nkS2!hz7`(*a4I+kHmJ-h~~p_ z$0_a_4^5{TCS9x2%#{1%_j+bC5oMA^74pG^g4B8cn-LF0Ri;K_(tP=O!eHk@Lb^Pb zkS_DQsix}RL5XsTH$X|U*6YhjX2byTr%gWo6xw!C3ia2&Sa-~qBwz5}MaeQhaYR^o z4TB5WC{~I<42EC;t6&$Ld{Rw*wty7Ori`WeGqQ;|-&n8%%s!ex|)CEVF0u4FJ{s+(Lcw&@a`S70sf<6KE{ zPx4Hz&F2f~+A_J0bfnP`8p^c+&Te>3x|FC3sV461 zgnbIG=rk|m+IW#@zDIpI3)L{=mRae^JR8QUO4|{h8(1*e7cp{Of`oEQh|Y(%P*-?Q>uo6d>Rs^3{WYJ)It(k zQVz=GbESg-qx|%G$sFT3E|!~5$&cWnX$28H(uPL~rB1)I4Nnlljd(?wO@!UpT~F*v6>-oX_C)Y*sqMETI8-Y9Vjl)=MQ$|fk=-cri&h+E$UCAH=YKc#Y;5XT z-qbUe4_fhM8@}n&QX>bQ>*z$Zqc(i68}z%{@DuS>{0eZ@O+Dd`EoyFh{?m}i_I$pD z1sH^dD8eEPM-Vfyn2{w|gB$PwLRgQb*n}JL3~oX@s&J6;4pj3qT86_|j+6WZe7IHL zTPVelr(t% z^ll(!sg1%LiY=u=htja#C^1_3>e=^3mUE;&7(FsQv1S6doMwvXE==On|uAuQzJ+ETfsW+Gx!JlRA$gA9rf4wKY zr(1D`k&I9|%OvYwZ%+;X>(o-4on#ev*45TaR zD(<;I66k8Sc;jnaN#3GKTpKSzMxNDA1qildXytZ_X+~}mIU{5=qc%dK8RO$`-)S{t z*;Xuzmp@)q&U>Zs;?X6OIo3f_Xexilu{DkJ&C;K9Xf9)3p6Lmyr0WbfT~7;W5&ZuF DNx`}F diff --git a/src/parallel/ParallelStreaming.java b/src/parallel/ParallelStreaming.java index ff4ffb8..bc77fd8 100644 --- a/src/parallel/ParallelStreaming.java +++ b/src/parallel/ParallelStreaming.java @@ -51,10 +51,9 @@ public void run(String referenceFile, String dir) throws IOException { List records = new LinkedList<>(); // we might want to parallelise this for maxmimum performance - for (String filename : ListGenbankFiles(dir)) { - System.out.println(filename); - records.add(Parse(filename)); - } + records = ListGenbankFiles(dir).parallelStream() + .map(Sequential::Parse) + .collect(Collectors.toList()); // have all dependencies ready to run in parallel stream for (GenbankRecord record : records) { @@ -68,8 +67,6 @@ public void run(String referenceFile, String dir) throws IOException { ForkJoinPool customThreadPool = new ForkJoinPool(11); - customThreadPool = new ForkJoinPool(11); - try { customThreadPool.submit( () -> { diff --git a/src/qut/Sequential.java b/src/qut/Sequential.java index a55f241..0ccd4e1 100644 --- a/src/qut/Sequential.java +++ b/src/qut/Sequential.java @@ -81,11 +81,15 @@ protected static List ListGenbankFiles(String dir) { return list; } - protected static GenbankRecord Parse(String file) throws IOException { + protected static GenbankRecord Parse(String file) { GenbankRecord record = new GenbankRecord(); - BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); - record.Parse(reader); - reader.close(); + try { + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file))); + record.Parse(reader); + reader.close(); + } catch (Exception e) { + // + } return record; }