I guess that the tuning parameters will vary depending on the specification of the target system and the line length of the data file. On your system the best performance (without narrowing it down further) looks to be with a 10,000 line buffer. On my rather elderly, vintage 2008 IIRC, Core 2 Duo laptop the sweet spot is around 1,000 lines for both the unpack and mask methods. Working on a 2,500,000 line file with 51 byte (inc. line terminator) lines I get the following ...
ok 1 - ANDmask ok 2 - unpackM Rate u950 u1050 u900 u1100 u1000 A950 A1050 A1100 A900 A10 +00 u950 1.28/s -- -0% -0% -1% -1% -39% -39% -39% -39% -4 +1% u1050 1.28/s 0% -- -0% -0% -1% -39% -39% -39% -39% -4 +1% u900 1.28/s 0% 0% -- -0% -1% -39% -39% -39% -39% -4 +1% u1100 1.28/s 1% 0% 0% -- -1% -39% -39% -39% -39% -4 +1% u1000 1.29/s 1% 1% 1% 1% -- -38% -38% -39% -39% -4 +0% A950 2.10/s 65% 64% 64% 64% 62% -- -0% -0% -0% - +3% A1050 2.10/s 65% 64% 64% 64% 63% 0% -- -0% -0% - +3% A1100 2.10/s 65% 64% 64% 64% 63% 0% 0% -- -0% - +3% A900 2.11/s 65% 65% 65% 64% 63% 0% 0% 0% -- - +2% A1000 2.16/s 69% 69% 69% 68% 67% 3% 3% 3% 2% +-- 1..2
... with this code.
use strict; use warnings; use Benchmark qw{ cmpthese }; use Test::More qw{ no_plan }; open my $inFH, q{<}, \ <<__EOD__ or die $!; TATCGC.TGCCC.ATTAAGCCATACTTCAAGGATCCCCCCG.GA.GGGCA GGAGTCGTC.ACACATCTTCACTAC.CATATCTTGCTACGGCCACTGACA CGA.A.CAATTTTCGAATGGAGGGCGAATGCCGTTGTGCGCTGCGTGACG TCT.AGCCT.CGAA.A.GCAGGCGTGGGGCGTACCACGGCTTGGCCTAT. ATGCACTT.AA.CCCTCGTAT.CTCTACTCT.ACAACCTTGGGCAG.T.T TAGGGC.CCGCG.TAAACTGCAGATAGTACTCCAAGAATCGCTCCGACCC .CGTCTAAACAATTAGCGGAGGTCGTC.CTGCAA.CAATGATCTTAACAC AG.AAGAAGT.CAAAGT.GTAGTGGCTGGGTACTTGATCA.TCAATTTCA ACC.TTCCAG.AAATGAGGTCTC.AAAAG.AGTT.CCTG.GGCTGTGTAG ACAT.A.TGTTAACACCACCTATAATAGAAGC.TTATATTCACC.TTAAC GAATTGT.TTCAATGCCACATA.GCTGGGCCAC.GCCTTAGGCTATTT.A AGCTTTACAGATGT.ACCAAA.CCAG.GACTAGATGGGGGGG.ATATGCC AGAAAGCCC.TCCCTGCGAAT.TGGGGGAGTACT.ATAGACTA.GGCCAG .CACTCCGG.CTAACACTACTTCCTGTAACAAA.CTGAAGAGA.CGTTTG CAGTCCTGAGCGTGCTAC..GTTCT.CTGTCG.TTAGATGGGCGC..GTA AGCAAGAAAACACGTACAGAAAAAGCCGACGGC.GGGGTTC.GTCAACC. T.ACTTATCGACATTAGCATCTGG.TGCCGCCTTT.GAACCCATTACTCG TAAACGA.CACCGCTTAGGGCTGACCTCCGAA.TTATCAGAGTACCGGGC ACGGATATCCTA.ATAGA.TACCTATGTAGGAAG.TCGAACTC.TAAACT ACCCCCGTCC.TATCCTC.CGA.ATTGCCCCCGGCCGTACTCCA.AAACC T..TC.AGCCGGG.TTG.TGTGGATCAGAAGTGATTGAGACTGG.GCCTA TCAACCTTGAAGTTGAAACTCCGAGAGT.CGCGGTCAACC.AGCCTGCGC GATCAGTGAGGAGCTGT.C.AAAGC.TC.AG.GCC.G.GA.GTGAGATT. G.C.AGTTTGCCACTT.CATCCAGAGTTCATCCAGCAGG.ATTGAAGTTA GCGCTGCA.CTGT.CATTTTTTATTCCCACCCGGTCTCCCCAACCCCGAT TGTATAGCCAGG.CGGAG.TTTCTT.TAGATCTAGTAAGACATT.CCCGA AC.CCCCT.G.TAGCTAAATCGACGGG..GTCAATATACACGGAT.CTTT AAGC.GGGCTGATGCTTATCTCCTAGCCCGC.CCTCGATGAT.ATTATTG TCACCC.ACCGC..T.CGGAACGAAAACT..AT.CCTATTAGAACATCCT .GCGTGAACC.TG.G.TACACCG.GATGGTCCCGAC.GGCTACCGAT.CA .CGCTGCCCCCATCTCCAGCTATC.AAA.AGGGACGCGATATGCGGAAGC CGTTCAGACAATCCTTTTGGGGTGTAAATGTCTCC.ACCTC.GAGA.CTG AAGACATAGGAG.CCAGAAT.A.CGTCATACAGAGGCACTC.TAT..TCT AGGCCTGCCTACTT.TTGGCTAA.C.AGACTTGG.AAG.ATGTAGAAC.G GTCAACCCGTGCTAACTGGGGTGAGGAATCTTCCGAGCC.TGCTCGTCGC TCG.TGGCAGCTC.AACTGGTGCGCGGCAG.CCTCCTGCCAAGTATTCAG CGGGGGCTAC..GTT.TCATCGAACACGGCACACTAACAAACTCCTGTGT TTCCGGGGTCGACCCTTTGGCCCAAGAGTGA.AGGGCTTCG.ACTGCG.. AA.CGTGCGGGTAGCCTAGACACGTAGC.TTGTGCGC.CG.CGCA.CAAG AT.TATGTCA.ACTTCCGCCGGG.CTTCTGTGTACATT.AA.GAGAATAA CGGCAAGGATTGCTCGACGTAGGAGTCCGTGGAGCTCGTGC.GACC.ACC .GTTAA.CACGCCTTAACTTTTCGGAACAGAGTAAC.AATCCCGG.TA.C CAGGTAATGTGTCACCAGGTTCGGGCCCT.CACCGTCCCAGCTAC.TGTT TAGCCCCTCTTTT.GAT.GGCCC.AGCGACATCAA.TGATC..CTGTAGG CGATCATATTTCATTGTTCCGC.TG.AGCGGT.A.TG.GCAAT.CAGCCG ATCGATGTTCTGATATG..GTGTGAATAC.AGAAACCGGCTTTGTCGGGG CCTTT.AGGAA.AC.TAGGT.TT.CTCAATGAAC.GACATCAAC.T.AGC AAGGAGGTACACAGCGTTCAGCGGATCC.CT.AAGC.TAGCATCTGCTGA GCAGATA.A.TCTGCTTCAACTGTGAAAGGGTTG.CTAATCAG.GCGTAG GGTACGATC.GCAAC.AGTCCACAAGTACACGGTGGAATT.CC..C.TTG __EOD__ my $fileStart = tell $inFH; my $offset = 9; # Column 10 if numbering from 1 my $testOK = q{C.T.AGCTGTTG..GAGACCCAGGCAGTCCCAGTTGCCGATCTTTCAC..}; my %methods = ( unpackM => sub { # Multi-line unpack suggested by LanX seek $inFH, 0, 0; my $buffer = <$inFH>; my $lineLen = length $buffer; my $nLines = shift; my $chunkSize = $lineLen * $nLines; seek $inFH, 0, 0; my $retStr; my $fmt = qq{(x${offset}ax@{ [ $lineLen - $offset - 1 ] })*}; while ( my $bytesRead = read $inFH, $buffer, $chunkSize ) { $retStr .= join q{}, unpack $fmt, $buffer; } return \ $retStr; }, ANDmask => sub { # Multi-line AND mask by johngg seek $inFH, 0, 0; my $buffer = <$inFH>; my $lineLen = length $buffer; my $nLines = shift; my $chunkSize = $lineLen * $nLines; seek $inFH, 0, 0; my $retStr; my $mask = qq{\x00} x ${offset} . qq{\xff} . qq{\x00} x ( $lineLen - $offset - 1 ); $mask x= $nLines; while ( my $bytesRead = read $inFH, $buffer, $chunkSize ) { ( my $anded = $buffer & $mask ) =~ tr{\x00}{}d; $retStr .= $anded; } return \ $retStr; }, ); foreach my $method ( sort keys %methods ) { ok( ${ $methods{ $method }->( 20 ) } eq $testOK, $method ); } close $inFH or die $!; my $filename = q{spw1202693.txt}; open $inFH, q{<}, $filename or die qq{open: < $filename: $!\n}; cmpthese( -15, { map { my $method = $_; map { my $nLines = $_; my $label = substr( $method, 0, 1 ) . $nLines; my $codeStr = q[sub { my $col = $methods{ ] . $method . q[ }->( $nLines ); }]; $label => eval $codeStr; } 900, 950, 1_000, 1_050, 1_100 } keys %methods } ); close $inFH or die qq{close: < $filename: $!\n};
Cheers,
JohnGG
In reply to Re^7: Faster and more efficient way to read a file vertically
by johngg
in thread Faster and more efficient way to read a file vertically
by Anonymous Monk
| For: | Use: | ||
| & | & | ||
| < | < | ||
| > | > | ||
| [ | [ | ||
| ] | ] |