Chopping from the back of each sequence in a single pass seems like a quick way as opposed to for example putting the sequence more than once through tr or some other regex processor.
I am not sure what your target performance is, but I picked up an example fastq file of just over 78MB and put it through this simple piece of code that uses one pass chopping. The fourteen seconds (on my 3.5GHz PC) suggest that 1 gygabyte would take about three minutes; note that the array of hash does NOT grow as more sequences are processed, so the prediction is scaleable.
which gave the following output:#!/usr/bin/perl use strict; use warnings; use Data::Dumper; my ($file) = glob '*.fastq'; open $fh, $file; my $analysis = []; # each element a sequence position # at which we will put a hash of freqs print localtime() . "\n"; while ( my $hdr = <$fh>) { my $seq = <$fh>; chomp $seq; analyse($seq, $analysis); $seq = <$fh> for (0 .. 1); # discard trailer } print localtime() . "\n"; print Dumper($analysis); sub analyse { my ($seq, $analysis) = @_; my $pos = length($seq); while ($pos) { $analysis->[--$pos]{chop $seq}++; } }
Wed Jun 8 14:19:04 2016 Wed Jun 8 14:19:18 2016 $VAR1 = [ { 'A' => 95617, 'T' => 108158, 'G' => 40710, 'C' => 64361 }, { 'T' => 75373, 'C' => 51943, 'G' => 72545, 'A' => 108985 }, { 'T' => 85539, 'C' => 57343, 'G' => 69604, 'A' => 96360 }, { 'A' => 92923, 'C' => 64361, 'T' => 87445, 'G' => 64117 }, { 'T' => 93456, 'G' => 60673, 'C' => 60057, 'A' => 94660 }, { 'A' => 94775, 'C' => 62953, 'G' => 63212, 'T' => 87906 }, { 'A' => 92453, 'G' => 64227, 'C' => 60365, 'T' => 91801 }, { 'C' => 60513, 'T' => 91077, 'G' => 61364, 'A' => 95892 }, { 'C' => 60524, 'G' => 60844, 'T' => 92885, 'A' => 94593 }, { 'A' => 94766, 'G' => 61648, 'C' => 60955, 'T' => 91477 }, { 'T' => 92978, 'G' => 60935, 'C' => 60316, 'A' => 94617 }, { 'C' => 60539, 'G' => 61581, 'T' => 92274, 'A' => 94452 }, { 'C' => 59940, 'G' => 60190, 'T' => 93538, 'A' => 95178 }, { 'G' => 60608, 'C' => 60446, 'T' => 93249, 'A' => 94543 }, { 'A' => 94666, 'T' => 93556, 'C' => 59989, 'G' => 60635 }, { 'C' => 60324, 'G' => 61720, 'T' => 93268, 'A' => 93534 }, { 'A' => 93717, 'C' => 60376, 'T' => 93535, 'G' => 61218 }, { 'G' => 61225, 'C' => 60992, 'T' => 93130, 'A' => 93499 }, { 'A' => 93515, 'C' => 61161, 'G' => 61151, 'T' => 93019 }, { 'A' => 92598, 'C' => 61351, 'T' => 93131, 'G' => 61766 }, { 'A' => 93152, 'C' => 61278, 'T' => 92889, 'G' => 61527 }, { 'A' => 93143, 'C' => 61225, 'G' => 62000, 'T' => 92478 }, { 'G' => 61474, 'T' => 92977, 'C' => 61546, 'A' => 92849 }, { 'A' => 92661, 'G' => 61980, 'C' => 61768, 'T' => 92437 }, { 'G' => 62070, 'C' => 61778, 'T' => 92189, 'A' => 92809 }, { 'A' => 92502, 'T' => 92386, 'C' => 61611, 'G' => 62347 }, { 'A' => 92432, 'T' => 91737, 'G' => 62648, 'C' => 62029 }, { 'T' => 91679, 'G' => 62252, 'C' => 62089, 'A' => 92826 }, { 'G' => 62453, 'T' => 91892, 'C' => 61765, 'A' => 92736 }, { 'G' => 62708, 'T' => 92085, 'C' => 61591, 'A' => 92462 }, { 'C' => 62291, 'T' => 91884, 'G' => 62821, 'A' => 91850 }, { 'G' => 62700, 'T' => 91738, 'C' => 62320, 'A' => 92088 }, { 'A' => 93030, 'C' => 61746, 'G' => 62788, 'T' => 91282 }, { 'T' => 91749, 'C' => 61744, 'G' => 62907, 'A' => 92446 }, { 'G' => 62296, 'T' => 91632, 'C' => 62216, 'A' => 92702 }, { 'T' => 91804, 'C' => 62383, 'G' => 62855, 'N' => 1, 'A' => 91803 }, { 'C' => 61646, 'G' => 62903, 'T' => 91341, 'A' => 92667, 'N' => 289 }, { 'C' => 62641, 'G' => 62872, 'T' => 91136, 'A' => 92197 }, { 'C' => 62057, 'G' => 62837, 'T' => 91154, 'N' => 620, 'A' => 92178 }, { 'G' => 62722, 'C' => 62043, 'T' => 91622, 'N' => 576, 'A' => 91883 }, { 'C' => 61811, 'T' => 91391, 'G' => 63307, 'A' => 92337 }, { 'A' => 91767, 'N' => 620, 'C' => 62239, 'G' => 63172, 'T' => 91048 }, { 'A' => 91871, 'N' => 53, 'G' => 63084, 'C' => 62200, 'T' => 91638 }, { 'T' => 91688, 'G' => 62924, 'C' => 61981, 'N' => 385, 'A' => 91868 }, { 'G' => 63108, 'C' => 62768, 'T' => 91005, 'N' => 419, 'A' => 91546 }, { 'A' => 91701, 'N' => 621, 'C' => 62578, 'G' => 63091, 'T' => 90855 }, { 'A' => 91816, 'N' => 71, 'C' => 62275, 'T' => 91096, 'G' => 63588 }, { 'A' => 91525, 'N' => 236, 'C' => 62607, 'T' => 91383, 'G' => 63095 }, { 'A' => 91080, 'N' => 26, 'G' => 63213, 'T' => 91594, 'C' => 62933 }, { 'A' => 91514, 'N' => 51, 'T' => 91087, 'G' => 63341, 'C' => 62853 }, { 'G' => 63314, 'C' => 62538, 'T' => 90800, 'N' => 593, 'A' => 91601 }, { 'G' => 63782, 'T' => 90949, 'C' => 62223, 'N' => 807, 'A' => 91085 }, { 'N' => 620, 'A' => 91394, 'T' => 90938, 'C' => 62302, 'G' => 63592 }, { 'A' => 91299, 'N' => 605, 'G' => 63677, 'C' => 62200, 'T' => 91065 }, { 'A' => 91432, 'N' => 622, 'G' => 63535, 'C' => 62496, 'T' => 90761 }, { 'G' => 63791, 'T' => 90401, 'C' => 62571, 'N' => 625, 'A' => 91458 }, { 'G' => 63397, 'C' => 62282, 'T' => 90597, 'A' => 91847, 'N' => 723 }, { 'C' => 62021, 'G' => 63202, 'T' => 91192, 'A' => 92137, 'N' => 294 }, { 'T' => 90904, 'G' => 63432, 'C' => 62605, 'A' => 91411, 'N' => 494 }, { 'A' => 91268, 'N' => 294, 'T' => 90559, 'C' => 63196, 'G' => 63529 }, { 'T' => 90368, 'G' => 63510, 'C' => 62999, 'N' => 371, 'A' => 91598 }, { 'A' => 91073, 'N' => 713, 'T' => 90533, 'C' => 62880, 'G' => 63647 }, { 'G' => 63387, 'C' => 62393, 'T' => 91009, 'N' => 729, 'A' => 91328 }, { 'C' => 62732, 'G' => 63781, 'T' => 90418, 'N' => 373, 'A' => 91542 }, { 'C' => 62653, 'T' => 90784, 'G' => 63484, 'N' => 290, 'A' => 91635 }, { 'N' => 140, 'A' => 91372, 'G' => 64039, 'T' => 90277, 'C' => 63018 }, { 'C' => 63377, 'T' => 90129, 'G' => 63458, 'A' => 91258, 'N' => 624 }, { 'T' => 90870, 'G' => 63823, 'C' => 62831, 'A' => 90667, 'N' => 655 }, { 'A' => 91640, 'N' => 428, 'G' => 63513, 'T' => 90278, 'C' => 62987 }, { 'A' => 90783, 'N' => 1166, 'C' => 62632, 'G' => 63972, 'T' => 90293 }, { 'A' => 91091, 'N' => 1004, 'G' => 64306, 'C' => 62924, 'T' => 89521 }, { 'A' => 90956, 'N' => 806, 'G' => 64238, 'T' => 89778, 'C' => 63068 }, { 'T' => 90931, 'G' => 64216, 'C' => 62586, 'A' => 90584, 'N' => 529 }, { 'C' => 62968, 'G' => 64399, 'T' => 89490, 'A' => 91272, 'N' => 717 }, { 'N' => 449, 'A' => 90544, 'G' => 64163, 'T' => 90620, 'C' => 63070 }, { 'A' => 90839, 'N' => 636, 'C' => 63188, 'G' => 64298, 'T' => 89885 }, { 'A' => 91013, 'N' => 932, 'G' => 63479, 'C' => 63121, 'T' => 90301 }, { 'T' => 89829, 'C' => 63165, 'G' => 64210, 'A' => 91346, 'N' => 296 }, { 'G' => 64529, 'C' => 62714, 'T' => 89936, 'A' => 91477, 'N' => 190 }, { 'N' => 301, 'A' => 90623, 'G' => 64859, 'C' => 63122, 'T' => 89941 }, { 'T' => 90099, 'C' => 63211, 'G' => 64084, 'N' => 131, 'A' => 91321 }, { 'N' => 1475, 'A' => 90609, 'G' => 63896, 'C' => 63063, 'T' => 89803 }, { 'N' => 1407, 'A' => 91461, 'T' => 88953, 'G' => 64306, 'C' => 62719 }, { 'T' => 89127, 'G' => 64231, 'C' => 62298, 'N' => 2177, 'A' => 91013 }, { 'A' => 90259, 'N' => 2131, 'C' => 62896, 'G' => 64105, 'T' => 89455 }, { 'C' => 63197, 'G' => 64079, 'T' => 89832, 'N' => 908, 'A' => 90830 }, { 'G' => 64777, 'T' => 89976, 'C' => 63068, 'N' => 357, 'A' => 90668 }, { 'T' => 89587, 'C' => 63200, 'G' => 64812, 'A' => 89978, 'N' => 1269 }, { 'N' => 1696, 'A' => 89904, 'G' => 64443, 'C' => 63402, 'T' => 89401 }, { 'A' => 89654, 'N' => 2703, 'C' => 63119, 'T' => 88903, 'G' => 64467 }, { 'N' => 1131, 'A' => 89889, 'T' => 89520, 'C' => 63477, 'G' => 64829 }, { 'G' => 64882, 'T' => 89466, 'C' => 63907, 'N' => 1347, 'A' => 89244 }, { 'C' => 63887, 'G' => 64353, 'T' => 89136, 'A' => 90096, 'N' => 1374 }, { 'G' => 65209, 'T' => 88684, 'C' => 63699, 'A' => 90123, 'N' => 1131 }, { 'C' => 63361, 'G' => 64956, 'T' => 89134, 'N' => 844, 'A' => 90551 }, { 'C' => 63860, 'T' => 88496, 'G' => 65164, 'A' => 89328, 'N' => 1998 }, { 'G' => 64476, 'C' => 63725, 'T' => 88534, 'N' => 1816, 'A' => 90295 }, { 'C' => 63638, 'T' => 88409, 'G' => 64782, 'N' => 1982, 'A' => 90035 }, { 'N' => 1594, 'A' => 89806, 'C' => 64095, 'T' => 88343, 'G' => 65008 }, { 'N' => 1572, 'A' => 89668, 'G' => 65516, 'C' => 63817, 'T' => 88273 } ];
One world, one people
In reply to Re: Weighted frequency of characters in an array of strings
by anonymized user 468275
in thread Weighted frequency of characters in an array of strings
by K_Edw
| For: | Use: | ||
| & | & | ||
| < | < | ||
| > | > | ||
| [ | [ | ||
| ] | ] |