in reply to Re: How to reslove this memory issue
in thread How to reslove this memory issue

Hi.. Thanks for the reply. Below is the minimum code. In this part records of 1gb i/p file(AH) is processed in such a way that they take the form of a hash reference. Now values of field 'bidirVolume' is stored in a new array AccUsage.

my $AccUsg; while(<AH>) { chomp; my $line=$_; $AccuCount++; my $MobileNumber; if( $line =~ /subscriberId:(\w+)\(\"(\d+)\"\)/ ) { $MobileNumber=$2; } my $plan=$line; $plan =~s/\\//g; my @AccVolume; if ( $plan =~ /usageControlAccum:(\w+)\(\"(.*)\"\)/ ) { my $p=$2; $p=~s/:\{/ => {/g; $p=~s/:\[/ => [/g; $p=~s/\"/\'/g; $p=~s/\':/\'=>/g; $p=~s/\}n/\}/g; #print $p,"\n"; my $e=eval($p); if ( @$ ) { push (@AccVolume,"error"); } else { #print Dumper($e); foreach my $value ( @{$e->{'reportingGroups'}} ) { if ( exists ( $value->{'absoluteAccumulated'}->{'counters' +} ) ) { $AccUsg->{$MobileNumber}->{$value->{'subscriberGroupName +'}}=$value->{'absoluteAccumulated'}->{'counters'}->[0]->{'bidirVolume +'}; } elsif ( exists ( $value->{'absoluteAccumulated'}->{'bidi +rVolume'} ) ) { $AccUsg->{$MobileNumber}->{$value->{'subscriberGroupName +'}}=$value->{'absoluteAccumulated'}->{'bidirVolume'}; } } } } } close(AH);

Now the the second 16gb i/p file (FH) matches its $planname value with the AccUsage hash and print the matched value to the output file.

while(<FH>) { chomp; my $line=$_; $SubsCount++; my $msisdn; my $IMEI; my $Circle; my $DeviceType; my $OPTIN; my $PlanType; my $familyId; my $trafficIds; if($line=~/userId:S(\d+)\(\"(\w+)\"\)/) { $msisdn = $2; } if ( $line=~/operatorInfo:A(\d+)\[(.*?)\]/ ) { my $opcinfo=$2; #ix0:S13("OptInState:3G")ix1:S11("CircleId:MH")ix2:S8("DevType +:")ix3:S9("imei:NULL")ix4:S16("PlanType:prepaid") #print $opcinfo,"\n"; if( $opcinfo =~ /ix(\d+):S(\d+)\(\"imei:(\w*)\"\)/ ) { $IMEI=$3; } if( $opcinfo =~ /ix(\d+):S(\d+)\(\"OptInState:(\w*)\"\)/ ) { $OPTIN=$3; } if( $opcinfo =~ /ix(\d+):S(\d+)\(\"CircleId:(\w*)\"\)/ ) { $Circle=$3; if( exists ( $lookup->{$Circle} ) ) { $Circle=$lookup->{$Circle}; } } if( $opcinfo =~ /ix(\d+):S(\d+)\(\"DevType:(\w*)\"\)/ ) { $DeviceType=$3; } if( $opcinfo =~ /ix(\d+):S(\d+)\(\"PlanType:(\w*)\"\)/ ) { $PlanType=$3; } } my @ValidPlan; #groups:A1[ix0:S10("3BASIC:100")] if($line=~/groups:A(\d+)\[(.*?)\]/) { my $plans=$2; my @AllPlans = split('ix\d+:S\d+\("',$plans); #print Dumper($AccUsg->{$msisdn}); foreach my $p ( @AllPlans ) { $p =~ s/\"\)//g; if ($p eq "" ) { next; } #my @planname=split(":",$p); if ( $p =~ /(\w+):(\d+)[:]?(.*)/) { my $planname=$1; my $priority=$2; my $expdate=$3; $expdate =~ s/,/\;/g; if( exists ( $AccUsg->{$msisdn}->{$planname} ) ) { if ( $expdate eq "") { push(@ValidPlan,"$planname;$priority;;;$AccUsg->{$ms +isdn}->{$planname}"); } elsif ( length($expdate) > 19 ) { push(@ValidPlan,"$planname;$priority;$expdate;$AccUs +g->{$msisdn}->{$planname}"); } else { push(@ValidPlan,"$planname;$priority;$expdate;;$AccU +sg->{$msisdn}->{$planname}"); } } else { if( $expdate eq "") { push(@ValidPlan,"$planname;$priority;;;"); } else { if( length($expdate) > 19 ) { push(@ValidPlan,"$planname;$priority;$ex +pdate;"); } else { push(@ValidPlan,"$planname;$priority +;$expdate;;"); } } } } } } if ( $line=~/familyId:S(\d+)\(\"(.*?)\"\)/ ) { $familyId=$2; } if ( $line=~/trafficIds:A(\d+)\[(.*?)\]/ ) { $trafficIds=$2; } #Now sorting the printplans priority wise # my $printPlan=join("|",@ValidPlan); my @validPlanSorted = sort { ($b =~ /(.*?);(\d+);(.*?)/)[1] <=> ($a +=~ /(.*?);(\d+);(.*?)/)[1] } @ValidPlan; my $printPlan=join("|",@validPlanSorted); $finalCount++; #print OUT "$msisdn;$IMEI;$Circle;$DeviceType;$OPTIN;$PlanType;$fami +lyId;$trafficIds;$printPlan\n"; print OUT "$msisdn,$IMEI,$Circle,$DeviceType,$OPTIN,$PlanType,$print +Plan\n"; } close(FH);

where one record of AH

P[containerVrsn:U(0)recordVrsn:U(0)size:U(560)ownGid:G[mdp:U(111817893 +5)seqNo:U(55)]logicalDbNo:U(1)classVrsn:U(1)timeStamp:U(0)dbRecord:T[ +classNo:U(1091971)size:U(532)updateVersion:U(1157)checksum:U(13183789 +05)EPC_UsageControlAccumulatedPot:R[subscriberId:S12("918970692483")u +sageControlAccum:S543("{\"reportingGroups\":[{\"absoluteAccumulated\" +:{\"counters\":[{\"bidirVolume\":3022011787,\"name\":\"base\"}],\"exp +iryDate\":{\"volume\":\"25-05-2013T00:00:00\"},\"previousExpiryDate\" +:{\"time\":\"25-04-2013T00:00:00\",\"volume\":\"25-04-2013T00:00:00\" +},\"reportingLevel\":\"totalTraffic\",\"resetPeriod\":{\"volume\":\"3 +0 days\"}},\"name\":\"110\",\"restartInfo\":\"25-04-2013T00:00:00\",\ +"selected\":\"no\",\"subscriberGroupName\":\"KA_FUP_2G80K_30\",\"subs +criptionDate\":\"25-04-2013T00:00:00\",\"validityTime\":1800}],\"vers +ion\":\"2.0\"}")]]]

where one record of FH

P[containerVrsn:U(0)recordVrsn:U(0)size:U(276)ownGid:G[mdp:U(109017151 +1)seqNo:U(28)]logicalDbNo:U(1)classVrsn:U(1)timeStamp:U(0)dbRecord:T[ +classNo:U(1064620)size:U(248)updateVersion:U(5)checksum:U(928324968)E +PC_SubscriberPot:R[userId:S12("919902995746")groups:A1[ix0:S12("KA_BA +SIC:100")]services:A0[]blacklist_services:A0[]operatorInfo:A5[ix0:S21 +("HomeMNC1:DefaultValue")ix1:S11("CircleId:KA")ix2:S13("OptInState:2G +")ix3:S9("imei:NULL")ix4:S16("PlanType:prepaid")]pccSubscriberPotRef: +M0[]notificationData:A0[]familyId:S0("")trafficIds:A0[]]]]

where one record of output file

MSISDN,IMEI,Circle,DeviceType,OPTIN,PlanType,PACKID1;priority;startdat +e;enddate;AccumulateUsage|PACKID2;priority;startdate;enddate;Accumula +teUsage|PACKID3;priorit y;startdate;enddate;AccumulateUsage|PACKID4;priority;startdate;enddate +;AccumulateUsage| 919164032638,NULL,KA,,2G,prepaid,KA_BASIC;100;;;|KA_FUP_2G80K_30;90;22 +-08-2014T00:00:00;20-09-2014T23:59:59;7032609129

Now I need to add four more fields from AH file (counter, expiry_date, prev_expirydate_volume & prev_expirydate_time) for this I need to add more AccUsage type hashes which are causing memory issue..so how much memory it will need more? I know code is badly written but thats how it is running here in production for few years now..& i can't help it :) ... thanks a lot