Hi All, thanks for reply. Below is the first part of my code where script is creating a sqlite database, manipulating the data of 3gb file and then storing it into the database..as I start running script CPU usage reaches 99-100%.. so this part has the problem..

#!/usr/bin/perl


use Data::Dumper;
use DBI;
use File::Basename;
#use warnings;

open(FH,"<",$ARGV[0]);
open(AH,"<",$ARGV[1]);
open(OUT,">",$ARGV[2]);
open(LOG,">",$ARGV[3]);

######################################################################
+#################################################
#Conneting to database
my $dbExt = "db";
my $driver   = "SQLite";

my $file = basename($ARGV[1]);
my @fileName = split(/\./, $file);
$file = $fileName[0].".$dbExt";
my $dbFileName = dirname($ARGV[1])."/". $file;


if(-e $dbFileName) {
 unlink ($dbFileName);
}

my $database = "$dbFileName";
my $dsn = "DBI:$driver:dbname=$database";
my $userid = "";
my $password = "";

my $dbh = DBI->connect($dsn, $userid, $password, { RaiseError => 1, Au
+toCommit => 0 })
                      or die $DBI::errstr;

my $stmt = qq(CREATE TABLE ACCU_USAGE (MOBILE VARCHAR2(50), PLANNAME  
+VARCHAR2(50), PLANUSAGE CHAR(50), COUNTER CHAR(20), STATUS VARCHAR2(2
+0), EXPIRY_DATE CHAR(
20), PREEXP_TIME CHAR(20), PREEXP_VOLUME CHAR(20)); );
my $rv =  $dbh->do($stmt);
if($rv < 0){
   unlink($dbFileName) if(-e $dbFileName);
   $dbh->disconnect();
   print LOG $DBI::errstr;
   exit(1);
}


######################################################################
+#################################################


my $SubsSize= -s $ARGV[0];
my $AccuSize= -s $ARGV[1];

my $SubsCount=0;
my $AccuCount=0;
my $finalCount=0;

#my $AccUsg;
my $rowCount = 0;


while(<AH>) {
  chomp;
  my $line=$_;
  $AccuCount++;
  my $MobileNumber;

  if( $line =~ /subscriberId:(\w+)\(\"(\d+)\"\)/ ) {
    $MobileNumber=$2;
  }
  else {
    $MobileNumber="''";
    }

  my $plan=$line;
  $plan =~s/\\//g;

  my @AccVolume;

  if ( $plan =~ /usageControlAccum:(\w+)\(\"(.*)\"\)/ ) {
        my $p=$2;
        $p=~s/:\{/ => {/g;
        $p=~s/:\[/ => [/g;
        $p=~s/\"/\'/g;
        $p=~s/\':/\'=>/g;
        $p=~s/\}n/\}/g;

        #print $p,"\n";
        my $e=eval($p);

        if ( @$ ) {
          push (@AccVolume,"error");
        } else {
        #print Dumper($e);
          foreach my $value ( @{$e->{'reportingGroups'}} ) {
            if ( exists ( $value->{'absoluteAccumulated'}->{'counters'
+} ) ) {

                $stmt = qq(INSERT INTO ACCU_USAGE  VALUES ($MobileNumb
+er, \'$value->{'subscriberGroupName'}\', $value->{'absoluteAccumulate
+d'}->{'counters'}->[0]-
>{'bidirVolume'}, \'$value->{'absoluteAccumulated'}->{'counters'}->[0]
+->{'name'}\', \'$value->{'selected'}\', \'$value->{'absoluteAccumulat
+ed'}->{'expiryDate'}->{
'volume'}\', \'$value->{'absoluteAccumulated'}->{'previousExpiryDate'}
+->{'time'}\', \'$value->{'absoluteAccumulated'}->{'previousExpiryDate
+'}->{'volume'}\' ););
            }
            elsif ( exists ( $value->{'absoluteAccumulated'}->{'bidirV
+olume'} ) ) {

                $stmt = qq(INSERT INTO ACCU_USAGE  VALUES ($MobileNumb
+er, \'$value->{'subscriberGroupName'}\', $value->{'absoluteAccumulate
+d'}->{'bidirVolume'}, \
'$value->{'absoluteAccumulated'}->{'name'}\', \'$value->{'selected'}\'
+, \'$value->{'absoluteAccumulated'}->{'expiryDate'}->{'volume'}\', \'
+$value->{'absoluteAccum
ulated'}->{'previousExpiryDate'}->{'time'}\', \'$value->{'absoluteAccu
+mulated'}->{'previousExpiryDate'}->{'volume'}\' ););
            }

######################################################################
+############################################################
$rv = $dbh->do($stmt) or die $DBI::errstr;
              if($rv < 0) {
                print LOG "Failed to insert $stmt query. Exiting...\n"
+;
                unlink($dbFileName) if(-e $dbFileName);
                print LOG $DBI::errstr;
                $dbh->disconnect();
                exit(1);
              }
              else {
                 $rowCount++;
                 if($rowCount == 5000) {
                   $dbh->commit();
                   $rowCount = 0;
                 }
              }
######################################################################
+#############################################################

          }
    }
  }
}
close(AH);

######################################################################
+######################################3
$dbh->commit();
[download]

here AH is my 3 gb file.. script manipulates it line by line and forms a hash which stores the values to be filled into the database...<\p>

By far the greatest amount of time is being spent evaling the data structure into existence:

        my $e = eval( $p );
[download]

There's not a lot you can do about that. Writing your own parser would be complicated, error prone and almost certainly eons slower.

The second largest amount of time is being spent building and executing those two complicated SQL statements:

$stmt = qq(INSERT INTO ACCU_USAGE  VALUES ($Mobil ...
...
$stmt = qq(INSERT INTO ACCU_USAGE  VALUES ($MobileNumber, ...
...
$rv = $dbh->do($stmt) or die $DBI::errstr;
...
[download]

Now that you can do something about. By pre-preparing the statement using placeholders:

my $sql = $dbh->prepare( q[ INSERT INTO ACCU_USAGE  VALUES( ?, ?, ?, ?
+, ?, ?, ?, ? ) ] ) 
    or die $DBI::errstr;
[download]

and then binding the vars when executing the statement:

                    $rv = $sql->execute(
                        $MobileNumber,
                        $value->{'subscriberGroupName'},
                        $value->{'absoluteAccumulated'}->{'counters'}-
+>[0]->{'bidirVolume'},
                        $value->{'absoluteAccumulated'}->{'counters'}-
+>[0]->{'name'},
                        $value->{'selected'},
                        $value->{'absoluteAccumulated'}->{'expiryDate'
+}->{'volume'},
                        $value->{'absoluteAccumulated'}->{'previousExp
+iryDate'}->{'time'},
                        $value->{'absoluteAccumulated'}->{'previousExp
+iryDate'}->{'volume'}
                    ) or die $DBI::errstr;
[download]

Not only will you give the DB engine the best chance of performing the inserts optimally; you can also make your code look a lot cleaner and simpler to maintain. It won't reduce your cpu usage; but it should result in your program finishing more quickly.

I hope you'll agree this is much nicer to look at and understand:

#!/usr/bin/perl
use Data::Dumper;
use DBI;
use File::Basename;
#use warnings;

@ARGV = qw[ NUL junk.dat CON CON ]; ## crude hack for local testing.

open( FH, "<", $ARGV[0] );
open( AH, "<", $ARGV[1] );
open( OUT, ">", $ARGV[2] );
open( LOG, ">", $ARGV[3] );

#Conneting to database
my $dbExt = "db";
my $driver = "SQLite";

#my $file = basename($ARGV[1]);
#my @fileName = split(/\./, $file);
#$file = $fileName[0] . ".$dbExt";
my $dbFileName = 'junk.db'; #dirname( $ARGV[1] )  ."/" . $file;


if(-e $dbFileName) {
    unlink ($dbFileName);
}

my $database = "$dbFileName";
my $dsn = "DBI:$driver:dbname=$database";
my $userid = "";
my $password = "";

my $dbh = DBI->connect($dsn, $userid, $password, { RaiseError => 1, Au
+toCommit => 0 })
    or die $DBI::errstr;

my $stmt = <<EOS;
    CREATE TABLE ACCU_USAGE (
        MOBILE VARCHAR2(50),
        PLANNAME  VARCHAR2(50),
        PLANUSAGE CHAR(50),
        COUNTER CHAR(20),
        STATUS VARCHAR2(20),
        EXPIRY_DATE CHAR(20),
        PREEXP_TIME CHAR(20),
        PREEXP_VOLUME CHAR(20)
    );
EOS

my $rv =  $dbh->do($stmt);
if($rv < 0){
    unlink($dbFileName) if(-e $dbFileName);
    $dbh->disconnect();
    print LOG $DBI::errstr;
    exit(1);
}

my $sql = $dbh->prepare( q[ INSERT INTO ACCU_USAGE  VALUES( ?, ?, ?, ?
+, ?, ?, ?, ? ) ] )
    or die $DBI::errstr;

my $SubsSize= -s $ARGV[0];
my $AccuSize= -s $ARGV[1];
my $SubsCount=0;
my $AccuCount=0;
my $finalCount=0;
my $rowCount = 0;

while( <AH> ) {
    chomp;
    my $line = $_;
    $AccuCount++;
    my $MobileNumber;

    if( $line =~ /subscriberId:(\w+)\(\"(\d+)\"\)/ ) {
        $MobileNumber = $2;
    }
    else {
        $MobileNumber = "''";
    }

    my $plan = $line;
    $plan =~ s/\\//g;
    my @AccVolume;

    if( $plan =~ /usageControlAccum:(\w+)\(\"(.*)\"\)/ ) {
        my $p = $2;
        $p =~ s/:\{/ => {/g;
        $p =~ s/:\[/ => [/g;
        $p =~s/\"/\'/g;
        $p =~  s/\':/\'=>/g;
        $p =~ s/\}n/\}/g;

#        print $p,"\n";
        my $e = eval( $p );

        if ( @$ ) {
            push (@AccVolume,"error");
        } else {
            #print Dumper($e);
            foreach my $value ( @{$e->{'reportingGroups'}} ) {
                if ( exists ( $value->{'absoluteAccumulated'}->{'count
+ers'} ) ) {
                    $rv = $sql->execute(
                        $MobileNumber,
                        $value->{'subscriberGroupName'},
                        $value->{'absoluteAccumulated'}->{'counters'}-
+>[0]->{'bidirVolume'},
                        $value->{'absoluteAccumulated'}->{'counters'}-
+>[0]->{'name'},
                        $value->{'selected'},
                        $value->{'absoluteAccumulated'}->{'expiryDate'
+}->{'volume'},
                        $value->{'absoluteAccumulated'}->{'previousExp
+iryDate'}->{'time'},
                        $value->{'absoluteAccumulated'}->{'previousExp
+iryDate'}->{'volume'}
                    ) or die $DBI::errstr;
                }
                elsif ( exists ( $value->{'absoluteAccumulated'}->{'bi
+dirVolume'} ) ) {
                    $rv = $sql->execute(
                        $MobileNumber,
                        $value->{'subscriberGroupName'},
                        $value->{'absoluteAccumulated'}->{'bidirVolume
+'},
                        $value->{'absoluteAccumulated'}->{'name'},
                        $value->{'selected'},
                        $value->{'absoluteAccumulated'}->{'expiryDate'
+}->{'volume'},
                        $value->{'absoluteAccumulated'}->{'previousExp
+iryDate'}->{'time'},
                        $value->{'absoluteAccumulated'}->{'previousExp
+iryDate'}->{'volume'}
                    ) or die $DBI::errstr;
                }
                if($rv < 0) {
                    print LOG "Failed to insert $stmt query. Exiting..
+.\n";
                    unlink($dbFileName) if(-e $dbFileName);
                    print LOG $DBI::errstr;
                    $dbh->disconnect();
                    exit(1);
                }
                else {
                    $rowCount++;
                    if($rowCount == 5000) {
                        $dbh->commit();
                        $rowCount = 0;
                    }
                }
            }
        }
    }
}
close(AH);
$dbh->commit();
[download]

---

With the rise and rise of 'Social' network sites: 'Computers are making people easier to use everyday'

Examine what is said, not who speaks -- Silence betokens consent -- Love the truth but pardon error.

"Science is about questioning the status quo. Questioning authority".

In the absence of evidence, opinion is indistinguishable from prejudice.

Thanks a lot for your efforts I will apply these changes and let you know the outcomes. meanwhile I ran the same script on my sunSO server and surprising there it was taking only 4 to 5% of cpu..on the other hand on my Linux server it was taking 99 to 100%.. why there is such a large difference... and it also shows that there is not much wrong with our script.. also please fiind my server details.

Linux server where I am supposed to run the script and its causing issues...

YZP4M2:~ # uname -a
Linux YZP4M2 2.6.5-7.308-smp #1 SMP Mon Dec 10 11:36:40 UTC 2007 x86_6
+4 x86_64 x86_64 GNU/Linux

YZP4M2:~ # less /proc/cpuinfo
processor       : 0
vendor_id       : GenuineIntel
cpu family      : 6
model           : 13
model name      : Intel(R) Xeon(R) CPU           E7420  @ 2.13GHz
stepping        : 1
cpu MHz         : 2133.414
cache size      : 3072 KB
physical id     : 0
siblings        : 4
core id         : 0
cpu cores       : 4
fpu             : yes
fpu_exception   : yes
cpuid level     : 11
wp              : yes
flags           : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge
+ mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm sysca
+ll lm pni monitor ds_cp
l est cmpxchg16b dca lahf_lm
bogomips        : 4227.07
clflush size    : 64
cache_alignment : 64
address sizes   : 40 bits physical, 48 bits virtual
power management:


YZP4M2:~ # cat /proc/cpuinfo | grep processor | wc -l
16

YZP4M2:~ # free
             total       used       free     shared    buffers     cac
+hed
Mem:       8158176    8132864      25312          0       4468    1314
+884
-/+ buffers/cache:    6813512    1344664
Swap:      2104472    2104132        340
[download]

details of solaris server where script is running fine

# psrinfo -v
Status of virtual processor 0 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:50.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 1 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 2 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 3 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 8 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 9 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 10 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 11 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 16 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1200 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 18 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1200 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 19 as of: 11/26/2014 10:48:52
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1200 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 512 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 513 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 514 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 515 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 520 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 521 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 522 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 523 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1800 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 528 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1200 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 530 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1200 MHz,
        and has a sparcv9 floating point processor.
Status of virtual processor 531 as of: 11/26/2014 10:48:53
  on-line since 12/16/2013 19:50:51.
  The sparcv9 processor operates at 1200 MHz,
        and has a sparcv9 floating point processor.


# prtconf | grep Memory
Memory size: 45056 Megabytes
[download]

Hi All, I am not able to reply on Corion's answer.. so please help me getting this clarification....as per his reply, it means that CPU usage on my linux machine is only around 25%. There is no problem with my script and I can go ahead with this?

Can you post a small sample of the file you are processing please ?

below are the sample of two rows from the 3gb input file.

P[containerVrsn:U(0)recordVrsn:U(0)size:U(564)ownGid:G[mdp:U(111817873
+8)seqNo:U(109)]logicalDbNo:U(1)classVrsn:U(1)timeStamp:U(0)dbRecord:T
+[classNo:U(1091971)size:U(536)updateVersion:U(4685)checksum:U(1697637
+779)EPC_UsageControlAccumulatedPot:R[subscriberId:S12("918722933152")
+usageControlAccum:S547("{\"reportingGroups\":[{\"absoluteAccumulated\
+":{\"counters\":[{\"bidirVolume\":782373665,\"name\":\"base\"}],\"exp
+iryDate\":{\"volume\":\"08-12-2014T00:00:00\"},\"previousExpiryDate\"
+:{\"time\":\"08-11-2014T00:00:00\",\"volume\":\"08-11-2014T00:00:00\"
+},\"reportingLevel\":\"totalTraffic\",\"resetPeriod\":{\"volume\":\"3
+0 days\"}},\"name\":\"110\",\"restartInfo\":\"08-11-2014T00:00:00\",\
+"selected\":\"yes\",\"subscriberGroupName\":\"KA_NF_1_25G20K_30\",\"s
+ubscriptionDate\":\"08-11-2014T00:00:00\",\"validityTime\":1800}],\"v
+ersion\":\"2.1\"}\n")]]]
P[containerVrsn:U(0)recordVrsn:U(0)size:U(564)ownGid:G[mdp:U(111817873
+8)seqNo:U(112)]logicalDbNo:U(1)classVrsn:U(1)timeStamp:U(0)dbRecord:T
+[classNo:U(1091971)size:U(536)updateVersion:U(159)checksum:U(13173327
+11)EPC_UsageControlAccumulatedPot:R[subscriberId:S12("919743708573")u
+sageControlAccum:S545("{\"reportingGroups\":[{\"absoluteAccumulated\"
+:{\"counters\":[{\"bidirVolume\":3533123434,\"name\":\"base\"}],\"exp
+iryDate\":{\"volume\":\"13-05-2013T23:59:59\"},\"previousExpiryDate\"
+:{\"time\":\"14-04-2013T00:00:00\",\"volume\":\"14-04-2013T00:00:00\"
+},\"reportingLevel\":\"totalTraffic\",\"resetPeriod\":{\"volume\":\"3
+0 days\"}},\"name\":\"110\",\"restartInfo\":\"14-04-2013T00:00:00\",\
+"selected\":\"no\",\"subscriberGroupName\":\"KA_3NFUP_3G80K_30\",\"su
+bscriptionDate\":\"14-04-2013T00:00:00\",\"validityTime\":1800}],\"ve
+rsion\":\"2.0\"}")]]]
[download]