in reply to Perl script speed

The following code which does (as far as I can tell) substantially what you want except that instead of creating a plethora of tables containing a single row, it generates a table per "signal" and adds a path number column. 4000 paths generates about a 30 MB "file" and runs in under 30 seconds.

#!usr/local/bin/perl
use strict;
use warnings;
use DBI;

my $fileStr = do {local $/; <DATA>};
my ($header, $meat) = $fileStr =~ /(.*)^PATH 1(.*)END_PATH 1.*/sm;
my $dbName = 'test1.sqlite';
my $paths  = 4000;

$fileStr = join '', $header, map {"PATH $_${meat}END_PATH $_\n"} 1 .. 
+$paths;

unlink $dbName;
open my $fin, '<', \$fileStr;

my $startTick = time ();
run($fin, $dbName);
my $seconds = time () - $startTick;

print "Took $seconds for $paths paths\n";

sub run {
    my ($fin, $dbName) = @_;
    my $self     = bless {fin => $fin};
    my $dsn      = "DBI:SQLite:dbname=$dbName";
    my $userid   = "";
    my $password = "";

    $self->{dbh} = DBI->connect($dsn, $userid, $password, {RaiseError 
+=> 1})
        or die $DBI::errstr;

    my %dispatch = (
        VERSION => sub { },
        PTDEF   => sub { },
        BANNER  => \&banner,
        PATH    => \&path,
    );

    $self->{dbh}->do('BEGIN TRANSACTION;');
    while (defined (my $line = <$fin>)) {
        my ($key, $tail) = $line =~ /^(\w+)\s*(.*)/ or next;

        die "Parser doesn't understand $line" if !exists $dispatch{$ke
+y};
        $dispatch{$key}($self, $key, $tail);
    }
    $self->{dbh}->do('COMMIT TRANSACTION;');
}


sub banner {
    my ($self, $key, $tail) = @_;
    my $fin = $self->{fin};
    my @cols;
    my @values;

    while (defined (my $line = <$fin>)) {
        last if $line =~ /END_$key/;

        my ($key, $value) = $line =~ /\{([^}]+)\}/g or next;
        push @cols,   $key;
        push @values, $value;
    }

    $self->table_create('FileInfo', @cols);
    $self->insertRow('FileInfo', @values);
}


sub path {
    my ($self, $key, $tail) = @_;
    my $fin      = $self->{fin};
    my %dispatch = (
        REQ_CLC         => \&rasGate,
        SLK_CLC         => \&rasGate,
        ARR_CLC         => \&rasGate,
        LAUNCH_CLK_PATH => \&ldcGate,
        DATA_PATH       => \&ldcGate,
        CAP_CLK_PATH    => \&ldcGate,
    );

    $self->{pathNum} = $tail;
    print "Processing path $tail\n";

    while (defined (my $line = <$fin>)) {
        last if $line =~ /END_$key/;

        my ($key, $tail) = $line =~ /^\s+(\w+)\s*(.*)/ or next;
        warn "Parser doesn't understand $line", next if !exists $dispa
+tch{$key};
        $dispatch{$key}($self, $key, $tail);
    }
}


sub rasGate {
    my ($self, $key, $tail) = @_;
    my $fin = $self->{fin};
    my @cols;
    my @values;
    my $rasToken;

    while (defined (my $line = <$fin>)) {
        last if $line =~ /END_$key/;

        my (undef, $key, $value) = $line =~ /\{([^}]+)\}/g or next;
        push @cols,   $key;
        push @values, $value;
    }

    $self->table_create($key, 'Path', @cols);
    $self->insertRow($key, $self->{pathNum}, @values);
}


sub ldcGate {
    my ($self, $key, $tail) = @_;
    my $fin      = $self->{fin};
    my %dispatch = (
        COLUMNS => \&ldcColumns,
        INST    => \&instLine,
        HPIN    => \&ldcLine,
        NET     => \&ldcLine,
        PORT    => \&ldcLine,
    );

    $self->{ldcKey} = $key;
    $self->{rows}   = [];

    while (defined (my $line = <$fin>)) {
        last if $line =~ /END_$key/;

        my ($key, $tail) = $line =~ /^\s*(\w+)\s*(.*)/ or next;
        die "Parser doesn't understand $line" if !exists $dispatch{$ke
+y};
        $dispatch{$key}($self, $key, $tail);
    }

    $self->{ldcSth}->execute($self->{pathNum}, @$_) for @{$self->{rows
+}};
}


sub ldcLine {
    my ($self, $key, $tail) = @_;

    push @{$self->{rows}},
        [map {defined ($_) ? $_ : ''} $tail =~ /\{([^}]*)\}/g];
}


sub instLine {
    my ($self, $key, $tail) = @_;
    my @values = map {defined ($_) ? $_ : ''} $tail =~ /\{([^}]*)\}/g;

    $values[-3] = "$values[-3]$values[-2]";
    splice @values, $#values - 1, 1;
    push @{$self->{rows}}, \@values;
}


sub ldcColumns {
    my ($self, $key, $tail) = @_;
    my @cols = $tail =~ /\{([^}]+)\}/g;

    $self->table_create($self->{ldcKey}, 'Path', @cols);

    my $places = join ', ', ('?') x @cols;
    my $stmt = qq(INSERT INTO '$self->{ldcKey}' VALUES(?, $places););

    $self->{ldcSth} = $self->{dbh}->prepare($stmt) or die $DBI::errstr
+;
}


sub table_create {
    my ($self, $tbname, @cols) = @_;
    my $columns = "'" . join ("', '", @cols) . "'";

    return if $self->{haveTable}{$tbname}++;

    my $sth = $self->{dbh}->prepare("CREATE TABLE $tbname ($columns)")
+;
    my $rv = $sth->execute() or die $DBI::errstr;

    if ($rv < 0) {
        print $DBI::errstr;
    } else {
        #    print "Table $tbname created";
    }
}


sub insertRow {
    my ($self, $tbname, @data) = @_;
    my $places = join ', ', ('?') x @data;
    my $stmt   = qq(INSERT INTO $tbname VALUES($places););
    my $sth    = $self->{dbh}->prepare($stmt) or die $DBI::errstr;

    $sth->execute(@data) or die $DBI::errstr;
}

[download]

Tack the following __DATA__ to the end of the script before running it, or alter the code to use an external file.

__DATA__
VERSION {1.0}
PTDEF {instance} {pin} {cell} {edge} {clock_edge} {clock} {phase}
BANNER
  {Module} {xyz}
  {Timing} {EARLY}
  {Slew Propagation} {WORST}
  {Operating Condition} {W_125_0.99_0.99}
  {PVT Mode} {min}
  {Tree Type} {balanced}
  {Process} {3.0000}
  {Voltage} {0.9900}
  {Temperature} {125.0000}
  {time unit} {1.0000 ps}
  {capacitance unit} {1.0000 fF}
  {resistance unit} {1.0000 kOhm}
  {TOOL} {v13.22-s020_1 ((64bit) 11/20/2013 12:28 (Linux 2.6))}
  {DATE} {May 16, 2014}
END_BANNER
PATH 1
  VIEW  func_STRONG_LT_MINC
  CHECK_TYPE {Removal Check}
  REF {xsd_gen/xdiv_sd/DLSB/reg2_reg} {clk}
  ENDPT {xsd_gen/xdiv_sd/DLSB/reg2_reg} {clrz} {dffcqs_f1_fs_dh} {^} {
+leading} {ADPLLS_CLKOUT} {ADPLLS_CLKOUT(C)(P)(func_STRONG_LT_MINC)*}
  BEGINPT {xsd_gen/div_code_u_reg_reg_1_} {q} {sdffpqs_f2_dh_xsvt} {^}
+ {leading} {DIVSD} {DIVSD(D)(P)(func_STRONG_LT_MINC)*}
  REQ_CLC
    {} {Other End Arrival Time} {307.3350}
    {+} {Removal} {61.1000}
    {+} {Phase Shift} {0.0000}
    {-} {CPPR Adjustment} {11.9700}
    {-} {Cycle Adjustment} {0.0000}
    {+} {Uncertainty} {20.0000}
    {=} {Required Time} {376.4650}
  END_REQ_CLC
  SLK_CLC
    {} {Arrival Time} {580.6149}
    {} {Slack Time} {204.1499}
  END_SLK_CLC
  SLK 204.1499
  ARR_CLC
    {} {Clock Rise Edge} {0.0000}
    {+} {Drive Adjustment} {1.3300}
    {=} {Beginpoint Arrival Time} {1.3300}
  END_ARR_CLC
  LAUNCH_CLK_PATH
    COLUMNS {instance} {fpin} {fedge} {tpin} {tedge} {net} {cell} {del
+ay} {incr_delay} {slew} {load} {arrival} {required} {stolen} {fanout}
+ {pin_location} {adjustment}
    PORT {} {adplls_clk} {^} {} {} {adplls_clk} {} {} {} {6.6000} {2.0
+255} {1.3300} {-202.8199} {} {1} {(0.00, 0.00) } {} 
    NET {} {} {} {} {} {adplls_clk} {} {0.4750} {0.0000} {6.6000} {2.0
+255} {1.8050} {-202.3449} {} {} {} {} 
    INST {dft_mux_adplls1_clk/tiboxv_clk_mx2_mux_0} {a} {^} {y} {^} {}
+ {ctmux2_f4} {40.4700} {0.0000} {17.9000} {} {42.2750} {-161.8749} {}
+ {6} {(16.47, 7.88) } {(16.11, 7.58)} {} 
    NET {} {} {} {} {} {mx_fref_adplls_clk} {} {1.1400} {0.0000} {18.1
+000} {17.3511} {43.4150} {-160.7349} {} {} {} {} 
    INST {dft_core_clock_leaker/func_clk_in_gate_dt/tiboxv_clk_icg_icg
+_0} {clkin} {^} {clkout} {^} {} {icg_f4} {26.4100} {0.0000} {7.8000} 
+{} {69.8250} {-134.3249} {} {1} {(66.33, 12.83) } {(65.43, 12.53)} {}
+ 
    NET {} {} {} {} {} {dft_core_clock_leaker/func_clk_in_gate} {} {0.
+0950} {0.0000} {7.8000} {2.4991} {69.9200} {-134.2299} {} {} {} {} 
    INST {dft_core_clock_leaker/func_clk_mux_dt/tiboxv_clk_mx2_mux_0} 
+{a} {^} {y} {^} {} {ctmux2_f4} {43.7950} {0.0000} {21.9000} {} {113.7
+150} {-90.4349} {} {9} {(64.22, 13.72) } {(64.58, 14.03)} {} 
    NET {} {} {} {} {} {cleaker_clk} {} {1.4250} {0.0000} {22.1000} {2
+1.7218} {115.1400} {-89.0099} {} {} {} {} 
    INST {icg_clk_983mhz/tiboxv_clk_icg_icg_0} {clkin} {^} {clkout} {^
+} {} {icg_f4} {85.5000} {0.0000} {103.6000} {} {200.6400} {-3.5099} {
+} {137} {(68.58, 8.78) } {(67.68, 9.08)} {} 
    NET {} {} {} {} {} {clk_2ghz} {} {9.8800} {0.0000} {105.1000} {174
+.7581} {210.5200} {6.3701} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/U0_clkin_gate0/tiboxh_clk_or2_or_0} {a} {^} 
+{y} {^} {} {ctor2_b4} {44.9350} {0.0000} {13.3000} {} {255.4550} {51.
+3051} {} {1} {(48.20, 31.88) } {(47.12, 31.73)} {} 
    NET {} {} {} {} {} {xsd_gen/xdiv_sd/clkin_0_} {} {0.2850} {0.0000}
+ {13.3000} {2.6745} {255.7400} {51.5901} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/U0_clkin0_buf/tiboxh_clk_buf_buf_0} {a} {^} 
+{y} {^} {} {ctb_f4} {21.9450} {0.0000} {9.7000} {} {277.6850} {73.535
+1} {} {5} {(47.07, 30.53) } {(48.33, 30.53)} {} 
    NET {} {} {} {} {} {xsd_gen/xdiv_sd/clkout_gdc_gated} {} {0.3800} 
+{0.0000} {9.7000} {10.3982} {278.0650} {73.9151} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/DLSB/reg4_reg} {clk} {^} {q} {^} {} {dffcqs_
+f1_fs_dh} {94.0500} {0.0000} {51.7000} {} {372.1150} {167.9651} {} {4
+} {(52.61, 40.12) } {(53.19, 41.27)} {DIVSD Adj. = 0.0000} 
    NET {} {} {} {} {} {xsd_gen/xdiv_sd/clkout_gdc} {} {3.8950} {0.000
+0} {51.8000} {11.1143} {376.0100} {171.8600} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/clkout_mux2/tiboxh_clk_mx2_mux_0} {b} {^} {y
+} {^} {} {ctmux2_f4} {64.1250} {0.0000} {34.2000} {} {440.1350} {235.
+9850} {} {8} {(54.36, 46.27) } {(53.28, 45.98)} {} 
    NET {} {} {} {} {} {clkgen_obs_clk} {} {4.1800} {0.0000} {34.5000}
+ {47.5230} {444.3150} {240.1650} {} {} {} {} 
  END_LAUNCH_CLK_PATH 
  DATA_PATH 
    COLUMNS {instance} {fpin} {fedge} {tpin} {tedge} {net} {cell} {del
+ay} {incr_delay} {slew} {load} {arrival} {required} {stolen} {fanout}
+ {pin_location} {adjustment} 
    INST {xsd_gen/div_code_u_reg_reg_1_} {clk} {^} {q} {^} {} {sdffpqs
+_f2_dh_xsvt} {39.3000} {0.0000} {16.0000} {} {483.6150} {279.4650} {}
+ {4} {(59.22, 41.18) } {(60.66, 41.18)} {} 
    NET {} {} {} {} {} {xsd_gen/div_code_u_reg_1_} {} {0.6000} {0.0000
+} {16.0000} {4.8318} {484.2150} {280.0650} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/g397} {a} {^} {y} {v} {} {inv_f1_xsvt} {16.6
+000} {0.0000} {12.5000} {} {500.8150} {296.6650} {} {3} {(56.97, 39.2
+3) } {(57.15, 39.08)} {} 
    NET {} {} {} {} {} {xsd_gen/xdiv_sd/n_16} {} {0.5000} {0.0000} {12
+.5000} {3.5621} {501.3150} {297.1650} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/g384} {b} {v} {y} {^} {} {nand3_f0p33_xsvt} 
+{17.0000} {0.0000} {10.6000} {} {518.3149} {314.1650} {} {1} {(56.72,
+ 37.73) } {(56.84, 37.43)} {} 
    NET {} {} {} {} {} {xsd_gen/xdiv_sd/n_7} {} {0.1000} {0.0000} {10.
+6000} {0.8820} {518.4149} {314.2650} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/g375} {c2} {^} {y} {^} {} {aoa112_f1_xsvt} {
+62.2000} {0.0000} {12.2000} {} {580.6149} {376.4650} {} {1} {(54.86, 
+37.73) } {(53.95, 37.58)} {} 
    NET {} {} {} {} {} {xsd_gen/xdiv_sd/n_25} {} {0.0000} {0.0000} {12
+.2000} {1.2445} {580.6149} {376.4650} {} {} {} {} 
  END_DATA_PATH
  OTHER_ARR_CLC
    {} {Clock Rise Edge} {0.0000}
    {+} {Drive Adjustment} {1.4700}
    {=} {Beginpoint Arrival Time} {1.4700}
  END_OTHER_ARR_CLC
  CAP_CLK_PATH
    COLUMNS {instance} {fpin} {fedge} {tpin} {tedge} {net} {cell} {del
+ay} {incr_delay} {slew} {load} {arrival} {required} {stolen} {fanout}
+ {pin_location} {adjustment}
    PORT {} {adplls_clk} {^} {} {} {adplls_clk} {} {} {} {6.6000} {2.1
+483} {1.4700} {205.6199} {} {1} {(0.00, 0.00) } {} 
    NET {} {} {} {} {} {adplls_clk} {} {0.5250} {0.0000} {6.6000} {2.1
+483} {1.9950} {206.1449} {} {} {} {} 
    INST {dft_mux_adplls1_clk/tiboxv_clk_mx2_mux_0} {a} {^} {y} {^} {}
+ {ctmux2_f4} {44.7300} {0.0000} {17.9000} {} {46.7250} {250.8749} {} 
+{6} {(16.47, 7.88) } {(16.11, 7.58)} {} 
    NET {} {} {} {} {} {mx_fref_adplls_clk} {} {1.2600} {0.0000} {18.1
+000} {17.5437} {47.9850} {252.1349} {} {} {} {} 
    INST {dft_core_clock_leaker/func_clk_in_gate_dt/tiboxv_clk_icg_icg
+_0} {clkin} {^} {clkout} {^} {} {icg_f4} {29.1900} {0.0000} {7.8000} 
+{} {77.1750} {281.3249} {} {1} {(66.33, 12.83) } {(65.43, 12.53)} {} 
    NET {} {} {} {} {} {dft_core_clock_leaker/func_clk_in_gate} {} {0.
+1050} {0.0000} {7.8000} {2.6219} {77.2800} {281.4299} {} {} {} {} 
    INST {dft_core_clock_leaker/func_clk_mux_dt/tiboxv_clk_mx2_mux_0} 
+{a} {^} {y} {^} {} {ctmux2_f4} {48.4050} {0.0000} {21.9000} {} {125.6
+850} {329.8349} {} {9} {(64.22, 13.72) } {(64.58, 14.03)} {} 
    NET {} {} {} {} {} {cleaker_clk} {} {1.5750} {0.0000} {22.1000} {2
+2.3134} {127.2600} {331.4099} {} {} {} {} 
    INST {icg_clk_983mhz/tiboxv_clk_icg_icg_0} {clkin} {^} {clkout} {^
+} {} {icg_f4} {94.5000} {0.0000} {103.7000} {} {221.7600} {425.9099} 
+{} {137} {(68.58, 8.78) } {(67.68, 9.08)} {} 
    NET {} {} {} {} {} {clk_2ghz} {} {10.9200} {0.0000} {105.1000} {18
+2.0893} {232.6800} {436.8299} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/U0_clkin_gate0/tiboxh_clk_or2_or_0} {a} {^} 
+{y} {^} {} {ctor2_b4} {49.6650} {0.0000} {13.3000} {} {282.3450} {486
+.4949} {} {1} {(48.20, 31.88) } {(47.12, 31.73)} {} 
    NET {} {} {} {} {} {xsd_gen/xdiv_sd/clkin_0_} {} {0.3150} {0.0000}
+ {13.3000} {2.8831} {282.6600} {486.8099} {} {} {} {} 
    INST {xsd_gen/xdiv_sd/U0_clkin0_buf/tiboxh_clk_buf_buf_0} {a} {^} 
+{y} {^} {} {ctb_f4} {24.2550} {0.0000} {9.7000} {} {306.9150} {511.06
+49} {} {5} {(47.07, 30.53) } {(48.33, 30.53)} {} 
    NET {} {} {} {} {} {xsd_gen/xdiv_sd/clkout_gdc_gated} {} {0.4200} 
+{0.0000} {9.7000} {10.7594} {307.3350} {511.4849} {} {} {} {} 
  END_CAP_CLK_PATH
END_PATH 1

[download]
Perl is the programming world's equivalent of English

Replies are listed 'Best First'.
Re^2: Perl script speed
by poj (Abbot) on May 31, 2014 at 06:24 UTC

    I think the regex in rasGate should to be

    my (undef, $key, $value) = $line =~ /\{([^}]*)\}/g or next; # * not +

    to parse the empty brackets {} correctly otherwise you get a column called 0.0000 in the table.

    ARR_CLC
    {} {Clock Rise Edge} {0.0000}
    {+} {Drive Adjustment} {1.3300}
    {=} {Beginpoint Arrival Time} {1.3300}
END_ARR_CLC

    [download]
    poj
[reply]
[d/l]
[select]
Re^2: Perl script speed
by rr27 (Initiate) on Jun 02, 2014 at 11:19 UTC
    Thanx a lot!! But I am facing an error of  DBD::SQLite::st execute failed: called with 16 variables when 17 are needed This is at the line 
     $self->{ldcSth}->execute($self->{pathNum}, @$_) for @{$self->{rows}};
}

    [download]
    of ldcGate. Please can you explain how to solve this error.
[reply]
[d/l]
[select]

      Am I right in thinking that some of lines only have 16 columns, ie no adjustment ?

      COLUMNS {instance} {fpin} {fedge} {tpin} {tedge} {net} {cell} {del
+ay} {incr_delay} {slew} {load} {arrival} {required} {stolen} {fanout}
+ {pin_location} {adjustment}

      [download]
      Update:
      I ask because in your code you have these parts which I guess deal with the extra column by concatenating 2 together 
      if ($l=~/adjustment/) {
  $flag=1; 
}

      [download]
      and 
      if ($flag==1 && $_=~/INST/) {
  for($i=0;$i<$#parts-2;++$i) {
     push(@data,$parts[$i]);
  }
  push(@data,"$parts[$i]$parts[$i+1]");
  push(@data,$parts[$i+2]);
} else {
  foreach $l (@parts) {
     push(@data,$l);
  }
}

      [download]

      poj
[reply]
[d/l]
[select]
        ya, I have portions in the file which have extra column. but it is not the problem area. Even the columns having 16 without adjustment is showing this error.
[reply]

In Section Seekers of Perl Wisdom