#!perl
use strict;
use warnings;
# start time
my $t0 = time();

# set up
my $src_file = 'src_file.txt';
my $out_file = 'out_file.txt';
my $hash_ref = { DELIMITER => '\|'};
my @lookup =(); my @header =(); my %fh =();

##Reading layout
#ACCT NUMBER|positionfile.delim|2|6
while (<DATA>){
  chomp;
  s/ +$//g;
  my ($head,$filename,$key,$val) = split /$hash_ref->{DELIMITER}/;
  push @header, $head;
  push @lookup,[ $filename ,$key-1,$val ];
  # open look-up files
  unless ( exists $fh{$filename} ){
    print "Open $filename\n";
    open $fh{$filename},'<',$filename or die "Could not open $filename
+";
  }
}

##Reading data file and printing final file
open my $SRC,'<',$src_file or die "Could not open $src_file";
open my $OUT,'>',$out_file or die "COuld not open $out_file";
print $OUT join '|',@header,"\n";
print "Processing $src_file\n";
my $count = 0;
while (<$SRC>){
  my @in = split '\|',$_;
  my @out   =();
  my %cache =();
  for my $ar (@lookup){
    my $filename = $ar->[0];
    my $key      = $in[$ar->[1]];
    unless ( exists $cache{$filename}{$key} ){
      $cache{$filename}{$key} = find_record($filename,$key); 
    }
    # get require column from record found in $filename
    push @out,$cache{$filename}{$key}->[$ar->[2]];
  }
  # print complete record
  print $OUT join '|',@out,"\n";
  ++$count;
  print "$count lines processed\n" unless ($count % 100);
}

## Close filehandles
for (keys %fh){
  print "Close file $_\n";
  close $fh{$_};
}

## Performance
my $dur = time()-$t0;
print "$count records written to $out_file in $dur seconds\n";

# Search for key in file
sub find_record {
  my ($filename,$key) = @_;
  seek $fh{$filename} , 0, 0;
  while ( readline($fh{$filename}) ){
    if (/\|$key\|/){
      chomp;
      return [ split /$hash_ref->{DELIMITER}/ ];
    }
  }
  warn "$key not found in $filename\n";
  return [];
}

__DATA__
ACCT NUMBER|positionfile.delim|2|6
ACCT SHORT NAME|accountfile.delim|6|6
PO TYPE|positionfile.delim|2|3
CUSIP|securityfile.delim|8|5
LOC CODE|positionfile.delim|2|47
LOC NAME|locationfile.delim|47|4
[download]