#!/usr/local/bin/perl -w
#
# Version 0.1
# http://www.dwoptimize.com/2007/05/data-type-validation-using-regular.html
# jag.singh@dwoptimize.com
#
# 1. Read specification file that defines the data file layout:
# 1.1. attribute name
# 1.2. attribute data type using regular expressions (http://www.perl.com/pub/a/2000/11/begperl3.html)
# 1.3. default data value 
#
# 2. Validate data file for data type; replace file attribute data with default value
#    if data type does not match specification
#
# 3. Create output data file with bad data values replaced by the default values
#
# 4. Create log file with results of data validation
#
# 5. Abort data validation process, if total number of errors reach max_errors
#
($spec_file, $data_in_file, $data_out_file, $log_file, $max_errors) = @ARGV; # read command line parameters
open (log_file, ">$log_file") or die "Can not open file $log_file, $!";
open (spec_file, "$spec_file") or die "Can not open file $spec_file, $!";
open (data_in_file, "$data_in_file") or die "Can not open file $data_in_file, $!";
open (data_out_file, ">$data_out_file") or die "Can not open file $data_out_file, $!";
print log_file "Spec File> ", $spec_file, "\n", "Data In File> ", $data_in_file, "\n", 
  "Data Out File> ", $data_out_file, "\n", "Log File> ", $log_file, "\n", "Max errors: ", $max_errors, "\n";
#
foreach $spec_line () { # Read full data file specification into memory from the spec file, 
  # this will be used for "lookup"
  chomp ($spec_line); # remove the newline from $spec_line.
  @spec_one_attribute = split(/\,/, $spec_line); # the spec file is ',' delimited
    # @spec_one_attribute contains: attribute name, attribute data type (regular expression), and default value
  push (@spec_all_attributes, [@spec_one_attribute]); # @spec_all_attributes contain the full data file specification
}
#
$line_number = 1; $total_errors = 0;
DATALINE: foreach $data_in_file () { # read data file, line by line
  chomp ($data_in_file); # remove the newline
  @data_in_attributes = split (/\|/, $data_in_file); # the data file is '|' delimited
  if ($#data_in_attributes != $#spec_all_attributes) { 
    # number of attributes on the data line do not match with the specification
    $total_errors++;
    print log_file "Error ", $total_errors, ". On the data line: ", $line_number, 
      ", # attributes: ", $#data_in_attributes + 1, 
      ", do not match # attributes in the file specification: ", $#spec_all_attributes + 1, "\n";
    last DATALINE if ($total_errors >= $max_errors); # terminate if too many errors
    next; # skip data attribute type validation
  }      
  $attribute_position = 0; @data_out_attributes = ();
  foreach $attribute (@data_in_attributes) {
    if ($attribute =~ m/$spec_all_attributes[$attribute_position][1]/) { # validate data attribute type by performing 
      # lookup for the regular expression from the spec memory structure
      push (@data_out_attributes, $attribute); # Correct data type, the output value is same as input value
    } else {
      push (@data_out_attributes, $spec_all_attributes[$attribute_position][2]); 
        # Bad data type, use default provided in the spec for output value
      $total_errors++;
      print log_file "Error ", $total_errors, ". Data type error on line: ", $line_number, 
        ", attribute: ", $attribute_position + 1, " (", $spec_all_attributes[$attribute_position][0], ")\n";
    }
    last DATALINE if ($total_errors >= $max_errors); # terminate if too many errors
    $attribute_position++;
  }
  print data_out_file join ("|", @data_out_attributes), "\n"; # the data out file is '|' delimited
} continue { # update line number counter even if the data attribute type validation is skipped
  $line_number++;
}
#
if ($total_errors >= $max_errors) {
  print log_file "Max error count reached: ", $total_errors, ", process terminated\n";
} else {
  print log_file "Process completed with: ", $total_errors, " errors\n";
}
# End