#!/usr/local/bin/perl -w
#
# Version 0.1
# http://www.dwoptimize.com/2007/05/data-type-validation-using-regular
+.html
# jag.singh@dwoptimize.com
#
# 1. Read specification file that defines the data file layout:
# 1.1. attribute name
# 1.2. attribute data type using regular expressions (http://www.perl.
+com/pub/a/2000/11/begperl3.html)
# 1.3. default data value
#
# 2. Validate data file for data type; replace file attribute data wit
+h default value
# if data type does not match specification
#
# 3. Create output data file with bad data values replaced by the defa
+ult values
#
# 4. Create log file with results of data validation
#
# 5. Abort data validation process, if total number of errors reach ma
+x_errors
#
($spec_file, $data_in_file, $data_out_file, $log_file, $max_errors) =
+@ARGV; # read command line parameters
open (log_file, ">$log_file") or die "Can not open file $log_file, $!"
+;
open (spec_file, "$spec_file") or die "Can not open file $spec_file, $
+!";
open (data_in_file, "$data_in_file") or die "Can not open file $data_i
+n_file, $!";
open (data_out_file, ">$data_out_file") or die "Can not open file $dat
+a_out_file, $!";
print log_file "Spec File> ", $spec_file, "\n", "Data In File> ", $dat
+a_in_file, "\n",
"Data Out File> ", $data_out_file, "\n", "Log File> ", $log_file, "\
+n", "Max errors: ", $max_errors, "\n";
#
foreach $spec_line () { # Read full data file specification into memor
+y from the spec file,
# this will be used for "lookup"
chomp ($spec_line); # remove the newline from $spec_line.
@spec_one_attribute = split(/\,/, $spec_line); # the spec file is ',
+' delimited
# @spec_one_attribute contains: attribute name, attribute data typ
+e (regular expression), and default value
push (@spec_all_attributes, [@spec_one_attribute]); # @spec_all_attr
+ibutes contain the full data file specification
}
#
$line_number = 1; $total_errors = 0;
DATALINE: foreach $data_in_file () { # read data file, line by line
chomp ($data_in_file); # remove the newline
@data_in_attributes = split (/\|/, $data_in_file); # the data file i
+s '|' delimited
if ($#data_in_attributes != $#spec_all_attributes) {
# number of attributes on the data line do not match with the spec
+ification
$total_errors++;
print log_file "Error ", $total_errors, ". On the data line: ", $l
+ine_number,
", # attributes: ", $#data_in_attributes + 1,
", do not match # attributes in the file specification: ", $#spe
+c_all_attributes + 1, "\n";
last DATALINE if ($total_errors >= $max_errors); # terminate if to
+o many errors
next; # skip data attribute type validation
}
$attribute_position = 0; @data_out_attributes = ();
foreach $attribute (@data_in_attributes) {
if ($attribute =~ m/$spec_all_attributes[$attribute_position][1]/)
+ { # validate data attribute type by performing
# lookup for the regular expression from the spec memory structu
+re
push (@data_out_attributes, $attribute); # Correct data type, th
+e output value is same as input value
} else {
push (@data_out_attributes, $spec_all_attributes[$attribute_posi
+tion][2]);
# Bad data type, use default provided in the spec for output v
+alue
$total_errors++;
print log_file "Error ", $total_errors, ". Data type error on li
+ne: ", $line_number,
", attribute: ", $attribute_position + 1, " (", $spec_all_attr
+ibutes[$attribute_position][0], ")\n";
}
last DATALINE if ($total_errors >= $max_errors); # terminate if to
+o many errors
$attribute_position++;
}
print data_out_file join ("|", @data_out_attributes), "\n"; # the da
+ta out file is '|' delimited
} continue { # update line number counter even if the data attribute t
+ype validation is skipped
$line_number++;
}
#
if ($total_errors >= $max_errors) {
print log_file "Max error count reached: ", $total_errors, ", proces
+s terminated\n";
} else {
print log_file "Process completed with: ", $total_errors, " errors\n
+";
}
# End
-
Are you posting in the right place? Check out Where do I post X? to know for sure.
-
Posts may use any of the Perl Monks Approved HTML tags. Currently these include the following:
<code> <a> <b> <big>
<blockquote> <br /> <dd>
<dl> <dt> <em> <font>
<h1> <h2> <h3> <h4>
<h5> <h6> <hr /> <i>
<li> <nbsp> <ol> <p>
<small> <strike> <strong>
<sub> <sup> <table>
<td> <th> <tr> <tt>
<u> <ul>
-
Snippets of code should be wrapped in
<code> tags not
<pre> tags. In fact, <pre>
tags should generally be avoided. If they must
be used, extreme care should be
taken to ensure that their contents do not
have long lines (<70 chars), in order to prevent
horizontal scrolling (and possible janitor
intervention).
-
Want more info? How to link
or How to display code and escape characters
are good places to start.