perl -ne '
 if(!/^2/) {
   $k = substr($_, 6, 6) . substr($_, 29, 10) . substr($_, 54, 12);
   print "$k|$_" 
 }'  newdata | sort -t "|" -k 1,1 >newdata.sorted

 join -t '|' -v 1 newdata.sorted alreadyprocessed  >needsprocessing
 cut -d '|' -f 2 needsprocessing >processinput

 # Then do the processing
 
 # ...
 # ...

 # If everything runs okay
 cut -d '|' -f 1 needsprocessing | 
         sort -m - alreadyprocessed >mergeout
 mv alreadyprocessed alreadyprocessed.bak
 mv mergeout alreadyprocessed