good evening - hello dear commmuntiy. i currently work out how to parse a site with a table (containing 6150 records) see here http://192.68.214.70/km/asps/schulsuche.asp?q=n&a=20 in order to get a good result with comma seperated values...: i have to make use of the Text::CSV - module 

#!/usr/bin/perl
    use strict;
    use warnings;
    use HTML::TableExtract;
    use LWP::Simple;
    use Text::CSV;
    use Cwd;
    use POSIX qw(strftime);
    my $te = HTML::TableExtract->new;
    my $total_records = 0;
    my $suchbegriffe = "e";
    my $treffer = 50;
    my $range = 0;
    my $url_to_process = "http://192.68.214.70/km/asps/schulsuche.asp?
+e=";
    my $processdir = "processing";
    my $counter = 50;
    my $displaydate = "";
    my $percent = 0;

    &workDir();
    chdir $processdir;
    &processURL();
    print "\nPress <enter> to continue\n";
    <>;
    $displaydate = strftime('%Y%m%d%H%M%S', localtime);
    open OUTFILE, ">webdata_for_$suchbegriffe\_$displaydate.txt";
    &processData();
    close OUTFILE;
    print "Finished processing $total_records records...\n";
    print "Processed data saved to $ENV{HOME}/$processdir/webdata_for_
+$suchbegriffe\_$displaydate.txt\n";
    unlink 'processing.html';
    die "\n";

    sub processURL() {
    print "\nProcessing $url_to_process$suchbegriffe&a=$treffer&s=$ran
+ge\n";
    getstore("$url_to_process$suchbegriffe&a=$treffer&s=$range", 'temp
+file.html') or die 'Unable to get page';

       while( <tempfile.html> ) {
          open( FH, "$_" ) or die;
          while( <FH> ) {
             if( $_ =~ /^.*?(Treffer <b>)(d+)( - )(d+)(</b> w+ w+ <b>)
+(d+).*/ ) {
                $total_records = $6;
                print "Total records to process is $total_records\n";
                }
             }
             close FH;
       }
       unlink 'tempfile.html';
    }

    sub processData() {
       while ( $range <= $total_records) {
          getstore("$url_to_process$suchbegriffe&a=$treffer&s=$range",
+ 'processing.html') or die 'Unable to get page';
          $te->parse_file('processing.html');
          my ($table) = $te->tables;
          for my $row ( $table->rows ) {
             cleanup(@$row);
             print OUTFILE "@$row\n";
          }
          $| = 1; 
          print "Processed records $range to $counter";
          print "\r";
          $counter = $counter + 50;
          $range = $range + 50;
          $te = HTML::TableExtract->new;
       }
    }

#    sub cleanup() {
#      for ( @_ ) {
#          s/s+/ /g;
#      }
#    }


my $html= get 'http://192.68.214.70/km/asps/schulsuche.asp?q=a&a=20';
$html =~ tr/\r//d;     # strip carriage returns
$html =~ s/&nbsp;/ /g; # expand spaces

my $te = new HTML::TableExtract();
$te->parse($html);

my @cols = qw(
    rownum
    number
    name
    phone
    type
    website
);

my @fields = qw(
    rownum
    number
    name
    street
    postal
    town
    phone
    fax
    type
    website
);

my $csv = Text::CSV->new({ binary => 1 });

foreach my $ts ($te->table_states) {
    foreach my $row ($ts->rows) {

        #  trim leading/trailing whitespace from base fields
        s/^\s+//, s/\s+$// for @$row;

        # load the fields into the hash using a "hash slice"
        my %h;
        @h{@cols} = @$row;

        # derive some fields from base fields, again using a hash slic
+e
        @h{qw/name street postal town/} = split /\n+/, $h{name};
        @h{qw/phone fax/} = split /\n+/, $h{phone};

        #  trim leading/trailing whitespace from derived fields
        s/^\s+//, s/\s+$// for @h{qw/name street postal town/};

        $csv->combine(@h{@fields});
        print $csv->string, "\n";
    }
}

#what do you think!? 
#
# 
    sub workDir() {
    # Use home directory to process data
    chdir or die "$!";
    if ( ! -d $processdir ) {
       mkdir ("$ENV{HOME}/$processdir", 0755) or die "Cannot make dire
+ctory $processdir: $!";
       }
    }

[download]




what do you think - i get some nasty errors with this.. But the concept is all right! I have some encoding errorss - utf-8 / iso 8859-1



In reply to HTML::TableExtract - combined with Text::CSV - character issues rise by Perlbeginner1

Title:
Use:  <p> text here (a paragraph) </p>
and:  <code> code here </code>
to format your post, it's "PerlMonks-approved HTML":


  • Posts are HTML formatted. Put <p> </p> tags around your paragraphs. Put <code> </code> tags around your code and data!
  • Titles consisting of a single word are discouraged, and in most cases are disallowed outright.
  • Read Where should I post X? if you're not absolutely sure you're posting in the right place.
  • Please read these before you post! —
  • Posts may use any of the Perl Monks Approved HTML tags:
    a, abbr, b, big, blockquote, br, caption, center, col, colgroup, dd, del, details, div, dl, dt, em, font, h1, h2, h3, h4, h5, h6, hr, i, ins, li, ol, p, pre, readmore, small, span, spoiler, strike, strong, sub, summary, sup, table, tbody, td, tfoot, th, thead, tr, tt, u, ul, wbr
  • You may need to use entities for some characters, as follows. (Exception: Within code tags, you can put the characters literally.)
            For:     Use:
    & &amp;
    < &lt;
    > &gt;
    [ &#91;
    ] &#93;
  • Link using PerlMonks shortcuts! What shortcuts can I use for linking?
  • See Writeup Formatting Tips and other pages linked from there for more info.