New Row: blah blah blah pattern found/not found, set department to ... final result: blah blah blah #### #!/usr/bin/perl use strict; use warnings; use WWW::Mechanize; use Data::Dumper; use HTML::TableExtract; use XML::FeedPP; use UTF8; use Encode qw(encode decode); # initialize my $cols; my $url; my $depth; my $count; my $data; my $tracking_code; my $location; my $job_title; my $date_posted; my $out_fh; my $key; my $value; my $department; my $input; # -------------------------------------------------------------------------------------- # Setup # -------------------------------------------------------------------------------------- # get the data from the web. The URL for CommVault's job listings is: # https://commvault.silkroad.com/epostings/index.cfm?fuseaction=app.jobsearch# # Either pass this in as --url when invoking or just set it. $url = "https://commvault.silkroad.com/epostings/index.cfm?fuseaction=app.jobsearch"; # the headings on the table columns for the job listings are, in order: $cols = 'tracking_code,job_title,location,date_posted'; # use a hash of patterns to look for within the job titles. If a pattern matches, assign the # category to the department label and stop processing. # the hash of jobs triggers to categories is below. Eventually this will just be a text # file to make maintenance easier - for now, just hardcode a couple of sample patterns and # category pairs. my %job_categories = ( 'account manager' => "Sales", 'systems engineer' => "PreSales" ); # define the output areas my $directory = "/Users/coblem/testing/"; my $outfile = "cvlt_jobs.csv"; open( $out_fh, '>', $ directory . $outfile) or die("Unable to create output file \"$out_fh\": $!\n"); print ("------ START -------------------- START -------------------- START -------------", "\n"); # -------------------------------------------------------------------------------------- # Processing # -------------------------------------------------------------------------------------- # bring in the table from the URL, break everything up into the columns and iterate over the rows # looking for patterns. if we find a match, set the department to the hash value. # first, bring in the row and extract along the column fields. my $m = WWW::Mechanize->new(); $m->get($url); $input = $m->content; my $te; if ( defined ($cols)) { my @headers = split(/,/, $cols); $te = HTML::TableExtract->new( headers => [ 'Tracking Code', 'Job Title', 'Location', 'Date Posted' ] ) or die qq{$!}; } # we shouldn't be in this section. the error handling below is not yet sophisticated enough. # fortunately they are not using embedded tables or lots of XML to sort through so we # shouldn't hit this. else { $te = new HTML::TableExtract( depth => $depth, count=>$count); }; # second, iterate over each row, looking for a pattern from the hash of categories. # start with breaking up the row into the fields from the column headings $te->parse($input); foreach my $row ($te->rows) { $tracking_code = $ { $row }[0]; $job_title = $ { $row }[1]; $location = $ { $row }[2]; $date_posted = $ { $row }[3]; print ("new row\: $tracking_code $job_title $location $date_posted \n"); # now look for a $key pattern to match inside $job_title. # If a key matches, set department to the category, $value # -------------------------------------------------------------------------------------- # THE WHILE LOOP STARTS HERE, ALONG WITH THE PROBLEM # -------------------------------------------------------------------------------------- while ( ($key, $value) = each %job_categories ) { print ( "looking in $job_title for\: $key \n"); if ($job_title =~ /$key/i) { $department = $value; print ( "found a match. setting department to $value \n"); last; } else { $department = 'Undefined'; print ( "no match found. department is now $department \n"); } } print (" final result\: $job_title \=\> $department \n \n"); # -------------------------------------------------------------------------------------- # Close out and clean-up # close file handles, any other items # not yet implemented # -------------------------------------------------------------------------------------- } #### ------ START -------------------- START -------------------- START ------------- new row: 306145-636 Sales Account Manager - Enterprise Seattle, Washington, United States 10/31/2013 looking in Sales Account Manager - Enterprise for: systems engineer no match found. department is now Undefined looking in Sales Account Manager - Enterprise for: account manager found a match. setting department to Sales final result: Sales Account Manager - Enterprise => Sales new row: 306144-636 Inside Sales Administrator Madrid, Madrid, Spain 10/30/2013 final result: Inside Sales Administrator => Sales new row: 306143-636 Inside Sales Administrator Milano, Lombardia, Italy 10/30/2013 looking in Inside Sales Administrator for: systems engineer no match found. department is now Undefined looking in Inside Sales Administrator for: account manager no match found. department is now Undefined final result: Inside Sales Administrator => Undefined new row: 306134-636 Senior Technical Consultant / Enterprise Solutions Architect Reading, West Berkshire, United Kingdom 10/30/2013 looking in Senior Technical Consultant / Enterprise Solutions Architect for: systems engineer no match found. department is now Undefined looking in Senior Technical Consultant / Enterprise Solutions Architect for: account manager no match found. department is now Undefined final result: Senior Technical Consultant / Enterprise Solutions Architect => Undefined new row: 306142-636 Product Manager - Database Oceanport, New Jersey, United States 10/29/2013 looking in Product Manager - Database for: systems engineer no match found. department is now Undefined looking in Product Manager - Database for: account manager no match found. department is now Undefined final result: Product Manager - Database => Undefined new row: 306141-636 Professional Services Project Manager Pleasanton, California, United States 10/29/2013 looking in Professional Services Project Manager for: systems engineer no match found. department is now Undefined looking in Professional Services Project Manager for: account manager no match found. department is now Undefined final result: Professional Services Project Manager => Undefined new row: 306140-636 Customer Support Engineer - Contract-to-Hire Oceanport, New Jersey, United States 10/29/2013 looking in Customer Support Engineer - Contract-to-Hire for: systems engineer no match found. department is now Undefined looking in Customer Support Engineer - Contract-to-Hire for: account manager no match found. department is now Undefined final result: Customer Support Engineer - Contract-to-Hire => Undefined new row: 306139-636 Systems Engineer - Houston Houston, Texas, United States 10/29/2013 looking in Systems Engineer - Houston for: systems engineer found a match. setting department to PreSales final result: Systems Engineer - Houston => PreSales new row: 306137-636 Systems Engineer New York, New York, United States 10/29/2013 looking in Systems Engineer for: account manager no match found. department is now Undefined final result: Systems Engineer => Undefined