in reply to Re^3: Extract CSS + JS + Image URLs from a HTML page?
in thread Extract CSS + JS + Image URLs from a HTML page?

Not the prettiest regexes, but they seem to work =) 
while ($string =~ m/\<script .*? src=\"(.+?)\Q.js"><\/script>\E/gis) {
        print "FOO: $1.js \n";
    }    

    while ($string =~ m/\<link .*? href=\"(.+?)\.css\" \/?\>/gis) {   
+ 
        print "FOO: $1.css \n";
    }

[download]
Cheers

Andy

Replies are listed 'Best First'.
Re^5: Extract CSS + JS + Image URLs from a HTML page?
by ultranerds (Hermit) on Jan 28, 2011 at 09:33 UTC
    Hi,

    Not sure if you wanna use my version, but this seems to work for CSS (JS should be a simple tweak); 
    sub get_css { 

    my $tmp = shift  ;

    my $self         ;
    my $url          ;
    my $html         ;

    my @result_arr   ;

    my $user_agent = "Html_Miner/0.01" ;
    my $timeout    = 60                ; 

    my $domain       ;
    

    ## First extract all required information.

    if( UNIVERSAL::isa( $tmp, 'HTML::Miner' )  ) { 

    $self = $tmp                        ;

    $url     =  $self->{ CURRENT_URL      } ;
    $html    =  $self->{ CURRENT_URL_HTML } ;
    $domain  =  $self->{ _BASE_DOMAIN     } ;

    } else { 
    
    $url = $tmp                         ;

    ## Check for validity of url! 
    my ( $tmp, $protocol, $domain, $uri ) =  
        _convert_to_valid_url( $url )   ;
    $url = $tmp                         ;

    my @params               = @_       ;
    my $html_has_been_passed = @params  ;

    
    if( $html_has_been_passed ) { 
        $html = shift                   ;
    } else { 

        ## Need to retrieve html 
    
        eval { 
        require LWP::UserAgent      ;
        require HTTP::Request       ;
        }; 
        croak( "LWP::UserAgent and HTTP::Request are required if the u
+rl is to be fetched!" ) 
        if( $@ );


        $html = _get_url_html( $url, $user_agent, $timeout )   ;
        
    } ## HTML Not passed


    }     ## Not called on Object.

    while( $html =~ m/\<link .*? href=\"(.+?)\.css\" \/?\>/gis ){
        push( @result_arr, "$1.css" );
    }

    return \@result_arr;

}

    [download]
    Cheers

    Andy
[reply]
[d/l]

      Thanks Andy

      Also I am sure you have thought of this but you might want to pass those links through 'HTML::Miner::get_absolute_url'

[reply]
        Hi,

        Good point =) Was just trying to get the basics going first, and then once I start looking at it more tonight, I'll look more in detail regarding how it works with relative URLs etc :)

        Thanks for such a cool modules - saved me ages trying to work out the code to do the same thing ;)

        Andy
[reply]
        BTW, another thing I just noticed (no biggie ;))

        You have this in your code: "Reletive" ... should be "Relative" :)
[reply]

In Section Seekers of Perl Wisdom