use strict; use warnings; use Selenium::Remote::Driver; use LWP::UserAgent; use Data::Dumper; my $wait_period = $ENV{ SP_WAIT_PERIOD } ? $ENV{ SP_WAIT_PERIOD } : 5; my $ua = LWP::UserAgent->new( "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0" ); $ua->default_header( 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'); $ua->default_header( 'Accept-Encoding' => 'gzip, deflate, br' ); $ua->default_header( 'Accept-Language' => 'de,en-US;q=0.7,en;q=0.3' ); $ua->default_header( 'Cache-Control' => 'no-cache' ); $ua->default_header( 'Connection' => 'keep-alive' ); $ua->default_header( 'DNT' => 1 ); $ua->default_header( 'Host' => 'myibd.investors.com' ); $ua->default_header( 'Pragma' => 'no-cache' ); $ua->default_header( 'Referer' => 'https://www.investors.com/' ); $ua->default_header( 'Upgrade-Insecure-Requests' => 1 ); #print STDERR "REQUEST HEADER FIELDS:\n"; #print STDERR join( "\n", $ua->default_headers->header_field_names() ); my $selen=Selenium::Remote::Driver->new( browser_name => 'firefox', ua => $ua ); $selen->debug_on; print STDERR "INFO: Created new Web Driver instance.\n"; my $url="https://myibd.investors.com/secure/signin.aspx?eurl=https://www.investors.com/"; $selen->set_timeout( 'page load', 60000); $selen->set_timeout( 'implicit', 100); $selen->get( $url ); sleep( $wait_period ); my $fh; open( $fh, ">", "first_page.htm"); print $fh $selen->get_page_source(); close( $fh ); my $elem; #eval{ $elem = $selen->find_element( "(//button[\@id='gdpr-accept-button'])" ); }; eval{ $elem = $selen->find_element( "(//a[\@id='signin-link'])" ); }; if ( $@ ) { die( "ERROR: Could not locate element using xpath " . "//a[\@id='signin-link']." ); } my $trials = 0; my $retcode; while ( $trials < 10 ) { eval { $retcode = $elem->click() }; if ( $@ ) { warn( "ERROR: Could not navigate to login page. $@" ); } if ( $retcode ) { print STDERR "Click successful, waiting for page to load ...\n"; sleep( 20 ); last; } $trials++; print STDERR "\$retcode: $retcode\n"; sleep( 1 ); } open( $fh, ">", "second_page.htm"); print $fh $selen->get_page_source(); close( $fh ); #### NFO: Created new Web Driver instance. REQ: POST, http://localhost:4444/wd/hub/session/349dc119-3d7d-48a6-921e-ff9041eeb3f2/timeouts, {"type":"page load","ms":60000} RES: {"state":"success","sessionId":null,"hCode":2038984705,"value":{},"class":"org.openqa.selenium.remote.Response","status":0} REQ: POST, http://localhost:4444/wd/hub/session/349dc119-3d7d-48a6-921e-ff9041eeb3f2/timeouts, {"type":"implicit","ms":100} RES: {"state":"success","sessionId":null,"hCode":1528004735,"value":{},"class":"org.openqa.selenium.remote.Response","status":0} REQ: POST, http://localhost:4444/wd/hub/session/349dc119-3d7d-48a6-921e-ff9041eeb3f2/url, {"url":"https://myibd.investors.com/secure/signin.aspx?eurl=https://www.investors.com/"} RES: {"state":"success","sessionId":null,"hCode":100444837,"value":{},"class":"org.openqa.selenium.remote.Response","status":0} REQ: GET, http://localhost:4444/wd/hub/session/349dc119-3d7d-48a6-921e-ff9041eeb3f2/source, {} RES: {"state":"success","sessionId":null,"hCode":1543153839,"value":"\n\n\nInvestor's Business Daily | GDPR Agreement...","class":"org.openqa.selenium.remote.Response","status":0} Wide character in print at ./stock_scraper.pl line 66. REQ: POST, http://localhost:4444/wd/hub/session/349dc119-3d7d-48a6-921e-ff9041eeb3f2/element, {"using":"xpath","value":"(//a[@id='signin-link'])"} RES: {"state":"no such element","sessionId":null,"hCode":1918752997,"value":{"additionalInformation":"\nDriver info: driver.version: unknown","localizedMessage":"Unable to locate element: (//a[@id='signin-link'])\nFor documentation on this error, please visit: http://seleniumhq.org/exceptions/no_such_element.html\nBuild info: version: '3.5.3', revision: 'a88d25fe6b', time: '2017-08-29T12:54:15.039Z'\nSystem info: host: 'v32523.1blu.de', ip: '178.254.37.13', os.name: 'Linux', os.arch: 'amd64', os.version: '3.16.0-042stab127.2', java.version: '1.8.0_172'\nDriver info: driver.version: unknown","supportUrl":"http://seleniumhq.org/exceptions/no_such_element.html","systemInformation":"System info: host: 'v32523.1blu.de', ip: '178.254.37.13', os.name: 'Linux', os.arch: 'amd64', os.version: '3.16.0-042stab127.2', java.version: '1.8.0_172'","cause":null,"stackTrace":[null,null,null,null,null,null,null,null,null,null,null,null,null,null],"suppressed":[],"message":"Unable to locate element: (//a[@id='signin-link'])\nFor documentation on this error, please visit: http://seleniumhq.org/exceptions/no_such_element.html\nBuild info: version: '3.5.3', revision: 'a88d25fe6b', time: '2017-08-29T12:54:15.039Z'\nSystem info: host: 'v32523.1blu.de', ip: '178.254.37.13', os.name: 'Linux', os.arch: 'amd64', os.version: '3.16.0-042stab127.2', java.version: '1.8.0_172'\nDriver info: driver.version: unknown","hCode":1792296,"class":"org.openqa.selenium.NoSuchElementException","buildInformation":null},"class":"org.openqa.selenium.remote.Response","status":7} ERROR: Could not locate element using xpath //a[@id='signin-link']. at ./stock_scraper.pl line 76. REQ: DELETE, http://localhost:4444/wd/hub/session/349dc119-3d7d-48a6-921e-ff9041eeb3f2, {} RES: {"state":"success","sessionId":null,"hCode":998727408,"value":{},"class":"org.openqa.selenium.remote.Response","status":0} #### Investor's Business Daily | Login investors.com IBD University | Welcome