use Tie::File;
use Fcntl;
 

#First year you want downloaded files for for:
my $startyear=2014;
#Last year you want files for:

my $endyear=2014;
#First qtr you want  files for (usually 1):
my $startqtr=1;

#Last year you want files for (usually 4):

my $endqtr=4;
#The directory you want your index files to be stored in.
my $inddirect="/Volumes/EDGAR1/Edgar/full-index";
#The directory you  are going to download filings to
my $direct="/Volumes/EDGAR1/Edgar/Edgar2/10K_10Q";
#The file that will contain the filings you want to download.
my $outfile="/Volumes/EDGAR1/Edgar/sizefiles1.txt";

my $formget1='(10-K )';
my $formget2='(10-K405 )';
my $formget3='(10KSB )';
my $formget4='(10-KSB )';
my $formget5='(10KSB40 )';
my $formget6='(10-KT )';
my $formget7='(10KT405 )';
my $formgetq1='(10-Q )';
my $formgetq2='(10QSB )';
my $formgetq3='(10-QSB )';
my $formgetq4='(10-QT )';
#if using windows, set to "\\" - if mac (or unix), set to "/";
my $slash='/';

#loop through all the index years you specfied

for($yr=$startyear;$yr<=$endyear;$yr++)

{
#loop through all the index quarters you specified
if($yr<$endyear){$eqtr=4}else{$eqtr=$endqtr}
for($qtr=$startqtr;$qtr<=$eqtr;$qtr++)

{
#Open the index file
open(INPUT, "$inddirect/company$qtr$yr.idx") || die "file for company$qtr$yr.idx: $!";
#Open the file you want to write to.  The first time through
#the file is opened to "replace" the existing file.
#After that, it is opened to append ">>".

if ($yr==$startyear && $qtr==$startqtr)
{$outfiler=">$outfile";}
else{$outfiler=">>$outfile";}
open(OUTPUT, "$outfiler") || die "file for 2006 1: $!";
$count=1;

while ($line=<INPUT>)

    {

#ignore the first 10 lines because they only contain header information

if ($.<11) {next};

$form_type=substr($line,62,12);
my $cik=substr($line,74,10);

$file_date=substr($line,86,10);

$file_date=~s/\-//g;
my $fullfilename=trim(substr($line,98,43));


if ($form_type=~/^$formget1(?!\/)/) 
    
{
    print OUTPUT "$fullfilename\n" ;
    
    $count++;
}
elsif ($form_type=~/^$formget2(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}
elsif ($form_type=~/^$formget3(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}
elsif ($form_type=~/^$formget4(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}
elsif ($form_type=~/^$formget5(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}
elsif ($form_type=~/^$formget6(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}
elsif ($form_type=~/^$formget7(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}

elsif ($form_type=~/^$formgetq1(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}
elsif ($form_type=~/^$formgetq2(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}
elsif ($form_type=~/^$formgetq3(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}
elsif ($form_type=~/^$formgetq4(?!\/)/)
{
    print OUTPUT "$fullfilename\n" ;
    
   $count++;
}

#if ($count>10){last;}
#end of the while loop <INPUT>
   }

close(INPUT);
close(OUTPUT);
# check to see if directory exists.  If not, create it.
unless(-d "$direct$slash$yr"){
    mkdir("$direct$slash$yr") or die;
}
#Open the directory and get put the names of all files into the array @old
opendir(DIR,"$direct$slash$yr")||die "Can't open directory";
@Old=readdir(DIR);

tie(@New1,Tie::File,"$outfile", mode=> O_RDWR)
or die "Cannot tie file BOO: $!n";
%seen=();
#defines an array called @aonly.
@aonly=();
foreach $item(@Old){$seen{$item}=1}
foreach $item(@New1){
         $item=~/(edgar\/data\/.*\/)(.*\.txt)/;
    unless($seen{$item}){
        push(@aonly,$item);

    }

}

ftpsignin();

foreach $filetoget(@aonly)

{
#    $filetoget=trim($filetoget);
    $fullfile="/$filetoget";
    $fonly=$filetoget;
    #Don't forget to put your directory in here.
    $fonly=~s/.*\/(.*)/$direct$slash$yr$slash$1/;
    #$ftp->get("$fullfile", "$fonly")
    #or  warn "can't get file",ftpsignin(),next; # "cannot get file",$ftp->message, next;
    

}
    $ftp->quit;
#end of qtr loop
}
#end of year loop
}

sub ftpsignin {
    use Net::FTP;
     $ftp = Net::FTP->new("ftp.sec.gov", Debug => 0, Passive => 1)
     or die "Cannot connect to some.host.name: $@";
     
     $ftp->login("anonymous",'-anonymous@')
     or next; #die "Cannot login ", $ftp->message;
     $ftp->binary(); # set binary mode 
       

}   

sub trim {
    my $new_phrase;
    my $phrase = shift(@_);
    $phrase =~ s/^\s+//;
    $phrase =~ s/\s+$//;
    $new_phrase = "$phrase";
    return "$new_phrase";
     }