use strict; use warnings; use File::Find; use Time::localtime; use File::Copy 'cp'; use File::Copy 'mv'; use File::Path qw(make_path); use threads; use Thread::Queue; # Package for debugging need to remove use diagnostics; use Data::Dumper; sub CreateIndividualArchive { my $srcDir = shift; my $destDir = shift; my $arrayFileList = shift; my $pathDelimiter = "/"; #my $tempFile = "dump.txt"; my @chars = ( '0' .. '9', 'A' .. 'F' ); my $len = 8; my $string; while ( $len-- ) { $string .= $chars[ rand @chars ] } my $tempFile = $string; #print "Temp fils is $tempFile\n"; my $fh; my $fileName = $destDir . $pathDelimiter . $tempFile; open $fh, '>', $fileName or die "Cannot open $tempFile :$!"; my $dirName = $arrayFileList->[0]; chop($dirName); for ( @$arrayFileList[ 1 .. $#$arrayFileList ] ) { print $fh "$_\n"; } close($fh); my $tarFileList = $destDir . $pathDelimiter . $tempFile; my $tarExt = ".tar.gz"; my $tarFileName = $destDir . $pathDelimiter . $dirName . $tarExt; my $cmd = "tar -zcf $tarFileName -C $srcDir -T $tarFileList"; #print "Cmd is $cmd\n"; print "CMD = $cmd\n"; my @tarOutput = `tar -zcf $tarFileName -C $srcDir -T $tarFileList 2>&1`; if ( $? == -1 ) { print "Archiving of the files fails : $!\n"; unlink $tarFileName; return 0; } unlink $tarFileList; return 1; } sub Thread { my $hashParm = shift; my %hashFileList = %$hashParm; my $sourcePath = shift; my $destinationPath = shift; my $keys; my $values; my @arrayValues; my @rowData = (); my $totalKey = keys %hashFileList; my $pathDelimiter = "/"; #print "In thread and total keys received $totalKey\n"; while ( ( $keys, $values ) = each(%hashFileList) ) { push( @arrayValues, $keys . $pathDelimiter ); my @row = ( $keys . $pathDelimiter, $keys . $pathDelimiter, 0 ); push( @rowData, \@row ); my @arrayParm = @{$values}; foreach my $value (@arrayParm) { my $fileName = $$value[0]; my $fileSize = $$value[1]; push( @arrayValues, $fileName ); my @row = ( $keys . $pathDelimiter, $fileName, $fileSize ); push( @rowData, \@row ); } #print "SourcePath $sourcePath and dest $destinationPath\n"; my $error = CreateIndividualArchive( $sourcePath, $destinationPath, \@arrayValues ); if ( $error eq 0 ) { print "Error while doing tar is $error\n"; } @arrayValues = (); @rowData = (); } } sub ScanDirWithPattern { my $sourcePath = shift; my $hashFileList = shift; my $pathDelimiter = "/"; my $pattern = ".txt"; if ( 0 eq opendir( DIR, $sourcePath ) ) { print "Failed to open directory $sourcePath\n"; return 0; } my @dirList; if ( 0 eq ( @dirList = readdir(DIR) ) ) { print "Failed to read directory $sourcePath\n"; closedir(DIR); return 0; } closedir(DIR); foreach my $dir (@dirList) { #print "Current directory is $dir\n"; next if ( $dir eq "." or $dir eq ".." ); my $currentDir = $sourcePath . $pathDelimiter . $dir; if ( -d $currentDir ) { if ( 0 eq opendir( DIR, $currentDir ) ) { print "Failed to open directory $currentDir\n"; return 0; } my @fileList; if ( 0 eq ( @fileList = readdir(DIR) ) ) { print "Failed to read directory $dir\n"; closedir(DIR); return 0; } closedir(DIR); my @relativeFileArray; foreach my $file (@fileList) { next if ( $file eq "." or $file eq ".." ); my $currentFile = $sourcePath . $pathDelimiter . $dir . $pathDelimiter . $file; next if ( -d $currentFile ); if ( -f $currentFile ) { if ( $currentFile =~ /$pattern/i ) { my $relativeFile = $dir . $pathDelimiter . $file; my $size = -s $currentFile; my @currentFileArray = ( $relativeFile, $size ); print "Inserting the $relativeFile in array\n"; push( @relativeFileArray, \@currentFileArray ); } } } $hashFileList->{$dir} = \@relativeFileArray; } } return 1; } sub Create { my ( $sourcePath, $destinationPath ) = @_; my $pathDelimiter = "/"; my %hashFileList; my $folderName = "temp"; my $error = ScanDirWithPattern( $sourcePath, \%hashFileList ); if ( $error eq 0 ) { print "Error while scaning $sourcePath for files\n"; return 0; } my $keys; my $values; my @arrayValues; my @rowData = (); my $totalKeys = keys %hashFileList; my $numThreads = 5; #For the time being if ( $totalKeys le $numThreads ) { $numThreads = $totalKeys; } my $bucketSize = $totalKeys / $numThreads; my @keys = keys %hashFileList; my @arrThreads; my $i = 0; my @arrHash; my $tempDir = $destinationPath . $pathDelimiter . $folderName; make_path($tempDir); $destinationPath = $tempDir; while ( my @keys2 = splice @keys, 0, $bucketSize ) { my %hash1; @hash1{@keys2} = @hashFileList{@keys2}; push @arrHash, \%hash1; } for my $href (@arrHash) { my $t = threads->create( \&Thread, \%$href, $sourcePath, $destinationPath ); push( @arrThreads, $t ); } foreach (@arrThreads) { my $num = $_->join; #print "done with $num\n"; } } my $srcDir = ""; my $destDir = ""; if ( @ARGV < 2 ) { die "$0 - Need source and destination directory\n" . "Usage: perl $0 src dest\n"; } $srcDir = shift; $destDir = shift; Create($srcDir, $destDir);