/fs001
/fs001/ITC
/fs001/ITC/LSC/
/fs001/ITC/LSC/users
/fs001/ITC/LSC/users/my_user
/fs001/ITC/LSC/users/some_user
/fs001/ITC/LSC/users/other_user

##</code><code>##

#!/usr/bin/perl

use File::Find;
use FileHandle;
use POSIX;
use URI::Escape;
use Time::HiRes qw ( usleep );

use strict;
use warnings;

#proggie to processs filesystems on celerra and work out which departments
#own what.

#5Mb, if it's smaller then it doesn't bother recursing.
my $size_threshold = 5242880;
my $config_file = "/usr/local/apache/htdocs/dusage/disk_usage.conf";
my @dirs = ( "/fs001", "/fs002", "/fs003", "/fs004", "/fs005", "/fs006", 
 	     "/fs007", "/fs008", "/fs009", "/fs010", "/fs011", "/fs012", );
#my @dirs = ( "/SiteWide/home/erolison" );
my $debug = 0;
my $global_recurse = 5; #default value but used for some interesting calcs.
my @excl_directories = ( '.', '..' ); #cos they're unusual
my %is_excluded;
for ( @excl_directories ) { $is_excluded{$_} = 1; }

my %sizes;
my %customers;
my %totals;

my ( $allfiles, $tenk, $fiftyk, $hundredk, $twohundredk, $fivehundredk, $onemeg );

sub data
{
  my $factor = 1024;
  my @sequence = ('b ', 'kb', 'Mb', 'Gb', 'Tb' );
  foreach my $input_number (@_)
  {
    my $seq_num = 0;
    my $output_number = $input_number;
    if ( ! $input_number ) { return sprintf("%3.2fb", 0 ); }
    while ( $output_number / $factor >= 1 )
    {
      $seq_num++;
      $output_number /= $factor;
    }
    return sprintf("%3.2f$sequence[$seq_num]", $output_number);
  }
}

sub getsize
{
#watch out for this. It's a _very_ expensive call.
#look out for optimisations lower down in the chain
  my @args = @_;
  my $sum = 0;
 
  if ( $debug ) { print "Getting size for @args\n"; }

  find sub { if ( -s ) { 
      $sum += -s ;
      $allfiles++;
      if  ( -s > 10240 ) { $tenk++ }
      if ( -s > 51200 ) { $fiftyk++ }
      if ( -s > 102400 ) { $hundredk++ }
      if ( -s > 204800 ) { $twohundredk++ }
      if ( -s > 512000 ) { $fivehundredk++ }
      if ( -s > 1048576 ) { $onemeg++ }
    }; 
    if ( $debug ) { usleep(1000) 
  } }, @args;

  return $sum;
}

sub get_size_of_files
{
  my @args= @_;
  my $sum = 0;
  if ( $debug ) { print "Sizing files in @args\n"; }
  foreach my $iggle ( @args )
  {
    opendir(IDIR, $iggle) or 
      print "WARNING: Couldn't open $iggle\n";
    while ( my $fname = readdir ( IDIR ) )
    {
      #print ( "$iggle/$fname" );
      if ( ( -s "$iggle/$fname" ) )
	{ $sum += -s "$iggle/$fname" }
    }
    close (IDIR);
  }
  return $sum;
}

sub dusage
{
  my $startpoint = shift(@_) || '.';
  my $recurse_depth = shift(@_) || 0;
  my @dusage_list;
  #function to show disk usage of all subdirectories.

  if ( $debug ) { print "Reading $startpoint\n"; }
  if ( -d $startpoint)
  {
    #if ( $recurse_depth-- > 0  && $stuff{$startpoint} > $size_threshold )
    if ( $recurse_depth-- > 0 )
    {
      $sizes{$startpoint} = get_size_of_files($startpoint);
      if ( $debug ) { print "adding $sizes{$startpoint} to $startpoint\n"; }

      #my $tmp = $startpoint;
      #$tmp =~ s,/[A-Za-z0-9_\.\,\- ]+$,,g;
      # $stuff{$tmp} += $stuff{$startpoint};
      # if ( $debug ) { print "in lo adding value of $startpoint ( $stuff{$startpoint} ) to $tmp = $stuff{$tmp}\n" };
    
      opendir ( DIR, $startpoint);
      while ( my $filename = readdir(DIR) )
      {
        if ( -d "$startpoint/$filename" && !($is_excluded{$filename}) )
        {
          $dusage_list[++$#dusage_list] = "$startpoint/$filename";
        }
      }
      for my $dir (@dusage_list)
      {
        dusage("$dir", $recurse_depth);
      }
    }
    else
    {
   #only process the expensive bit, if we're not going to recurse 'deeper'
      $sizes{$startpoint} = getsize($startpoint);
    }
  }
}


sub do_output
{
  my $target = pop(@_);
  my $base_indent = ( $target =~ tr,/,, );
  $target =~ s,/,,g;
  if ( $debug ) { print "$target\n"; }

  open ( FILESTAT, ">file_sizes${target}.html" );
  print FILESTAT "Total number of files = ", $allfiles, "<BR/>\n";
  print FILESTAT "files over 10k in size = ", $tenk, "<BR/>\n";
  print FILESTAT "files over 50k in size = ", $fiftyk, "<BR/>\n";
  print FILESTAT "files over 100k in size = ", $hundredk, "<BR/>\n";
  print FILESTAT "files over 200k in size = ", $twohundredk, "<BR/>\n";
  print FILESTAT "files over 500k in size = ", $fivehundredk, "<BR/>\n";
  print FILESTAT "files over 1M in size = ", $onemeg, "<BR/>\n";
  close ( FILESTAT );

  open ( REPORT, ">disk_usage${target}.xml" );
  open ( CSV, ">disk_usage${target}.csv" );
  my %output = %sizes;
  my %basic_sizes = %sizes;
  
  if ( $debug ) 
  {
    foreach my $item ( sort ( keys ( %sizes ) ) ) 
    {
      print ("directory size: $sizes{$item} = $item \n");
    }
  }
  while ( keys(%sizes) )
  {
    foreach my $value ( sort ( keys ( %sizes ) ) ) 
    {
      my $upd = $value;
      $upd =~ s,/[A-Za-z0-9_\.\,\- ]+$,,g;
      if ( $debug) { print "upd = $upd value = $value\n"; }
      if ( ! ( "$upd" eq "$value" ) )
      {
        $output{$upd} += $sizes{$value};
        $sizes{$upd} += $sizes{$value};
        if ( $debug ) { print "adding $value ( $sizes{$value} ) to $upd\n" }
      }
      delete($sizes{$value});
    }
  }
  if ( $debug ) 
  {
    foreach my $item ( sort ( keys ( %output ) ) )
    {
      print ("$output{$item} = $item \n");
    }
  }
  print REPORT "<?xml version=\"1.0\" encoding=\"ISO8859-1\" ?>\n";
  print REPORT "<?xml-stylesheet type=\"text/xsl\" href=\"/dusage/disk_usage.xsl\"?>\n";
  print REPORT "<ALL>\n";
  print REPORT "<TITLE>Disk usage report for $target </TITLE>\n";
  print REPORT "<DATE>",strftime("%d/%m/%y", localtime(time)), "</DATE>\n";
  print REPORT "<BYDIRECTORY>\n";
  print REPORT "<TITLE>Listing by Directory</TITLE>\n";

  my $current_indent = -1;

  foreach my $item ( sort ( keys ( %output ) ))
  {
    #if ( $output{$item} > $size_threshold )
    #{
      my $base_object = $item; 
      $base_object =~ s,.*/,/,g;
      my $indent_depth = ( $item =~ tr,/,, );
      #my $indent_html = join("", "<L DIR=\"",$base_object,"\" DEPTH=\"", $indent_depth,"\" SIZE=\"", data($output{$item}),"\">\n");
      my $indent_html = join("", 
                           "<L", $indent_depth,
                           " DIR=\"",
                           uri_escape($base_object,"^A-Za-z0-9\-_.!~+ *'()\/"),
                           "\" SIZE=\"", data($output{$item}),
                           "\" DEPTH=\"", $indent_depth, "\">"
                            );
      #now we work out who 'owns' that data by doing substring matches
      #with the config array.
      
      my @dir_list = split("/", $item);
      my $owner = "";
      while ( !$owner && @dir_list )
      {
        my $srch_string = join("/", @dir_list);
                         #the 'dir' to look for in the customers array.
                         #might or might not have a trailing '/'
        $srch_string =~ 's,/$,,g'; #strip training /
        #if ( $debug ) { print "$item: checking for \"$srch_string\"\n"; }
        if ( $customers{$srch_string} ) 
   	{
	   $owner = $customers{$srch_string};
        }
        pop (@dir_list);
      }
      if ( $debug ) { print "$owner\n"; }
      if ( $debug ) { print "$indent_depth to $current_indent\n"; }
      if ( $indent_depth <= $current_indent ) 
      {  
        for ( my $i = $current_indent; $i >= $indent_depth; $i-- )
        {
           print REPORT "</L",$i,">\n";
        }
      }
      if ( $indent_depth > $current_indent + 1 )
      {
        for ( my $i = $current_indent + 1; $i > $indent_depth; $i++ )
        {
          print REPORT "<L",$i," PATH=\"more...\" DEPTH=\"$i\">";
        }
      }

      $current_indent = $indent_depth;
      print REPORT "\n",$indent_html,"\n";
      print REPORT "<SIZE>", data($output{$item}), "</SIZE>\n";
      print REPORT "<DIR>", uri_escape($base_object,"^A-Za-z0-9\-_.!~+ *'()\/"), "</DIR>\n";
      print REPORT "<FULL_PATH>", uri_escape($item,"^A-Za-z0-9\-_.!~+ *'()\/"), "</FULL_PATH>\n";
      print REPORT "<BSIZE>", $output{$item}, "</BSIZE>\n";
      my ( $lcust, $ldept ) = split(":", $owner);
      if ( not $lcust ) { $lcust = "Unknown"; };
      if ( not $ldept ) { $ldept = "Unknown"; };
      print REPORT "<CUSTOMER>", $lcust, "</CUSTOMER>\n";
      print REPORT "<DEPT>", $ldept,"</DEPT>\n";

      if ( $debug ) 
      { 
        printf ("%${indent_depth}s", data($output{$item})); 
        print ("\t $base_object\n"); 
      }
      if ( !$owner ) { $owner = "unknown:unknown" };

      my ( $customer, $dept ) = split (":", $owner );
      unless ( $totals{$customer}{'Total'}{'du'} ) 
           { $totals{$customer}{'Total'}{'du'} = 0; }
      if ( $basic_sizes{$item} )
      {
        $totals{$customer}{'Total'}{'du'} += $basic_sizes{$item}; 
      }
      push ( @{$totals{$customer}{'Total'}{'dirs'}}, $item );
      if ( $debug ) { print "$customer $dept = $basic_sizes{$item}\n"; }
      if ( $basic_sizes{$item} )
      {
        $totals{$customer}{$dept}{'du'} += $basic_sizes{$item}; 
      }
      push ( @{$totals{$customer}{$dept}{'dirs'}}, $item ); 
      #push ( @customer_chain, join(" ", $owner, $item, data($output{$item}) ) );
    #} #if size
  }
  for ( my $i = $current_indent; $i >= 0; $i--)
  {
    print REPORT "</L",$i,">\n";
  }

  print REPORT "</BYDIRECTORY>\n\n<BYCUSTOMER>\n";
  if ( $debug ) { print "Listing by Customer and Department\n"; }
  print REPORT "<TITLE>Listing by Customer and Department</TITLE>\n";
  print CSV "Customer, Dept, total usage (bytes),\n";
  foreach my $customer ( sort ( keys ( %totals ) ) )
  {
    foreach my $dept ( sort ( keys ( %{$totals{$customer}} ) ) )
    {
      if ( $debug ) { print "$customer $dept ", data($totals{$customer}{$dept}{'du'}), "\n"; }
      #print REPORT "<TR><TD>$customer $dept ", data($totals{$customer}{$dept}{'du'}), "</TD></TR>\n";
      print CSV $customer,",",$dept,",",$totals{$customer}{$dept}{'du'},",\n";
      print REPORT "<CUSTOMER G=\"$customer $dept\"><NAME>$customer</NAME>";
      print REPORT "<DEPT>", $dept, "</DEPT>";
      print REPORT "<USAGE>", data($totals{$customer}{$dept}{'du'});
      print REPORT "</USAGE></CUSTOMER>\n";
    }
  }

print REPORT "</BYCUSTOMER></ALL>\n";
close REPORT;
close CSV;
}

#MAIN

#print get_size_of_files("test");

my ($idir) = @ARGV;

if ( -f $config_file ) 
{
  open ( CONF, "$config_file");
  while ( <CONF> )
  {
    chomp;
    my ( $fs, $cust, $dept ) = split(":");
    $customers{$fs} = join(":", $cust, $dept);
    if ( $debug ) { print "got $fs - $cust - $dept\n"; }
  }
  close ( CONF );
}

if ( $debug ) { print keys ( %customers ); } 

STDOUT -> autoflush(1);

if ( ! $idir )
{ 
  foreach my $dir ( @dirs )
  {
    if ( $debug ) { print "\nSTARTING $dir\n"; }
    dusage ( $dir, $global_recurse );
  }
  do_output ( "ALL" );
}
else
{
  dusage ( $idir, $global_recurse );
  do_output($idir);
}