If you could translate ':' to '.', you could use:

  use strict;
  use Set::Infinite;

  my @array = (
      [ '03.30', '05.45' ],
      [ '05.45', '09.15' ],
      [ '10.00', '11.35' ],
      [ '11.00', '15.40' ]
    );

  my $set = Set::Infinite->new;

  $set = $set->union( $_ ) for @array;

  print "intervals: $set\n";

  my $gaps = $set->complement->intersection( '00.00' , '24.00' );

  print "gaps: $gaps\n";

# intervals: [03.30..09.15],[10.00..15.40]
# gaps: [00.00..03.30),(09.15..10.00),(15.40..24.00]
[download]

update: tested.