Okay, I hacked up Term::Visual a bit and came up with a patch one can apply to Term/Visual.pm:

@@ -148,6 +148,7 @@

   $kernel->alias_set( $alias );
   $console = POE::Wheel::Curses->new( InputEvent => 'private_input');
+  use_default_colors();
   my $old_mouse_events = 0;
   mousemask(0, $old_mouse_events);

@@ -1354,6 +1355,7 @@
      re      => COLOR_RED,      red     => COLOR_RED,
      wh      => COLOR_WHITE,    white   => COLOR_WHITE,
      ye      => COLOR_YELLOW,   yellow  => COLOR_YELLOW,
+        de              => -1,                         default => -1,
    );

   my %attribute_table =
[download]

Works a charm now ^_^ you just have to set colors as 'de' or 'default'.