Like this?

sub dequote {
   local $_ = @_ ? $_[0] : $_;
   return $_ unless (s/^"(.*)"$/$1/s);
   s/\\(.)/$1/gs;
   return $_;
}

my @data;
while (<DATA>) {
   chomp;
   my @fields;
   $_ = $_ . ',';
   push(@fields, $1) while (/\G("(?:[^"\\]|\\.)*"|[^,]*)(?:,)/g);
   @fields = map { dequote } @fields;
   push(@data, \@fields);
}

require Data::Dumper;
print(Data::Dumper::Dumper(\@data));

__DATA__
"hi",3,20.6,"green","32"
16,"alpha",0.00
[download]