How about this:

#! /usr/bin/perl -w
use strict;
use Data::Dumper;

my %data;

while (<DATA>)
{
  chomp;
  my ($column, $value) = /^\s*(\S+):\s*(.*?)$/;
  my @fields = $value =~ /"([^"]*")/g;
  push @{$data{$column}}, @fields;
}

print Dumper (\%data);

__DATA__
COLUMN_A: "Y","N"
  COLUMN_B:
COLUMN_C: "something", "something else", "and more"
[download]