#!/usr/bin/perl -w

use strict;

chdir('dir1');

open(my $fh, '<', 'list.name') or die("open() failed: $!");

my %all = map { $_ => undef } grep { -d } glob("*");
delete($all{$_}) for ( map { chomp; $_ } <$fh> );

system('rm', '-ir', $_) for (keys(%all));

__END__
format of dir1/list.name:

dir1
dir2
dir3
dir4
dirx