#!/usr/bin/env perl
use IO::Socket::INET;
 
# auto-flush on socket
$| = 1;
 
# creating a listening socket
my $socket = new IO::Socket::INET (
    LocalHost => '0.0.0.0',
    LocalPort => '5101',
    Proto => 'tcp',
    Listen => 10,
    Reuse => 1
);
die "cannot create socket $!\n" unless $socket;
print "server waiting for client connection on port 5101\n";
 
while(1)
{
    # waiting for a new client connection
    my $client_socket = $socket->accept();
 
    # get information about a newly connected client
    my $client_address = $client_socket->peerhost();
    my $client_port = $client_socket->peerport();
    print "connection from $client_address:$client_port\n";
 
    # read up to 16 characters from the connected client
    my $theirtime = 0;
    $client_socket->recv($theirtime, 16);
    print "received data: $theirtime\n";
	$theirtime =~s/[^\d]//g; # untaint
 
    # write response data to the connected client
	my $their_drift = time - $theirtime ;
    $client_socket->send($their_drift);
 
    # notify client that response has been sent
    shutdown($client_socket, 1);
}
 
$socket->close();

__END__

Problem

Your program has forked and you want to tell the other end that you're done sending data. You've tried close on the socket, but the remote end never gets an EOF or SIGPIPE.
Solution

Use shutdown:

shutdown(SOCKET, 0);                # I/we have stopped reading data
shutdown(SOCKET, 1);                # I/we have stopped writing data
shutdown(SOCKET, 2);                # I/we have stopped using this socket