#!/bin/bash

# Unix GNUbash script to monitor a dir for '*.txt' files
# containing a tab-separated pair of local-filename remote-object
# which will copy to the cloud
# if successful the file is moved to the done dir else to failed dir
# (subdirs of monitor dir)
# the process can be parallelised to up to NUMTHREADS threads using
# GNU's creme-de-la-creme sem (which uses Perl)
# The monitor dir is given as the only input param and should already exists
# The idea is that a separate Perl script will select the files to transfer
# and create a signal file inside the monitor dir containing the details
# of the transfer.
# by bliako
# for https://perlmonks.org/?node_id=11101534
# 20/06/2019
#####

NUMTHREADS=3
SLEEPTIME=2s

### nothing to change below
###
MONITOR_DIR=$1
if [ "${MONITOR_DIR}" == "" ] || [ ! -d "${MONITOR_DIR}" ]; then
	echo "$0 : a 'monitor-dir' name must be given as 1st param pointing to an existing, readable dir"
	exit 1
fi
DONE_DIR="${MONITOR_DIR}/done"
mkdir -p "${DONE_DIR}" &> /dev/null
FAILED_DIR="${MONITOR_DIR}/failed"
mkdir -p "${FAILED_DIR}" &> /dev/null
if [ ! -d "${DONE_DIR}" ] || [ ! -d "${FAILED_DIR}" ]; then
	echo "$0 : failed to create dir '${DONE_DIR}' and/or '${FAILED_DIR}'"
	exit 1
fi

function execu {
	local cmd="$1"
	local asignalfile="$2"
	local done_dir="$3"
	local failed_dir="$4"
	echo "execu() : called with cmd='${cmd}', asignalfile='${asignalfile}', done_dir='${done_dir}', failed_dir='${failed_dir}'"
	eval ${cmd}
	if [ $? -eq 0 ]; then
		echo "$0 : success executing ${cmd}" 1>&2
		mv "${asignalfile}" "${done_dir}"
	else
		echo "$0 : command has failed ${cmd}" 1>&2
		mv "${asignalfile}" "${failed_dir}"
	fi
}; export -f execu

while true; do
	nowdone=0
	while IFS= read -r -d '' afwf; do
		# we found a file in the dir we are monitoring
		# it must containing the fullpath to the file to transfer
		# then tab and then the remote
		echo "checking '${afwf}'"
		declare -a fde=($(head -1 "${afwf}" | cut -d$'\t' -f1,2))
		CMD="gsutil cp '${fde[0]}' 'gs://${fde[1]}'"
		echo "$0: executing ${CMD} ..."
		if [ "${NUMTHREADS}" -gt 1 ]; then
			echo "$0 : parallelising over ${NUMTHREADS} ..."
			sem -j${NUMTHREADS} execu "'${CMD}'" "'${afwf}'" "'${DONE_DIR}'" "'${FAILED_DIR}'"
		else
			echo "$0 : executing ..."
			execu "${CMD}" "${afwf}" "${DONE_DIR}" "${FAILED_DIR}"
		fi
	done < <(find "${MONITOR_DIR}" -maxdepth 1 -type f -name '*.txt' -print0)
	totaldone=$((${totaldone}+${nowdone}))
	echo "$0 : sleeping some before next monitor, done ${totaldone} so far"
	sleep ${SLEEPTIME} # sleep some
done