splunk_colddb_migration/tasks/rsync_until_completed.sh

#! /bin/bash
#
# rsync takes a long time and can timeout, so repeat
# until it's done.
# 
# Recommendation, in another window/screen session, run:
#
FOLDERS=(did-it-aws elis2-aws-pii)
TARGET=AWS-Indexers
SLEEPTIME=300 # Time to rest between retries
FORKS=10 # Maximum number of parallel actions

function join_by { local IFS="$1"; shift; echo "$*"; } # joins strings
FOLDERS_SHELL=\{$(join_by , ${FOLDERS[@]})\}

echo Recommendation: In another window/screen session, run:
echo 	watch --interval=30 \"ansible $TARGET --sudo --sudo-user=splunk -m shell -a \'du --summarize -h /opt/splunk/var/lib/\{splunk,splunkcold\}/$FOLDERS_SHELL/colddb\*\'\"

# Store our resullt
result=1
while [[ $result -ne 0 ]] 
do
  result=0
  for folder in "${FOLDERS[@]}"
  do
    echo Synchronizing ${folder}...
    time ansible-playbook rsync_colddb.yml --forks=$FORKS --extra-vars="target=$TARGET folder=${folder}"
    result=$((result+$?))
  done
  if [[ $result -ne 0 ]]
  then
    echo Not finished. Resting for $SLEEPTIME seconds...
    sleep 300
  fi
done