# call:
#	[gap=N #h]
#	mirror SOURCE_RSYNC_URL DESTINATION_DIRECTORY [FILTER...]
# Creates files in ~ directory.
# 2009-04-23
# 2011-09-14 .. min ago -> + .. h ago
# 2011-09-23 same change in another place
# 2011-09-29 default gap - 2 h
# 2011-11-17 default rsync options and path
# 2011-11-22 -f to force mirroring (despite the $gap time has not passed); floting point calculations; -h - help; -n - dry run
# 2011-11-28 printf formats corrections; -29 \n before "MIRRORING"
# 2011-12-12 $p alias for /packages
# 2011-12-13 date after "MIRRORING"
# 2011-12-30 corr. for fractional gaps; typo
# 2011-01-02 check rsync error and recover the old time stamp
# 2012-01-31 --timeout=3600 spotted in Debian's ftpsync; rsync error was checked reversed!
# 2013-11-08 +$optionslocal for rsync to enable adding options to standard $options - if $options are to be changed, the script should redeclare them
# 2014-01-10 reset LC_NUMERIC to "C" - just unsetting it is not enough for printf command
# 2014-01-29 $options: +--chmod=og-w cos openmandriva mirror uses 666/777 permissions
# 2014-05-15 if default destination path is empty use $p / basename $0; remote duplicate slashes, e.g. in /$p (//packages)
# 2015-01-22 e-mail on error, to whoami by default; better rsync return code check
# 2015-02-20 recently mirrored -> recently successfully mirrored ... last attempt
# 2015-02-25 check if .stamp, .filter or .stamp.old files exist (recent mirroring: date of .filter, recent successfull mirroring: date od .stamp)
# 2016-10-13 defoptions variable for use in restoring options when changed in a script
# 2016-11-19 do not run rsync if free space low
# 2017-03-22 add 0 to free space to cover case of non-existent fs
# 2017-04-12 adding 0 not enough, we should handle $NR==0
# 2017-05-30,06-01 main switch
# 2017-09-20 load limit; 09-21 error correction, RUNNING… line on start
# 2017-01-18 on load to high inform about the limit; check existence of the destination directory before df; ... -> …
# 2018-03-09 count mirror even if no target dir
# 2018-03-10 show load in MIRRORING line; 04-16 do not round it
# 2018-05-25 cmp load *1000
# 2018-07-27 load presented in original form by injecting .
# 2019-04-08 -F rises maxloadp to 50.00
# 2019-08-03 success->Success; summarising mirror-success script
# 2020-02-26 set defoptions and options in the script's main part; check load in every call to mirror()
# 2020-09-02 --delete-before when space low; 09-04 delfree: 16->64 GB; 09-06 defoptions: + -y; 09-30 ~/.mirror-functions configration file
# 2020-10-10 earlier report about last mirroring; message an e-mail about being above the gap; using te previous time stamp if we encounter the mirror running already; show mirror # in file; repeat[onerror] to try several times, after a random delay
# 2020-10-16 waitload=1 to wait on low load; print information about mirroring result even when skipping
# 2020-10-18 only one instance should wait to not to exhaust pids: move low load waiting after locking; manipulate .stamp files after locking and low load waiting to not to touch to early
# 2021-09-11 report more on errors, print version; 09-12 the default mirrordir should be $p/`basename $remote1 not $p/$name [of the script]
# 2021-10-07 -vanished from the list of errors cos it is temporary; 10-20 non-empty e-mails about error only; 10-22 + missing [
# 2024-02-29 / after $mirrordir@line 159 to avoid being fooled by a symlink
# TODO: file marker: UPDATING; list recent times option; -w option to wait for lock relasing instead of skipping (for regular random mirrors); timeout rsync for redhat archives; TIMEOUT=86400 & use of timeout cmd; wait for low load
unset LANG
export LC_NUMERIC=C
p=/packages	# alias
delfree=65536 # MB = 64 GB - switch to --delete-before
minfree=1024 # MB
minfree=1 # MB
maxloadp=20.00
off=~/MIRROR_OFF
defoptions="-vaSHy --partial --delete --delete-after --delete-excluded --timeout=3600 --chmod=og-w"
gapfactor=1.5		# alarm if last successful mirroring more than $gapfactor * $gap ago
repeat=0		# try several times
repeatonerror=0		# try several times if there was an error
waitload=0		# wait until the load falls down
if [ -f ~/.mirror-functions ]; then . ~/.mirror-functions; fi

versioncur=`stat -c %y $0 | sed 's/\..*$//'`
versionfunc=`stat -c %y /local/scripts/mirror/mirror-functions | sed 's/\..*$//'`
echo "RUNNING $0 $*	 Versions: $versioncur (the current script), $versionfunc (mirror-functions)"

if [ -f $off ]; then echo "Mirroring blocked (`cat $off`). Remove $off to start again."; exit 1; fi

count=0; select=0; force=0; echo=""
while [[ "$1" =~ -[fFhn] ]]; do
	case "$1" in
		"-h") shift; cat <<EOT
$0 [-h] [-f [NUM]] [-n]
Script for mirroring rsync packages backed by mirror-functions library.
-h		this help
-f [NUM]	mirror even if the gap time set in the script have not passed.
                With NUM converns only package number NUM (conting from 1)
-F [NUM]	as -f and bypass the load limit by setting it to 1000
-n		dry run - do not execute rsync commands [unfinished]
EOT
		exit;;
		"-f"|"-F") force=1; if [ "$1" == "-F" ]; then maxloadp=1000.00; fi
			shift; if [[ "$1" =~ ^[0-9] ]]; then select=$1; shift; fi;;
		"-n")	shift; echo=echo; maxloadp=1000;;	# unfinished
	esac
done

maxload=`echo $maxloadp | awk '{ print int(1000*$1+0.5) }'`	# *1000 for integer comparisons

if [ "$options" == "" ]; then options="$defoptions"; fi

mirror() {
	local remote=$1
	local mirrordir
	local name
	name=`basename $0`
	if [ "$2" == "" ]; then
		mirrordir="$p/`basename $remote`"	# if the destination path not specified
	else
		mirrordir=$2				# otherwise use the specified one
	fi
	mirrordir=`echo "$mirrordir" | sed 's%//%/%g'`	# remove duplicate slashes
	local lock=~/`echo $mirrordir | tr / .`
	local filter="${lock}.filter"
	local stamp="${lock}.stamp"
	if [ "$email" == "" ]; then email="`whoami`"; fi
	dorepeat=0
	repeatnow=0
	repeatonerrornow=0
	if [ "$repeat" -ne 0 ]; then repeatnow="$repeat"; fi
	if [ "$repeatonerror" -ne 0 ]; then repeatonerrornow="$repeatonerror"; fi

	if [ ! -e "$stamp" ]; then touch -d "100000 hours ago" "$stamp"; fi	# just in case, to get a reasonable result from substraction
	if [ ! -e "$filter" ]; then touch -d "100000 hours ago" "$filter"; fi
	# to not to nag to often
	if [ "$gap" == "" ]; then gap=2; fi	# in h
	if [ "$rsync" == "" ]; then rsync="rsync"; fi	# default path
	load=`uptime | awk '{ gsub(",", "", $(NF-2)); print int($(NF-2)*1000+0.5) }'`; loadp=`echo $load | sed -e 's/\(^.*\)\(...\)/\1.\2/' -e 's/0$//'`	# presentation
	count=$(( $count + 1 ))				# submirrors counter

	printf '\nMIRRORING: %s -> %s   @%s	mirror %d	load: %s\n' $remote $mirrordir "`date`" "$count" "$loadp"
	if [ -f "$lock" ]; then if ! ps hp `cat $lock`; then echo Deleting a stale lock file.; rm -f $lock; fi; fi	# remove lock if no such process

	# time stamps of filter (recently touched) and stamp (backed off on unsuccessfull attempt)
	tcurr=`date "+%s"`; tstamp=`stat -c "%Y" "$stamp"`; tfilter=`stat -c "%Y" "$filter"`
	if [ -f "$lock" ]; then tstamp=`stat -c "%Y" "$stamp".old`; fi		# just running, use the previous stamp
	if [ "$tstamp" == "" ]; then tstamp=0; fi
	if [ "$tfilter" == "" ]; then tstamp=0; fi
	abovegap=`echo "$tcurr - $tstamp - $gap * 3600" | bc -lq | sed 's/\..*//'`
	min=`echo "($tcurr - $tstamp) / 60" | bc -lq`
	hours=`echo "($tcurr - $tstamp) / 3600" | bc -lq`
	minf=`echo "($tcurr - $tfilter) / 60" | bc -lq`
	hoursf=`echo "($tcurr - $tfilter) / 3600" | bc -lq`
	abovegapmax=`echo "$gapfactor * $gap * 3600 - $gap * 3600" | bc -lq | sed 's/\..*//'`
	gaptmess=""
	if [ "$abovegap" -gt "$abovegapmax" ]; then
			gaptimes=`echo "$abovegap / $gap / 3600 + 1" | bc -lq`
			gaptmess=`printf "The delay for $remote is %.1f times the gap (%.1f h)." $gaptimes $gap`
			gaptimes2=`printf "%.2f" $gaptimes`
			echo "$gaptmess" | mail -s "WARNING: $name $remote delay is $gaptimes2 times the gap ($gap h)" "$email"
	fi
	if [ $abovegap -le 0 -a $force == 0 ]; then
		printf 'Successfully mirrored only %.0f min (%.1f h) ago (last attempt %.1f h ago) which is less than %.1f h, skipping.\n' $min $hours $hoursf $gap
		if [ "$gaptmess" != "" ]; then echo "$gaptmess"; fi
		return
	fi

	if [ $force != 0 ]; then
		if [  $select == 0 -o $select == $count ]; then
			printf 'Forcing another mirroring of this package recently successfully mirrored %.0f min (%.1f h) ago (last attempt %.1f h ago), while gap is %.1f h, continuing…\n' $min $hours $hoursf $gap
			if [ "$gaptmess" != "" ]; then echo "$gaptmess"; fi
		else
			printf 'Successfully mirrored %.0f min (%.1f h) ago (last attempt %.1f h ago), which is more than %.1f h, continuing…' $min $hours $hoursf $gap
			printf '   Skipping.\n'
			return
		fi
	else
		printf 'Successfully mirrored %.0f min (%.1f h) ago (last attempt %.1f h ago), which is more than %.1f h, continuing…\n' $min $hours $hoursf $gap
		if [ "$gaptmess" != "" ]; then echo "$gaptmess"; fi
	fi

	if [ ! -d "$mirrordir"/ ]; then printf "NO DESTINATION DIRECTORY $mirrordir . EXITING…\n"; return; fi
	free=`df -PB1M $mirrordir/ | awk '$0 !~ /^Filesystem/ { print $(NF-2) + 0 } END { if ($NR==0) print 0 }'`
	if [ "$free" -le "$minfree" ]; then printf "THE DESTINATION FS FILLED, %d MB <= %d MB (minfree). EXITING…\n" "$free" "$minfree"; return; fi
	if [ "$free" -le "$delfree" ]; then
		printf "THE DESTINATION FS IS ALMOST FULL, %d MB <= %d MB (delfree). Using --delete-before now.\n" "$free" "$delfree"
		defoptions="-varSH --partial --delete --delete-before --delete-excluded --timeout=3600 --chmod=og-w"
	fi

	dorun=1; runs=0
	echo -n > "$filter"
	while [ "$3" != "" ]; do echo "$3" >> "$filter"; shift; done	# shift removes the arguments so it cannot be used in a loop
echo "filter: $filter"; cat $filter
	while [ "$dorun" != 0 ]; do
		dorun=0; runs=$(( $runs + 1 ))
		if [ "$runs" -gt 1 ]; then echo "Run # $runs"; fi
		if lockfile -3 -r 1 -'!' $lock; then
			echo Lock: skipping this section.
		else
			chmod u+w $lock; echo $$ > $lock; chmod u-w $lock
			insidel=0
			while [ "$load" -gt "$maxload" ]; do
				insidel=1
				echo -n "Load $loadp > $maxloadp, "
				if [ "$waitload" != 0 ]; then
					awk 'BEGIN { srand(); t=100*rand(); printf "waiting %d s…   ", t; system("sleep "t) }'	# up to 100 s delay
					load=`uptime | awk '{ gsub(",", "", $(NF-2)); print int($(NF-2)*1000+0.5) }'`; loadp=`echo $load | sed -e 's/\(^.*\)\(...\)/\1.\2/' -e 's/0$//'`
				else 
					echo "skipping mirroring."
					return
				fi
			done
			if [ "$insidel"==1 ]; then echo ''; fi

			mv "$stamp" "$stamp".old; touch "$stamp"
			$echo $rsync --filter ". $filter" $options $optionslocal $remote $mirrordir > ~/".$name.$$" 2>&1
			if [ $? != 0 ]; then			# on error e-mail and don't count this attempt
				errpatt='@ERROR|WARNING|failed|could not|THE DESTINATION FS IS ALMOST FULL|IO error|verification failed|rsync: connection|rsync error|lockfile:|Lock:|Deletions stopped|out of range'
				if [ `egrep "$errpatt" ~/".$name.$$" | wc -l` -gt 0 ]; then	# any errors fitting the pattern? we ignore vanishing files
					( echo "$remote -> $mirrordir"; echo ''; egrep "$errpatt" ~/".$name.$$") | mail -s "ERROR: $name $remote" "$email"
				fi
				if [ -f "$stamp".old ]; then mv "$stamp".old "$stamp"; fi
				if [ "$repeatonerrornow" -gt 0 ]; then repeatonerrornow=$(( $repeatonerrornow - 1 )); dorun=1; fi
			fi
			cat ~/".$name.$$"; rm ~/".$name.$$"
			rm -f $lock
			if [ "$repeatnow" -gt 0 ]; then repeatnow=$(( $repeatnow - 1 )); dorun=1; fi
		fi
		echo ''
		if [ "$dorun" != 0 ]; then
			awk 'BEGIN { srand(); t=100*rand(); print "Waiting", t, "s…"; system("sleep "t) }'	# up to 100 s delay
		fi
	done
}