#!/bin/bash
# vim: set noexpandtab tabstop=4 shiftwidth=4 softtabstop=4 :

basename="${0##*/}"

. /usr/lib/unbound/iptools.sh
. "${OMR_LIB_DIR:-/usr/share/omr/lib}/omr-state.sh"
. "${OMR_LIB_DIR:-/usr/share/omr/lib}/omr-vps-curl.sh"

export OMR_TRACKER_STATUS
export OMR_TRACKER_PREV_STATUS
export OMR_TRACKER_STATUS_MSG
export OMR_TRACKER_LATENCY
export OMR_TRACKER_LOSS
export OMR_TRACKER_SERVER_IP
export OMR_TRACKER_SERVER_NAME

_log() {
	logger -p daemon.info -t "${basename}" "$@"
}

_post_tracking() {
	[ ! -d /usr/share/omr/server-post-tracking.d/ ] && return
	for tracker_bin in /usr/share/omr/server-post-tracking.d/*; do
		[ -x "$tracker_bin" ] && (
			_log() {
				logger -t "post-tracking-${tracker_bin##*/}" "$*"
			}
			. "$tracker_bin" 2>&1
		)
	done
}

# Parse loss% and avg latency from ping -q output stored in $ret.
# Sets globals: loss (packet loss %), latency (avg ms, empty if no rtt line)
_parse_ping() {
	# the field ending with "%", as in omr-tracker: "+N errors" comes before
	# it when the host is unreachable, and $6 then read "+1", an answer
	loss=$(printf '%s\n' "$ret" | awk '/packet loss/ { for (i = 1; i <= NF; i++) if ($i ~ /%$/) { sub("%", "", $i); print int($i) } }')
	latency=$(printf '%s\n' "$ret" | awk -F/ '/rtt/{print int($5)}')
}

_ping_server() {
	local host=$1
	ret=$(ping -w "$timeout" -c 1 -q "${host}" 2>&1)
	_parse_ping
	if [ -n "$loss" ] && [ "$loss" != "100" ]; then
		[ -n "$latency" ] && OMR_TRACKER_LATENCY="$latency"
		OMR_TRACKER_LOSS="$loss"
		server_ping=true
	fi
}

_ping_server_intf() {
	local host=$1
	for intf in $mp_default_intfs; do
		local k=0
		while [ "$server_ping" = false ] && [ "$k" -le "$retry" ]; do
			ret=$(ping -w "$timeout" -c 1 -q -I "$intf" "${host}" 2>&1)
			_parse_ping
			if [ -n "$loss" ] && [ "$loss" != "100" ]; then
				[ -n "$latency" ] && OMR_TRACKER_LATENCY="$latency"
				OMR_TRACKER_LOSS="$loss"
				server_ping=true
				if [ "$OMR_TRACKER_CHECK_QUALITY" = "1" ]; then
					if [ "$OMR_TRACKER_PREV_STATUS" = "OK" ]; then
						if [ -n "$OMR_TRACKER_LOSS" ] && [ -n "$OMR_TRACKER_FAILURE_LOSS" ] && [ "$OMR_TRACKER_LOSS" -ge "$OMR_TRACKER_FAILURE_LOSS" ]; then
							OMR_TRACKER_STATUS="ERROR"
							OMR_TRACKER_STATUS_MSG="Packet loss is $OMR_TRACKER_LOSS this is more than failure limit defined at $OMR_TRACKER_FAILURE_LOSS"
							server_ping=false
						elif [ -n "$OMR_TRACKER_LATENCY" ] && [ -n "$OMR_TRACKER_FAILURE_LATENCY" ] && [ "$OMR_TRACKER_LATENCY" -ge "$OMR_TRACKER_FAILURE_LATENCY" ]; then
							OMR_TRACKER_STATUS="ERROR"
							OMR_TRACKER_STATUS_MSG="Latency is $OMR_TRACKER_LATENCY this is more than failure limit defined at $OMR_TRACKER_FAILURE_LATENCY"
							server_ping=false
						fi
					elif [ "$OMR_TRACKER_PREV_STATUS" = "ERROR" ]; then
						if [ -n "$OMR_TRACKER_LOSS" ] && [ -n "$OMR_TRACKER_RECOVERY_LOSS" ] && [ "$OMR_TRACKER_LOSS" -ge "$OMR_TRACKER_RECOVERY_LOSS" ]; then
							OMR_TRACKER_STATUS="ERROR"
							OMR_TRACKER_STATUS_MSG="Packet loss is $OMR_TRACKER_LOSS this is more than recovery limit defined at $OMR_TRACKER_RECOVERY_LOSS"
							server_ping=false
						elif [ -n "$OMR_TRACKER_LATENCY" ] && [ -n "$OMR_TRACKER_RECOVERY_LATENCY" ] && [ "$OMR_TRACKER_LATENCY" -ge "$OMR_TRACKER_RECOVERY_LATENCY" ]; then
							OMR_TRACKER_STATUS="ERROR"
							OMR_TRACKER_STATUS_MSG="Latency is $OMR_TRACKER_LATENCY this is more than recovery limit defined at $OMR_TRACKER_RECOVERY_LATENCY"
							server_ping=false
						fi
					fi
				fi
			fi
			k=$((k+1))
			[ "$server_ping" = false ] && sleep "${intervaltries}"
		done
	done
}

_check_server() {
	local host=$1 port=$2 k=0 url
	if [ "$(valid_subnet6 "$host")" = "ok" ]; then
		url="https://[${host}]:${port}/"
	else
		url="https://${host}:${port}/"
	fi
	while [ "$server_ping" = false ] && [ "$k" -le "$retry" ]; do
		ret=$(omr_vps_curl --max-time "$timeout" -s -k "$url")
		[ -n "$ret" ] && server_ping=true
		k=$((k+1))
		[ "$server_ping" = false ] && sleep "${intervaltries}"
	done
}

_check_server_intf() {
	local host=$1 port=$2 valid_ip6 intf k url
	valid_ip6=$(valid_subnet6 "$host")
	for intf in $mp_default_intfs; do
		if [ "$valid_ip6" = "ok" ]; then
			url="https://[${host}%${intf}]:${port}/"
		else
			url="https://${host}:${port}/"
		fi
		k=0
		while [ "$server_ping" = false ] && [ "$k" -le "$retry" ]; do
			if [ "$valid_ip6" = "ok" ]; then
				ret=$(omr_vps_curl --max-time "$timeout" -s -k "$url")
			else
				ret=$(omr_vps_curl --max-time "$timeout" -s -k --interface "$intf" "$url")
			fi
			[ -n "$ret" ] && server_ping=true
			k=$((k+1))
			[ "$server_ping" = false ] && sleep "${intervaltries}"
		done
	done
}

_get_server_name() {
	local serverid=$1
	local serverip=$2
	config_get serveripc "$serverid" server
	if [ "$serveripc" = "$serverip" ]; then
		servername=$serverid
	fi
}

# _disable_current <server> <server in use>: the one in use is skipped. It
# was cleared too, right after set_ip marked it, so the restarts of
# _apply_server_config found no current server and mptcpovervpn and
# openvpnbonding went back to the master that had just failed.
_disable_current() {
	local serv=$1
	[ "$serv" = "$2" ] && return
	omr_state_set openmptcprouter "${serv}" current 0
}

# Set a proxy/VPN option and flag it in $changes, only if the value differs:
# the commit and restart at the end of a check are for this tracker's edits
_set_option() {
	[ "$(uci -q get "$1")" = "$2" ] && return
	uci -q set "$1=$2" && changes="1"
}

_disable_redir() {
	local redir="$1"
	local serverdisable="$2"
	local shadowsocks="$3"
	config_get serverss "$redir" server
	if [ "$serverss" = "$serverdisable" ]; then
		_set_option "${shadowsocks}.${redir}.disabled" 1
	fi
}

_enable_redir() {
	local redir="$1"
	local shadowsocks="$2"
	config_get serverss "$redir" server
	if [ "$serverss" = "sss${count}" ]; then
		_set_option "${shadowsocks}.${redir}.disabled" 0
	fi
}

_send_mail_alert() {
	local type="$1" name="$2" ip="$3"
	local mail_alert mail_to OMR_SYSNAME mail_subject mail_message
	mail_alert="$(uci -q get omr-tracker.server.mail_alert)"
	[ "$mail_alert" != "1" ] && return
	mail_to="$(uci -q get mail.default.to | tr -d '\r\n')"
	case "$mail_to" in
		-*|*[[:space:]]*) mail_to="" ;;
	esac
	[ -z "$mail_to" ] && return
	OMR_SYSNAME="$(uci -q get system.@system[0].hostname)"
	mail_subject="$(uci -q get omr-tracker.defaults.mail_${type}_subject)"
	mail_message="$(uci -q get omr-tracker.defaults.mail_${type}_message)"
	if [ -n "$mail_subject" ] && [ -n "$mail_message" ]; then
		mail_subject=$(echo "$mail_subject" | sed -e "s/%SYSNAME%/$OMR_SYSNAME/g" -e "s/%INTERFACE%/Server ${name}/g" -e "s/%DEVICE%/${ip}/g" -e "s/%MESSAGE%/$OMR_TRACKER_STATUS_MSG/g")
		mail_subject="$(printf '%s' "$mail_subject" | tr -d '\r\n')"
		mail_message=$(echo "$mail_message" | sed -e "s/%SYSNAME%/$OMR_SYSNAME/g" -e "s/%INTERFACE%/Server ${name}/g" -e "s/%DEVICE%/${ip}/g" -e "s/%MESSAGE%/$OMR_TRACKER_STATUS_MSG/g")
		printf 'Subject: %s\n\n%s\n' "$mail_subject" "$mail_message" | sendmail "$mail_to"
	else
		local status_word
		[ "$type" = "up" ] && status_word="UP" || status_word="down"
		mail_subject="$(printf '%s' "$OMR_SYSNAME: Server ${name} (${ip}) is ${status_word}" | tr -d '\r\n')"
		printf 'Subject: %s\n\nThe reason is "%s".\n' "$mail_subject" "$OMR_TRACKER_STATUS_MSG" | sendmail "$mail_to"
	fi
}

_apply_server_config() {
	local ip="$1" name="$2"
	uci -q batch <<-EOF >/dev/null
		set xray.omrout.s_vmess_address="$ip"
		set xray.omrout.s_vless_address="$ip"
		set xray.omrout.s_vless_reality_address="$ip"
		set xray.omrout.s_trojan_address="$ip"
		set xray.omrout.s_socks_address="$ip"
		set xray.omrout.s_shadowsocks_address="$ip"
		set v2ray.omrout.s_vmess_address="$ip"
		set v2ray.omrout.s_vless_address="$ip"
		set v2ray.omrout.s_trojan_address="$ip"
		set v2ray.omrout.s_socks_address="$ip"
		commit v2ray
		commit xray
		set glorytun.vpn.host="$ip"
		commit glorytun
		set glorytun-udp.vpn.host="$ip"
		commit glorytun-udp
		set dsvpn.vpn.host="$ip"
		commit dsvpn
		set mlvpn.general.host="$ip"
		commit mlvpn
		set mqvpn.server.ip="$ip"
		commit mqvpn
		del openvpn.omr.remote
		add_list openvpn.omr.remote="$ip"
		commit openvpn
	EOF
	/etc/init.d/openmptcprouter-vps get_openvpn_key "$name" >/dev/null 2>&1
	/etc/init.d/v2ray restart >/dev/null 2>&1
	/etc/init.d/xray restart >/dev/null 2>&1
	/etc/init.d/glorytun restart >/dev/null 2>&1
	/etc/init.d/glorytun-udp restart >/dev/null 2>&1
	/etc/init.d/mlvpn restart >/dev/null 2>&1
	/etc/init.d/mqvpn restart >/dev/null 2>&1
	/etc/init.d/dsvpn restart >/dev/null 2>&1
	/etc/init.d/mptcpovervpn restart >/dev/null 2>&1
	# openvpnbonding points omr_bonding_* at the server in use, and its
	# "openvpn start <instance>" restarts a client whose config changed. Not
	# "openvpn restart" after it: that recreated the taps it had just put in
	# bonding-omrvpn, out of it.
	if [ "$(uci -q get openmptcprouter.settings.vpn)" = "openvpn_bonding" ]; then
		/etc/init.d/openvpnbonding restart >/dev/null 2>&1
	fi
}

# Enable SS redirs for both libev and rust (shared by master and backup set_ip)
_enable_ss_redirs() {
	config_load shadowsocks-libev
	config_foreach _enable_redir ss_redir "shadowsocks-libev"
	config_load shadowsocks-rust
	config_foreach _enable_redir ss_redir "shadowsocks-rust"
}

# Disable SS redirs for the server that owns $1 (IP address)
_disable_ss_redirs_by_ip() {
	local ip=$1
	servername=""
	config_load shadowsocks-libev
	config_foreach _get_server_name server "$ip"
	[ -n "$servername" ] && config_foreach _disable_redir ss_redir "$servername" "shadowsocks-libev"
	servername=""
	config_load shadowsocks-rust
	config_foreach _get_server_name server "$ip"
	[ -n "$servername" ] && config_foreach _disable_redir ss_redir "$servername" "shadowsocks-rust"
}

# Resolve hostname/IP $1 to a numeric address; sets global $serverip.
# This runs on every check cycle, so a literal IP short-circuits without the
# resolveip fork, and hostname lookups are cached for 300s (a failed lookup
# is not cached, so it retries next cycle).
declare -A _resolve_cache _resolve_cache_ts
_resolve_ip() {
	local ipd=$1 r now
	case "$ipd" in
		*:*) serverip="$ipd"; return ;;      # literal IPv6 (hostnames have no colon)
		*[!0-9.]*) ;;                         # letters present: hostname, resolve below
		*[0-9]*) serverip="$ipd"; return ;;  # digits and dots only: literal IPv4
	esac
	printf -v now '%(%s)T' -1
	if [ -n "${_resolve_cache[$ipd]}" ] && [ $((now - _resolve_cache_ts[$ipd])) -lt 300 ]; then
		serverip="${_resolve_cache[$ipd]}"
		return
	fi
	r=$(resolveip -t 5 -4 "$ipd" | head -n 1)
	if [ -n "$r" ]; then
		serverip="$r"
	else
		r=$(resolveip -t 5 -6 "$ipd" | head -n 1)
		[ -n "$r" ] && serverip="$r"
	fi
	if [ -n "$serverip" ]; then
		_resolve_cache[$ipd]="$serverip"
		_resolve_cache_ts[$ipd]=$now
	fi
}

# Restart proxy/DNS services after a config change
_restart_proxy_services() {
	/etc/init.d/shadowsocks-libev restart >/dev/null 2>&1
	/etc/init.d/shadowsocks-rust restart >/dev/null 2>&1
	/etc/init.d/openvpn restart omr >/dev/null 2>&1
	/etc/init.d/unbound restart >/dev/null 2>&1
	/etc/init.d/dnsmasq restart >/dev/null 2>&1
}

# Common check_ip body used by both _check_master and _check_backup.
# Resolves ipd, runs the configured check type, then calls set_ip (defined
# by the caller before config_list_foreach).
# Checks address $1 of a server (API port in $port) the configured way:
# sets serverip to the resolved address and server_ping
_probe_ip() {
	server_ping=false
	serverip=""
	_resolve_ip "$1"
	[ -z "$serverip" ] && return
	if [ "$OMR_TRACKER_TYPE" = "api" ]; then
		_check_server_intf "$serverip" "$port"
	elif [ "$OMR_TRACKER_TYPE" = "apiping" ]; then
		_check_server_intf "$serverip" "$port"
		[ "$server_ping" = false ] && _ping_server_intf "$serverip"
	elif [ "$OMR_TRACKER_TYPE" = "ping" ]; then
		_ping_server_intf "$serverip"
	elif [ "$OMR_TRACKER_TYPE" = "none" ]; then
		server_ping=true
	fi
}

_do_check_ip() {
	_probe_ip "$1"
	[ -z "$serverip" ] && return
	[ "$server_ping" = true ] && server_answered="1"
	countips=$((countips+1))
	server_held=false
	set_ip
	# A master kept waiting by the failback setting answers, but is not
	# the server in use: the backups are still checked
	[ "$server_ping" = true ] && [ "$server_held" = false ] && oneserverup="1"
}

# Whether server $1 answers ($2: 1 or 0), for the status page: in the
# state, written on a change only
_set_reachable() {
	[ "${_srv_reachable[$1]}" = "$2" ] && return
	omr_state_set openmptcprouter "$1" reachable "$2"
	_srv_reachable[$1]="$2"
}

# The rounds check the backups only while no master answers. With
# omr-tracker.server.backup_check_interval, the backups that a round did
# not check are checked in the background, once per interface, to show on
# the status page whether they would answer. Nothing is switched from here.
_probe_backup() {
	local name=$1 backup disabled ip port ipd answered=0
	local OMR_VPS_SERVER="$name"
	config_get backup "$1" backup
	config_get disabled "$1" disabled
	config_get ip "$1" ip
	config_get port "$1" port "65500"
	[ "$backup" = "1" ] && [ -n "$ip" ] && [ "$disabled" != "1" ] || return
	case " $checked_servers " in
		*" $name "*) return ;;
	esac
	for ipd in $ip; do
		_probe_ip "$ipd"
		[ "$server_ping" = true ] && { answered=1; break; }
	done
	[ "$(omr_state_get openmptcprouter."$name".reachable)" = "$answered" ] || \
		omr_state_set openmptcprouter "$name" reachable "$answered"
}

_get_backup_in_use() {
	local backup
	config_get backup "$1" backup
	[ "$backup" = "1" ] && [ "$(omr_state_get openmptcprouter."$1".current)" = "1" ] && backup_in_use="$1"
}

# omr-tracker.server.failback: whether to stay on the backup server in use
# although the master answers again (state key $1). Moving back restarts
# the proxy and the VPNs, which drops every open connection: "delayed"
# waits until the master has answered for failback_delay seconds, "manual"
# until "/etc/init.d/omr-tracker failback". Returns 0 to stay on the backup.
_failback_hold() {
	local key="$1" now
	[ "$failback_force" = "1" ] && return 1
	case "$failback" in
		delayed|manual) ;;
		*) return 1 ;;
	esac
	# Without a backup in use this is no failback, and the master is set at
	# once: after a reboot (the state is gone) or a change of the config
	backup_in_use=""
	config_load openmptcprouter
	config_foreach _get_backup_in_use server
	[ -z "$backup_in_use" ] && return 1
	if [ "$(omr_state_get openmptcprouter.omr.failback_now)" = "1" ]; then
		omr_state_del openmptcprouter omr failback_now
		logger -t "OMR-Tracker-Server" "Failback to the master server requested"
		failback_force="1"
		return 1
	fi
	if [ "$failback" = "delayed" ]; then
		printf -v now '%(%s)T' -1
		[ -z "${_failback_since[$key]}" ] && _failback_since[$key]="$now"
		[ $((now - _failback_since[$key])) -ge "$failback_delay" ] && return 1
	fi
	return 0
}

_check_master() {
	local name=$1
	local count=0
	local countips=0
	local changes="0"
	local cfg_changes="0"
	# The API check connects to the resolved address: name the server
	# section for omr_vps_curl, which otherwise only finds it (and its
	# api_pin) when the address is spelled the same in the ip list, i.e.
	# never for a VPS configured by hostname
	local OMR_VPS_SERVER="$name"
	config_get master "$1" master
	config_get ip "$1" ip
	config_get port "$1" port "65500"
	config_get disabled "$1" disabled
	serverip=""
	[ "$master" = "1" ] && [ -n "$ip" ] && [ "$disabled" != "1" ] && {
		set_ip() {
			ip="$serverip"
			local ss_libev_server ss_rust_server
			local state_key="m:${name}:${countips}"
			local alert_up="" alert_down=""
			ss_libev_server=$(uci -q get shadowsocks-libev.sss${count}.server)
			ss_rust_server=$(uci -q get shadowsocks-rust.sss${count}.server)
			if [ "$server_ping" = true ]; then
				if { [ -n "$(uci -q get shadowsocks-libev.sss${count})" ] && [ "$ss_libev_server" != "$ip" ]; } || \
				   { [ -n "$(uci -q get shadowsocks-rust.sss${count})" ] && [ "$ss_rust_server" != "$ip" ]; } || \
				   { [ -n "$(uci -q get openvpn.omr)" ] && [ -z "$(uci -q get openvpn.omr.remote | grep "$ip")" ]; }; then
					if _failback_hold "$state_key"; then
						server_held=true
						failback_held="1"
						if [ "${_failback_logged[$state_key]}" != "1" ]; then
							logger -t "OMR-Tracker-Server" "Master server ${name} up ($ip), stay on backup server ${backup_in_use} (failback ${failback})"
							_failback_logged[$state_key]="1"
						fi
						count=$((count+1))
						return
					fi
					logger -t "OMR-Tracker-Server" "Master server ${name} up ($ip), set it back"
					changes="1"
					uci -q batch <<-EOF >/dev/null
						set shadowsocks-libev.sss${count}.server="$ip"
						set shadowsocks-rust.sss${count}.server="$ip"
					EOF
					# runtime, in /var/state (see omr-state.sh)
					omr_state_set openmptcprouter "${name}" current '1'
					omr_state_del openmptcprouter omr detected_ss_ipv4
					omr_state_del openmptcprouter omr detected_public_ipv4
					omr_state_del openmptcprouter omr detected_ss_ipv6
					omr_state_del openmptcprouter omr detected_public_ipv6
					[ -z "$(uci -q get openvpn.omr.remote | grep "$ip")" ] && \
						uci -q add_list openvpn.omr.remote="$ip"
					if [ "$count" -eq "0" ]; then
						config_load openmptcprouter
						config_foreach _disable_current server "$name"
						_apply_server_config "$ip" "$name"
					fi
				fi
				_failback_since[$state_key]=""
				_failback_logged[$state_key]=""
				OMR_TRACKER_STATUS_MSG="Answer to ping and to API check"
				# Steady state (already up last cycle, config untouched): the
				# redir re-enable walk, ip_down bookkeeping and up-alert only
				# matter on a down->up transition or after a config fix, and
				# their config_load/uci churn was most of this daemon's
				# idle-cycle CPU cost. The mail/script alerts also belong on
				# the transition only: one alert per event, not one per cycle.
				if [ "${_srv_prev_up[$state_key]}" != "1" ] || [ "$changes" = "1" ]; then
					_proxy=$(uci -q get openmptcprouter.settings.proxy)
					_vpn=$(uci -q get openmptcprouter.settings.vpn)
					[ "$_proxy" = "shadowsocks-rust" ] && _set_option shadowsocks-rust.sss${count}.disabled 0
					[ "$_proxy" = "shadowsocks" ]      && _set_option shadowsocks-libev.sss${count}.disabled 0
					[ "$_vpn"   = "openvpn" ]          && _set_option openvpn.omr.enabled 1
					_enable_ss_redirs
					[ -n "$ip" ] && [ -n "$(uci -q get openmptcprouter.${name}.ip_down | grep -w "$ip")" ] && \
						uci -q del_list openmptcprouter.${name}.ip_down="$ip" && cfg_changes="1"
					omr_state_set openmptcprouter ${name} current '1'
					_send_mail_alert "up" "$name" "$ip"
					alert_up="1"
				fi
				_srv_prev_up[$state_key]="1"
				OMR_TRACKER_SERVER_IP="$ip"
				OMR_TRACKER_SERVER_NAME="$name"
				OMR_TRACKER_STATUS="OK"
				# Like the mail, once per transition, after the variables
				# the script may read are set
				[ "$alert_up" = "1" ] && {
					script_alert_up=$(uci -q get omr-tracker.proxy.script_alert_up)
					[ -n "$script_alert_up" ] && eval "$script_alert_up"
				}
				count=$((count+1))
			else
				OMR_TRACKER_STATUS_MSG="No answer to ping and to API check"
				if [ "${_srv_prev_up[$state_key]}" != "0" ]; then
					[ -n "$ip" ] && \
						[ -z "$(uci -q get openmptcprouter.${name}.ip_down | grep -w "$ip")" ] && \
						uci -q add_list openmptcprouter.${name}.ip_down="$ip" && \
						cfg_changes="1" && \
						logger -t "OMR-Tracker-Server" "Master server ${name} down ($ip)"
					_disable_ss_redirs_by_ip "$ip"
					_send_mail_alert "down" "$name" "$ip"
					alert_down="1"
				fi
				_srv_prev_up[$state_key]="0"
				_failback_since[$state_key]=""
				_failback_logged[$state_key]=""
				OMR_TRACKER_STATUS="ERROR"
				[ "$alert_down" = "1" ] && {
					script_alert_down=$(uci -q get omr-tracker.proxy.script_alert_down)
					[ -n "$script_alert_down" ] && eval "$script_alert_down"
				}
			fi
		}
		server_answered="0"
		config_load openmptcprouter
		config_list_foreach "$1" ip _do_check_ip
		_set_reachable "$name" "$server_answered"
		checked_servers="$checked_servers $name"
		[ "$oneserverup" = "0" ] && omr_state_set openmptcprouter ${name} current '0'
		# ip_down is the only persistent openmptcprouter option written here
		# (current and the global state are in /var/state): commit it, not
		# whatever other processes have pending in the shared /tmp/.uci
		[ "$cfg_changes" = "1" ] && uci -q commit openmptcprouter
		# Commit and restart only after an edit of this tracker: other scripts
		# can have edits pending in the shared /tmp/.uci (omr-schedule's
		# interface lists, for one), and those are no server change
		[ "$changes" = "1" ] && {
			uci -q commit shadowsocks-libev
			uci -q commit shadowsocks-rust
			uci -q commit openvpn
			_restart_proxy_services
		}
		return
	}
}

_check_backup() {
	local name=$1
	local count=0
	local countips=0
	local changes="0"
	local cfg_changes="0"
	# A backup server that answered earlier in this round is the one in use:
	# checking the next ones would make each answering backup take the VPN
	# and the proxy over in turn, every round
	[ "$oneserverup" = "1" ] && return
	# See _check_master
	local OMR_VPS_SERVER="$name"
	config_get backup "$1" backup
	config_get ip "$1" ip
	config_get port "$1" port "65500"
	config_get disabled "$1" disabled
	serverip=""
	[ "$backup" = "1" ] && [ -n "$ip" ] && [ "$disabled" != "1" ] && {
		set_ip() {
			ip="$serverip"
			local ss_libev_server ss_rust_server
			local state_key="b:${name}:${countips}"
			local alert_up="" alert_down=""
			ss_libev_server=$(uci -q get shadowsocks-libev.sss${count}.server)
			ss_rust_server=$(uci -q get shadowsocks-rust.sss${count}.server)
			if [ "$server_ping" = true ]; then
				if { [ -n "$(uci -q get shadowsocks-libev.sss${count})" ] && [ "$ss_libev_server" != "$ip" ]; } || \
				   { [ -n "$(uci -q get shadowsocks-rust.sss${count})" ] && [ "$ss_rust_server" != "$ip" ]; } || \
				   { [ -n "$(uci -q get openvpn.omr)" ] && [ -z "$(uci -q get openvpn.omr.remote | grep "$ip")" ]; }; then
					logger -t "OMR-Tracker-Server" "Use backup server $name ($ip)"
					changes="1"
					uci -q batch <<-EOF >/dev/null
						set shadowsocks-libev.sss${count}.server="$ip"
						set shadowsocks-rust.sss${count}.server="$ip"
					EOF
					# runtime, in /var/state (see omr-state.sh)
					omr_state_set openmptcprouter "${name}" current '1'
					omr_state_del openmptcprouter omr detected_ss_ipv4
					omr_state_del openmptcprouter omr detected_public_ipv4
					omr_state_del openmptcprouter omr detected_ss_ipv6
					omr_state_del openmptcprouter omr detected_public_ipv6
					if [ "$count" -eq "0" ]; then
						config_load openmptcprouter
						config_foreach _disable_current server "$name"
						_apply_server_config "$ip" "$name"
					fi
					sleep "$waittest"
				fi
				OMR_TRACKER_STATUS_MSG="Answer to ping and to API check"
				# Same steady-state gate as _check_master: only a down->up
				# transition or a config fix needs the redir/ip_down/alert work
				if [ "${_srv_prev_up[$state_key]}" != "1" ] || [ "$changes" = "1" ]; then
					_proxy=$(uci -q get openmptcprouter.settings.proxy)
					[ "$_proxy" = "shadowsocks-rust" ] && _set_option shadowsocks-rust.sss${count}.disabled 0
					[ "$_proxy" = "shadowsocks" ]      && _set_option shadowsocks-libev.sss${count}.disabled 0
					_enable_ss_redirs
					[ -n "$ip" ] && [ -n "$(uci -q get openmptcprouter.${name}.ip_down | grep -w "$ip")" ] && \
						uci -q del_list openmptcprouter.${name}.ip_down="$ip" && cfg_changes="1"
					omr_state_set openmptcprouter ${name} current '1'
					_send_mail_alert "up" "$name" "$ip"
					alert_up="1"
				fi
				_srv_prev_up[$state_key]="1"
				OMR_TRACKER_SERVER_IP="$ip"
				OMR_TRACKER_SERVER_NAME="$name"
				OMR_TRACKER_STATUS="OK"
				# Like the mail, once per transition, after the variables
				# the script may read are set
				[ "$alert_up" = "1" ] && {
					script_alert_up=$(uci -q get omr-tracker.proxy.script_alert_up)
					[ -n "$script_alert_up" ] && eval "$script_alert_up"
				}
				count=$((count+1))
			else
				[ -z "$OMR_TRACKER_STATUS_MSG" ] && OMR_TRACKER_STATUS_MSG="No answer to ping and to API check"
				if [ "${_srv_prev_up[$state_key]}" != "0" ]; then
					[ -n "$ip" ] && \
						[ -z "$(uci -q get openmptcprouter.${name}.ip_down | grep -w "$ip")" ] && \
						uci -q add_list openmptcprouter.${name}.ip_down="$ip" && \
						cfg_changes="1" && \
						logger -t "OMR-Tracker-Server" "Backup server ${name} down ($ip)"
					omr_state_set openmptcprouter ${name} current '0'
					_disable_ss_redirs_by_ip "$ip"
					_send_mail_alert "down" "$name" "$ip"
					alert_down="1"
				fi
				_srv_prev_up[$state_key]="0"
				OMR_TRACKER_STATUS="ERROR"
				[ "$alert_down" = "1" ] && {
					script_alert_down=$(uci -q get omr-tracker.proxy.script_alert_down)
					[ -n "$script_alert_down" ] && eval "$script_alert_down"
				}
			fi
			countips=$((countips+1))
		}
		server_answered="0"
		config_load openmptcprouter
		config_list_foreach "$1" ip _do_check_ip
		_set_reachable "$name" "$server_answered"
		checked_servers="$checked_servers $name"
		[ "$oneserverup" = "0" ] && omr_state_set openmptcprouter ${name} current '0'
		# ip_down is the only persistent openmptcprouter option written here
		# (current and the global state are in /var/state): commit it, not
		# whatever other processes have pending in the shared /tmp/.uci
		[ "$cfg_changes" = "1" ] && uci -q commit openmptcprouter
		# Only after an edit of this tracker (see _check_master)
		[ "$changes" = "1" ] && {
			uci -q commit shadowsocks-libev
			uci -q commit shadowsocks-rust
			uci -q commit openvpn
			_restart_proxy_services
		}
		[ "$server_ping" = true ] && return
	}
}

# One round: the masters, then the backups while no master is in use
_check_servers() {
	local name
	failback_held="0"
	failback_force="0"
	checked_servers=""
	config_load openmptcprouter
	config_foreach _check_master server
	[ "$oneserverup" = "0" ] && {
		config_load openmptcprouter
		config_foreach _check_backup server
	}
	# A master kept waiting by the failback setting is taken back as soon
	# as no backup answers anymore
	[ "$oneserverup" = "0" ] && [ "$failback_held" = "1" ] && {
		failback_held="0"
		failback_force="1"
		config_load openmptcprouter
		config_foreach _check_master server
	}
	# openmptcprouter-vps leaves a master kept waiting alone: its config
	# would move the VPNs back to it
	if [ "$failback_held" != "$_failback_state" ]; then
		if [ "$failback_held" = "1" ]; then
			omr_state_set openmptcprouter omr failback_hold 1
		else
			omr_state_del openmptcprouter omr failback_hold
			omr_state_del openmptcprouter omr failback_now
		fi
		_failback_state="$failback_held"
	fi
	# Without the background check, a backup that the round did not check
	# has no known state anymore: drop the one of its last check
	if [ "$backup_check_interval" = "0" ]; then
		for name in "${!_srv_reachable[@]}"; do
			case " $checked_servers " in
				*" $name "*) ;;
				*)
					omr_state_del openmptcprouter "$name" reachable
					unset "_srv_reachable[$name]"
					;;
			esac
		done
	fi
}

# Starts _probe_backup in the background every backup_check_interval
# seconds while a master is in use, one run at a time: a backup that does
# not answer takes up to a timeout per interface, which the rounds, and so
# the failover, must not wait for
_check_backups_background() {
	local now
	[ "$backup_check_interval" -gt 0 ] && [ "$oneserverup" = "1" ] || return
	printf -v now '%(%s)T' -1
	[ $((now - _bc_last)) -ge "$backup_check_interval" ] || return
	[ -n "$_bc_pid" ] && kill -0 "$_bc_pid" 2>/dev/null && return
	_bc_last="$now"
	# the state it writes is not in this shell's cache
	_srv_reachable=()
	(
		retry=0
		config_load openmptcprouter
		config_foreach _probe_backup server
	) &
	_bc_pid=$!
}

. /lib/functions.sh

timeout=${OMR_TRACKER_TIMEOUT:-5}
interval=${OMR_TRACKER_INTERVAL:-10}
intervaltries=${OMR_TRACKER_INTERVAL_TRIES:-2}
retry=${OMR_TRACKER_TRIES:-4}
waittest=${OMR_TRACKER_WAIT_TEST:-0}
# Per-"server:ip" up/down state from the previous cycle, so the check
# functions can tell a transition apart from the steady state
declare -A _srv_prev_up
# omr-tracker.server.failback: immediate (default), delayed or manual
failback=${OMR_TRACKER_FAILBACK:-immediate}
failback_delay=${OMR_TRACKER_FAILBACK_DELAY:-300}
case "$failback_delay" in
	''|*[!0-9]*) failback_delay=300 ;;
esac
# Per master "server:ip": when it started to answer while a backup is in
# use, and whether that was logged
declare -A _failback_since _failback_logged
# Last value written to openmptcprouter.omr.failback_hold
_failback_state=""
# omr-tracker.server.backup_check_interval, in seconds, 0 to disable
backup_check_interval=${OMR_TRACKER_BACKUP_CHECK_INTERVAL:-60}
case "$backup_check_interval" in
	''|*[!0-9]*) backup_check_interval=60 ;;
esac
# Per server: last "reachable" written to the state
declare -A _srv_reachable
_bc_last=0
_bc_pid=""
# `multipath` is itself a shell script whose no-arg listing forks per network
# device, so refresh the default-route interface list at most every 30s
# instead of on every check. Worst case a hotplugged/removed WAN is seen up
# to 30s late here; the per-interface checks tolerate a stale entry (a dead
# interface just fails its probe and the loop moves on).
# The interfaces in backup mode are listed too, after the others: with only
# the default ones, every server was declared down (proxies, VPN and DNS
# restarted) as soon as a backup WAN was the only one left.
mp_default_intfs=""
_mp_intfs_ts=0
# Last value written to openmptcprouter.omr.server, so the global up/down
# state is committed only on a transition, not every cycle
_omr_server_state=""
while true; do
	server_ping=false
	oneserverup="0"
	printf -v _mp_now '%(%s)T' -1
	if [ $((_mp_now - _mp_intfs_ts)) -ge 30 ]; then
		mp_default_intfs="$(multipath 2>/dev/null | awk '/ is in default mode/ { d = d " " $1 } / is in backup mode/ { b = b " " $1 } END { print d b }')"
		_mp_intfs_ts=$_mp_now
	fi
	_check_servers
	_check_backups_background
	# Global server state for consumers (LuCI, scripts): "up" while at least
	# one master or backup server answers, "down" otherwise. The
	# server-post-tracking.d hooks get the same, as their status.
	if [ "$oneserverup" = "1" ]; then
		srv_state="up"
		OMR_TRACKER_STATUS="OK"
	else
		srv_state="down"
		OMR_TRACKER_STATUS="ERROR"
	fi
	if [ "$srv_state" != "$_omr_server_state" ]; then
		[ "$(uci -q get openmptcprouter.omr)" = "router" ] || { uci -q set openmptcprouter.omr=router; uci -q commit openmptcprouter; }
		omr_state_set openmptcprouter omr server "$srv_state"
		_omr_server_state="$srv_state"
	fi
	_post_tracking
	OMR_TRACKER_PREV_STATUS="$OMR_TRACKER_STATUS"
	sleep "${interval}"
done
