#!/QOpenSys/pkgs/bin/bash
# zendopsctl -- starts, stops, restarts, and shows the services of the Zend
# OpenSearch application: OpenSearch, OpenSearch Dashboards, and the data
# importers, one by one or in groups.
#
# It works the same with and without zend-initd:
# - stop checks both ways a service can run, under zend-initd and from its
#   control script, and stops it the way it was started. It does not matter
#   how the service was started, or whether zend-initd was installed since;
# - start uses zend-initd when it is installed and its supervisor runs, and
#   the control scripts otherwise. --zendinit and --ctl choose explicitly.
# A restart of services started with their control scripts after zend-initd
# was installed therefore moves them to zend-initd.

PATH=/QOpenSys/pkgs/bin:/QOpenSys/usr/bin:/usr/bin:/bin:$PATH
export PATH

# The overrides are meant for testing outside of an installed system
install_root=${ZENDOPSCTL_ROOT:-/opt/zend}
zi_bin=${ZENDOPSCTL_ZENDINIT:-/QOpenSys/pkgs/bin/zendinit}
zi_confdir=${ZENDOPSCTL_ZENDINIT_CONFDIR:-/QOpenSys/etc/zend-initd/services.d}

app_home=$install_root/opensearch-app
opensearch_ready=$install_root/opensearch/bin/opensearch_ready.sh
# Seconds zendinit waits for a stop, and for a start including dependencies,
# the same as the initialisation script
zi_stop_timeout=240
zi_start_timeout=900
# Seconds to wait for OpenSearch to answer before starting what needs it
ready_wait=${ZENDOPSCTL_READY_WAIT:-600}
ready_poll=5

# Short names, zend-initd services, control scripts, and groups, in start
# order. Everything after OpenSearch needs it.
all_names=(
	opensearch
	dashboards
	logs-importer
	httpd-metrics
	php-metrics
	otel-receiver
)
all_services=(
	opensearch-svc
	opensearch-dashboards-svc
	opensearch-logs-importer-svc
	opensearch-httpd-metrics-svc
	opensearch-php-metrics-svc
	opensearch-otel-receiver-svc
)
all_ctls=(
	"$install_root/opensearch/bin/opensearch_ctl.sh"
	"$install_root/opensearch-dashboards/bin/opensearch-dashboards_ctl.sh"
	"$app_home/bin/logs_importer_ctl.sh"
	"$app_home/bin/httpd_metrics_importer_ctl.sh"
	"$app_home/bin/php_metrics_importer_ctl.sh"
	"$app_home/bin/otel_receiver_ctl.sh"
)
all_groups=(
	""
	""
	importers
	importers
	importers
	importers
)

requested_mode=auto
mode=
disable=no
selected=()
failures=()

show_usage() {
	cat <<EOF
Usage: ${0##*/} [--zendinit|--ctl] {start|restart} [target...]
       ${0##*/} [--disable] stop [target...]
       ${0##*/} {status|active|enabled} [target...]

Targets, default all:
  all            every installed service
  importers      logs-importer httpd-metrics php-metrics otel-receiver
  opensearch     OpenSearch                   (opensearch-svc)
  dashboards     OpenSearch Dashboards        (opensearch-dashboards-svc)
  logs-importer  logs importer                (opensearch-logs-importer-svc)
  httpd-metrics  HTTP server metrics importer (opensearch-httpd-metrics-svc)
  php-metrics    PHP metrics importer         (opensearch-php-metrics-svc)
  otel-receiver  OTLP receiver                (opensearch-otel-receiver-svc)
The zend-initd service names are accepted too. Services are started in the
order above, OpenSearch first, and stopped in the reverse order.

stop finds a service under zend-initd and as started with its control script,
and stops it either way. start uses zend-initd when it is installed and its
supervisor runs, the control scripts otherwise:
  --zendinit     start with zend-initd, start its supervisor when needed
  --ctl          start with the control scripts
Under zend-initd a service is enabled before it is started, with --ctl it is
disabled there. restart moves a service started with its control script to
zend-initd, and back with --ctl. stop does not disable a service unless
  --disable      stop also disables the services in zend-initd, so the
                 supervisor does not start them when it starts

status exits with 0 when every selected service runs, 3 otherwise.
active prints the services that run, or that zend-initd will start, and
enabled those enabled in zend-initd, one name per line. Both exit with 0
when they print a name, 1 otherwise.
EOF
}

fail() {
	failures+=("$1")
	echo "$1" >&2
}

zi_installed() {
	[ -x "$zi_bin" ]
}

# zi_known <service> -- 0 when zend-initd is installed and has the service
zi_known() {
	zi_installed && [ -f "$zi_confdir/$1.conf" ]
}

# zi_field <service> <field> -- one field of "zendinit status", first word
zi_field() {
	"$zi_bin" status "$1" 2>/dev/null |
		awk -v field="$2" '$1 == field { print $2; exit }'
}

# zi_active <service> -- 0 when zend-initd runs the service or will start it
zi_active() {
	zi_known "$1" || return 1
	[ "$(zi_field "$1" target)" = started ] && return 0
	case "$(zi_field "$1" state)" in
		running|starting) return 0 ;;
	esac
	return 1
}

ctl_active() {
	"$1" status >/dev/null 2>&1
}

supervisor_running() {
	"$zi_bin" system status 2>/dev/null | grep -q '^supervisor *running'
}

# wait_until <seconds> <command...>
wait_until() {
	limit=$1
	shift
	waited=0
	until "$@"; do
		if [ "$waited" -ge "$limit" ]; then
			return 1
		fi
		sleep "$ready_poll"
		waited=$((waited + ready_poll))
	done
}

opensearch_answers() {
	[ -x "$opensearch_ready" ] && "$opensearch_ready" >/dev/null 2>&1
}

# --- Targets ---

# select_target <target> -- marks the services of a target in selected[]
select_target() {
	found=no
	for i in "${!all_names[@]}"; do
		case "$1" in
			all) ;;
			"${all_names[$i]}"|"${all_services[$i]}") ;;
			"${all_groups[$i]}") [ -n "$1" ] || continue ;;
			*) continue ;;
		esac
		found=yes
		if [ -x "${all_ctls[$i]}" ]; then
			selected[i]=1
		elif [ "$1" = "${all_names[$i]}" ] || [ "$1" = "${all_services[$i]}" ]; then
			echo "${all_names[$i]} is not installed, ${all_ctls[$i]} was not found." >&2
		fi
	done
	if [ "$found" = no ]; then
		echo "Unknown target: $1" >&2
		show_usage >&2
		exit 2
	fi
}

any_selected() {
	[ "${#selected[@]}" -gt 0 ]
}

# --- How services are started ---

resolve_mode() {
	case "$requested_mode" in
		ctl)
			mode=ctl
			;;
		zendinit)
			if ! zi_installed; then
				echo "zend-initd is not installed, install it with: yum install zend-initd" >&2
				exit 1
			fi
			if ! supervisor_running; then
				echo "Starting the zend-initd supervisor."
				if ! "$zi_bin" system start || ! wait_until 60 supervisor_running; then
					echo "The zend-initd supervisor did not start." >&2
					exit 1
				fi
			fi
			mode=zendinit
			;;
		auto)
			mode=ctl
			if zi_installed; then
				if supervisor_running; then
					mode=zendinit
				else
					check_not_enabled
					echo "zend-initd is installed but its supervisor is not running, the"
					echo "control scripts are used. To use zend-initd, run 'zendinit system start'"
					echo "and restart the services, or use --zendinit."
				fi
			fi
			;;
	esac
}

# check_not_enabled -- exits when a selected service is enabled in zend-initd.
# The supervisor would start it when it starts, next to the instance of the
# control script. Checked before anything is stopped.
check_not_enabled() {
	enabled=()
	for i in "${!all_names[@]}"; do
		[ -n "${selected[$i]}" ] || continue
		if zi_known "${all_services[$i]}" &&
			[ "$(zi_field "${all_services[$i]}" enabled)" = yes ]; then
			enabled+=("${all_names[$i]}")
		fi
	done
	[ "${#enabled[@]}" -eq 0 ] && return 0
	echo "Enabled in zend-initd, whose supervisor is not running: ${enabled[*]}" >&2
	echo "Nothing is changed. Start the supervisor with 'zendinit system start' and" >&2
	echo "rerun, or choose with --zendinit, or with --ctl, which disables them there." >&2
	exit 1
}

# uses_zi <index> -- 0 when start runs the service under zend-initd
uses_zi() {
	[ "$mode" = zendinit ] && zi_known "${all_services[$1]}"
}

# --- Stop ---

stop_one() {
	name=${all_names[$1]}
	svc=${all_services[$1]}
	ctl=${all_ctls[$1]}
	stopped_any=no
	stop_rc=0
	# Disabled first, so the supervisor does not start it again
	if [ "$disable" = yes ] && zi_known "$svc" &&
		[ "$(zi_field "$svc" enabled)" != no ]; then
		echo "Disabling $name in zend-initd."
		if ! "$zi_bin" disable "$svc" >/dev/null; then
			fail "$name could not be disabled, disable it with: zendinit disable $svc"
			stop_rc=1
		fi
	fi
	# zendinit stop keeps the service stopped, a kill from outside would only
	# be undone by a restart. It goes first, a control script started service
	# may run next to it after a migration gone wrong.
	if zi_active "$svc"; then
		stopped_any=yes
		echo "Stopping $name with zend-initd."
		if ! "$zi_bin" --timeout "$zi_stop_timeout" stop "$svc"; then
			fail "$name did not stop, stop it with: zendinit --timeout $zi_stop_timeout stop $svc"
			return 1
		fi
	fi
	if ctl_active "$ctl"; then
		stopped_any=yes
		echo "Stopping $name with its control script."
		if ! "$ctl" stop; then
			fail "$name did not stop, stop it with: $ctl stop"
			return 1
		fi
	fi
	[ "$stopped_any" = yes ] || echo "$name is stopped."
	return "$stop_rc"
}

# stop_selected -- dependents first. Returns 1 when a service did not stop.
stop_selected() {
	rc=0
	for ((i = ${#all_names[@]} - 1; i >= 0; i--)); do
		[ -n "${selected[$i]}" ] || continue
		stop_one "$i" || rc=1
	done
	return "$rc"
}

# --- Start ---

start_one() {
	name=${all_names[$1]}
	svc=${all_services[$1]}
	ctl=${all_ctls[$1]}
	if ctl_active "$ctl"; then
		echo "$name is already running, started with its control script."
		if uses_zi "$1"; then
			echo "  'zendopsctl restart $name' moves it to zend-initd."
		fi
		return 0
	fi
	if uses_zi "$1"; then
		# zendinit start refuses a disabled service. Enabled, it also comes
		# back when the supervisor starts after an IPL.
		if [ "$(zi_field "$svc" enabled)" != yes ] &&
			! "$zi_bin" enable "$svc" >/dev/null; then
			fail "$name could not be enabled, enable it with: zendinit enable $svc"
			return 1
		fi
		echo "Starting $name with zend-initd."
		"$zi_bin" --timeout "$zi_start_timeout" start "$svc" && return 0
		fail "$name did not start, see: zendinit status $svc"
		return 1
	fi
	if zi_active "$svc"; then
		# A second instance would conflict with it
		echo "$name is run by zend-initd ($(zi_field "$svc" state)), it is not started with its control script."
		if [ "$requested_mode" = ctl ]; then
			echo "  'zendopsctl --ctl restart $name' moves it to its control script."
		fi
		return 0
	fi
	# An enabled service is started by the supervisor when it starts, next the one started here. Without --ctl, check_not_enabled has
	# already refused this.
	if zi_known "$svc" && [ "$(zi_field "$svc" enabled)" = yes ]; then
		echo "Disabling $svc in zend-initd, it runs with its control script now."
		if ! "$zi_bin" disable "$svc" >/dev/null; then
			fail "$name could not be disabled, disable it with: zendinit disable $svc"
			return 1
		fi
	fi
	echo "Starting $name with its control script."
	"$ctl" start && return 0
	fail "$name did not start."
	return 1
}

# start_selected -- OpenSearch first, the others once it answers
start_selected() {
	if [ -n "${selected[0]}" ]; then
		if ! zi_active "${all_services[0]}" && ! ctl_active "${all_ctls[0]}" &&
			opensearch_answers; then
			fail "Something else already answers on the OpenSearch port, opensearch is not started."
		elif ! start_one 0; then
			fail "The services that need OpenSearch are not started."
			return 1
		fi
	fi
	dependents=no
	for ((i = 1; i < ${#all_names[@]}; i++)); do
		[ -n "${selected[$i]}" ] && dependents=yes
	done
	[ "$dependents" = yes ] || return 0

	if ! opensearch_answers; then
		if [ -n "${selected[0]}" ]; then
			echo "Waiting up to $ready_wait seconds for OpenSearch to answer..."
			if ! wait_until "$ready_wait" opensearch_answers; then
				fail "OpenSearch did not answer within $ready_wait seconds, the services that need it are not started."
				return 1
			fi
			echo "OpenSearch answers."
		elif [ "$mode" = zendinit ]; then
			echo "OpenSearch does not answer, zend-initd starts the services once it does."
		else
			echo "OpenSearch does not answer, start it with: zendopsctl start opensearch" >&2
		fi
	fi
	for ((i = 1; i < ${#all_names[@]}; i++)); do
		[ -n "${selected[$i]}" ] || continue
		start_one "$i"
	done
}

# --- Status ---

show_status() {
	if ! zi_installed; then
		echo "zend-initd: not installed"
	elif supervisor_running; then
		echo "zend-initd: supervisor running"
	else
		echo "zend-initd: supervisor not running"
	fi
	rc=0
	printf '%-14s %-30s %-9s %s\n' NAME SERVICE STATE "STARTED BY"
	for i in "${!all_names[@]}"; do
		[ -n "${selected[$i]}" ] || continue
		svc=${all_services[$i]}
		state=stopped
		by=-
		if zi_known "$svc"; then
			zi_state=$(zi_field "$svc" state)
			zi_enabled=$(zi_field "$svc" enabled)
			zi_target=$(zi_field "$svc" target)
			case "$zi_state" in
				running|starting)
					state=$zi_state
					by=zend-initd
					;;
				*)
					# A crashed service zend-initd brings back
					if [ "$zi_target" = started ]; then
						state=${zi_state:-unknown}
						by="zend-initd, target started"
					fi
					;;
			esac
		fi
		if ctl_active "${all_ctls[$i]}"; then
			if [ "$by" = - ]; then
				state=running
				by="control script"
			else
				# Two instances, stop both and start one again
				by="$by and control script"
			fi
		fi
		if zi_known "$svc"; then
			by="$by (enabled ${zi_enabled:-?})"
		fi
		[ "$state" = running ] || rc=3
		printf '%-14s %-30s %-9s %s\n' "${all_names[$i]}" "$svc" "$state" "$by"
	done
	return "$rc"
}

# --- Queries for scripts ---

# list_active -- the services that run either way, or zend-initd will start
list_active() {
	listed=1
	for i in "${!all_names[@]}"; do
		[ -n "${selected[$i]}" ] || continue
		if zi_active "${all_services[$i]}" || ctl_active "${all_ctls[$i]}"; then
			echo "${all_names[$i]}"
			listed=0
		fi
	done
	return "$listed"
}

# list_enabled -- the services enabled in zend-initd
list_enabled() {
	listed=1
	for i in "${!all_names[@]}"; do
		[ -n "${selected[$i]}" ] || continue
		if zi_known "${all_services[$i]}" &&
			[ "$(zi_field "${all_services[$i]}" enabled)" = yes ]; then
			echo "${all_names[$i]}"
			listed=0
		fi
	done
	return "$listed"
}

# --- Main ---

command_name=
targets=()
while [ "$#" -gt 0 ]; do
	case "$1" in
		--zendinit) requested_mode=zendinit ;;
		--ctl) requested_mode=ctl ;;
		--disable) disable=yes ;;
		-h|--help|help)
			show_usage
			exit 0
			;;
		-*)
			echo "Unknown option: $1" >&2
			show_usage >&2
			exit 2
			;;
		*)
			if [ -z "$command_name" ]; then
				command_name=$1
			else
				targets+=("$1")
			fi
			;;
	esac
	shift
done

case "$command_name" in
	start|stop|restart|status|active|enabled) ;;
	"")
		show_usage >&2
		exit 2
		;;
	*)
		echo "Unknown command: $command_name" >&2
		show_usage >&2
		exit 2
		;;
esac
if [ "$disable" = yes ] && [ "$command_name" != stop ]; then
	echo "--disable goes with stop only." >&2
	exit 2
fi
if [ "$requested_mode" != auto ]; then
	case "$command_name" in
		start|restart) ;;
		*)
			echo "--zendinit and --ctl go with start and restart only." >&2
			exit 2
			;;
	esac
fi

[ "${#targets[@]}" -gt 0 ] || targets=(all)
for target in "${targets[@]}"; do
	select_target "$target"
done
if ! any_selected; then
	echo "None of the selected services is installed." >&2
	exit 1
fi

case "$command_name" in
	start)
		resolve_mode
		start_selected
		;;
	stop)
		stop_selected
		;;
	restart)
		# Decided before anything is stopped, it may refuse
		resolve_mode
		# Nothing is started while an old instance may still run
		if ! stop_selected; then
			echo "Stopping failed, nothing is started." >&2
			exit 1
		fi
		start_selected
		;;
	status)
		show_status
		exit $?
		;;
	active)
		list_active
		exit $?
		;;
	enabled)
		list_enabled
		exit $?
		;;
esac

if [ "${#failures[@]}" -gt 0 ]; then
	exit 1
fi
exit 0
