#!/QOpenSys/pkgs/bin/bash
# zendopsctl -- starts, stops, restarts, and shows the services of the Zend
# OpenSearch application: OpenSearch, OpenSearch Dashboards, and the data
# importers, one by one or in groups. The services run under zend-initd.

PATH=/QOpenSys/pkgs/bin:/QOpenSys/usr/bin:/usr/bin:/bin:$PATH
export PATH

# The overrides are meant for testing outside of an installed system
install_root=${ZENDOPSCTL_ROOT:-/opt/zend}
zi_bin=${ZENDOPSCTL_ZENDINIT:-/QOpenSys/pkgs/bin/zendinit}
zi_confdir=${ZENDOPSCTL_ZENDINIT_CONFDIR:-/QOpenSys/etc/zend-initd/services.d}

opensearch_ready=$install_root/opensearch/bin/opensearch_ready.sh
# Seconds zendinit waits for a stop, and for a start including dependencies,
# the same as the initialisation script
zi_stop_timeout=240
zi_start_timeout=900
# Seconds to wait for OpenSearch to answer before starting what needs it
ready_wait=${ZENDOPSCTL_READY_WAIT:-600}
ready_poll=5

# zend-initd services and their groups, in start order. Everything after
# OpenSearch needs it.
all_names=(
	opensearch
	dashboards
	logs-importer
	httpd-metrics
	php-metrics
	otel-receiver
)
all_groups=(
	""
	""
	importers
	importers
	importers
	importers
)

disable=no
selected=()
failures=()

show_usage() {
	cat <<EOF
Usage: ${0##*/} {start|restart} [target...]
       ${0##*/} [--disable] stop [target...]
       ${0##*/} {status|active|enabled} [target...]

Targets, default all:
  all            every installed service
  importers      logs-importer httpd-metrics php-metrics otel-receiver
  opensearch     OpenSearch
  dashboards     OpenSearch Dashboards
  logs-importer  logs importer
  httpd-metrics  HTTP server metrics importer
  php-metrics    PHP metrics importer
  otel-receiver  OTLP receiver
The targets are the zend-initd service names. Services are started in the
order above, OpenSearch first, and stopped in the reverse order.

The services run under zend-initd. start and restart start its supervisor
when it is not running, and enable a service before they start it, so the
supervisor also starts it when it starts, after an IPL for example. stop
does not disable a service unless
  --disable      stop also disables the services in zend-initd, so the
                 supervisor does not start them when it starts

status exits with 0 when every selected service runs, 3 otherwise.
active prints the services that run, or that zend-initd will start, and
enabled those enabled in zend-initd, one name per line. Both exit with 0
when they print a name, 1 otherwise.
EOF
}

fail() {
	failures+=("$1")
	echo "$1" >&2
}

# zi_known <service> -- 0 when zend-initd has the service
zi_known() {
	[ -f "$zi_confdir/$1.conf" ]
}

# zi_field <service> <field> -- one field of "zendinit status", first word
zi_field() {
	"$zi_bin" status "$1" 2>/dev/null |
		awk -v field="$2" '$1 == field { print $2; exit }'
}

# zi_active <service> -- 0 when zend-initd runs the service or will start it
zi_active() {
	zi_known "$1" || return 1
	[ "$(zi_field "$1" target)" = started ] && return 0
	case "$(zi_field "$1" state)" in
		running|starting) return 0 ;;
	esac
	return 1
}

supervisor_running() {
	"$zi_bin" system status 2>/dev/null | grep -q '^supervisor *running'
}

# wait_until <seconds> <command...>
wait_until() {
	limit=$1
	shift
	waited=0
	until "$@"; do
		if [ "$waited" -ge "$limit" ]; then
			return 1
		fi
		sleep "$ready_poll"
		waited=$((waited + ready_poll))
	done
}

opensearch_answers() {
	[ -x "$opensearch_ready" ] && "$opensearch_ready" >/dev/null 2>&1
}

# --- Targets ---

# select_target <target> -- marks the services of a target in selected[]
select_target() {
	found=no
	for i in "${!all_names[@]}"; do
		case "$1" in
			all) ;;
			"${all_names[$i]}") ;;
			"${all_groups[$i]}") [ -n "$1" ] || continue ;;
			*) continue ;;
		esac
		found=yes
		if zi_known "${all_names[$i]}"; then
			selected[i]=1
		elif [ "$1" = "${all_names[$i]}" ]; then
			echo "${all_names[$i]} is not installed, $zi_confdir/${all_names[$i]}.conf was not found." >&2
		fi
	done
	if [ "$found" = no ]; then
		echo "Unknown target: $1" >&2
		show_usage >&2
		exit 2
	fi
}

any_selected() {
	[ "${#selected[@]}" -gt 0 ]
}

# ensure_supervisor -- starts the zend-initd supervisor when it does not run
ensure_supervisor() {
	supervisor_running && return 0
	echo "Starting the zend-initd supervisor."
	if ! "$zi_bin" system start || ! wait_until 60 supervisor_running; then
		echo "The zend-initd supervisor did not start." >&2
		exit 1
	fi
}

# --- Stop ---

stop_one() {
	svc=${all_names[$1]}
	stop_rc=0
	# Disabled first, so the supervisor does not start it again
	if [ "$disable" = yes ] && [ "$(zi_field "$svc" enabled)" != no ]; then
		echo "Disabling $svc in zend-initd."
		if ! "$zi_bin" disable "$svc" >/dev/null; then
			fail "$svc could not be disabled, disable it with: zendinit disable $svc"
			stop_rc=1
		fi
	fi
	if ! zi_active "$svc"; then
		echo "$svc is stopped."
		return "$stop_rc"
	fi
	echo "Stopping $svc."
	if ! "$zi_bin" --timeout "$zi_stop_timeout" stop "$svc"; then
		fail "$svc did not stop, stop it with: zendinit --timeout $zi_stop_timeout stop $svc"
		return 1
	fi
	return "$stop_rc"
}

# stop_selected -- dependents first. Returns 1 when a service did not stop.
stop_selected() {
	rc=0
	for ((i = ${#all_names[@]} - 1; i >= 0; i--)); do
		[ -n "${selected[$i]}" ] || continue
		stop_one "$i" || rc=1
	done
	return "$rc"
}

# --- Start ---

start_one() {
	svc=${all_names[$1]}
	# zendinit start refuses a disabled service. Enabled, it also comes back
	# when the supervisor starts after an IPL.
	if [ "$(zi_field "$svc" enabled)" != yes ] &&
		! "$zi_bin" enable "$svc" >/dev/null; then
		fail "$svc could not be enabled, enable it with: zendinit enable $svc"
		return 1
	fi
	echo "Starting $svc."
	"$zi_bin" --timeout "$zi_start_timeout" start "$svc" && return 0
	fail "$svc did not start, see: zendinit status $svc"
	return 1
}

# start_selected -- OpenSearch first, the others once it answers
start_selected() {
	if [ -n "${selected[0]}" ]; then
		if ! zi_active "${all_names[0]}" && opensearch_answers; then
			fail "Something else already answers on the OpenSearch port, opensearch is not started."
		elif ! start_one 0; then
			fail "The services that need OpenSearch are not started."
			return 1
		fi
	fi
	dependents=no
	for ((i = 1; i < ${#all_names[@]}; i++)); do
		[ -n "${selected[$i]}" ] && dependents=yes
	done
	[ "$dependents" = yes ] || return 0

	if ! opensearch_answers; then
		if [ -n "${selected[0]}" ]; then
			echo "Waiting up to $ready_wait seconds for OpenSearch to answer..."
			if ! wait_until "$ready_wait" opensearch_answers; then
				fail "OpenSearch did not answer within $ready_wait seconds, the services that need it are not started."
				return 1
			fi
			echo "OpenSearch answers."
		else
			echo "OpenSearch does not answer, zend-initd starts the services once it does."
		fi
	fi
	for ((i = 1; i < ${#all_names[@]}; i++)); do
		[ -n "${selected[$i]}" ] || continue
		start_one "$i"
	done
}

# --- Status ---

show_status() {
	if supervisor_running; then
		echo "zend-initd: supervisor running"
	else
		echo "zend-initd: supervisor not running"
	fi
	rc=0
	printf '%-14s %-9s %s\n' SERVICE STATE ENABLED
	for i in "${!all_names[@]}"; do
		[ -n "${selected[$i]}" ] || continue
		svc=${all_names[$i]}
		state=$(zi_field "$svc" state)
		state=${state:-unknown}
		# A crashed service zend-initd brings back
		case "$state" in
			running|starting) ;;
			*)
				if [ "$(zi_field "$svc" target)" = started ]; then
					state="$state (target started)"
				fi
				;;
		esac
		enabled=$(zi_field "$svc" enabled)
		[ "$state" = running ] || rc=3
		printf '%-14s %-9s %s\n' "$svc" "$state" "${enabled:-?}"
	done
	return "$rc"
}

# --- Queries for scripts ---

# list_active -- the services that run, or zend-initd will start
list_active() {
	listed=1
	for i in "${!all_names[@]}"; do
		[ -n "${selected[$i]}" ] || continue
		if zi_active "${all_names[$i]}"; then
			echo "${all_names[$i]}"
			listed=0
		fi
	done
	return "$listed"
}

# list_enabled -- the services enabled in zend-initd
list_enabled() {
	listed=1
	for i in "${!all_names[@]}"; do
		[ -n "${selected[$i]}" ] || continue
		if [ "$(zi_field "${all_names[$i]}" enabled)" = yes ]; then
			echo "${all_names[$i]}"
			listed=0
		fi
	done
	return "$listed"
}

# --- Main ---

command_name=
targets=()
while [ "$#" -gt 0 ]; do
	case "$1" in
		--disable) disable=yes ;;
		-h|--help|help)
			show_usage
			exit 0
			;;
		-*)
			echo "Unknown option: $1" >&2
			show_usage >&2
			exit 2
			;;
		*)
			if [ -z "$command_name" ]; then
				command_name=$1
			else
				targets+=("$1")
			fi
			;;
	esac
	shift
done

case "$command_name" in
	start|stop|restart|status|active|enabled) ;;
	"")
		show_usage >&2
		exit 2
		;;
	*)
		echo "Unknown command: $command_name" >&2
		show_usage >&2
		exit 2
		;;
esac
if [ "$disable" = yes ] && [ "$command_name" != stop ]; then
	echo "--disable goes with stop only." >&2
	exit 2
fi
if [ ! -x "$zi_bin" ]; then
	echo "zend-initd is not installed, $zi_bin was not found." >&2
	echo "Install it with: yum install zend-initd" >&2
	exit 1
fi

[ "${#targets[@]}" -gt 0 ] || targets=(all)
for target in "${targets[@]}"; do
	select_target "$target"
done
if ! any_selected; then
	echo "None of the selected services is installed." >&2
	exit 1
fi

case "$command_name" in
	start)
		ensure_supervisor
		start_selected
		;;
	stop)
		stop_selected
		;;
	restart)
		ensure_supervisor
		# Nothing is started while an old instance may still run
		if ! stop_selected; then
			echo "Stopping failed, nothing is started." >&2
			exit 1
		fi
		start_selected
		;;
	status)
		show_status
		exit $?
		;;
	active)
		list_active
		exit $?
		;;
	enabled)
		list_enabled
		exit $?
		;;
esac

if [ "${#failures[@]}" -gt 0 ]; then
	exit 1
fi
exit 0
