#!/QOpenSys/usr/bin/sh
# zendinitd -- the zend-initd supervisor.
#
# A reconcile loop, not an event system.  Each pass establishes what should be
# running (config plus admin intent), establishes what is actually running
# (the platform adapter), and moves one towards the other.  That shape is what
# lets the supervisor be restarted, upgraded or killed outright without
# disturbing the services it manages: on the way back up it reads the job ids
# it recorded and adopts what is still there.
#
# POSIX sh only; see lib/common.sh for the house rules.  The shebang and the
# library path are rewritten by `make install`.
set -u

ZI_VERSION='0.1.2'
ZI_LIBDIR_INSTALLED='/QOpenSys/pkgs/libexec/zend-initd/lib'

# ${1+"$@"} rather than "$@", throughout.  With set -u and no positional
# parameters, PASE ksh treats "$@" as an unset parameter and dies with
# "@: parameter not set" -- and the supervisor is invoked with no arguments at
# all by STRZINITD, so this would have made IPL autostart fail silently while
# every manual test passed.  bash, dash and ksh93 do not do this, so nothing
# off-platform reproduces it.
#
# On IBM i, /bin/sh is the QSH shell (/QSYS.LIB/QSHELL.LIB/QZSHSH.PGM), not
# PASE.  A source-tree copy still carrying a #!/bin/sh shebang therefore runs
# under an interpreter that does not handle PASE signals: the supervisor was
# seen to die on SIGTERM with "PASE for i ended for signal 15" instead of
# running its stop handler, leaving its services behind.
#
# `make install` rewrites the shebang, so an installed copy is trusted and left
# alone -- an operator who chose a different SHEBANG gets what they chose.  An
# unsubstituted placeholder means a checkout being run in place, and that is
# re-executed under the PASE shell exactly once.
zi_pase_reexec() {
    case $ZI_LIBDIR_INSTALLED in
        /*) return 0 ;;
    esac
    [ -z "${ZI_PASE_REEXEC:-}" ] || return 0
    [ -x /QOpenSys/usr/bin/sh ] || return 0
    ZI_PASE_REEXEC=1
    export ZI_PASE_REEXEC
    exec /QOpenSys/usr/bin/sh "$0" ${1+"$@"}
}

ZI_ONCE=no
ZI_STOP=no
ZI_RELOAD=no
ZI_SLEEP_PID=
ZI_KICKED=no
ZI_PASS=0
# Set during a pass when a service is in transition; the next sleep is then
# fast_poll_interval instead of poll_interval.
ZID_UNSETTLED=no
# Services that another configured service depends on, space separated with a
# leading and trailing blank, so membership is a plain case match.
ZID_DEPENDED_ON=' '

# ---------------------------------------------------------------------------
# Bootstrap
# ---------------------------------------------------------------------------

zid_bootstrap() {
    zi_pase_reexec ${1+"$@"}
    case $0 in
        */*) ZI_SELF_DIR=${0%/*} ;;
        *)   ZI_SELF_DIR=. ;;
    esac

    if [ -n "${ZENDINIT_LIBDIR:-}" ]; then
        ZI_LIBDIR=$ZENDINIT_LIBDIR
    else
        ZI_LIBDIR=
        case $ZI_LIBDIR_INSTALLED in
            /*) [ -d "$ZI_LIBDIR_INSTALLED" ] && ZI_LIBDIR=$ZI_LIBDIR_INSTALLED ;;
        esac
        if [ -z "$ZI_LIBDIR" ]; then
            for zid_cand in "$ZI_SELF_DIR/../lib" "$ZI_SELF_DIR/lib"; do
                if [ -f "$zid_cand/config.sh" ]; then
                    ZI_LIBDIR=$zid_cand
                    break
                fi
            done
        fi
    fi
    if [ -z "$ZI_LIBDIR" ] || [ ! -f "$ZI_LIBDIR/config.sh" ]; then
        printf 'zendinitd: cannot locate the library directory\n' >&2
        exit 1
    fi

    . "$ZI_LIBDIR/common.sh"
    . "$ZI_LIBDIR/paths.sh"
    . "$ZI_LIBDIR/log.sh"
    . "$ZI_LIBDIR/config.sh"
    . "$ZI_LIBDIR/adapter.sh"
    . "$ZI_LIBDIR/state.sh"
    . "$ZI_LIBDIR/lock.sh"
    . "$ZI_LIBDIR/service.sh"
    . "$ZI_LIBDIR/depend.sh"

    # shellcheck disable=SC2034  # consumed by lib/log.sh
    LOG_TAG=zendinitd
    case $ZI_VERSION in
        [0-9]*) ;;
        *) ZI_VERSION=$(cat "$ZI_SELF_DIR/../VERSION" 2>/dev/null || printf 'unknown') ;;
    esac
}

zid_usage() {
    cat <<'EOF'
Usage: zendinitd [options]

Options:
  --once            run a single reconcile pass and exit
  --foreground      stay in the foreground (the default; the supervisor is a
                    submitted batch job, so it does not daemonise itself)
  --log FILE        write the supervisor log here
  --level LEVEL     debug, info, warn, error or none
  -h, --help        show this help
  -V, --version     show the version

Signals:
  TERM, INT         stop every running service, then exit
  HUP               re-read the configuration
  USR1              wake up and reconcile now
EOF
}

# ---------------------------------------------------------------------------
# Signals
# ---------------------------------------------------------------------------

# The sleep runs in the background and is waited on, because a foreground
# sleep delays trap delivery until it finishes -- a supervisor that ignores a
# stop request for a full poll interval is not acceptable.
#
# A signal is remembered in ZI_KICKED as well as cutting the sleep short.  A
# signal that arrives during a pass finds no sleep to interrupt, and before the
# flag existed it was simply lost: `zendinit start` sent USR1 while a pass was
# running and then waited out a whole poll_interval for the next one.  Now the
# sleep that follows such a pass is skipped.
zid_sleep() {
    [ "$1" -gt 0 ] 2>/dev/null || return 0
    if [ "$ZI_KICKED" = yes ]; then
        ZI_KICKED=no
        log_debug "woken during the pass; not sleeping"
        return 0
    fi
    sleep "$1" &
    ZI_SLEEP_PID=$!
    # A signal between the check above and ZI_SLEEP_PID being set had no sleep
    # to interrupt, so it is acted on here.
    if [ "$ZI_KICKED" = yes ]; then
        kill "$ZI_SLEEP_PID" 2>/dev/null
    fi
    log_debug "sleeping ${1}s (sleep pid $ZI_SLEEP_PID)"
    wait "$ZI_SLEEP_PID" 2>/dev/null
    log_debug "awake"
    ZI_SLEEP_PID=
    ZI_KICKED=no
    return 0
}

# Invoked from the trap strings below.  Which code the linter reports for that
# depends on its version, so both are disabled.
# shellcheck disable=SC2329,SC2317
zid_interrupt_sleep() {
    ZI_KICKED=yes
    [ -n "$ZI_SLEEP_PID" ] || return 0
    kill "$ZI_SLEEP_PID" 2>/dev/null
    return 0
}

# ---------------------------------------------------------------------------
# Reconcile
# ---------------------------------------------------------------------------

# Desired state: what the configuration says AND what an administrator asked
# for.  Both must agree before anything is started.
zid_desired_running() {
    [ "$SVC_ENABLED" = yes ] || return 1
    [ "$ST_TARGET" = started ] || return 1
    return 0
}

# zid_record_failure <service> -- the service was running and is not any more.
#
# Returns 0 if a restart should be attempted once any backoff has elapsed, and
# 1 if it must not be: policy says never, or the service has been marked
# failed.  The caller must honour that, or the restart policy is decoration.
zid_record_failure() {
    zid_rf_name=$1
    cmn_clock
    zid_rf_now=$CMN_NOW

    if svc_read_exit "$zid_rf_name"; then
        ST_LAST_EXIT=$CMN_R
    else
        ST_LAST_EXIT=
    fi
    ST_JOBID=
    ST_PID=
    ST_READY=no

    # A window that has expired starts again, so occasional failures spread
    # over days never accumulate into a flap verdict.
    if [ "$ST_WINDOW_START" -eq 0 ] || \
       [ $((zid_rf_now - ST_WINDOW_START)) -gt "$SVC_RESTART_WINDOW" ]; then
        ST_WINDOW_START=$zid_rf_now
        ST_RESTARTS=0
    fi
    ST_RESTARTS=$((ST_RESTARTS + 1))

    log_warn "$zid_rf_name stopped (exit ${ST_LAST_EXIT:-unknown}), failure $ST_RESTARTS in this window"

    if ! svc_wants_restart "$SVC_RESTART" "$ST_LAST_EXIT"; then
        state_set_state "$zid_rf_name" stopped "not restarting: restart=$SVC_RESTART"
        return 1
    fi

    if [ "$ST_RESTARTS" -ge "$SVC_RESTART_BURST" ]; then
        ST_BACKOFF_UNTIL=0
        state_set_state "$zid_rf_name" failed \
            "$ST_RESTARTS failures within ${SVC_RESTART_WINDOW}s"
        log_error "$zid_rf_name has failed $ST_RESTARTS times within ${SVC_RESTART_WINDOW}s; giving up until it is started again"
        if [ "$SVC_NOTIFY_QSYSOPR" = yes ]; then
            ibmi_sndmsg_qsysopr "zend-initd: service $zid_rf_name failed $ST_RESTARTS times and will not be restarted"
        fi
        return 1
    fi

    svc_backoff_delay "$ST_RESTARTS" "$SVC_RESTART_DELAY" "$SVC_RESTART_DELAY_MAX"
    ST_BACKOFF_UNTIL=$((zid_rf_now + CMN_R))
    state_set_state "$zid_rf_name" stopped "restarting in ${CMN_R}s"
    return 0
}

# zid_try_spawn <service> -- start it if policy and backoff allow.
zid_try_spawn() {
    zid_ts_name=$1
    cmn_clock
    zid_ts_now=$CMN_NOW

    if [ "$ST_STATE" = failed ]; then
        return 1
    fi
    if [ "$ST_STATE" = completed ]; then
        # A oneshot that has run.  `zendinit start` clears this when an
        # administrator wants it run again.
        return 1
    fi
    if [ "$ST_BACKOFF_UNTIL" -gt "$zid_ts_now" ]; then
        log_debug "$zid_ts_name: waiting $((ST_BACKOFF_UNTIL - zid_ts_now))s more before restarting"
        # A backoff that ends before the next regular pass would otherwise be
        # stretched to a whole poll_interval.
        if [ $((ST_BACKOFF_UNTIL - zid_ts_now)) -lt "$CFG_POLL_INTERVAL" ]; then
            ZID_UNSETTLED=yes
        fi
        return 1
    fi
    if [ "$ZID_STARTS_THIS_PASS" -ge "$CFG_MAX_CONCURRENT_STARTS" ]; then
        log_debug "$zid_ts_name: deferring start, $CFG_MAX_CONCURRENT_STARTS already starting this pass"
        ZID_UNSETTLED=yes
        return 1
    fi
    # Dependencies are checked here rather than in the ordering, because
    # ordering only decides who is considered first: a dependency may be
    # ordered ahead and still not be up yet.
    if ! svc_deps_satisfied "$zid_ts_name"; then
        log_debug "$zid_ts_name: waiting on $SVC_DEPS_WHY"
        if [ "$ST_NOTE" != "waiting on $SVC_DEPS_WHY" ]; then
            ST_NOTE="waiting on $SVC_DEPS_WHY"
            state_save "$zid_ts_name"
        fi
        return 1
    fi

    ZID_STARTS_THIS_PASS=$((ZID_STARTS_THIS_PASS + 1))
    if svc_spawn "$zid_ts_name"; then
        state_set_state "$zid_ts_name" starting
        # Seen running on the next pass, which is a fast one.
        ZID_UNSETTLED=yes
        return 0
    fi
    state_save "$zid_ts_name"
    return 1
}

# zid_reconcile_service <service>
zid_reconcile_service() {
    zid_rs_name=$1
    cmn_clock
    zid_rs_now=$CMN_NOW

    paths_service_conf "$zid_rs_name"
    zid_rs_conf=$CMN_R
    state_load "$zid_rs_name"

    if [ ! -f "$zid_rs_conf" ]; then
        # Configuration removed while the service was running.  Stop what we
        # started, then forget it: leaving an unmanaged job behind would be
        # worse than either alternative.
        if [ -n "$ST_JOBID" ]; then
            log_warn "$zid_rs_name: configuration is gone; stopping $ST_JOBID"
            config_service_defaults "$zid_rs_name"
            config_service_postcheck 2>/dev/null
            svc_stop "$zid_rs_name"
        fi
        state_remove "$zid_rs_name"
        return 0
    fi

    config_reset_diagnostics
    if ! config_load_service "$zid_rs_conf"; then
        log_error "$zid_rs_name: configuration could not be read; leaving it alone this pass"
        return 0
    fi
    if [ "$CFG_ERROR_COUNT" -gt 0 ]; then
        # A service whose config is broken is left exactly as it is: not
        # started, and not stopped either if it is already running.
        log_error "$zid_rs_name: configuration has $CFG_ERROR_COUNT error(s); leaving it alone"
        log_debug "$CFG_ERRORS"
        return 0
    fi

    # Tier 2 is asked on the first pass, then every status_probe_every passes,
    # and whenever tier 1 is inconclusive.  See PLAN.md 4.1.
    zid_rs_tier2=
    if [ "$ZI_PASS" -le 1 ] || \
       [ $((ZI_PASS % CFG_STATUS_PROBE_EVERY)) -eq 0 ]; then
        zid_rs_tier2=tier2
    fi
    svc_liveness "$zid_rs_name" "$zid_rs_tier2"

    if zid_desired_running; then
        case $SVC_LIVE in
            running)
                if [ "$ST_STATE" != running ]; then
                    log_info "$zid_rs_name is running as $ST_JOBID"
                    state_set_state "$zid_rs_name" running
                fi
                if [ -z "$ST_PID" ] && svc_read_pid "$zid_rs_name"; then
                    ST_PID=$CMN_R
                    state_save "$zid_rs_name"
                fi
                # Readiness is what dependents wait on, so it is evaluated
                # until it passes and then remembered.
                if [ "$ST_READY" != yes ]; then
                    if svc_ready "$zid_rs_name"; then
                        log_info "$zid_rs_name is ready"
                        ST_READY=yes
                        state_save "$zid_rs_name"
                    elif [ "$ST_SINCE" -gt 0 ] && \
                         [ $((zid_rs_now - ST_SINCE)) -lt "$SVC_READINESS_TIMEOUT" ]; then
                        # Something waits on this one: check again soon, so a
                        # dependent starts within seconds of it becoming ready.
                        case $ZID_DEPENDED_ON in
                            *" $zid_rs_name "*) ZID_UNSETTLED=yes ;;
                        esac
                    elif [ "$ST_SINCE" -gt 0 ]; then
                        # Running but never became ready.  Reported once and
                        # then left alone: the service is up, and forcing a
                        # restart of something that may be working would be
                        # worse than letting its dependents wait.
                        if [ "$ST_NOTE" != "not ready after ${SVC_READINESS_TIMEOUT}s" ]; then
                            log_error "$zid_rs_name has been running for ${SVC_READINESS_TIMEOUT}s without becoming ready ($SVC_READINESS)"
                            ST_NOTE="not ready after ${SVC_READINESS_TIMEOUT}s"
                            state_save "$zid_rs_name"
                        fi
                    fi
                fi
                # Stability resets the failure count, so a service that has
                # behaved for a full window starts again with a clean slate.
                if [ "$ST_RESTARTS" -gt 0 ] && [ "$ST_SINCE" -gt 0 ] && \
                   [ $((zid_rs_now - ST_SINCE)) -ge "$SVC_RESTART_WINDOW" ]; then
                    log_info "$zid_rs_name has been up for ${SVC_RESTART_WINDOW}s; clearing its failure count"
                    ST_RESTARTS=0
                    ST_WINDOW_START=0
                    ST_BACKOFF_UNTIL=0
                    state_save "$zid_rs_name"
                fi
                ;;
            msgw)
                # Alive but not working, and it will stay that way until
                # somebody answers a message nobody is watching for.
                log_error "$zid_rs_name is in MSGW ($ST_JOBID): the process is alive but the work has stopped"
                if [ "$SVC_NOTIFY_QSYSOPR" = yes ]; then
                    ibmi_sndmsg_qsysopr "zend-initd: service $zid_rs_name is in MSGW and will be restarted"
                fi
                svc_stop "$zid_rs_name"
                ST_LAST_EXIT=
                zid_record_failure "$zid_rs_name"
                ;;
            queued)
                # Submitted and waiting for a slot.  Resubmitting here is the
                # duplicate-job bug; the only correct action is to wait.
                if [ "$ST_STATE" != starting ]; then
                    state_set_state "$zid_rs_name" starting "queued"
                fi
                # A job that waits for a free slot can wait a long time, so
                # it is only looked at often within its start_timeout.
                if [ $((zid_rs_now - ST_SINCE)) -lt "$SVC_START_TIMEOUT" ]; then
                    ZID_UNSETTLED=yes
                fi
                log_debug "$zid_rs_name: job $ST_JOBID is on the queue"
                ;;
            ended)
                zid_rs_restart=yes
                # A oneshot that exited cleanly did what it was asked.  It is
                # not restarted, and it is not a failure either -- 'completed'
                # is a state of its own so that `status` can say so and
                # dependents can treat it as satisfied.
                if [ "$SVC_TYPE" = oneshot ]; then
                    case $ST_STATE in
                        running|starting)
                            if svc_read_exit "$zid_rs_name" && [ "$CMN_R" = 0 ]; then
                                ST_LAST_EXIT=0
                                ST_JOBID=
                                ST_PID=
                                ST_READY=no
                                log_info "$zid_rs_name completed"
                                state_set_state "$zid_rs_name" completed
                                return 0
                            fi
                            ;;
                        completed)
                            return 0
                            ;;
                    esac
                fi
                case $ST_STATE in
                    running|starting)
                        # It died.  Whether it comes back is the policy's call,
                        # and a decision not to restart must stick -- spawning
                        # anyway would make restart=never meaningless and would
                        # overwrite the reason recorded in state.
                        zid_record_failure "$zid_rs_name" || zid_rs_restart=no
                        ;;
                esac
                if [ "$zid_rs_restart" = yes ]; then
                    zid_try_spawn "$zid_rs_name" || true
                fi
                ;;
            *)
                log_warn "$zid_rs_name: state could not be established; taking no action this pass"
                ;;
        esac
    else
        case $SVC_LIVE in
            running|msgw|queued)
                log_info "$zid_rs_name: stopping (enabled=$SVC_ENABLED, target=$ST_TARGET)"
                state_set_state "$zid_rs_name" stopping
                svc_stop "$zid_rs_name"
                ST_JOBID=
                ST_PID=
                ST_READY=no
                state_set_state "$zid_rs_name" stopped
                ;;
            ended)
                if [ "$ST_STATE" != stopped ] && [ "$ST_STATE" != failed ]; then
                    ST_JOBID=
                    ST_PID=
                    state_set_state "$zid_rs_name" stopped
                fi
                ;;
        esac
    fi
    return 0
}

# The order services are considered in: dependencies first, priority breaking
# ties within a level.
#
# Services that have state but no configuration are placed last, so that a
# deleted configuration results in a stopped job rather than an orphan.
zid_service_order() {
    zid_so_graph=
    ZID_DEPENDED_ON=' '
    config_list_services
    zid_so_rest=$CMN_R
    while [ -n "$zid_so_rest" ]; do
        cmn_pop_line "$zid_so_rest"
        zid_so_name=$CMN_R
        zid_so_rest=$CMN_REST
        paths_service_conf "$zid_so_name"
        config_reset_diagnostics
        if config_load_service "$CMN_R" 2>/dev/null; then
            # depend.sh wants the dependency list comma-separated on one line.
            zid_so_deps=
            zid_so_deprest=$SVC_DEPENDS
            while [ -n "$zid_so_deprest" ]; do
                cmn_pop_line "$zid_so_deprest"
                zid_so_deprest=$CMN_REST
                [ -n "$CMN_R" ] || continue
                zid_so_deps=${zid_so_deps:+$zid_so_deps,}$CMN_R
            done
            zid_so_ifs=$IFS
            IFS=,
            for zid_so_dep in $zid_so_deps; do
                case $ZID_DEPENDED_ON in
                    *" $zid_so_dep "*) ;;
                    *) ZID_DEPENDED_ON="$ZID_DEPENDED_ON$zid_so_dep " ;;
                esac
            done
            IFS=$zid_so_ifs
            zid_so_graph="$zid_so_graph$SVC_PRIORITY:$zid_so_name:$zid_so_deps
"
        else
            zid_so_graph="${zid_so_graph}50:$zid_so_name:
"
        fi
    done

    DEP_CYCLE=
    depend_order "$zid_so_graph"
    zid_so_ordered=$CMN_R
    if [ -n "$DEP_CYCLE" ]; then
        log_error "dependency cycle among: $DEP_CYCLE -- these are started in priority order instead"
    fi

    state_list
    zid_so_rest=$CMN_R
    while [ -n "$zid_so_rest" ]; do
        cmn_pop_line "$zid_so_rest"
        zid_so_name=$CMN_R
        zid_so_rest=$CMN_REST
        paths_service_conf "$zid_so_name"
        if [ ! -f "$CMN_R" ]; then
            zid_so_ordered="$zid_so_ordered$zid_so_name
"
        fi
    done
    CMN_R=$zid_so_ordered
}

zid_reconcile_pass() {
    ZI_PASS=$((ZI_PASS + 1))
    ZID_STARTS_THIS_PASS=0
    ZID_UNSETTLED=no
    log_debug "reconcile pass $ZI_PASS"

    zid_service_order
    zid_rp_rest=$CMN_R
    while [ -n "$zid_rp_rest" ]; do
        cmn_pop_line "$zid_rp_rest"
        zid_rp_name=$CMN_R
        zid_rp_rest=$CMN_REST
        [ -n "$zid_rp_name" ] || continue
        zid_reconcile_service "$zid_rp_name"
    done
    return 0
}

# Stop everything we are running, for a controlled shutdown.
zid_stop_all() {
    log_info "stopping all services"
    zid_service_order
    # Reverse of the start order, so a service is stopped before the things it
    # depends on go away underneath it.
    depend_reverse "$CMN_R"
    zid_sa_rest=$CMN_R
    while [ -n "$zid_sa_rest" ]; do
        cmn_pop_line "$zid_sa_rest"
        zid_sa_name=$CMN_R
        zid_sa_rest=$CMN_REST
        [ -n "$zid_sa_name" ] || continue
        state_load "$zid_sa_name"
        [ -n "$ST_JOBID" ] || continue
        paths_service_conf "$zid_sa_name"
        config_reset_diagnostics
        config_load_service "$CMN_R" 2>/dev/null || config_service_defaults "$zid_sa_name"
        state_set_state "$zid_sa_name" stopping
        svc_stop "$zid_sa_name"
        ST_JOBID=
        ST_PID=
        state_set_state "$zid_sa_name" stopped "supervisor shutting down"
    done
    return 0
}

zid_reload() {
    log_info "re-reading configuration"
    config_reset_diagnostics
    config_load_global "$ZI_GLOBAL_CONF"
    if [ "$CFG_ERROR_COUNT" -gt 0 ]; then
        log_error "global configuration has errors; keeping the previous settings"
        log_error "$CFG_ERRORS"
        return 1
    fi
    log_set_level "$CFG_LOG_LEVEL"
    # shellcheck disable=SC2034  # consumed by lib/log.sh
    LOG_MAX_SIZE=$CFG_LOG_MAX_SIZE
    # shellcheck disable=SC2034  # consumed by lib/log.sh
    LOG_KEEP=$CFG_LOG_KEEP
    # shellcheck disable=SC2034  # read by lib/ibmi.sh
    IBMI_LIBRARY=$CFG_LIBRARY
    return 0
}

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

zid_bootstrap ${1+"$@"}

ZI_LOG_ARG=
ZI_LEVEL_ARG=
while [ $# -gt 0 ]; do
    case $1 in
        --once) ZI_ONCE=yes ;;
        --foreground) ;;
        --log) ZI_LOG_ARG=${2:-}; shift ;;
        --log=*) ZI_LOG_ARG=${1#--log=} ;;
        --level) ZI_LEVEL_ARG=${2:-}; shift ;;
        --level=*) ZI_LEVEL_ARG=${1#--level=} ;;
        -h|--help) zid_usage; exit 0 ;;
        -V|--version) printf 'zendinitd %s\n' "$ZI_VERSION"; exit 0 ;;
        *)
            printf 'zendinitd: unknown argument: %s\n' "$1" >&2
            zid_usage >&2
            exit 2
            ;;
    esac
    shift
done

config_reset_diagnostics
config_load_global "$ZI_GLOBAL_CONF"
if [ "$CFG_ERROR_COUNT" -gt 0 ]; then
    printf 'zendinitd: global configuration has %s error(s):\n%s' \
        "$CFG_ERROR_COUNT" "$CFG_ERRORS" >&2
    exit 6
fi
log_set_level "$CFG_LOG_LEVEL"
# shellcheck disable=SC2034  # consumed by lib/log.sh
LOG_MAX_SIZE=$CFG_LOG_MAX_SIZE
# shellcheck disable=SC2034  # consumed by lib/log.sh
LOG_KEEP=$CFG_LOG_KEEP
[ -n "$ZI_LEVEL_ARG" ] && log_set_level "$ZI_LEVEL_ARG"

for zid_dir in "$ZI_STATE_DIR" "$ZI_RUN_DIR" "$ZI_LOG_DIR"; do
    [ -d "$zid_dir" ] || mkdir -p "$zid_dir" 2>/dev/null
done

if [ -n "$ZI_LOG_ARG" ]; then
    log_set_file "$ZI_LOG_ARG" || log_warn "cannot write $ZI_LOG_ARG; logging to stderr"
elif [ "$ZI_ONCE" != yes ]; then
    log_set_file "$ZI_SUPERVISOR_LOG" || \
        log_warn "cannot write $ZI_SUPERVISOR_LOG; logging to stderr"
fi

if ! adapter_load "$ZI_LIBDIR"; then
    log_die 1 "no usable platform adapter"
fi
# shellcheck disable=SC2034  # read by lib/ibmi.sh
IBMI_LIBRARY=$CFG_LIBRARY

if ! lock_acquire; then
    # QSYSOPR as well as the log, because at IPL the log is exactly what this
    # cannot write.  A supervisor that failed to start as the autostart job's
    # profile left no trace anywhere at all: no log line, because the log
    # directory was not writable either, and no joblog message, because the
    # failure happened inside PASE.  Nothing started and nothing said why.
    # QSYSOPR is where an operator looks after an IPL.
    ibmi_sndmsg_qsysopr "zend-initd: the supervisor could not start -- it did not get its lock ($ZI_RUN_DIR). No services are running." 2>/dev/null
    log_die 1 "refusing to start: could not take the supervisor lock"
fi
# Traps are installed here, at the top level, and deliberately NOT inside a
# function.  In PASE ksh a trap set inside a function is local to that function
# and is discarded when it returns, so a supervisor that installed its handlers
# from a helper had no handlers at all: it died on SIGTERM with "PASE for i
# ended for signal 15" and left every service it had started still running.
# bash and dash do not behave this way, so nothing off-platform catches it.
#
# The stop request is logged from inside the trap: when a supervisor goes away,
# the first question is always whether it was asked to.
trap 'lock_release; ibmi_cleanup' EXIT
trap 'ZI_STOP=yes; log_info "stop requested (signal)"; zid_interrupt_sleep' TERM INT
trap 'ZI_RELOAD=yes; zid_interrupt_sleep' HUP
# USR1 means "reconcile now": interrupting the sleep is the whole mechanism, so
# there is nothing to remember about it.
trap 'zid_interrupt_sleep' USR1

log_info "zendinitd $ZI_VERSION starting (adapter=$IBMI_KIND, poll=${CFG_POLL_INTERVAL}s, fast poll=${CFG_FAST_POLL_INTERVAL}s)"

if [ "$CFG_STARTUP_DELAY" -gt 0 ]; then
    log_info "waiting ${CFG_STARTUP_DELAY}s before the first pass"
    zid_sleep "$CFG_STARTUP_DELAY"
fi

# Adoption is not a special code path: the first pass reads the job ids that
# were recorded before, asks the platform what is still there, and reports it.
state_list
zid_adopt_rest=$CMN_R
zid_adopted=0
while [ -n "$zid_adopt_rest" ]; do
    cmn_pop_line "$zid_adopt_rest"
    zid_adopt_name=$CMN_R
    zid_adopt_rest=$CMN_REST
    [ -n "$zid_adopt_name" ] || continue
    state_load "$zid_adopt_name"
    [ -n "$ST_JOBID" ] || continue
    svc_liveness "$zid_adopt_name" tier2
    case $SVC_LIVE in
        running|msgw|queued)
            log_info "adopting $zid_adopt_name, already $SVC_LIVE as $ST_JOBID"
            zid_adopted=$((zid_adopted + 1))
            ;;
    esac
done
[ "$zid_adopted" -gt 0 ] && log_info "adopted $zid_adopted service(s) from a previous supervisor"

while :; do
    zid_reconcile_pass

    if [ "$ZI_ONCE" = yes ]; then
        break
    fi
    if [ "$ZI_STOP" = yes ]; then
        break
    fi

    if [ "$ZID_UNSETTLED" = yes ] && \
       [ "$CFG_FAST_POLL_INTERVAL" -lt "$CFG_POLL_INTERVAL" ]; then
        zid_sleep "$CFG_FAST_POLL_INTERVAL"
    else
        zid_sleep "$CFG_POLL_INTERVAL"
    fi

    if [ "$ZI_RELOAD" = yes ]; then
        ZI_RELOAD=no
        zid_reload || true
    fi
    if [ "$ZI_STOP" = yes ]; then
        break
    fi
done

if [ "$ZI_STOP" = yes ]; then
    zid_stop_all
fi

log_info "zendinitd stopping"
lock_release
exit 0
