#!/bin/bash
# vim: cindent:shiftwidth=4:tabstop=4:smarttab:textwidth=100:noexpandtab

set -o errexit
set -o nounset
#set -o xtrace

#$title$ Clearcable SOE System Check
#$check$ system compliance with a deployment checklist, unexpected changes in system state
#$ref$ KB: SOE4SystemCheck
#$author$ Rafal Rzeczkowski
#$version$ 1.3.7

#level_check supervisor

#CHANGELOG
#0.20	experimental
#0.21	plugin state support
#0.22	exit if there are no plugins
#0.25	run short/long groups or display
#0.26	allow individual plugins to exit independently
#0.27	replace clear{} with ok{} since clear(1) is defined
#0.28	add an explicit return statement for functions
#0.29	add check name when displaying results
#0.30	do not run in unprivileged mode if there is no data yet
#0.31	run plugins without checking in PATH; avoid name conflicts
#0.40	record OK status, keep/display first event date
#0.41	colour support in status display, summary
#0.42	ignore lack of colour support on dumb terminals
#0.43	handle state changes between warning<->critical
#0.44	handle state changes to unknown
#0.45	echodebug function to use for printing debug messages
#0.46	display the severity of failure in debug messages
#0.47	is_worktime function to intelligently set severity
#0.48	explicit return code in echodebug() to avoid failing plugin
#0.49	generate an internal warning if plugin fails
#0.50	report plugin execution runtime
#0.51	auto-detect and select the most appropriate IPC directory location
#0.52	safer, global lock for all execution levels
#0.53	added support for a help message and additional display verbosity levels
#0.54	selective SYS checks, support for clear and enable/disable actions
#0.55	append to arrays via the official += operator for bash 4.3 compatibility
#0.56	allow processing of escape sequences in echodebug()
#0.57	export TERM to avoid tput warnings on dumb terminals
#0.58	verbose output is now paragraph based
#0.59	display partial verbose output for errors without helpmsg
#0.60	support verbose display of syscheck supervisor plugin execution errors
#0.61	support displaying of plugin author metadata field
#0.62	support command line debug switch to increase plugin debug level
#0.63	check_error_counter_list() available to use by plugins (nic-errors, io-errors)
#0.64	support (and ignore) device hot-plug when checking error_counter_list
#0.65	support problem acknowledgement
#0.66	more comprehensive plugin status statistical breakdown
#0.67	"failed" selector automatically selects previously failed plugins for a re-check
#0.68	added reusable is_element_of() library function (former name = isElementOf)
#0.69	log recoveries to syslog
#0.70	added restrict/release cron API
#0.71	parse an explicit version header in preference to "$V" version variables
#0.72	correct shellcheck issues
#0.73	record PPID in restrict/release lock file
#0.74	add 'release-all' option to wipe all restrict/release records
#0.75	fix globbing for 'rm' command not working when quoted
#0.76	fix filename pattern for 'release-all' option
#0.77	add 'query' function for external integration (such as munin-node)
#0.78	support 'caution' priority level
#0.79	support getting status of single plugin
#0.80	colourize output only when standard output is connected to a terminal
#0.81	harmonize syscheck version encoding with that of plugins
#0.82	move explicit PATH definition directly to syscheck script itself
#0.83	support additional properties in plugin level processing
#0.84	unify runtime delay randomization
#0.85	reverse logic on DEBUG flag handling
#0.86	support terminals without setaf or even sgr capabilities
#0.87	restyle according to https://kb.clearcable.ca/KB/ProgrammingStyleStandards
#0.88	check for superuser permissions only when initialization is pending
#0.89	add a second argument to level_check API so it is called only once
#0.90	ignore submission of performance counters (NATS API preparation)
#1.0.0	publish plugin results via NATS Messaging
#1.0.1	recognize two-part customer codes
#1.1.0	export machine readable problem characteristics
#1.1.1	provide plural_text API
#1.1.2	provide print_size_scaled API
#1.1.3	support time calculations in print_size_scaled
#1.1.4	replace the use of BASHPID with $$ on bash 3 (SOE4.0)
#1.1.5	provide BCFG2_PARCEL and BCFG2_DEBIAN_VERSION exports
#1.1.6	provide TIME_UNIT_* exports
#1.1.7	provide BCFG2_SUPPLEMENT export
#1.1.8	provide BCFG2_GROUPS export
#1.1.9	provide STATE_DIR query response
#1.1.10	ignore Debian version range tags in Bcfg2
#1.2.0	modify disable state to work more like ack state -Joshua Boniface
#1.2.1	add additional meta values to ack and disable -Joshua Boniface
#1.2.2	show ack and disable outputs along with issue levels -Joshua Boniface
#1.2.3	sort output by level -Joshua Boniface
#1.2.4	refactor ack/disable storage to preserve actual state in prep for 1.2.5 -Joshua Boniface
#1.2.5	store state for failed plugins even if ack'd -Joshua Boniface
#1.2.6	show hour+minute in short mod time -Joshua Boniface
#1.2.7	add additional verbosity level with nicer output -Joshua Boniface
#1.2.8	fix bug with all-ok output -Joshua Boniface
#1.2.9	include 2022-adopted SI prefixes in print_size_scaled
#1.2.10	detect interactive use via SUDO_COMMAND rather than the absence of MAILTO
#1.2.11	separate plugin debug output into labelled sections
#1.2.12	wait for plugin execution lock when running as a non-interactive job
#1.2.13	provide sd_booted() function export
#1.2.14	provide check_expiry() function export
#1.3.0	support Perl plugins
#1.3.1	provide update_cache_file() function export
#1.3.2	timeout NATS connection more quickly when DNS lookup or ping fails
#1.3.3	report NATS errors unconditionally via echo_err() instead echodebug()
#1.3.4	provide DEBUG export (Perl)
#1.3.5	publish machine info from hostnamectl
#1.3.6	skip publishing of uptime and machine info during shutdown
#1.3.7	export VERSION_ID and VERSION_CODENAME from os-release(5)

VERSION=$(awk -F'^#[$]version[$][\t ]+' '{if ($2){print $2}}' "$0")
BASENAME=$(basename "$0")
SHARE_DIR="/usr/local/share/$BASENAME"
PATH='/sbin:/bin:/usr/sbin:/usr/bin'

# auto-detect the most optimal location for IPC files
# keep synchronized with the Munin syscheck plugin
if [[ -d '/run' ]]; then
	# preferred (Wheezy+)
	# http://wiki.debian.org/ReleaseGoals/RunDirectory
	RUN='/run'
	LOCK="$RUN/lock/$BASENAME"
elif [[ -d '/dev/shm' ]]; then
	# not technically correct, but tmpfs is desirable
	RUN='/dev/shm'
	LOCK="$RUN/$BASENAME.lock"
else
	# fall-back default
	RUN='/var/run/'
	LOCK="/var/lock/$BASENAME"
fi
declare -r -i LOCK_TIMEOUT=300
  IPC_DIR="$RUN/$BASENAME.ipc"
 META_DIR="$RUN/$BASENAME.meta"  # Stores ack/disable metadata (who/ticket)
STATE_DIR="$RUN/$BASENAME.state"
 HELP_DIR="$RUN/$BASENAME.help"
  FIX_DIR="$RUN/$BASENAME.fix"
 PID_FILE="$RUN/$BASENAME.pid"
 SHUTDOWN="$RUN/$BASENAME.shutdown"
CACHE_DIR='/var/cache/syscheck'

# NATS Messaging telemetry output driver (SOE4.1+)
NATS_SOCKET=0
declare -r NATS_SERVER='nats.clearcable.net.'
declare -r NATS_PORT=4222
declare -r NATS_AUTH_TOKEN='oYMv9WcLN5H70qabJUc8XuzdNbxUCHfn'
declare -r NATS_PING_TIMEOUT=5
declare -r SITE_CODE=$(awk '{if ($0~/^[[:alnum:]][[:alnum:]](_[[:alnum:]][[:alnum:]])?$/){print $0;exit}}' /etc/bcfg2.group)

function nats_reply_check {
	local REPLY_EXPECTED=$1

	if ! IFS=$'\r\n' read -r REPLY <&${NATS_SOCKET}; then
		echo_err 'NATS: timeout receiving a reply'
		NATS_SOCKET=0
		return 1
	fi

	# handle pending PING request
	if [[ $REPLY = 'PING' ]]; then
		echo -ne "PONG\r\n" >&${NATS_SOCKET}
		echodebug 'NATS: PING->PONG (nats_reply_check)'
		if ! IFS=$'\r\n' read -r REPLY <&${NATS_SOCKET}; then
			echo_err 'NATS: timeout receiving a reply after processing PING'
			NATS_SOCKET=0
			return 1
		fi
	fi

	if [[ $REPLY != $REPLY_EXPECTED ]]; then
		echo_err "NATS: invalid reply {$REPLY}, expected {$REPLY_EXPECTED}"
		NATS_SOCKET=0
		return 1
	fi

	return 0
}

function nats_init {
	TMOUT=6 # default timeout for the read builtin

	local ERR_MSG_PREFIX='NATS transport disabled'

	# https://wiki.bash-hackers.org/scripting/bashchanges
	# {varname} style automatic file descriptor allocation | 4.1-alpha
	if [[ ( ${BASH_VERSINFO[0]} -eq 4 && ${BASH_VERSINFO[1]} -lt 1 ) || ${BASH_VERSINFO[0]} -lt 4 ]]; then
		echo_err "$ERR_MSG_PREFIX: NATS client code requires Bash version 4.1+"
		return
	fi

	if ! timeout $NATS_PING_TIMEOUT fping -q $NATS_SERVER; then
		echo_err "$ERR_MSG_PREFIX: timeout on server name resolution and ICMP probe"
		return
	fi

	if ! exec {NATS_SOCKET}<>/dev/tcp/$NATS_SERVER/$NATS_PORT; then
		echo_err "$ERR_MSG_PREFIX: connection to server failed"
		return
	fi

	# https://docs.nats.io/nats-protocol/nats-protocol
	if ! read -r OP_NAME JSON_INFO <&${NATS_SOCKET}; then
		echo_err "$ERR_MSG_PREFIX: failed to receive initial server greeting from FD $NATS_SOCKET"
		NATS_SOCKET=0
		return
	fi
	# INFO {
	# server_id":"NAN36TBAPXUHV7AQYZBLJA2DYLF5PZVTFO5VUPENQOQOEQPC77RNSQC5",
	# "server_name":"NAN36TBAPXUHV7AQYZBLJA2DYLF5PZVTFO5VUPENQOQOEQPC77RNSQC5",
	# "version":"2.1.8",
	# "proto":1,
	# "git_commit":"c0b574f",
	# "go":"go1.14.8",
	# "host":"::",
	# "port":4222,
	# "max_payload":1048576,
	# "client_id":58,
	# "client_ip":"::1"
	# }
	NATS_SERVER_VERSION=${JSON_INFO##*\"version\":\"}
	NATS_SERVER_VERSION=${NATS_SERVER_VERSION%%\",*}

	if [[ $OP_NAME != 'INFO' ]]; then
		echo_err "$ERR_MSG_PREFIX: unexpected greeting from NATS server: expected {INFO}, received {$OP_NAME}"
		NATS_SOCKET=0
		return
	fi

	declare -A CONNECT_INFO
	CONNECT_INFO[verbose]='true'
	CONNECT_INFO[pedantic]='true'
	CONNECT_INFO[tls_required]='false'
	CONNECT_INFO[name]='NATS Publisher for syscheck'
	CONNECT_INFO[lang]='bash'
	CONNECT_INFO[version]=$VERSION
	CONNECT_INFO[protocol]=1
	CONNECT_INFO[echo]='true'
	CONNECT_INFO[auth_token]=$NATS_AUTH_TOKEN

	CONNECT_INFO_JSON=''
	for KEY in "${!CONNECT_INFO[@]}"; do
		VALUE=${CONNECT_INFO[$KEY]}
		if [[ -n "$CONNECT_INFO_JSON" ]]; then
			CONNECT_INFO_JSON+=','
		fi
		if [[ $VALUE = 'true' || $VALUE = 'false' || $VALUE =~ ^-?[0-9]+$ ]]; then
			CONNECT_INFO_JSON+="\"$KEY\":$VALUE"
		else
			CONNECT_INFO_JSON+="\"$KEY\":\"$VALUE\""
		fi
	done

	echo -ne "CONNECT {$CONNECT_INFO_JSON}\r\n" >&${NATS_SOCKET}
	if ! nats_reply_check '+OK'; then
		NATS_SOCKET=0
		return
	fi

	echodebug "publishing NATS updates to $NATS_SERVER:$NATS_PORT (server version $NATS_SERVER_VERSION) via FD $NATS_SOCKET"
}

function nats_keepalive_validate {
	local REQUEST
	if read -t 0 <&${NATS_SOCKET}; then
		# handle pending PING request
		IFS=$'\r\n' read -r REQUEST <&${NATS_SOCKET}
		if [[ $REQUEST = 'PING' ]]; then
			echo -ne "PONG\r\n" >&${NATS_SOCKET}
			echodebug 'NATS: PING->PONG (nats_keepalive_validate)'
		else
			echo_err "NATS: received unexpected message {$REQUEST}"
			NATS_SOCKET=0
		fi
	fi
}

function nats_pub {
	local SUBJECT_SUFFIX=$1
	shift
	local MSG="$@"

	if [[ $NATS_SOCKET -eq 0 ]]; then
		# NATS transport not initialized / failed
		return
	fi

	nats_keepalive_validate

	SUBJECT="$SITE_CODE.${HOSTNAME//./_}.syscheck.$SUBJECT_SUFFIX"

	echo -ne "PUB $SUBJECT ${#MSG}\r\n$MSG\r\n" >&${NATS_SOCKET}
	nats_reply_check '+OK'
}

function nats_plugin_start {
	local SYS=$1
	nats_pub $SYS.execution_start $(date +%s.%N)
}

function nats_plugin_end {
	local SYS=$1
	nats_pub $SYS.execution_end $(date +%s.%N)
}

function nats_host_status {
	local status=$1
	nats_plugin_start $SYS

	if [[ $status = 'up' ]]; then
		publish_uptime
		publish_machine_info
	fi

	nats_pub $SYS.status $status
	echodebug "$HOSTNAME syscheck service status: $status"
	nats_plugin_end $SYS
}

function echodebug {
	if [[ "$DEBUG" -gt 0 ]]; then
		echo -e "$*"
	fi
	return 0
}

function echo_err {
	local msg="$@"

	echo -e "$msg" 2>&1
	return 0
}

function helpmsg {
	local MSG="$@"

	echo "$MSG" > "$HELP_DIR/$SYS"
}

function loadmeta {
	set +o nounset
	if [[ -z $MODULE ]]; then
		local MODULE=$PLUGIN
	fi
	set -o nounset

	for STATE in ack disable; do
		if [[ -f "$META_DIR/$MODULE.${STATE}" ]]; then
			META_FILE="$META_DIR/$MODULE.${STATE}"
			META=$(<"$META_FILE")
			META_MSG="$(awk -F'|' '{ print $1 }' <<<"$META")"
			META_WHO="$(awk -F'|' '{ print $2 }' <<<"$META")"
			META_TICKET="$(awk -F'|' '{ print $3 }' <<<"$META")"
			break
		else
			META_MSG="Unknown"
			META_WHO="Nobody"
			META_TICKET="N/A"
		fi
	done
}

function problem {
	local ID=$1
	local PARTICULAR=$2

	if ! is_element_of $ID ${PROBLEMS[*]}; then
		echo "internal API failure: ID=$ID is not in the set PROBLEMS=(${PROBLEMS[*]})" 1>&2
		exit 1
	fi

	if [[ ! -d "$FIX_DIR" ]]; then
		# for systems updated with FIX_DIR API while syscheck was already initialized
		mkdir "$FIX_DIR"
	fi

	echo -e "$ID\t$PARTICULAR" > "$FIX_DIR/$SYS"
}

function plural_text {
	local ITEM_COUNT=$1
	local SINGULAR_FORM=$2
	if [[ -n "${3+xxx}" ]]; then
		local PLURAL_FORM=$3
	fi

	# https://en.wikipedia.org/wiki/English_plurals
	if [[ $ITEM_COUNT -eq 0 || $ITEM_COUNT -gt 1 ]]; then
		if [[ -n "${PLURAL_FORM+xxx}" ]]; then
			echo $PLURAL_FORM
		else
			if [[ "$SINGULAR_FORM" =~ (sh|ch|s)$ ]]; then
				echo ${SINGULAR_FORM}es
			else
				echo ${SINGULAR_FORM}s
			fi
		fi
	elif [[ $ITEM_COUNT -eq 1 ]]; then
		echo $SINGULAR_FORM
	else
		echo 'internal API failure: item count out of bounds' 1>&2
		exit 1
	fi
	return
}

function print_size_scaled() {
	local SIZE=$1
	local BASE=$2

	declare -r AUTO_RANGE_MIN=$(( 2**3 ))

	#https://en.wikipedia.org/wiki/Binary_prefix
	#https://en.wikipedia.org/wiki/Metric_prefix
	declare -r -a BINARY_PREFIX=('' Ki Mi Gi Ti Pi Ei Zi Yi Ri Qi)
	declare -r -a DECIMAL_PREFIX=('' k M G T P E Z Y R Q)

	#https://en.wikipedia.org/wiki/Sexagesimal
	#https://en.wikipedia.org/wiki/Unit_of_time
	#https://en.wikipedia.org/wiki/Minute
	#https://en.wikipedia.org/wiki/Hour
	#https://en.wikipedia.org/wiki/Day
	#https://en.wikipedia.org/wiki/Year
	#https://en.wikipedia.org/wiki/Century
	declare -r -a TIME_PREFIX=(s min h d a c)
	declare -r -a TIME_SCALE=(1 60 60 24 365 100)

	if [[ $SIZE -lt 0 ]]; then
		echo 'internal API failure: size must be a positive integer' 1>&2
		exit 1
	fi

	case $BASE in
		 2) declare -r -a PREFIX=("${BINARY_PREFIX[@]}");;
		10) declare -r -a PREFIX=("${DECIMAL_PREFIX[@]}");;
		60) declare -r -a PREFIX=("${TIME_PREFIX[@]}");;
		*)
			echo 'internal API failure: base must be 2, 10, or 60' 1>&2
			exit 1
			;;
	esac

	local i
	for ((i=$((${#PREFIX[@]}-1));i>=0;i--)); do
		case $BASE in
			 2) PREFIX_SIZE=$((2**(10*i)));;
			10) PREFIX_SIZE=$((10**(3*i)));;
			60) PREFIX_SIZE=1
				for ((j=i;j>=0;j--)); do
					PREFIX_SIZE=$((PREFIX_SIZE*${TIME_SCALE[$j]}))
				done
				;;
		esac
		if [[ $PREFIX_SIZE -eq 0 ]]; then
			# 64-bit overflow for large prefixes
			continue
		fi
		if [[ $((SIZE/PREFIX_SIZE)) -gt $AUTO_RANGE_MIN || $i -eq 0 ]]; then
			echo $((SIZE/PREFIX_SIZE))${PREFIX[$i]}
			break
		fi
	done
}

function fail {
	LEVEL=$1
	MSG="$2"

	case $LEVEL in
		warning)	rm --force "$IPC_DIR/$SYS".{ok,critical,caution};;
		critical)	rm --force "$IPC_DIR/$SYS".{ok,warning,caution};;
		caution)	rm --force "$IPC_DIR/$SYS".{ok,warning,critical};;
	esac
	nats_pub $SYS.status $LEVEL
	nats_pub $SYS.problem $MSG
	if [[ -s "$IPC_DIR/$SYS.ack" ]]; then
		nats_pub $SYS.acknowledged $(<$IPC_DIR/$SYS.ack)
		LEVEL="ack"
	fi
	if [[ -s "$IPC_DIR/$SYS.$LEVEL" ]]; then
		MSG_OLD=$(<"$IPC_DIR/$SYS.$LEVEL")
	else
		MSG_OLD=""
	fi
	if [[ "$MSG" != "$MSG_OLD" ]]; then
		echo "$MSG" > "$IPC_DIR/$SYS.$LEVEL"
		logger -t "$0" -p daemon.warning "FAIL[$LEVEL]: $SYS ($MSG)"
	fi
	echo "FAIL[$LEVEL]: $SYS ($MSG)"
	echodebug ''
	return 0
}

function ok {
	local CHECK_EXEC_DURATION
	CHECK_EXEC_DURATION=$((SECONDS - CHECK_START_TIME))
	rm --force "$HELP_DIR/$SYS"
	rm --force "$FIX_DIR/$SYS"
	rm --force "$IPC_DIR/$SYS".{warning,critical,ack,caution}
	rm --force "$META_DIR/$SYS".ack
	if [[ ! -f "$IPC_DIR/$SYS.ok" ]]; then
		touch "$IPC_DIR/$SYS.ok"
		logger -t "$0" -p daemon.info "OK: $SYS"
	fi
	nats_pub $SYS.status ok
	echo "OK: $SYS"
	echodebug ''
	return 0
}

function unknown {
	local MSG=${1:-}
	rm --force "$IPC_DIR/$SYS".*
	rm --force "$META_DIR/$SYS".ack
	nats_pub $SYS.status unknown
	if [[ -n "$MSG" ]]; then
		echodebug "$MSG"
		nats_pub $SYS.problem $MSG
	fi
	echo "UNKNOWN: $SYS"
	echodebug ''
	return 0
}

function is_disabled {
	if [[ "$SYS" == "syscheck" ]]; then
		# The "syscheck" check is never "disabled"
		return 1
	elif [[ -f "$IPC_DIR/$SYS.disable" ]]; then
		# We have a .disable sys, so the plugin is disabled
		return 0
	elif [[ ! -f "$SHARE_DIR/$SYS" ]]; then
		# The plugin does not exist
		echodebug "Plugin $SYS does not exist"
		return 0
	elif [[ ! -x "$SHARE_DIR/$SYS" ]]; then
		# The plugin is not executable, old-style disable
		echodebug "Plugin $SYS is not executable"
		return 0
	else
		# The plugin is not disabled
		return 1
	fi
}

function sysack {
	mv --force --no-target-directory "$IPC_DIR/$SYS".* "$IPC_DIR/$SYS.ack" 2>/dev/null || true
	# Temporary migration workaround until every system reboots
	if [[ ! -d "$META_DIR" ]]; then
		mkdir $META_DIR
	fi
	echo "$REASON|$WHO|$TICKET" > "$META_DIR/$SYS.ack"
	echo "ACK: $SYS"
	echodebug ''
	return 0
}

function sysdisable {
	nats_init
	nats_plugin_start $SYS
	rm --force "$IPC_DIR/$SYS".*
	echo "N/A" > "$IPC_DIR/$SYS.disable"
	# Temporary migration workaround until every system reboots
	if [[ ! -d "$META_DIR" ]]; then
		mkdir $META_DIR
	fi
	echo "$REASON|$WHO|$TICKET" > "$META_DIR/$SYS.disable"
	nats_pub $SYS.status unknown
	nats_pub $SYS.problem 'administrative disable'
	echodebug "DISABLE: $SYS."
	nats_plugin_end $SYS
	return 0
}

function sysenable {
	if [[ ! -f "$SHARE_DIR/$SYS" ]]; then
		echo 'plugin not found'
		exit 1
	fi
	if [[ ! -x "$SHARE_DIR/$SYS" ]]; then
		chmod +x "$SHARE_DIR/$SYS"
	fi
	rm --force "$IPC_DIR/$SYS".*
	rm --force "$META_DIR/$SYS".*
	echodebug "ENABLE: $SYS."
	return 0
}

function performance {
	local METRIC=$1
	local VALUE=$2
	nats_pub $SYS.@performance "$METRIC=$VALUE"
}

function publish_uptime {
	local uptime idle_time

	# proc(5) : /proc/uptime
	# This file contains two numbers: the uptime of the system (seconds), and
	# the amount of time spent in idle process (seconds).
	read -r uptime idle_time < /proc/uptime

	performance uptime $uptime
	performance idle_time $idle_time
}

function publish_machine_info {
	if ! sd_booted; then
		return
	fi

	local key value
	while IFS=" : " read key value; do
		if [[ -z "$value" ]]; then
			continue
		fi
		value=${value%,}
		key=${key#*\"}
		key=${key%\"}
		if [[ "$value" = 'null' ]]; then
			continue
		fi
		nats_pub $SYS.@machine_info "$key=$value"
	done < <(hostnamectl --json="pretty")
}

function level_check {
	LEVEL_OF_PLUGIN="$1"
	BEHAVIOUR_FLAG="${2:-NULL}"

	case $LEVEL_REQUESTED in
		all)
			;;
		on_boot)
			case $BEHAVIOUR_FLAG in
				settling|extended)
					echo $LEVEL_OF_PLUGIN > $LEVEL_EXCEPTION
					exit
					;;
			esac
			;;
		short)
			case $LEVEL_OF_PLUGIN in
				long)
					echo $LEVEL_OF_PLUGIN > $LEVEL_EXCEPTION
					exit
					;;
			esac
			;;
		long)
			case $LEVEL_OF_PLUGIN in
				short)
					echo $LEVEL_OF_PLUGIN > $LEVEL_EXCEPTION
					exit
					;;
			esac
			;;
		*)
			echo "${TPUT_RED}ERROR: unknown level request {$LEVEL_REQUESTED} for plugin {$MODULE}${TPUT_NORMAL}"
			exit 1
			;;
	esac

	nats_plugin_start $SYS
	echodebug "[$SYS]"
}

function unattended_random_delay {
	local MAX_DELAY="$1"

	if [[ -n "${SUDO_COMMAND+xxx}" ]]; then
		# interactive
		return
	elif [[ "$LEVEL_REQUESTED" == 'on_boot' ]]; then
		# desynchronized via a reboot
		return
	else
		# this is an unattended, time-scheduled job
		sleep $(( RANDOM % MAX_DELAY ))
	fi
}

function state_load {
	local STATE_FILE="$STATE_DIR/$SYS"
	if [[ -s "$STATE_FILE" ]]; then
		cat "$STATE_FILE"
	else
		echo ''
	fi
}

function state_save {
	local STATE_FILE="$STATE_DIR/$SYS"
	cat > "$STATE_FILE"
}

function is_worktime {
	is_worktime=0
	is_not_worktime=1
	#reversed meaning since 0 shell exit code is OK

	export TZ='America/Toronto'
	#%a	locale's abbreviated weekday name (e.g., Sun)
	#%H	hour (00..23)
	weekday_name=$(date +%a)
	hour=$(date +%H)
	unset TZ

	case $weekday_name in
		Sat|Sun)
			echodebug "now is_not_worktime since weekday_name=$weekday_name"
			return $is_not_worktime;;
		*)
		case $hour in
			09|10|11|12|13|14|15|16)
				echodebug "now is_worktime since hour in 09|10|11|12|13|14|15|16"
				return $is_worktime
				;;
			*)
				echodebug "now is_not_worktime"
				return $is_not_worktime
				;;
		esac
		;;
	esac
}

function check_error_counter_list {
	if [[ ! ( "${OBJECT_DESC+xxx}" && "${FIELD_LIST+xxx}" && "${STATE_CUR+xxx}" && "${HELPMSG+xxx}") ]]; then
		fail 'warning' 'check_error_counter_list() called without required environment: OBJECT_DESC FIELD_LIST STATE_CUR HELPMSG'
		helpmsg 'plugin API fault - contact plugin maintainer'
		return
	fi

	IFS_SAVED="$IFS"
	IFS=$'\n'
	STATE_CUR_LIST=($STATE_CUR)

	unset IFS
	OBJECTs=($(
	IFS=$'\n'
	for line in ${STATE_CUR_LIST[*]}
	do
		obj=${line%%	*}
		echo $obj
	done
	))

	if [[ ${#OBJECTs[*]} -eq 0 ]]; then
		echodebug 'nothing to check - no objects found'
		unknown
		IFS="$IFS_SAVED"
		return
	fi
	echodebug "${#OBJECTs[*]} $OBJECT_DESC(s): ${OBJECTs[*]}"

	STATE_OLD=$(state_load)
	echo "$STATE_CUR" | state_save
	if [[ -z "$STATE_OLD" ]]; then
		echodebug 'no previous state to compare at this time - expected during the first run'
		unknown
		IFS="$IFS_SAVED"
		return
	fi
	IFS=$'\n'
	STATE_OLD_LIST=($STATE_OLD)

	i=0
	while [[ $i -lt ${#OBJECTs[*]} ]]; do
		if [[ -z ${STATE_OLD_LIST[$i]+xxx} ]]; then
			IFS=$'\t'
			OBJECT_CUR=(${STATE_CUR_LIST[$i]})
			unset IFS
			OBJECT_NAME=${OBJECT_CUR[0]}
			echodebug "new $OBJECT_DESC $OBJECT_NAME detected; deferring further analysis to next cycle"
			IFS="$IFS_SAVED"
			return
		elif [[ ${STATE_CUR_LIST[$i]} != ${STATE_OLD_LIST[$i]} ]]; then
			IFS=$'\t'
			OBJECT_CUR=(${STATE_CUR_LIST[$i]})
			OBJECT_OLD=(${STATE_OLD_LIST[$i]})
			unset IFS
			OBJECT_NAME=${OBJECT_CUR[0]}
			unset OBJECT_CUR[0]
			unset OBJECT_OLD[0]
			fail 'warning' "newly detected errors on $OBJECT_DESC $OBJECT_NAME: ${OBJECT_OLD[*]} -> ${OBJECT_CUR[*]} (${FIELD_LIST[*]})"
			helpmsg "$HELPMSG"
			IFS="$IFS_SAVED"
			return
		fi
		i=$((i+1))
	done
	ok
	IFS="$IFS_SAVED"
}

function is_element_of {
	local TO_FIND=$1
	shift

	for ARRAY_ELEMENT in "$@"; do
		if [[ "$TO_FIND" = "$ARRAY_ELEMENT" ]]; then
			return 0
		fi
	done
	return 1
}

if [[ ! -d "$IPC_DIR" ]]; then
	if [[ "$UID" -gt 0 ]]; then
		echo "$BASENAME: initialization pending" 1>&2
		exit
	else
		mkdir "$IPC_DIR" "$META_DIR" "$STATE_DIR" "$HELP_DIR" "$FIX_DIR"
	fi
fi

function source_os_release {
	#man:os-release(5)
	source '/etc/os-release'

	# debian-lenny and debian-squeeze are supported via a Bcfg2 update

	export VERSION_ID VERSION_CODENAME
}

function bcfg2_groups_parse {
	local BCFG2_GROUP
	BCFG2_PARCEL=''
	BCFG2_SUPPLEMENT=''
	BCFG2_GROUPS=()
	while read BCFG2_GROUP; do
		BCFG2_GROUPS+=($BCFG2_GROUP)
		case $BCFG2_GROUP in
			parcel-*) BCFG2_PARCEL=${BCFG2_GROUP##parcel-} ;;
			debian-EOL);;
			debian-ge[0-9]*);;
			debian-le[0-9]*);;
			debian-*) BCFG2_DEBIAN_VERSION=${BCFG2_GROUP##debian-} ;;
			supplement-*)
				if [[ -z "$BCFG2_SUPPLEMENT" ]]; then
					BCFG2_SUPPLEMENT=${BCFG2_GROUP##supplement-}
				fi
				;;
		esac
	done < /etc/bcfg2.group
	export BCFG2_PARCEL BCFG2_DEBIAN_VERSION BCFG2_SUPPLEMENT BCFG2_GROUPS
}

# sd_booted(3) - Test whether the system is running the systemd init system
function sd_booted() {
	test -d '/run/systemd/system/'
}

function check_expiry() {
	local resource=$1
	local notBefore_epoch=$2
	local notAfter_epoch=$3
	local expires_in_caution=$4
	local expires_in_warning=$5

	local now_epoch=$(date '+%s')

	local expires_in=$(( notAfter_epoch - now_epoch ))
	local expires_in_scaled

	if [[ $now_epoch -lt $notBefore_epoch ]]; then
		fail critical "$resource not valid yet"
		problem NOT_VALID_YET "$resource"
		exit
	elif [[ $expires_in -lt 0 ]]; then
		expires_in_scaled=$(print_size_scaled -$expires_in 60)
		fail critical "$resource expired $expires_in_scaled ago"
		problem EXPIRED "$resource"
		exit
	elif [[ $expires_in -lt $expires_in_warning ]]; then
		expires_in_scaled=$(print_size_scaled $expires_in 60)
		fail warning "$resource expires in $expires_in_scaled"
		problem EXPIRY_IMMINENT "$resource"
		exit
	elif [[ $expires_in -lt $expires_in_caution ]]; then
		expires_in_scaled=$(print_size_scaled $expires_in 60)
		fail caution "$resource expires in $expires_in_scaled"
		problem EXPIRY_IMMINENT "$resource"
		exit
	else
		expires_in_scaled=$(print_size_scaled $expires_in 60)
		echodebug "$resource OK (expires in $expires_in_scaled)"
	fi
}

function update_cache_file() {
	local src=$1
	local dst="$CACHE_DIR/$SYS"

	if ! cmp --silent "$src" "$dst"; then
		cp --archive "$src" "$dst"
		chmod 444 "$dst"
	fi
}

#
# export of constants

# https://en.wikipedia.org/wiki/Unit_of_time
declare -r -i TIME_UNIT_SECOND=1
declare -r -i TIME_UNIT_MINUTE=$(( 60 * TIME_UNIT_SECOND ))
declare -r -i TIME_UNIT_HOUR=$(( 60 * TIME_UNIT_MINUTE ))
declare -r -i TIME_UNIT_DAY=$(( 24 * TIME_UNIT_HOUR ))
declare -r -i TIME_UNIT_WEEK=$(( 7 * TIME_UNIT_DAY ))
declare -r -i TIME_UNIT_FORTNIGHT=$(( 2 * TIME_UNIT_WEEK ))
declare -r -i TIME_UNIT_YEAR=$(( 365 * TIME_UNIT_DAY + \
	5 * TIME_UNIT_HOUR + 49 * TIME_UNIT_MINUTE + 12 * TIME_UNIT_SECOND )) # Gregorian year (365.2425)
declare -r -i TIME_UNIT_MONTH=$(( TIME_UNIT_YEAR / 12 ))
export TIME_UNIT_SECOND TIME_UNIT_MINUTE TIME_UNIT_HOUR TIME_UNIT_DAY TIME_UNIT_WEEK \
	TIME_UNIT_FORTNIGHT TIME_UNIT_MONTH TIME_UNIT_YEAR

# standard output is not connected to a terminal / dumb terminal
TPUT_BLACK=
TPUT_RED=
TPUT_GREEN=
TPUT_YELLOW=
TPUT_BLUE=
TPUT_MAGENTA=
TPUT_CYAN=
TPUT_WHITE=
TPUT_GREY=
TPUT_NORMAL=

# colourize the output
if [ -t 1 ]; then
	# file descriptor 1 (stdout) is opened on a terminal

	# terminfo(5)
	COLOR_BLACK=0
	COLOR_RED=1
	COLOR_GREEN=2
	COLOR_YELLOW=3
	COLOR_BLUE=4
	COLOR_MAGENTA=5
	COLOR_CYAN=6
	COLOR_WHITE=7

	ATTRIBUTE_STANDOUT='	1'
	ATTRIBUTE_UNDERLINE='	0 1'
	ATTRIBUTE_REVERSE='		0 0 1'
	ATTRIBUTE_BLINK='		0 0 0 1'
	ATTRIBUTE_DIM='			0 0 0 0 1'
	ATTRIBUTE_BOLD='		0 0 0 0 0 1'
	ATTRIBUTE_INVIS='		0 0 0 0 0 0 1'
	ATTRIBUTE_PROTECT='		0 0 0 0 0 0 0 1'
	ATTRIBUTE_ALTCHARSET='	0 0 0 0 0 0 0 0 1'

	if tput setaf >/dev/null; then
		# set foreground colour using ANSI escape
		TPUT_BLACK=$(tput setaf "$COLOR_BLACK")
		TPUT_RED=$(tput setaf "$COLOR_RED")
		TPUT_GREEN=$(tput setaf "$COLOR_GREEN")
		TPUT_YELLOW=$(tput setaf "$COLOR_YELLOW")
		TPUT_BLUE=$(tput setaf "$COLOR_BLUE")
		TPUT_MAGENTA=$(tput setaf "$COLOR_MAGENTA")
		TPUT_CYAN=$(tput setaf "$COLOR_CYAN")
		TPUT_GREY=$(tput setaf "$COLOR_WHITE")
		TPUT_WHITE=$(tput bold)
		TPUT_NORMAL=$(tput sgr0)
	elif tput sgr >/dev/null; then
		# terminal supports display attributes (but no colours)
		TPUT_BLACK=$(tput sgr "$ATTRIBUTE_INVIS")
		TPUT_RED=$(tput sgr "$ATTRIBUTE_BLINK")
		TPUT_GREEN=$(tput sgr "$ATTRIBUTE_DIM")
		TPUT_YELLOW=$(tput sgr "$ATTRIBUTE_UNDERLINE")
		TPUT_BLUE=$(tput sgr "$ATTRIBUTE_STANDOUT")
		TPUT_MAGENTA=$(tput sgr "$ATTRIBUTE_REVERSE")
		TPUT_CYAN=$(tput sgr "$ATTRIBUTE_REVERSE")
		TPUT_GREY=$(tput sgr "$ATTRIBUTE_BOLD")
		TPUT_WHITE=$(tput sgr "$ATTRIBUTE_BOLD")
		TPUT_NORMAL=$(tput sgr0)
	fi
fi

function usage() {
	cat << 'EOF'
usage:
	$ syscheck [-v|-s]
	$ syscheck status PLUGIN_NAME
	$ syscheck all|short|long|failed
	$ syscheck check|clear|enable PLUGIN_NAME
	$ syscheck ack PLUGIN_NAME "<administrator name>" "<ticket#> or <N/A>" "<reason text>"
	$ syscheck disable PLUGIN_NAME "<administrator name>" "<ticket#> or <N/A>" "<reason text>"
	$ syscheck restrict APP_NAME duration
	$ syscheck release APP_NAME
	$ syscheck query RESOURCE
EOF
	exit 2
}


if ! options=$(getopt --options vsd --longoptions verbose,silent,debug -- "$@")
then
	usage
fi

DEBUG=0
if [[ -n "${SUDO_COMMAND+xxx}" ]]; then
	DEBUG=1
fi
export DEBUG
VERBOSE=1
eval set -- "$options"
while [ $# -gt 0 ]; do
	case $1 in
		-d|--debug) VERBOSE=$((VERBOSE+2));;
		-v|--verbose) VERBOSE=$((VERBOSE+1));;
		-s|--silent) VERBOSE=$((VERBOSE-1));;

		(--) shift; break;;
		(-*) echo "$0: error - unrecognized option $1" 1>&2; exit 2;;
		(*) break;;
	esac
	shift
done
# Enure if -v and -d, only -d
if [[ $VERBOSE -gt 3 ]]; then
	VERBOSE=3
fi

LEVEL_REQUESTED=
SYS_LIMIT=
if [[ -n "${1+xxx}" ]]; then
	case ${1} in
		all|short|long|on_boot)
			LEVEL_REQUESTED="$1"
			SYS_SELECT='*'
			;;
		check)
			LEVEL_REQUESTED='all'
			SYS_SELECT="$2"
			;;
		failed)
			LEVEL_REQUESTED='all'
			SYS_SELECT=""
			for FILE in $IPC_DIR/*; do
				FILENAME=${FILE##*/}
				MODULE=${FILENAME%%.*}
				PRIORITY=${FILENAME##*.}
				if [[ "$PRIORITY" != 'ok' ]]; then
					SYS_SELECT+=" $MODULE"
				fi
			done
			if [[ -z "$SYS_SELECT" ]]; then
				echo 'no failed plugins to recheck - all reported OK'
			fi
			;;
		status)
			SYS_LIMIT="$2"
			;;
		clear)
			SYS="$2"
			if is_disabled; then
				echodebug "$SYS disabled"
				exit 1
			fi
			nats_init
			nats_plugin_start $SYS
			unknown 'administrative clear'
			nats_plugin_end $SYS
			exit 0
			;;
		enable)
			SYS="$2"
			sysenable
			exit 0
			;;
		disable|ack)
			# Allow more graceful error handling here
			set +o nounset

			# Validate that all 5 expected arguments are given (e.g. ack myplugin "Joshua Boniface" "RT12345" "This is a good reason")
			if [[ $# -lt 5 ]]; then
				usage
			fi

			MODE="$1"
			SYS="$2"
			WHO="$3"
			TICKET="$4"
			REASON="$5"
			shift;shift;shift;shift;shift
			REASON="$REASON $*"

			# Check if plugin exists
			if [[ ! -f "$SHARE_DIR/$SYS" ]]; then
				echo "Plugin $SYS not found"
				exit 1
			fi

			# Check if plugin is disable
			if is_disabled; then
				echo "Plugin $SYS is disabled"
				exit 1
			fi

			# Validate that ${TICKET} is N/A or all-numbers
			if [[ ${TICKET} != 'N/A' ]]; then
				if [[ -z "${TICKET##*[!0-9]*}" ]]; then
					echo "A ticket number must be provided."
					exit 1
				fi
			fi

			# End our graceful error handling
			set -o nounset

			# Handle the actual differences in the disable and ack modes
			case $MODE in
				disable) sysdisable;;
				ack) sysack;;
			esac
			exit 0
			;;
		restrict)
			APP="$2"
			DURATION="$3"
			DURATION=$(( DURATION ))
			APP_LOCK=$LOCK.$APP.$PPID
			if [[ -s "$APP_LOCK" ]]; then
				echo 'already restricted'
				exit 1
			else
				START=$(date --utc '+%s')
				echo -e "$PPID\t$START\t$DURATION" > "$APP_LOCK"
				exit 0
			fi
			;;
		release)
			APP="$2"
			APP_LOCK=$LOCK.$APP.$PPID
			if [[ ! -s "$APP_LOCK" ]]; then
				echo 'nothing to release'
				exit 1
			else
				read -r PPID_REC START DURATION STOP < "$APP_LOCK"
				if [[ -n "$STOP" ]]; then
					rm "$APP_LOCK"
					echo "cleared record of overrun for $PPID_REC"
					exit 1
				else
					STOP=$(date --utc '+%s')
				fi

				if [[ $(( START + DURATION )) -ge "$STOP" ]]; then
					# clean finish
					rm "$APP_LOCK"
				else
					# overrun
					echo -e "$PPID\t$START\t$DURATION\t$STOP" > "$APP_LOCK"
				fi
				exit 0
			fi
			;;
		release-all)
			LOCKDIR=$(dirname "$LOCK")
			find "$LOCKDIR" -maxdepth 1 -type f -name "$BASENAME.*" -print0 |
				xargs --null --no-run-if-empty -- rm --verbose
			exit 0
			;;
		query)
			case $2 in
				IPC_DIR) echo $IPC_DIR;;
				STATE_DIR) echo $STATE_DIR;;
				CACHE_DIR) echo $CACHE_DIR;;
			esac
			exit 0
			;;
		shutdown)
			touch $SHUTDOWN
			if [[ -s $PID_FILE ]]; then
				PID=$(<"$PID_FILE")
				kill -STOP $PID
				pkill --signal QUIT --parent $PID
				kill -QUIT $PID
				kill -CONT $PID
				rm $PID_FILE
			fi
			nats_init
			SYS='_' nats_host_status down
			exit 0
			;;
		test)
			case ${2} in
				terminal)
					echo "${TPUT_RED}critical${TPUT_NORMAL}"
					echo "${TPUT_YELLOW}warning${TPUT_NORMAL}"
					echo "${TPUT_GREY}caution${TPUT_NORMAL}"
					echo "${TPUT_GREEN}OK${TPUT_NORMAL}"
					echo "${TPUT_CYAN}acknowledged${TPUT_NORMAL}"
					echo "${TPUT_BLUE}disabled${TPUT_NORMAL}"
					echo "${TPUT_WHITE}undef${TPUT_NORMAL}"
					;;
				print_size_scaled)
					for ((i=1;i<$((2**62));i=i*7)); do
						B2=$(print_size_scaled $i 2)
						B10=$(print_size_scaled $i 10)
						B60=$(print_size_scaled $i 60)
						echo -e "$i\t$B2\t$B10\t$B60"
					done
					;;
			esac
			exit 0
			;;
		*)
			usage
			;;
	esac
fi

if [[ -d "$SHARE_DIR" ]]; then
	cd "$SHARE_DIR"
else
	echo "SHARE_DIR $SHARE_DIR missing"
	exit 1
fi

# display test results
INSTALLED_PLUGINs=(*)
declare -a ACK_PLUGINs=()
declare -a DISABLED_PLUGINs=()
declare -a PLUGINs=()
if [[ -z "$LEVEL_REQUESTED" ]]; then
	FAIL_COUNT=0
	OK_COUNT=0
	for FILE in $IPC_DIR/*.critical $IPC_DIR/*.warning $IPC_DIR/*.caution $IPC_DIR/*.ack $IPC_DIR/*.disable $IPC_DIR/*.ok; do
		if [[ -f "$FILE" ]]; then
			MOD_TIME_FULL=$(stat --format="%y" "$FILE")
			MOD_TIME_SHORT=${MOD_TIME_FULL%:*}
			DATA=$(<"$FILE")
			FILENAME=${FILE##*/}
			MODULE=${FILENAME%%.*}
			PRIORITY=${FILENAME##*.}
			PLUGINs+=("$MODULE")
			if [[ -n "$SYS_LIMIT" && "$SYS_LIMIT" != "$MODULE" ]]; then
				continue
			fi
			META="N/A"
			case $PRIORITY in
				warning)	COLOUR="$TPUT_YELLOW"; FAIL_COUNT=$(( FAIL_COUNT + 1 ));;
				critical)	COLOUR="$TPUT_RED"; FAIL_COUNT=$(( FAIL_COUNT + 1 ));;
				caution)	COLOUR="$TPUT_GREY"; FAIL_COUNT=$(( FAIL_COUNT + 1 ));;
				ack)		ACK_PLUGINs+=("$MODULE")
							COLOUR="$TPUT_CYAN"
							loadmeta
							if [[ ${META_TICKET} == 'N/A' ]]; then
								META="$META_MSG (by $META_WHO)"
							else
								META="$META_MSG (by $META_WHO ticket# ${META_TICKET})"
							fi
							;;
				disable)	DISABLED_PLUGINs+=("$MODULE")
							COLOUR="$TPUT_BLUE"
							loadmeta
							if [[ ${META_TICKET} == 'N/A' ]]; then
								META="$META_MSG (by $META_WHO)"
							else
								META="$META_MSG (by $META_WHO ticket# ${META_TICKET})"
							fi
							;;
				ok)			OK_COUNT=$(( OK_COUNT + 1 ))
							if [[ -z $SYS_LIMIT ]]; then
								continue
							else
								COLOUR="$TPUT_GREEN"
								DATA="check passed"
							fi
							;;
				*) echo "${TPUT_RED}ERROR: unknown priority {$PRIORITY} for plugin {$MODULE}${TPUT_NORMAL}"; exit 1;;
			esac
			if [[ $VERBOSE -eq 0 ]]; then
				echo -n "$MODULE.$PRIORITY "
			elif [[ $VERBOSE -eq 1 ]]; then
				# "Short" output
				echo "$MOD_TIME_SHORT [$COLOUR$PRIORITY$TPUT_NORMAL] $MODULE: $DATA"
				if [[ "$META" != "N/A" ]]; then
					# Indent past the modtime (exactly 17 chars)
					echo -e "                 > $META"
				fi
			elif [[ $VERBOSE -eq 2 ]]; then
				# "Long" output
				if [[ "$MODULE" = 'syscheck' ]]; then
					MODULE_FILE=$0
				else
					MODULE_FILE=$SHARE_DIR/$MODULE
				fi
				MODULE_VER=$(   awk -F'^#[$]version[$][\t ]+' '{if ($2){print $2}}'	"$MODULE_FILE")
				if [[ -z "$MODULE_VER" ]]; then
					MODULE_VER=$(awk -F'[=\t ]' '{if ($1=="V"){V=$2}}END{print V}'	"$MODULE_FILE")
				fi
				MODULE_TITLE=$( awk -F'^#[$]title[$][\t ]+'  '{if ($2){print $2}}'	"$MODULE_FILE")
				MODULE_CHECK=$( awk -F'^#[$]check[$][\t ]+'  '{if ($2){print $2}}'	"$MODULE_FILE")
				MODULE_REF=$(   awk -F'^#[$]ref[$][\t ]+'    '{if ($2){print $2}}'	"$MODULE_FILE")
				MODULE_AUTHOR=$(awk -F'^#[$]author[$][\t ]+' '{if ($2){print $2}}'	"$MODULE_FILE")
				MODULE_LEVEL=$( awk -F'^#?level_check[\t ]+' '{if ($2){print $2}}'	"$MODULE_FILE")
				HELP_MSG=$( awk 'BEGINFILE {if (ERRNO){print "undef";exit}}{print}' "$HELP_DIR/$MODULE" 2>/dev/null )
				if [[ ! -f "$FIX_DIR/$MODULE" ]]; then
					FIX_SPEC="undef"
				else
					FIX_SPEC=$( awk 'BEGINFILE {if (ERRNO){print "undef";exit}}{print}' "$FIX_DIR/$MODULE" 2>/dev/null )
				fi
				if [[ $FIX_SPEC != "undef" ]]; then
					HAS_FIX="Yes"
				else
					HAS_FIX="No"
				fi
				######## "0123456789"
				echo -e "${TPUT_WHITE}$MODULE_TITLE check ($MODULE) $MODULE_VER${TPUT_NORMAL}"
				echo -e "${TPUT_WHITE}Level:${TPUT_NORMAL} $MODULE_LEVEL  ${TPUT_WHITE}Author:${TPUT_NORMAL} $MODULE_AUTHOR  ${TPUT_WHITE}Sysfix:${TPUT_NORMAL} $HAS_FIX"
				echo -e "${TPUT_WHITE}Checks:${TPUT_NORMAL} $MODULE_CHECK"
				echo -e "${TPUT_WHITE}Status:${TPUT_NORMAL} $COLOUR$PRIORITY${TPUT_NORMAL} (since $MOD_TIME_FULL)"
				echo -e "${TPUT_WHITE}Output:${TPUT_NORMAL} $COLOUR$DATA${TPUT_NORMAL}"
				echo -e "${TPUT_WHITE}Detail:${TPUT_NORMAL} $HELP_MSG  ${TPUT_WHITE}References:${TPUT_NORMAL} $MODULE_REF"
				if [[ $META != "N/A" ]]; then
				echo -e "${TPUT_WHITE}Meta:${TPUT_NORMAL} $META"
				fi

				echo
			elif [[ $VERBOSE -eq 3 ]]; then
				# "Debug" output
				function print_kv {
					KEY="$1"
					VALUE="$2"
					printf '[%-10s]: %s\n' "$KEY" "$VALUE"
				}
				if [[ "$MODULE" = 'syscheck' ]]; then
					MODULE_FILE=$0
				else
					MODULE_FILE=$SHARE_DIR/$MODULE
				fi
				MODULE_VER=$(   awk -F'^#[$]version[$][\t ]+' '{if ($2){print $2}}'	"$MODULE_FILE")
				if [[ -z "$MODULE_VER" ]]; then
					MODULE_VER=$(awk -F'[=\t ]' '{if ($1=="V"){V=$2}}END{print V}'	"$MODULE_FILE")
				fi
				MODULE_TITLE=$( awk -F'^#[$]title[$][\t ]+'  '{if ($2){print $2}}'	"$MODULE_FILE")
				MODULE_CHECK=$( awk -F'^#[$]check[$][\t ]+'  '{if ($2){print $2}}'	"$MODULE_FILE")
				MODULE_REF=$(   awk -F'^#[$]ref[$][\t ]+'    '{if ($2){print $2}}'	"$MODULE_FILE")
				MODULE_AUTHOR=$(awk -F'^#[$]author[$][\t ]+' '{if ($2){print $2}}'	"$MODULE_FILE")
				MODULE_LEVEL=$( awk -F'^#?level_check[\t ]+' '{if ($2){print $2}}'	"$MODULE_FILE")
				HELP_MSG=$( awk 'BEGINFILE {if (ERRNO){print "undef";exit}}{print}' "$HELP_DIR/$MODULE" 2>/dev/null )
				if [[ ! -f "$FIX_DIR/$MODULE" ]]; then
					FIX_SPEC="undef"
				else
					FIX_SPEC=$( awk 'BEGINFILE {if (ERRNO){print "undef";exit}}{print}' "$FIX_DIR/$MODULE" 2>/dev/null )
				fi
				######## "0123456789"
				print_kv "sequence"	"$FAIL_COUNT"
				print_kv "name"		"$MODULE"
				print_kv "version"	"$MODULE_VER"
				print_kv "author"	"$MODULE_AUTHOR"
				print_kv "title"	"$MODULE_TITLE"
				print_kv "checks for"	"$MODULE_CHECK"
				print_kv "reference"	"$MODULE_REF"
				print_kv "level"	"$MODULE_LEVEL"
				print_kv "severity"	"$PRIORITY"
				print_kv "timestamp"	"$MOD_TIME_FULL"
				print_kv "error text"	"$DATA"
				print_kv "meta msg"	"$META"
				print_kv "help text"	"$HELP_MSG"
				print_kv "fix helper"	"$FIX_SPEC"
				echo
			fi
		fi
	done
	if [[ $VERBOSE -eq 0 && ( $FAIL_COUNT -gt 0 || -n "$SYS_LIMIT" ) ]]; then
		echo
	fi
	if [[ ${#PLUGINs[*]} -gt 0 && $FAIL_COUNT -eq 0 && ${#ACK_PLUGINs[*]} -eq 0 && ${#DISABLED_PLUGINs[*]} -eq 0 && $VERBOSE -gt 0 && -z "$SYS_LIMIT" ]]; then
		echo -n "syscheck health status: "
		echo -n "$OK_COUNT/${#INSTALLED_PLUGINs[*]} plugin(s) ${TPUT_GREEN}OK${TPUT_NORMAL}"
		echo
	fi
	exit
fi

# run tests
(
	if [[ -n ${SUDO_COMMAND+xxx} ]]; then
		# interactive
		if ! flock --exclusive --nonblock 200; then
			PID=$(<"$PID_FILE")
			echo "plugin execution locked by PID $PID" 1>&2
			exit 1
		fi
	else
		# scheduled job
		if ! flock --exclusive --timeout $LOCK_TIMEOUT 200; then
			echo "unsuccessfully waited $LOCK_TIMEOUT seconds for plugin execution lock" 1>&2
			exit 1
		fi
	fi
	if [[ -z "${BASHPID+xxx}" ]]; then
		BASHPID=$$ #not available in bash 3 (SOE4.0)
		# shutdown handling may not work with $$ in edge cases
	fi
	echo $BASHPID > "$PID_FILE"
	echodebug "$BASENAME version $VERSION PID $BASHPID"
	nats_init
	SYS='_' nats_host_status up

	source_os_release
	bcfg2_groups_parse

	export LEVEL_REQUESTED
	LEVEL_EXCEPTION=$(mktemp)
	for SYS in $SYS_SELECT; do
		if is_disabled; then
			continue
		fi
		CHECK_START_TIME=$SECONDS
		mime_type=$(file --brief --mime-type "$SYS")
		case $mime_type in
			text/x-shellscript) interpreter='.';; # bash
			text/x-perl) interpreter='perl';;
			*) interpreter='false';;
		esac
		export CMD_FILE="$META_DIR/$SYS.cmd"
		: > $LEVEL_EXCEPTION
		if ( $interpreter "./$SYS" ); then
			if [[ -s "$CMD_FILE" ]]; then
				. "$CMD_FILE"
				rm "$CMD_FILE"
			fi
			if [[ ! -s $LEVEL_EXCEPTION ]]; then
				nats_plugin_end $SYS
			fi
		else
			DATE=$(date)
			# shellcheck disable=SC2097,SC2098
			SYS='syscheck' fail warning "failed to run $SYS plugin at $DATE"
			# shellcheck disable=SC2097,SC2098
			SYS='syscheck' helpmsg "report the logic/design fault to the $SYS plugin maintainer"
		fi
	done
	rm "$LEVEL_EXCEPTION" "$PID_FILE"
) 200>"$LOCK"

# all run successfully
touch "$IPC_DIR"
