#!/bin/bash
# vim: cindent:shiftwidth=4:tabstop=4:smarttab:textwidth=100

#set -o posix
set -o errexit
set -o pipefail
set -o nounset
#set -o xtrace

#$title$ File integrity monitoring (FIM)
#$check$ missing or mismatches checksums of OS packaged files
#$ref$ debsums, /var/lib/dpkg/info/*.md5sums
#$author$ Rafal Rzeczkowski
#$version$ 0.8.50

level_check long

#CHANGELOG
#0.5.0	create an initial version
#0.6.0	add file exclusions specific to the Xen profile
#0.6.1	add file exclusions for a router
#0.7.0	respond to NATS keepalive messages while debsums is running
#0.7.1	add file exclusions for the SOE4 base profile
#0.7.2	add file exclusions specific to the Prefix profile
#0.7.3	add file exclusions for the SOE4 base profile
#0.7.4	add file exclusions specific to the LVS profile
#0.7.5	add file exclusions specific to the RPKI profile
#0.7.6	add file exclusions specific to the mirror profile
#0.7.7	add file exclusions specific to the mail profile
#0.7.8	add file exclusions specific to the master profile
#0.7.9	add file exclusions specific to Apache2
#0.8.0	exclude customized configuration files automatically
#0.8.1	add file exclusions specific to the backup profile
#0.8.2	add file exclusion specific to the backup profile
#0.8.3	add file exclusion specific to PVC deploys
#0.8.4	add file exclusion for upgraded grub-pc package
#0.8.5	add file exclusion specific to the ns profile
#0.8.6	activate checks for the VPN profile
#0.8.7	add file exclusions for the SOE4 base profile
#0.8.8	add file exclusion specific to the rt profile
#0.8.9	add file exclusion specific to the base profile
#0.8.10	add file exclusion for /etc/issue for OVAs
#0.8.11	exclude customized proc_pri Munin plugin
#0.8.12	exclude customized threads Munin plugin
#0.8.13	exclude customized spf_engine URL
#0.8.14	exclude customized /etc/init.d/supervisor script
#0.8.15	exclude customized snmpd service script/config
#0.8.16	exclude customized cpuspeed Munin plugin
#0.8.17	add file exclusion specific to the prov profile
#0.8.18	exclude corrected /etc/init.d/alsa-utils script
#0.8.19	exclude customized TMPFS configuration
#0.8.20	exclude SOE5 base configuration files
#0.8.21	exclude bcfg2-managed SOE4.2 Munin plugins
#0.8.22	add file exclusion specific to the kb profile
#0.8.23	revert customization of usb.ids to package default
#0.8.24	exclude isc-dhcp-client hook files on SOE4.2
#0.8.25	exclude AppArmor abstraction patches
#0.8.26	exclude Z shell configuration files
#0.8.27	exclude patch for Python 3 libapt-pkg
#0.8.28	exclude patched /etc/init.d/bootlogs script
#0.8.29	exclude System-V init script for TAYGA
#0.8.30	exclude systemd root filesystem fsck unit
#0.8.31	exclude systemd gpt-auto-generator patch
#0.8.32	exclude GNSS-related services
#0.8.33	exclude tcpdump AppArmor profile
#0.8.34	exclude fan control service
#0.8.35	exclude PHC service definition
#0.8.36	exclude systemd BIND9 service customization
#0.8.37	exclude systemd update-initramfs autofs patch
#0.8.38	exclude procps hack for sysctl module pre-load
#0.8.39	exclude D4 patch reversion for environment load via PAM
#0.8.40	exclude fail2ban configuration patch for systemd journal
#0.8.41	exclude Vim syntax file patch for BIND 9
#0.8.42	exclude D4 patch reversion for environment load via PAM
#0.8.43	remove systemd BIND9 service customization
#0.8.44	exclude open-vm-tools tools.conf config
#0.8.45	exclude Routinator 3000 configs
#0.8.46	exclude updated Net::Ping on SOE4.3 LSN
#0.8.47	exclude AppArmor default abstractions (python)
#0.8.48	exclude bcfg2-managed SOE5 Munin plugins
#0.8.49	exclude cloudinit config for SOE5
#0.8.50	exclude systemd journal stream transport configuration
#0.8.50	exclude nftables.conf
#0.8.51	exclude ldapscripts.conf, ldapscripts.passwd
#0.8.52	exclude prometheus exporter for mysqld and postgres
#0.8.53	exclude prometheus exporter for bind

declare -r -a PROBLEMS=(CHECKSUMS_MISSING BAD_EXIT_STATUS INTEGRITY_VIOLATION)
test -n ${#PROBLEMS[@]}

declare -r -a BCFG2_TRACKED=(
/etc/apparmor.d/abstractions/consoles
/etc/apparmor.d/abstractions/nameservice
/etc/apparmor.d/abstractions/python
/etc/default/bcfg2
/etc/bind/bind.keys
/etc/bind/named.conf
/etc/apparmor.d/usr.sbin.named
/etc/bind/named.conf.options
/etc/crontab
/usr/share/perl5/Net/Server.pm
/etc/security/limits.conf
/etc/apparmor.d/usr.bin.man
/usr/share/perl5/Munin/Node/OS.pm
/etc/munin/munin-node.conf
/usr/share/munin/plugins/bind9
/usr/share/munin/plugins/bind9_rndc
/usr/share/munin/plugins/cpu
/usr/share/munin/plugins/diskstats
/usr/share/munin/plugins/entropy
/usr/share/munin/plugins/fw_conntrack
/usr/share/munin/plugins/if_
/usr/share/munin/plugins/interrupts
/usr/share/munin/plugins/irqstats
/usr/share/munin/plugins/nfs_client
/usr/share/munin/plugins/nfsd
/usr/share/munin/plugins/proc_pri
/usr/share/munin/plugins/sensors_
/usr/share/munin/plugins/slapd_bdb_cache_
/usr/share/munin/plugins/threads
/etc/services
/etc/ntp.conf
/etc/apparmor.d/usr.sbin.ntpd
/etc/ssh/ssh_config
/etc/ssl/openssl.cnf
/etc/init.d/rsync
/etc/snmp/snmpd.conf
/etc/sudoers
/etc/init.d/sysfsutils
/etc/syslog-ng/syslog-ng.conf
/etc/default/syslog-ng
/etc/tayga.conf
/etc/default/tayga
/etc/apparmor.d/usr.sbin.tcpdump
/etc/ulogd.conf
/etc/vim/vimrc
/etc/init.d/binfmt-support
/usr/share/munin/plugins/proc_pri
/usr/share/munin/plugins/threads
/etc/default/tmpfs
/etc/init.d/tayga
/etc/nftables.conf
/etc/ldapscripts/ldapscripts.conf
/etc/ldapscripts/ldapscripts.passwd

/etc/default/acpid
/etc/init.d/cryptdisks
/usr/sbin/drbd-overview
/etc/drbd.d/global_common.conf
/etc/default/ipmievd
/etc/init.d/ladvd
/etc/lvm/lvm.conf
/etc/grub.d/20_memtest86+
/usr/share/munin/plugins/ipmi_
/etc/init.d/radvd
/etc/screenrc
/etc/watchdog.conf
/etc/default/grub
/etc/default/grub.d/xen.cfg
/etc/init.d/xen
/etc/xen/scripts/vif-common.sh
/etc/init.d/xendomains
/etc/default/xend
/etc/xen/xl.conf

/etc/dhcpcd.conf
/etc/init.d/dhcpcd
/etc/hdparm.conf
/etc/linuxptp/ptp4l.conf
/etc/default/ntp
/etc/default/rsync
/etc/init.d/samba-ad-dc

/usr/share/arno-iptables-firewall/environment
/etc/arno-iptables-firewall/custom-rules
/etc/arno-iptables-firewall/firewall.conf
/etc/bind/named.conf.default-zones
/etc/dnsmasq.conf
/etc/default/gpsd
/usr/share/perl5/Net/GPSD3.pm
/usr/share/munin/plugins/bind9_rndc
/etc/ntpsec/ntp.conf
/etc/default/ntpsec
/etc/ser2net.conf
/usr/share/vim/vim81/syntax/bindzone.vim
/etc/default/watchdog
/etc/wide-dhcpv6/dhcp6c.conf

/etc/default/auditd
/etc/ImageMagick-6/policy.xml
/etc/init.d/lvm2
/etc/rsnapshot.conf
/etc/cron.d/rsnapshot

/etc/amavis/conf.d/01-debian
/etc/amavis/conf.d/05-domain_id
/etc/amavis/conf.d/05-node_id
/etc/amavis/conf.d/15-av_scanners
/etc/amavis/conf.d/50-user
/etc/apparmor.d/usr.bin.freshclam
/etc/apparmor.d/usr.sbin.clamd
/etc/cron.d/clamav-unofficial-sigs
/etc/default/amavisd-snmp-subagent
/etc/default/amavis-mc
/etc/default/spamassassin
/etc/fail2ban/filter.d/postfix-sasl.conf
/etc/fail2ban/filter.d/sshd.conf
/etc/init.d/opendmarc
/etc/init.d/sqlgrey
/etc/opendkim.conf
/etc/opendmarc.conf
/etc/postfix-policyd-spf-python/policyd-spf.conf
/etc/sqlgrey/sqlgrey.conf
/usr/sbin/clamav-unofficial-sigs
/usr/sbin/sqlgrey
/usr/share/munin/plugins/postfix_mailqueue
/usr/lib/python3/dist-packages/spf_engine/__init__.py

/etc/init.d/ps-watcher
/etc/default/ps-watcher

/etc/default/keepalived
/etc/init.d/keepalived
/usr/sbin/ldirectord
/etc/init.d/ldirectord
/usr/share/perl5/Mail/POP3Client.pm
/usr/share/perl5/Net/IMAP/Simple.pm

/etc/rpki-rtr-server/application.properties
/etc/rpki-validator/application.properties

/etc/apache2/ports.conf
/etc/apache2/mods-available/mpm_event.conf
/etc/bind/db.root
/etc/vsftpd.conf

/etc/courier/authdaemonrc
/etc/init.d/courier-authdaemon
/etc/courier/authldaprc
/etc/courier/imapd
/etc/courier/imapd-ssl
/etc/courier/pop3d
/etc/courier/pop3d-ssl
/etc/maildroprc
/etc/default/opendmarc
/etc/default/saslauthd

/etc/init.d/bcfg2-server

/usr/share/backuppc/lib/BackupPC/Xfer/Tar.pm
/etc/backuppc/apache.conf
/etc/backuppc/config.pl
/etc/init.d/tftpd-hpa
/etc/backuppc/hosts

/etc/dhcp/dhclient.conf

/etc/default/slapd

/etc/init.d/request-tracker4
/etc/apparmor.d/usr.sbin.mysqld
/etc/request-tracker4/apache2-modperl2.conf

/etc/issue

/etc/init.d/supervisor

/etc/init.d/snmpd
/etc/default/snmpd

/usr/share/munin/plugins/cpuspeed

/etc/init.d/alsa-utils
/etc/init.d/bootlogs
/etc/init.d/procps

# prov SOE4.2
/etc/default/dhcpstats
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/warnings.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/imp.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/tokenize.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/base64.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/_bootlocale.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/_collections_abc.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/posixpath.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/tempfile.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/hashlib.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/abc.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/copyreg.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/site.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/linecache.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/types.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/re.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/token.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/ntpath.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/tarfile.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/os.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/shutil.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/io.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/codecs.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/_weakrefset.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/locale.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/fnmatch.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/sre_constants.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/__future__.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/operator.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/stat.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/random.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/keyword.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/sre_compile.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/functools.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/bisect.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/genericpath.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/reprlib.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/heapq.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/weakref.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/hmac.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/copy.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/struct.cpython-34.pyc
/opt/clearcable/dhcpstats/venv/lib/python3.4/__pycache__/sre_parse.cpython-34.pyc
/usr/share/munin/plugins/interrupts
/usr/share/munin/plugins/fail2ban
/etc/perl/sitecustomize.pl
/etc/snmp/snmp.conf
/etc/default/stun
/usr/bin/unattended-upgrade
/etc/dhcp/dhcpd.conf
/usr/share/vim/vim74/syntax/bindzone.vim
/etc/dhcp/dhclient-enter-hooks.d/debug
/etc/dhcp/dhclient-exit-hooks.d/debug
/etc/dhcp/dhclient-exit-hooks.d/rfc3442-classless-routes

# SOE5 base
/etc/systemd/system.conf
/etc/systemd/resolved.conf
/etc/systemd/timesyncd.conf
/etc/systemd/zram-generator.conf
/usr/lib/python3/dist-packages/apt/cache.py
/lib/systemd/system/systemd-fsck-root.service
/lib/systemd/system-generators/systemd-gpt-auto-generator
/etc/apparmor.d/usr.bin.tcpdump
/usr/sbin/update-initramfs
/etc/pam.d/sshd
/etc/pam.d/login
/etc/pam.d/su
/etc/fail2ban/fail2ban.conf
/etc/cloud/cloud.cfg

# KB
/etc/apache2/sites-available/000-default.conf

# Z shell
/etc/zsh/zprofile
/etc/zsh/zshenv

# GNSS
/etc/apparmor.d/usr.sbin.gpsd
/lib/systemd/system/gpsd.service
/lib/systemd/system/ntpsec.service
/lib/systemd/system/ser2net.service

# hardware
/lib/systemd/system/thinkfan.service
/lib/systemd/system/phc2sys@.service

# DNS
/usr/share/vim/vim90/syntax/bindzone.vim
/etc/init.d/bind9

# OVA open-vm-tools
/etc/vmware-tools/tools.conf

# RPKI (SOE5)
/etc/caddy/Caddyfile
/etc/routinator/routinator.conf

# LSN (SOE4.3)
/usr/share/perl/5.28.1/Net/Ping.pm

# systemd journal stream transport
/etc/systemd/journal-upload.conf
/etc/systemd/journal-remote.conf

# Apache RT#55577 (SOE 4.3)
/etc/apache2/conf-available/security.conf

# Prometheus configuration
/etc/default/prometheus-node-exporter
/etc/default/prometheus-nats-ambassador
/etc/default/prometheus-fping-exporter
/etc/default/prometheus-blackbox-exporter
/etc/default/prometheus-bind-exporter
/etc/default/prometheus-alertmanager
/etc/default/prometheus-mysqld-exporter
/etc/default/prometheus-postgres-exporter
)

mapfile -t CHECKSUMS_MISSING < <(debsums --list-missing)
if [[ ${#CHECKSUMS_MISSING[@]} -eq 0 ]]; then
	echodebug 'all Debian package checksums are present'
else
	CHECKSUM_S=$(plural_text ${#CHECKSUMS_MISSING[@]} checksum)
	fail caution "${#CHECKSUMS_MISSING[@]} package $CHECKSUM_S missing: ${CHECKSUMS_MISSING[*]}"
	helpmsg 'reinstall these packages to download/regenerate checksums'
	problem CHECKSUMS_MISSING "$(printf '%s;' "${CHECKSUMS_MISSING[@]}")"
	exit
fi

case "$BCFG2_PARCEL" in
	'')
		echodebug 'starting checksum validation for SOE base'
		;;
	lsn|xen|vdi|prefix|lvs|rpki|mirror|mail|master|backup|ns|vpn|rt|pad|prov|dns|kb)
		echodebug "starting checksum validation for SOE parcel $BCFG2_PARCEL"
		;;
	*)
		unknown "checksum validation skipped on unsupported parcel type $BCFG2_PARCEL"
		exit
		;;
esac

(
cd /
while sleep 30; do
	nats_keepalive_validate
done
)&
KEEPALIVE_LOOP=$!

TEMP_FILE=$(mktemp)
if nice ionice -c3 debsums --silent --all 2>"$TEMP_FILE"; then
	# 0=>success
	EXIT_STATUS=$?
	kill $KEEPALIVE_LOOP
	fail caution "unexpected debsums exit status $EXIT_STATUS"
	problem BAD_EXIT_STATUS $EXIT_STATUS
	rm "$TEMP_FILE"
	exit
else
	# 2=>Changed or missing package files, or checksum mismatch on an archive.
	EXIT_STATUS=$?
	kill $KEEPALIVE_LOOP
	if [[ $EXIT_STATUS -ne 2 ]]; then
		fail caution "unexpected debsums exit status $EXIT_STATUS"
		problem BAD_EXIT_STATUS $EXIT_STATUS
		rm "$TEMP_FILE"
		exit
	fi
fi
DEBSUMS_OUTPUT=$(<"$TEMP_FILE")
rm "$TEMP_FILE"

declare -i EXCLUDED_COUNT=0
#debsums: changed file /etc/ulogd.conf (from ulogd2 package)
while read -r DEBSUMS_KEY PROBLEM FILE_KEY FILENAME _; do
	if [[ "$DEBSUMS_KEY" = 'debsums:' && "$FILE_KEY" = 'file' ]]; then
		:
	else
		exit 1
	fi

	if [[ "$PROBLEM" = 'missing' ]]; then
		continue
	fi

	if [[ "$FILENAME" =~ ^/etc/ && -s "$FILENAME".orig ]]; then
		# customized configuration file
		EXCLUDED_COUNT=$((EXCLUDED_COUNT+1))
		continue
	fi

	if is_element_of "$FILENAME" "${BCFG2_TRACKED[@]}"; then
		EXCLUDED_COUNT=$((EXCLUDED_COUNT+1))
		continue
	else
		fail warning "$PROBLEM file $FILENAME"
		problem INTEGRITY_VIOLATION "$FILENAME"
		exit
	fi
done <<< "$DEBSUMS_OUTPUT"

echodebug "all Debian file checksums match ($EXCLUDED_COUNT excluded)"
ok
