#!/usr/bin/sh
# vim: cindent:shiftwidth=4:tabstop=4:smarttab:textwidth=100

# [security2rtbh]
# Remotely Triggered Black Hole (RTBH)
# Security Blacklist Importer
#
# - downloads security-related IP blacklists via HTTPS and DNS
# - imports updates into the FRR SECURITY table

#CHANGELOG
#0.50	start development based on bogon2rtbh
#0.51	add Spamhaus Project - Don't Route Or Peer lists
#0.52	remove unmaintained CISA resources
#0.6.0	replace Cisco Talos with AlienVault + Emerging Threats
#0.6.1	add DShield.org (SANS Internet Storm Center) feed
#0.6.2	remove AlienVault feed
#0.7.0	update Spamhaus DROP lists to JSON format
#0.8.0	restore Cisco Talos list (Snort Rules)

set -o errexit
#set -o pipefail
set -o nounset
#set -o xtrace

RTBH_TYPE='SECURITY'
WORK_DIR='/var/cache/rtbh'
CTL_PROG='rtbh-ctl'

# Regular Expressions Cookbook (O'Reilly Media)
# Matching IPv4 Addresses
# https://www.oreilly.com/library/view/regular-expressions-cookbook/9780596802837/ch07s16.html
REGEX_IPV4='^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)[.]){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'

# AlienVault - Open Threat Exchange
# AT&T Cybersecurity
# https://cybersecurity.att.com/open-threat-exchange
# NO LONGER MAINTAINED as of 2021-04
ALIENVAULT_URL='https://reputation.alienvault.com/reputation.data'
ALIENVAULT_FILE=${ALIENVAULT_URL##*/}

# Emerging Threats
# Proofpoint
# https://www.proofpoint.com/us
EMERGINGTHREATS_URL='https://rules.emergingthreats.net/blockrules/compromised-ips.txt'
EMERGINGTHREATS_FILE=${EMERGINGTHREATS_URL##*/}

# DShield.org
# SANS Internet Storm Center
# https://dshield.org/
DSHIELD_URL='https://feeds.dshield.org/block.txt'
DSHIELD_FILE=${DSHIELD_URL##*/}

# Talos Reputation Center
# https://talosintelligence.com/reputation
# =
# Snort Rules and IDS Software Download
# https://www.snort.org/downloads
TALOS_URL='https://snort.org/downloads/ip-block-list'
TALOS_FILE=${TALOS_URL##*/}

# DROP - Don't Route Or Peer
# The Spamhaus Project
# https://www.spamhaus.org/drop/
SPAMHAUS_DROP_BASE_URL='https://www.spamhaus.org/drop'
SPAMHAUS_DROP_FILE='drop_v4.json'
SPAMHAUS_DROPv6_FILE='drop_v6.json'
SPAMHAUS_ASN_DROP_FILE='asndrop.json'
SPAMHAUS_DROP_URL="$SPAMHAUS_DROP_BASE_URL/$SPAMHAUS_DROP_FILE"
SPAMHAUS_DROPv6_URL="$SPAMHAUS_DROP_BASE_URL/$SPAMHAUS_DROPv6_FILE"
SPAMHAUS_ASN_DROP_URL="$SPAMHAUS_DROP_BASE_URL/$SPAMHAUS_ASN_DROP_FILE"

# Clearcable internally maintained lists
CCN_IPV4_RR='ipv4.rtbh.clearcable.net.'
CCN_IPV6_RR='ipv6.rtbh.clearcable.net.'

DELAY_RANDOM_MAX=240
RUNTIME_MAX=900

curl_timestamping() {
	URL=$1
	FILE=${URL##*/}

	BYTES_DL=$(curl --silent --output "$FILE" --write-out '%{size_download}' \
		--location --remote-time --time-cond "$FILE" \
		--url "$URL")
}

cd $WORK_DIR # expected to fail when rtbh-init not used yet

# delay only when running under CRON
if [ -n "${MAILTO+xxx}" ]; then
	DELAY=$(( RANDOM % DELAY_RANDOM_MAX ))
	echo "desynchronization adjustment of $DELAY second(s)"
	sleep $DELAY
fi

# cleanup obsolete files
rm --force ip-filter.blf TA-17-164A_csv.csv TA-17-318A-IOCs.csv TA18-149A.CSV \
	$ALIENVAULT_FILE \
	drop.txt dropv6.txt edrop.txt

syscheck restrict security2rtbh $RUNTIME_MAX

# aggregate file for all feeds: validated and formatted
LIST_HTTP=$(mktemp)

#[block.txt]
#Columns (tab delimited):
#(1) start of netblock
#(2) end of netblock
#(3) subnet (/24 for class C)
#(4) number of targets scanned
#(5) name of Network
#(6) Country
#(7) contact email address
#Start	End	Netmask	Attacks	Name	Country	email
#194.147.140.0	194.147.140.255	24	11546	INT-NETWORK	SC	abuse@ipvolume.net
curl_timestamping "$DSHIELD_URL"
awk --assign REGEX_IPV4="$REGEX_IPV4" '
function errprint(msg) {
	print "ERROR: " msg > "/dev/stderr"
}
BEGIN {
	FS="\t"
}
{
	Start=$1
	End=$2
	Netmask=$3
	Attacks=$4
	Name=$5
	Country=$6
	email=$7	#appears to be optional
	
	if ($0~/^[[:space:]]*#/)
		next
	else if ($1 == "Start")
		next
	else if ($8) {
		errprint("input file contains too many fields:" $8)
		exit 1
	}
	else if (Start!~REGEX_IPV4) {
		errprint("input file first field contains something other than an IPv4 address: " Start)
		exit 1
	}
	else if (Netmask!~/^[0-9]{1,2}$/) {
		errprint("input file Netmask field contains something other than an integer: " Netmask)
		exit 1
	}
	else
		print Start "/" Netmask
}' $DSHIELD_FILE >> $LIST_HTTP

#[compromised-ips.txt]
#101.108.160.69
#101.108.184.102
#101.108.218.250
#101.108.61.226
#101.109.149.160
curl_timestamping "$EMERGINGTHREATS_URL"
awk --assign REGEX_IPV4="$REGEX_IPV4" '
function errprint(msg) {
	print "ERROR: " msg > "/dev/stderr"
}
{
	ip=$1

	if ($2) {
		errprint("input file contains too many fields")
		exit 1
	}
	else if (ip!~REGEX_IPV4) {
		errprint("input file first field contains something other than an IPv4 address: " ip)
		exit 1
	}
	else
		print $1 "/32"
}' $EMERGINGTHREATS_FILE >> $LIST_HTTP

# [drop_v4.json]
# {"type":"metadata","timestamp":1716310745,"size":84389,"records":1383,"copyright":"(c) 2024 The Spamhaus Project SLU","terms":"https://www.spamhaus.org/drop/terms/"}
# {"cidr":"223.254.0.0/16","sblid":"SBL212803","rir":"apnic"}
curl_timestamping $SPAMHAUS_DROP_URL
curl_timestamping $SPAMHAUS_DROPv6_URL
jq --raw-output 'select(.rir) | .cidr' $SPAMHAUS_DROP_FILE $SPAMHAUS_DROPv6_FILE >> $LIST_HTTP

# [TALOS] - CURRENTLY UNAVAILABLE
# 193.120.55.242
# 222.179.104.186
# 174.139.83.11
#curl_timestamping $TALOS_URL
#awk --assign REGEX_IPV4="$REGEX_IPV4" -- '
#function errprint(msg) {
#	print "ERROR: " msg > "/dev/stderr"
#}
#{
#	if (!$1)
#		next
#
#	ip_address=$1
#
#	if ($2) {
#		errprint("input file contains too many fields")
#		exit 1
#	}
#	else if (ip_address !~ REGEX_IPV4) {
#		errprint("input file contains something other than an IPv4 address: {" ip_address "}")
#		exit 1
#	}
#	else
#		print ip_address "/32"
#}' "$TALOS_FILE" >> $LIST_HTTP


CCN_IPV4_NETBLOCKs=$(dig +short $CCN_IPV4_RR txt|awk '{print substr(substr($0,1,length($0)-1),2)}')
CCN_IPV6_NETBLOCKs=$(dig +short $CCN_IPV6_RR txt|awk '{print substr(substr($0,1,length($0)-1),2)}')
for CCN_NETBLOCK in $CCN_IPV4_NETBLOCKs $CCN_IPV6_NETBLOCKs; do
	NETBLOCK_CLEAN=$(netmask --cidr $CCN_NETBLOCK)
	echo $NETBLOCK_CLEAN >> $LIST_HTTP
done

LIST_BGPD=$(mktemp)
$CTL_PROG --table $RTBH_TYPE --list > $LIST_BGPD

if ADD=$(grep --fixed-strings --line-regexp --invert-match --file=$LIST_BGPD $LIST_HTTP); then
	for BOGON in $(echo $ADD); do
		$CTL_PROG --verbose --volatile --table $RTBH_TYPE --append $BOGON
	done
	$CTL_PROG --verbose --commit
fi

if DEL=$(grep --fixed-strings --line-regexp --invert-match --file=$LIST_HTTP $LIST_BGPD); then
	for BOGON in $(echo $DEL); do
		$CTL_PROG --verbose --volatile --table $RTBH_TYPE --delete $BOGON
	done
	$CTL_PROG --verbose --commit
fi

rm $LIST_HTTP $LIST_BGPD

syscheck release security2rtbh
