Skip to content

Commit

Permalink
️️⚡️ SYNC ⚡️ 2024/12/25 19:08
Browse files Browse the repository at this point in the history
  • Loading branch information
github-actions committed Dec 25, 2024
0 parents commit 5355c13
Show file tree
Hide file tree
Showing 1,379 changed files with 33,355,710 additions and 0 deletions.
15 changes: 15 additions & 0 deletions .cron/Justfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
root := `git rev-parse --show-toplevel`

default: cron aggregate stats hallofshame

cron:
pstats {{root}}/.cron/jobs/abuseipdb/cron

aggregate:
pstats {{root}}/.cron/jobs/abuseipdb/aggregate

stats:
pstats {{root}}/.cron/jobs/abuseipdb/stats

hallofshame: stats
pstats {{root}}/.cron/jobs/abuseipdb/hallofshame
107 changes: 107 additions & 0 deletions .cron/jobs/abuseipdb/aggregate
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
#!/usr/bin/env bash

set -e

# Setup
cd "$(dirname $0)"
GIT_ROOT=$(git rev-parse --show-toplevel)

. $GIT_ROOT/.cron/scripts/ciutil

DB_PATH=$GIT_ROOT/db
README_PATH=$GIT_ROOT/README.md

DATE=$(date +%F)
DATE_DIR=$DB_PATH/$DATE
LATEST="$DATE_DIR/$DATE.ipv4"

aggregate() {
local DAYS=$1
local OUTPUT=$2
cd $DB_PATH
___
echo "ℹ $OUTPUT"; echo

PATTERN=$(daterange $DAYS --separator '|')
fd -tf "(${PATTERN}).ipv4$" -x bash -c "printf '%s - ' '{/}'; wc -l < {}" | sort -V | column -t

fd -tf "(${PATTERN}.ipv4)" -x cat | grepip | \
iprange - -1 --except $GIT_ROOT/.cron/jobs/abuseipdb/bogons.ipv4 >| $GIT_ROOT/$OUTPUT.tmp


TS=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
cat <<EOF >| $GIT_ROOT/$OUTPUT
#
# Aggregated Blocklist for AbuseIPDB: A list of the most reported IP addresses.
#
# Last updated: $TS
# Confidence level: ~100%
# Filename: $OUTPUT
# Number of ips: $(wc -l < $GIT_ROOT/$OUTPUT.tmp)
#
# Source: https://github.com/borestad/blocklist-abuseipdb
# Stats: https://github.com/borestad/blocklist-abuseipdb/tree/main/stats
# Credits 1: https://www.abuseipdb.com - please support them!
# Credits 2: https://ipinfo.io - The Trusted Source For IP Address Data
#
EOF

cat $GIT_ROOT/$OUTPUT.tmp >> $GIT_ROOT/$OUTPUT
echo
echo "Total: (`wc -l < $GIT_ROOT/$OUTPUT.tmp` ip)"

rm -f $GIT_ROOT/$OUTPUT.tmp
}


decorate-with-asn-info() {
___
echo "✨ Decorate ips with ASN info"
cd $GIT_ROOT && \
fd -tf '.ipv4$' --max-depth=1 --min-depth=1 -x \
bash -c "cat {} | .cron/scripts/ip2ipinfo.ts >| {}.tmp && mv {}.tmp {}" || true
}

update-footer() {
___
echo "✨ Update footer"

# Delete everything below placeholder
sed -i '/ABUSEIPDB-STATS-PLACEHOLDER/q' $README_PATH

update=$(date -u '+%Y-%m-%d - %H:%M:%S')
echo "Last check: \`$update\` (UTC)" >> $README_PATH
echo '```' >> $README_PATH

cd $GIT_ROOT

fd 'abuseipdb-s100.*.ipv4$' . --min-depth 1 --max-depth 1 -x bash -c 'printf "❯ %s" "{.}"; echo " ($(wc -l < {}) ips)"' | \
sort -V \
>> $README_PATH

echo '```' >> $README_PATH
}

fd '\.ipv4$' $DB_PATH -x cat | \
grepip | \
iprange - -1 --except $GIT_ROOT/.cron/jobs/abuseipdb/bogons.ipv4 | \
sponge $GIT_ROOT/abuseipdb-s100-all.ipv4 &

# c = confidence
aggregate 2 "abuseipdb-s100-1d.ipv4" # Compensate +24h to ensure we have a full day of data
aggregate 3 "abuseipdb-s100-3d.ipv4"
aggregate 7 "abuseipdb-s100-7d.ipv4"
aggregate 14 "abuseipdb-s100-14d.ipv4"
aggregate 30 "abuseipdb-s100-30d.ipv4"
aggregate 60 "abuseipdb-s100-60d.ipv4"
aggregate 90 "abuseipdb-s100-90d.ipv4"
aggregate 120 "abuseipdb-s100-120d.ipv4"
aggregate 180 "abuseipdb-s100-180d.ipv4"
aggregate 365 "abuseipdb-s100-365d.ipv4"

wait

echo
update-footer

decorate-with-asn-info
12 changes: 12 additions & 0 deletions .cron/jobs/abuseipdb/asn
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/usr/bin/env bash

set -e
export LC_ALL=C

# Setup
cd "$(dirname $0)"
GIT_ROOT=$(git rev-parse --show-toplevel)

cat $GIT_ROOT/.cron/jobs/abuseipdb/asn.cfg | shfmt -mn | \
xargs -P2 -I% bkt --ttl=1d -- curl -sL https://raw.githubusercontent.com/ipverse/asn-ip/master/as/%/ipv4-aggregated.txt | \
iprange --min-prefix 24
19 changes: 19 additions & 0 deletions .cron/jobs/abuseipdb/asn.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# These ASNS are broken. Relative to their assigned IP space, they have an extremely high number of malicious traffic.
# Continously check for maliscoius activity, even if Confidence Level < 100%

215766 # AS215766 (EMANUELHOSTING) Emanuel Hosting Ltd. - United Kingdom https://www.abuseipdb.com/check-block/79.110.62.0/24
201814 # AS201814 (MEVSPACE) MEVSPACE sp. z o.o. - Poland
267784 # AS267784 (FLYSERVERS) Flyservers S.A https://www.abuseipdb.com/check-block/45.43.64.0/24 - Panama
211298 # AS211298 (INTERNET-MEASUREMENT) Constantine Cybersecurity Ltd. - United Kingdom https://www.abuseipdb.com/check-block/87.236.176.0/24
202425 # AS202425 (INT-NETWORK) IP Volume inc https://cleantalk.org/blacklists/as202425 - United Kingdom
208843 # AS208843 (ALPHASTRIKE-RESEARCH) Alpha Strike Labs GmbH - Germany https://www.abuseipdb.com/check-block/45.83.67.0/24
51396 # AS51396 (PFCLOUD) Pfcloud UG
198953 # AS198953 (PROTON66) Proton66 OOO - Russia
211680 # AS211680 (BITSIGHT) NSEC - Sistemas Informaticos, S.A.
214961 # AS214961 (STELLARGROUPSAS) Stellar Groups SAS - France
215365 # AS215365 (THREATOFF) Tom Gewiese
216240 # AS216240 (MORTALSOFT) MortalSoft Ltd.
9465 # AS9465 (AGOTOZPTELTD-AP) AGOTOZ PTE. LTD.
210743 # AS210743 (BABBAR) Babbar SAS
214940 # AS214940 (KPRONET) KPROHOST LLC
273113 # ONERED JWG532 SRL, DO
13 changes: 13 additions & 0 deletions .cron/jobs/abuseipdb/bogons.ipv4
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
0.0.0.0/8
10.0.0.0/8
100.64.0.0/10
127.0.0.0/8
169.254.0.0/16
172.16.0.0/12
192.0.0.0/24
192.0.2.0/24
192.168.0.0/16
198.18.0.0/15
198.51.100.0/24
203.0.113.0/24
224.0.0.0/3
137 changes: 137 additions & 0 deletions .cron/jobs/abuseipdb/cron
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
#!/usr/bin/env bash

set -e

# Setup
cd "$(dirname $0)"

GIT_ROOT=$(git rev-parse --show-toplevel)
DB_PATH=$GIT_ROOT/db
mkdir -p $DB_PATH
TEMPFILE=$(mktemp)
TEMPDIR=$(mktemp -d)

function check-block() {
local URL="$1"
curl -sL "$URL" | \
grepip | xargs -n1 $GIT_ROOT/.cron/scripts/abuseipdb-check | jq 'select(.numReports > 1)' | grepip
}

cd $TEMPDIR

# Debug
echo "Public IP:"
echo $(timeout 2s curl --no-progress-meter ipv4.icanhazip.com)
echo

echo '✔ Debug...'
date '+%Y/%m/%d %H:%M:%S'
bkt --ttl=6h -- date '+%Y/%m/%d %H:%M:%S'
echo


echo '✔ Download abuseipdb...'
# Use a TTL of ~2.5 hours (~ 9/10 requests if verified webmaster)
bkt --ttl=180min -- curl https://api.abuseipdb.com/api/v2/blacklist \
--get \
--max-time 10 \
--user-agent "" \
--no-progress-meter \
-d confidenceMinimum=100 \
-d limit=9999999 \
-H "Key: $ABUSEIPDB_TOKEN" \
-H "Accept: text/plain" \
--fail \
-w "\n" \
-o TEMPFILE.1 || true

echo '✔ Download & decorate with extra sources ...'
echo '#2: abuseipdb.tmiland.com/'
curl -sL https://abuseipdb.tmiland.com/abuseipdb.txt \
--compressed --max-time 10 -G -sL --fail -o TEMPFILE.2 || true

echo '#3: LittleJake'
curl -sL https://raw.githubusercontent.com/LittleJake/ip-blacklist/main/abuseipdb_blacklist_ip_score_100.txt \
--compressed --max-time 10 -G -sL --fail -o TEMPFILE.3 || true

#
echo '#4: 💩 Whitelisted scanners'
check-block https://raw.githubusercontent.com/borestad/iplists/refs/heads/main/paloaltonetworks/paloaltonetworks.ipv4 >| TEMPFILE.${RANDOM} || true
check-block https://raw.githubusercontent.com/borestad/iplists/refs/heads/main/censys/censys.ipv4 >| TEMPFILE.${RANDOM} || true
check-block https://raw.githubusercontent.com/borestad/iplists/refs/heads/main/project-sonar/project-sonar.ipv4 >| TEMPFILE.${RANDOM} || true
check-block https://raw.githubusercontent.com/borestad/iplists/refs/heads/main/openai/gptbot.ipv4 >| TEMPFILE.${RANDOM} || true


echo '#7: Broken ASNS'
$GIT_ROOT/.cron/jobs/abuseipdb/asn | \
xargs -I% $GIT_ROOT/.cron/scripts/abuseipdb-check % | jq 'select(.numReports > 1)' | grepip >| TEMPFILE.${RANDOM} || true

# Redundancy:
# - Separate private cache (1 of 5 requests / day) to avoid breaking the 5 free run limit / day
# - If above urls fail due to github actions being flaky, still have somewhat fresh data.
# echo '✔ Download from cache'
echo '#8: Private cache'
curl "$CRONSRC_URL" --compressed --max-time 10 -G -sL -w "\n\n" --fail -o TEMPFILE.${RANDOM} || true

echo '#9: Private honeypots with verified abuse score'
curl "$HONEYPOT1_URL/$(date '+%Y-%m-%d').ipv4" --compressed --max-time 10 -G -sL -w "\n\n" --fail -o TEMPFILE.111 || true
cat TEMPFILE.111 | grepip | iprange - -1 --except $GIT_ROOT/abuseipdb-s100-1d.ipv4 || true

curl "$HONEYPOT2_URL/$(date '+%Y-%m-%d').ipv4" --compressed --max-time 10 -G -sL -w "\n\n" --fail -o TEMPFILE.222 || true
cat TEMPFILE.222 | grepip | iprange - -1 --except $GIT_ROOT/abuseipdb-s100-1d.ipv4 || true


# echo '✔ Stats'
# for FILE in TEMPFILE.*; do printf "$FILE "; wc -l < $FILE; done

echo '✔ Squash all sources (by design: fail if no sources worked)'
grep -h "" TEMPFILE.* >> $TEMPFILE

echo '✔ Validate: Clean comments'
cat $TEMPFILE | shfmt -mn | sponge $TEMPFILE

echo '✔ Validate: Extract ipv6 data'
grep ':' $TEMPFILE | sort | tac | cidr-merger | sponge $TEMPFILE.ipv6

echo '✔ Validate: Extract ipv4 data'
grep -v ":" $TEMPFILE | \
iprange - -1 --except $GIT_ROOT/.cron/jobs/abuseipdb/bogons.ipv4 \
> $TEMPFILE.ipv4

# 3. Validate data
LINES=`wc -l < $TEMPFILE.ipv4`
if [[ "$LINES" -gt "1000" ]]; then
echo "✔ Validate: File contains: $LINES lines"
mv $TEMPFILE.ipv4 $DB_PATH/abuseipdb-s100-latest.ipv4
mv $TEMPFILE.ipv6 $DB_PATH/abuseipdb-s100-latest.ipv6
else
echo "❌ Validation failed"
echo
echo "-----------------------------------------------------"
cat $TEMPFILE
echo "-----------------------------------------------------"
cat $TEMPFILE.ipv4
echo "-----------------------------------------------------"
exit 1
fi

echo
echo '✔ Aggregate: Create folders'
DATE=$(date +%F)
DATE_DIR=$DB_PATH/$DATE
mkdir -pv $DATE_DIR && cd $DATE_DIR

echo '✔ Aggregate: Copy latest to correct date folder'
cp $DB_PATH/abuseipdb-s100-latest.ipv4 "$DATE_DIR/tmp-$(date +%H-%m-%S).ipv4"
cp $DB_PATH/abuseipdb-s100-latest.ipv6 "$DATE_DIR/tmp-$(date +%H-%m-%S).ipv6"

echo '✔ Aggregate: Squash ipv4 data'
iprange -1 *.ipv4 | sponge $(date +%Y-%m-%d).ipv4

echo '✔ Aggregate: Squash ipv6 data'
cat *.ipv6 | grep ':' | sort | uniq | sort | sponge $(date +%Y-%m-%d).ipv6

echo
echo '✔ Cleanup: Remove temp files'
rm -f tmp*.ipv4
rm -f tmp*.ipv6
84 changes: 84 additions & 0 deletions .cron/jobs/abuseipdb/hallofshame
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
#!/usr/bin/env bash

set -e
export LC_ALL=C

# Setup
cd "$(dirname $0)"
GIT_ROOT=$(git rev-parse --show-toplevel)
mkdir -p $GIT_ROOT/stats/hallofshame/subnets

cd $GIT_ROOT

update-hall-of-shame() {
local days=$1
local percent=$2
local nr=$(($percent * 256 / 100))

OUTPUT="$GIT_ROOT/stats/hallofshame/subnets/abuseipdb-s99-hallofshame-${days}-${percent}percent.ipv4"
echo $OUTPUT
TS=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
TEMPFILE=$(mktemp)


# Hall of shame, where > x % of the ips in a cidr-block is malicious traffic
cat "./abuseipdb-s100-${days}.ipv4" | \
iprange -1 | \
sed 's/\./ /g' | \
awk '{print $1, $2, $3}' | \
sed 's/[[:space:]]/./g' | \
sort | uniq -c | sort | \
sed 's/$/.0\/24/' | \
awk "\$1 >= $nr {print \$2}" | \
iprange --min-prefix 24 \
>> $TEMPFILE


echo "#" >| $OUTPUT
echo "# Hall of Shame: $days" >> $OUTPUT
echo "# An aggregated list of ip ranges, where more than ${percent}% the traffic from a /24 range is malicious from the last ${days}ays" >> $OUTPUT
echo "#" >> $OUTPUT
echo "#" >> $OUTPUT
echo "# Last updated: $TS" >> $OUTPUT
echo "# Days: $days" >> $OUTPUT
echo "# Malicious level: > ${percent}%" >> $OUTPUT
echo "# Filename: $(basename $OUTPUT)" >> $OUTPUT
echo "# Number of ips: $(iprange -1 $TEMPFILE | wc -l)" >> $OUTPUT
echo "#" >> $OUTPUT
echo "# Source: https://github.com/borestad/blocklist-abuseipdb" >> $OUTPUT
echo "# Credits: https://www.abuseipdb.com - please support them!" >> $OUTPUT
echo "# Example: https://www.abuseipdb.com/check-block/64.62.156.0/24" >> $OUTPUT

echo "#" >> $OUTPUT
echo "" >> $OUTPUT

cat $TEMPFILE >> $OUTPUT

}

decorate-with-asn-info() {
cd $GIT_ROOT && \
fd -tf '.ipv4$' $GIT_ROOT/stats/hallofshame/subnets --max-depth=1 --min-depth=1 -x \
bash -c "cat {} | .cron/scripts/ip2ipinfo.ts >| {}.tmp && mv {}.tmp {}" || true
}



for i in 1 5 10 15 20 25 50 75; do
update-hall-of-shame 1d $i &
update-hall-of-shame 3d $i &
update-hall-of-shame 7d $i &
update-hall-of-shame 14d $i &
update-hall-of-shame 30d $i &
update-hall-of-shame 60d $i &
update-hall-of-shame 90d $i &
update-hall-of-shame 120d $i &
update-hall-of-shame 180d $i &
update-hall-of-shame 365d $i &
update-hall-of-shame all $i &
done

wait

decorate-with-asn-info

Loading

0 comments on commit 5355c13

Please sign in to comment.