From 44e84dad6bbf032ec7b57a4a4c2370b1778a256b Mon Sep 17 00:00:00 2001 From: Chocolate4U Date: Mon, 2 Oct 2023 18:57:02 +0330 Subject: [PATCH] Bug fix Added fallback for Iranian CDNs IP lists to get data from local repository if remote URL was inaccessible. Added multiple checks for 'malware' and 'phishing' domains lists to filter out invalid URLs. --- .github/workflows/release.yml | 11 +++++++++-- config.json | 8 ++++---- data/arvancloud-ip.txt | 15 +++++++++++++++ data/derakcloud-ipv4.txt | 22 ++++++++++++++++++++++ data/derakcloud-ipv6.txt | 1 + data/iranserver-ip.txt | 11 +++++++++++ 6 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 data/arvancloud-ip.txt create mode 100644 data/derakcloud-ipv4.txt create mode 100644 data/derakcloud-ipv6.txt create mode 100644 data/iranserver-ip.txt diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29279326..ecd8d183 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,6 +37,13 @@ jobs: run: | curl -sSL https://raw.githubusercontent.com/Chocolate4U/ito.gov.ir-Mirror/main/data/Messengers.csv | awk -F"," '{print $2}' | sed '1d' > messengers-ip.txt + - name: Get domestic CDNs IP list + run: | + curl --connect-timeout 15 -sSL https://www.arvancloud.ir/en/ips.txt -o arvancloud-ip.txt || cp data/arvancloud-ip.txt arvancloud-ip.txt + curl --connect-timeout 15 -sSL https://api.derak.cloud/public/ipv4 -o derakcloud-ipv4.txt || cp data/derakcloud-ipv4.txt derakcloud-ipv4.txt + curl --connect-timeout 15 -sSL https://api.derak.cloud/public/ipv6 -o derakcloud-ipv6.txt || cp data/derakcloud-ipv6.txt derakcloud-ipv6.txt + curl --connect-timeout 15 -sSL https://ips.f95.com/ip.txt -o iranserver-ip.txt || cp data/iranserver-ip.txt iranserver-ip.txt + - name: Get GeoLite2 env: LICENSE_KEY: ${{ secrets.MAXMIND_GEOLITE2_LICENSE }} @@ -82,14 +89,14 @@ jobs: - name: Generate malware list run: | - curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt | sed '/#/d' > malware.txt + curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > malware.txt curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-ips-online.txt | sed '/#/d' > malware-ip.txt echo "TOTAL_MALWARE=$(wc -l < malware.txt)" >> $GITHUB_ENV mv malware.txt release - name: Generate phishing list run: | - curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt | sed '/#/d' > phishing.txt + curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > phishing.txt curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-ips.txt | sed '/#/d' > phishing-ip.txt echo "TOTAL_PHISHING=$(wc -l < phishing.txt)" >> $GITHUB_ENV mv phishing.txt release diff --git a/config.json b/config.json index a46921f7..75c9731b 100644 --- a/config.json +++ b/config.json @@ -25,7 +25,7 @@ "action": "add", "args": { "name": "arvancloud", - "uri": "https://www.arvancloud.ir/en/ips.txt" + "uri": "../arvancloud-ip.txt" } }, { @@ -33,7 +33,7 @@ "action": "add", "args": { "name": "derakcloud", - "uri": "https://api.derak.cloud/public/ipv4" + "uri": "../derakcloud-ipv4.txt" } }, { @@ -41,7 +41,7 @@ "action": "add", "args": { "name": "derakcloud", - "uri": "https://api.derak.cloud/public/ipv6" + "uri": "../derakcloud-ipv6.txt" } }, { @@ -49,7 +49,7 @@ "action": "add", "args": { "name": "iranserver", - "uri": "https://ips.f95.com/ip.txt" + "uri": "../iranserver-ip.txt" } }, { diff --git a/data/arvancloud-ip.txt b/data/arvancloud-ip.txt new file mode 100644 index 00000000..942e6ac3 --- /dev/null +++ b/data/arvancloud-ip.txt @@ -0,0 +1,15 @@ +185.143.232.0/22 +92.114.16.80/28 +188.229.116.16/29 +94.182.182.28/30 +185.228.238.0/28 +94.182.153.24/29 +94.101.182.0/27 +2.144.3.128/28 +89.45.48.64/28 +37.32.16.0/27 +37.32.17.0/27 +37.32.18.0/27 +37.32.19.0/27 +185.215.232.0/22 +109.230.200.48/29 \ No newline at end of file diff --git a/data/derakcloud-ipv4.txt b/data/derakcloud-ipv4.txt new file mode 100644 index 00000000..b8cff5a0 --- /dev/null +++ b/data/derakcloud-ipv4.txt @@ -0,0 +1,22 @@ +5.145.115.0/24 +5.145.112.0/24 +185.24.255.0/24 +185.169.6.0/24 +5.145.117.0/24 +5.145.118.0/24 +5.145.119.0/24 +178.62.222.208/28 +159.69.229.224/28 +116.202.90.176/28 +165.232.92.112/28 +216.155.152.176/28 +139.180.159.176/28 +45.77.71.48/28 +45.77.87.48/28 +209.246.143.48/28 +139.180.159.192/28 +45.76.37.144/28 +185.24.252.192/28 +217.69.10.128/28 +5.145.113.192/28 +185.24.254.64/28 \ No newline at end of file diff --git a/data/derakcloud-ipv6.txt b/data/derakcloud-ipv6.txt new file mode 100644 index 00000000..3ced7f38 --- /dev/null +++ b/data/derakcloud-ipv6.txt @@ -0,0 +1 @@ +2a04:2f00::/29 \ No newline at end of file diff --git a/data/iranserver-ip.txt b/data/iranserver-ip.txt new file mode 100644 index 00000000..df9c0460 --- /dev/null +++ b/data/iranserver-ip.txt @@ -0,0 +1,11 @@ +5.9.198.122/32 +5.182.45.23/32 +5.182.45.37/32 +85.208.253.161/32 +85.208.255.159/32 +94.182.97.43/32 +94.182.97.44/31 +94.182.97.46/32 +144.76.216.57/32 +168.119.4.117/32 +185.116.161.33/32