Bug fix

Added fallback for Iranian CDNs IP lists to get data from local repository if remote URL was inaccessible.

Added multiple checks for 'malware' and 'phishing' domains lists to filter out invalid URLs.

.github/workflows/release.yml

@@ -37,6 +37,13 @@ jobs:
         run: |
           curl -sSL https://raw.githubusercontent.com/Chocolate4U/ito.gov.ir-Mirror/main/data/Messengers.csv | awk -F"," '{print $2}' | sed '1d' > messengers-ip.txt
 
+      - name: Get domestic CDNs IP list
+        run: |
+          curl --connect-timeout 15 -sSL https://www.arvancloud.ir/en/ips.txt -o arvancloud-ip.txt || cp data/arvancloud-ip.txt arvancloud-ip.txt
+          curl --connect-timeout 15 -sSL https://api.derak.cloud/public/ipv4 -o derakcloud-ipv4.txt || cp data/derakcloud-ipv4.txt derakcloud-ipv4.txt
+          curl --connect-timeout 15 -sSL https://api.derak.cloud/public/ipv6 -o derakcloud-ipv6.txt || cp data/derakcloud-ipv6.txt derakcloud-ipv6.txt
+          curl --connect-timeout 15 -sSL https://ips.f95.com/ip.txt -o iranserver-ip.txt || cp data/iranserver-ip.txt iranserver-ip.txt
+
       - name: Get GeoLite2
         env:
           LICENSE_KEY: ${{ secrets.MAXMIND_GEOLITE2_LICENSE }}
@@ -82,14 +89,14 @@ jobs:
 
       - name: Generate malware list
         run: |
-          curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt | sed '/#/d' > malware.txt
+          curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > malware.txt
           curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-ips-online.txt | sed '/#/d' > malware-ip.txt
           echo "TOTAL_MALWARE=$(wc -l < malware.txt)" >> $GITHUB_ENV
           mv malware.txt release
 
       - name: Generate phishing list
         run: |
-          curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt | sed '/#/d' > phishing.txt
+          curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > phishing.txt
           curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-ips.txt | sed '/#/d' > phishing-ip.txt
           echo "TOTAL_PHISHING=$(wc -l < phishing.txt)" >> $GITHUB_ENV
           mv phishing.txt release

config.json

@@ -25,31 +25,31 @@
             "action": "add",
             "args": {
                 "name": "arvancloud",
-                "uri": "https://www.arvancloud.ir/en/ips.txt"
+                "uri": "../arvancloud-ip.txt"
             }
         },
         {
             "type": "text",
             "action": "add",
             "args": {
                 "name": "derakcloud",
-                "uri": "https://api.derak.cloud/public/ipv4"
+                "uri": "../derakcloud-ipv4.txt"
             }
         },
         {
             "type": "text",
             "action": "add",
             "args": {
                 "name": "derakcloud",
-                "uri": "https://api.derak.cloud/public/ipv6"
+                "uri": "../derakcloud-ipv6.txt"
             }
         },
         {
             "type": "text",
             "action": "add",
             "args": {
                 "name": "iranserver",
-                "uri": "https://ips.f95.com/ip.txt"
+                "uri": "../iranserver-ip.txt"
             }
         },
         {

data/arvancloud-ip.txt

@@ -0,0 +1,15 @@
+185.143.232.0/22
+92.114.16.80/28
+188.229.116.16/29
+94.182.182.28/30
+185.228.238.0/28
+94.182.153.24/29
+94.101.182.0/27
+2.144.3.128/28
+89.45.48.64/28
+37.32.16.0/27
+37.32.17.0/27
+37.32.18.0/27
+37.32.19.0/27
+185.215.232.0/22
+109.230.200.48/29

data/derakcloud-ipv4.txt

@@ -0,0 +1,22 @@
+5.145.115.0/24
+5.145.112.0/24
+185.24.255.0/24
+185.169.6.0/24
+5.145.117.0/24
+5.145.118.0/24
+5.145.119.0/24
+178.62.222.208/28
+159.69.229.224/28
+116.202.90.176/28
+165.232.92.112/28
+216.155.152.176/28
+139.180.159.176/28
+45.77.71.48/28
+45.77.87.48/28
+209.246.143.48/28
+139.180.159.192/28
+45.76.37.144/28
+185.24.252.192/28
+217.69.10.128/28
+5.145.113.192/28
+185.24.254.64/28

data/derakcloud-ipv6.txt

@@ -0,0 +1 @@
+2a04:2f00::/29

data/iranserver-ip.txt

@@ -0,0 +1,11 @@
+5.9.198.122/32
+5.182.45.23/32
+5.182.45.37/32
+85.208.253.161/32
+85.208.255.159/32
+94.182.97.43/32
+94.182.97.44/31
+94.182.97.46/32
+144.76.216.57/32
+168.119.4.117/32
+185.116.161.33/32