diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_ASN_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_ASN_CL.json new file mode 100644 index 00000000000..62a2c1a5cce --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_ASN_CL.json @@ -0,0 +1,33 @@ +{ + "Name": "Ipinfo_ASN_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "asn", + "Type": "String" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "domain", + "Type": "String" + }, + { + "Name": "route", + "Type": "String" + }, + { + "Name": "asn_type", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Abuse_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Abuse_CL.json new file mode 100644 index 00000000000..96c6b8c402c --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Abuse_CL.json @@ -0,0 +1,37 @@ +{ + "Name": "Ipinfo_Abuse_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "email", + "Type": "String" + }, + { + "Name": "address", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "phone", + "Type": "String" + }, + { + "Name": "network", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Carrier_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Carrier_CL.json new file mode 100644 index 00000000000..f418168a591 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Carrier_CL.json @@ -0,0 +1,33 @@ +{ + "Name": "Ipinfo_Carrier_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "carrier", + "Type": "String" + }, + { + "Name": "mcc", + "Type": "String" + }, + { + "Name": "mnc", + "Type": "String" + }, + { + "Name": "cc", + "Type": "String" + }, + { + "Name": "network", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Country_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Country_CL.json new file mode 100644 index 00000000000..2c1b1d0883b --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Country_CL.json @@ -0,0 +1,41 @@ +{ + "Name": "Ipinfo_Country_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "as_domain", + "Type": "String" + }, + { + "Name": "as_name", + "Type": "String" + }, + { + "Name": "asn", + "Type": "String" + }, + { + "Name": "continent", + "Type": "String" + }, + { + "Name": "continent_name", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "country_name", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Domain_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Domain_CL.json new file mode 100644 index 00000000000..49a2e0b2a37 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Domain_CL.json @@ -0,0 +1,21 @@ +{ + "Name": "Ipinfo_Domain_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "domains", + "Type": "String" + }, + { + "Name": "total", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Location_extended_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Location_extended_CL.json new file mode 100644 index 00000000000..5a69511ecf6 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Location_extended_CL.json @@ -0,0 +1,57 @@ +{ + "Name": "Ipinfo_Location_extended_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "city", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "country_name", + "Type": "String" + }, + { + "Name": "latitude", + "Type": "String" + }, + { + "Name": "longitude", + "Type": "String" + }, + { + "Name": "postal_code", + "Type": "String" + }, + { + "Name": "radius", + "Type": "String" + }, + { + "Name": "region_name", + "Type": "String" + }, + { + "Name": "region", + "Type": "String" + }, + { + "Name": "timezone", + "Type": "String" + }, + { + "Name": "geoname_id", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Privacy_extended_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Privacy_extended_CL.json new file mode 100644 index 00000000000..e27f26b0dbc --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_Privacy_extended_CL.json @@ -0,0 +1,65 @@ +{ + "Name": "Ipinfo_Privacy_extended_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "anycast", + "Type": "String" + }, + { + "Name": "census", + "Type": "String" + }, + { + "Name": "census_port", + "Type": "String" + }, + { + "Name": "device_activity", + "Type": "String" + }, + { + "Name": "hosting", + "Type": "String" + }, + { + "Name": "network", + "Type": "String" + }, + { + "Name": "proxy", + "Type": "String" + }, + { + "Name": "relay", + "Type": "String" + }, + { + "Name": "tor", + "Type": "String" + }, + { + "Name": "vpn", + "Type": "String" + }, + { + "Name": "vpn_config", + "Type": "String" + }, + { + "Name": "vpn_name", + "Type": "String" + }, + { + "Name": "whois", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_RIRWHOIS_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_RIRWHOIS_CL.json new file mode 100644 index 00000000000..2f5587e4a07 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_RIRWHOIS_CL.json @@ -0,0 +1,89 @@ +{ + "Name": "Ipinfo_RIRWHOIS_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "whois_id", + "Type": "String" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "status", + "Type": "String" + }, + { + "Name": "tech", + "Type": "String" + }, + { + "Name": "maintainer", + "Type": "String" + }, + { + "Name": "admin", + "Type": "String" + }, + { + "Name": "source", + "Type": "String" + }, + { + "Name": "whois_domain", + "Type": "String" + }, + { + "Name": "updated", + "Type": "String" + }, + { + "Name": "org", + "Type": "String" + }, + { + "Name": "rdns_domain", + "Type": "String" + }, + { + "Name": "domain", + "Type": "String" + }, + { + "Name": "geoloc", + "Type": "String" + }, + { + "Name": "org_address", + "Type": "String" + }, + { + "Name": "asn", + "Type": "String" + }, + { + "Name": "as_name", + "Type": "String" + }, + { + "Name": "as_domain", + "Type": "String" + }, + { + "Name": "as_type", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_RWHOIS_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_RWHOIS_CL.json new file mode 100644 index 00000000000..e4cfc750677 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_RWHOIS_CL.json @@ -0,0 +1,65 @@ +{ + "Name": "Ipinfo_RWHOIS_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "whois_id", + "Type": "String" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "whois_desc", + "Type": "String" + }, + { + "Name": "host", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "email", + "Type": "String" + }, + { + "Name": "abuse", + "Type": "String" + }, + { + "Name": "domain", + "Type": "String" + }, + { + "Name": "city", + "Type": "String" + }, + { + "Name": "street", + "Type": "String" + }, + { + "Name": "postal", + "Type": "String" + }, + { + "Name": "updated", + "Type": "String" + }, + { + "Name": "imported", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_ASN_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_ASN_CL.json new file mode 100644 index 00000000000..2a91a5861ce --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_ASN_CL.json @@ -0,0 +1,37 @@ +{ + "Name": "Ipinfo_WHOIS_ASN_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "whois_id", + "Type": "String" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "org_id", + "Type": "String" + }, + { + "Name": "created", + "Type": "String" + }, + { + "Name": "updated", + "Type": "String" + }, + { + "Name": "source", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_MNT_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_MNT_CL.json new file mode 100644 index 00000000000..df86c921768 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_MNT_CL.json @@ -0,0 +1,41 @@ +{ + "Name": "Ipinfo_WHOIS_MNT_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "whois_id", + "Type": "String" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "admin_id", + "Type": "String" + }, + { + "Name": "tech_id", + "Type": "String" + }, + { + "Name": "org_id", + "Type": "String" + }, + { + "Name": "created", + "Type": "String" + }, + { + "Name": "updated", + "Type": "String" + }, + { + "Name": "source", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_NET_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_NET_CL.json new file mode 100644 index 00000000000..aacdb181c20 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_NET_CL.json @@ -0,0 +1,65 @@ +{ + "Name": "Ipinfo_WHOIS_NET_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "whois_id", + "Type": "String" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "domain", + "Type": "String" + }, + { + "Name": "org_id", + "Type": "String" + }, + { + "Name": "status", + "Type": "String" + }, + { + "Name": "tech_id", + "Type": "String" + }, + { + "Name": "mnt_id", + "Type": "String" + }, + { + "Name": "admin_id", + "Type": "String" + }, + { + "Name": "abuse_id", + "Type": "String" + }, + { + "Name": "created", + "Type": "String" + }, + { + "Name": "updated", + "Type": "String" + }, + { + "Name": "source", + "Type": "String" + }, + { + "Name": "range", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_ORG_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_ORG_CL.json new file mode 100644 index 00000000000..ce2440a8214 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_ORG_CL.json @@ -0,0 +1,77 @@ +{ + "Name": "Ipinfo_WHOIS_ORG_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "whois_id", + "Type": "String" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "address", + "Type": "String" + }, + { + "Name": "street", + "Type": "String" + }, + { + "Name": "city", + "Type": "String" + }, + { + "Name": "state", + "Type": "String" + }, + { + "Name": "postalcode", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "admin_id", + "Type": "String" + }, + { + "Name": "tech_id", + "Type": "String" + }, + { + "Name": "abuse_id", + "Type": "String" + }, + { + "Name": "mnt_id", + "Type": "String" + }, + { + "Name": "email", + "Type": "String" + }, + { + "Name": "domain", + "Type": "String" + }, + { + "Name": "created", + "Type": "String" + }, + { + "Name": "updated", + "Type": "String" + }, + { + "Name": "source", + "Type": "String" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_POC_CL.json b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_POC_CL.json new file mode 100644 index 00000000000..1df0b7e0279 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/Ipinfo_WHOIS_POC_CL.json @@ -0,0 +1,57 @@ +{ + "Name": "Ipinfo_WHOIS_POC_CL", + "Properties": [ + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "whois_id", + "Type": "String" + }, + { + "Name": "name", + "Type": "String" + }, + { + "Name": "mobilephone", + "Type": "String" + }, + { + "Name": "officephone", + "Type": "String" + }, + { + "Name": "fax", + "Type": "String" + }, + { + "Name": "address", + "Type": "String" + }, + { + "Name": "country", + "Type": "String" + }, + { + "Name": "email", + "Type": "String" + }, + { + "Name": "abuse_email", + "Type": "String" + }, + { + "Name": "created", + "Type": "String" + }, + { + "Name": "updated", + "Type": "String" + }, + { + "Name": "source", + "Type": "String" + } + ] +} diff --git a/Sample Data/Custom/Ipinfo_ASN_CL.json b/Sample Data/Custom/Ipinfo_ASN_CL.json new file mode 100644 index 00000000000..f4c65498c9a --- /dev/null +++ b/Sample Data/Custom/Ipinfo_ASN_CL.json @@ -0,0 +1,62 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "asn":"AS13335", + "domain":"cloudflare.com", + "name":"Cloudflare, Inc.", + "range":"1.0.0.0/24", + "route":"1.0.0.0/24", + "asn_type":"hosting", + "Type": "Ipinfo_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.341 PM", + "asn":"AS206617", + "domain":"neomedia.it", + "name":"Neomedia S.r.l.", + "range":"1.0.1.0/24", + "route":"1.0.1.0/24", + "asn_type":"business", + "Type": "Ipinfo_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.342 PM", + "asn":"AS206617", + "domain":"neomedia.it", + "name":"Neomedia S.r.l.", + "range":"1.0.2.0/24", + "route":"1.0.2.0/24", + "asn_type":"business", + "Type": "Ipinfo_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.343 PM", + "asn":"AS38803", + "domain":"gtelecom.com.au", + "name":"Wirefreebroadband Pty Ltd", + "range":"1.0.4.0/24", + "route":"1.0.4.0/22", + "asn_type":"isp", + "Type": "Ipinfo_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.344 PM", + "asn":"AS38803", + "domain":"gtelecom.com.au", + "name":"Wirefreebroadband Pty Ltd", + "range":"1.0.5.0/24", + "route":"1.0.5.0/24", + "asn_type":"isp", + "Type": "Ipinfo_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.345 PM", + "asn":"AS38803", + "domain":"gtelecom.com.au", + "name":"Wirefreebroadband Pty Ltd", + "range":"1.0.6.0/23", + "route":"1.0.4.0/22", + "asn_type":"isp", + "Type": "Ipinfo_ASN_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_Abuse_CL.json b/Sample Data/Custom/Ipinfo_Abuse_CL.json new file mode 100644 index 00000000000..8e18da21046 --- /dev/null +++ b/Sample Data/Custom/Ipinfo_Abuse_CL.json @@ -0,0 +1,68 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "name":"ABUSE APNICRANDNETAU", + "email":"sanitized@sanitized.com", + "address":"PO Box 3646, South Brisbane, QLD 4101, Australia", + "country":"AU", + "phone":"+000000000", + "network":"1.0.0.0/24", + "range":"1.0.0.0/24", + "Type": "Ipinfo_Abuse_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.341 PM", + "address":"No.31 ,jingrong street,beijing, 100032", + "country":"CN", + "email":"sanitized@sanitized.com", + "name":"ABUSE CHINANETCN", + "network":"1.0.1.0/24", + "phone":"+000000000", + "range":"1.0.1.0/24", + "Type": "Ipinfo_Abuse_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.342 PM", + "address":"No.31 ,jingrong street,beijing, 100032", + "country":"CN", + "email":"sanitized@sanitized.com", + "name":"ABUSE CHINANETCN", + "network":"1.0.2.0/23", + "phone":"+000000000", + "range":"1.0.2.0/23", + "Type": "Ipinfo_Abuse_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.343 PM", + "address":"1/18 Deblin drive, Narre warren, vic 3805, Melbourne victoria 3805", + "country":"AU", + "email":"sanitized@sanitized.com", + "name":"ABUSE WPLAU", + "network":"1.0.4.0/24", + "phone":"+000000000", + "range":"1.0.4.0/24", + "Type": "Ipinfo_Abuse_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.344 PM", + "address":"1/18 Deblin drive, Narre warren, vic 3805, Melbourne victoria 3805", + "country":"AU", + "email":"sanitized@sanitized.com", + "name":"ABUSE WPLAU", + "network":"1.0.5.0/24", + "phone":"+000000000", + "range":"1.0.5.0/24", + "Type": "Ipinfo_Abuse_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.345 PM", + "address":"1/18 Deblin drive, Narre warren, vic 3805, Melbourne victoria 3805", + "country":"AU", + "email":"sanitized@sanitized.com", + "name":"ABUSE WPLAU", + "network":"1.0.6.0/24", + "phone":"+000000000", + "range":"1.0.6.0/24", + "Type": "Ipinfo_Abuse_CL" + } +] diff --git a/Sample Data/Custom/Ipinfo_Carrier_CL.json b/Sample Data/Custom/Ipinfo_Carrier_CL.json new file mode 100644 index 00000000000..0ac663759b3 --- /dev/null +++ b/Sample Data/Custom/Ipinfo_Carrier_CL.json @@ -0,0 +1,62 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "carrier":"dtac-T", + "cc":"TH", + "mcc":"520", + "mnc":"47", + "network":"1.0.178.0-1.0.178.255", + "range":"1.0.178.0/24", + "Type": "Ipinfo_Carrier_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.341 PM", + "carrier":"NTT docomo", + "cc":"JP", + "mcc":"440", + "mnc":"10", + "network":"1.1.125.0-1.1.125.255", + "range":"1.1.125.0/24", + "Type": "Ipinfo_Carrier_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.342 PM", + "carrier":"dtac-T", + "cc":"TH", + "mcc":"520", + "mnc":"47", + "network":"1.1.240.0-1.1.240.255", + "range":"1.1.240.0/24", + "Type": "Ipinfo_Carrier_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.343 PM", + "carrier":"AIS", + "cc":"TH", + "mcc":"520", + "mnc":"01", + "network":"1.2.164.0-1.2.164.255", + "range":"1.2.164.0/24", + "Type": "Ipinfo_Carrier_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.344 PM", + "carrier":"AIS", + "cc":"TH", + "mcc":"520", + "mnc":"01", + "network":"1.2.177.0-1.2.177.255", + "range":"1.2.177.0/24", + "Type": "Ipinfo_Carrier_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.345 PM", + "carrier":"dtac-T", + "cc":"TH", + "mcc":"520", + "mnc":"47", + "network":"1.2.238.0-1.2.238.255", + "range":"1.2.238.0/24", + "Type": "Ipinfo_Carrier_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_Country_CL.json b/Sample Data/Custom/Ipinfo_Country_CL.json new file mode 100644 index 00000000000..ff8c0615829 --- /dev/null +++ b/Sample Data/Custom/Ipinfo_Country_CL.json @@ -0,0 +1,73 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:07:33.199 PM", + "as_domain":"cloudflare.com", + "as_name":"Cloudflare, Inc.", + "asn":"AS13335", + "continent":"OC", + "continent_name":"Oceania", + "country":"AU", + "country_name":"Australia", + "ip_range":"1.0.0.0/25", + "Type": "Ipinfo_Country_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:07:33.199 PM", + "as_domain":"cloudflare.com", + "as_name":"Cloudflare, Inc.", + "asn":"AS13335","continent":"OC", + "continent_name":"Oceania", + "country":"AU", + "country_name":"Australia", + "ip_range":"1.0.0.128/26", + "Type": "Ipinfo_Country_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:07:33.199 PM", + "as_domain":"cloudflare.com", + "as_name":"Cloudflare, Inc.", + "asn":"AS13335", + "continent":"OC", + "continent_name":"Oceania", + "country":"AU", + "country_name":"Australia", + "ip_range":"1.0.0.192/27", + "Type": "Ipinfo_Country_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:07:33.199 PM", + "as_domain":"cloudflare.com", + "as_name":"Cloudflare, Inc.", + "asn":"AS13335", + "continent":"OC", + "continent_name":"Oceania", + "country":"AU", + "country_name":"Australia", + "ip_range":"1.0.0.224/28", + "Type": "Ipinfo_Country_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:07:33.199 PM", + "as_domain":"cloudflare.com", + "as_name":"Cloudflare, Inc.", + "asn":"AS13335", + "continent":"OC", + "continent_name":"Oceania", + "country":"AU", + "country_name":"Australia", + "ip_range":"1.0.0.240/29", + "Type": "Ipinfo_Country_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:07:33.199 PM", + "as_domain":"cloudflare.com", + "as_name":"Cloudflare, Inc.", + "asn":"AS13335", + "continent":"OC", + "continent_name":"Oceania", + "country":"AU", + "country_name":"Australia", + "ip_range":"1.0.0.248/30", + "Type": "Ipinfo_Country_CL" + } +] diff --git a/Sample Data/Custom/Ipinfo_Domain_CL.json b/Sample Data/Custom/Ipinfo_Domain_CL.json new file mode 100644 index 00000000000..63b49305cd3 --- /dev/null +++ b/Sample Data/Custom/Ipinfo_Domain_CL.json @@ -0,0 +1,44 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "domains":"xn--uhr444azd877g.cn,xn--ugr712atyak08d.cn", + "range":"1.0.0.4/32", + "total":"2", + "Type": "Ipinfo_Domain_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.341 PM", + "domains":"zfdnf.com,xn--ugr712atyak08d.cn,xn--uhr444azd877g.cn", + "range":"1.0.0.5/32", + "total":"3", + "Type": "Ipinfo_Domain_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.342 PM", + "domains":"xn--uhr444azd877g.cn,xn--ugr712atyak08d.cn", + "range":"1.0.0.6/32", + "total":"2", + "Type": "Ipinfo_Domain_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.343 PM", + "domains":"xn--uhr444azd877g.cn,xn--ugr712atyak08d.cn,tcm-database.cf", + "range":"1.0.0.7/32", + "total":"3", + "Type": "Ipinfo_Domain_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.344 PM", + "domains":"xn--uhr444azd877g.cn,dimeji.com,xn--ugr712atyak08d.cn", + "range":"1.0.0.8/32", + "total":"3", + "Type": "Ipinfo_Domain_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.345 PM", + "domains":"xn--ugr712atyak08d.cn,xn--uhr444azd877g.cn", + "range":"1.0.0.9/32", + "total":"2", + "Type": "Ipinfo_Domain_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_Location_extended_CL.json b/Sample Data/Custom/Ipinfo_Location_extended_CL.json new file mode 100644 index 00000000000..9095ee0bb0c --- /dev/null +++ b/Sample Data/Custom/Ipinfo_Location_extended_CL.json @@ -0,0 +1,98 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "city":"Parepare", + "country":"ID", + "country_name":"Indonesia", + "geoname_id":"1632353", + "latitude":"-4.0135", + "longitude":"119.6255", + "postal_code":"", + "radius":"20", + "range":"1.0.0.255/32", + "region":"SN", + "region_name":"South Sulawesi", + "timezone":"Asia/Makassar", + "Type": "Ipinfo_Location_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.341 PM", + "city":"Xiamen", + "country":"CN", + "country_name":"China", + "geoname_id":"1790645", + "latitude":"24.47979", + "longitude":"118.08187", + "postal_code":"", + "radius":"1000", + "range":"1.0.1.0/24", + "region":"FJ", + "region_name":"Fujian", + "timezone":"Asia/Shanghai", + "Type": "Ipinfo_Location_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.342 PM", + "city":"Xiamen", + "country":"CN", + "country_name":"China", + "geoname_id":"1790645", + "latitude":"24.47979", + "longitude":"118.08187", + "postal_code":"", + "radius":"1000", + "range":"1.0.2.0/23", + "region":"FJ", + "region_name":"Fujian", + "timezone":"Asia/Shanghai", + "Type": "Ipinfo_Location_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.343 PM", + "city":"Frankston East", + "country":"AU", + "country_name":"Australia", + "geoname_id":"2166143", + "latitude":"-38.03357", + "longitude":"145.305", + "postal_code":"3805", + "radius":"1000", + "range":"1.0.4.0/22", + "region":"VIC", + "region_name":"Victoria", + "timezone":"Australia/Melbourne", + "Type": "Ipinfo_Location_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.344 PM", + "city":"Shenzhen", + "country":"CN", + "country_name":"China", + "geoname_id":"1795565", + "latitude":"22.54554", + "longitude":"114.0683", + "postal_code":"", + "radius":"1000", + "range":"1.0.8.0/21", + "region":"GD", + "region_name":"Guangdong", + "timezone":"Asia/Shanghai", + "Type": "Ipinfo_Location_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.345 PM", + "city":"Tokyo", + "country":"JP", + "country_name":"Japan", + "geoname_id":"1850147", + "latitude":"35.6895", + "longitude":"139.69171", + "postal_code":"101-8656", + "radius":"5000", + "range":"1.0.16.0/29", + "region":"13", + "region_name":"Tokyo", + "timezone":"Asia/Tokyo", + "Type": "Ipinfo_Location_extended_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_Privacy_extended_CL.json b/Sample Data/Custom/Ipinfo_Privacy_extended_CL.json new file mode 100644 index 00000000000..07653a5a34b --- /dev/null +++ b/Sample Data/Custom/Ipinfo_Privacy_extended_CL.json @@ -0,0 +1,110 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "anycast":"true", + "census":"false", + "census_port":"", + "device_activity":"true", + "hosting":"true", + "network":"1.0.0.0-1.0.0.0", + "proxy":"", + "range":"1.0.0.0/32", + "relay":"", + "tor":"", + "vpn":"", + "vpn_config":"false", + "vpn_name":"", + "whois":"false", + "Type": "Ipinfo_Privacy_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.341 PM", + "anycast":"true", + "census":"true", + "census_port":"500", + "device_activity":"true", + "hosting":"true", + "network":"1.0.0.1-1.0.0.2", + "proxy":"", + "range":"1.0.0.1/32", + "relay":"", + "tor":"", + "vpn":"true", + "vpn_config":"false", + "vpn_name":"", + "whois":"false", + "Type": "Ipinfo_Privacy_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.342 PM", + "anycast":"true", + "census":"true", + "census_port":"500", + "device_activity":"true", + "hosting":"true", + "network":"1.0.0.1-1.0.0.2", + "proxy":"", + "range":"1.0.0.2/32", + "relay":"", + "tor":"", + "vpn":"true", + "vpn_config":"false", + "vpn_name":"", + "whois":"false", + "Type": "Ipinfo_Privacy_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.343 PM", + "anycast":"true", + "census":"false", + "census_port":"", + "device_activity":"true", + "hosting":"true", + "network":"1.0.0.3-1.0.0.255", + "proxy":"", + "range":"1.0.0.3/32", + "relay":"", + "tor":"", + "vpn":"", + "vpn_config":"false", + "vpn_name":"", + "whois":"false", + "Type": "Ipinfo_Privacy_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.344 PM", + "anycast":"true", + "census":"false", + "census_port":"", + "device_activity":"true", + "hosting":"true", + "network":"1.0.0.3-1.0.0.255", + "proxy":"", + "range":"1.0.0.4/30", + "relay":"", + "tor":"", + "vpn":"", + "vpn_config":"false", + "vpn_name":"", + "whois":"false", + "Type": "Ipinfo_Privacy_extended_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.345 PM", + "anycast":"true", + "census":"false", + "census_port":"", + "device_activity":"true", + "hosting":"true", + "network":"1.0.0.3-1.0.0.255", + "proxy":"", + "range":"1.0.0.8/29", + "relay":"", + "tor":"", + "vpn":"", + "vpn_config":"false", + "vpn_name":"", + "whois":"false", + "Type": "Ipinfo_Privacy_extended_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_RIRWHOIS_CL.json b/Sample Data/Custom/Ipinfo_RIRWHOIS_CL.json new file mode 100644 index 00000000000..0bb34a2cef3 --- /dev/null +++ b/Sample Data/Custom/Ipinfo_RIRWHOIS_CL.json @@ -0,0 +1,146 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "177.101.205.240/30", + "whois_id": "PAC154", + "name": "THIAGO GARCIA CARO FORNECEDOR DE ACESSO A INTERNET", + "country": "BR", + "status": "REALLOCATED", + "tech": "GLBRA31", + "maintainer": "", + "admin": "", + "source": "lacnic", + "whois_domain": "vogeltelecom.com", + "updated": "2019-08-23", + "org": "", + "rdns_domain": "", + "domain": "vogeltelecom.com", + "geoloc": "", + "org_address": "", + "asn": "AS25933", + "as_name": "Vogel Solu\u00e7\u00f5es em Telecom e Inform\u00e1tica S/A", + "as_domain": "vogeltelecom.com", + "as_type": "isp", + "Type": "Ipinfo_RIRWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "141.95.214.108/30", + "whois_id": "OVH_408693513", + "name": "OVH HOSTING OY", + "country": "FI", + "status": "ASSIGNED PA", + "tech": "OTC15-RIPE", + "maintainer": "OVH-MNT", + "admin": "OTC15-RIPE", + "source": "ripe", + "whois_domain": "ovh.net", + "updated": "2022-03-14", + "org": "ORG-OH6-RIPE", + "rdns_domain": "", + "domain": "ovh.net", + "geoloc": "", + "org_address": "Malminkatu 28\\n00100 Helsinki\\nFinland", + "asn": "AS16276", + "as_name": "OVH SAS", + "as_domain": "ovhcloud.com", + "as_type": "hosting", + "Type": "Ipinfo_RIRWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "31.147.223.128/29", + "whois_id": "CARNET-XVIIIGIMZG", + "name": "XVIII. GIMNAZIJA ZAGREB", + "country": "HR", + "status": "ASSIGNED PA", + "tech": "CIA22-RIPE", + "maintainer": "RIPE-NCC-HM-MNT\\NAS2108-MNT", + "admin": "CIA22-RIPE", + "source": "ripe", + "whois_domain": "carnet.hr", + "updated": "2021-03-11", + "org": "ORG-CAAR1-RIPE", + "rdns_domain": "", + "domain": "carnet.hr", + "geoloc": "", + "org_address": "Josipa Marohnica 5\\n10000\\nZagreb\\nCROATIA", + "asn": "AS2108", + "as_name": "Croatian Academic and Research Network", + "as_domain": "carnet.hr", + "as_type": "education", + "Type": "Ipinfo_RIRWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "79.187.91.160/30", + "whois_id": "CUSTOMER-IDSL-207981", + "name": "static IP", + "country": "PL", + "status": "ASSIGNED PA", + "tech": "TPHT", + "maintainer": "RIPE-NCC-HM-MNT\\NTPNET", + "admin": "TPHT", + "source": "ripe", + "whois_domain": "tpnet.pl", + "updated": "2010-09-25", + "org": "ORG-PT1-RIPE", + "rdns_domain": "", + "domain": "tpnet.pl", + "geoloc": "", + "org_address": "Al. Jerozolimskie 160\\n02-326\\nWarszawa\\nPOLAND", + "asn": "AS5617", + "as_name": "Orange Polska Spolka Akcyjna", + "as_domain": "orange.pl", + "as_type": "isp", + "Type": "Ipinfo_RIRWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "77.60.231.232/29", + "whois_id": "OTS136779", + "name": "Blok Textiel", + "country": "NL", + "status": "ASSIGNED PA", + "tech": "BB3481-RIPE", + "maintainer": "RIPE-NCC-HM-MNT\\NKPN-MNT", + "admin": "BB3481-RIPE", + "source": "ripe", + "whois_domain": "wxs.nl", + "updated": "2007-04-20", + "org": "ORG-KOVN1-RIPE", + "rdns_domain": "", + "domain": "wxs.nl", + "geoloc": "", + "org_address": "Wilhelminakade 123\\n3072AP\\nRotterdam\\nNETHERLANDS", + "asn": "AS1136", + "as_name": "KPN B.V.", + "as_domain": "idstrategy.com.ua", + "as_type": "isp", + "Type": "Ipinfo_RIRWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "209.121.11.128/26", + "whois_id": "TELUS-HSIA-PTCMBC2", + "name": "TELUS-HSIA-PTCMBC02", + "country": "CA", + "status": "REASSIGNMENT", + "tech": "", + "maintainer": "", + "admin": "", + "source": "arin", + "whois_domain": "", + "updated": "2021-10-20", + "org": "C08072763", + "rdns_domain": "telus.net", + "domain": "telus.net", + "geoloc": "", + "org_address": "2990 Gordon Ave, Port Coquitlam, BC, CA", + "asn": "AS852", + "as_name": "TELUS Communications Inc.", + "as_domain": "telus.com", + "as_type": "isp", + "Type": "Ipinfo_RIRWHOIS_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_RWHOIS_CL.json b/Sample Data/Custom/Ipinfo_RWHOIS_CL.json new file mode 100644 index 00000000000..88d2fd76a6e --- /dev/null +++ b/Sample Data/Custom/Ipinfo_RWHOIS_CL.json @@ -0,0 +1,110 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "172.17.68.238", + "whois_id": "NET-154835.172.17.68.238", + "name": "Web-hosting.com", + "whois_desc": "NC-PH-3045 (IPMI)", + "host": "whois.namecheaphosting.com:4321", + "country": "US", + "email": "sanitized@sanitized.com", + "abuse": "", + "domain": "", + "city": "Phoenix", + "street": "3402 East University Drive", + "postal": "85034", + "updated": "2020-12-24", + "imported": "2024-08-01", + "Type": "Ipinfo_RWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "50.120.213.8/30", + "whois_id": "NET-50-120-213-8-30", + "name": "Wyalusing Hotel", + "whois_desc": "50-120-213-8-30", + "host": "rwhois.frontiernet.net:4321", + "country": "US", + "email": "sanitized@sanitized.com", + "abuse": "", + "domain": "wyalusinghotel.com", + "city": "Wyalusing", + "street": "54 Main Street", + "postal": "18853", + "updated": "2012-05-16", + "imported": "2024-07-24", + "Type": "Ipinfo_RWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "208.72.23.225", + "whois_id": "WEBLINE4 208.72.20.0/22", + "name": "Justin Apperson", + "whois_desc": "Justin Apperson-208.72.23.225", + "host": "rwhois.webline-services.com:4321", + "country": "", + "email": "sanitized@sanitized.com", + "abuse": "", + "domain": "", + "city": "", + "street": "8535 Baymeadows Rd", + "postal": "", + "updated": "2017-03-15", + "imported": "2024-08-01", + "Type": "Ipinfo_RWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "206.130.64.200/29", + "whois_id": "NETBLK-VIANET.206.130.64.0/24", + "name": "Vianet Internet Solutions", + "whois_desc": "VIANET-206.130.64.200/29", + "host": "rwhois.vianet.ca:4321", + "country": "CA", + "email": "sanitized@sanitized.com", + "abuse": "", + "domain": "vianet.ca", + "city": "Sudbury", + "street": "128 Larch St.", + "postal": "P3E 5J8", + "updated": "2012-01-11", + "imported": "2024-08-01", + "Type": "Ipinfo_RWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "67.223.118.149", + "whois_id": "NET-227820.67.223.118.149", + "name": "Web-hosting.com", + "whois_desc": "premium100.web-hosting.com", + "host": "whois.namecheaphosting.com:4321", + "country": "US", + "email": "sanitized@sanitized.com", + "abuse": "", + "domain": "", + "city": "Phoenix", + "street": "3402 East University Drive", + "postal": "85034", + "updated": "2022-04-17", + "imported": "2024-08-01", + "Type": "Ipinfo_RWHOIS_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "108.179.233.202", + "whois_id": "NETBLK-BO.108.179.233.202/32", + "name": "structuresgo3d.com", + "whois_desc": "BO-108.179.233.202/32", + "host": "rwhois.websitewelcome.com:4321", + "country": "US", + "email": "sanitized@sanitized.com", + "abuse": "", + "domain": "", + "city": "", + "street": "", + "postal": "", + "updated": "2013-07-17", + "imported": "2024-08-01", + "Type": "Ipinfo_RWHOIS_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_WHOIS_ASN_CL.json b/Sample Data/Custom/Ipinfo_WHOIS_ASN_CL.json new file mode 100644 index 00000000000..ba625c1af10 --- /dev/null +++ b/Sample Data/Custom/Ipinfo_WHOIS_ASN_CL.json @@ -0,0 +1,68 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "AS133315", + "name": "Icontechnext", + "country": "IN", + "org_id": "", + "created": "", + "updated": "2017-03-06", + "source": "apnic", + "Type": "Ipinfo_WHOIS_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "AS60413", + "name": "Innovative solutions Ltd.", + "country": "RU", + "org_id": "ORG-ISL54-RIPE", + "created": "2013-07-23", + "updated": "2019-06-05", + "source": "ripe", + "Type": "Ipinfo_WHOIS_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "AS61670", + "name": "New Net Informatica e Telecomunica\u00e7\u00f5es", + "country": "BR", + "org_id": "", + "created": "2014-09-03", + "updated": "2016-08-24", + "source": "lacnic", + "Type": "Ipinfo_WHOIS_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "AS209758", + "name": "CryptoCentre LLC", + "country": "RU", + "org_id": "ORG-CL530-RIPE", + "created": "2018-12-10", + "updated": "2021-05-19", + "source": "ripe", + "Type": "Ipinfo_WHOIS_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "AS141236", + "name": "Mandalay City Development Committee", + "country": "MM", + "org_id": "", + "created": "2020-10-06", + "updated": "2020-10-06", + "source": "apnic", + "Type": "Ipinfo_WHOIS_ASN_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "AS205501", + "name": "Telus Comunicaciones S.L.", + "country": "ES", + "org_id": "ORG-TCS31-RIPE", + "created": "2017-08-03", + "updated": "2018-09-04", + "source": "ripe", + "Type": "Ipinfo_WHOIS_ASN_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_WHOIS_MNT_CL.json b/Sample Data/Custom/Ipinfo_WHOIS_MNT_CL.json new file mode 100644 index 00000000000..f022a4aaa0d --- /dev/null +++ b/Sample Data/Custom/Ipinfo_WHOIS_MNT_CL.json @@ -0,0 +1,74 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "LIR-RU-REDBYTES-1-MNT", + "name": "", + "admin_id": "DUMY-RIPE", + "tech_id": "", + "org_id": "", + "created": "2022-02-16", + "updated": "2023-02-21", + "source": "ripe", + "Type": "Ipinfo_WHOIS_MNT_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "LIR-RS-RTS-SERBIA-1-MNT", + "name": "", + "admin_id": "DUMY-RIPE", + "tech_id": "", + "org_id": "", + "created": "2021-04-19", + "updated": "2021-04-19", + "source": "ripe", + "Type": "Ipinfo_WHOIS_MNT_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "IR-FOURSE-1-MNT", + "name": "Startup maintainer", + "admin_id": "TS36964-RIPE", + "tech_id": "TS36964-RIPE", + "org_id": "", + "created": "2016-01-26", + "updated": "2016-01-31", + "source": "ripe", + "Type": "Ipinfo_WHOIS_MNT_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "DE-KIS2-1-MNT", + "name": "Startup maintainer", + "admin_id": "WK2587-RIPE", + "tech_id": "WK2587-RIPE", + "org_id": "", + "created": "2018-06-08", + "updated": "2018-06-08", + "source": "ripe", + "Type": "Ipinfo_WHOIS_MNT_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "MNT-DK-IT-FORSYNINGEN-1", + "name": "Startup maintainer", + "admin_id": "AD14826-RIPE", + "tech_id": "AD14826-RIPE", + "org_id": "", + "created": "2018-11-09", + "updated": "2018-11-09", + "source": "ripe", + "Type": "Ipinfo_WHOIS_MNT_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "GENERATED-FXRUAVLTJUMFE2GLJSHZB7C9FLFCVYHX-MNT", + "name": "Auto-generated maintainer", + "admin_id": "AGMT2310-AFRINIC", + "tech_id": "", + "org_id": "", + "created": "", + "updated": "", + "source": "afrinic", + "Type": "Ipinfo_WHOIS_MNT_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_WHOIS_NET_CL.json b/Sample Data/Custom/Ipinfo_WHOIS_NET_CL.json new file mode 100644 index 00000000000..b985b576836 --- /dev/null +++ b/Sample Data/Custom/Ipinfo_WHOIS_NET_CL.json @@ -0,0 +1,110 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "195.99.253.56/29", + "whois_id": "BTNET-CUSTOMER", + "name": "FTIP003328088 IP MASTER", + "country": "GB", + "domain": "bt.net", + "org_id": "ORG-BPIS1-RIPE", + "status": "ASSIGNED PA", + "tech_id": "BY73-RIPE", + "mnt_id": "BTNET-MNT", + "admin_id": "BY73-RIPE", + "abuse_id": "AR13878-RIPE", + "created": "2013-11-11", + "updated": "2023-12-15", + "source": "ripe", + "Type": "Ipinfo_WHOIS_NET_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "76.202.106.56/29", + "whois_id": "SBC-76-202-106-56-29-0704022956", + "name": "JEFFERSON BANK TRUST-070402182948", + "country": "US", + "domain": "", + "org_id": "C01608463", + "status": "REASSIGNMENT", + "tech_id": "", + "mnt_id": "", + "admin_id": "", + "abuse_id": "", + "created": "2007-04-02", + "updated": "2018-01-10", + "source": "arin", + "Type": "Ipinfo_WHOIS_NET_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "201.144.11.0/24", + "whois_id": "DCA", + "name": "Gesti\u00f3n de direccionamiento UniNet", + "country": "MX", + "domain": "uninet.net.mx", + "org_id": "", + "status": "REASSIGNED", + "tech_id": "DCA", + "mnt_id": "", + "admin_id": "", + "abuse_id": "SRU", + "created": "2007-09-15", + "updated": "2012-09-01", + "source": "lacnic", + "Type": "Ipinfo_WHOIS_NET_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "62.207.162.64/29", + "whois_id": "OTS582089", + "name": "Zeeuws Landgoed", + "country": "NL", + "domain": "zeeuwslandgoed.nl", + "org_id": "ORG-KOVN1-RIPE", + "status": "ASSIGNED PA", + "tech_id": "AVO78-RIPE", + "mnt_id": "AS286-MNT", + "admin_id": "AVO78-RIPE", + "abuse_id": "KPN-RIPE", + "created": "2011-12-14", + "updated": "2011-12-14", + "source": "ripe", + "Type": "Ipinfo_WHOIS_NET_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "173.166.203.128/29", + "whois_id": "HILTONGARDENINN", + "name": "HILTON GARDEN INN", + "country": "US", + "domain": "hilton.com", + "org_id": "C04858711", + "status": "REASSIGNMENT", + "tech_id": "", + "mnt_id": "", + "admin_id": "", + "abuse_id": "", + "created": "2014-01-24", + "updated": "2014-01-24", + "source": "arin", + "Type": "Ipinfo_WHOIS_NET_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "range": "83.13.228.104/30", + "whois_id": "CUSTOMER-IDSL-143791", + "name": "static IP", + "country": "PL", + "domain": "tpnet.pl", + "org_id": "ORG-PT1-RIPE", + "status": "ASSIGNED PA", + "tech_id": "TPHT", + "mnt_id": "TPNET", + "admin_id": "TPHT", + "abuse_id": "OPHT", + "created": "2010-09-25", + "updated": "2010-09-25", + "source": "ripe", + "Type": "Ipinfo_WHOIS_NET_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_WHOIS_ORG_CL.json b/Sample Data/Custom/Ipinfo_WHOIS_ORG_CL.json new file mode 100644 index 00000000000..c3fc8a36e36 --- /dev/null +++ b/Sample Data/Custom/Ipinfo_WHOIS_ORG_CL.json @@ -0,0 +1,128 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "C06122127", + "name": "TWINING", + "address": "", + "street": "1572 SANTA ANA AVE\\nFL 1st", + "city": "SACRAMENTO", + "state": "CA", + "postalcode": "95838", + "country": "US", + "admin_id": "", + "tech_id": "", + "abuse_id": "", + "mnt_id": "", + "email": "sanitized@sanitized.com", + "domain": "", + "created": "2016-05-10", + "updated": "2016-05-10", + "source": "arin", + "Type": "Ipinfo_WHOIS_ORG_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "C08143236", + "name": "KIMBERLY CLARK", + "address": "", + "street": "58 PICKETT DISTRICT RD", + "city": "NEW MILFORD", + "state": "CT", + "postalcode": "06776", + "country": "US", + "admin_id": "", + "tech_id": "", + "abuse_id": "", + "mnt_id": "", + "email": "sanitized@sanitized.com", + "domain": "kimberly-clark.com", + "created": "2021-12-22", + "updated": "2021-12-22", + "source": "arin", + "Type": "Ipinfo_WHOIS_ORG_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "C02003329", + "name": "1 Up Software", + "address": "", + "street": "701 Congressional Boulevard\\nSuite 100", + "city": "Carmel", + "state": "IN", + "postalcode": "46032", + "country": "US", + "admin_id": "", + "tech_id": "", + "abuse_id": "", + "mnt_id": "", + "email": "sanitized@sanitized.com", + "domain": "", + "created": "2008-07-31", + "updated": "2011-03-19", + "source": "arin", + "Type": "Ipinfo_WHOIS_ORG_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "C07074935", + "name": "Private Customer - AT&T Internet Services", + "address": "", + "street": "Private Residence", + "city": "Richardson", + "state": "TX", + "postalcode": "75082", + "country": "US", + "admin_id": "", + "tech_id": "", + "abuse_id": "", + "mnt_id": "", + "email": "sanitized@sanitized.com", + "domain": "", + "created": "2018-10-03", + "updated": "2018-10-03", + "source": "arin", + "Type": "Ipinfo_WHOIS_ORG_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "HSS-116", + "name": "HostFlyte Server Solutions", + "address": "", + "street": "PO Box 32", + "city": "Dingwall", + "state": "NS", + "postalcode": "B0C 1G0", + "country": "CA", + "admin_id": "ADMIN6928-ARIN", + "tech_id": "ADMIN6928-ARIN", + "abuse_id": "ABUSE7319-ARIN", + "mnt_id": "ADMIN6928-ARIN", + "email": "sanitized@sanitized.com", + "domain": "hostflyte.com", + "created": "2018-12-13", + "updated": "2019-05-30", + "source": "arin", + "Type": "Ipinfo_WHOIS_ORG_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "C04673155", + "name": "Scott Kostler08162013121202551", + "address": "", + "street": "2701 N. Central Expwy", + "city": "Richardson", + "state": "TX", + "postalcode": "75080", + "country": "US", + "admin_id": "", + "tech_id": "", + "abuse_id": "", + "mnt_id": "", + "email": "sanitized@sanitized.com", + "domain": "", + "created": "2013-08-16", + "updated": "2018-07-19", + "source": "arin", + "Type": "Ipinfo_WHOIS_ORG_CL" + } +] \ No newline at end of file diff --git a/Sample Data/Custom/Ipinfo_WHOIS_POC_CL.json b/Sample Data/Custom/Ipinfo_WHOIS_POC_CL.json new file mode 100644 index 00000000000..799c114116c --- /dev/null +++ b/Sample Data/Custom/Ipinfo_WHOIS_POC_CL.json @@ -0,0 +1,98 @@ +[ + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "DD768-RIPE", + "name": "Domenico D'alessandro", + "mobilephone": "", + "officephone": "+39 040365722", + "fax": "+39 040365722", + "address": "NORDSPEDIZIONIERI DUE SRL\\nVIA MALASPINA 34\\n34100 TRIESTE\\nItaly", + "country": "IT", + "email": "sanitized@sanitized.com", + "abuse_email": "sanitized@sanitized.com", + "created": "2003-08-01", + "updated": "2020-06-03", + "source": "ripe", + "Type": "Ipinfo_WHOIS_POC_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "LC15352-RIPE", + "name": "Luc Chamourin", + "mobilephone": "", + "officephone": "+33 326540758", + "fax": "", + "address": "8 RUE DU PRE BREDA 51530 MARDEUIL FRA", + "country": "FR", + "email": "sanitized@sanitized.com", + "abuse_email": "sanitized@sanitized.com", + "created": "2022-03-21", + "updated": "2022-03-21", + "source": "ripe", + "Type": "Ipinfo_WHOIS_POC_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "FS5029-RIPE", + "name": "FEDERICA SABA", + "mobilephone": "", + "officephone": "+39356689950", + "fax": "+39356689950", + "address": "SABA FEDERICA\\nVIA DANIMARCA 14\\n09100 CAGLIARI\\nItaly", + "country": "IT", + "email": "sanitized@sanitized.com", + "abuse_email": "sanitized@sanitized.com", + "created": "2007-02-08", + "updated": "2020-06-03", + "source": "ripe", + "Type": "Ipinfo_WHOIS_POC_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "AN33089-RIPE", + "name": "ADRIANO NUZZO", + "mobilephone": "", + "officephone": "+39789598399", + "fax": "+39789598399", + "address": "A.L. SRL\\nVIA COREA 10\\n07026 OLBIA\\nItaly", + "country": "IT", + "email": "sanitized@sanitized.com", + "abuse_email": "sanitized@sanitized.com", + "created": "2023-01-09", + "updated": "2023-01-09", + "source": "ripe", + "Type": "Ipinfo_WHOIS_POC_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "CCMFR7", + "name": "Caio C\u00e9sar Martins Fraporti", + "mobilephone": "", + "officephone": "", + "fax": "", + "address": "", + "country": "", + "email": "sanitized@sanitized.com", + "abuse_email": "sanitized@sanitized.com", + "created": "2016-07-05", + "updated": "2016-07-05", + "source": "lacnic", + "Type": "Ipinfo_WHOIS_POC_CL" + }, + { + "TimeGenerated": "4/30/2024, 4:01:17.340 PM", + "whois_id": "WYH19-TW", + "name": "Ming Her Wu", + "mobilephone": "", + "officephone": "", + "fax": "", + "address": "Jeng Sheng Lee\\nNo.91, Fu Shing Rd., Yunlin\\nYunlin Taiwan", + "country": "TW", + "email": "sanitized@sanitized.com", + "abuse_email": "sanitized@sanitized.com", + "created": "2001-08-07", + "updated": "2001-08-07", + "source": "twnic", + "Type": "Ipinfo_WHOIS_POC_CL" + } +] \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/constants.py b/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/constants.py new file mode 100644 index 00000000000..0cd1e754099 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/constants.py @@ -0,0 +1,79 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'ASN_DCR_NAME', 'ASN_TABLE_NAME', 'ASN_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'MMDB_NAME', + 'ASN_TABLE_SCHEMA', 'ASN_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +ASN_DCR_NAME = "ipinfo_rule_for_ASN_tables" +ASN_TABLE_NAME = "Ipinfo_ASN_CL" +ASN_STREAM_DECLARATION = "Custom-Ipinfo_ASN_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +MMDB_NAME = "asn.mmdb" + +ASN_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": ASN_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "asn", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "route", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "asn_type", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{ASN_TABLE_NAME}", + "name": ASN_TABLE_NAME, +} + +ASN_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "asn", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "domain", "type": "string"}, + {"name": "route", "type": "string"}, + {"name": "asn_type", "type": "string"}, + {"name": "range", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/function.json b/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/main.py b/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/main.py new file mode 100644 index 00000000000..b1a570664a6 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/main.py @@ -0,0 +1,84 @@ +import logging +import time +import maxminddb +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo ASN timer trigger function executed.") + + def upload_data_to_ASN_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + mmdb_file_path = "/tmp/asn.mmdb" + reader = maxminddb.open_database(mmdb_file_path) + chunk_size = 10000 + data_chunk = [] + logging.info("Uploading Standard ASN Data.\n") + for ip, ip_data in reader: + result = {} + result["asn"] = ip_data.get("asn", "") + result["name"] = ip_data.get("name", "") + result["domain"] = ip_data.get("domain", "") + result["route"] = ip_data.get("route", "") + result["asn_type"] = ip_data.get("type", "") + result["range"] = str(ip) + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + reader.close() + logging.info("Standard ASN Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(ASN_TABLE_NAME, ASN_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(ASN_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + ASN_dcr_immutableid, ASN_stream_name = check_and_create_data_collection_rules( + access_token, + ASN_DCR_NAME, + ASN_STREAM_DECLARATION, + ASN_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_ASN_table(dce_endpoint, ASN_dcr_immutableid, ASN_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/utils.py b/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/utils.py new file mode 100644 index 00000000000..53397fd0b2b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/AzureFunctionIPinfoASN/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{MMDB_NAME}?token=" + logging.info(f"Downloading '{MMDB_NAME}'...") + file_path = os.path.join("/tmp/", MMDB_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{MMDB_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{MMDB_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{MMDB_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/ASN/IPifnoASNConn.zip b/Solutions/IPinfo/Data Connectors/ASN/IPifnoASNConn.zip new file mode 100644 index 00000000000..8063658e640 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/ASN/IPifnoASNConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/ASN/IPinfo_ASN_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/ASN/IPinfo_ASN_API_AzureFunctionApp.json new file mode 100644 index 00000000000..b7cd31c15fb --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/IPinfo_ASN_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoASNDataConnector", + "title": "IPinfo ASN Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_ASN datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "ASN Data", + "legend": "Ipinfo_ASN_CL", + "baseQuery": "Ipinfo_ASN_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_ASN_CL", + "query": "Ipinfo_ASN_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_ASN_CL", + "lastDataReceivedQuery": "Ipinfo_ASN_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_ASN_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-ASN-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-ASN-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/ASN/azuredeploy_Connector_IPinfo_ASN_AzureFunction.json b/Solutions/IPinfo/Data Connectors/ASN/azuredeploy_Connector_IPinfo_ASN_AzureFunction.json new file mode 100644 index 00000000000..c0c1895d233 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/azuredeploy_Connector_IPinfo_ASN_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo ASN", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-ASN-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/ASN/host.json b/Solutions/IPinfo/Data Connectors/ASN/host.json new file mode 100644 index 00000000000..e53d8df199b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "00:10:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/ASN/proxies.json b/Solutions/IPinfo/Data Connectors/ASN/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/ASN/requirements.txt b/Solutions/IPinfo/Data Connectors/ASN/requirements.txt new file mode 100644 index 00000000000..facd64c3bf9 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/ASN/requirements.txt @@ -0,0 +1,9 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests +maxminddb diff --git a/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/constants.py b/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/constants.py new file mode 100644 index 00000000000..c3354b13af8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/constants.py @@ -0,0 +1,81 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'ABUSE_DCR_NAME', 'ABUSE_TABLE_NAME', 'ABUSE_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'MMDB_NAME', + 'ABUSE_TABLE_SCHEMA', 'ABUSE_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +ABUSE_DCR_NAME = "ipinfo_rule_for_abuse_tables" +ABUSE_TABLE_NAME = "Ipinfo_Abuse_CL" +ABUSE_STREAM_DECLARATION = "Custom-Ipinfo_Abuse_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +MMDB_NAME = "standard_abuse.mmdb" + +ABUSE_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": ABUSE_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "email", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "address", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "phone", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "network", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{ABUSE_TABLE_NAME}", + "name": ABUSE_TABLE_NAME, +} + +ABUSE_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "name", "type": "string"}, + {"name": "email", "type": "string"}, + {"name": "address", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "phone", "type": "string"}, + {"name": "network", "type": "string"}, + {"name": "range", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/function.json b/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/main.py b/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/main.py new file mode 100644 index 00000000000..aa73eac9cf9 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/main.py @@ -0,0 +1,85 @@ +import logging +import time +import maxminddb +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo Abuse timer trigger function executed.") + + def upload_data_to_abuse_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + mmdb_file_path = "/tmp/standard_abuse.mmdb" + reader = maxminddb.open_database(mmdb_file_path) + chunk_size = 10000 + data_chunk = [] + logging.info("Uploading Standard Abuse Data.\n") + for ip, ip_data in reader: + result = {} + result["name"] = ip_data.get("name", "") + result["email"] = ip_data.get("email", "") + result["address"] = ip_data.get("address", "") + result["country"] = ip_data.get("country", "") + result["phone"] = ip_data.get("phone", "") + result["network"] = ip_data.get("network", "") + result["range"] = str(ip) + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + reader.close() + logging.info("Standard Abuse Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(ABUSE_TABLE_NAME, ABUSE_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(ABUSE_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + abuse_dcr_immutableid, abuse_stream_name = check_and_create_data_collection_rules( + access_token, + ABUSE_DCR_NAME, + ABUSE_STREAM_DECLARATION, + ABUSE_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_abuse_table(dce_endpoint, abuse_dcr_immutableid, abuse_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/utils.py b/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/utils.py new file mode 100644 index 00000000000..53397fd0b2b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/AzureFunctionIPinfoAbuse/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{MMDB_NAME}?token=" + logging.info(f"Downloading '{MMDB_NAME}'...") + file_path = os.path.join("/tmp/", MMDB_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{MMDB_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{MMDB_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{MMDB_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/Abuse/IPinfoAbuseConn.zip b/Solutions/IPinfo/Data Connectors/Abuse/IPinfoAbuseConn.zip new file mode 100644 index 00000000000..358d2d79d31 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/Abuse/IPinfoAbuseConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/Abuse/IPinfo_Abuse_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/Abuse/IPinfo_Abuse_API_AzureFunctionApp.json new file mode 100644 index 00000000000..41d96727678 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/IPinfo_Abuse_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoAbuseDataConnector", + "title": "IPinfo Abuse Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_abuse datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "Abuse Data", + "legend": "Ipinfo_Abuse_CL", + "baseQuery": "Ipinfo_Abuse_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Abuse_CL", + "query": "Ipinfo_Abuse_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Abuse_CL", + "lastDataReceivedQuery": "Ipinfo_Abuse_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Abuse_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Abuse-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-Abuse-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Abuse/azuredeploy_Connector_IPinfo_Abuse_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Abuse/azuredeploy_Connector_IPinfo_Abuse_AzureFunction.json new file mode 100644 index 00000000000..0ac8c54ba57 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/azuredeploy_Connector_IPinfo_Abuse_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo Abuse", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-Abuse-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Abuse/host.json b/Solutions/IPinfo/Data Connectors/Abuse/host.json new file mode 100644 index 00000000000..eed246c12bf --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "03:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/Abuse/proxies.json b/Solutions/IPinfo/Data Connectors/Abuse/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/Abuse/requirements.txt b/Solutions/IPinfo/Data Connectors/Abuse/requirements.txt new file mode 100644 index 00000000000..facd64c3bf9 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Abuse/requirements.txt @@ -0,0 +1,9 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests +maxminddb diff --git a/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/constants.py b/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/constants.py new file mode 100644 index 00000000000..73a5c6ff04c --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/constants.py @@ -0,0 +1,79 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'CARRIER_DCR_NAME', 'CARRIER_TABLE_NAME', 'CARRIER_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'MMDB_NAME', + 'CARRIER_TABLE_SCHEMA', 'CARRIER_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +CARRIER_DCR_NAME = "ipinfo_rule_for_carrier_tables" +CARRIER_TABLE_NAME = "Ipinfo_Carrier_CL" +CARRIER_STREAM_DECLARATION = "Custom-Ipinfo_Carrier_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +MMDB_NAME = "carrier.mmdb" + +CARRIER_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": CARRIER_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "carrier", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "mcc", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "mnc", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "cc", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "network", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{CARRIER_TABLE_NAME}", + "name": CARRIER_TABLE_NAME, +} + +CARRIER_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "carrier", "type": "string"}, + {"name": "mcc", "type": "string"}, + {"name": "mnc", "type": "string"}, + {"name": "cc", "type": "string"}, + {"name": "network", "type": "string"}, + {"name": "range", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/function.json b/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/main.py b/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/main.py new file mode 100644 index 00000000000..55bd7133f81 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/main.py @@ -0,0 +1,84 @@ +import logging +import time +import maxminddb +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo Carrier timer trigger function executed.") + + def upload_data_to_carrier_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + mmdb_file_path = "/tmp/carrier.mmdb" + reader = maxminddb.open_database(mmdb_file_path) + chunk_size = 10000 + data_chunk = [] + logging.info("Uploading Standard Carrier Data.\n") + for ip, ip_data in reader: + result = {} + result["carrier"] = ip_data.get("carrier", "") + result["mcc"] = ip_data.get("mcc", "") + result["mnc"] = ip_data.get("mnc", "") + result["cc"] = ip_data.get("cc", "") + result["network"] = ip_data.get("network", "") + result["range"] = str(ip) + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + reader.close() + logging.info("Standard Carrier Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(CARRIER_TABLE_NAME, CARRIER_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(CARRIER_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + carrier_dcr_immutableid, carrier_stream_name = check_and_create_data_collection_rules( + access_token, + CARRIER_DCR_NAME, + CARRIER_STREAM_DECLARATION, + CARRIER_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_carrier_table(dce_endpoint, carrier_dcr_immutableid, carrier_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/utils.py b/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/utils.py new file mode 100644 index 00000000000..53397fd0b2b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/AzureFunctionIPinfoCarrier/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{MMDB_NAME}?token=" + logging.info(f"Downloading '{MMDB_NAME}'...") + file_path = os.path.join("/tmp/", MMDB_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{MMDB_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{MMDB_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{MMDB_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/Carrier/IPinfoCarrierConn.zip b/Solutions/IPinfo/Data Connectors/Carrier/IPinfoCarrierConn.zip new file mode 100644 index 00000000000..0d240d4bee1 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/Carrier/IPinfoCarrierConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/Carrier/IPinfo_Carrier_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/Carrier/IPinfo_Carrier_API_AzureFunctionApp.json new file mode 100644 index 00000000000..c707653669c --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/IPinfo_Carrier_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoCarrierDataConnector", + "title": "IPinfo Carrier Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_carrier datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "Carrier Data", + "legend": "Ipinfo_Carrier_CL", + "baseQuery": "Ipinfo_Carrier_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Carrier_CL", + "query": "Ipinfo_Carrier_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Carrier_CL", + "lastDataReceivedQuery": "Ipinfo_Carrier_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Carrier_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Carrier-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-Carrier-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Carrier/azuredeploy_Connector_IPinfo_Carrier_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Carrier/azuredeploy_Connector_IPinfo_Carrier_AzureFunction.json new file mode 100644 index 00000000000..c9e05939bb3 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/azuredeploy_Connector_IPinfo_Carrier_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo Carrier", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-Carrier-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Carrier/host.json b/Solutions/IPinfo/Data Connectors/Carrier/host.json new file mode 100644 index 00000000000..e53d8df199b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "00:10:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/Carrier/proxies.json b/Solutions/IPinfo/Data Connectors/Carrier/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/Carrier/requirements.txt b/Solutions/IPinfo/Data Connectors/Carrier/requirements.txt new file mode 100644 index 00000000000..facd64c3bf9 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Carrier/requirements.txt @@ -0,0 +1,9 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests +maxminddb diff --git a/Solutions/IPinfo/Data Connectors/Company/azuredeploy_Connector_IPinfo_Company_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Company/azuredeploy_Connector_IPinfo_Company_AzureFunction.json index cf67667aa15..0e075d92183 100644 --- a/Solutions/IPinfo/Data Connectors/Company/azuredeploy_Connector_IPinfo_Company_AzureFunction.json +++ b/Solutions/IPinfo/Data Connectors/Company/azuredeploy_Connector_IPinfo_Company_AzureFunction.json @@ -32,11 +32,11 @@ }, "RETENTION_IN_DAYS": { "type": "string", - "defaultValue": "IPinfo Token" + "defaultValue": "10" }, "TOTAL_RETENTION_IN_DAYS": { "type": "string", - "defaultValue": "IPinfo Token" + "defaultValue": "30" }, "SCHEDULE": { "type": "string", @@ -114,7 +114,7 @@ "reserved": true, "maximumElasticWorkerCount": 20, "siteConfig": { - "linuxFxVersion": "Python|3.10" + "linuxFxVersion": "python|3.11" } } }, diff --git a/Solutions/IPinfo/Data Connectors/Company/host.json b/Solutions/IPinfo/Data Connectors/Company/host.json index f65ed7c548a..e65054b4c51 100644 --- a/Solutions/IPinfo/Data Connectors/Company/host.json +++ b/Solutions/IPinfo/Data Connectors/Company/host.json @@ -11,6 +11,6 @@ }, "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[4.*, 5.0.0)" + "version": "[3.*, 4.0.0)" } } diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/constants.py b/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/constants.py new file mode 100644 index 00000000000..21cf652e1bb --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/constants.py @@ -0,0 +1,83 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', + 'WORKSPACE_NAME', 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', + 'DATA_COLLECTION_ENDPOINT_NAME', 'COUNTRY_DCR_NAME', 'COUNTRY_TABLE_NAME', + 'COUNTRY_STREAM_DECLARATION', 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', + 'MMDB_NAME', 'COUNTRY_TABLE_SCHEMA', 'COUNTRY_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +COUNTRY_DCR_NAME = "ipinfo_rule_for_country_table" +COUNTRY_TABLE_NAME = "Ipinfo_Country_CL" +COUNTRY_STREAM_DECLARATION = "Custom-Ipinfo_Country_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data/free" +MMDB_NAME = "country_asn.mmdb" + +COUNTRY_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": COUNTRY_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "as_domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "as_name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "asn", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "continent", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "continent_name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country_name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{COUNTRY_TABLE_NAME}", + "name": COUNTRY_TABLE_NAME, +} + +COUNTRY_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "as_domain", "type": "string"}, + {"name": "as_name", "type": "string"}, + {"name": "asn", "type": "string"}, + {"name": "continent", "type": "string"}, + {"name": "continent_name", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "country_name", "type": "string"}, + {"name": "range", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/function.json b/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/function.json new file mode 100644 index 00000000000..b0027ce9e99 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/function.json @@ -0,0 +1,12 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] + } + \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/main.py b/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/main.py new file mode 100644 index 00000000000..f1f6c8201f6 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/main.py @@ -0,0 +1,86 @@ +import logging +import maxminddb +import time +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo Country ASN timer trigger function executed.") + + def upload_data_to_country_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + mmdb_file_path = "/tmp/country_asn.mmdb" + reader = maxminddb.open_database(mmdb_file_path) + chunk_size = 10000 + data_chunk = [] + logging.info("Uploading Country ASN Data.\n") + for ip, ip_data in reader: + result = {} + result["as_domain"] = ip_data.get("as_domain", "") + result["as_name"] = ip_data.get("as_name", "") + result["asn"] = ip_data.get("asn", "") + result["continent"] = ip_data.get("continent", "") + result["continent_name"] = ip_data.get("continent_name", "") + result["country"] = ip_data.get("country", "") + result["country_name"] = ip_data.get("country_name", "") + result["range"] = str(ip) + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + reader.close() + logging.info("Country ASN Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(COUNTRY_TABLE_NAME, COUNTRY_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(COUNTRY_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + country_dcr_immutableid, country_stream_name = check_and_create_data_collection_rules( + access_token, + COUNTRY_DCR_NAME, + COUNTRY_STREAM_DECLARATION, + COUNTRY_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_country_table(dce_endpoint, country_dcr_immutableid, country_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/utils.py b/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/utils.py new file mode 100644 index 00000000000..53397fd0b2b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/AzureFunctionIPinfoCountryASN/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{MMDB_NAME}?token=" + logging.info(f"Downloading '{MMDB_NAME}'...") + file_path = os.path.join("/tmp/", MMDB_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{MMDB_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{MMDB_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{MMDB_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/IPinfoCountryConn.zip b/Solutions/IPinfo/Data Connectors/Country ASN/IPinfoCountryConn.zip new file mode 100644 index 00000000000..ac65c0d40a4 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/Country ASN/IPinfoCountryConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/IPinfo_Country_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/Country ASN/IPinfo_Country_API_AzureFunctionApp.json new file mode 100644 index 00000000000..74878b4bac3 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/IPinfo_Country_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoCountryDataConnector", + "title": "IPinfo Country ASN Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download country_asn datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "Country Data", + "legend": "Ipinfo_Country_CL", + "baseQuery": "Ipinfo_Country_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Country_CL", + "query": "Ipinfo_Country_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Country_CL", + "lastDataReceivedQuery": "Ipinfo_Country_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Country_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Country-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-Country-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/azuredeploy_Connector_IPinfo_Country_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Country ASN/azuredeploy_Connector_IPinfo_Country_AzureFunction.json new file mode 100644 index 00000000000..a3325ea3929 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/azuredeploy_Connector_IPinfo_Country_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo Country", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-Country-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/host.json b/Solutions/IPinfo/Data Connectors/Country ASN/host.json new file mode 100644 index 00000000000..2b8b7bb60bd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "01:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/proxies.json b/Solutions/IPinfo/Data Connectors/Country ASN/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/Country ASN/requirements.txt b/Solutions/IPinfo/Data Connectors/Country ASN/requirements.txt new file mode 100644 index 00000000000..facd64c3bf9 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Country ASN/requirements.txt @@ -0,0 +1,9 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests +maxminddb diff --git a/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/constants.py b/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/constants.py new file mode 100644 index 00000000000..bf2ff3f2306 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/constants.py @@ -0,0 +1,73 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'DOMAIN_DCR_NAME', 'DOMAIN_TABLE_NAME', 'DOMAIN_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'MMDB_NAME', + 'DOMAIN_TABLE_SCHEMA', 'DOMAIN_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +DOMAIN_DCR_NAME = "ipinfo_rule_for_domain_tables" +DOMAIN_TABLE_NAME = "Ipinfo_Domain_CL" +DOMAIN_STREAM_DECLARATION = "Custom-Ipinfo_Domain_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +MMDB_NAME = "standard_ip_hosted_domains.mmdb" + +DOMAIN_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": DOMAIN_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "domains", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "total", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{DOMAIN_TABLE_NAME}", + "name": DOMAIN_TABLE_NAME, +} + +DOMAIN_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "domains", "type": "string"}, + {"name": "total", "type": "string"}, + {"name": "range", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/function.json b/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/main.py b/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/main.py new file mode 100644 index 00000000000..1d221f095d0 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/main.py @@ -0,0 +1,81 @@ +import logging +import time +import maxminddb +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo Domain timer trigger function executed.") + + def upload_data_to_domain_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + mmdb_file_path = "/tmp/standard_ip_hosted_domains.mmdb" + reader = maxminddb.open_database(mmdb_file_path) + chunk_size = 10000 + data_chunk = [] + logging.info("Uploading Standard Domain Data.\n") + for ip, ip_data in reader: + result = {} + result["domains"] = ip_data.get("domains", "") + result["total"] = ip_data.get("total", "") + result["range"] = str(ip) + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + reader.close() + logging.info("Standard Domain Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(DOMAIN_TABLE_NAME, DOMAIN_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(DOMAIN_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + domain_dcr_immutableid, domain_stream_name = check_and_create_data_collection_rules( + access_token, + DOMAIN_DCR_NAME, + DOMAIN_STREAM_DECLARATION, + DOMAIN_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_domain_table(dce_endpoint, domain_dcr_immutableid, domain_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/utils.py b/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/utils.py new file mode 100644 index 00000000000..53397fd0b2b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/AzureFunctionIPinfoDomain/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{MMDB_NAME}?token=" + logging.info(f"Downloading '{MMDB_NAME}'...") + file_path = os.path.join("/tmp/", MMDB_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{MMDB_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{MMDB_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{MMDB_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/Domain/IPinfoDomainConn.zip b/Solutions/IPinfo/Data Connectors/Domain/IPinfoDomainConn.zip new file mode 100644 index 00000000000..7b1732da87d Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/Domain/IPinfoDomainConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/Domain/IPinfo_Domain_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/Domain/IPinfo_Domain_API_AzureFunctionApp.json new file mode 100644 index 00000000000..ef38b69e7e7 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/IPinfo_Domain_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoDomainDataConnector", + "title": "IPinfo Domain Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_domain datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "Domain Data", + "legend": "Ipinfo_Domain_CL", + "baseQuery": "Ipinfo_Domain_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Domain_CL", + "query": "Ipinfo_Domain_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Domain_CL", + "lastDataReceivedQuery": "Ipinfo_Domain_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Domain_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Domain-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-Domain-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Domain/azuredeploy_Connector_IPinfo_Domain_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Domain/azuredeploy_Connector_IPinfo_Domain_AzureFunction.json new file mode 100644 index 00000000000..fb5715e0df4 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/azuredeploy_Connector_IPinfo_Domain_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo Domain", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-Domain-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Domain/host.json b/Solutions/IPinfo/Data Connectors/Domain/host.json new file mode 100644 index 00000000000..eed246c12bf --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "03:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/Domain/proxies.json b/Solutions/IPinfo/Data Connectors/Domain/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/Domain/requirements.txt b/Solutions/IPinfo/Data Connectors/Domain/requirements.txt new file mode 100644 index 00000000000..facd64c3bf9 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Domain/requirements.txt @@ -0,0 +1,9 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests +maxminddb diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/constants.py b/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/constants.py new file mode 100644 index 00000000000..b4f0d70d17f --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/constants.py @@ -0,0 +1,91 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', + 'WORKSPACE_NAME', 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', + 'DATA_COLLECTION_ENDPOINT_NAME', 'LOCATION_EXTENDED_DCR_NAME', 'LOCATION_EXTENDED_TABLE_NAME', + 'LOCATION_EXTENDED_STREAM_DECLARATION', 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', + 'MMDB_NAME', 'LOCATION_EXTENDED_TABLE_SCHEMA', 'LOCATION_EXTENDED_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +LOCATION_EXTENDED_DCR_NAME = "ipinfo_rule_for_location_extended_table" +LOCATION_EXTENDED_TABLE_NAME = "Ipinfo_Location_extended_CL" +LOCATION_EXTENDED_STREAM_DECLARATION = "Custom-Ipinfo_Location_extended_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +MMDB_NAME = "location_extended_v2.mmdb" + +LOCATION_EXTENDED_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": LOCATION_EXTENDED_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "city", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country_name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "latitude", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "longitude", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "postal_code", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "radius", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "region_name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "region", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "timezone", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "geoname_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{LOCATION_EXTENDED_TABLE_NAME}", + "name": LOCATION_EXTENDED_TABLE_NAME, +} + +LOCATION_EXTENDED_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "city", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "country_name", "type": "string"}, + {"name": "latitude", "type": "string"}, + {"name": "longitude", "type": "string"}, + {"name": "postal_code", "type": "string"}, + {"name": "radius", "type": "string"}, + {"name": "region_name", "type": "string"}, + {"name": "region", "type": "string"}, + {"name": "timezone", "type": "string"}, + {"name": "geoname_id", "type": "string"}, + {"name": "range", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/function.json b/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/function.json new file mode 100644 index 00000000000..b0027ce9e99 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/function.json @@ -0,0 +1,12 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] + } + \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/main.py b/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/main.py new file mode 100644 index 00000000000..e568c9659a6 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/main.py @@ -0,0 +1,90 @@ +import logging +import time +import maxminddb +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo Iplocation Extended timer trigger function executed.") + + def upload_data_to_location_extended_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + mmdb_file_path = "/tmp/location_extended_v2.mmdb" + reader = maxminddb.open_database(mmdb_file_path) + chunk_size = 10000 + data_chunk = [] + logging.info("Uploading Standard Location Extended Data.\n") + for ip, ip_data in reader: + result = {} + result["city"] = ip_data.get("city", "") + result["country"] = ip_data.get("country", "") + result["country_name"] = ip_data.get("country_name", "") + result["latitude"] = ip_data.get("latitude", "") + result["longitude"] = ip_data.get("longitude", "") + result["postal_code"] = ip_data.get("postal_code", "") + result["radius"] = ip_data.get("radius", "") + result["region_name"] = ip_data.get("region_name", "") + result["region"] = ip_data.get("region", "") + result["timezone"] = ip_data.get("timezone", "") + result["geoname_id"] = ip_data.get("geoname_id", "") + result["range"] = str(ip) + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + reader.close() + logging.info("Standard Location Extended Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(LOCATION_EXTENDED_TABLE_NAME, LOCATION_EXTENDED_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(LOCATION_EXTENDED_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + location_extended_dcr_immutableid, location_extended_stream_name = check_and_create_data_collection_rules( + access_token, + LOCATION_EXTENDED_DCR_NAME, + LOCATION_EXTENDED_STREAM_DECLARATION, + LOCATION_EXTENDED_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_location_extended_table(dce_endpoint, location_extended_dcr_immutableid, location_extended_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/utils.py b/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/utils.py new file mode 100644 index 00000000000..53397fd0b2b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/AzureFunctionIPinfoIplocationExtended/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{MMDB_NAME}?token=" + logging.info(f"Downloading '{MMDB_NAME}'...") + file_path = os.path.join("/tmp/", MMDB_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{MMDB_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{MMDB_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{MMDB_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/IPinfoIplocationExtendedConn.zip b/Solutions/IPinfo/Data Connectors/Iplocation Extended/IPinfoIplocationExtendedConn.zip new file mode 100644 index 00000000000..a2559aec5b1 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/Iplocation Extended/IPinfoIplocationExtendedConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/IPinfo_Iplocation_Extended_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/Iplocation Extended/IPinfo_Iplocation_Extended_API_AzureFunctionApp.json new file mode 100644 index 00000000000..348522b272d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/IPinfo_Iplocation_Extended_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoIplocationExtendedDataConnector", + "title": "IPinfo Iplocation Extended Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_location_extended datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "Location Extended Data", + "legend": "Ipinfo_Location_extended_CL", + "baseQuery": "Ipinfo_Location_extended_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Location_extended_CL", + "query": "Ipinfo_Location_extended_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Location_extended_CL", + "lastDataReceivedQuery": "Ipinfo_Location_extended_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Location_extended_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Iplocation-Extended-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-Iplocation-Extended-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/azuredeploy_Connector_IPinfo_Iplocation_Extended_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Iplocation Extended/azuredeploy_Connector_IPinfo_Iplocation_Extended_AzureFunction.json new file mode 100644 index 00000000000..e65c0a718bb --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/azuredeploy_Connector_IPinfo_Iplocation_Extended_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo Iplocation Extended", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-Iplocation-extended-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/host.json b/Solutions/IPinfo/Data Connectors/Iplocation Extended/host.json new file mode 100644 index 00000000000..cf39731eec6 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "15:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/proxies.json b/Solutions/IPinfo/Data Connectors/Iplocation Extended/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/Iplocation Extended/requirements.txt b/Solutions/IPinfo/Data Connectors/Iplocation Extended/requirements.txt new file mode 100644 index 00000000000..facd64c3bf9 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Iplocation Extended/requirements.txt @@ -0,0 +1,9 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests +maxminddb diff --git a/Solutions/IPinfo/Data Connectors/Iplocation/azuredeploy_Connector_IPinfo_Iplocation_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Iplocation/azuredeploy_Connector_IPinfo_Iplocation_AzureFunction.json index 7855097241e..0124b50069b 100644 --- a/Solutions/IPinfo/Data Connectors/Iplocation/azuredeploy_Connector_IPinfo_Iplocation_AzureFunction.json +++ b/Solutions/IPinfo/Data Connectors/Iplocation/azuredeploy_Connector_IPinfo_Iplocation_AzureFunction.json @@ -32,11 +32,11 @@ }, "RETENTION_IN_DAYS": { "type": "string", - "defaultValue": "IPinfo Token" + "defaultValue": "10" }, "TOTAL_RETENTION_IN_DAYS": { "type": "string", - "defaultValue": "IPinfo Token" + "defaultValue": "30" }, "SCHEDULE": { "type": "string", @@ -114,7 +114,7 @@ "reserved": true, "maximumElasticWorkerCount": 20, "siteConfig": { - "linuxFxVersion": "Python|3.10" + "linuxFxVersion": "python|3.11" } } }, diff --git a/Solutions/IPinfo/Data Connectors/Iplocation/host.json b/Solutions/IPinfo/Data Connectors/Iplocation/host.json index 57c55bc2913..cf39731eec6 100644 --- a/Solutions/IPinfo/Data Connectors/Iplocation/host.json +++ b/Solutions/IPinfo/Data Connectors/Iplocation/host.json @@ -11,6 +11,6 @@ }, "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[4.*, 5.0.0)" + "version": "[3.*, 4.0.0)" } } diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/constants.py b/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/constants.py new file mode 100644 index 00000000000..1dca4096ef5 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/constants.py @@ -0,0 +1,95 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'PRIVACY_EXTENDED_DCR_NAME', 'PRIVACY_EXTENDED_TABLE_NAME', 'PRIVACY_EXTENDED_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'MMDB_NAME', + 'PRIVACY_EXTENDED_TABLE_SCHEMA', 'PRIVACY_EXTENDED_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +PRIVACY_EXTENDED_DCR_NAME = "ipinfo_rule_for_privacy_extended_tables" +PRIVACY_EXTENDED_TABLE_NAME = "Ipinfo_Privacy_extended_CL" +PRIVACY_EXTENDED_STREAM_DECLARATION = "Custom-Ipinfo_Privacy_extended_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +MMDB_NAME = "privacy_extended.mmdb" + +PRIVACY_EXTENDED_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": PRIVACY_EXTENDED_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "anycast", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "census", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "census_port", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "device_activity", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "hosting", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "network", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "proxy", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "relay", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "tor", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "vpn", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "vpn_config", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "vpn_name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{PRIVACY_EXTENDED_TABLE_NAME}", + "name": PRIVACY_EXTENDED_TABLE_NAME, +} + +PRIVACY_EXTENDED_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "anycast", "type": "string"}, + {"name": "census", "type": "string"}, + {"name": "census_port", "type": "string"}, + {"name": "device_activity", "type": "string"}, + {"name": "hosting", "type": "string"}, + {"name": "network", "type": "string"}, + {"name": "proxy", "type": "string"}, + {"name": "relay", "type": "string"}, + {"name": "tor", "type": "string"}, + {"name": "vpn", "type": "string"}, + {"name": "vpn_config", "type": "string"}, + {"name": "vpn_name", "type": "string"}, + {"name": "whois", "type": "string"}, + {"name": "range", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/function.json b/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/main.py b/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/main.py new file mode 100644 index 00000000000..2aa838c3c41 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/main.py @@ -0,0 +1,92 @@ +import logging +import time +import maxminddb +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo Privacy Extended timer trigger function executed.") + + def upload_data_to_privacy_extended_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + mmdb_file_path = "/tmp/privacy_extended.mmdb" + reader = maxminddb.open_database(mmdb_file_path) + chunk_size = 10000 + data_chunk = [] + logging.info("Uploading Standard Privacy Extended Data.\n") + for ip, ip_data in reader: + result = {} + result["anycast"] = ip_data.get("anycast", "") + result["census"] = ip_data.get("census", "") + result["census_port"] = ip_data.get("census_port", "") + result["device_activity"] = ip_data.get("device_activity", "") + result["hosting"] = ip_data.get("hosting", "") + result["network"] = ip_data.get("network", "") + result["proxy"] = ip_data.get("proxy", "") + result["relay"] = ip_data.get("relay", "") + result["tor"] = ip_data.get("tor", "") + result["vpn"] = ip_data.get("vpn", "") + result["vpn_config"] = ip_data.get("vpn_config", "") + result["vpn_name"] = ip_data.get("vpn_name", "") + result["whois"] = ip_data.get("whois", "") + result["range"] = str(ip) + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + reader.close() + logging.info("Standard Privacy Extended Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(PRIVACY_EXTENDED_TABLE_NAME, PRIVACY_EXTENDED_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(PRIVACY_EXTENDED_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + privacy_extended_dcr_immutableid, privacy_extended_stream_name = check_and_create_data_collection_rules( + access_token, + PRIVACY_EXTENDED_DCR_NAME, + PRIVACY_EXTENDED_STREAM_DECLARATION, + PRIVACY_EXTENDED_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_privacy_extended_table(dce_endpoint, privacy_extended_dcr_immutableid, privacy_extended_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/utils.py b/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/utils.py new file mode 100644 index 00000000000..53397fd0b2b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/AzureFunctionIPinfoPrivacyExtended/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{MMDB_NAME}?token=" + logging.info(f"Downloading '{MMDB_NAME}'...") + file_path = os.path.join("/tmp/", MMDB_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{MMDB_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{MMDB_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{MMDB_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/IPinfoPrivacyExtendedConn.zip b/Solutions/IPinfo/Data Connectors/Privacy Extended/IPinfoPrivacyExtendedConn.zip new file mode 100644 index 00000000000..5beab5aa1bf Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/Privacy Extended/IPinfoPrivacyExtendedConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/IPinfo_Privacy_Extended_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/Privacy Extended/IPinfo_Privacy_Extended_API_AzureFunctionApp.json new file mode 100644 index 00000000000..96c6ddad7c7 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/IPinfo_Privacy_Extended_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoPrivacyExtendedDataConnector", + "title": "IPinfo Privacy Extended Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_privacy datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "Privacy Extended Data", + "legend": "Ipinfo_Privacy_extended_CL", + "baseQuery": "Ipinfo_Privacy_extended_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Privacy_extended_CL", + "query": "Ipinfo_Privacy_extended_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Privacy_extended_CL", + "lastDataReceivedQuery": "Ipinfo_Privacy_extended_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Privacy_extended_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Privacy-Extended-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-Privacy-Extended-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/azuredeploy_Connector_IPinfo_Privacy_Extended_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Privacy Extended/azuredeploy_Connector_IPinfo_Privacy_Extended_AzureFunction.json new file mode 100644 index 00000000000..66bfd6a7ad2 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/azuredeploy_Connector_IPinfo_Privacy_Extended_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo Privacy Extended", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-Privacy-Extended-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/host.json b/Solutions/IPinfo/Data Connectors/Privacy Extended/host.json new file mode 100644 index 00000000000..cf39731eec6 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "15:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/proxies.json b/Solutions/IPinfo/Data Connectors/Privacy Extended/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/Privacy Extended/requirements.txt b/Solutions/IPinfo/Data Connectors/Privacy Extended/requirements.txt new file mode 100644 index 00000000000..facd64c3bf9 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/Privacy Extended/requirements.txt @@ -0,0 +1,9 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests +maxminddb diff --git a/Solutions/IPinfo/Data Connectors/Privacy/azuredeploy_Connector_IPinfo_Privacy_AzureFunction.json b/Solutions/IPinfo/Data Connectors/Privacy/azuredeploy_Connector_IPinfo_Privacy_AzureFunction.json index 157e54dd8c0..90920e8b1cc 100644 --- a/Solutions/IPinfo/Data Connectors/Privacy/azuredeploy_Connector_IPinfo_Privacy_AzureFunction.json +++ b/Solutions/IPinfo/Data Connectors/Privacy/azuredeploy_Connector_IPinfo_Privacy_AzureFunction.json @@ -32,11 +32,11 @@ }, "RETENTION_IN_DAYS": { "type": "string", - "defaultValue": "IPinfo Token" + "defaultValue": "10" }, "TOTAL_RETENTION_IN_DAYS": { "type": "string", - "defaultValue": "IPinfo Token" + "defaultValue": "30" }, "SCHEDULE": { "type": "string", @@ -114,7 +114,7 @@ "reserved": true, "maximumElasticWorkerCount": 20, "siteConfig": { - "linuxFxVersion": "Python|3.10" + "linuxFxVersion": "python|3.11" } } }, diff --git a/Solutions/IPinfo/Data Connectors/Privacy/host.json b/Solutions/IPinfo/Data Connectors/Privacy/host.json index f65ed7c548a..e65054b4c51 100644 --- a/Solutions/IPinfo/Data Connectors/Privacy/host.json +++ b/Solutions/IPinfo/Data Connectors/Privacy/host.json @@ -11,6 +11,6 @@ }, "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[4.*, 5.0.0)" + "version": "[3.*, 4.0.0)" } } diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/constants.py b/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/constants.py new file mode 100644 index 00000000000..5eb00a52831 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/constants.py @@ -0,0 +1,107 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'RIRWHOIS_DCR_NAME', 'RIRWHOIS_TABLE_NAME', 'RIRWHOIS_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'CSV_NAME', + 'RIRWHOIS_TABLE_SCHEMA', 'RIRWHOIS_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +RIRWHOIS_DCR_NAME = "ipinfo_rule_for_RIRWHOIS_tables" +RIRWHOIS_TABLE_NAME = "Ipinfo_RIRWHOIS_CL" +RIRWHOIS_STREAM_DECLARATION = "Custom-Ipinfo_RIRWHOIS_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +CSV_NAME = "rir.csv.gz" + +RIRWHOIS_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": RIRWHOIS_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "status", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "tech", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "maintainer", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "admin", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "source", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "updated", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "org", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "rdns_domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "geoloc", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "org_address", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "asn", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "as_name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "as_domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "as_type", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{RIRWHOIS_TABLE_NAME}", + "name": RIRWHOIS_TABLE_NAME, +} + +RIRWHOIS_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "range", "type": "string"}, + {"name": "whois_id", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "status", "type": "string"}, + {"name": "tech", "type": "string"}, + {"name": "maintainer", "type": "string"}, + {"name": "admin", "type": "string"}, + {"name": "source", "type": "string"}, + {"name": "whois_domain", "type": "string"}, + {"name": "updated", "type": "string"}, + {"name": "org", "type": "string"}, + {"name": "rdns_domain", "type": "string"}, + {"name": "domain", "type": "string"}, + {"name": "geoloc", "type": "string"}, + {"name": "org_address", "type": "string"}, + {"name": "asn", "type": "string"}, + {"name": "as_name", "type": "string"}, + {"name": "as_domain", "type": "string"}, + {"name": "as_type", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/function.json b/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/main.py b/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/main.py new file mode 100644 index 00000000000..bbe98c38a47 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/main.py @@ -0,0 +1,98 @@ +import logging +import time +import csv +import gzip +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo RIRWHOIS timer trigger function executed.") + + def upload_data_to_RIRWHOIS_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + csv_file_path = "/tmp/rir.csv.gz" + chunk_size = 10000 + data_chunk = [] + with gzip.open(csv_file_path, mode='rt') as csvfile: + reader = csv.DictReader(csvfile) + for ip_data in reader: + result = {} + result["whois_id"] = ip_data.get("id", "") + result["name"] = ip_data.get("name", "") + result["country"] = ip_data.get("country", "") + result["status"] = ip_data.get("status", "") + result["tech"] = ip_data.get("tech", "") + result["maintainer"] = ip_data.get("maintainer", "") + result["admin"] = ip_data.get("admin", "") + result["source"] = ip_data.get("source", "") + result["whois_domain"] = ip_data.get("whois_domain", "") + result["updated"] = ip_data.get("updated", "") + result["org"] = ip_data.get("org", "") + result["rdns_domain"] = ip_data.get("rdns_domain", "") + result["domain"] = ip_data.get("domain", "") + result["geoloc"] = ip_data.get("geoloc", "") + result["org_address"] = ip_data.get("org_address", "") + result["asn"] = ip_data.get("asn", "") + result["as_name"] = ip_data.get("as_name", "") + result["as_domain"] = ip_data.get("as_domain", "") + result["as_type"] = ip_data.get("as_type", "") + result["range"] = ip_data.get("range", "") + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("RIRWHOIS Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(RIRWHOIS_TABLE_NAME, RIRWHOIS_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(RIRWHOIS_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + RIRWHOIS_dcr_immutableid, RIRWHOIS_stream_name = check_and_create_data_collection_rules( + access_token, + RIRWHOIS_DCR_NAME, + RIRWHOIS_STREAM_DECLARATION, + RIRWHOIS_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_RIRWHOIS_table(dce_endpoint, RIRWHOIS_dcr_immutableid, RIRWHOIS_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/utils.py b/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/utils.py new file mode 100644 index 00000000000..ff1865193c1 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/AzureFunctionIPinfoRIRWHOIS/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{CSV_NAME}?token=" + logging.info(f"Downloading '{CSV_NAME}'...") + file_path = os.path.join("/tmp/", CSV_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{CSV_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{CSV_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{CSV_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/IPinfoRIRWHOISConn.zip b/Solutions/IPinfo/Data Connectors/RIRWHOIS/IPinfoRIRWHOISConn.zip new file mode 100644 index 00000000000..07c7a004258 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/RIRWHOIS/IPinfoRIRWHOISConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/IPinfo_RIRWHOIS_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/RIRWHOIS/IPinfo_RIRWHOIS_API_AzureFunctionApp.json new file mode 100644 index 00000000000..15d5a286a14 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/IPinfo_RIRWHOIS_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoRIRWHOISDataConnector", + "title": "IPinfo RIRWHOIS Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download RIRWHOIS datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "RIRWHOIS Data", + "legend": "Ipinfo_RIRWHOIS_CL", + "baseQuery": "Ipinfo_RIRWHOIS_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_RIRWHOIS_CL", + "query": "Ipinfo_RIRWHOIS_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_RIRWHOIS_CL", + "lastDataReceivedQuery": "Ipinfo_RIRWHOIS_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_RIRWHOIS_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-RIRWHOIS-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-RIRWHOIS-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/azuredeploy_Connector_IPinfo_RIRWHOIS_AzureFunction.json b/Solutions/IPinfo/Data Connectors/RIRWHOIS/azuredeploy_Connector_IPinfo_RIRWHOIS_AzureFunction.json new file mode 100644 index 00000000000..53806380ea4 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/azuredeploy_Connector_IPinfo_RIRWHOIS_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo RIRWHOIS", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-RIRWHOIS-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/host.json b/Solutions/IPinfo/Data Connectors/RIRWHOIS/host.json new file mode 100644 index 00000000000..eed246c12bf --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "03:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/proxies.json b/Solutions/IPinfo/Data Connectors/RIRWHOIS/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/RIRWHOIS/requirements.txt b/Solutions/IPinfo/Data Connectors/RIRWHOIS/requirements.txt new file mode 100644 index 00000000000..5a811d57a2d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RIRWHOIS/requirements.txt @@ -0,0 +1,8 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/constants.py b/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/constants.py new file mode 100644 index 00000000000..54a665f0a90 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/constants.py @@ -0,0 +1,95 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'RWHOIS_DCR_NAME', 'RWHOIS_TABLE_NAME', 'RWHOIS_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'CSV_NAME', + 'RWHOIS_TABLE_SCHEMA', 'RWHOIS_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +RWHOIS_DCR_NAME = "ipinfo_rule_for_RWHOIS_tables" +RWHOIS_TABLE_NAME = "Ipinfo_RWHOIS_CL" +RWHOIS_STREAM_DECLARATION = "Custom-Ipinfo_RWHOIS_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +CSV_NAME = "rwhois.csv.gz" + +RWHOIS_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": RWHOIS_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_desc", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "host", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "email", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "abuse", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "city", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "street", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "postal", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "updated", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "imported", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{RWHOIS_TABLE_NAME}", + "name": RWHOIS_TABLE_NAME, +} + +RWHOIS_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "range", "type": "string"}, + {"name": "whois_id", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "whois_desc", "type": "string"}, + {"name": "host", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "email", "type": "string"}, + {"name": "abuse", "type": "string"}, + {"name": "domain", "type": "string"}, + {"name": "city", "type": "string"}, + {"name": "street", "type": "string"}, + {"name": "postal", "type": "string"}, + {"name": "updated", "type": "string"}, + {"name": "imported", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/function.json b/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/main.py b/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/main.py new file mode 100644 index 00000000000..8bcdad3b1d7 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/main.py @@ -0,0 +1,93 @@ +import logging +import time +import csv +import gzip +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo RWHOIS timer trigger function executed.") + + def upload_data_to_RWHOIS_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + csv_file_path = "/tmp/rwhois.csv.gz" + chunk_size = 10000 + data_chunk = [] + logging.info("Uploading RWHOIS Data.\n") + with gzip.open(csv_file_path, mode='rt') as csvfile: + reader = csv.DictReader(csvfile) + for ip_data in reader: + result = {} + result["whois_id"] = ip_data.get("id", "") + result["name"] = ip_data.get("name", "") + result["whois_desc"] = ip_data.get("descr", "") + result["host"] = ip_data.get("host", "") + result["country"] = ip_data.get("country", "") + result["email"] = ip_data.get("email", "") + result["abuse"] = ip_data.get("abuse", "") + result["domain"] = ip_data.get("domain", "") + result["city"] = ip_data.get("city", "") + result["street"] = ip_data.get("street", "") + result["postal"] = ip_data.get("postal", "") + result["updated"] = ip_data.get("updated", "") + result["imported"] = ip_data.get("imported", "") + result["range"] = ip_data.get("range", "") + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("RWHOIS Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(RWHOIS_TABLE_NAME, RWHOIS_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(RWHOIS_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + RWHOIS_dcr_immutableid, RWHOIS_stream_name = check_and_create_data_collection_rules( + access_token, + RWHOIS_DCR_NAME, + RWHOIS_STREAM_DECLARATION, + RWHOIS_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_RWHOIS_table(dce_endpoint, RWHOIS_dcr_immutableid, RWHOIS_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/utils.py b/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/utils.py new file mode 100644 index 00000000000..5e210f02b97 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/AzureFunctionIPinfoRWHOIS/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{CSV_NAME}?token=" + logging.info(f"Downloading '{CSV_NAME}'...") + file_path = os.path.join("/tmp/", CSV_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{CSV_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{CSV_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{CSV_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/IPinfoRWHOISConn.zip b/Solutions/IPinfo/Data Connectors/RWHOIS/IPinfoRWHOISConn.zip new file mode 100644 index 00000000000..4a7da01a4e0 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/RWHOIS/IPinfoRWHOISConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/IPinfo_RWHOIS_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/RWHOIS/IPinfo_RWHOIS_API_AzureFunctionApp.json new file mode 100644 index 00000000000..e01d5902da4 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/IPinfo_RWHOIS_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoRWHOISDataConnector", + "title": "IPinfo RWHOIS Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download RWHOIS datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "RWHOIS Data", + "legend": "Ipinfo_RWHOIS_CL", + "baseQuery": "Ipinfo_RWHOIS_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_RWHOIS_CL", + "query": "Ipinfo_RWHOIS_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_RWHOIS_CL", + "lastDataReceivedQuery": "Ipinfo_RWHOIS_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_RWHOIS_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-RWHOIS-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-RWHOIS-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/azuredeploy_Connector_IPinfo_RWHOIS_AzureFunction.json b/Solutions/IPinfo/Data Connectors/RWHOIS/azuredeploy_Connector_IPinfo_RWHOIS_AzureFunction.json new file mode 100644 index 00000000000..fe6a7bb5017 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/azuredeploy_Connector_IPinfo_RWHOIS_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo RWHOIS", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-RWHOIS-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/host.json b/Solutions/IPinfo/Data Connectors/RWHOIS/host.json new file mode 100644 index 00000000000..e53d8df199b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "00:10:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/proxies.json b/Solutions/IPinfo/Data Connectors/RWHOIS/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/RWHOIS/requirements.txt b/Solutions/IPinfo/Data Connectors/RWHOIS/requirements.txt new file mode 100644 index 00000000000..5a811d57a2d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/RWHOIS/requirements.txt @@ -0,0 +1,8 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/constants.py b/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/constants.py new file mode 100644 index 00000000000..665ee34a7da --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/constants.py @@ -0,0 +1,81 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'WHOIS_ASN_DCR_NAME', 'WHOIS_ASN_TABLE_NAME', 'WHOIS_ASN_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'CSV_NAME', + 'WHOIS_ASN_TABLE_SCHEMA', 'WHOIS_ASN_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +WHOIS_ASN_DCR_NAME = "ipinfo_rule_for_WHOIS_ASN_tables" +WHOIS_ASN_TABLE_NAME = "Ipinfo_WHOIS_ASN_CL" +WHOIS_ASN_STREAM_DECLARATION = "Custom-Ipinfo_WHOIS_ASN_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +CSV_NAME = "whois_asn.csv.gz" + +WHOIS_ASN_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": WHOIS_ASN_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "org_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "created", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "updated", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "source", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{WHOIS_ASN_TABLE_NAME}", + "name": WHOIS_ASN_TABLE_NAME, +} + +WHOIS_ASN_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "whois_id", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "org_id", "type": "string"}, + {"name": "created", "type": "string"}, + {"name": "updated", "type": "string"}, + {"name": "source", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/function.json b/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/main.py b/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/main.py new file mode 100644 index 00000000000..ae4db0ec224 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/main.py @@ -0,0 +1,88 @@ +import logging +import time +import csv +import gzip +import sys +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo WHOIS_ASN timer trigger function executed.") + + def upload_data_to_WHOIS_ASN_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + csv_file_path = "/tmp/whois_asn.csv.gz" + chunk_size = 10000 + data_chunk = [] + csv.field_size_limit(sys.maxsize) + logging.info("Uploading WHOIS_ASN Data.\n") + with gzip.open(csv_file_path, mode='rt') as csvfile: + reader = csv.DictReader(csvfile) + for ip_data in reader: + result = {} + result["whois_id"] = ip_data.get("id", "") + result["name"] = ip_data.get("name", "") + result["country"] = ip_data.get("country", "") + result["org_id"] = ip_data.get("org_id", "") + result["created"] = ip_data.get("created", "") + result["updated"] = ip_data.get("updated", "") + result["source"] = ip_data.get("source", "") + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("WHOIS_ASN Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(WHOIS_ASN_TABLE_NAME, WHOIS_ASN_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(WHOIS_ASN_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + WHOIS_ASN_dcr_immutableid, WHOIS_ASN_stream_name = check_and_create_data_collection_rules( + access_token, + WHOIS_ASN_DCR_NAME, + WHOIS_ASN_STREAM_DECLARATION, + WHOIS_ASN_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_WHOIS_ASN_table(dce_endpoint, WHOIS_ASN_dcr_immutableid, WHOIS_ASN_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/utils.py b/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/utils.py new file mode 100644 index 00000000000..f44b4b0a099 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/AzureFunctionIPinfoWHOISASN/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{CSV_NAME}?token=" + logging.info(f"Downloading '{CSV_NAME}'...") + file_path = os.path.join("/tmp/", CSV_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{CSV_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{CSV_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{CSV_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/IPInfoWHOISASNConn.zip b/Solutions/IPinfo/Data Connectors/WHOIS ASN/IPInfoWHOISASNConn.zip new file mode 100644 index 00000000000..4991ff5d0e6 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/WHOIS ASN/IPInfoWHOISASNConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/IPinfo_WHOIS_ASN_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/WHOIS ASN/IPinfo_WHOIS_ASN_API_AzureFunctionApp.json new file mode 100644 index 00000000000..cdf70cf33eb --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/IPinfo_WHOIS_ASN_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoWHOISASNDataConnector", + "title": "IPinfo WHOIS ASN Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download WHOIS_ASN datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "WHOIS_ASN Data", + "legend": "Ipinfo_WHOIS_ASN_CL", + "baseQuery": "Ipinfo_WHOIS_ASN_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_ASN_CL", + "query": "Ipinfo_WHOIS_ASN_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_ASN_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_ASN_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_ASN_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-ASN-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-WHOIS-ASN-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/azuredeploy_Connector_IPinfo_WHOIS_ASN_AzureFunction.json b/Solutions/IPinfo/Data Connectors/WHOIS ASN/azuredeploy_Connector_IPinfo_WHOIS_ASN_AzureFunction.json new file mode 100644 index 00000000000..7cc4abb21e0 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/azuredeploy_Connector_IPinfo_WHOIS_ASN_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo WHOIS ASN", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-WHOIS-ASN-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/host.json b/Solutions/IPinfo/Data Connectors/WHOIS ASN/host.json new file mode 100644 index 00000000000..e53d8df199b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "00:10:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/proxies.json b/Solutions/IPinfo/Data Connectors/WHOIS ASN/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ASN/requirements.txt b/Solutions/IPinfo/Data Connectors/WHOIS ASN/requirements.txt new file mode 100644 index 00000000000..5a811d57a2d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ASN/requirements.txt @@ -0,0 +1,8 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/constants.py b/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/constants.py new file mode 100644 index 00000000000..d085460b631 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/constants.py @@ -0,0 +1,83 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'WHOIS_MNT_DCR_NAME', 'WHOIS_MNT_TABLE_NAME', 'WHOIS_MNT_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'CSV_NAME', + 'WHOIS_MNT_TABLE_SCHEMA', 'WHOIS_MNT_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +WHOIS_MNT_DCR_NAME = "ipinfo_rule_for_WHOIS_MNT_tables" +WHOIS_MNT_TABLE_NAME = "Ipinfo_WHOIS_MNT_CL" +WHOIS_MNT_STREAM_DECLARATION = "Custom-Ipinfo_WHOIS_MNT_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +CSV_NAME = "whois_mnt.csv.gz" + +WHOIS_MNT_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": WHOIS_MNT_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "admin_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "tech_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "org_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "created", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "updated", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "source", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{WHOIS_MNT_TABLE_NAME}", + "name": WHOIS_MNT_TABLE_NAME, +} + +WHOIS_MNT_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "whois_id", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "admin_id", "type": "string"}, + {"name": "tech_id", "type": "string"}, + {"name": "org_id", "type": "string"}, + {"name": "created", "type": "string"}, + {"name": "updated", "type": "string"}, + {"name": "source", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/function.json b/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/main.py b/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/main.py new file mode 100644 index 00000000000..dd924f436a0 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/main.py @@ -0,0 +1,89 @@ +import logging +import time +import csv +import gzip +import sys +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo WHOIS_MNT timer trigger function executed.") + + def upload_data_to_WHOIS_MNT_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + csv_file_path = "/tmp/whois_mnt.csv.gz" + chunk_size = 10000 + data_chunk = [] + csv.field_size_limit(sys.maxsize) + logging.info("Uploading WHOIS_MNT Data.\n") + with gzip.open(csv_file_path, mode='rt') as csvfile: + reader = csv.DictReader(csvfile) + for ip_data in reader: + result = {} + result["whois_id"] = ip_data.get("id", "") + result["name"] = ip_data.get("name", "") + result["admin_id"] = ip_data.get("admin_id", "") + result["tech_id"] = ip_data.get("tech_id", "") + result["org_id"] = ip_data.get("org_id", "") + result["created"] = ip_data.get("created", "") + result["updated"] = ip_data.get("updated", "") + result["source"] = ip_data.get("source", "") + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("WHOIS_MNT Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(WHOIS_MNT_TABLE_NAME, WHOIS_MNT_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(WHOIS_MNT_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + WHOIS_MNT_dcr_immutableid, WHOIS_MNT_stream_name = check_and_create_data_collection_rules( + access_token, + WHOIS_MNT_DCR_NAME, + WHOIS_MNT_STREAM_DECLARATION, + WHOIS_MNT_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_WHOIS_MNT_table(dce_endpoint, WHOIS_MNT_dcr_immutableid, WHOIS_MNT_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/utils.py b/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/utils.py new file mode 100644 index 00000000000..f44b4b0a099 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/AzureFunctionIPinfoWHOISMNT/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{CSV_NAME}?token=" + logging.info(f"Downloading '{CSV_NAME}'...") + file_path = os.path.join("/tmp/", CSV_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{CSV_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{CSV_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{CSV_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/IPinfoWHOISMNTConn.zip b/Solutions/IPinfo/Data Connectors/WHOIS MNT/IPinfoWHOISMNTConn.zip new file mode 100644 index 00000000000..e6bee2db1d2 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/WHOIS MNT/IPinfoWHOISMNTConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/IPinfo_WHOIS_MNT_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/WHOIS MNT/IPinfo_WHOIS_MNT_API_AzureFunctionApp.json new file mode 100644 index 00000000000..f9cf8455a55 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/IPinfo_WHOIS_MNT_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoWHOISMNTDataConnector", + "title": "IPinfo WHOIS MNT Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download WHOIS_MNT datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "WHOIS_MNT Data", + "legend": "Ipinfo_WHOIS_MNT_CL", + "baseQuery": "Ipinfo_WHOIS_MNT_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_MNT_CL", + "query": "Ipinfo_WHOIS_MNT_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_MNT_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_MNT_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_MNT_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-MNT-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-WHOIS-MNT-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/azuredeploy_Connector_IPinfo_WHOIS_MNT_AzureFunction.json b/Solutions/IPinfo/Data Connectors/WHOIS MNT/azuredeploy_Connector_IPinfo_WHOIS_MNT_AzureFunction.json new file mode 100644 index 00000000000..240a02f1c4d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/azuredeploy_Connector_IPinfo_WHOIS_MNT_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo WHOIS MNT", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-WHOIS-MNT-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/host.json b/Solutions/IPinfo/Data Connectors/WHOIS MNT/host.json new file mode 100644 index 00000000000..e53d8df199b --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "00:10:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/proxies.json b/Solutions/IPinfo/Data Connectors/WHOIS MNT/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/WHOIS MNT/requirements.txt b/Solutions/IPinfo/Data Connectors/WHOIS MNT/requirements.txt new file mode 100644 index 00000000000..5a811d57a2d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS MNT/requirements.txt @@ -0,0 +1,8 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/constants.py b/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/constants.py new file mode 100644 index 00000000000..b9222f5201f --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/constants.py @@ -0,0 +1,95 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'WHOIS_NET_DCR_NAME', 'WHOIS_NET_TABLE_NAME', 'WHOIS_NET_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'CSV_NAME', + 'WHOIS_NET_TABLE_SCHEMA', 'WHOIS_NET_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +WHOIS_NET_DCR_NAME = "ipinfo_rule_for_WHOIS_NET_tables" +WHOIS_NET_TABLE_NAME = "Ipinfo_WHOIS_NET_CL" +WHOIS_NET_STREAM_DECLARATION = "Custom-Ipinfo_WHOIS_NET_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +CSV_NAME = "whois_net.csv.gz" + +WHOIS_NET_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": WHOIS_NET_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + {"name": "ip_range", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "org_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "status", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "tech_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "mnt_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "admin_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "abuse_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "created", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "updated", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "source", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{WHOIS_NET_TABLE_NAME}", + "name": WHOIS_NET_TABLE_NAME, +} + +WHOIS_NET_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "range", "type": "string"}, + {"name": "whois_id", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "domain", "type": "string"}, + {"name": "org_id", "type": "string"}, + {"name": "status", "type": "string"}, + {"name": "tech_id", "type": "string"}, + {"name": "mnt_id", "type": "string"}, + {"name": "admin_id", "type": "string"}, + {"name": "abuse_id", "type": "string"}, + {"name": "created", "type": "string"}, + {"name": "updated", "type": "string"}, + {"name": "source", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/function.json b/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/main.py b/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/main.py new file mode 100644 index 00000000000..a447e7d56c5 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/main.py @@ -0,0 +1,95 @@ +import logging +import time +import csv +import gzip +import sys +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo WHOIS_NET timer trigger function executed.") + + def upload_data_to_WHOIS_NET_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + csv_file_path = "/tmp/whois_net.csv.gz" + chunk_size = 10000 + data_chunk = [] + csv.field_size_limit(sys.maxsize) + logging.info("Uploading WHOIS_NET Data.\n") + with gzip.open(csv_file_path, mode='rt') as csvfile: + reader = csv.DictReader(csvfile) + for ip_data in reader: + result = {} + result["whois_id"] = ip_data.get("id", "") + result["name"] = ip_data.get("name", "") + result["country"] = ip_data.get("country", "") + result["domain"] = ip_data.get("domain", "") + result["org_id"] = ip_data.get("org_id", "") + result["status"] = ip_data.get("status", "") + result["tech_id"] = ip_data.get("tech_id", "") + result["mnt_id"] = ip_data.get("mnt_id", "") + result["admin_id"] = ip_data.get("admin_id", "") + result["abuse_id"] = ip_data.get("abuse_id", "") + result["created"] = ip_data.get("created", "") + result["updated"] = ip_data.get("updated", "") + result["source"] = ip_data.get("source", "") + result["range"] = ip_data.get("range", "") + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("WHOIS_NET Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(WHOIS_NET_TABLE_NAME, WHOIS_NET_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(WHOIS_NET_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + WHOIS_NET_dcr_immutableid, WHOIS_NET_stream_name = check_and_create_data_collection_rules( + access_token, + WHOIS_NET_DCR_NAME, + WHOIS_NET_STREAM_DECLARATION, + WHOIS_NET_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_WHOIS_NET_table(dce_endpoint, WHOIS_NET_dcr_immutableid, WHOIS_NET_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/utils.py b/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/utils.py new file mode 100644 index 00000000000..5e210f02b97 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/AzureFunctionIPinfoWHOISNET/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{CSV_NAME}?token=" + logging.info(f"Downloading '{CSV_NAME}'...") + file_path = os.path.join("/tmp/", CSV_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{CSV_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{CSV_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{CSV_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n| project-rename ip_range=range\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/IPinfoWHOISNETConn.zip b/Solutions/IPinfo/Data Connectors/WHOIS NET/IPinfoWHOISNETConn.zip new file mode 100644 index 00000000000..c9f5cf23804 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/WHOIS NET/IPinfoWHOISNETConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/IPinfo_WHOIS_NET_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/WHOIS NET/IPinfo_WHOIS_NET_API_AzureFunctionApp.json new file mode 100644 index 00000000000..dc5e8ec0a48 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/IPinfo_WHOIS_NET_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoWHOISNETDataConnector", + "title": "IPinfo WHOIS NET Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download WHOIS_NET datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "WHOIS_NET Data", + "legend": "Ipinfo_WHOIS_NET_CL", + "baseQuery": "Ipinfo_WHOIS_NET_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_NET_CL", + "query": "Ipinfo_WHOIS_NET_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_NET_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_NET_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_NET_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-NET-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-WHOIS-NET-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/azuredeploy_Connector_IPinfo_WHOIS_NET_AzureFunction.json b/Solutions/IPinfo/Data Connectors/WHOIS NET/azuredeploy_Connector_IPinfo_WHOIS_NET_AzureFunction.json new file mode 100644 index 00000000000..de54faff11f --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/azuredeploy_Connector_IPinfo_WHOIS_NET_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo WHOIS NET", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-WHOIS-NET-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/host.json b/Solutions/IPinfo/Data Connectors/WHOIS NET/host.json new file mode 100644 index 00000000000..eed246c12bf --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "03:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/proxies.json b/Solutions/IPinfo/Data Connectors/WHOIS NET/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/WHOIS NET/requirements.txt b/Solutions/IPinfo/Data Connectors/WHOIS NET/requirements.txt new file mode 100644 index 00000000000..5a811d57a2d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS NET/requirements.txt @@ -0,0 +1,8 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/constants.py b/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/constants.py new file mode 100644 index 00000000000..779ea8f95c1 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/constants.py @@ -0,0 +1,101 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'WHOIS_ORG_DCR_NAME', 'WHOIS_ORG_TABLE_NAME', 'WHOIS_ORG_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'CSV_NAME', + 'WHOIS_ORG_TABLE_SCHEMA', 'WHOIS_ORG_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +WHOIS_ORG_DCR_NAME = "ipinfo_rule_for_WHOIS_ORG_tables" +WHOIS_ORG_TABLE_NAME = "Ipinfo_WHOIS_ORG_CL" +WHOIS_ORG_STREAM_DECLARATION = "Custom-Ipinfo_WHOIS_ORG_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +CSV_NAME = "whois_org.csv.gz" + +WHOIS_ORG_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": WHOIS_ORG_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "address", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "street", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "city", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "state", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "postalcode", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "admin_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "tech_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "abuse_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "mnt_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "email", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "domain", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "created", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "updated", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "source", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{WHOIS_ORG_TABLE_NAME}", + "name": WHOIS_ORG_TABLE_NAME, +} + +WHOIS_ORG_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "whois_id", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "address", "type": "string"}, + {"name": "street", "type": "string"}, + {"name": "city", "type": "string"}, + {"name": "state", "type": "string"}, + {"name": "postalcode", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "admin_id", "type": "string"}, + {"name": "tech_id", "type": "string"}, + {"name": "abuse_id", "type": "string"}, + {"name": "mnt_id", "type": "string"}, + {"name": "email", "type": "string"}, + {"name": "domain", "type": "string"}, + {"name": "created", "type": "string"}, + {"name": "updated", "type": "string"}, + {"name": "source", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/function.json b/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/main.py b/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/main.py new file mode 100644 index 00000000000..0320b190063 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/main.py @@ -0,0 +1,98 @@ +import logging +import time +import csv +import gzip +import sys +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo WHOIS_ORG timer trigger function executed.") + + def upload_data_to_WHOIS_ORG_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + csv_file_path = "/tmp/whois_org.csv.gz" + chunk_size = 10000 + data_chunk = [] + csv.field_size_limit(sys.maxsize) + logging.info("Uploading WHOIS_ORG Data.\n") + with gzip.open(csv_file_path, mode='rt') as csvfile: + reader = csv.DictReader(csvfile) + for ip_data in reader: + result = {} + result["whois_id"] = ip_data.get("id", "") + result["name"] = ip_data.get("name", "") + result["address"] = ip_data.get("address", "") + result["street"] = ip_data.get("street", "") + result["city"] = ip_data.get("city", "") + result["state"] = ip_data.get("state", "") + result["postalcode"] = ip_data.get("postalcode", "") + result["country"] = ip_data.get("country", "") + result["admin_id"] = ip_data.get("admin_id", "") + result["tech_id"] = ip_data.get("tech_id", "") + result["abuse_id"] = ip_data.get("abuse_id", "") + result["mnt_id"] = ip_data.get("mnt_id", "") + result["email"] = ip_data.get("email", "") + result["domain"] = ip_data.get("domain", "") + result["created"] = ip_data.get("created", "") + result["updated"] = ip_data.get("updated", "") + result["source"] = ip_data.get("source", "") + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("WHOIS_ORG Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(WHOIS_ORG_TABLE_NAME, WHOIS_ORG_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(WHOIS_ORG_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + WHOIS_ORG_dcr_immutableid, WHOIS_ORG_stream_name = check_and_create_data_collection_rules( + access_token, + WHOIS_ORG_DCR_NAME, + WHOIS_ORG_STREAM_DECLARATION, + WHOIS_ORG_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_WHOIS_ORG_table(dce_endpoint, WHOIS_ORG_dcr_immutableid, WHOIS_ORG_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/utils.py b/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/utils.py new file mode 100644 index 00000000000..f44b4b0a099 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/AzureFunctionIPinfoWHOISORG/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{CSV_NAME}?token=" + logging.info(f"Downloading '{CSV_NAME}'...") + file_path = os.path.join("/tmp/", CSV_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{CSV_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{CSV_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{CSV_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/IPinfoWHOISORGConn.zip b/Solutions/IPinfo/Data Connectors/WHOIS ORG/IPinfoWHOISORGConn.zip new file mode 100644 index 00000000000..d4c416e4c76 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/WHOIS ORG/IPinfoWHOISORGConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/IPinfo_WHOIS_ORG_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/WHOIS ORG/IPinfo_WHOIS_ORG_API_AzureFunctionApp.json new file mode 100644 index 00000000000..f7c8562669e --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/IPinfo_WHOIS_ORG_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoWHOISORGDataConnector", + "title": "IPinfo WHOIS ORG Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download WHOIS_ORG datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "WHOIS_ORG Data", + "legend": "Ipinfo_WHOIS_ORG_CL", + "baseQuery": "Ipinfo_WHOIS_ORG_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_ORG_CL", + "query": "Ipinfo_WHOIS_ORG_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_ORG_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_ORG_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_ORG_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-ORG-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-WHOIS-ORG-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/azuredeploy_Connector_IPinfo_WHOIS_ORG_AzureFunction.json b/Solutions/IPinfo/Data Connectors/WHOIS ORG/azuredeploy_Connector_IPinfo_WHOIS_ORG_AzureFunction.json new file mode 100644 index 00000000000..8c55a7c040d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/azuredeploy_Connector_IPinfo_WHOIS_ORG_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo WHOIS ORG", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-WHOIS-ORG-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/host.json b/Solutions/IPinfo/Data Connectors/WHOIS ORG/host.json new file mode 100644 index 00000000000..2b8b7bb60bd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "01:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/proxies.json b/Solutions/IPinfo/Data Connectors/WHOIS ORG/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/WHOIS ORG/requirements.txt b/Solutions/IPinfo/Data Connectors/WHOIS ORG/requirements.txt new file mode 100644 index 00000000000..5a811d57a2d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS ORG/requirements.txt @@ -0,0 +1,8 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/constants.py b/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/constants.py new file mode 100644 index 00000000000..4e3d1569798 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/constants.py @@ -0,0 +1,91 @@ +import os + +__all__ = [ + 'RESOURCE_ID', 'IPINFO_TOKEN', 'TENANT_ID', 'CLIENT_ID', 'CLIENT_SECRET', + 'LOCATION', 'SUBCRIPTION_ID', 'RESOURCE_GROUP_NAME', 'WORKSPACE_NAME', + 'RETENTION_IN_DAYS', 'TOTAL_RETENTION_IN_DAYS', 'DATA_COLLECTION_ENDPOINT_NAME', + 'WHOIS_POC_DCR_NAME', 'WHOIS_POC_TABLE_NAME', 'WHOIS_POC_STREAM_DECLARATION', + 'AZURE_SCOPE', 'AZURE_BASE_URL', 'IPINFO_BASE_URL', 'CSV_NAME', + 'WHOIS_POC_TABLE_SCHEMA', 'WHOIS_POC_TABLE_COLUMNS' +] + +# Enviornment Virables +RESOURCE_ID = os.environ["RESOURCE_ID"] +IPINFO_TOKEN = os.environ["IPINFO_TOKEN"] +TENANT_ID = os.environ["TENANT_ID"] +CLIENT_ID = os.environ["CLIENT_ID"] +CLIENT_SECRET = os.environ["CLIENT_SECRET"] +RETENTION_IN_DAYS = os.environ["RETENTION_IN_DAYS"] +TOTAL_RETENTION_IN_DAYS = os.environ["TOTAL_RETENTION_IN_DAYS"] +LOCATION = os.environ["LOCATION"] + +parts = RESOURCE_ID.split("/") +SUBCRIPTION_ID = parts[2] +RESOURCE_GROUP_NAME = parts[4] +WORKSPACE_NAME = parts[8] + +DATA_COLLECTION_ENDPOINT_NAME = "ipinfo-logs-ingestion" +WHOIS_POC_DCR_NAME = "ipinfo_rule_for_WHOIS_POC_tables" +WHOIS_POC_TABLE_NAME = "Ipinfo_WHOIS_POC_CL" +WHOIS_POC_STREAM_DECLARATION = "Custom-Ipinfo_WHOIS_POC_CL" + +AZURE_SCOPE = "https://management.azure.com/.default" +AZURE_BASE_URL = f"https://management.azure.com/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft." +IPINFO_BASE_URL = "https://ipinfo.io/data" +CSV_NAME = "whois_poc.csv.gz" + +WHOIS_POC_TABLE_SCHEMA = { + "properties": { + "totalRetentionInDays": TOTAL_RETENTION_IN_DAYS, + "archiveRetentionInDays": 0, + "plan": "Analytics", + "retentionInDaysAsDefault": True, + "totalRetentionInDaysAsDefault": True, + "schema": { + "tableSubType": "DataCollectionRuleBased", + "name": WHOIS_POC_TABLE_NAME, + "tableType": "CustomLog", + "description": "Range based table", + "columns": [ + {"name": "TimeGenerated", "type": "datetime", "isDefaultDisplay": False, "isHidden": False}, + {"name": "whois_id", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "name", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "mobilephone", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "officephone", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "fax", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "address", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "country", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "email", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "abuse_email", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "created", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "updated", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + {"name": "source", "type": "string", "isDefaultDisplay": False, "isHidden": False}, + ], + "standardColumns": [{"name": "TenantId", "type": "guid", "isDefaultDisplay": False, "isHidden": False}], + "solutions": ["LogManagement"], + "isTroubleshootingAllowed": True, + }, + "provisioningState": "Succeeded", + "retentionInDays": RETENTION_IN_DAYS, + }, + "id": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{WHOIS_POC_TABLE_NAME}", + "name": WHOIS_POC_TABLE_NAME, +} + +WHOIS_POC_TABLE_COLUMNS = { + "columns": [ + {"name": "TimeGenerated", "type": "datetime"}, + {"name": "whois_id", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "mobilephone", "type": "string"}, + {"name": "officephone", "type": "string"}, + {"name": "fax", "type": "string"}, + {"name": "address", "type": "string"}, + {"name": "country", "type": "string"}, + {"name": "email", "type": "string"}, + {"name": "abuse_email", "type": "string"}, + {"name": "created", "type": "string"}, + {"name": "updated", "type": "string"}, + {"name": "source", "type": "string"}, + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/function.json b/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/function.json new file mode 100644 index 00000000000..194890db3dd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "main.py", + "bindings": [ + { + "name": "myTimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "%SCHEDULE%" + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/main.py b/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/main.py new file mode 100644 index 00000000000..0e402f0bfb0 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/main.py @@ -0,0 +1,93 @@ +import logging +import time +import csv +import gzip +import sys +import azure.functions as func +from azure.identity import ClientSecretCredential +from azure.monitor.ingestion import LogsIngestionClient +from .constants import * +from .utils import download_mmdbs +from .utils import check_and_create_data_collection_endpoint +from .utils import check_and_create_table +from .utils import check_and_create_data_collection_rules +from .utils import get_table + +def main(myTimer: func.TimerRequest) -> None: + if myTimer.past_due: + logging.info("The timer is past due!") + + logging.info("Ipinfo WHOIS_POC timer trigger function executed.") + + def upload_data_to_WHOIS_POC_table(dce_endpoint, dcr_immutableid, stream_name): + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + client = LogsIngestionClient(endpoint=dce_endpoint, credential=credential, logging_enable=True) + csv_file_path = "/tmp/whois_poc.csv.gz" + chunk_size = 10000 + data_chunk = [] + csv.field_size_limit(sys.maxsize) + logging.info("Uploading WHOIS_POC Data.\n") + with gzip.open(csv_file_path, mode='rt') as csvfile: + reader = csv.DictReader(csvfile) + for ip_data in reader: + result = {} + result["whois_id"] = ip_data.get("id", "") + result["name"] = ip_data.get("name", "") + result["mobilephone"] = ip_data.get("mobilephone", "") + result["officephone"] = ip_data.get("officephone", "") + result["fax"] = ip_data.get("fax", "") + result["address"] = ip_data.get("address", "") + result["country"] = ip_data.get("country", "") + result["email"] = ip_data.get("email", "") + result["abuse_email"] = ip_data.get("abuse_email", "") + result["created"] = ip_data.get("created", "") + result["updated"] = ip_data.get("updated", "") + result["source"] = ip_data.get("source", "") + data_chunk.append(result) + if len(data_chunk) >= chunk_size: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("Wait for the next schedule run.") + break + data_chunk = [] + if data_chunk: + try: + client.upload(rule_id=dcr_immutableid, stream_name=stream_name, logs=data_chunk) + except Exception as e: + logging.error(f"Upload failed: {e}") + logging.info("WHOIS_POC Data uploading completed.") + + # Function flow starts here; above this line are function definitions + credential = ClientSecretCredential(TENANT_ID, CLIENT_ID, CLIENT_SECRET) + access_token = credential.get_token(AZURE_SCOPE).token + if access_token: + logging.info("\nAccess Token Retrieved\n") + logging.info(access_token) + else: + logging.error("\nFailed to retrieve access token\n") + + download_mmdbs() + dce_endpoint = check_and_create_data_collection_endpoint(DATA_COLLECTION_ENDPOINT_NAME, access_token) + check_and_create_table(WHOIS_POC_TABLE_NAME, WHOIS_POC_TABLE_SCHEMA, access_token) + retries = 3 + while retries > 0: + if get_table(WHOIS_POC_TABLE_NAME, access_token): + logging.info("Waiting for the table to be created properly, creating the data collection rule in 1 minute...") + time.sleep(60) + WHOIS_POC_dcr_immutableid, WHOIS_POC_stream_name = check_and_create_data_collection_rules( + access_token, + WHOIS_POC_DCR_NAME, + WHOIS_POC_STREAM_DECLARATION, + WHOIS_POC_TABLE_COLUMNS, + DATA_COLLECTION_ENDPOINT_NAME, + ) + upload_data_to_WHOIS_POC_table(dce_endpoint, WHOIS_POC_dcr_immutableid, WHOIS_POC_stream_name) + break + else: + logging.info("Table not created yet, retrying in 1 minute...") + time.sleep(60) + retries -= 1 + if retries == 0: + logging.error("Table creation timed out after 3 retries. Data collection rules were not created.") diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/utils.py b/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/utils.py new file mode 100644 index 00000000000..f44b4b0a099 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/AzureFunctionIPinfoWHOISPOC/utils.py @@ -0,0 +1,167 @@ +import requests +import logging +import os +from .constants import * + +def generate_url(resource_type, **kwargs): + url_templates = { + "dataCollectionEndpoint": f"{AZURE_BASE_URL}Insights/dataCollectionEndpoints/{{endpoint_name}}?api-version=2022-06-01", + "dataCollectionRule": f"{AZURE_BASE_URL}Insights/dataCollectionRules/{{rule_name}}?api-version=2022-06-01", + "table": f"{AZURE_BASE_URL}OperationalInsights/workspaces/{WORKSPACE_NAME}/tables/{{table_name}}?api-version=2022-10-01", + } + template = url_templates.get(resource_type) + if template: + return template.format(**kwargs) + return "Invalid resource type" + +def download_with_retry(url, file_path, retries=3): + for attempt in range(retries): + try: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(file_path, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + file.write(chunk) + return True + except Exception as e: + logging.error(f"Attempt {attempt + 1} failed: {e}") + if attempt < retries - 1: + logging.info("Retrying...") + continue + return False + +def download_mmdbs(): + url = f"{IPINFO_BASE_URL}/{CSV_NAME}?token=" + logging.info(f"Downloading '{CSV_NAME}'...") + file_path = os.path.join("/tmp/", CSV_NAME) + if os.path.exists(file_path): + os.remove(file_path) + logging.info(f"Previous file '{CSV_NAME}' deleted.") + success = download_with_retry(url + IPINFO_TOKEN, file_path) + if success: + logging.info(f"File '{CSV_NAME}' downloaded successfully.") + else: + logging.error(f"Failed to download the file '{CSV_NAME}'.") + +def create_data_collection_endpoint(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + payload = {"location": LOCATION, "properties": {"networkAcls": {"publicNetworkAccess": "Enabled"}}} + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info("\nData collection endpoint created successfully.\n") + else: + logging.error(f"Failed to create data collection endpoint. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_data_collection_endpoint_url(data_collection_endpoint_name, access_token): + url = generate_url("dataCollectionEndpoint", endpoint_name=data_collection_endpoint_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + endpoint = data.get("properties", {}).get("logsIngestion", {}).get("endpoint") + if endpoint: + return endpoint + logging.info(f"\nData collection endpoint not exist. Status code: {response.status_code}. Creating ...") + create_data_collection_endpoint(data_collection_endpoint_name, access_token) + return get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + +def check_and_create_data_collection_endpoint(data_collection_endpoint_name, access_token): + endpoint = get_data_collection_endpoint_url(data_collection_endpoint_name, access_token) + logging.info(f"Endpoint: {endpoint}\n") + return endpoint + +def create_table(table_name, schema_payload, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + response = requests.put(url, json=schema_payload, headers=headers) + if response.status_code == 200: + logging.info(f"\n{table_name} Table created successfully.\n") + elif response.status_code == 202: + logging.info(f"\n{table_name} Table creation initiated successfully.\n") + else: + logging.error(f"Failed to create {table_name} Table. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + +def get_table(table_name, access_token): + url = generate_url("table", table_name=table_name) + headers = {"Authorization": f"Bearer {access_token}"} + response = requests.get(url, headers=headers) + if response.status_code == 404: + logging.info(f"\n{table_name} table not exists.\n") + return False + elif response.status_code == 200: + logging.info(f"\n{table_name} table already exists.\n") + return True + else: + logging.error(f"Failed to check {table_name}. Status code: {response.status_code}") + logging.error("Response body: %s", response.text) + return False + +def check_and_create_table(table_name, schema_payload, access_token): + table_status = get_table(table_name, access_token) + if table_status == False: + create_table(table_name, schema_payload, access_token) + +def get_data_collection_rule(access_token, data_collection_rule_name): + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + headers = {"Authorization": "Bearer " + access_token} + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json() + immutableId = data["properties"]["immutableId"] + streamDeclarations = list(data["properties"]["streamDeclarations"].keys())[0] + return immutableId, streamDeclarations + + logging.info(f"{data_collection_rule_name} Data Rule endpoint not exist. Status code:{response.status_code}") + return None, None + +def create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint): + headers = {"Authorization": "Bearer " + access_token, "Content-Type": "application/json"} + url = generate_url("dataCollectionRule", rule_name=data_collection_rule_name) + payload = { + "properties": { + "dataCollectionEndpointId": f"/subscriptions/{SUBCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP_NAME}/providers/Microsoft.Insights/dataCollectionEndpoints/{endpoint}", + "streamDeclarations": {stream_declaration: {"columns": columns["columns"]}}, + "dataSources": {}, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": f"/subscriptions/{SUBCRIPTION_ID}/resourcegroups/{RESOURCE_GROUP_NAME}/providers/microsoft.operationalinsights/workspaces/{WORKSPACE_NAME}", + "name": WORKSPACE_NAME, + } + ] + }, + "dataFlows": [ + { + "streams": [stream_declaration], + "destinations": [WORKSPACE_NAME], + "transformKql": "source\n| extend TimeGenerated = now()\n", + "outputStream": stream_declaration, + } + ], + }, + "location": LOCATION, + } + response = requests.put(url, json=payload, headers=headers) + if response.status_code == 200: + logging.info(f"\nData collection Rule for {data_collection_rule_name} created successfully.\n") + else: + logging.error( + f"Failed to create data collection Rule for {data_collection_rule_name}. Status code: {response.status_code}" + + ) + logging.error("Response body: %s", response.text) + +def check_and_create_data_collection_rules( + access_token, data_collection_rule_name, stream_declaration, columns, endpoint +): + dcr_immutableid, stream_name = get_data_collection_rule(access_token, data_collection_rule_name) + if dcr_immutableid is not None and stream_name is not None: + logging.info(f"\nData collection Rule `{data_collection_rule_name}` already exists.") + return dcr_immutableid, stream_name + logging.info(f"\nData collection Rule for {data_collection_rule_name} doesn't exist. Creating...") + create_data_collection_rule(access_token, data_collection_rule_name, stream_declaration, columns, endpoint) + return get_data_collection_rule(access_token, data_collection_rule_name) diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/IPinfoWHOISPOCConn.zip b/Solutions/IPinfo/Data Connectors/WHOIS POC/IPinfoWHOISPOCConn.zip new file mode 100644 index 00000000000..c835e727117 Binary files /dev/null and b/Solutions/IPinfo/Data Connectors/WHOIS POC/IPinfoWHOISPOCConn.zip differ diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/IPinfo_WHOIS_POC_API_AzureFunctionApp.json b/Solutions/IPinfo/Data Connectors/WHOIS POC/IPinfo_WHOIS_POC_API_AzureFunctionApp.json new file mode 100644 index 00000000000..2dc9233cd5a --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/IPinfo_WHOIS_POC_API_AzureFunctionApp.json @@ -0,0 +1,114 @@ +{ + "id": "IPinfoWHOISPOCDataConnector", + "title": "IPinfo WHOIS POC Data Connector", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download WHOIS_POC datasets and insert it into custom log table in Microsoft Sentinel", + "graphQueries": [ + { + "metricName": "WHOIS_POC Data", + "legend": "Ipinfo_WHOIS_POC_CL", + "baseQuery": "Ipinfo_WHOIS_POC_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_POC_CL", + "query": "Ipinfo_WHOIS_POC_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_POC_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_POC_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_POC_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-POC-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-Ipinfo-WHOIS-POC-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/azuredeploy_Connector_IPinfo_WHOIS_POC_AzureFunction.json b/Solutions/IPinfo/Data Connectors/WHOIS POC/azuredeploy_Connector_IPinfo_WHOIS_POC_AzureFunction.json new file mode 100644 index 00000000000..e8d952a10d7 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/azuredeploy_Connector_IPinfo_WHOIS_POC_AzureFunction.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "FunctionName": { + "defaultValue": "IPinfo WHOIS POC", + "minLength": 1, + "maxLength": 11, + "type": "string" + }, + "RESOURCE_ID": { + "type": "string", + "defaultValue": "Resouce ID", + "metadata": { + "description": "Use 'Log Analytic Workspace-->Properties' blade having 'Resource ID' property value. This is a fully qualified resourceId which is in format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + } + }, + "TENANT_ID": { + "type": "string", + "defaultValue": "Tenant ID" + }, + "CLIENT_ID": { + "type": "string", + "defaultValue": "Client ID" + }, + "CLIENT_SECRET": { + "type": "securestring" + }, + "IPINFO_TOKEN": { + "type": "string", + "defaultValue": "IPinfo Token" + }, + "RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "10" + }, + "TOTAL_RETENTION_IN_DAYS": { + "type": "string", + "defaultValue": "30" + }, + "SCHEDULE": { + "type": "string", + "defaultValue": "0 30 9 * * *" + }, + "LOCATION": { + "type": "string" + } + }, + "variables": { + "FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]", + "StorageSuffix": "[environment().suffixes.storage]" + }, + "resources": [ + { + "type": "Microsoft.Insights/components", + "apiVersion": "2020-02-02", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "kind": "web", + "properties": { + "Application_Type": "web", + "ApplicationId": "[variables('FunctionName')]", + "WorkspaceResourceId": "[parameters('RESOURCE_ID')]" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2023-04-01", + "name": "[tolower(variables('FunctionName'))]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "StorageV2", + "properties": { + "networkAcls": { + "bypass": "AzureServices", + "defaultAction": "Allow" + }, + "supportsHttpsTrafficOnly": true, + "encryption": { + "services": { + "file": { + "keyType": "Account", + "enabled": true + }, + "blob": { + "keyType": "Account", + "enabled": true + } + }, + "keySource": "Microsoft.Storage" + } + } + }, + + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "sku": { + "name": "EP2", + "tier": "ElasticPremium", + "family": "EP" + }, + "kind": "elastic", + "properties": { + "name": "[variables('FunctionName')]", + "targetWorkerCount": 1, + "targetWorkerSizeId": 3, + "reserved": true, + "maximumElasticWorkerCount": 20, + "siteConfig": { + "linuxFxVersion": "python|3.11" + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "properties": { + "deleteRetentionPolicy": { + "enabled": false + } + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]" + ], + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + } + }, + + { + "type": "Microsoft.Web/sites", + "apiVersion": "2023-01-01", + "name": "[variables('FunctionName')]", + "location": "[parameters('LOCATION')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]", + "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "[resourceId('Microsoft.Insights/components', variables('FunctionName'))]" + ], + "kind": "functionapp", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "name": "[variables('FunctionName')]", + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('FunctionName'))]", + "httpsOnly": true, + "clientAffinityEnabled": true, + "alwaysOn": true + }, + "resources": [ + { + "apiVersion": "2023-01-01", + "type": "config", + "name": "appsettings", + "dependsOn": [ + "[resourceId('Microsoft.Web/sites/', variables('FunctionName'))]" + ], + "properties": { + "FUNCTIONS_EXTENSION_VERSION": "~4", + "FUNCTIONS_WORKER_RUNTIME": "python", + "APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2020-02-02').InstrumentationKey]", + "APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2020-02-02').ConnectionString]", + "AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=', listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2023-04-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]", + "WEBSITE_CONTENTSHARE": "[toLower(variables('FunctionName'))]", + "RESOURCE_ID": "[parameters('RESOURCE_ID')]", + "TENANT_ID": "[parameters('TENANT_ID')]", + "CLIENT_ID": "[parameters('CLIENT_ID')]", + "CLIENT_SECRET": "[parameters('CLIENT_SECRET')]", + "IPINFO_TOKEN": "[parameters('IPINFO_TOKEN')]", + "RETENTION_IN_DAYS": "[parameters('RETENTION_IN_DAYS')]", + "TOTAL_RETENTION_IN_DAYS": "[parameters('TOTAL_RETENTION_IN_DAYS')]", + "SCHEDULE": "[parameters('SCHEDULE')]", + "LOCATION": "[parameters('LOCATION')]", + "WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-IPinfo-WHOIS-POC-functionapp" + } + } + ] + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/blobServices/containers", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "publicAccess": "None" + } + }, + + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2023-04-01", + "name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]", + "[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]" + ], + "properties": { + "shareQuota": 5120 + } + } + ] +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/host.json b/Solutions/IPinfo/Data Connectors/WHOIS POC/host.json new file mode 100644 index 00000000000..2b8b7bb60bd --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/host.json @@ -0,0 +1,16 @@ +{ + "version": "2.0", + "functionTimeout": "01:00:00", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[3.*, 4.0.0)" + } +} diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/proxies.json b/Solutions/IPinfo/Data Connectors/WHOIS POC/proxies.json new file mode 100644 index 00000000000..13ca746ccf8 --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/proxies.json @@ -0,0 +1,4 @@ +{ + "$schema": "http://json.schemastore.org/proxies", + "proxies": {} +} \ No newline at end of file diff --git a/Solutions/IPinfo/Data Connectors/WHOIS POC/requirements.txt b/Solutions/IPinfo/Data Connectors/WHOIS POC/requirements.txt new file mode 100644 index 00000000000..5a811d57a2d --- /dev/null +++ b/Solutions/IPinfo/Data Connectors/WHOIS POC/requirements.txt @@ -0,0 +1,8 @@ +# DO NOT include azure-functions-worker in this file +# The Python Worker is managed by Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions +azure.identity +azure.monitor.ingestion +requests diff --git a/Solutions/IPinfo/Data/Solution_IPinfo.json b/Solutions/IPinfo/Data/Solution_IPinfo.json index e6a0130b45d..cede55b4ac4 100644 --- a/Solutions/IPinfo/Data/Solution_IPinfo.json +++ b/Solutions/IPinfo/Data/Solution_IPinfo.json @@ -13,13 +13,27 @@ "Hunting Queries": [], "Data Connectors": [ "Solutions/IPinfo/Data Connectors/Company/IPinfo_Company_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/Company/IPinfo_Country_API_AzureFunctionApp.json", "Solutions/IPinfo/Data Connectors/Iplocation/IPinfo_Iplocation_API_AzureFunctionApp.json", - "Solutions/IPinfo/Data Connectors/Privacy/IPinfo_Privacy_API_AzureFunctionApp.json" + "Solutions/IPinfo/Data Connectors/Privacy/IPinfo_Privacy_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/Abuse/IPinfo_Abuse_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/ASN/IPinfo_ASN_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/Carrier/IPinfo_Carrier_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/Domain/IPinfo_Domain_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/Iplocation Extended/IPinfo_Iplocation_Extended_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/Privacy Extended/IPinfo_Privacy_Extended_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/RWHOIS/IPinfo_RWHOIS_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/RIRWHOIS/IPinfo_RIRWHOIS_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/WHOIS ASN/IPinfo_WHOIS_ASN_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/WHOIS MNT/IPinfo_WHOIS_MNT_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/WHOIS NET/IPinfo_WHOIS_NET_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/WHOIS ORG/IPinfo_WHOIS_ORG_API_AzureFunctionApp.json", + "Solutions/IPinfo/Data Connectors/WHOIS POC/IPinfo_WHOIS_POC_API_AzureFunctionApp.json" ], "Watchlists": [], "WatchlistDescription": [], "BasePath": "C:\\GitHub\\Azure-Sentinel\\Solutions\\IPinfo", - "Version": "3.0.0", + "Version": "3.0.1", "Metadata": "SolutionMetadata.json", "TemplateSpec": true, "Is1PConnector": false diff --git a/Solutions/IPinfo/Package/3.0.1.zip b/Solutions/IPinfo/Package/3.0.1.zip new file mode 100644 index 00000000000..c81ccbf9203 Binary files /dev/null and b/Solutions/IPinfo/Package/3.0.1.zip differ diff --git a/Solutions/IPinfo/Package/mainTemplate.json b/Solutions/IPinfo/Package/mainTemplate.json index 13cad67609a..ea03e1903fe 100644 --- a/Solutions/IPinfo/Package/mainTemplate.json +++ b/Solutions/IPinfo/Package/mainTemplate.json @@ -27,7 +27,7 @@ }, "variables": { "_solutionName": "IPinfo", - "_solutionVersion": "3.0.0", + "_solutionVersion": "3.1.0", "solutionId": "idbllc1687537942583.microsoft-sentinel-solution-ipinfo-ipintelligence", "_solutionId": "[variables('solutionId')]", "uiConfigId1": "IPinfoCompanyDataConnector", @@ -57,9 +57,137 @@ "dataConnectorTemplateSpecName3": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId3'))))]", "dataConnectorVersion3": "1.0.0", "_dataConnectorcontentProductId3": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId3'),'-', variables('dataConnectorVersion3'))))]", + "uiConfigId4": "IPinfoAbuseDataConnector", + "_uiConfigId4": "[variables('uiConfigId4')]", + "dataConnectorContentId4": "IPinfoAbuseDataConnector", + "_dataConnectorContentId4": "[variables('dataConnectorContentId4')]", + "dataConnectorId4": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId4'))]", + "_dataConnectorId4": "[variables('dataConnectorId4')]", + "dataConnectorTemplateSpecName4": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId4'))))]", + "dataConnectorVersion4": "1.0.0", + "_dataConnectorcontentProductId4": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId4'),'-', variables('dataConnectorVersion4'))))]", + "uiConfigId5": "IPinfoASNDataConnector", + "_uiConfigId5": "[variables('uiConfigId5')]", + "dataConnectorContentId5": "IPinfoASNDataConnector", + "_dataConnectorContentId5": "[variables('dataConnectorContentId5')]", + "dataConnectorId5": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId5'))]", + "_dataConnectorId5": "[variables('dataConnectorId5')]", + "dataConnectorTemplateSpecName5": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId5'))))]", + "dataConnectorVersion5": "1.0.0", + "_dataConnectorcontentProductId5": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId5'),'-', variables('dataConnectorVersion5'))))]", + "uiConfigId6": "IPinfoCarrierDataConnector", + "_uiConfigId6": "[variables('uiConfigId6')]", + "dataConnectorContentId6": "IPinfoCarrierDataConnector", + "_dataConnectorContentId6": "[variables('dataConnectorContentId6')]", + "dataConnectorId6": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId6'))]", + "_dataConnectorId6": "[variables('dataConnectorId6')]", + "dataConnectorTemplateSpecName6": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId6'))))]", + "dataConnectorVersion6": "1.0.0", + "_dataConnectorcontentProductId6": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId6'),'-', variables('dataConnectorVersion6'))))]", + "uiConfigId7": "IPinfoDomainDataConnector", + "_uiConfigId7": "[variables('uiConfigId7')]", + "dataConnectorContentId7": "IPinfoDomainDataConnector", + "_dataConnectorContentId7": "[variables('dataConnectorContentId7')]", + "dataConnectorId7": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId7'))]", + "_dataConnectorId7": "[variables('dataConnectorId7')]", + "dataConnectorTemplateSpecName7": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId7'))))]", + "dataConnectorVersion7": "1.0.0", + "_dataConnectorcontentProductId7": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId7'),'-', variables('dataConnectorVersion7'))))]", + "uiConfigId8": "IPinfoIplocationExtendedDataConnector", + "_uiConfigId8": "[variables('uiConfigId8')]", + "dataConnectorContentId8": "IPinfoIplocationExtendedDataConnector", + "_dataConnectorContentId8": "[variables('dataConnectorContentId8')]", + "dataConnectorId8": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId8'))]", + "_dataConnectorId8": "[variables('dataConnectorId8')]", + "dataConnectorTemplateSpecName8": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId8'))))]", + "dataConnectorVersion8": "1.0.0", + "_dataConnectorcontentProductId8": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId8'),'-', variables('dataConnectorVersion8'))))]", + "uiConfigId9": "IPinfoPrivacyExtendedDataConnector", + "_uiConfigId9": "[variables('uiConfigId9')]", + "dataConnectorContentId9": "IPinfoPrivacyExtendedDataConnector", + "_dataConnectorContentId9": "[variables('dataConnectorContentId9')]", + "dataConnectorId9": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId9'))]", + "_dataConnectorId9": "[variables('dataConnectorId9')]", + "dataConnectorTemplateSpecName9": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId9'))))]", + "dataConnectorVersion9": "1.0.0", + "_dataConnectorcontentProductId9": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId9'),'-', variables('dataConnectorVersion9'))))]", + "uiConfigId10": "IPinfoRWHOISDataConnector", + "_uiConfigId10": "[variables('uiConfigId10')]", + "dataConnectorContentId10": "IPinfoRWHOISDataConnector", + "_dataConnectorContentId10": "[variables('dataConnectorContentId10')]", + "dataConnectorId10": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId10'))]", + "_dataConnectorId10": "[variables('dataConnectorId10')]", + "dataConnectorTemplateSpecName10": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId10'))))]", + "dataConnectorVersion10": "1.0.0", + "_dataConnectorcontentProductId10": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId10'),'-', variables('dataConnectorVersion10'))))]", + "uiConfigId11": "IPinfoRIRWHOISDataConnector", + "_uiConfigId11": "[variables('uiConfigId11')]", + "dataConnectorContentId11": "IPinfoRIRWHOISDataConnector", + "_dataConnectorContentId11": "[variables('dataConnectorContentId11')]", + "dataConnectorId11": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId11'))]", + "_dataConnectorId11": "[variables('dataConnectorId11')]", + "dataConnectorTemplateSpecName11": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId11'))))]", + "dataConnectorVersion11": "1.0.0", + "_dataConnectorcontentProductId11": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId11'),'-', variables('dataConnectorVersion11'))))]", + "uiConfigId12": "IPinfoWHOISASNDataConnector", + "_uiConfigId12": "[variables('uiConfigId12')]", + "dataConnectorContentId12": "IPinfoWHOISASNDataConnector", + "_dataConnectorContentId12": "[variables('dataConnectorContentId12')]", + "dataConnectorId12": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId12'))]", + "_dataConnectorId12": "[variables('dataConnectorId12')]", + "dataConnectorTemplateSpecName12": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId12'))))]", + "dataConnectorVersion12": "1.0.0", + "_dataConnectorcontentProductId12": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId12'),'-', variables('dataConnectorVersion12'))))]", + "uiConfigId13": "IPinfoWHOISMNTDataConnector", + "_uiConfigId13": "[variables('uiConfigId13')]", + "dataConnectorContentId13": "IPinfoWHOISMNTDataConnector", + "_dataConnectorContentId13": "[variables('dataConnectorContentId13')]", + "dataConnectorId13": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId13'))]", + "_dataConnectorId13": "[variables('dataConnectorId13')]", + "dataConnectorTemplateSpecName13": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId13'))))]", + "dataConnectorVersion13": "1.0.0", + "_dataConnectorcontentProductId13": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId13'),'-', variables('dataConnectorVersion13'))))]", + "uiConfigId14": "IPinfoWHOISNETDataConnector", + "_uiConfigId14": "[variables('uiConfigId14')]", + "dataConnectorContentId14": "IPinfoWHOISNETDataConnector", + "_dataConnectorContentId14": "[variables('dataConnectorContentId14')]", + "dataConnectorId14": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId14'))]", + "_dataConnectorId14": "[variables('dataConnectorId14')]", + "dataConnectorTemplateSpecName14": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId14'))))]", + "dataConnectorVersion14": "1.0.0", + "_dataConnectorcontentProductId14": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId14'),'-', variables('dataConnectorVersion14'))))]", + "uiConfigId15": "IPinfoWHOISORGDataConnector", + "_uiConfigId15": "[variables('uiConfigId15')]", + "dataConnectorContentId15": "IPinfoWHOISORGDataConnector", + "_dataConnectorContentId15": "[variables('dataConnectorContentId15')]", + "dataConnectorId15": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId15'))]", + "_dataConnectorId15": "[variables('dataConnectorId15')]", + "dataConnectorTemplateSpecName15": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId15'))))]", + "dataConnectorVersion15": "1.0.0", + "_dataConnectorcontentProductId15": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId15'),'-', variables('dataConnectorVersion15'))))]", + "uiConfigId16": "IPinfoWHOISPOCDataConnector", + "_uiConfigId16": "[variables('uiConfigId16')]", + "dataConnectorContentId16": "IPinfoWHOISPOCDataConnector", + "_dataConnectorContentId16": "[variables('dataConnectorContentId16')]", + "dataConnectorId16": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId16'))]", + "_dataConnectorId16": "[variables('dataConnectorId16')]", + "dataConnectorTemplateSpecName16": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId16'))))]", + "dataConnectorVersion16": "1.0.0", + "_dataConnectorcontentProductId16": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId16'),'-', variables('dataConnectorVersion16'))))]", + "uiConfigId17": "IPinfoCountryDataConnector", + "_uiConfigId17": "[variables('uiConfigId17')]", + "dataConnectorContentId17": "IPinfoCountryDataConnector", + "_dataConnectorContentId17": "[variables('dataConnectorContentId17')]", + "dataConnectorId17": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId17'))]", + "_dataConnectorId17": "[variables('dataConnectorId17')]", + "dataConnectorTemplateSpecName17": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat(parameters('workspace'),'-dc-',uniquestring(variables('_dataConnectorContentId17'))))]", + "dataConnectorVersion17": "1.0.0", + "_dataConnectorcontentProductId17": "[concat(take(variables('_solutionId'),50),'-','dc','-', uniqueString(concat(variables('_solutionId'),'-','DataConnector','-',variables('_dataConnectorContentId17'),'-', variables('dataConnectorVersion17'))))]", + "_solutioncontentProductId": "[concat(take(variables('_solutionId'),50),'-','sl','-', uniqueString(concat(variables('_solutionId'),'-','Solution','-',variables('_solutionId'),'-', variables('_solutionVersion'))))]" }, "resources": [ + { "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", "apiVersion": "2023-04-01-preview", @@ -69,7 +197,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "IPinfo Company data connector with template version 3.0.0", + "description": "IPinfo Company data connector with template version 3.0.1", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('dataConnectorVersion1')]", @@ -187,7 +315,7 @@ }, { "title": "Step 2 - Configure the Function App", - "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." } ] }, @@ -376,7 +504,7 @@ }, { "title": "Step 2 - Configure the Function App", - "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." } ] }, @@ -389,7 +517,7 @@ } } }, - + { "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", "apiVersion": "2023-04-01-preview", @@ -399,7 +527,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "IPinfo data connector with template version 3.0.0", + "description": "IPinfo Iplocation data connector with template version 3.0.1", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('dataConnectorVersion2')]", @@ -517,7 +645,7 @@ }, { "title": "Step 2 - Configure the Function App", - "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." } ] }, @@ -706,7 +834,7 @@ }, { "title": "Step 2 - Configure the Function App", - "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." } ] }, @@ -715,11 +843,11 @@ ] } ], - "id": "[variables('_uiConfigId1')]" + "id": "[variables('_uiConfigId2')]" } } }, - + { "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", "apiVersion": "2023-04-01-preview", @@ -729,7 +857,7 @@ "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" ], "properties": { - "description": "IPinfo data connector with template version 3.0.0", + "description": "IPinfo Privacy data connector with template version 3.0.1", "mainTemplate": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "[variables('dataConnectorVersion3')]", @@ -847,7 +975,7 @@ }, { "title": "Step 2 - Configure the Function App", - "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." } ] }, @@ -934,7 +1062,7 @@ "kind": "GenericUI", "properties": { "connectorUiConfig": { - "title": "IPinfo (using Azure Functions)", + "title": "IPinfo Privacy (using Azure Functions)", "publisher": "IPinfo", "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_privacy datasets and insert them into custom log tables in Microsoft Sentinel.", "graphQueries": [ @@ -1036,7 +1164,7 @@ }, { "title": "Step 2 - Configure the Function App", - "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION \n5. Once all application settings have been entered, click **Save**." + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." } ] }, @@ -1045,7 +1173,4627 @@ ] } ], - "id": "[variables('_uiConfigId1')]" + "id": "[variables('_uiConfigId3')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName4')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo Abuse data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion4')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId4'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId4')]", + "title": "IPinfo Abuse (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_abuse datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Abuse Data", + "legend": "Ipinfo_Abuse_CL", + "baseQuery": "Ipinfo_Abuse_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Abuse_CL", + "query": "Ipinfo_Abuse_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Abuse_CL", + "lastDataReceivedQuery": "Ipinfo_Abuse_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Abuse_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Abuse-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Abuse-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId4'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId4'))]", + "contentId": "[variables('_dataConnectorContentId4')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion1')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId4')]", + "contentKind": "DataConnector", + "displayName": "IPinfo Abuse (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId4')]", + "id": "[variables('_dataConnectorcontentProductId4')]", + "version": "[variables('dataConnectorVersion1')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId4'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId4')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId4'))]", + "contentId": "[variables('_dataConnectorContentId4')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion1')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId4'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo Abuse (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_abuse datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Abuse Data", + "legend": "Ipinfo_Abuse_CL", + "baseQuery": "Ipinfo_Abuse_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Abuse_CL", + "query": "Ipinfo_Abuse_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Abuse_CL", + "lastDataReceivedQuery": "Ipinfo_Abuse_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Abuse_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-abuse-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-abuse-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId4')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName5')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo ASN data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion5')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId5'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId5')]", + "title": "IPinfo ASN (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download ASN datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "ASN Data", + "legend": "Ipinfo_ASN_CL", + "baseQuery": "Ipinfo_ASN_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_ASN_CL", + "query": "Ipinfo_ASN_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_ASN_CL", + "lastDataReceivedQuery": "Ipinfo_ASN_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_ASN_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-ASN-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-ASN-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId5'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId5'))]", + "contentId": "[variables('_dataConnectorContentId5')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion5')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId5')]", + "contentKind": "DataConnector", + "displayName": "IPinfo ASN (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId5')]", + "id": "[variables('_dataConnectorcontentProductId5')]", + "version": "[variables('dataConnectorVersion5')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId5'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId5')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId5'))]", + "contentId": "[variables('_dataConnectorContentId5')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion5')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId5'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo ASN (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download ASN datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "ASN Data", + "legend": "Ipinfo_ASN_CL", + "baseQuery": "Ipinfo_ASN_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_ASN_CL", + "query": "Ipinfo_ASN_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_ASN_CL", + "lastDataReceivedQuery": "Ipinfo_ASN_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_ASN_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-ASN-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-ASN-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId5')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName6')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo Carrier data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion6')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId6'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId6')]", + "title": "IPinfo Carrier (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download carrier datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Carrier Data", + "legend": "Ipinfo_Carrier_CL", + "baseQuery": "Ipinfo_Carrier_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Carrier_CL", + "query": "Ipinfo_Carrier_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Carrier_CL", + "lastDataReceivedQuery": "Ipinfo_Carrier_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Carrier_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Carrier-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Carrier-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId6'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId6'))]", + "contentId": "[variables('_dataConnectorContentId6')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion6')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId6')]", + "contentKind": "DataConnector", + "displayName": "IPinfo Carrier (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId6')]", + "id": "[variables('_dataConnectorcontentProductId6')]", + "version": "[variables('dataConnectorVersion6')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId6'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId6')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId6'))]", + "contentId": "[variables('_dataConnectorContentId6')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion6')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId6'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo Carrier (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download carrier datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Carrier Data", + "legend": "Ipinfo_Carrier_CL", + "baseQuery": "Ipinfo_Carrier_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Carrier_CL", + "query": "Ipinfo_Carrier_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Carrier_CL", + "lastDataReceivedQuery": "Ipinfo_Carrier_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Carrier_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Carrier-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Carrier-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId6')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName7')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo Domain data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion7')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId7'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId7')]", + "title": "IPinfo Domain (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_ip_hosted_domains datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Domain Data", + "legend": "Ipinfo_Domain_CL", + "baseQuery": "Ipinfo_Domain_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Domain_CL", + "query": "Ipinfo_Domain_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Domain_CL", + "lastDataReceivedQuery": "Ipinfo_Domain_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Domain_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Domain-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Domain-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId7'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId7'))]", + "contentId": "[variables('_dataConnectorContentId7')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion7')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId7')]", + "contentKind": "DataConnector", + "displayName": "IPinfo Domain (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId7')]", + "id": "[variables('_dataConnectorcontentProductId7')]", + "version": "[variables('dataConnectorVersion7')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId7'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId7')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId7'))]", + "contentId": "[variables('_dataConnectorContentId7')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion7')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId7'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo Domain (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_ip_hosted_domains datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Domain Data", + "legend": "Ipinfo_Domain_CL", + "baseQuery": "Ipinfo_Domain_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Domain_CL", + "query": "Ipinfo_Domain_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Domain_CL", + "lastDataReceivedQuery": "Ipinfo_Domain_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Domain_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Domain-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Domain-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId7')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName8')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo Iplocation Extended data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion8')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId8'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId8')]", + "title": "IPinfo Iplocation Extended (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download location_extended datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Iplocation Extended Data", + "legend": "Ipinfo_Location_extended_CL", + "baseQuery": "Ipinfo_Location_extended_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Location_extended_CL", + "query": "Ipinfo_Location_extended_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Location_extended_CL", + "lastDataReceivedQuery": "Ipinfo_Location_extended_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Location_extended_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Iplocation_extended-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Iplocation_extended-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId8'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId8'))]", + "contentId": "[variables('_dataConnectorContentId8')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion8')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId8')]", + "contentKind": "DataConnector", + "displayName": "IPinfo Iplocation Extended (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId8')]", + "id": "[variables('_dataConnectorcontentProductId8')]", + "version": "[variables('dataConnectorVersion8')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId8'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId8')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId8'))]", + "contentId": "[variables('_dataConnectorContentId8')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion8')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId8'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo Iplocation Extended (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download location_extended datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Iplocation Extended Data", + "legend": "Ipinfo_Location_extended_CL", + "baseQuery": "Ipinfo_Iplocation_extended_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Iplocation_extended_CL", + "query": "Ipinfo_Iplocation_extended_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Iplocation_extended_CL", + "lastDataReceivedQuery": "Ipinfo_Iplocation_extended_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Iplocation_extended_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Iplocation_extended-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Iplocation_extended-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId8')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName9')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo Privacy Extended data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion9')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId9'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId9')]", + "title": "IPinfo Privacy Extended (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download privacy_extended datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Privacy Extended Data", + "legend": "Ipinfo_Privacy_extended_CL", + "baseQuery": "Ipinfo_Privacy_extended_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Privacy_extended_CL", + "query": "Ipinfo_Privacy_extended_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Privacy_extended_CL", + "lastDataReceivedQuery": "Ipinfo_Privacy_extended_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Privacy_extended_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Privacy_extended-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Privacy_extended-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId9'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId9'))]", + "contentId": "[variables('_dataConnectorContentId9')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion9')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId9')]", + "contentKind": "DataConnector", + "displayName": "IPinfo Privacy Extended (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId9')]", + "id": "[variables('_dataConnectorcontentProductId9')]", + "version": "[variables('dataConnectorVersion9')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId9'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId9')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId9'))]", + "contentId": "[variables('_dataConnectorContentId9')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion9')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId9'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo Privacy Extended (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download privacy_extended datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Privacy Extended Data", + "legend": "Ipinfo_Privacy_extended_CL", + "baseQuery": "Ipinfo_Privacy_extended_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Privacy_extended_CL", + "query": "Ipinfo_Privacy_extended_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Privacy_extended_CL", + "lastDataReceivedQuery": "Ipinfo_Privacy_extended_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Privacy_extended_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Privacy_extended-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Privacy_extended-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId9')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName10')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo RWHOIS data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion10')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId10'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId10')]", + "title": "IPinfo RWHOIS (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_RWHOIS datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "RWHOIS Data", + "legend": "Ipinfo_RWHOIS_CL", + "baseQuery": "Ipinfo_RWHOIS_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_RWHOIS_CL", + "query": "Ipinfo_RWHOIS_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_RWHOIS_CL", + "lastDataReceivedQuery": "Ipinfo_RWHOIS_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_RWHOIS_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-RWHOIS-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-RWHOIS-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId10'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId10'))]", + "contentId": "[variables('_dataConnectorContentId10')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion10')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId10')]", + "contentKind": "DataConnector", + "displayName": "IPinfo RWHOIS (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId10')]", + "id": "[variables('_dataConnectorcontentProductId10')]", + "version": "[variables('dataConnectorVersion10')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId10'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId10')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId10'))]", + "contentId": "[variables('_dataConnectorContentId10')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion10')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId10'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo RWHOIS (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_RWHOIS datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "RWHOIS Data", + "legend": "Ipinfo_RWHOIS_CL", + "baseQuery": "Ipinfo_RWHOIS_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_RWHOIS_CL", + "query": "Ipinfo_RWHOIS_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_RWHOIS_CL", + "lastDataReceivedQuery": "Ipinfo_RWHOIS_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_RWHOIS_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-RWHOIS-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-RWHOIS-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId10')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName11')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo RIRWHOIS data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion11')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId11'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId11')]", + "title": "IPinfo RIRWHOIS (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_RIRWHOIS datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "RIRWHOIS Data", + "legend": "Ipinfo_RIRWHOIS_CL", + "baseQuery": "Ipinfo_RIRWHOIS_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_RIRWHOIS_CL", + "query": "Ipinfo_RIRWHOIS_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_RIRWHOIS_CL", + "lastDataReceivedQuery": "Ipinfo_RIRWHOIS_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_RIRWHOIS_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-RIRWHOIS-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-RIRWHOIS-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId11'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId11'))]", + "contentId": "[variables('_dataConnectorContentId11')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion11')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId11')]", + "contentKind": "DataConnector", + "displayName": "IPinfo RIRWHOIS (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId11')]", + "id": "[variables('_dataConnectorcontentProductId11')]", + "version": "[variables('dataConnectorVersion11')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId11'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId11')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId11'))]", + "contentId": "[variables('_dataConnectorContentId11')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion11')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId11'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo RIRWHOIS (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_RIRWHOIS datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "RIRWHOIS Data", + "legend": "Ipinfo_RIRWHOIS_CL", + "baseQuery": "Ipinfo_RIRWHOIS_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_RIRWHOIS_CL", + "query": "Ipinfo_RIRWHOIS_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_RIRWHOIS_CL", + "lastDataReceivedQuery": "Ipinfo_RIRWHOIS_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_RIRWHOIS_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-RIRWHOIS-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-RIRWHOIS-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId11')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName12')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo WHOIS ASN data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion12')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId12'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId12')]", + "title": "IPinfo WHOIS ASN (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_ASN datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_ASN Data", + "legend": "Ipinfo_WHOIS_ASN_CL", + "baseQuery": "Ipinfo_WHOIS_ASN_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_ASN_CL", + "query": "Ipinfo_WHOIS_ASN_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_ASN_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_ASN_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_ASN_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-ASN-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-ASN-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId12'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId12'))]", + "contentId": "[variables('_dataConnectorContentId12')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion12')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId12')]", + "contentKind": "DataConnector", + "displayName": "IPinfo WHOIS ASN (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId12')]", + "id": "[variables('_dataConnectorcontentProductId12')]", + "version": "[variables('dataConnectorVersion12')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId12'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId12')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId12'))]", + "contentId": "[variables('_dataConnectorContentId12')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion12')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId12'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo WHOIS ASN (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_ASN datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_ASN Data", + "legend": "Ipinfo_WHOIS_ASN_CL", + "baseQuery": "Ipinfo_WHOIS_ASN_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_ASN_CL", + "query": "Ipinfo_WHOIS_ASN_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_ASN_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_ASN_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_ASN_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-ASN-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-ASN-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId12')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName13')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo WHOIS MNT data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion13')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId13'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId13')]", + "title": "IPinfo WHOIS MNT (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_MNT datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_MNT Data", + "legend": "Ipinfo_WHOIS_MNT_CL", + "baseQuery": "Ipinfo_WHOIS_MNT_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_MNT_CL", + "query": "Ipinfo_WHOIS_MNT_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_MNT_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_MNT_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_MNT_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-MNT-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-MNT-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId13'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId13'))]", + "contentId": "[variables('_dataConnectorContentId13')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion13')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId13')]", + "contentKind": "DataConnector", + "displayName": "IPinfo WHOIS MNT (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId13')]", + "id": "[variables('_dataConnectorcontentProductId13')]", + "version": "[variables('dataConnectorVersion13')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId13'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId13')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId13'))]", + "contentId": "[variables('_dataConnectorContentId13')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion13')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId13'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo WHOIS MNT (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_MNT datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_MNT Data", + "legend": "Ipinfo_WHOIS_MNT_CL", + "baseQuery": "Ipinfo_WHOIS_MNT_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_MNT_CL", + "query": "Ipinfo_WHOIS_MNT_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_MNT_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_MNT_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_MNT_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-MNT-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId13')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName14')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo WHOIS NET data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion14')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId14'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId14')]", + "title": "IPinfo WHOIS NET (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_NET datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_NET Data", + "legend": "Ipinfo_WHOIS_NET_CL", + "baseQuery": "Ipinfo_WHOIS_NET_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_NET_CL", + "query": "Ipinfo_WHOIS_NET_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_NET_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_NET_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_NET_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-NET-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-NET-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId14'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId14'))]", + "contentId": "[variables('_dataConnectorContentId14')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion14')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId14')]", + "contentKind": "DataConnector", + "displayName": "IPinfo WHOIS NET (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId14')]", + "id": "[variables('_dataConnectorcontentProductId14')]", + "version": "[variables('dataConnectorVersion14')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId14'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId14')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId14'))]", + "contentId": "[variables('_dataConnectorContentId14')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion14')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId14'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo WHOIS NET (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_NET datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_NET Data", + "legend": "Ipinfo_WHOIS_NET_CL", + "baseQuery": "Ipinfo_WHOIS_NET_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_NET_CL", + "query": "Ipinfo_WHOIS_NET_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_NET_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_NET_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_NET_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-NET-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId14')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName15')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo WHOIS ORG data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion15')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId15'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId15')]", + "title": "IPinfo WHOIS ORG (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_ORG datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_ORG Data", + "legend": "Ipinfo_WHOIS_ORG_CL", + "baseQuery": "Ipinfo_WHOIS_ORG_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_ORG_CL", + "query": "Ipinfo_WHOIS_ORG_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_ORG_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_ORG_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_ORG_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-ORG-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-ORG-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId15'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId15'))]", + "contentId": "[variables('_dataConnectorContentId15')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion15')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId15')]", + "contentKind": "DataConnector", + "displayName": "IPinfo WHOIS ORG (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId15')]", + "id": "[variables('_dataConnectorcontentProductId15')]", + "version": "[variables('dataConnectorVersion15')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId15'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId15')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId15'))]", + "contentId": "[variables('_dataConnectorContentId15')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion15')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId15'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo WHOIS ORG (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_ORG datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_ORG Data", + "legend": "Ipinfo_WHOIS_ORG_CL", + "baseQuery": "Ipinfo_WHOIS_ORG_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_ORG_CL", + "query": "Ipinfo_WHOIS_ORG_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_ORG_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_ORG_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_ORG_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOI-ORG-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-ORG-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId15')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName16')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo WHOIS POC data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion16')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId16'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId16')]", + "title": "IPinfo WHOIS POC (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_POC datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_POC Data", + "legend": "Ipinfo_WHOIS_POC_CL", + "baseQuery": "Ipinfo_WHOIS_POC_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_POC_CL", + "query": "Ipinfo_WHOIS_POC_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_POC_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_POC_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_POC_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-POC-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-POC-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId16'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId16'))]", + "contentId": "[variables('_dataConnectorContentId16')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion16')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId16')]", + "contentKind": "DataConnector", + "displayName": "IPinfo WHOIS POC (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId16')]", + "id": "[variables('_dataConnectorcontentProductId16')]", + "version": "[variables('dataConnectorVersion16')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId16'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId16')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId16'))]", + "contentId": "[variables('_dataConnectorContentId16')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion16')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId16'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo WHOIS POC (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download standard_WHOIS_POC datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "WHOIS_POC Data", + "legend": "Ipinfo_WHOIS_POC_CL", + "baseQuery": "Ipinfo_WHOIS_POC_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_WHOIS_POC_CL", + "query": "Ipinfo_WHOIS_POC_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_WHOIS_POC_CL", + "lastDataReceivedQuery": "Ipinfo_WHOIS_POC_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_WHOIS_POC_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-POC-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-WHOIS-POC-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId16')]" + } + } + }, + + { + "type": "Microsoft.OperationalInsights/workspaces/providers/contentTemplates", + "apiVersion": "2023-04-01-preview", + "name": "[variables('dataConnectorTemplateSpecName17')]", + "location": "[parameters('workspace-location')]", + "dependsOn": [ + "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/contentPackages', variables('_solutionId'))]" + ], + "properties": { + "description": "IPinfo Country ASN data connector with template version 3.0.1", + "mainTemplate": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "[variables('dataConnectorVersion17')]", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId17'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "id": "[variables('_uiConfigId17')]", + "title": "IPinfo Country ASN (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download country_asn datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Country Data", + "legend": "Ipinfo_Country_CL", + "baseQuery": "Ipinfo_Country_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Country_CL", + "query": "Ipinfo_Country_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Country_CL", + "lastDataReceivedQuery": "Ipinfo_Country_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Country_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-WHOIS-POC-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Country-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ] + } + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId17'),'/'))))]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId17'))]", + "contentId": "[variables('_dataConnectorContentId17')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion17')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + } + ] + }, + "packageKind": "Solution", + "packageVersion": "[variables('_solutionVersion')]", + "packageName": "[variables('_solutionName')]", + "packageId": "[variables('_solutionId')]", + "contentSchemaVersion": "1.0.0", + "contentId": "[variables('_dataConnectorContentId17')]", + "contentKind": "DataConnector", + "displayName": "IPinfo Country (using Azure Functions)", + "contentProductId": "[variables('_dataConnectorcontentProductId17')]", + "id": "[variables('_dataConnectorcontentProductId17')]", + "version": "[variables('dataConnectorVersion17')]" + } + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/metadata", + "apiVersion": "2023-04-01-preview", + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',concat('DataConnector-', last(split(variables('_dataConnectorId17'),'/'))))]", + "dependsOn": ["[variables('_dataConnectorId17')]"], + "location": "[parameters('workspace-location')]", + "properties": { + "parentId": "[extensionResourceId(resourceId('Microsoft.OperationalInsights/workspaces', parameters('workspace')), 'Microsoft.SecurityInsights/dataConnectors', variables('_dataConnectorContentId17'))]", + "contentId": "[variables('_dataConnectorContentId17')]", + "kind": "DataConnector", + "version": "[variables('dataConnectorVersion17')]", + "source": { + "kind": "Solution", + "name": "IPinfo", + "sourceId": "[variables('_solutionId')]" + }, + "author": { + "name": "IPinfo" + }, + "support": { + "name": "IPinfo", + "email": "support@ipinfo.io", + "tier": "Partner", + "link": "https://www.ipinfo.io/" + } + } + }, + { + "name": "[concat(parameters('workspace'),'/Microsoft.SecurityInsights/',variables('_dataConnectorContentId17'))]", + "apiVersion": "2021-03-01-preview", + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "location": "[parameters('workspace-location')]", + "kind": "GenericUI", + "properties": { + "connectorUiConfig": { + "title": "IPinfo Country ASN (using Azure Functions)", + "publisher": "IPinfo", + "descriptionMarkdown": "This IPinfo data connector installs an Azure Function app to download country_asn datasets and insert them into custom log tables in Microsoft Sentinel.", + "graphQueries": [ + { + "metricName": "Country Data", + "legend": "Ipinfo_Country_CL", + "baseQuery": "Ipinfo_Country_CL" + } + ], + "sampleQueries": [ + { + "description": "Ipinfo_Country_CL", + "query": "Ipinfo_Country_CL" + } + ], + "dataTypes": [ + { + "name": "Ipinfo_Country_CL", + "lastDataReceivedQuery": "Ipinfo_Country_CL | summarize Time = max(TimeGenerated)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "Ipinfo_Country_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1, + "isPreview": true + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions on the workspace are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ], + "customs": [ + { + "name": "Microsoft.Web/sites permissions", + "description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)." + }, + { + "name": "IPinfo API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + } + ] + }, + "instructionSteps": [ + { + "title": "1. Retrieve API Token", + "description": "Retrieve your IPinfo API Token [here](https://ipinfo.io/)." + }, + { + "title": "2. In your Azure AD tenant, create an Azure Active Directory (AAD) application", + "description": "In your Azure AD tenant, create an Azure Active Directory (AAD) application and acquire Tenant ID, Client ID, and Client Secret: Use this Link." + }, + { + "title": "3. Assign the AAD application the Microsoft Sentinel Contributor Role.", + "description": "Assign the AAD application you just created to the Contributor(Privileged administrator roles) and Monitoring Metrics Publisher(Job function roles) in the same “Resource Group” you use for “Log Analytic Workspace” on which “Microsoft Sentinel” is added: Use this Link." + }, + { + "title": "4. Get Workspace Resource ID", + "description": "Use the Log Analytic Workspace -> Properties blade having the 'Resource ID' property value. This is a fully qualified resourceId which is in the format '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}'" + }, + { + "title": "5. Deploy the Azure Function", + "description": "Use this for automated deployment of the IPinfo data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[![Deploy To Azure](https://aka.ms/deploytoazurebutton)](https://aka.ms/sentinel-IPinfo-Country-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **RESOURCE_ID**, **IPINFO_TOKEN**, **TENANT_ID**, **CLIENT_ID**, **CLIENT_SECRET**." + }, + { + "title": "Manual Deployment of Azure Functions", + "description": "Use the following step-by-step instructions to deploy the IPinfo data connector manually with Azure Functions (Deployment via Visual Studio Code).", + "instructions": [ + { + "parameters": { + "instructionSteps": [ + { + "title": "Step 1 - Deploy a Function App", + "description": "1. Download the Azure Function App file. Extract the archive to your local development computer [Azure Function App](https://aka.ms/sentinel-IPinfo-Country-functionapp). \n2. Create Function App using Hosting Functions Premium or App service plan using advanced option using VSCode. \n3. Follow the function app manual deployment instructions to deploy the Azure Functions app using VSCode. \n4. After successful deployment of the function app, follow the next steps for configuring it." + }, + { + "title": "Step 2 - Configure the Function App", + "description": "1. Go to Azure Portal for the Function App configuration.\n2. In the Function App, select the Function App Name and select **Settings** -> **Configuration** or **Environment variables**. \n3. In the **Application settings** tab, select **+ New application setting**.\n4. Add each of the following application settings individually, with their respective string values (case-sensitive):\n\t\tRESOURCE_ID\n\t\tIPINFO_TOKEN\n\t\tTENANT_ID\n\t\tCLIENT_ID\n\t\tCLIENT_SECRET\n\t\tSCHEDULE\n\t\tLOCATION\n\t\tRETENTION_IN_DAYS\n\t\tTOTAL_RETENTION_IN_DAYS \n5. Once all application settings have been entered, click **Save**." + } + ] + }, + "type": "InstructionStepsGroup" + } + ] + } + ], + "id": "[variables('_uiConfigId17')]" } } }, @@ -1055,9 +5803,9 @@ "apiVersion": "2023-04-01-preview", "location": "[parameters('workspace-location')]", "properties": { - "version": "3.0.0", + "version": "3.1.0", "kind": "Solution", - "contentSchemaVersion": "3.0.0", + "contentSchemaVersion": "3.1.0", "displayName": "IPinfo", "publisherDisplayName": "IPinfo", "descriptionHtml": "See Refrence", diff --git a/Solutions/IPinfo/ReleaseNotes.md b/Solutions/IPinfo/ReleaseNotes.md index 37ec7c18a96..367be8feb50 100644 --- a/Solutions/IPinfo/ReleaseNotes.md +++ b/Solutions/IPinfo/ReleaseNotes.md @@ -1,3 +1,4 @@ | **Version** | **Date Modified (DD-MM-YYYY)** | **Change History** | |-------------|--------------------------------|---------------------------------------------| -| 3.0.0 | 10-07-2024 | Initial Solution Release | +| 3.0.1 | 19-08-2024 | IPinfo New Data Connectors | +| 3.0.0 | 10-07-2024 | Initial Solution Release |