Merge pull request #152 from Ola-Mamdouh234/develop

Develop-hashlocal
egirna · Jan 16, 2024 · e888b01 · e888b01
2 parents 5b61912 + 17d2fd5
commit e888b01
Show file tree

Hide file tree

Showing 14 changed files with 525 additions and 57 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-<h1 align="center">Go ICAP Server</h1>
+<h1 align="center">Go ICAPeg Server</h1>
 <p align="center">
     <em>ICAPeg Server.</em>
 </p>
@@ -25,7 +25,7 @@
 
 Open Source multi-vendor ICAP server
 
-Scan files requested via a proxy server using ICAPeg ICAP server, ICAPeg is an ICAP server connecting web proxies with API-based scanning services and more soon. ICAPeg currently supports [**VirusTotal**](https://www.virustotal.com/gui/home/upload), [**Cloudmersive**](https://cloudmersive.com/) & [**Clamav**](https://www.clamav.net/)  for scanning the files following the ICAP protocol. If you don't know about the ICAP protocol, here is a bit about it: 
+Scan files requested via a proxy server using ICAPeg ICAP server, ICAPeg is an ICAP server connecting web proxies with API-based scanning services and more soon. ICAPeg currently supports [**hashlocal**](#hashlocal), [**clhashlookup**](#clhashlookup) & [**Clamav**](https://www.clamav.net/)  for scanning the files following the ICAP protocol. If you don't know about the ICAP protocol, here is a bit about it: 
 
 ## What is ICAP?
 
@@ -37,15 +37,15 @@ To know more about the ICAP protocol, [check this out](https://tools.ietf.org/ht
 ## Table of Contents
 
 - [Prerequisites](#prerequisites)
+- [Service Describtion](#Service_Describtion)
 - [Configuration](#configuration)
 - [Adding a new vendor to ICAPeg](#adding-a-new-vendor-to-ICAPeg)
 - [Developer Guide](#developer-guide)
 - [How to Setup Existed Services](#how-to-setup-existed-services)
   - [Echo](#echo)
-  - [Virustotal](#virustotal)
   - [ClamAV](#clamav)
-  - [Cloudmersive](#cloudmersive)
-
+
+  
 - [Things to keep in mind](#things-to-keep-in-mind)
 - [More on ICAPeg](#more-on-icapeg)
 - [Contributing](#contributing)
@@ -93,6 +93,21 @@ $ ./icapeg
 
 You should see something like, ```ICAP server is running on localhost:1344 ...```. This tells you the ICAP server is up and running.
 
+## Service_Describtion
+
+- **echo**
+> simply it's like when you bing url,so what we send will be received 
+
+- ## clamav**
+> ClamAV is an open source (GPLv2) anti-virus toolkit, designed especially for e-mail scanning on mail gateways. It provides a number of utilities including a flexible and scalable multi-threaded daemon also helps to scan file quickly.
+
+- ## clhashlookup**
+> simply it helps to scan each file we need to check before we send to Api.
+
+- ## hashlocal**
+>  we will pretend that you want to download a file. Still, you don't know if it is safe or not, so     using this service helps you calculate the hash value of any file you download through the icapeg, then it checks if this hash value is available or not in hash_file if it is in the hash_file returns back an exception page, so if you tried to downloadany file of eicar test vrius it will appear an exception page,cause the hash file of thosefile is foun in our hash_file.
+
+
 ## Configuration
 
 	You can change the default configuration file whatever you want to customize the application.
@@ -138,7 +153,9 @@ You should see something like, ```ICAP server is running on localhost:1344 ...``
           - **false**: Debugging headers should not be displayed with ICAP headers.
 
           - Any port number that isn't used in your machine.
-
+
+      ## echo
+
       - **[echo] section** 
 
         >  **Note**: Variables explained in **echo** service are mandatory with any service integrated with **ICAPeg**.
@@ -279,7 +296,6 @@ You should see something like, ```ICAP server is running on localhost:1344 ...``
         - ### **Optional variables** (Variables that depends on the service)
 
           > **Notes:** 
-          >
           > - You may not use these variables in your service and you may use them, It depends on your service and It's up to you.
           > - We will pretend that this service is for file processing and it sends that file to an external API to process it then it gets it back again, So all optional variables depend on that scenario in this service. (It's just a fake scenario service that can do anything not just for processing files).
 
@@ -297,7 +313,104 @@ You should see something like, ```ICAP server is running on localhost:1344 ...``
             - **false**: Returning **400 Bad request**.
 
             Get more details about **request mode** from [here](https://datatracker.ietf.org/doc/html/rfc3507#section-3.1).
-
+
+      ## hashlocal
+
+      - **[hashlocal] section**
+        ```toml
+        [hashlocal]
+        vendor = "hashlocal"
+        service_caption= "hashlocal"   #Service
+        service_tag = "hashlocal ICAP"  #ISTAG
+        req_mode=true
+        resp_mode=true
+        shadow_service=false
+        preview_bytes = "1024" #byte
+        preview_enabled = true# options send preview header or not
+        bypass_extensions = ["*"]
+        process_extensions = ["pdf","exe", "zip"] # * = everything except the ones in bypass, unknown = system couldn't find out the type of the file
+        reject_extensions = ["docx"]
+        Hash_File = "./hash_file/hash_file_path.txt" #
+        max_filesize = 0 #bytes
+        return_original_if_max_file_size_exceeded=true
+        return_400_if_file_ext_rejected=false
+        http_exception_response_code = 403
+        http_exception_has_body = true
+        exception_page = "./temp/exception-page.html" # Location of the exception page for this service
+        ```
+       - ### **New used variables **
+
+       - **HashFile**
+       It is a string variable has the link of the hash_file_path.txt ,where we save the hashes value we want to be compared with any file we try to download through icapeg.
+
+      ## clhashlookup
+
+      - **[clhashlookup] section**
+
+        ```toml
+       [clhashlookup]
+        vendor = "clhashlookup"
+        service_caption= "cl-hashlookup"   #Service
+        service_tag = "cl-hashlookup ICAP"  #ISTAG
+        req_mode=true
+        resp_mode=true
+        shadow_service=false
+        preview_bytes = "1024" #byte
+        preview_enabled = true# options send preview header or not
+        bypass_extensions = ["*"]
+        process_extensions = ["pdf","exe", "zip"] # * = everything except the ones in bypass, unknown = system couldn't find out the type of the file
+        reject_extensions = ["docx"]
+        scan_url = "https://hashlookup.circl.lu/lookup/sha256/" #
+        timeout  = 300 #seconds , ICAP will return 408 - Request timeout
+        fail_threshold = 2
+        max_filesize = 0 #bytes
+        return_original_if_max_file_size_exceeded=true
+        return_400_if_file_ext_rejected=false
+        verify_server_cert=true
+        bypass_on_api_error=false
+        http_exception_response_code = 403
+        http_exception_has_body = true
+        exception_page = "./temp/exception-page.html" # Location of the exception page for this service
+        ```
+        - ### **New used variables **
+
+       - **Scan_url**
+       It is a string variable has the link of Api ,where we send the files we scanned to.
+
+       ## clamav
+
+      - **[clamav] section**
+
+        ```toml
+        [clamav]
+        vendor = "clamav"
+        service_caption= "clamav service"   #Service
+        service_tag = "CLAMAV ICAP"  #ISTAG
+        req_mode=true
+        resp_mode=true
+        shadow_service=false
+        preview_bytes = "1024" #byte
+        preview_enabled = true# options send preview header or not
+        process_extensions = ["pdf", "zip", "com"] # * = everything except the ones in bypass, unknown = system couldn't find out the type of the file
+        reject_extensions = ["docx"]
+        bypass_extensions = ["*"]
+        socket_path = "/var/run/clamav/clamd.ctl"
+        fail_threshold = 2
+        timeout = 10 #seconds, the time upto which the server will wait for clamav to scan the results
+        #max file size value from 1 to 9223372036854775807, and value of zero means unlimited
+        max_filesize = 0 #bytes
+        return_original_if_max_file_size_exceeded=false
+        return_400_if_file_ext_rejected=false
+        verify_server_cert=true
+        bypass_on_api_error=false
+        http_exception_response_code = 403
+        http_exception_has_body = true
+        exception_page = "./temp/exception-page.html" # Location of the exception page for this service
+        ```
+      - ### **New used variables **
+
+       - **socket_path**
+       It is a string variable it helps sending the HTTP msg body to the ClamAV through antivirus socket.
 
 ## Adding a new vendor to ICAPeg
 
@@ -311,16 +424,12 @@ You should see something like, ```ICAP server is running on localhost:1344 ...``
 
   This is a developer guide which includes a lot of functions to help the developer while implementing his new service.
 
-## How to Setup Existed Services
+## How to Setup Existing Services
 
 - #### **Echo**: It doesn't need setup, it takes the HTTP message and returns it as it is. **Echo** is just an example service.
 
-- #### [**Virustotal**](/vendors-markdowns/virustotal/VIRUSTOTALAPI.md).
-
 - #### [**ClamAV**](/vendors-markdowns/clamav/CLAMAVSETUP.md).
 
-- #### [**Cloudmersive**](/vendors-markdowns/cloudmersive/CLOUDMERSIVEAPI.md).
-
 ## Testing
 
 - [How to test **ICAPeg**](Testing.md)

diff --git a/config.toml b/config.toml
@@ -21,7 +21,7 @@ title = "ICAP configuration file"
 port = 1344
 log_level="debug"
 write_logs_to_console= false
-services= ["echo", "clhashlookup", "clamav"]
+services= ["echo", "clhashlookup", "clamav","hashlocal"]
 debugging_headers=true
 web_server_host = "$_WEB_SERVER_HOST" #Example: "localhost:8081" , replace localhost with the ICAP server IP address.
 web_server_endpoint = "/service/message"  
@@ -68,6 +68,26 @@ http_exception_response_code = 403
 http_exception_has_body = true
 exception_page = "./temp/exception-page.html" # Location of the exception page for this service
 
+[hashlocal]
+vendor = "hashlocal"
+service_caption= "hashlocal"   #Service
+service_tag = "hashlocal ICAP"  #ISTAG
+req_mode=true
+resp_mode=true
+shadow_service=false
+preview_bytes = "1024" #byte
+preview_enabled = true# options send preview header or not
+bypass_extensions = ["*"]
+process_extensions = ["pdf","exe", "zip"] # * = everything except the ones in bypass, unknown = system couldn't find out the type of the file
+reject_extensions = ["docx"]
+hash_file = "./hash_file/hash_file_path.txt" #
+max_filesize = 0 #bytes
+return_original_if_max_file_size_exceeded=true
+return_400_if_file_ext_rejected=false
+http_exception_response_code = 403
+http_exception_has_body = true
+exception_page = "./temp/exception-page.html" # Location of the exception page for this service
+
 [clamav]
 vendor = "clamav"
 service_caption= "clamav service"   #Service

diff --git a/hash_file/hash_file_path.txt b/hash_file/hash_file_path.txt
@@ -0,0 +1,3 @@
+2546dcffc5ad854d4ddc64fbf056871cd5a00f2471cb7a5bfd4ac23b6e9eedad
+e1105070ba828007508566e28a2b8d4c65d192e9eaf3b7868382b7cae747b397
+275a021bbfb6489e54d471899f7db9d1663fc695ec2fe2a2c4538aabf651fd0f
diff --git a/service/service.go b/service/service.go
@@ -6,6 +6,7 @@ import (
 	"icapeg/service/services/clamav"
 	"icapeg/service/services/clhashlookup"
 	"icapeg/service/services/echo"
+	"icapeg/service/services/hashlocal"
 	"net/textproto"
 )
 
@@ -14,6 +15,7 @@ const (
 	VendorEcho       = "echo"
 	VendorClamav     = "clamav"
 	VendorHashlookup = "clhashlookup"
+	VendorHashlocal  = "hashlocal"
 )
 
 type (
@@ -32,12 +34,13 @@ func GetService(vendor, serviceName, methodName string, httpMsg *http_message.Ht
 	switch vendor {
 	case VendorEcho:
 		return echo.NewEchoService(serviceName, methodName, httpMsg, xICAPMetadata)
-	case VendorClamav:
-		return clamav.NewClamavService(serviceName, methodName, httpMsg, xICAPMetadata)
 	case VendorHashlookup:
 		return clhashlookup.NewHashlookupService(serviceName, methodName, httpMsg, xICAPMetadata)
+	case VendorClamav:
+		return clamav.NewClamavService(serviceName, methodName, httpMsg, xICAPMetadata)
+	case VendorHashlocal:
+		return hashlocal.NewHashlocalService(serviceName, methodName, httpMsg, xICAPMetadata)
 	}
-
 	return nil
 }
 
@@ -47,9 +50,11 @@ func InitServiceConfig(vendor, serviceName string) {
 	switch vendor {
 	case VendorEcho:
 		echo.InitEchoConfig(serviceName)
-	case VendorClamav:
-		clamav.InitClamavConfig(serviceName)
 	case VendorHashlookup:
 		clhashlookup.InitHashlookupConfig(serviceName)
+	case VendorClamav:
+		clamav.InitClamavConfig(serviceName)
+	case VendorHashlocal:
+		hashlocal.InitHashlocalConfig(serviceName)
 	}
 }
diff --git a/service/services/hashlocal/config.go b/service/services/hashlocal/config.go
@@ -0,0 +1,81 @@
+package hashlocal
+
+import (
+	http_message "icapeg/http-message"
+	"icapeg/logging"
+	"icapeg/readValues"
+	services_utilities "icapeg/service/services-utilities"
+	general_functions "icapeg/service/services-utilities/general-functions"
+	"net/textproto"
+	"sync"
+	//"time"
+)
+
+var doOnce sync.Once
+var HashlocalConfig *Hashlocal
+
+// Hashlookup represents the information regarding the Hashlookup service
+type Hashlocal struct {
+	xICAPMetadata              string
+	httpMsg                    *http_message.HttpMsg
+	serviceName                string
+	methodName                 string
+	maxFileSize                int
+	bypassExts                 []string
+	processExts                []string
+	rejectExts                 []string
+	extArrs                    []services_utilities.Extension
+	hashfile                   string
+	returnOrigIfMaxSizeExc     bool
+	return400IfFileExtRejected bool
+	generalFunc                *general_functions.GeneralFunc
+	BypassOnApiError           bool
+	FileHash                   string
+	CaseBlockHttpResponseCode  int
+	CaseBlockHttpBody          bool
+	ExceptionPage              string
+	IcapHeaders                textproto.MIMEHeader
+}
+
+func InitHashlocalConfig(serviceName string) {
+	logging.Logger.Debug("loading " + serviceName + " service configurations")
+	doOnce.Do(func() {
+		HashlocalConfig = &Hashlocal{
+			maxFileSize:                readValues.ReadValuesInt(serviceName + ".max_filesize"),
+			bypassExts:                 readValues.ReadValuesSlice(serviceName + ".bypass_extensions"),
+			processExts:                readValues.ReadValuesSlice(serviceName + ".process_extensions"),
+			rejectExts:                 readValues.ReadValuesSlice(serviceName + ".reject_extensions"),
+			hashfile:                   readValues.ReadValuesString(serviceName + ".hash_file"),
+			returnOrigIfMaxSizeExc:     readValues.ReadValuesBool(serviceName + ".return_original_if_max_file_size_exceeded"),
+			return400IfFileExtRejected: readValues.ReadValuesBool(serviceName + ".return_400_if_file_ext_rejected"),
+			BypassOnApiError:           readValues.ReadBoolFromEnv(serviceName + ".bypass_on_api_error"),
+			CaseBlockHttpResponseCode:  readValues.ReadValuesInt(serviceName + ".http_exception_response_code"),
+			CaseBlockHttpBody:          readValues.ReadValuesBool(serviceName + ".http_exception_has_body"),
+			ExceptionPage:              readValues.ReadValuesString(serviceName + ".exception_page"),
+		}
+		HashlocalConfig.extArrs = services_utilities.InitExtsArr(HashlocalConfig.processExts, HashlocalConfig.rejectExts, HashlocalConfig.bypassExts)
+	})
+}
+
+// NewHashlookupService returns a new populated instance of the Hashlookup service
+func NewHashlocalService(serviceName, methodName string, httpMsg *http_message.HttpMsg, xICAPMetadata string) *Hashlocal {
+	return &Hashlocal{
+		xICAPMetadata:              xICAPMetadata,
+		httpMsg:                    httpMsg,
+		serviceName:                serviceName,
+		methodName:                 methodName,
+		maxFileSize:                HashlocalConfig.maxFileSize,
+		bypassExts:                 HashlocalConfig.bypassExts,
+		processExts:                HashlocalConfig.processExts,
+		rejectExts:                 HashlocalConfig.rejectExts,
+		extArrs:                    HashlocalConfig.extArrs,
+		hashfile:                   HashlocalConfig.hashfile,
+		returnOrigIfMaxSizeExc:     HashlocalConfig.returnOrigIfMaxSizeExc,
+		return400IfFileExtRejected: HashlocalConfig.return400IfFileExtRejected,
+		generalFunc:                general_functions.NewGeneralFunc(httpMsg, xICAPMetadata),
+		BypassOnApiError:           HashlocalConfig.BypassOnApiError,
+		CaseBlockHttpResponseCode:  HashlocalConfig.CaseBlockHttpResponseCode,
+		CaseBlockHttpBody:          HashlocalConfig.CaseBlockHttpBody,
+		ExceptionPage:              HashlocalConfig.ExceptionPage,
+	}
+}