ip2location db added & config extended

CHANGELOG.md

@@ -11,6 +11,14 @@ Updates should follow the [Keep a CHANGELOG](http://keepachangelog.com/) princip
 ### Added
 - NaN
 
+## [1.1.0] - 2020-01-19
+- ip2location db added
+- Get ISP information
+- Get domain information
+- Get usage information
+- Get proxy information
+- Available config parameters extended
+
 ## [1.0.4] - 2020-01-18
 - Population density omitted if empty or null
 - Detect tor users

server/api.go

@@ -5,6 +5,7 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"encoding/xml"
+	"fmt"
 	"github.com/go-web/httpmux"
 	"github.com/mileusna/useragent"
 	"golang.org/x/text/language"
@@ -19,15 +20,10 @@ import (
 
 // openDB opens and returns the IP database file or URL.
 func (s *Server) openDB() error {
-	if _, err := s.Api.db.OpenURL(); err != nil {
-		return err
-	}
-	if _, err := s.Api.asnDB.OpenURL(); err != nil {
-		return err
-	}
-	if _, err := s.Api.torDB.OpenURL(); err != nil {
-		return err
-	}
+	if _, err := s.Api.i2lDB.Start(); err != nil {return err}
+	if _, err := s.Api.torDB.Start(); err != nil {return err}
+	if _, err := s.Api.db.Start(); err != nil {return err}
+	if _, err := s.Api.asnDB.Start(); err != nil {return err}
 
 	return nil
 }
@@ -43,14 +39,18 @@ func (s *Server) IpLookUp(writer writerFunc) http.HandlerFunc {
 
 		ip, q := ips[rand.Intn(len(ips))], &GeoIpQuery{}
 		if err := s.Api.db.Lookup(ip, &q.DefaultQuery); err != nil {
+			fmt.Println(err)
 			http.Error(w, "Try again later.", http.StatusServiceUnavailable)
 			return
 		}
 		if err := s.Api.asnDB.Lookup(ip, &q.ASNDefaultQuery); err != nil {
 			http.Error(w, "Try again later.", http.StatusServiceUnavailable)
 			return
 		}
-		w.Header().Set("X-Database-Date", s.Api.db.Date().Format(http.TimeFormat))
+
+		q.ProxyDefaultQuery = s.Api.i2lDB.Lookup(ip)
+
+		w.Header().Set("X-Database-Date", s.Api.db.Updater.Date().Format(http.TimeFormat))
 		lang := getRequestParam(r, "lang")
 
 		q.IsTorUser = s.Api.torDB.Lookup(ip)
@@ -66,14 +66,6 @@ func (q *GeoIpQuery) Translate(names map[string]string, lang string) string {
 	return names["en"]
 }
 
-func convertFloat32ToFloat64(q []float32) []float64{
-	result := make([]float64, len(q))
-	for i, val := range q {
-		result[i] = float64(val)
-	}
-	return result
-}
-
 func getMostPreferredLanguage(tag []language.Tag, q []float32) language.Tag {
 	lang := language.Tag{}
 	score := 0.0
@@ -91,19 +83,21 @@ func (q *GeoIpQuery) Record(ip net.IP, lang string, request *http.Request) *resp
 	lang = parseAcceptLanguage(lang, q.Country.Names)
 
 	r := &responseRecord{
-		IP:          ip.String(),
-		IsInEuropeanUnion: q.Country.IsInEuropeanUnion,
-		CountryCode: q.Country.ISOCode,
-		ContinentCode: q.Translate(q.Continent.Names, lang),
-		CountryName: q.Translate(q.Country.Names, lang),
-		City:        q.Translate(q.City.Names, lang),
-		ZipCode:     q.Postal.Code,
-		TimeZone:    q.Location.TimeZone,
-		Latitude:    roundFloat(q.Location.Latitude, .5, 4),
-		Longitude:   roundFloat(q.Location.Longitude, .5, 4),
-		MetroCode:   q.Location.MetroCode,
-		PopulationDensity:   q.Location.PopulationDensity,
-		AccuracyRadius:   q.Location.AccuracyRadius,
+		IP:          		ip.String(),
+		Isp:				q.Isp,
+		Domain:				q.Domain,
+		IsInEuropeanUnion: 	q.Country.IsInEuropeanUnion,
+		CountryCode: 		q.Country.ISOCode,
+		ContinentCode: 		q.Translate(q.Continent.Names, lang),
+		CountryName: 		q.Translate(q.Country.Names, lang),
+		City:        		q.Translate(q.City.Names, lang),
+		ZipCode:     		q.Postal.Code,
+		TimeZone:    		q.Location.TimeZone,
+		Latitude:    		roundFloat(q.Location.Latitude, .5, 4),
+		Longitude:   		roundFloat(q.Location.Longitude, .5, 4),
+		MetroCode:   		q.Location.MetroCode,
+		PopulationDensity:  q.Location.PopulationDensity,
+		AccuracyRadius:   	q.Location.AccuracyRadius,
 		ASN: &ASNRecord{
 			AutonomousSystemNumber:       q.AutonomousSystemNumber,
 			AutonomousSystemOrganization: q.AutonomousSystemOrganization,
@@ -131,16 +125,20 @@ func (q *GeoIpQuery) Record(ip net.IP, lang string, request *http.Request) *resp
 			},
 
 			System:   &SystemRecord{
-				OS:      userAgent.OS,
-				Browser: userAgent.Name,
-				Version: userAgent.Version,
-				OSVersion: userAgent.OSVersion,
-				Device: userAgent.Device,
-				Mobile: userAgent.Mobile,
-				Tablet: userAgent.Tablet,
-				Desktop: userAgent.Desktop,
-				Bot: userAgent.Bot,
-				Tor: q.IsTorUser,
+				OS:      	userAgent.OS,
+				Browser: 	userAgent.Name,
+				Version: 	userAgent.Version,
+				OSVersion: 	userAgent.OSVersion,
+				Device: 	userAgent.Device,
+				Mobile: 	userAgent.Mobile,
+				Tablet: 	userAgent.Tablet,
+				Desktop: 	userAgent.Desktop,
+				Bot: 		userAgent.Bot,
+				Tor: 		q.IsTorUser,
+				ProxyType: 	q.ProxyType,
+				Proxy: 		q.Proxy,
+				UsageType: 	q.UsageType,
+				LastSeen:	uint(q.LastSeen),
 			},
 		}
 	}
@@ -163,9 +161,12 @@ func (rr *responseRecord) String() string {
 		var SystemDesktop int;if rr.User.System.Desktop {SystemDesktop = 1}
 		var SystemBot int;if rr.User.System.Bot {SystemBot = 1}
 		var SystemTor int;if rr.User.System.Tor {SystemTor = 1}
+		var SystemIsProxy int;if rr.User.System.Proxy {SystemIsProxy = 1}
 
 		err = w.Write([]string{
 			rr.IP,
+			rr.Isp,
+			rr.Domain,
 			strconv.Itoa(inEU),
 			rr.ContinentCode,
 			rr.CountryCode,
@@ -195,10 +196,16 @@ func (rr *responseRecord) String() string {
 			strconv.Itoa(SystemDesktop),
 			strconv.Itoa(SystemBot),
 			strconv.Itoa(SystemTor),
+			rr.User.System.ProxyType,
+			strconv.Itoa(SystemIsProxy),
+			rr.User.System.UsageType,
+			strconv.Itoa(int(rr.User.System.LastSeen)),
 		})
 	}else{
 		err = w.Write([]string{
 			rr.IP,
+			rr.Isp,
+			rr.Domain,
 			strconv.Itoa(inEU),
 			rr.ContinentCode,
 			rr.CountryCode,

server/backend.go

@@ -11,6 +11,8 @@ import (
 	_ "net/http/pprof"
 
 	"github.com/fiorix/go-listener/listener"
+
+	"../utils/updater"
 )
 
 type writerFunc func(w http.ResponseWriter, r *http.Request, d *responseRecord)
@@ -27,7 +29,6 @@ func (s *Server) NewHandler() (http.Handler, error) {
 	mux.GET("/csv/*host",  s.registerHandler(csvResponse))
 	mux.GET("/xml/*host",  s.registerHandler(xmlResponse))
 	mux.GET("/json/*host", s.registerHandler(jsonResponse))
-	go s.watchEvents()
 	return mux, nil
 }
 
@@ -40,17 +41,19 @@ func (s *Server) initCors() {
 }
 
 // watchEvents logs and collect metrics of database events.
-func (s *Server) watchEvents() {
-	for {
-		select {
-		case file := <-s.Api.db.NotifyOpen():
-			log.Println("database loaded:", file)
-		case err := <-s.Api.db.NotifyError():
-			log.Println("database error:", err)
-		case msg := <-s.Api.db.NotifyInfo():
-			log.Println("database info:", msg)
-		case <-s.Api.db.NotifyClose():
-			return
+func (s *Server) watchEvents(updater *updater.Config) {
+	if !s.Config.Silent {
+		for {
+			select {
+			case file := <-updater.NotifyOpen():
+				log.Println("database loaded:", file)
+			case err := <-updater.NotifyError():
+				log.Println("database error:", err)
+			case msg := <-updater.NotifyInfo():
+				log.Println("database info:", msg)
+			case <-updater.NotifyClose():
+				return
+			}
 		}
 	}
 }
@@ -71,7 +74,9 @@ func (s *Server) listenerOpts() []listener.Option {
 }
 
 func (s *Server) runServer(f http.Handler) {
-	log.Println("geoip http server starting on", s.Config.ServerAddr)
+	if !s.Config.Silent {
+		log.Println("geoip http server starting on", s.Config.ServerAddr)
+	}
 	ln, err := listener.New(s.Config.ServerAddr, s.listenerOpts()...)
 	if err != nil {
 		log.Fatal(err)

server/main.go

@@ -2,7 +2,8 @@ package server
 
 import (
 	"../utils/config"
-	"../utils/db"
+	"../utils/i2ldb"
+	"../utils/mmdb"
 	"../utils/tor"
 	"encoding/xml"
 	"github.com/rs/cors"
@@ -32,14 +33,17 @@ type Server struct {
 }
 
 type GeoIpQuery struct {
-	db.DefaultQuery
-	db.ASNDefaultQuery
-	db.TorDefaultQuery
+	i2ldb.ProxyDefaultQuery
+	mmdb.DefaultQuery
+	mmdb.ASNDefaultQuery
+	mmdb.TorDefaultQuery
 }
 
 type responseRecord struct {
 	XMLName     		xml.Name	`xml:"Response" json:"-"`
 	IP          		string  	`json:"ip"`
+	Isp 				string		`json:"isp"`
+	Domain 				string		`json:"domain"`
 	IsInEuropeanUnion 	bool  		`json:"is_in_european_union"`
 	ContinentCode 		string  	`json:"continent_code"`
 	CountryCode 		string  	`json:"country_code"`
@@ -85,12 +89,17 @@ type SystemRecord struct {
 	Desktop   	bool 	`json:"desktop"`
 	Bot   		bool 	`json:"bot"`
 	Tor   		bool 	`json:"tor"`
+	LastSeen 	uint	`json:"last_seen"`
+	UsageType 	string	`json:"usage_type"`
+	ProxyType 	string	`json:"proxy_type"`
+	Proxy 		bool	`json:"proxy"`
 }
 
 type ApiHandler struct {
-	db    *db.DB
-	asnDB *db.DB
+	db    *mmdb.DB
+	asnDB *mmdb.DB
 	torDB *tor.Config
+	i2lDB *i2ldb.Config
 	cors  *cors.Cors
 }
 
@@ -113,18 +122,23 @@ func NewServerConfig(c *config.Config) *Server {
 		RateLimit: NewRateLimit(rate.Limit(c.RateLimitLimit), c.RateLimitBurst),
 
 		Api: &ApiHandler{
-			db: db.NewDefaultConfig(c, c.ProductID),
-			asnDB: db.NewDefaultConfig(c, c.ASNProductID),
+			db:    mmdb.NewDefaultConfig(c, c.MMProductID),
+			asnDB: mmdb.NewDefaultConfig(c, c.MMASNProductID),
 			torDB: tor.NewDefaultConfig(c),
+			i2lDB: i2ldb.NewDefaultConfig(c),
 		},
 	}
 
 	return conf
 }
 
 func (s *Server) Start() {
+	go s.watchEvents(s.Api.torDB.Updater)
+	go s.watchEvents(s.Api.db.Updater)
+	go s.watchEvents(s.Api.asnDB.Updater)
+	go s.watchEvents(s.Api.i2lDB.Updater)
+
 	_ = s.openDB()
-	go s.watchEvents()
 
 	if s.Config.LogToStdout {
 		log.SetOutput(os.Stdout)