From 8716690f368e09a7fbab3bbbf45a0c4f456ce6a5 Mon Sep 17 00:00:00 2001 From: Harry Denholm Date: Sun, 4 Nov 2018 21:55:25 +0000 Subject: [PATCH] big batch of updates adding various security features - manual TLS and Let's Encrypt support / rate-limiting / add-sync live toggling. Bunch of new documentation and some testing on AWS. --- Dockerfile | 3 +- README.md | 49 +++++++++---- config.go | 22 +++++- docker-build | 7 +- main.go | 194 +++++++++++++++++++++++++++++++++++++++++---------- prod.toml | 28 +++++--- 6 files changed, 242 insertions(+), 61 deletions(-) diff --git a/Dockerfile b/Dockerfile index ec0023a..4b8fa4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,6 @@ WORKDIR /app COPY prod.toml /app/ COPY xsyn-deploy /app/ -EXPOSE 8080 +EXPOSE 80 +EXPOSE 443 ENTRYPOINT ["./xsyn-deploy"] diff --git a/README.md b/README.md index e82d43c..9494de1 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,52 @@ -# xSyn +![xSyn Logo](https://raw.githubusercontent.com/ishani/xSyn/master/logo.jpg) + Compact server implementing xBrowserSync API using Golang and BoltDB; supports API version 1.1.4 (Oct 2018) -Easy to deploy via Docker, xSyn provides a lean server for privately hosting your own bookmarks sync store. As of writing, [xBrowserSync](https://www.xbrowsersync.org/) is available for Chrome, Firefox, Android and iOS. It's really good! +Easy to deploy via Docker, xSyn provides a lean server for privately hosting your own bookmarks sync store. As of writing, [xBrowserSync](https://www.xbrowsersync.org/) is available for Chrome, Firefox, Android - It's really good! -### Todo +### Configuring -* Actually check and reject a bundle of data that exceeds the advertised limits -* Rate limiting -* Tests +xSyn pulls configuration from a TOML file during boot and allows environment variable overloads for all the values. Easy to setup and easy to tune. + +Check `prod.toml` for all available settings and override names. + +### Securing + +xSyn can be run unsecured, with TLS via provided keys or automatically secured via *Let's Encrypt*. -As it stands, xSyn works great for a private xBrowserSync server - I've been running with it for about a year - but bear in mind it doesn't have a lot of defence against people trying to flood or attack (yet) +It is possible to run a special route that toggles the `Accepting New Syncs` value while running, so one can open/close the gates on a public server to limit users manually. +Rate-limiting is enabled by default on all routes and is easily configurable. + +-- ### DockerHub -A recent build is available at [hdenholm/xsyn:latest](https://hub.docker.com/r/hdenholm/xsyn/) -If running locally, ensure you map a volume to `/data` so that the BoltDB file persists. +An up-to-date build is available at [hdenholm/xsyn:latest](https://hub.docker.com/r/hdenholm/xsyn/) + +Note that build dates are stamped into the published images, which you can view in the log on startup (with `release_mode` / `XS_SRV_RELEASE` set to false so you can see the Info logs) ### Azure + I have a test instance running on Azure using their slightly restrictive Docker support for App Services. -In Application Settings, make sure `WEBSITES_ENABLE_APP_SERVICE_STORAGE` is enabled. With a default configuration file, add `WEBSITES_PORT` and set it to 8080. +In the portal, navigate to *Application Settings*, make sure `WEBSITES_ENABLE_APP_SERVICE_STORAGE` is enabled. With a default configuration file, add `WEBSITES_PORT` and set it to 80 - remember to update it if you set the `port` / `XS_SRV_PORT` config value. + +Because Azure doesn't let you manually configure volume mapping, we have to override the default file for the BoltDB. Set `XS_BOLT_FILE` to `/home/site/store.db` (or anything under the `/home/site` folder) + +Note that currently there is no way to use *Let's Encrypt* on Azure App Services as of writing as it requires more than one port to be exposed, and Azure doesn't allow this. + +### AWS + +xSyn works on ECS easily, including full *Let's Encrypt* support if you assign an EIP and map it to an owned domain. + +I have a simple example task template [over here](https://gist.github.com/ishani/06a99050500069319493facd31b6576e) - tested, but not a lot. Note this has LE enabled, so either disable that or set your own domain name up before you kick it off. + + +-- +#### Todo + +* Tests -Because Azure doesn't let you manually configure volume mapping, we have to override the default file for the BoltDB. Set `XS_BOLT_FILE` to `/home/site/store.db` (or anything under the /home/site folder) +As it stands, xSyn works great for a private xBrowserSync server - I've been running with it for about a year - and I've poked it about on a few different platforms, but it really needs some actual tests and fuzzing done -Check the `config.go` file for other envvars you can set to modify default behaviour. diff --git a/config.go b/config.go index 002cc95..bc76639 100644 --- a/config.go +++ b/config.go @@ -30,8 +30,9 @@ import ( ) type tomlConfig struct { - Server tomlServer - Bolt tomlBolt + Server tomlServer + Bolt tomlBolt + Security tomlSecurity } type tomlBolt struct { StorageFile string `toml:"file" env:"XS_BOLT_FILE"` @@ -44,6 +45,14 @@ type tomlServer struct { Port int32 `toml:"port" env:"XS_SRV_PORT"` StatusRoute string `toml:"status_route" env:"XS_SRV_STATUS"` } +type tomlSecurity struct { + ReqPerSecond float64 `toml:"max_requests_per_second" env:"XS_SEC_RPS"` + AcceptNewSyncs bool `toml:"accept_new_syncs" env:"XS_SEC_ACCEPT_NEW_SYNC"` + SyncToggleRoute string `toml:"sync_toggle_route" env:"XS_SEC_SYNCTOGGLE"` + TLSCert string `toml:"tls_cert" env:"XS_SEC_TLSCERT"` + UseLetsEncrypt string `toml:"lets_encrypt" env:"XS_SEC_LE"` + LetsEncryptCache string `toml:"lets_encrypt_cache" env:"XS_SEC_LE_CACHE"` +} // AppConfig is the config data parsed from disk var AppConfig tomlConfig @@ -105,7 +114,7 @@ func checkOverrides(configData interface{}, cfgLog *zap.Logger) error { overrideFromEnv := os.Getenv(envOverride) if overrideFromEnv != "" { - cfgLog.Info("Overriding config", + cfgLog.Debug("Overriding config", zap.String("key", envOverride), zap.String("value", overrideFromEnv), ) @@ -121,6 +130,13 @@ func checkOverrides(configData interface{}, cfgLog *zap.Logger) error { } field.Set(reflect.ValueOf(int32(ivalue))) + case reflect.Float64: + fvalue, err := strconv.ParseFloat(overrideFromEnv, 64) + if err != nil { + return err + } + field.Set(reflect.ValueOf(float64(fvalue))) + case reflect.Bool: bvalue, err := strconv.ParseBool(overrideFromEnv) if err != nil { diff --git a/docker-build b/docker-build index 7de7aa3..b5e2564 100644 --- a/docker-build +++ b/docker-build @@ -1,6 +1,11 @@ #!/bin/bash -env GOOS=linux GOARCH=amd64 go build -o xsyn-deploy *.go +# Define a timestamp function +timestamp() { + date +"%Y-%m-%d_%H-%M-%S" +} + +env GOOS=linux GOARCH=amd64 go build -ldflags "-X main.BuildStamp=$(timestamp)" -o xsyn-deploy *.go docker build -f Dockerfile -t hdenholm/xsyn:latest . docker tag xsyn:latest hdenholm/xsyn:latest diff --git a/main.go b/main.go index ccaed94..1c8dfe3 100644 --- a/main.go +++ b/main.go @@ -20,17 +20,28 @@ import ( "fmt" "html/template" "io" + "os" "time" "github.com/boltdb/bolt" + "github.com/didip/tollbooth" + "github.com/didip/tollbooth/limiter" + "github.com/didip/tollbooth_gin" "github.com/fatih/structs" + "github.com/gin-contrib/size" + "github.com/gin-gonic/autotls" "github.com/gin-gonic/gin" uuid "github.com/satori/go.uuid" "go.uber.org/zap" + "golang.org/x/crypto/acme/autocert" ) +// tbd; make the Debug/Prod choice here configurable var zLog, _ = zap.NewProduction() +// BuildStamp can be written to externally during a go build to apply a build-time string, like a timestamp +var BuildStamp string = "[unstamped]" + // names for buckets where we hide our data var boltDataBucket = []byte("BM") var boltTimestampBucket = []byte("TS") @@ -46,11 +57,36 @@ type RequestData struct { EncodedBookmarks string `json:"bookmarks"` } +// by default we accept new sync IDs - ie. new users for the service; +// this can be overridden in the config and toggled live, if required +var newSyncsAllowed = true + +func synAcceptTOS(tosURL string) bool { + zLog.Info("Autocert TOS", zap.String("URL", tosURL)) + return true +} + func main() { // fetch config from toml, apply env overrides, etc LoadConfig() + // log out the build stamp and record when we booted up to show on /stats + // helps me ensure that webhooks et al are firing and servers are up to date as expected + zLog.Info("xSyn", zap.String("Build", BuildStamp)) + bootTime := time.Now().UTC() + + // if a cache path was given for LetsEncrypt, trial-run the creation of it + // so we know early on that the storage has been configured correctly + if len(AppConfig.Security.LetsEncryptCache) > 0 { + if err := os.MkdirAll(AppConfig.Security.LetsEncryptCache, 0700); err != nil { + zLog.Panic("LE cache path test", + zap.String("path", AppConfig.Security.LetsEncryptCache), + zap.Error(err), + ) + } + } + // open or create the Bolt DB storage file db, err := bolt.Open( AppConfig.Bolt.StorageFile, @@ -62,7 +98,7 @@ func main() { } defer db.Close() - // ensure the bucket collection exists + // ensure the bucket collection exists, create them if not err = db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists(boltDataBucket) if err != nil { @@ -82,22 +118,61 @@ func main() { zLog.Panic("Bucket creation", zap.Error(err)) } - // switch to release + db.Sync() + if _, err := os.Stat(AppConfig.Bolt.StorageFile); os.IsNotExist(err) { + zLog.Panic("BoltDB file check", zap.Error(err)) + } + + // switch to release? if AppConfig.Server.ReleaseMode { gin.SetMode(gin.ReleaseMode) } - // line up the routes + // build a Gin instance with default middleware router := gin.Default() + + // apply rate limiting middleware if specified + if AppConfig.Security.ReqPerSecond > 0 { + + zLog.Info("Adding rate-limiting", zap.Float64("RPS", AppConfig.Security.ReqPerSecond)) + + // I've chosen a fairly arbitrary burst limit to allow XBS to poll a few things during a sync without + // exhausting the limits immediately as this limit is applied to all routes + limiter := tollbooth.NewLimiter(AppConfig.Security.ReqPerSecond, &limiter.ExpirableOptions{DefaultExpirationTTL: time.Hour}) + limiter.SetBurst(20) + router.Use(tollbooth_gin.LimitHandler(limiter)) + } + + // magic route to toggle new-sync option + if len(AppConfig.Security.SyncToggleRoute) > 0 { + + zLog.Info("Enabling sync toggling route") + + router.GET(AppConfig.Security.SyncToggleRoute, func(c *gin.Context) { + newSyncsAllowed = !newSyncsAllowed + c.String(200, fmt.Sprintf("Toggled accept_new_syncs to [%t]", newSyncsAllowed)) + }) + } + + // route to create a new sync ID router.POST("/bookmarks", func(c *gin.Context) { + // sorry, we're closed for business + if newSyncsAllowed == false { + c.JSON(409, gin.H{ + "code": "NotAllowed", + "message": "Not accepting new sync users", + }) + return + } + var bookmarkData CreateBookmarkData if err := c.ShouldBindJSON(&bookmarkData); err != nil { handleError(c, "MissingParameter", "/bookmarks POST missing", err) return } - zLog.Info("New SyncID requested", zap.String("Client", bookmarkData.ClientVersion)) + zLog.Debug("New SyncID requested", zap.String("Client", bookmarkData.ClientVersion)) newID := "invalid" imprintTime := createTimestampString() @@ -140,7 +215,12 @@ func main() { // will loop forever, paranoia suggests we should have // a counter and terminate after N runs uniqueIDRetryCount++ - zLog.Info("Duplicate UUID, retrying", zap.Int("Count", uniqueIDRetryCount)) + zLog.Warn("Duplicate UUID, retrying", zap.Int("Count", uniqueIDRetryCount)) + + // .. so do that + if uniqueIDRetryCount > 8 { + return fmt.Errorf("too many UUID collisions") + } } // copy out the ID @@ -162,7 +242,7 @@ func main() { return } - zLog.Info("New key created", zap.String("key", newID)) + zLog.Debug("New key created", zap.String("key", newID)) c.JSON(200, gin.H{ "id": newID, @@ -219,41 +299,46 @@ func main() { }) }) - // replace bookmarks data for the given SyncID - router.PUT("/bookmarks/:id", func(c *gin.Context) { - markID := c.Param("id") - markIDBytes := []byte(markID) + maxSyncSizeBytes := int64(1024 * AppConfig.Server.MaxSyncSizeKb) - var bookmarkData RequestData - if err := c.ShouldBindJSON(&bookmarkData); err != nil { - handleError(c, "MissingParameter", "No bookmarks provided", err) - return - } + sizeLimitedRoutes := router.Group("/", limits.RequestSizeLimiter(maxSyncSizeBytes)) + { + // replace bookmarks data for the given SyncID + sizeLimitedRoutes.PUT("/bookmarks/:id", func(c *gin.Context) { + markID := c.Param("id") + markIDBytes := []byte(markID) - imprintTime := createTimestampString() + var bookmarkData RequestData + if err := c.ShouldBindJSON(&bookmarkData); err != nil { + handleError(c, "MissingParameter", "No bookmarks provided", err) + return + } - err = db.Update(func(tx *bolt.Tx) error { + imprintTime := createTimestampString() - bkData := tx.Bucket(boltDataBucket) + err = db.Update(func(tx *bolt.Tx) error { - if err = bkData.Put(markIDBytes, []byte(bookmarkData.EncodedBookmarks)); err != nil { - return err - } + bkData := tx.Bucket(boltDataBucket) - bkTs := tx.Bucket(boltTimestampBucket) + if err = bkData.Put(markIDBytes, []byte(bookmarkData.EncodedBookmarks)); err != nil { + return err + } - err = bkTs.Put(markIDBytes, []byte(imprintTime)) - return err - }) + bkTs := tx.Bucket(boltTimestampBucket) - if handleError(c, "InternalError", "", err) { - return - } + err = bkTs.Put(markIDBytes, []byte(imprintTime)) + return err + }) - c.JSON(200, gin.H{ - "lastUpdated": imprintTime, + if handleError(c, "InternalError", "", err) { + return + } + + c.JSON(200, gin.H{ + "lastUpdated": imprintTime, + }) }) - }) + } // return the timestamp of the last update for the given SyncID router.GET("/bookmarks/:id/lastUpdated", func(c *gin.Context) { @@ -314,11 +399,18 @@ func main() { }) router.GET("/info", func(c *gin.Context) { + + serviceStatus := 1 + if newSyncsAllowed == false { + serviceStatus = 3 + } + c.JSON(200, gin.H{ - "status": 1, + "status": serviceStatus, "message": AppConfig.Server.ServiceMessage, - "version": "1.1.4", - "maxSyncSize": 1024 * AppConfig.Server.MaxSyncSizeKb, + "version": "1.1.5", + "buildstamp": BuildStamp, + "maxSyncSize": maxSyncSizeBytes, }) }) @@ -367,7 +459,9 @@ func main() { dbstat := make(map[string]interface{}) dbstat["key count"] = keyCount dbstat["db size (bytes)"] = dbSize - datamap["Data Bucket"] = dbstat + dbstat["build stamp"] = BuildStamp + dbstat["boot time"] = bootTime.Format(time.RFC850) + datamap["State"] = dbstat // .. and then the other maps extracted from bolt datamap["Bolt-Db"] = stats @@ -386,7 +480,35 @@ func main() { }) launchString := fmt.Sprintf(":%d", AppConfig.Server.Port) - router.Run(launchString) + + if len(AppConfig.Security.TLSCert) > 0 { + + zLog.Info("Starting server", zap.String("mode", "https")) + + zLog.Fatal("exited", zap.Error(router.RunTLS( + launchString, + fmt.Sprintf("%s.pem", AppConfig.Security.TLSCert), + fmt.Sprintf("%s.key", AppConfig.Security.TLSCert), + ))) + + } else if len(AppConfig.Security.UseLetsEncrypt) > 0 { + + zLog.Info("Starting server", zap.String("mode", "https-lets-encrypt")) + + autocertmgr := autocert.Manager{ + Prompt: synAcceptTOS, + HostPolicy: autocert.HostWhitelist(AppConfig.Security.UseLetsEncrypt), + Cache: autocert.DirCache(AppConfig.Security.LetsEncryptCache), + } + + zLog.Fatal("exited", zap.Error(autotls.RunWithManager(router, &autocertmgr))) + + } else { + + zLog.Info("Starting server", zap.String("mode", "http")) + + zLog.Fatal("exited", zap.Error(router.Run(launchString))) + } } // xbs seems to want a 409 when things go wrong; this is a simple wrapper to generate diff --git a/prod.toml b/prod.toml index 34b34a0..d2aa676 100644 --- a/prod.toml +++ b/prod.toml @@ -1,12 +1,24 @@ +# configuration k:v # envvar override # usage + [server] -release_mode = true -service_message = "Hello from [github.com/ishani/xSyn] the compact Go server for xBrowserSync" -max_sync_size_kb = 1000 -port = 8080 -status_route = "/stat" +service_message = "Hello from [github.com/ishani/xSyn], the compact Go server for xBrowserSync" +release_mode = true # XS_SRV_RELEASE # run Gin in release mode, hiding all debug and verbose logging +max_sync_size_kb = 500 # XS_SRV_MAXSYNC # maximum amount of data to allow per SyncID (remember it comes in compressed, so 500kb is _a lot_ of bookmarks) +port = 80 # XS_SRV_PORT # port to serve on; + # NOTE: ..unless in Lets Encrypt mode, in which case both :80 and :443 are used and cannot be overridden +status_route = "/stat" # XS_SRV_STATUS # route that shows more comprehensive server stats; obfuscate this if you like + +[security] +max_requests_per_second = 1.5 # XS_SEC_RPS # set to <= 0 to disable rate-limiting, otherwise N rps +accept_new_syncs = true # XS_SEC_ACCEPT_NEW_SYNC + # false to disable any new XBS SyncIDs to be made (ie. no new users) +sync_toggle_route = "" # XS_SEC_SYNCTOGGLE # path that, if visited, toggles the runtime state of 'Accept New Syncs'. Set to "" to disable this feature. + # example : "/vnXNXLZU4oSzVnEFjnmfQ9cBBYUkTu" +tls_cert = "" # XS_SEC_TLSCERT # file prefix for SSL certs - if supplied, runs with TLS (eg. MyCert.pem and MyCert.key) + # NOTE: this takes priority over LE options below +lets_encrypt = "" # XS_SEC_LE # supply a domain name to enable autotls manager; uses go's autocert acme library +lets_encrypt_cache = "" # XS_SEC_LE_CACHE # path to directory to store LE cache, or "" to use in-memory cache (not generally recommended) [bolt] -file = "marks.db" +file = "marks.db" # XS_BOLT_FILE # path to where to store the database init_timeout = 5 - -