Merge pull request #8 from ties/feature/rtrmon-add-last-modified-support

Add last modified support

cmd/rtrmon/rtrmon.go

@@ -44,7 +44,8 @@ var (
 	MetricsPath = flag.String("metrics", "/metrics", "Metrics path")
 	OutFile     = flag.String("file", "diff.json", "Diff file (or URL path without /)")
 
-	UserAgent = flag.String("useragent", fmt.Sprintf("StayRTR-%v (+https://github.com/bgp/stayrtr)", AppVersion), "User-Agent header")
+	UserAgent                  = flag.String("useragent", fmt.Sprintf("StayRTR-%v (+https://github.com/bgp/stayrtr)", AppVersion), "User-Agent header")
+	DisableConditionalRequests = flag.Bool("disable.conditional.requests", false, "Disable conditional requests (using If-None-Match/If-Modified-Since headers)")
 
 	PrimaryHost            = flag.String("primary.host", "tcp://rtr.rpki.cloudflare.com:8282", "primary server")
 	PrimaryValidateCert    = flag.Bool("primary.tls.validate", true, "Validate TLS")
@@ -685,6 +686,8 @@ func main() {
 	log.SetLevel(lvl)
 
 	fc := utils.NewFetchConfig()
+	fc.EnableEtags = !*DisableConditionalRequests
+	fc.EnableLastModified = !*DisableConditionalRequests
 	fc.UserAgent = *UserAgent
 
 	c1 := NewClient()

cmd/stayrtr/stayrtr.go

@@ -12,13 +12,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	rtr "github.com/bgp/stayrtr/lib"
-	"github.com/bgp/stayrtr/prefixfile"
-	"github.com/bgp/stayrtr/utils"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/crypto/ssh"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -28,6 +21,14 @@ import (
 	"sync"
 	"syscall"
 	"time"
+
+	rtr "github.com/bgp/stayrtr/lib"
+	"github.com/bgp/stayrtr/prefixfile"
+	"github.com/bgp/stayrtr/utils"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/crypto/ssh"
 )
 
 const (
@@ -85,6 +86,7 @@ var (
 	UseSerial = flag.String("useserial", "disable", "Use serial contained in file (disable, startup, full)")
 
 	Etag            = flag.Bool("etag", true, "Enable Etag header")
+	LastModified    = flag.Bool("last.modified", true, "Enable Last-Modified header")
 	UserAgent       = flag.String("useragent", fmt.Sprintf("StayRTR-%v (+https://github.com/bgp/stayrtr)", AppVersion), "User-Agent header")
 	Mime            = flag.String("mime", "application/json", "Accept setting format (some servers may prefer text/json)")
 	RefreshInterval = flag.Int("refresh", 600, "Refresh interval in seconds")
@@ -589,6 +591,7 @@ func main() {
 	s.fetchConfig.UserAgent = *UserAgent
 	s.fetchConfig.Mime = *Mime
 	s.fetchConfig.EnableEtags = *Etag
+	s.fetchConfig.EnableLastModified = *LastModified
 
 	if serialId, ok := serialToId[*UseSerial]; ok {
 		s.useSerial = serialId

utils/utils.go

@@ -7,24 +7,27 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"time"
 	"sync"
+	"time"
 )
 
 type FetchConfig struct {
 	UserAgent string
 	Mime      string
 
-	etags       map[string]string
-	etagsLock   *sync.RWMutex
-	EnableEtags bool
+	etags                  map[string]string
+	lastModified           map[string]time.Time
+	conditionalRequestLock *sync.RWMutex
+	EnableEtags            bool
+	EnableLastModified     bool
 }
 
 func NewFetchConfig() *FetchConfig {
 	return &FetchConfig{
-		etags: make(map[string]string),
-		etagsLock: &sync.RWMutex{},
-		Mime: "application/json",
+		etags:                  make(map[string]string),
+		lastModified:           make(map[string]time.Time),
+		conditionalRequestLock: &sync.RWMutex{},
+		Mime:                   "application/json",
 	}
 }
 
@@ -75,12 +78,20 @@ func (c *FetchConfig) FetchFile(file string) ([]byte, int, bool, error) {
 			req.Header.Set("Accept", c.Mime)
 		}
 
-		c.etagsLock.RLock()
-		etag, ok := c.etags[file]
-		c.etagsLock.RUnlock()
-		if c.EnableEtags && ok {
-			req.Header.Set("If-None-Match", etag)
+		c.conditionalRequestLock.RLock()
+		if c.EnableEtags {
+			etag, ok := c.etags[file]
+			if ok {
+				req.Header.Set("If-None-Match", etag)
+			}
+		}
+		if c.EnableLastModified {
+			lastModified, ok := c.lastModified[file]
+			if ok {
+				req.Header.Set("If-Modified-Since", lastModified.UTC().Format(http.TimeFormat))
+			}
 		}
+		c.conditionalRequestLock.RUnlock()
 
 		proxyurl, err := http.ProxyFromEnvironment(req)
 		if err != nil {
@@ -109,9 +120,10 @@ func (c *FetchConfig) FetchFile(file string) ([]byte, int, bool, error) {
 				File: file,
 			}
 		} else if fhttp.StatusCode != 200 {
-			c.etagsLock.Lock()
+			c.conditionalRequestLock.Lock()
 			delete(c.etags, file)
-			c.etagsLock.Unlock()
+			delete(c.lastModified, file)
+			c.conditionalRequestLock.Unlock()
 			return nil, fhttp.StatusCode, true, fmt.Errorf("HTTP %s", fhttp.Status)
 		}
 		//LastRefresh.WithLabelValues(file).Set(float64(s.lastts.UnixNano() / 1e9))
@@ -121,15 +133,28 @@ func (c *FetchConfig) FetchFile(file string) ([]byte, int, bool, error) {
 		newEtag := fhttp.Header.Get("ETag")
 
 		if !c.EnableEtags || newEtag == "" || newEtag != c.etags[file] { // check lock here
-			c.etagsLock.Lock()
+			c.conditionalRequestLock.Lock()
 			c.etags[file] = newEtag
-			c.etagsLock.Unlock()
+			c.conditionalRequestLock.Unlock()
 		} else {
 			return nil, fhttp.StatusCode, true, IdenticalEtag{
 				File: file,
 				Etag: newEtag,
 			}
 		}
+
+		if c.EnableLastModified {
+			// Accept any valid Last-Modified values. Because of the 1s resolution,
+			// getting the same value is not an error (c.f. the IdenticalEtag error).
+			ifModifiedSince, err := http.ParseTime(fhttp.Header.Get("Last-Modified"))
+			c.conditionalRequestLock.Lock()
+			if err == nil {
+				c.lastModified[file] = ifModifiedSince
+			} else {
+				delete(c.lastModified, file)
+			}
+			c.conditionalRequestLock.Unlock()
+		}
 	} else {
 		f, err = os.Open(file)
 		if err != nil {