Runtime placeholders (#224)

* Runtime placeholders: remove IPs validation in UnmarshalCaddyfile

* Runtime placeholders: add placeholder replacement for IPs in Provision

* Runtime placeholders: add placeholder replacement for other strings

* Runtime placeholders: update README

* Runtime placeholders: remove unused ParseNetworks

README.md

@@ -583,3 +583,14 @@ While only allowing connections from a specific network and requiring a username
 }
 ```
 </details>
+
+## Placeholders support
+
+Environment variables having `{$VAR}` syntax are supported in Caddyfile only. They are evaluated once at launch before Caddyfile is parsed.
+
+Runtime placeholders having `{...}` syntax, including environment variables referenced as `{env.VAR}`, are supported in both Caddyfile and pure JSON, with some caveats described below.
+- Options of *int*, *float*, *big.int*, *duration*, and other numeric types don't support runtime placeholders at all.
+- Options of *string* type containing IPs or CIDRs (e.g. `dial` in `upstream` of `proxy` handler), regular expressions (e.g. `cookie_hash_regexp` of `rdp` matcher), or special values (e.g. `commands` and `credentials` of `socks5` handler)  support runtime placeholders, but they are evaluated __once at provision__ due to the existing optimizations.
+- Other options of *string* type (e.g. `alpn` of `tls` matcher) generally support runtime placeholders, and they are evaluated __each time at match or handle__. However, there are some exceptions, e.g. `tls_*` options inside `upstream` of `proxy` handler, and all options inside `connection_policy` of `tls` handler, that don't support runtime placeholders at all.
+
+Please note that runtime placeholders support depends on handler/matcher implementations. Given some matchers and handlers are outside of this repository, it's up to their developers to support or restrict usage of runtime placeholders.

layer4/matchers.go

@@ -19,10 +19,10 @@ import (
 	"fmt"
 	"net"
 	"net/netip"
-	"strings"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"go.uber.org/zap"
 )
 
@@ -125,10 +125,15 @@ func (*MatchRemoteIP) CaddyModule() caddy.ModuleInfo {
 }
 
 // Provision parses m's IP ranges, either from IP or CIDR expressions.
-func (m *MatchRemoteIP) Provision(_ caddy.Context) (err error) {
-	m.cidrs, err = ParseNetworks(m.Ranges)
-	if err != nil {
-		return err
+func (m *MatchRemoteIP) Provision(_ caddy.Context) error {
+	repl := caddy.NewReplacer()
+	for _, addrOrCIDR := range m.Ranges {
+		addrOrCIDR = repl.ReplaceAll(addrOrCIDR, "")
+		prefix, err := caddyhttp.CIDRExpressionToPrefix(addrOrCIDR)
+		if err != nil {
+			return err
+		}
+		m.cidrs = append(m.cidrs, prefix)
 	}
 	return nil
 }
@@ -173,13 +178,13 @@ func (m *MatchRemoteIP) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 		return d.ArgErr()
 	}
 
-	prefixes, err := ParseNetworks(d.RemainingArgs())
-	if err != nil {
-		return err
-	}
-
-	for _, prefix := range prefixes {
-		m.Ranges = append(m.Ranges, prefix.String())
+	for d.NextArg() {
+		val := d.Val()
+		if val == "private_ranges" {
+			m.Ranges = append(m.Ranges, caddyhttp.PrivateRangesCIDR()...)
+			continue
+		}
+		m.Ranges = append(m.Ranges, val)
 	}
 
 	// No blocks are supported
@@ -207,11 +212,15 @@ func (*MatchLocalIP) CaddyModule() caddy.ModuleInfo {
 
 // Provision parses m's IP ranges, either from IP or CIDR expressions.
 func (m *MatchLocalIP) Provision(_ caddy.Context) error {
-	ipnets, err := ParseNetworks(m.Ranges)
-	if err != nil {
-		return err
+	repl := caddy.NewReplacer()
+	for _, addrOrCIDR := range m.Ranges {
+		addrOrCIDR = repl.ReplaceAll(addrOrCIDR, "")
+		prefix, err := caddyhttp.CIDRExpressionToPrefix(addrOrCIDR)
+		if err != nil {
+			return err
+		}
+		m.cidrs = append(m.cidrs, prefix)
 	}
-	m.cidrs = ipnets
 	return nil
 }
 
@@ -255,13 +264,13 @@ func (m *MatchLocalIP) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 		return d.ArgErr()
 	}
 
-	prefixes, err := ParseNetworks(d.RemainingArgs())
-	if err != nil {
-		return err
-	}
-
-	for _, prefix := range prefixes {
-		m.Ranges = append(m.Ranges, prefix.String())
+	for d.NextArg() {
+		val := d.Val()
+		if val == "private_ranges" {
+			m.Ranges = append(m.Ranges, caddyhttp.PrivateRangesCIDR()...)
+			continue
+		}
+		m.Ranges = append(m.Ranges, val)
 	}
 
 	// No blocks are supported
@@ -393,29 +402,3 @@ var (
 	_ ConnMatcher           = (*MatchNot)(nil)
 	_ caddyfile.Unmarshaler = (*MatchNot)(nil)
 )
-
-// ParseNetworks parses a list of string IP addresses or CIDR subnets into a slice of net.IPNet's.
-// It accepts for example ["127.0.0.1", "127.0.0.0/8", "::1", "2001:db8::/32"].
-func ParseNetworks(networks []string) (ipNets []netip.Prefix, err error) {
-	for _, str := range networks {
-		if strings.Contains(str, "/") {
-			ipNet, err := netip.ParsePrefix(str)
-			if err != nil {
-				return nil, fmt.Errorf("parsing CIDR expression: %v", err)
-			}
-			ipNets = append(ipNets, ipNet)
-			continue
-		}
-
-		addr, err := netip.ParseAddr(str)
-		if err != nil {
-			return nil, err
-		}
-		bits := 32
-		if addr.Is6() {
-			bits = 128
-		}
-		ipNets = append(ipNets, netip.PrefixFrom(addr, bits))
-	}
-	return ipNets, nil
-}

layer4/server.go

@@ -56,7 +56,9 @@ func (s *Server) Provision(ctx caddy.Context, logger *zap.Logger) error {
 		s.MatchingTimeout = caddy.Duration(MatchingTimeoutDefault)
 	}
 
+	repl := caddy.NewReplacer()
 	for i, address := range s.Listen {
+		address = repl.ReplaceAll(address, "")
 		addr, err := caddy.ParseNetworkAddress(address)
 		if err != nil {
 			return fmt.Errorf("parsing listener address '%s' in position %d: %v", address, i, err)
@@ -182,7 +184,7 @@ func (s *Server) handle(conn net.Conn) {
 
 // UnmarshalCaddyfile sets up the Server from Caddyfile tokens. Syntax:
 //
-//	<addresses> {
+//	<address:port> [<address:port>] {
 //		matching_timeout <duration>
 //		@a <matcher> [<matcher_args>]
 //		@b {
@@ -205,11 +207,7 @@ func (s *Server) handle(conn net.Conn) {
 func (s *Server) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	// Wrapper name and all same-line options are treated as network addresses
 	for ok := true; ok; ok = d.NextArg() {
-		addr := d.Val()
-		if _, err := caddy.ParseNetworkAddress(addr); err != nil {
-			return d.Errf("parsing network address '%s': %v", addr, err)
-		}
-		s.Listen = append(s.Listen, addr)
+		s.Listen = append(s.Listen, d.Val())
 	}
 
 	if err := ParseCaddyfileNestedRoutes(d, &s.Routes, &s.MatchingTimeout); err != nil {

modules/l4proxy/proxy.go

@@ -57,6 +57,8 @@ type Handler struct {
 	// Ref: https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
 	ProxyProtocol string `json:"proxy_protocol,omitempty"`
 
+	proxyProtocolVersion uint8
+
 	ctx    caddy.Context
 	logger *zap.Logger
 }
@@ -83,8 +85,14 @@ func (h *Handler) Provision(ctx caddy.Context) error {
 		h.LoadBalancing.SelectionPolicy = mod.(Selector)
 	}
 
-	if h.ProxyProtocol != "" && h.ProxyProtocol != "v1" && h.ProxyProtocol != "v2" {
-		return fmt.Errorf("proxy_protocol: \"%s\" should be empty, or one of \"v1\" \"v2\"", h.ProxyProtocol)
+	repl := caddy.NewReplacer()
+	proxyProtocol := repl.ReplaceAll(h.ProxyProtocol, "")
+	if proxyProtocol == "v1" {
+		h.proxyProtocolVersion = 1
+	} else if proxyProtocol == "v2" {
+		h.proxyProtocolVersion = 2
+	} else if proxyProtocol != "" {
+		return fmt.Errorf("proxy_protocol: \"%s\" should be empty, or one of \"v1\" \"v2\"", proxyProtocol)
 	}
 
 	// prepare upstreams
@@ -220,12 +228,12 @@ func (h *Handler) dialPeers(upstream *Upstream, repl *caddy.Replacer, down *laye
 		// Send the PROXY protocol header.
 		if err == nil {
 			downConn := l4proxyprotocol.GetConn(down)
-			switch h.ProxyProtocol {
-			case "v1":
+			switch h.proxyProtocolVersion {
+			case 1:
 				var h proxyprotocol.HeaderV1
 				h.FromConn(downConn, false)
 				_, err = h.WriteTo(up)
-			case "v2":
+			case 2:
 				var h proxyprotocol.HeaderV2
 				h.FromConn(downConn, false)
 				_, err = h.WriteTo(up)
@@ -401,13 +409,8 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	_, wrapper := d.Next(), d.Val() // consume wrapper name
 
 	// Treat all same-line options as upstream addresses
-	for i := 0; d.NextArg(); i++ {
-		val := d.Val()
-		_, err := caddy.ParseNetworkAddress(val)
-		if err != nil {
-			return d.Errf("parsing %s upstream on position %d: %v", wrapper, i, err)
-		}
-		h.Upstreams = append(h.Upstreams, &Upstream{Dial: []string{val}})
+	for d.NextArg() {
+		h.Upstreams = append(h.Upstreams, &Upstream{Dial: []string{d.Val()}})
 	}
 
 	var (
@@ -591,13 +594,6 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				return d.Errf("duplicate %s option '%s'", wrapper, optionName)
 			}
 			_, h.ProxyProtocol, hasProxyProtocol = d.NextArg(), d.Val(), true
-			switch h.ProxyProtocol {
-			case "v1", "v2":
-				continue
-			default:
-				return d.Errf("malformed %s option '%s': unrecognized value '%s'",
-					wrapper, optionName, h.ProxyProtocol)
-			}
 		case "upstream":
 			u := &Upstream{}
 			if err := u.UnmarshalCaddyfile(d.NewFromNextSegment()); err != nil {

modules/l4proxy/upstream.go

@@ -56,19 +56,23 @@ func (u *Upstream) String() string {
 }
 
 func (u *Upstream) provision(ctx caddy.Context, h *Handler) error {
+	repl := caddy.NewReplacer()
 	for _, dialAddr := range u.Dial {
+		// replace runtime placeholders
+		replDialAddr := repl.ReplaceAll(dialAddr, "")
+
 		// parse and validate address
-		addr, err := caddy.ParseNetworkAddress(dialAddr)
+		addr, err := caddy.ParseNetworkAddress(replDialAddr)
 		if err != nil {
 			return err
 		}
 		if addr.PortRangeSize() != 1 {
-			return fmt.Errorf("%s: port ranges not currently supported", dialAddr)
+			return fmt.Errorf("%s: port ranges not currently supported", replDialAddr)
 		}
 
 		// create or load peer info
 		p := &peer{address: addr}
-		existingPeer, loaded := peers.LoadOrStore(dialAddr, p)
+		existingPeer, loaded := peers.LoadOrStore(dialAddr, p) // peers are deleted in Handler.Cleanup
 		if loaded {
 			p = existingPeer.(*peer)
 		}
@@ -368,13 +372,7 @@ func (u *Upstream) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 	if len(shortcutArgs) == 0 {
 		return d.Errf("malformed %s block: at least one %s address must be provided", wrapper, shortcutOptionName)
 	}
-	for _, arg := range shortcutArgs {
-		_, err := caddy.ParseNetworkAddress(arg)
-		if err != nil {
-			return d.Errf("parsing %s option '%s': %v", wrapper, shortcutOptionName, err)
-		}
-		u.Dial = append(u.Dial, arg)
-	}
+	u.Dial = append(u.Dial, shortcutArgs...)
 
 	return nil
 }

modules/l4proxyprotocol/handler.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	"github.com/mastercactapus/proxyprotocol"
 	"go.uber.org/zap"
 
@@ -55,10 +56,12 @@ func (*Handler) CaddyModule() caddy.ModuleInfo {
 
 // Provision sets up the module.
 func (h *Handler) Provision(ctx caddy.Context) error {
-	for _, s := range h.Allow {
-		_, n, err := net.ParseCIDR(s)
+	repl := caddy.NewReplacer()
+	for _, allowCIDR := range h.Allow {
+		allowCIDR = repl.ReplaceAll(allowCIDR, "")
+		_, n, err := net.ParseCIDR(allowCIDR)
 		if err != nil {
-			return fmt.Errorf("invalid subnet '%s': %w", s, err)
+			return fmt.Errorf("invalid subnet '%s': %w", allowCIDR, err)
 		}
 		h.rules = append(h.rules, proxyprotocol.Rule{Timeout: time.Duration(h.Timeout), Subnet: n})
 	}
@@ -190,12 +193,13 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 			if d.CountRemainingArgs() == 0 {
 				return d.ArgErr()
 			}
-			prefixes, err := layer4.ParseNetworks(d.RemainingArgs())
-			if err != nil {
-				return d.Errf("parsing %s option '%s': %v", wrapper, optionName, err)
-			}
-			for _, prefix := range prefixes {
-				h.Allow = append(h.Allow, prefix.String())
+			for d.NextArg() {
+				val := d.Val()
+				if val == "private_ranges" {
+					h.Allow = append(h.Allow, caddyhttp.PrivateRangesCIDR()...)
+					continue
+				}
+				h.Allow = append(h.Allow, val)
 			}
 		case "timeout":
 			if hasTimeout {