diff --git a/modules/caddytls/matchers.go b/modules/caddytls/matchers.go
index aee0e726e3a..d41492ffe22 100644
--- a/modules/caddytls/matchers.go
+++ b/modules/caddytls/matchers.go
@@ -86,14 +86,14 @@ func (m *MatchRemoteIP) Provision(ctx caddy.Context) error {
 		if err != nil {
 			return err
 		}
-		m.cidrs = cidrs
+		m.cidrs = append(m.cidrs, cidrs...)
 	}
 	for _, str := range m.NotRanges {
 		cidrs, err := m.parseIPRange(str)
 		if err != nil {
 			return err
 		}
-		m.notCidrs = cidrs
+		m.notCidrs = append(m.notCidrs, cidrs...)
 	}
 	return nil
 }