feat: Implement the admission server and a validation webhook for plu…

…gins (#573)

cmd/ingress/ingress.go

@@ -139,6 +139,7 @@ the apisix cluster and others are created`,
 	cmd.PersistentFlags().StringVar(&cfg.LogLevel, "log-level", "info", "error log level")
 	cmd.PersistentFlags().StringVar(&cfg.LogOutput, "log-output", "stderr", "error log output file")
 	cmd.PersistentFlags().StringVar(&cfg.HTTPListen, "http-listen", ":8080", "the HTTP Server listen address")
+	cmd.PersistentFlags().StringVar(&cfg.HTTPSListen, "https-listen", ":8443", "the HTTPS Server listen address")
 	cmd.PersistentFlags().BoolVar(&cfg.EnableProfiling, "enable-profiling", true, "enable profiling via web interface host:port/debug/pprof")
 	cmd.PersistentFlags().StringVar(&cfg.Kubernetes.Kubeconfig, "kubeconfig", "", "Kubernetes configuration file (by default in-cluster configuration will be used)")
 	cmd.PersistentFlags().DurationVar(&cfg.Kubernetes.ResyncInterval.Duration, "resync-interval", time.Minute, "the controller resync (with Kubernetes) interval, the minimum resync interval is 30s")

conf/config-default.yaml

@@ -27,7 +27,11 @@ log_output: "stderr" # the output file path of error log, default is stderr, whe
                      # are marshalled in JSON format, which can be parsed by
                      # programs easily.
 
+cert_file: "/etc/webhook/certs/cert.pem" # the TLS certificate file path.
+key_file: "/etc/webhook/certs/key.pem"   # the TLS key file path.
+
 http_listen: ":8080"   # the HTTP Server listen address, default is ":8080"
+https_listen: ":8443"   # the HTTPS Server listen address, default is ":8443"
 enable_profiling: true # enable profiling via web interfaces
                        # host:port/debug/pprof, default is true.
 

go.mod

@@ -3,6 +3,7 @@ module github.com/apache/apisix-ingress-controller
 go 1.13
 
 require (
+	github.com/fsnotify/fsnotify v1.5.0 // indirect
 	github.com/gin-gonic/gin v1.6.3
 	github.com/google/uuid v1.2.0 // indirect
 	github.com/hashicorp/go-memdb v1.0.4
@@ -11,15 +12,17 @@ require (
 	github.com/imdario/mergo v0.3.11 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/onsi/ginkgo v1.16.4 // indirect
-	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_golang v1.10.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/procfs v0.2.0 // indirect
+	github.com/slok/kubewebhook/v2 v2.1.0
 	github.com/spf13/cobra v1.1.1
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
+	github.com/xeipuuv/gojsonschema v1.2.0
 	go.uber.org/multierr v1.3.0
 	go.uber.org/zap v1.13.0
 	golang.org/x/net v0.0.0-20210510120150-4163338589ed
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20210819072135-bce67f096156 // indirect
 	golang.org/x/tools v0.1.5 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.21.1

pkg/api/router/router.go

@@ -12,6 +12,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package router
 
 import (

pkg/api/router/router_test.go

@@ -22,6 +22,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/apache/apisix-ingress-controller/pkg/apisix"
 )
 
 func TestHealthz(t *testing.T) {
@@ -50,3 +52,14 @@ func TestMetrics(t *testing.T) {
 
 	assert.Equal(t, w.Code, http.StatusOK)
 }
+
+func TestWebhooks(t *testing.T) {
+	w := httptest.NewRecorder()
+	c, r := gin.CreateTestContext(w)
+	req, err := http.NewRequest("POST", "/validation", nil)
+	assert.Nil(t, err, nil)
+	c.Request = req
+	MountWebhooks(r, &apisix.ClusterOptions{})
+
+	assert.Equal(t, w.Code, http.StatusOK)
+}

pkg/api/router/webhook.go

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+	"github.com/gin-gonic/gin"
+
+	"github.com/apache/apisix-ingress-controller/pkg/api/validation"
+	"github.com/apache/apisix-ingress-controller/pkg/apisix"
+)
+
+// MountWebhooks mounts webhook related routes.
+func MountWebhooks(r *gin.Engine, co *apisix.ClusterOptions) {
+	// init the schema client, it will be used to query schema of objects.
+	_, _ = validation.GetSchemaClient(co)
+	r.POST("/validation/apisixroutes/plugin", gin.WrapH(validation.NewPluginValidatorHandler()))
+}

pkg/api/server.go

@@ -12,26 +12,33 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package api
 
 import (
+	"context"
+	"crypto/tls"
 	"net"
 	"net/http"
 	"net/http/pprof"
+	"time"
 
 	"github.com/gin-gonic/gin"
+	"go.uber.org/zap"
 
 	apirouter "github.com/apache/apisix-ingress-controller/pkg/api/router"
+	"github.com/apache/apisix-ingress-controller/pkg/apisix"
 	"github.com/apache/apisix-ingress-controller/pkg/config"
 	"github.com/apache/apisix-ingress-controller/pkg/log"
 	"github.com/apache/apisix-ingress-controller/pkg/types"
 )
 
 // Server represents the API Server in ingress-apisix-controller.
 type Server struct {
-	router       *gin.Engine
-	httpListener net.Listener
-	pprofMu      *http.ServeMux
+	httpServer      *gin.Engine
+	admissionServer *http.Server
+	httpListener    net.Listener
+	pprofMu         *http.ServeMux
 }
 
 // NewServer initializes the API Server.
@@ -41,23 +48,47 @@ func NewServer(cfg *config.Config) (*Server, error) {
 		return nil, err
 	}
 	gin.SetMode(gin.ReleaseMode)
-	router := gin.New()
-	router.Use(gin.Recovery(), gin.Logger())
-	apirouter.Mount(router)
+	httpServer := gin.New()
+	httpServer.Use(gin.Recovery(), gin.Logger())
+	apirouter.Mount(httpServer)
 
 	srv := &Server{
-		router:       router,
+		httpServer:   httpServer,
 		httpListener: httpListener,
 	}
-
 	if cfg.EnableProfiling {
 		srv.pprofMu = new(http.ServeMux)
 		srv.pprofMu.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 		srv.pprofMu.HandleFunc("/debug/pprof/profile", pprof.Profile)
 		srv.pprofMu.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 		srv.pprofMu.HandleFunc("/debug/pprof/trace", pprof.Trace)
 		srv.pprofMu.HandleFunc("/debug/pprof/", pprof.Index)
-		router.GET("/debug/pprof/*profile", gin.WrapF(srv.pprofMu.ServeHTTP))
+		httpServer.GET("/debug/pprof/*profile", gin.WrapF(srv.pprofMu.ServeHTTP))
+	}
+
+	cert, err := tls.LoadX509KeyPair(cfg.CertFilePath, cfg.KeyFilePath)
+	if err != nil {
+		log.Warnw("failed to load x509 key pair, will not start admission server",
+			zap.String("Error", err.Error()),
+			zap.String("CertFilePath", cfg.CertFilePath),
+			zap.String("KeyFilePath", cfg.KeyFilePath),
+		)
+	} else {
+		admission := gin.New()
+		admission.Use(gin.Recovery(), gin.Logger())
+		apirouter.MountWebhooks(admission, &apisix.ClusterOptions{
+			Name:     cfg.APISIX.DefaultClusterName,
+			AdminKey: cfg.APISIX.DefaultClusterAdminKey,
+			BaseURL:  cfg.APISIX.DefaultClusterBaseURL,
+		})
+
+		srv.admissionServer = &http.Server{
+			Addr:    cfg.HTTPSListen,
+			Handler: admission,
+			TLSConfig: &tls.Config{
+				Certificates: []tls.Certificate{cert},
+			},
+		}
 	}
 
 	return srv, nil
@@ -67,13 +98,57 @@ func NewServer(cfg *config.Config) (*Server, error) {
 func (srv *Server) Run(stopCh <-chan struct{}) error {
 	go func() {
 		<-stopCh
-		if err := srv.httpListener.Close(); err != nil {
-			log.Errorf("failed to close http listener: %s", err)
+
+		closed := make(chan struct{}, 2)
+		go srv.closeHttpServer(closed)
+		go srv.closeAdmissionServer(closed)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		cnt := 2
+		for cnt > 0 {
+			select {
+			case <-ctx.Done():
+				log.Errorf("close servers timeout")
+				return
+			case <-closed:
+				cnt--
+				log.Debug("close a server")
+			}
+		}
+	}()
+
+	go func() {
+		log.Debug("starting http server")
+		if err := srv.httpServer.RunListener(srv.httpListener); err != nil && !types.IsUseOfClosedNetConnErr(err) {
+			log.Errorf("failed to start http server: %s", err)
 		}
 	}()
-	if err := srv.router.RunListener(srv.httpListener); err != nil && !types.IsUseOfClosedNetConnErr(err) {
-		log.Errorf("failed to start API Server: %s", err)
-		return err
+
+	if srv.admissionServer != nil {
+		go func() {
+			log.Debug("starting admission server")
+			if err := srv.admissionServer.ListenAndServeTLS("", ""); err != nil && !types.IsUseOfClosedNetConnErr(err) {
+				log.Errorf("failed to start admission server: %s", err)
+			}
+		}()
 	}
+
 	return nil
 }
+
+func (srv *Server) closeHttpServer(closed chan struct{}) {
+	if err := srv.httpListener.Close(); err != nil {
+		log.Errorf("failed to close http listener: %s", err)
+	}
+	closed <- struct{}{}
+}
+
+func (srv *Server) closeAdmissionServer(closed chan struct{}) {
+	if srv.admissionServer != nil {
+		if err := srv.admissionServer.Shutdown(context.TODO()); err != nil {
+			log.Errorf("failed to shutdown admission server: %s", err)
+		}
+	}
+	closed <- struct{}{}
+}

pkg/api/server_test.go

@@ -15,8 +15,10 @@
 package api
 
 import (
+	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"testing"
 	"time"
 
@@ -25,17 +27,91 @@ import (
 	"github.com/apache/apisix-ingress-controller/pkg/config"
 )
 
+const (
+	_tlsCert = `-----BEGIN CERTIFICATE-----
+MIIDBTCCAe0CFHoW964zOGe29tXJwA4WWsrUxyggMA0GCSqGSIb3DQEBCwUAMD8x
+PTA7BgNVBAMMNGFwaXNpeC1pbmdyZXNzLWNvbnRyb2xsZXItd2ViaG9vay5pbmdy
+ZXNzLWFwaXNpeC5zdmMwHhcNMjEwODIxMDgxNDQzWhcNMjIwODIxMDgxNDQzWjA/
+MT0wOwYDVQQDDDRhcGlzaXgtaW5ncmVzcy1jb250cm9sbGVyLXdlYmhvb2suaW5n
+cmVzcy1hcGlzaXguc3ZjMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+2mMDnAkHbpmMPMgZHTh5VKnUXRrHKMY3OEzTyDs4MxBSrxBsIrRYXjXBi6A75IRU
+XD9/W8DyIENclLRTrYdLt03OD8n5a2Z6+DW8XfAO0FZ058QnyKOo9v1/RKqHkPtV
+PwbCjUvCCClsgihOSzxcgcF2oHm2x1JaATBicWNS4cze6LrkmVSI2BL/6liU9hSJ
+15MtyNRqe18sQ/7z6cWZBkAfwW9pY4lC0JWNHntFdnQJzPlw/jM9rzHamnBrMas9
+R2TDVqfgURqKmJaQBd0lDtc9Zrp2G9dCqmF8UP3OvH3cBa8UKBzo4kPRsjKEf5+r
+zzMrwNG7kX47K82JZNhKlwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBBJ3881kMF
+DaYXJ9xlIo0lWijt8yoDn5bGXrvT0Q+tLJhbFmVh9Mr+/NwaythKPM4dcXXWKlwN
+Ham8OpqfFP2BZ93nv+CXgQxpdNAGQPNmJ3146o8sJpbnNwQCTcoe9nm66DTW6340
+SCdDwuwkNRMsc24EnTdmwe7Z0XBgz+jx0WGlzxmeQKJQVUChp7w1qNiUfNWjK3Ud
+hCUjmUwiqVpk9+I997a9/DNu6CEt7SIJK3nbuLWDuXa4S3ebMgVlCGXAapb5QfDe
+S3BTAjguuygwbpo4M+S6hyObMpdNbr9dVhFLGj02lzL3a+mM1C19kJCpbJggu1Y3
+oXDF4V2XHbzJ
+-----END CERTIFICATE-----
+`
+	_tlsKey = `-----BEGIN RSA PRIVATE KEY-----
+MIIEpgIBAAKCAQEA2mMDnAkHbpmMPMgZHTh5VKnUXRrHKMY3OEzTyDs4MxBSrxBs
+IrRYXjXBi6A75IRUXD9/W8DyIENclLRTrYdLt03OD8n5a2Z6+DW8XfAO0FZ058Qn
+yKOo9v1/RKqHkPtVPwbCjUvCCClsgihOSzxcgcF2oHm2x1JaATBicWNS4cze6Lrk
+mVSI2BL/6liU9hSJ15MtyNRqe18sQ/7z6cWZBkAfwW9pY4lC0JWNHntFdnQJzPlw
+/jM9rzHamnBrMas9R2TDVqfgURqKmJaQBd0lDtc9Zrp2G9dCqmF8UP3OvH3cBa8U
+KBzo4kPRsjKEf5+rzzMrwNG7kX47K82JZNhKlwIDAQABAoIBAQDQJY9LKU/sGm2P
+gShusWTzTOsb0moAcuwuvQsdzVPDV8t3EDAA4+NV5+aRLifnpjjBs8OvsDcWiR20
+nisjOdDw5TeB1P/lXcfWy2C+KA/2gnDqdgt1MIfa4cJrsB2GEgcuC0NjaNGG9fR2
+GfSFwQJqqfpm+Zs8X0Fp4LPzXregfd//sgnNi5dorWxZ142lJvAStC/inEzLFBLW
+hC+tDq9zIXUmAhlMzfmJ3cf8gU7z+RMOYkNFaz7EGM6wWZSppiWBk9A7BiknV5AJ
+cQRv2woGy2ZgP7MXZVg8RNaX5w6P6GFEK5NbdoyHkGL2olvf8tN7f9oNLdv9apQf
+6F3l7OABAoGBAP6sX+tSqs/oAouyZQ4v9NnrnhBKgPgnMwcKaohg4jo58TMJ5ldQ
+U10AkZyfVcQ/gE7531N+6D/fzEYSwiiZdsOFVEMHQitIXIZMDeyU+EPoZawyHCpn
+h6NuaStkXqowtEdkscJgiCRBNncnKwvCuLu8copoglfwPaaLMzrBilzNAoGBANuG
+P6f3XLfvyDyVDM6oAbLVQGIfEBrSueyoLIackSe1a1mJ7pTmMnY9S/9W+i3ZR6Kp
+tAKUnEkoN90l8R/1V0x7AobOhMWicblo23eAw9r6jXKZtUxlhbjNKYzfQRVetbT4
+ix/qKdme1dXLAeM4YgF1CKxO1ccf6fOJArWpSwTzAoGBAOoux+U0ly2nQvACkzqA
+jr71EtwYJpAKO7n1shDGRkEUlt8/8zfG/WE/7KYBPnS/j9UPoHS+9gIGYWjuRuve
+cn9IUztvqUDzwWEc/pDWS5TmVtgJHC1CFlAKb1sfaI1HS/96cJs0+Pudm9/lfIfL
+/uNjXlA32ePTXl2PEwSsg/bhAoGBAIthmss/8LvM4BsvG9merK1qXx2t0WDmiSws
+v1Cc2kEXHFjWjgg2fLW8R6ORCvnPan9qNqQozW5ZvdaJP6bl9I7Xz4veVkjR0llB
+rY8bz78atHKeC5G9KAFlKkuKeN1jrAWChXs3B2loQyciZUlqxDdeoqocx/lNVxLM
+3E6RddNnAoGBAMCjs0qKwT5ENMsaQxFlwPEKuC5Sl0ejKgUnoHsVl9VuhAMcwE70
+hMJMGXv2p1BbBuuW35jH92LBSBjS/Zv4b86DG2VQsDWNI4u3lPFd1zif6dhE8yvU
+bKS1uxKukPFp6zxFwR7YZIiwo3tGkcudpHdTNurNMQiSTN97LTo8KL8y
+-----END RSA PRIVATE KEY-----
+`
+)
+
+func generateCertFiles() (certFileName string, keyFileName string) {
+	certFile, _ := ioutil.TempFile("", "cert.*.pem")
+	certFileName = certFile.Name()
+	_, _ = certFile.Write([]byte(_tlsCert))
+
+	keyFile, _ := ioutil.TempFile("", "key.*.pem")
+	keyFileName = keyFile.Name()
+	_, _ = keyFile.Write([]byte(_tlsKey))
+	return
+}
+
+func deleteFiles(filename ...string) {
+	for _, fn := range filename {
+		_ = os.Remove(fn)
+	}
+}
+
 func TestServer(t *testing.T) {
 	cfg := &config.Config{HTTPListen: "127.0.0.1:0"}
-	srv, err := NewServer(cfg)
-	assert.Nil(t, err, "see non-nil error: ", err)
-
-	err = srv.httpListener.Close()
+	_, err := NewServer(cfg)
 	assert.Nil(t, err, "see non-nil error: ", err)
 }
 
 func TestServerRun(t *testing.T) {
-	cfg := &config.Config{HTTPListen: "127.0.0.1:0"}
+	certFileName, keyFileName := generateCertFiles()
+	defer deleteFiles(certFileName, keyFileName)
+
+	cfg := &config.Config{
+		HTTPListen:   "127.0.0.1:0",
+		HTTPSListen:  "127.0.0.1:0",
+		CertFilePath: certFileName,
+		KeyFilePath:  keyFileName,
+	}
+
 	srv, err := NewServer(cfg)
 	assert.Nil(t, err, "see non-nil error: ", err)