diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 710d616f63..48b70d30ac 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -200,7 +200,7 @@ func main() { faController := fleetallocation.NewController(wh, allocationMutex, kubeClient, extClient, agonesClient, agonesInformerFactory) gasController := gameserverallocations.NewController(api, health, gsCounter, topNGSForAllocation, - kubeClient, agonesClient, agonesInformerFactory) + kubeClient, kubeInformerFactory, agonesClient, agonesInformerFactory) fasController := fleetautoscalers.NewController(wh, health, kubeClient, extClient, agonesClient, agonesInformerFactory) diff --git a/pkg/gameserverallocations/controller.go b/pkg/gameserverallocations/controller.go index 36d9ed450a..d1c2c1f9d3 100644 --- a/pkg/gameserverallocations/controller.go +++ b/pkg/gameserverallocations/controller.go @@ -15,6 +15,10 @@ package gameserverallocations import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" "fmt" "io/ioutil" "math/rand" @@ -48,9 +52,11 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + corev1lister "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" ) @@ -63,6 +69,12 @@ var ( ErrConflictInGameServerSelection = errors.New("The Gameserver was already allocated") ) +const ( + secretClientCertName = "client-cert" + secretClientKeyName = "client-key" + secretCaCertName = "ca-cert" +) + // Controller is a the GameServerAllocation controller type Controller struct { baseLogger *logrus.Entry @@ -75,6 +87,7 @@ type Controller struct { gameServerGetter getterv1alpha1.GameServersGetter gameServerLister listerv1alpha1.GameServerLister allocationPolicyLister multiclusterlisterv1alpha1.GameServerAllocationPolicyLister + secretLister corev1lister.SecretLister stop <-chan struct{} workerqueue *workerqueue.WorkerQueue recorder record.EventRecorder @@ -98,6 +111,7 @@ func NewController(apiServer *apiserver.APIServer, counter *gameservers.PerNodeCounter, topNGameServerCnt int, kubeClient kubernetes.Interface, + kubeInformerFactory informers.SharedInformerFactory, agonesClient versioned.Interface, agonesInformerFactory externalversions.SharedInformerFactory, ) *Controller { @@ -110,6 +124,7 @@ func NewController(apiServer *apiserver.APIServer, gameServerGetter: agonesClient.StableV1alpha1(), gameServerLister: agonesInformer.GameServers().Lister(), allocationPolicyLister: agonesInformerFactory.Multicluster().V1alpha1().GameServerAllocationPolicies().Lister(), + secretLister: kubeInformerFactory.Core().V1().Secrets().Lister(), } c.baseLogger = runtime.NewLoggerWithType(c) c.workerqueue = workerqueue.NewWorkerQueue(c.syncGameServers, c.baseLogger, logfields.GameServerKey, stable.GroupName+".GameServerUpdateController") @@ -290,19 +305,21 @@ func (c *Controller) allocateFromLocalCluster(gsa *v1alpha1.GameServerAllocation // applyMultiClusterAllocation retrieves allocation policies and iterate on policies. // Then allocate gameservers from local or remote cluster accordingly. -func (c *Controller) applyMultiClusterAllocation(gsa *v1alpha1.GameServerAllocation) (*v1alpha1.GameServerAllocation, error) { - var result *v1alpha1.GameServerAllocation +func (c *Controller) applyMultiClusterAllocation(gsa *v1alpha1.GameServerAllocation) (result *v1alpha1.GameServerAllocation, err error) { - selector, err := metav1.LabelSelectorAsSelector(&gsa.Spec.MultiClusterSetting.PolicySelector) - if err != nil { - return nil, err + selector := labels.Everything() + if len(gsa.Spec.MultiClusterSetting.PolicySelector.MatchLabels)+len(gsa.Spec.MultiClusterSetting.PolicySelector.MatchExpressions) != 0 { + selector, err = metav1.LabelSelectorAsSelector(&gsa.Spec.MultiClusterSetting.PolicySelector) + if err != nil { + return nil, err + } } policies, err := c.allocationPolicyLister.GameServerAllocationPolicies(gsa.ObjectMeta.Namespace).List(selector) if err != nil { return nil, err } else if len(policies) == 0 { - return c.allocateFromLocalCluster(gsa) + return nil, errors.New("no multi-cluster allocation policy is specified") } it := multiclusterv1alpha1.NewConnectionInfoIterator(policies) @@ -328,8 +345,98 @@ func (c *Controller) applyMultiClusterAllocation(gsa *v1alpha1.GameServerAllocat // allocateFromRemoteCluster allocates gameservers from a remote cluster by making // an http call to allocation service in that cluster. func (c *Controller) allocateFromRemoteCluster(gsa v1alpha1.GameServerAllocation, connectionInfo *multiclusterv1alpha1.ClusterConnectionInfo, namespace string) (*v1alpha1.GameServerAllocation, error) { - // TODO: implement getting secrets and making rest call to remote cluster - return nil, nil + var gsaResult v1alpha1.GameServerAllocation + + // TODO: handle converting error to apiserver error + // TODO: cache the client + client, err := c.createRemoteClusterRestClient(namespace, connectionInfo.SecretName) + if err != nil { + return nil, err + } + + // Forward the game server allocation request to another cluster, + // and disable multicluster settings to avoid the target cluster + // forward the allocation request again. + gsa.Spec.MultiClusterSetting.Enabled = false + body, err := json.Marshal(gsa) + if err != nil { + return nil, err + } + + // TODO: Retry on transient error --> response.StatusCode >= 500 + response, err := client.Post(connectionInfo.AllocationEndpoint, "application/json", bytes.NewBuffer(body)) + if err != nil { + return nil, err + } + defer response.Body.Close() // nolint: errcheck + + data, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + if response.StatusCode >= 400 { + return nil, errors.New(string(data)) + } + + err = json.Unmarshal(data, &gsaResult) + if err != nil { + return nil, err + } + + return &gsaResult, nil +} + +// createRemoteClusterRestClient creates a rest client with proper certs to make a remote call. +func (c *Controller) createRemoteClusterRestClient(namespace, secretName string) (*http.Client, error) { + clientCert, clientKey, caCert, err := c.getClientCertificates(namespace, secretName) + if err != nil { + return nil, err + } + if clientCert == nil || clientKey == nil { + return nil, fmt.Errorf("missing client certificate key pair in secret %s", secretName) + } + + // Load client cert + cert, err := tls.X509KeyPair(clientCert, clientKey) + if err != nil { + return nil, err + } + + tlsConfig := &tls.Config{Certificates: []tls.Certificate{cert}} + if len(caCert) != 0 { + // Load CA cert, if provided and trust the server certificate. + // This is required for self-signed certs. + tlsConfig.RootCAs = x509.NewCertPool() + ca, err := x509.ParseCertificate(caCert) + if err != nil { + return nil, err + } + tlsConfig.RootCAs.AddCert(ca) + } + + // Setup HTTPS client + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + }, nil +} + +// getClientCertificates returns the client certificates and CA cert for remote allocation cluster call +func (c *Controller) getClientCertificates(namespace, secretName string) (clientCert, clientKey, caCert []byte, err error) { + secret, err := c.secretLister.Secrets(namespace).Get(secretName) + if err != nil { + return nil, nil, nil, err + } + if secret == nil || len(secret.Data) == 0 { + return nil, nil, nil, fmt.Errorf("secert %s does not have data", secretName) + } + + // Create http client using cert + clientCert = secret.Data[secretClientCertName] + clientKey = secret.Data[secretClientKeyName] + caCert = secret.Data[secretCaCertName] + return clientCert, clientKey, caCert, nil } // allocationDeserialization processes the request and namespace, and attempts to deserialise its values diff --git a/pkg/gameserverallocations/controller_test.go b/pkg/gameserverallocations/controller_test.go index 6c7bfbc733..c8ec32910d 100644 --- a/pkg/gameserverallocations/controller_test.go +++ b/pkg/gameserverallocations/controller_test.go @@ -16,6 +16,8 @@ package gameserverallocations import ( "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" "fmt" "net/http" @@ -34,6 +36,7 @@ import ( "github.com/heptiolabs/healthcheck" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -728,22 +731,12 @@ func TestGetRandomlySelectedGS(t *testing.T) { assert.Equal(t, "gs1", selectedGS.ObjectMeta.Name) } -func TestMultiClusterAllocation(t *testing.T) { +func TestMultiClusterAllocationFromLocal(t *testing.T) { t.Parallel() t.Run("Handle allocation request locally", func(t *testing.T) { c, m := newFakeController() fleetName := addReactorForGameServer(&m) - stop, cancel := agtesting.StartInformers(m) - defer cancel() - - // This call initializes the cache - err := c.syncReadyGSServerCache() - assert.Nil(t, err) - - err = c.counter.Run(0, stop) - assert.Nil(t, err) - m.AgonesClient.AddReactor("list", "gameserverallocationpolicies", func(action k8stesting.Action) (bool, k8sruntime.Object, error) { return true, &multiclusterv1alpha1.GameServerAllocationPolicyList{ Items: []multiclusterv1alpha1.GameServerAllocationPolicy{ @@ -758,18 +751,23 @@ func TestMultiClusterAllocation(t *testing.T) { }, }, ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"cluster": "onprem"}, + Labels: map[string]string{"cluster": "onprem"}, + Namespace: defaultNs, }, }, }, }, nil }) - m.AgonesClient.AddReactor("get", "secrets", - func(action k8stesting.Action) (bool, k8sruntime.Object, error) { - t.Error("Should not get secrets for local cluster") - return false, nil, nil - }) + stop, cancel := agtesting.StartInformers(m) + defer cancel() + + // This call initializes the cache + err := c.syncReadyGSServerCache() + assert.Nil(t, err) + + err = c.counter.Run(0, stop) + assert.Nil(t, err) gsa := &v1alpha1.GameServerAllocation{ ObjectMeta: metav1.ObjectMeta{ @@ -801,6 +799,12 @@ func TestMultiClusterAllocation(t *testing.T) { c, m := newFakeController() fleetName := addReactorForGameServer(&m) + m.AgonesClient.AddReactor("list", "gameserverallocationpolicies", func(action k8stesting.Action) (bool, k8sruntime.Object, error) { + return true, &multiclusterv1alpha1.GameServerAllocationPolicyList{ + Items: []multiclusterv1alpha1.GameServerAllocationPolicy{}, + }, nil + }) + stop, cancel := agtesting.StartInformers(m) defer cancel() @@ -811,42 +815,260 @@ func TestMultiClusterAllocation(t *testing.T) { err = c.counter.Run(0, stop) assert.Nil(t, err) + gsa := &v1alpha1.GameServerAllocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNs, + Name: "alloc1", + ClusterName: "multicluster", + }, + Spec: v1alpha1.GameServerAllocationSpec{ + MultiClusterSetting: v1alpha1.MultiClusterSetting{ + Enabled: true, + PolicySelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "cluster": "onprem", + }, + }, + }, + Required: metav1.LabelSelector{MatchLabels: map[string]string{stablev1alpha1.FleetNameLabel: fleetName}}, + }, + } + + _, err = executeAllocation(gsa, c) + assert.Error(t, err) + }) +} + +func TestMultiClusterAllocationFromRemote(t *testing.T) { + t.Parallel() + t.Run("Handle allocation request remotely", func(t *testing.T) { + c, m := newFakeController() + fleetName := addReactorForGameServer(&m) + + // Mock server + expectedGSAName := "mocked" + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + serverResponse := v1alpha1.GameServerAllocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: expectedGSAName, + }, + } + response, _ := json.Marshal(serverResponse) + _, _ = w.Write(response) + })) + defer server.Close() + + // Set client CA for server + certpool := x509.NewCertPool() + certpool.AppendCertsFromPEM(clientCert) + server.TLS.ClientCAs = certpool + server.TLS.ClientAuth = tls.RequireAndVerifyClientCert + + // Allocation policy reactor + clusterName := "remotecluster" + secretName := clusterName + "secret" m.AgonesClient.AddReactor("list", "gameserverallocationpolicies", func(action k8stesting.Action) (bool, k8sruntime.Object, error) { return true, &multiclusterv1alpha1.GameServerAllocationPolicyList{ - Items: []multiclusterv1alpha1.GameServerAllocationPolicy{}, + Items: []multiclusterv1alpha1.GameServerAllocationPolicy{ + { + Spec: multiclusterv1alpha1.GameServerAllocationPolicySpec{ + Priority: 1, + Weight: 200, + ConnectionInfo: multiclusterv1alpha1.ClusterConnectionInfo{ + AllocationEndpoint: server.URL, + ClusterName: clusterName, + SecretName: secretName, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNs, + }, + }, + }, }, nil }) - m.AgonesClient.AddReactor("get", "secrets", + m.KubeClient.AddReactor("list", "secrets", func(action k8stesting.Action) (bool, k8sruntime.Object, error) { - t.Error("Should not get secrets for local cluster") - return false, nil, nil + return true, getTestSecret(secretName, server.TLS.Certificates[0].Certificate[0]), nil }) + stop, cancel := agtesting.StartInformers(m) + defer cancel() + + // This call initializes the cache + err := c.syncReadyGSServerCache() + assert.Nil(t, err) + + err = c.counter.Run(0, stop) + assert.Nil(t, err) + gsa := &v1alpha1.GameServerAllocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: defaultNs, Name: "alloc1", - ClusterName: "multicluster", + ClusterName: "localcluster", }, Spec: v1alpha1.GameServerAllocationSpec{ MultiClusterSetting: v1alpha1.MultiClusterSetting{ Enabled: true, - PolicySelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "cluster": "onprem", + }, + Required: metav1.LabelSelector{MatchLabels: map[string]string{stablev1alpha1.FleetNameLabel: fleetName}}, + }, + } + + result, err := executeAllocation(gsa, c) + if assert.NoError(t, err) { + assert.Equal(t, expectedGSAName, result.ObjectMeta.Name) + } + }) + + t.Run("Remote server returns error", func(t *testing.T) { + c, m := newFakeController() + fleetName := addReactorForGameServer(&m) + + // Mock server to return error + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "test error message", 500) + })) + defer server.Close() + + // Set client CA for server + certpool := x509.NewCertPool() + certpool.AppendCertsFromPEM(clientCert) + server.TLS.ClientCAs = certpool + server.TLS.ClientAuth = tls.RequireAndVerifyClientCert + + // Allocation policy reactor + clusterName := "remotecluster" + secretName := clusterName + "secret" + m.AgonesClient.AddReactor("list", "gameserverallocationpolicies", func(action k8stesting.Action) (bool, k8sruntime.Object, error) { + return true, &multiclusterv1alpha1.GameServerAllocationPolicyList{ + Items: []multiclusterv1alpha1.GameServerAllocationPolicy{ + { + Spec: multiclusterv1alpha1.GameServerAllocationPolicySpec{ + Priority: 1, + Weight: 200, + ConnectionInfo: multiclusterv1alpha1.ClusterConnectionInfo{ + AllocationEndpoint: server.URL, + ClusterName: clusterName, + SecretName: secretName, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNs, }, }, }, + }, nil + }) + + m.KubeClient.AddReactor("list", "secrets", + func(action k8stesting.Action) (bool, k8sruntime.Object, error) { + return true, getTestSecret(secretName, server.TLS.Certificates[0].Certificate[0]), nil + }) + + stop, cancel := agtesting.StartInformers(m) + defer cancel() + + // This call initializes the cache + err := c.syncReadyGSServerCache() + assert.Nil(t, err) + + err = c.counter.Run(0, stop) + assert.Nil(t, err) + + gsa := &v1alpha1.GameServerAllocation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNs, + Name: "alloc1", + ClusterName: "localcluster", + }, + Spec: v1alpha1.GameServerAllocationSpec{ + MultiClusterSetting: v1alpha1.MultiClusterSetting{ + Enabled: true, + }, Required: metav1.LabelSelector{MatchLabels: map[string]string{stablev1alpha1.FleetNameLabel: fleetName}}, }, } - ret, err := executeAllocation(gsa, c) - assert.NoError(t, err) - assert.Equal(t, gsa.Spec.Required, ret.Spec.Required) - expectedState := v1alpha1.GameServerAllocationAllocated - assert.True(t, expectedState == ret.Status.State, "Failed: %s vs %s", expectedState, ret.Status.State) + _, err = executeAllocation(gsa, c) + assert.Error(t, err) + assert.Contains(t, err.Error(), "test error message") + }) +} + +func TestCreateRestClientError(t *testing.T) { + t.Parallel() + t.Run("Missing secret", func(t *testing.T) { + c, _ := newFakeController() + _, err := c.createRemoteClusterRestClient(defaultNs, "secret-name") + assert.Error(t, err) + assert.Contains(t, err.Error(), "secret-name") + }) + t.Run("Missing cert", func(t *testing.T) { + c, m := newFakeController() + + m.KubeClient.AddReactor("list", "secrets", + func(action k8stesting.Action) (bool, k8sruntime.Object, error) { + return true, &corev1.SecretList{ + Items: []corev1.Secret{{ + Data: map[string][]byte{ + "client-cert": clientCert, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-name", + Namespace: defaultNs, + }, + }}}, nil + }) + + _, cancel := agtesting.StartInformers(m) + defer cancel() + + _, err := c.createRemoteClusterRestClient(defaultNs, "secret-name") + assert.Error(t, err) + assert.Contains(t, err.Error(), "missing client certificate key pair in secret secret-name") + }) + t.Run("Bad client cert", func(t *testing.T) { + c, m := newFakeController() + + m.KubeClient.AddReactor("list", "secrets", + func(action k8stesting.Action) (bool, k8sruntime.Object, error) { + return true, &corev1.SecretList{ + Items: []corev1.Secret{{ + Data: map[string][]byte{ + "client-cert": []byte("XXX"), + "client-key": []byte("XXX"), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-name", + Namespace: defaultNs, + }, + }}}, nil + }) + + _, cancel := agtesting.StartInformers(m) + defer cancel() + + _, err := c.createRemoteClusterRestClient(defaultNs, "secret-name") + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to find any PEM data in certificate input") + }) + t.Run("Bad CA cert", func(t *testing.T) { + c, m := newFakeController() + + m.KubeClient.AddReactor("list", "secrets", + func(action k8stesting.Action) (bool, k8sruntime.Object, error) { + return true, getTestSecret("secret-name", []byte("XXX")), nil + }) + + _, cancel := agtesting.StartInformers(m) + defer cancel() + + _, err := c.createRemoteClusterRestClient(defaultNs, "secret-name") + assert.Error(t, err) + assert.Contains(t, err.Error(), "certificate") }) } @@ -927,7 +1149,77 @@ func newFakeController() (*Controller, agtesting.Mocks) { m.Mux = http.NewServeMux() counter := gameservers.NewPerNodeCounter(m.KubeInformerFactory, m.AgonesInformerFactory) api := apiserver.NewAPIServer(m.Mux) - c := NewController(api, healthcheck.NewHandler(), counter, 1, m.KubeClient, m.AgonesClient, m.AgonesInformerFactory) + c := NewController(api, healthcheck.NewHandler(), counter, 1, m.KubeClient, m.KubeInformerFactory, m.AgonesClient, m.AgonesInformerFactory) c.recorder = m.FakeRecorder return c, m } + +func getTestSecret(secretName string, serverCert []byte) *corev1.SecretList { + return &corev1.SecretList{ + Items: []corev1.Secret{ + { + Data: map[string][]byte{ + "ca-cert": serverCert, + "client-key": clientKey, + "client-cert": clientCert, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: defaultNs, + }, + }, + }, + } +} + +var clientCert = []byte(`-----BEGIN CERTIFICATE----- +MIIDuzCCAqOgAwIBAgIUduDWtqpUsp3rZhCEfUrzI05laVIwDQYJKoZIhvcNAQEL +BQAwbTELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9u +ZG9uMRgwFgYDVQQKDA9HbG9iYWwgU2VjdXJpdHkxFjAUBgNVBAsMDUlUIERlcGFy +dG1lbnQxCjAIBgNVBAMMASowHhcNMTkwNTAyMjIzMDQ3WhcNMjkwNDI5MjIzMDQ3 +WjBtMQswCQYDVQQGEwJHQjEPMA0GA1UECAwGTG9uZG9uMQ8wDQYDVQQHDAZMb25k +b24xGDAWBgNVBAoMD0dsb2JhbCBTZWN1cml0eTEWMBQGA1UECwwNSVQgRGVwYXJ0 +bWVudDEKMAgGA1UEAwwBKjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AKGDasjadVwe0bXUEQfZCkMEAkzn0qTud3RYytympmaS0c01SWFNZwPRO0rpdIOZ +fyXVXVOAhgmgCR6QuXySmyQIoYl/D6tVhc5r9FyWPIBtzQKCJTX0mZOZwMn22qvo +bfnDnVsZ1Ny3RLZIF3um3xovvePXyg1z7D/NvCogNuYpyUUEITPZX6ss5ods/U78 +BxLhKrT8iyu61ZC+ZegbHQqFRngbeb348gE1JwKTslDfe4oH7tZ+bNDZxnGcvh9j +eyagpM0zys4gFfQf/vfD2aEsUJ+GesUQC6uGVoGnTFshFhBsAK6vpIQ4ZQujaJ0r +NKgJ/ccBJFiJXMCR44yWFY0CAwEAAaNTMFEwHQYDVR0OBBYEFEe1gDd8JpzgnvOo +1AEloAXxmxHCMB8GA1UdIwQYMBaAFEe1gDd8JpzgnvOo1AEloAXxmxHCMA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAI5GyuakVgunerCGCSN7Ghsr +ys9vJytbyT+BLmxNBPSXWQwcm3g9yCDdgf0Y3q3Eef7IEZu4I428318iLzhfKln1 +ua4fxvmTFKJ65lQKNkc6Y4e3w1t+C2HOl6fOIVT231qsCoM5SAwQQpqAzEUj6kZl +x+3avw9KSlXqR/mCAkePyoKvprxeb6RVDdq92Ug0qzoAHLpvIkuHdlF0dNp6/kO0 +1pVL0BqW+6UTimSSvH8F/cMeYKbkhpE1u2c/NtNwsR2jN4M9kl3KHqkynk67PfZv +pwlCqZx4M8FpdfCbOZeRLzClUBdD5qzev0L3RNUx7UJzEIN+4LCBv37DIojNOyA= +-----END CERTIFICATE-----`) + +var clientKey = []byte(`-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQChg2rI2nVcHtG1 +1BEH2QpDBAJM59Kk7nd0WMrcpqZmktHNNUlhTWcD0TtK6XSDmX8l1V1TgIYJoAke +kLl8kpskCKGJfw+rVYXOa/RcljyAbc0CgiU19JmTmcDJ9tqr6G35w51bGdTct0S2 +SBd7pt8aL73j18oNc+w/zbwqIDbmKclFBCEz2V+rLOaHbP1O/AcS4Sq0/IsrutWQ +vmXoGx0KhUZ4G3m9+PIBNScCk7JQ33uKB+7WfmzQ2cZxnL4fY3smoKTNM8rOIBX0 +H/73w9mhLFCfhnrFEAurhlaBp0xbIRYQbACur6SEOGULo2idKzSoCf3HASRYiVzA +keOMlhWNAgMBAAECggEAaRPDjEq8IaOXUdFXByEIERNxn7EOlOjj5FjEGgt9pKwO +PJBXXitqQsyD47fAasGZO/b1EZdDHM32QOFtG4OR1T6cQYTdn90zAVmwj+/aCr/k +qaYcKV8p7yIPkBW+rCq6Kc0++X7zwmilFmYOiQ7GhRXcV3gTZu8tG1FxAoMU1GYA +WoGiu+UsEm0MFIOwV/DOukDaj6j4Q9wD0tqi2MsjrugjDI8/mSx5mlvo3yZHubl0 +ChQaWZyUlL2B40mQJc3qsRZzso3sbU762L6G6npQJ19dHgsBfBBs/Q4/DdeqcOb4 +Q9OZ8Q3Q5nXQ7359Sh94LvLOoaWecRTBPGaRvGAGLQKBgQDTOZPEaJJI9heUQ0Ar +VvUuyjILv8CG+PV+rGZ7+yMFCUlmM/m9I0IIc3WbmxxiRypBv46zxpczQHwWZRf2 +7IUZdyrBXRtNoaXbWh3dSgqa7WuHGUzqmn+98sQDodewCyGon8LG9atyge8vFo/l +N0Y21duYj4NeJod82Y0RAKsuzwKBgQDDwCuvbq0FkugklUr5WLFrYTzWrTYPio5k +ID6Ku57yaZNVRv52FTF3Ac5LoKGCv8iPg+x0SiTmCbW2DF2ohvTuJy1H/unJ4bYG +B9vEVOiScbvrvuQ6iMgfxNUCEEQvmn6+uc+KHVwPixY4j6/q1ZLXLPbjqXYHPYi+ +lx9ZG0As4wKBgDj52QAr7Pm9WBLoKREHvc9HP0SoDrjZwu7Odj6POZ0MKj5lWsJI +FnHNIzY8GuXvqFhf4ZBgyzxJ8q7fyh0TI7wAxwmtocXJCsImhtPAOygbTtv8WSEX +V8nXCESqjVGxTvz7S0D716llny0met4rkMcN3NREMf1di0KENGcXtRVFAoGBAKs3 +bD5/NNF6RJizCKf+fvjoTVmMmYuQaqmDVpDsOMPZumfNuAa61NA+AR4/OuXtL9Tv +1COHMq0O8yRvvoAIwzWHiOC/Q+g0B41Q1FXu2po05uT1zBSyzTCUbqfmaG2m2ZOj +XLd2pK5nvqDsdTeXZV/WUYCiGb2Ngg0Ki/3ZixF3AoGACwPxxoAWkuD6T++35Vdt +OxAh/qyGMtgfvdBJPfA3u4digTckBDTwYBhrmvC2Vuc4cpb15RYuUT/M+c3gS3P0 +q+2uLIuwciETPD7psK76NsQM3ZL/IEaZB3VMxbMMFn/NQRbmntTd/twZ42zieX+R +2VpXYUjoRcuir2oU0wh3Hic= +-----END PRIVATE KEY-----`)