Skip to content

Commit

Permalink
feat: User service ports Traefik Docker labels (#1871)
Browse files Browse the repository at this point in the history
## Description:
As part of the authenticated public http ports feature, we are bringing
Traefik inside the Docker cluster to route HTTP traffic to the user
service HTTP ports. This PR sets the required Docker labels.

This was tested with Traefik running inside the engine enclave with the
following static config and the user service enclave network added to
the Traefik container list of networks.

```
version: '3'

services:
  reverse-proxy:
    image: traefik:v2.10
    # Enables the web UI and tells Traefik to listen to docker
    command:
      - --accesslog=true
      - --api.debug=true
      - --api.insecure=true
      - --api.dashboard=true
      - --api.disabledashboardad=true
      - --providers.docker
      - --entrypoints.web.address=:8000
      - --providers.docker.network=bridge
      - --providers.docker.exposedByDefault=false
      - --log.level=DEBUG
    ports:
      # The HTTP port
      - "8000:8000"
      # The Web UI (enabled by --api.insecure=true)
      - "8080:8080"
    volumes:
      # So that Traefik can listen to the Docker events
      - /var/run/docker.sock:/var/run/docker.sock
```

User service "nginx" port labels:
Enclave short UUID: 65d2fb6d6732
Service short UUID: 3771c85af16a
HTTP Port number: 80
```
"traefik.enable": "true"
"traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Host(`80-3771c85af16a-65d2fb6d6732`)"
"traefik.http.routers.65d2fb6d6732-3771c85af16a-80.service": "65d2fb6d6732-3771c85af16a-80"
"traefik.http.services.65d2fb6d6732-3771c85af16a-80.loadbalancer.server.port": "80"
```

```
$ curl -I http://localhost:8000 -H "Host: 80-3771c85af16a-65d2fb6d6732"
HTTP/1.1 200 OK
Accept-Ranges: bytes
Content-Length: 615
Content-Type: text/html
Date: Wed, 29 Nov 2023 21:32:51 GMT
Etag: "6537cac7-267"
Last-Modified: Tue, 24 Oct 2023 13:46:47 GMT
Server: nginx/1.25.
```

## Is this change user facing?
NO

## References (if applicable):

https://www.notion.so/kurtosistech/Public-user-service-HTTP-ports-bdf1107b0d1c4ca990c346fd87473174
  • Loading branch information
laurentluce authored Nov 30, 2023
1 parent d11cd37 commit d18f20e
Show file tree
Hide file tree
Showing 4 changed files with 192 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ func ValidateUserCustomLabelKey(str string) error {
return nil
}

// CreateNewDockerUserCustomLabelKey creates a Traefik Docker label with the Traefik label key prefix
func CreateNewDockerTraefikLabelKey(str string) (*DockerLabelKey, error) {
labelKeyStr := traefikLabelKeyPrefixStr + str
return createNewDockerLabelKey(labelKeyStr)
}

func createNewDockerLabelKey(str string) (*DockerLabelKey, error) {
if err := validate(str); err != nil {
return nil, stacktrace.NewError("Label key string '%v' is not valid", str)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ const (
logsServiceUuidDockerLabelKey = "service_uuid"
logsServiceShortUuidDockerLabelKey = "service_short_uuid"
logsServiceNameDockerLabelKey = "service_name"

// Traefik label keys
traefikLabelKeyPrefixStr = "traefik."
)

// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@ package object_attributes_provider
import (
"crypto/md5"
"encoding/hex"
"fmt"
"net"
"strconv"
"strings"
"time"

"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_value"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_object_name"
Expand All @@ -13,9 +20,6 @@ import (
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator"
"github.com/kurtosis-tech/stacktrace"
"net"
"strings"
"time"
)

const (
Expand Down Expand Up @@ -250,6 +254,14 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForUserServiceContain
labels[dockerLabelKey] = dockerLabelValue
}

traefikLabels, err := provider.getTraefikLabelsForEnclaveObject(serviceUuidStr, privatePorts)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting traefik labels for enclave object with UUID '%v'", serviceUuid)
}
for traefikLabelKey, traefikLabelValue := range traefikLabels {
labels[traefikLabelKey] = traefikLabelValue
}

objectAttributes, err := newDockerObjectAttributesImpl(name, labels)
if err != nil {
return nil, stacktrace.Propagate(
Expand Down Expand Up @@ -516,6 +528,94 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb
return labels, nil
}

// Return Traefik labels
// Including the labels required to route traffic to the user service ports based on the Host header:
// <port number>-<service short uuid>-<enclave short uuid>
// The Traefik service name format is: <enclave short uuid>-<service short uuid>-<port number>
// With the following input:
// Enclave short UUID: 65d2fb6d6732
// Service short UUID: 3771c85af16a
// HTTP Port 1 number: 80
// HTTP Port 2 number: 81
// the following labels are returned:
// "traefik.enable": "true",
// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Host(`80-3771c85af16a-65d2fb6d6732`)",
// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.service": "65d2fb6d6732-3771c85af16a-80",
// "traefik.http.services.65d2fb6d6732-3771c85af16a-80.loadbalancer.server.port": "80"
// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Host(`81-3771c85af16a-65d2fb6d6732`)",
// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.service": "65d2fb6d6732-3771c85af16a-81",
// "traefik.http.services.65d2fb6d6732-3771c85af16a-81.loadbalancer.server.port": "81"
func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEnclaveObject(serviceUuid string, ports map[string]*port_spec.PortSpec) (map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue, error) {
labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{}

for _, portSpec := range ports {
maybeApplicationProtocol := ""
if portSpec.GetMaybeApplicationProtocol() != nil {
maybeApplicationProtocol = *portSpec.GetMaybeApplicationProtocol()
}
if maybeApplicationProtocol == consts.HttpApplicationProtocol {
shortEnclaveUuid := uuid_generator.ShortenedUUIDString(provider.enclaveId.GetString())
shortServiceUuid := uuid_generator.ShortenedUUIDString(serviceUuid)
servicePortStr := fmt.Sprintf("%s-%s-%d", shortEnclaveUuid, shortServiceUuid, portSpec.GetNumber())

// Host rule
ruleKeySuffix := fmt.Sprintf("http.routers.%s.rule", servicePortStr)
ruleLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey(ruleKeySuffix)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting the traefik rule label key with suffix '%v'", ruleKeySuffix)
}
ruleValue := fmt.Sprintf("Host(`%d-%s-%s`)", portSpec.GetNumber(), shortServiceUuid, shortEnclaveUuid)
ruleLabelValue, err := docker_label_value.CreateNewDockerLabelValue(ruleValue)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred creating the traefik rule label value with value '%v'", ruleValue)
}
labels[ruleLabelKey] = ruleLabelValue

// Service name
serviceKeySuffix := fmt.Sprintf("http.routers.%s.service", servicePortStr)
serviceLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey(serviceKeySuffix)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting the traefik service label key with suffix '%v'", serviceKeySuffix)
}
serviceValue := servicePortStr
serviceLabelValue, err := docker_label_value.CreateNewDockerLabelValue(serviceValue)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred creating the traefik service label value with value '%v'", serviceValue)
}
labels[serviceLabelKey] = serviceLabelValue

// Service port number
portKeySuffix := fmt.Sprintf("http.services.%s.loadbalancer.server.port", servicePortStr)
portLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey(portKeySuffix)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting the traefik port label key with suffix '%v'", portKeySuffix)
}
portValue := strconv.Itoa(int(portSpec.GetNumber()))
portLabelValue, err := docker_label_value.CreateNewDockerLabelValue(portValue)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred creating the traefik port label value with value '%v'", portValue)
}
labels[portLabelKey] = portLabelValue
}
}

if len(labels) > 0 {
// Enable Traefik for this service is there is at least one traefik label
traefikEnableLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey("enable")
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting the traefik enable label key")
}
traefikEnableValue := "true"
traefikEnableLabelValue, err := docker_label_value.CreateNewDockerLabelValue(traefikEnableValue)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred creating the traefik enable label value with value '%v'", traefikEnableValue)
}
labels[traefikEnableLabelKey] = traefikEnableLabelValue
}

return labels, nil
}

func getLabelKeyValuesAsStrings(labels map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue) map[string]string {
result := map[string]string{}
for key, value := range labels {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package object_attributes_provider

import (
"net"
"testing"
"time"

"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
"github.com/stretchr/testify/require"
)

const (
enclaveUuid = "65d2fb6d673249b8b4a91a2f4ae616de"
)

var (
portWaitForTest = port_spec.NewWait(5 * time.Second)
)

func TestForUserServiceContainer(t *testing.T) {
objAttrsProvider := GetDockerObjectAttributesProvider()
enclaveObjAttrsProvider, err := objAttrsProvider.ForEnclave(enclaveUuid)
require.NoError(t, err, "An unexpected error occurred getting the enclave object attributes provider")

serviceName := service.ServiceName("nginx")
serviceUuid := service.ServiceUUID("3771c85af16a40a18201acf4b4b5ad28")
privateIpAddr := net.IP("1.2.3.4")
port1Id := "port1"
port1Num := uint16(23)
port1Protocol := port_spec.TransportProtocol_TCP
port1Spec, err := port_spec.NewPortSpec(port1Num, port1Protocol, "", portWaitForTest)
require.NoError(t, err, "An unexpected error occurred creating port 1 spec")
port2Id := "port2"
port2Num := uint16(45)
port2Protocol := port_spec.TransportProtocol_TCP
port2ApplicationProtocol := consts.HttpApplicationProtocol
port2Spec, err := port_spec.NewPortSpec(port2Num, port2Protocol, port2ApplicationProtocol, portWaitForTest)
require.NoError(t, err, "An unexpected error occurred creating port 2 spec")
privatePorts := map[string]*port_spec.PortSpec{
port1Id: port1Spec,
port2Id: port2Spec,
}
userLabels := map[string]string{}
containerAttrs, err := enclaveObjAttrsProvider.ForUserServiceContainer(
serviceName,
serviceUuid,
privateIpAddr,
privatePorts,
userLabels,
)
require.NoError(t, err, "An unexpected error occurred getting the container attributes")
objName := containerAttrs.GetName()
require.Equal(t, objName.GetString(), "nginx--3771c85af16a40a18201acf4b4b5ad28")
objLabels := containerAttrs.GetLabels()
for labelKey, labelValue := range objLabels {
switch labelKey.GetString() {
case docker_label_key.AppIDDockerLabelKey.GetString():
require.Equal(t, labelValue.GetString(), "kurtosis")
case docker_label_key.ContainerTypeDockerLabelKey.GetString():
require.Equal(t, labelValue.GetString(), "user-service")
case docker_label_key.EnclaveUUIDDockerLabelKey.GetString():
require.Equal(t, labelValue.GetString(), "65d2fb6d673249b8b4a91a2f4ae616de")
case "traefik.enable":
require.Equal(t, labelValue.GetString(), "true")
case "traefik.http.routers.65d2fb6d6732-3771c85af16a-23.rule":
require.Fail(t, "A traefik label for port 23 should not be present")
case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.rule":
require.Equal(t, labelValue.GetString(), "Host(`45-3771c85af16a-65d2fb6d6732`)")
case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.service":
require.Equal(t, labelValue.GetString(), "65d2fb6d6732-3771c85af16a-45")
case "traefik.http.services.65d2fb6d6732-3771c85af16a-45.loadbalancer.server.port":
require.Equal(t, labelValue.GetString(), "45")
default:
break
}
}
}

0 comments on commit d18f20e

Please sign in to comment.