diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 55a5b960..2ead75f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,3 +76,5 @@ jobs: files: | bin/flintlockd_amd64 bin/flintlockd_arm64 + bin/flintlock-metrics_amd64 + bin/flintlock-metrics_arm64 diff --git a/.golangci.yml b/.golangci.yml index 804a6f09..bc23dbbc 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,8 +26,6 @@ linters-settings: local-prefixes: github.com/weaveworks/flintlock govet: check-shadowing: true - misspell: - locale: GB nolintlint: allow-leading-space: false allow-unused: false diff --git a/Makefile b/Makefile index de6b2ded..b7bdce32 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ REPO_ROOT := $(shell git rev-parse --show-toplevel) BIN_DIR := bin OUT_DIR := out FLINTLOCKD_CMD := cmd/flintlockd +FLINTLOCK_METRICS_CMD := cmd/flintlock-metrics TOOLS_DIR := hack/tools TOOLS_BIN_DIR := $(TOOLS_DIR)/bin TOOLS_SHARE_DIR := $(TOOLS_DIR)/share @@ -54,13 +55,28 @@ test_image = weaveworks/flintlock-e2e ##@ Build .PHONY: build -build: $(BIN_DIR) ## Build the binaries - go build -o $(BIN_DIR)/flintlockd ./cmd/flintlockd +build: build-flintlockd build-flintlock-metrics ## Build the binaries + +.PHONY: build-flintlockd +build-flintlockd: $(BIN_DIR) ## Build flintlockd binary + go build -o $(BIN_DIR)/flintlockd ./$(FLINTLOCKD_CMD) + +.PHONY: build-flintlock-metrics ## Build flintlock-metrics binary +build-flintlock-metrics: $(BIN_DIR) + go build -o $(BIN_DIR)/flintlock-metrics ./$(FLINTLOCK_METRICS_CMD) .PHONY: build-release -build-release: $(BIN_DIR) ## Build the release binaries - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o $(BIN_DIR)/flintlockd_amd64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./cmd/flintlockd - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/flintlockd_arm64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./cmd/flintlockd +build-release: build-release-flintlockd build-release-flintlock-metrics ## Build the release binaries + +.PHONY: build-release-flintlockd +build-release-flintlockd: $(BIN_DIR) ## Build flintlockd release binaries + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o $(BIN_DIR)/flintlockd_amd64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./$(FLINTLOCKD_CMD) + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/flintlockd_arm64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./$(FLINTLOCKD_CMD) + +.PHONY: build-release-flintlock-metrics +build-release-flintlock-metrics: $(BIN_DIR) ## Build flintlock-metrics release binaries + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o $(BIN_DIR)/flintlock-metrics_amd64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./$(FLINTLOCK_METRICS_CMD) + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/flintlock-metrics_arm64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./$(FLINTLOCK_METRICS_CMD) ##@ Generate diff --git a/cmd/flintlock-metrics/main.go b/cmd/flintlock-metrics/main.go new file mode 100644 index 00000000..53bf5f82 --- /dev/null +++ b/cmd/flintlock-metrics/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "os" + + "github.com/sirupsen/logrus" + + "github.com/weaveworks/flintlock/internal/command/metrics" +) + +func main() { + app := metrics.NewApp(os.Stdout) + + if err := app.Run(os.Args); err != nil { + logrus.Error(err) + } +} diff --git a/core/ports/services.go b/core/ports/services.go index 021d00cb..3cc78497 100644 --- a/core/ports/services.go +++ b/core/ports/services.go @@ -21,12 +21,19 @@ type MicroVMService interface { Start(ctx context.Context, vm *models.MicroVM) error // State returns the state of a microvm. State(ctx context.Context, id string) (MicroVMState, error) + // Metrics returns with the metrics of a microvm. + Metrics(ctx context.Context, id models.VMID) (MachineMetrics, error) } // This state represents the state of the Firecracker MVM process itself // The state for the entire Flintlock MVM is represented in models.MicroVMState. type MicroVMState string +// MachineMetrics is a metrics interface for providers. +type MachineMetrics interface { + ToPrometheus() []byte +} + const ( MicroVMStateUnknown MicroVMState = "unknown" MicroVMStatePending MicroVMState = "pending" diff --git a/docs/quick-start.md b/docs/quick-start.md index 59ba2573..c251d0ad 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -8,7 +8,7 @@ and run: mdtoc -inplace docs/quick-start.md - [MacOS Users](#macos-users) - [Configure network](#configure-network) - - [Install packages and start libvirtd](#install-packages-and-start-) + - [Install packages and start libvirtd](#install-packages-and-start-libvirtd) - [Create kvm network](#create-kvm-network) - [Create and connect tap device](#create-and-connect-tap-device) - [Containerd](#containerd) @@ -20,13 +20,15 @@ and run: mdtoc -inplace docs/quick-start.md - [Set up Firecracker](#set-up-firecracker) - [Set up and start flintlock](#set-up-and-start-flintlock) - [Interacting with the service](#interacting-with-the-service) + - [hammertime](#hammertime) - [grpc-client-cli](#grpc-client-cli) - [Example](#example) - [BloomRPC](#bloomrpc) - [Import](#import) - [Example](#example-1) +- [Start metrics exporter](#start-metrics-exporter) - [Troubleshooting](#troubleshooting) - - [flintlockd fails to start with failed to reconcile vmid](#flintlockd-fails-to-start-with-) + - [flintlockd fails to start with failed to reconcile vmid](#flintlockd-fails-to-start-with-failed-to-reconcile-vmid) ## MacOS Users @@ -380,6 +382,26 @@ Click the green `>` in the centre of the screen. The response should come immedi [grpcurl]: https://github.com/fullstorydev/grpcurl [bloomrpc]: https://github.com/uw-labs/bloomrpc +## Start metrics exporter + +Flintlock has a metrics exporter called `flintlock-metrics`. It listens on an +HTTP port and serves Prometheus compatible output. + +``` +sudo ./bin/flintlock-metrics serve \ + --containerd-socket=/run/containerd-dev/containerd.sock \ + --http-endpoint=0.0.0.0:8000 +``` + +Available endpoints: + +* `/machine/uid/{uid}`: Metrics for a specific MicroVM. +* `/machine/{namespace}/{name}`: Metrics for all MicroVMs with given name and namespace. +* `/machine/{namespace}`: Metrics for all MicroVMs under a specific Namespace. +* `/machine`: Metrics for all MicroVMs from all Namespaces. + +For testing/development, there is a minimal docker compose setup under `hack/scripts/monitoring/metrics`. + ## Troubleshooting ### flintlockd fails to start with `failed to reconcile vmid` diff --git a/go.mod b/go.mod index 303a4a9b..60ab0a23 100644 --- a/go.mod +++ b/go.mod @@ -41,8 +41,11 @@ require ( ) require ( + github.com/gorilla/mux v1.8.0 + github.com/urfave/cli/v2 v2.3.0 github.com/weaveworks/flintlock/api v0.0.0-20211217111250-5f8d70c4a581 github.com/weaveworks/flintlock/client v0.0.0-00010101000000-000000000000 + github.com/yitsushi/file-tailor v1.0.0 ) require ( @@ -59,6 +62,7 @@ require ( github.com/containerd/ttrpc v1.1.0 // indirect github.com/containernetworking/cni v0.8.1 // indirect github.com/containernetworking/plugins v0.9.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-openapi/analysis v0.19.10 // indirect @@ -98,6 +102,7 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect diff --git a/go.sum b/go.sum index c4207f8f..0f7b9fa2 100644 --- a/go.sum +++ b/go.sum @@ -275,6 +275,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -569,6 +570,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -872,6 +875,7 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= @@ -953,7 +957,10 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -975,6 +982,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yitsushi/file-tailor v1.0.0 h1:f9tqOqmZtEGumL3vZCFMDWpT4M8K6O72Kg1Jl3N/af0= +github.com/yitsushi/file-tailor v1.0.0/go.mod h1:GXj00IZZ8qPqs0SARahDSJ+O3ee+WewXVorQ0YiGB98= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/hack/scripts/monitoring/metrics/docker-compose.yaml b/hack/scripts/monitoring/metrics/docker-compose.yaml new file mode 100644 index 00000000..505efd85 --- /dev/null +++ b/hack/scripts/monitoring/metrics/docker-compose.yaml @@ -0,0 +1,12 @@ +version: "3.9" +services: + prom: + image: prom/prometheus + ports: + - 9090 + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + grafana: + image: grafana/grafana:8.3.3 + ports: + - 3000:3000 diff --git a/hack/scripts/monitoring/metrics/prometheus.yml b/hack/scripts/monitoring/metrics/prometheus.yml new file mode 100644 index 00000000..b77e104b --- /dev/null +++ b/hack/scripts/monitoring/metrics/prometheus.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 15s + evaluation_interval: 30s + +scrape_configs: +- job_name: 'prometheus' + static_configs: + - targets: ['127.0.0.1:9090'] + +- job_name: flintlock + scrape_interval: 5s + metrics_path: '/machine' + static_configs: + - targets: ['192.168.100.35:8090'] diff --git a/hack/scripts/payload/CreateMicroVM.json b/hack/scripts/payload/CreateMicroVM.json index 2c94f478..2f0dd052 100644 --- a/hack/scripts/payload/CreateMicroVM.json +++ b/hack/scripts/payload/CreateMicroVM.json @@ -29,8 +29,11 @@ ], "interfaces": [ { - "guest_device_name": "eth1", - "type": 0 + "device_id": "eth1", + "type": 1, + "address": { + "address": "192.168.100.30/32" + } } ], "metadata": { diff --git a/infrastructure/firecracker/metrics.go b/infrastructure/firecracker/metrics.go new file mode 100644 index 00000000..f7c87683 --- /dev/null +++ b/infrastructure/firecracker/metrics.go @@ -0,0 +1,39 @@ +package firecracker + +import ( + "fmt" + "strings" +) + +type MachineMetrics struct { + Namespace string `json:"Namespace"` + MachineName string `json:"MachineName"` + MachineUID string `json:"MachineUID"` + Data Metrics `json:"Data"` +} + +type Metrics map[string]map[string]int64 + +func (mm MachineMetrics) ToPrometheus() []byte { + output := []string{} + labels := strings.Join( + []string{ + metricsLabel("namespace", mm.Namespace), + metricsLabel("name", mm.MachineName), + metricsLabel("uid", mm.MachineUID), + }, + ",", + ) + + for prefix, group := range mm.Data { + for key, value := range group { + output = append(output, fmt.Sprintf("%s_%s{%s} %d", prefix, key, labels, value)) + } + } + + return []byte(strings.Join(output, "\n")) +} + +func metricsLabel(key, value string) string { + return fmt.Sprintf("%s=\"%s\"", key, value) +} diff --git a/infrastructure/firecracker/provider.go b/infrastructure/firecracker/provider.go index cbeaa89a..622d2842 100644 --- a/infrastructure/firecracker/provider.go +++ b/infrastructure/firecracker/provider.go @@ -2,12 +2,15 @@ package firecracker import ( "context" + "encoding/json" "fmt" + "os" "syscall" "time" "github.com/sirupsen/logrus" "github.com/spf13/afero" + tailor "github.com/yitsushi/file-tailor" "github.com/weaveworks/flintlock/core/models" "github.com/weaveworks/flintlock/core/ports" @@ -148,3 +151,34 @@ func (p *fcProvider) State(ctx context.Context, id string) (ports.MicroVMState, return ports.MicroVMStateRunning, nil } + +func (p *fcProvider) Metrics(ctx context.Context, vmid models.VMID) (ports.MachineMetrics, error) { + machineMetrics := MachineMetrics{ + Namespace: vmid.Namespace(), + MachineName: vmid.Name(), + MachineUID: vmid.UID(), + Data: Metrics{}, + } + + vmState := NewState(vmid, p.config.StateRoot, p.fs) + + file, err := os.Open(vmState.MetricsPath()) + if err != nil { + return machineMetrics, fmt.Errorf("unable to open metrics file: %w", err) + } + + defer file.Close() + + content, err := tailor.Tail(file, 1) + if err != nil { + return machineMetrics, fmt.Errorf("unable to read the last line of the metrics file: %w", err) + } + + // It can throw an error, but we don't care. + // For example the utc_timestamp_ms field is in the root of the metrics JSON, + // and it does not follow the map[string]string pattern, but we don't care + // about that value. + _ = json.Unmarshal(content, &machineMetrics.Data) + + return machineMetrics, nil +} diff --git a/infrastructure/mock/ports.go b/infrastructure/mock/ports.go index 41c98cf8..e91b2ddf 100644 --- a/infrastructure/mock/ports.go +++ b/infrastructure/mock/ports.go @@ -78,6 +78,21 @@ func (mr *MockMicroVMServiceMockRecorder) Delete(arg0, arg1 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockMicroVMService)(nil).Delete), arg0, arg1) } +// Metrics mocks base method. +func (m *MockMicroVMService) Metrics(arg0 context.Context, arg1 models.VMID) (ports.MachineMetrics, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Metrics", arg0, arg1) + ret0, _ := ret[0].(ports.MachineMetrics) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Metrics indicates an expected call of Metrics. +func (mr *MockMicroVMServiceMockRecorder) Metrics(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Metrics", reflect.TypeOf((*MockMicroVMService)(nil).Metrics), arg0, arg1) +} + // Start mocks base method. func (m *MockMicroVMService) Start(arg0 context.Context, arg1 *models.MicroVM) error { m.ctrl.T.Helper() diff --git a/internal/command/flags/urfave.go b/internal/command/flags/urfave.go new file mode 100644 index 00000000..d76f639a --- /dev/null +++ b/internal/command/flags/urfave.go @@ -0,0 +1,74 @@ +package flags + +import ( + "github.com/urfave/cli/v2" + + "github.com/weaveworks/flintlock/internal/config" + "github.com/weaveworks/flintlock/pkg/defaults" +) + +type WithFlagsFunc func() []cli.Flag + +func CLIFlags(options ...WithFlagsFunc) []cli.Flag { + flags := []cli.Flag{} + + for _, group := range options { + flags = append(flags, group()...) + } + + return flags +} + +func WithContainerDFlags() WithFlagsFunc { + return func() []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: containerdSocketFlag, + Value: defaults.ContainerdSocket, + Usage: "The path to the containerd socket.", + }, + &cli.StringFlag{ + Name: containerdNamespace, + Value: defaults.ContainerdNamespace, + Usage: "The name of the containerd namespace to use.", + }, + } + } +} + +func WithHTTPEndpointFlags() WithFlagsFunc { + return func() []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: httpEndpointFlag, + Value: defaults.HTTPAPIEndpoint, + Usage: "The endpoint for the HTTP server to listen on.", + }, + } + } +} + +func WithGlobalConfigFlags() WithFlagsFunc { + return func() []cli.Flag { + return []cli.Flag{ + &cli.StringFlag{ + Name: "state-dir", + Value: defaults.StateRootDir, + Usage: "The directory to use for the as the root for runtime state.", + }, + } + } +} + +func ParseFlags(cfg *config.Config) cli.BeforeFunc { + return func(ctx *cli.Context) error { + cfg.HTTPAPIEndpoint = ctx.String(httpEndpointFlag) + + cfg.CtrSocketPath = ctx.String(containerdSocketFlag) + cfg.CtrNamespace = ctx.String(containerdNamespace) + + cfg.StateRootDir = ctx.String("state-dir") + + return nil + } +} diff --git a/internal/command/metrics/main.go b/internal/command/metrics/main.go new file mode 100644 index 00000000..a09ee2e5 --- /dev/null +++ b/internal/command/metrics/main.go @@ -0,0 +1,27 @@ +package metrics + +import ( + "io" + + "github.com/urfave/cli/v2" +) + +func NewApp(out io.Writer) *cli.App { + app := cli.NewApp() + + if out != nil { + app.Writer = out + } + + app.Name = "flintlock-metrics" + app.EnableBashCompletion = true + app.Commands = commands() + + return app +} + +func commands() []*cli.Command { + return []*cli.Command{ + serveCommand(), + } +} diff --git a/internal/command/metrics/serve.go b/internal/command/metrics/serve.go new file mode 100644 index 00000000..5c81166d --- /dev/null +++ b/internal/command/metrics/serve.go @@ -0,0 +1,169 @@ +package metrics + +import ( + "context" + "fmt" + "net/http" + + "github.com/gorilla/mux" + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" + + "github.com/weaveworks/flintlock/core/models" + "github.com/weaveworks/flintlock/core/ports" + "github.com/weaveworks/flintlock/internal/command/flags" + "github.com/weaveworks/flintlock/internal/config" + "github.com/weaveworks/flintlock/internal/inject" +) + +type serveFunc func(http.ResponseWriter, *http.Request) + +func serveCommand() *cli.Command { + cfg := &config.Config{} + + return &cli.Command{ + Name: "serve", + Aliases: []string{"s"}, + Usage: "Listen and serve HTTP.", + Before: flags.ParseFlags(cfg), + Flags: flags.CLIFlags( + flags.WithContainerDFlags(), + flags.WithHTTPEndpointFlags(), + flags.WithGlobalConfigFlags(), + ), + Action: func(c *cli.Context) error { + return serve(cfg) + }, + } +} + +func serve(cfg *config.Config) error { + aports, err := inject.InitializePorts(cfg) + if err != nil { + return fmt.Errorf("initialising ports for application: %w", err) + } + + router := mux.NewRouter() + + router.HandleFunc("/machine/uid/{uid}", serveMachineByUID(aports)) + router.HandleFunc("/machine/{namespace}/{name}", serveMachinesByName(aports)) + router.HandleFunc("/machine/{namespace}", serveMachinesByNamespace(aports)) + router.HandleFunc("/machine", serveAllMachines(aports)) + + logrus.Infof("Start listening on %s", cfg.HTTPAPIEndpoint) + + return http.ListenAndServe(cfg.HTTPAPIEndpoint, router) +} + +func getAllMachineMetrics(ctx context.Context, aports *ports.Collection, query models.ListMicroVMQuery) ([]ports.MachineMetrics, error) { + mms := []ports.MachineMetrics{} + + machines, err := aports.Repo.GetAll(ctx, query) + if err != nil { + return mms, err + } + + for _, machine := range machines { + metrics, err := aports.Provider.Metrics(ctx, machine.ID) + if err != nil { + return mms, err + } + + mms = append(mms, metrics) + } + + return mms, nil +} + +func serveMachineByUID(aports *ports.Collection) serveFunc { + return func(response http.ResponseWriter, request *http.Request) { + vars := mux.Vars(request) + + vm, err := aports.Repo.Get(context.Background(), ports.RepositoryGetOptions{ + UID: vars["uid"], + }) + if err != nil { + logrus.Error(err.Error()) + response.WriteHeader(http.StatusInternalServerError) + + return + } + + metrics, err := aports.Provider.Metrics(context.Background(), vm.ID) + if err != nil { + logrus.Error(err.Error()) + response.WriteHeader(http.StatusInternalServerError) + + return + } + + response.WriteHeader(http.StatusOK) + + _, _ = response.Write(metrics.ToPrometheus()) + } +} + +func serveMachinesByName(aports *ports.Collection) serveFunc { + return func(response http.ResponseWriter, request *http.Request) { + vars := mux.Vars(request) + + mms, err := getAllMachineMetrics( + context.Background(), + aports, + models.ListMicroVMQuery{ + "namespace": vars["namespace"], + "name": vars["name"], + }, + ) + if err != nil { + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(err.Error())) + + return + } + + response.WriteHeader(http.StatusOK) + + for _, mm := range mms { + _, _ = response.Write(append(mm.ToPrometheus(), '\n')) + } + } +} + +func serveMachinesByNamespace(aports *ports.Collection) serveFunc { + return func(response http.ResponseWriter, request *http.Request) { + vars := mux.Vars(request) + + mms, err := getAllMachineMetrics(context.Background(), aports, models.ListMicroVMQuery{"namespace": vars["namespace"]}) + if err != nil { + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(err.Error())) + + return + } + + response.WriteHeader(http.StatusOK) + + for _, mm := range mms { + _, _ = response.Write(append(mm.ToPrometheus(), '\n')) + } + } +} + +func serveAllMachines(aports *ports.Collection) serveFunc { + return func(response http.ResponseWriter, request *http.Request) { + mms, err := getAllMachineMetrics(context.Background(), aports, models.ListMicroVMQuery{}) + if err != nil { + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(err.Error())) + + return + } + + response.WriteHeader(http.StatusOK) + + for _, mm := range mms { + _, _ = response.Write(append(mm.ToPrometheus(), '\n')) + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 7879ef56..02d195dc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,7 +14,7 @@ type Config struct { Logging log.Config // GRPCEndpoint is the endpoint for the gRPC server. GRPCAPIEndpoint string - // HTTPAPIEndpoint is the endpoint for the HHTP proxy for the gRPC service.. + // HTTPAPIEndpoint is the endpoint for the HTTP proxy for the gRPC service.. HTTPAPIEndpoint string // FirecrackerBin is the firecracker binary to use. FirecrackerBin string diff --git a/userdocs/docs/getting-started/extras/metrics.md b/userdocs/docs/getting-started/extras/metrics.md new file mode 100644 index 00000000..43e67cba --- /dev/null +++ b/userdocs/docs/getting-started/extras/metrics.md @@ -0,0 +1,19 @@ +# Start metrics exporter + +Flintlock has a metrics exporter called `flintlock-metrics`. It listens on an +HTTP port and serves Prometheus compatible output. + +``` +sudo ./bin/flintlock-metrics serve \ + --containerd-socket=/run/containerd-dev/containerd.sock \ + --http-endpoint=0.0.0.0:8000 +``` + +Available endpoints: + +* `/machine/uid/{uid}`: Metrics for a specific MicroVM. +* `/machine/{namespace}/{name}`: Metrics for all MicroVMs with given name and namespace. +* `/machine/{namespace}`: Metrics for all MicroVMs under a specific Namespace. +* `/machine`: Metrics for all MicroVMs from all Namespaces. + +For testing/development, there is a minimal docker compose setup under `hack/scripts/monitoring/metrics`.