Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Expose firecracker metrics #357

Merged
merged 3 commits into from
Feb 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,3 +76,5 @@ jobs:
files: |
bin/flintlockd_amd64
bin/flintlockd_arm64
bin/flintlock-metrics_amd64
bin/flintlock-metrics_arm64
2 changes: 0 additions & 2 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ linters-settings:
local-prefixes: github.com/weaveworks/flintlock
govet:
check-shadowing: true
misspell:
locale: GB
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Duplicated entry, removed one of them.

nolintlint:
allow-leading-space: false
allow-unused: false
Expand Down
26 changes: 21 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ REPO_ROOT := $(shell git rev-parse --show-toplevel)
BIN_DIR := bin
OUT_DIR := out
FLINTLOCKD_CMD := cmd/flintlockd
FLINTLOCK_METRICS_CMD := cmd/flintlock-metrics
TOOLS_DIR := hack/tools
TOOLS_BIN_DIR := $(TOOLS_DIR)/bin
TOOLS_SHARE_DIR := $(TOOLS_DIR)/share
Expand Down Expand Up @@ -54,13 +55,28 @@ test_image = weaveworks/flintlock-e2e
##@ Build

.PHONY: build
build: $(BIN_DIR) ## Build the binaries
go build -o $(BIN_DIR)/flintlockd ./cmd/flintlockd
build: build-flintlockd build-flintlock-metrics ## Build the binaries

.PHONY: build-flintlockd
build-flintlockd: $(BIN_DIR) ## Build flintlockd binary
go build -o $(BIN_DIR)/flintlockd ./$(FLINTLOCKD_CMD)

.PHONY: build-flintlock-metrics ## Build flintlock-metrics binary
build-flintlock-metrics: $(BIN_DIR)
go build -o $(BIN_DIR)/flintlock-metrics ./$(FLINTLOCK_METRICS_CMD)

.PHONY: build-release
build-release: $(BIN_DIR) ## Build the release binaries
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o $(BIN_DIR)/flintlockd_amd64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./cmd/flintlockd
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/flintlockd_arm64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./cmd/flintlockd
build-release: build-release-flintlockd build-release-flintlock-metrics ## Build the release binaries

.PHONY: build-release-flintlockd
build-release-flintlockd: $(BIN_DIR) ## Build flintlockd release binaries
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o $(BIN_DIR)/flintlockd_amd64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./$(FLINTLOCKD_CMD)
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/flintlockd_arm64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./$(FLINTLOCKD_CMD)

.PHONY: build-release-flintlock-metrics
build-release-flintlock-metrics: $(BIN_DIR) ## Build flintlock-metrics release binaries
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o $(BIN_DIR)/flintlock-metrics_amd64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./$(FLINTLOCK_METRICS_CMD)
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o $(BIN_DIR)/flintlock-metrics_arm64 -ldflags "-X $(VERSION_PKG).Version=$(VERSION) -X $(VERSION_PKG).BuildDate=$(BUILD_DATE) -X $(VERSION_PKG).CommitHash=$(GIT_COMMIT)" ./$(FLINTLOCK_METRICS_CMD)

##@ Generate

Expand Down
17 changes: 17 additions & 0 deletions cmd/flintlock-metrics/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package main

import (
"os"

"github.com/sirupsen/logrus"

"github.com/weaveworks/flintlock/internal/command/metrics"
)

func main() {
app := metrics.NewApp(os.Stdout)

if err := app.Run(os.Args); err != nil {
logrus.Error(err)
}
}
7 changes: 7 additions & 0 deletions core/ports/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,19 @@ type MicroVMService interface {
Start(ctx context.Context, vm *models.MicroVM) error
// State returns the state of a microvm.
State(ctx context.Context, id string) (MicroVMState, error)
// Metrics returns with the metrics of a microvm.
Metrics(ctx context.Context, id models.VMID) (MachineMetrics, error)
}

// This state represents the state of the Firecracker MVM process itself
// The state for the entire Flintlock MVM is represented in models.MicroVMState.
type MicroVMState string

// MachineMetrics is a metrics interface for providers.
type MachineMetrics interface {
ToPrometheus() []byte
}

const (
MicroVMStateUnknown MicroVMState = "unknown"
MicroVMStatePending MicroVMState = "pending"
Expand Down
26 changes: 24 additions & 2 deletions docs/quick-start.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ and run: mdtoc -inplace docs/quick-start.md
<!-- toc -->
- [MacOS Users](#macos-users)
- [Configure network](#configure-network)
- [Install packages and start <code>libvirtd</code>](#install-packages-and-start-)
- [Install packages and start <code>libvirtd</code>](#install-packages-and-start-libvirtd)
- [Create kvm network](#create-kvm-network)
- [Create and connect tap device](#create-and-connect-tap-device)
- [Containerd](#containerd)
Expand All @@ -20,13 +20,15 @@ and run: mdtoc -inplace docs/quick-start.md
- [Set up Firecracker](#set-up-firecracker)
- [Set up and start flintlock](#set-up-and-start-flintlock)
- [Interacting with the service](#interacting-with-the-service)
- [hammertime](#hammertime)
- [grpc-client-cli](#grpc-client-cli)
- [Example](#example)
- [BloomRPC](#bloomrpc)
- [Import](#import)
- [Example](#example-1)
- [Start metrics exporter](#start-metrics-exporter)
- [Troubleshooting](#troubleshooting)
- [flintlockd fails to start with <code>failed to reconcile vmid</code>](#flintlockd-fails-to-start-with-)
- [flintlockd fails to start with <code>failed to reconcile vmid</code>](#flintlockd-fails-to-start-with-failed-to-reconcile-vmid)
<!-- /toc -->

## MacOS Users
Expand Down Expand Up @@ -380,6 +382,26 @@ Click the green `>` in the centre of the screen. The response should come immedi
[grpcurl]: https://github.com/fullstorydev/grpcurl
[bloomrpc]: https://github.com/uw-labs/bloomrpc

## Start metrics exporter

Flintlock has a metrics exporter called `flintlock-metrics`. It listens on an
HTTP port and serves Prometheus compatible output.

```
sudo ./bin/flintlock-metrics serve \
--containerd-socket=/run/containerd-dev/containerd.sock \
--http-endpoint=0.0.0.0:8000
```

Available endpoints:

* `/machine/uid/{uid}`: Metrics for a specific MicroVM.
* `/machine/{namespace}/{name}`: Metrics for all MicroVMs with given name and namespace.
* `/machine/{namespace}`: Metrics for all MicroVMs under a specific Namespace.
* `/machine`: Metrics for all MicroVMs from all Namespaces.

For testing/development, there is a minimal docker compose setup under `hack/scripts/monitoring/metrics`.

## Troubleshooting

### flintlockd fails to start with `failed to reconcile vmid`
Expand Down
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,11 @@ require (
)

require (
github.com/gorilla/mux v1.8.0
github.com/urfave/cli/v2 v2.3.0
github.com/weaveworks/flintlock/api v0.0.0-20211217111250-5f8d70c4a581
github.com/weaveworks/flintlock/client v0.0.0-00010101000000-000000000000
github.com/yitsushi/file-tailor v1.0.0
)

require (
Expand All @@ -59,6 +62,7 @@ require (
github.com/containerd/ttrpc v1.1.0 // indirect
github.com/containernetworking/cni v0.8.1 // indirect
github.com/containernetworking/plugins v0.9.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-openapi/analysis v0.19.10 // indirect
Expand Down Expand Up @@ -98,6 +102,7 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
Expand Down Expand Up @@ -569,6 +570,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
Expand Down Expand Up @@ -872,6 +875,7 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
Expand Down Expand Up @@ -953,7 +957,10 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
Expand All @@ -975,6 +982,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yitsushi/file-tailor v1.0.0 h1:f9tqOqmZtEGumL3vZCFMDWpT4M8K6O72Kg1Jl3N/af0=
github.com/yitsushi/file-tailor v1.0.0/go.mod h1:GXj00IZZ8qPqs0SARahDSJ+O3ee+WewXVorQ0YiGB98=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
12 changes: 12 additions & 0 deletions hack/scripts/monitoring/metrics/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
version: "3.9"
services:
prom:
image: prom/prometheus
ports:
- 9090
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
grafana:
image: grafana/grafana:8.3.3
ports:
- 3000:3000
14 changes: 14 additions & 0 deletions hack/scripts/monitoring/metrics/prometheus.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
global:
scrape_interval: 15s
evaluation_interval: 30s

scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['127.0.0.1:9090']

- job_name: flintlock
scrape_interval: 5s
metrics_path: '/machine'
static_configs:
- targets: ['192.168.100.35:8090']
7 changes: 5 additions & 2 deletions hack/scripts/payload/CreateMicroVM.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,11 @@
],
"interfaces": [
{
"guest_device_name": "eth1",
"type": 0
"device_id": "eth1",
"type": 1,
"address": {
"address": "192.168.100.30/32"
}
}
],
"metadata": {
Expand Down
39 changes: 39 additions & 0 deletions infrastructure/firecracker/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package firecracker

import (
"fmt"
"strings"
)

type MachineMetrics struct {
Namespace string `json:"Namespace"`
MachineName string `json:"MachineName"`
MachineUID string `json:"MachineUID"`
Data Metrics `json:"Data"`
}

type Metrics map[string]map[string]int64

func (mm MachineMetrics) ToPrometheus() []byte {
output := []string{}
labels := strings.Join(
[]string{
metricsLabel("namespace", mm.Namespace),
metricsLabel("name", mm.MachineName),
metricsLabel("uid", mm.MachineUID),
},
",",
)

for prefix, group := range mm.Data {
for key, value := range group {
output = append(output, fmt.Sprintf("%s_%s{%s} %d", prefix, key, labels, value))
}
}

return []byte(strings.Join(output, "\n"))
}

func metricsLabel(key, value string) string {
return fmt.Sprintf("%s=\"%s\"", key, value)
}
34 changes: 34 additions & 0 deletions infrastructure/firecracker/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@ package firecracker

import (
"context"
"encoding/json"
"fmt"
"os"
"syscall"
"time"

"github.com/sirupsen/logrus"
"github.com/spf13/afero"
tailor "github.com/yitsushi/file-tailor"

"github.com/weaveworks/flintlock/core/models"
"github.com/weaveworks/flintlock/core/ports"
Expand Down Expand Up @@ -148,3 +151,34 @@ func (p *fcProvider) State(ctx context.Context, id string) (ports.MicroVMState,

return ports.MicroVMStateRunning, nil
}

func (p *fcProvider) Metrics(ctx context.Context, vmid models.VMID) (ports.MachineMetrics, error) {
machineMetrics := MachineMetrics{
Namespace: vmid.Namespace(),
MachineName: vmid.Name(),
MachineUID: vmid.UID(),
Data: Metrics{},
}

vmState := NewState(vmid, p.config.StateRoot, p.fs)

file, err := os.Open(vmState.MetricsPath())
if err != nil {
return machineMetrics, fmt.Errorf("unable to open metrics file: %w", err)
}

defer file.Close()

content, err := tailor.Tail(file, 1)
if err != nil {
return machineMetrics, fmt.Errorf("unable to read the last line of the metrics file: %w", err)
}

// It can throw an error, but we don't care.
// For example the utc_timestamp_ms field is in the root of the metrics JSON,
// and it does not follow the map[string]string pattern, but we don't care
// about that value.
_ = json.Unmarshal(content, &machineMetrics.Data)

return machineMetrics, nil
}
15 changes: 15 additions & 0 deletions infrastructure/mock/ports.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading