Skip to content

Commit

Permalink
feat: Expose firecracker metrics
Browse files Browse the repository at this point in the history
* Basic logic for the metrics exporter.
* Basic docker compose file to start a prometheus and grafana instance
  for testing.
* Metrics collection under provider.
* Documentation

Fixes #210
  • Loading branch information
yitsushi committed Jan 27, 2022
1 parent 1e52881 commit 7e31a76
Show file tree
Hide file tree
Showing 17 changed files with 468 additions and 7 deletions.
2 changes: 0 additions & 2 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ linters-settings:
local-prefixes: github.com/weaveworks/flintlock
govet:
check-shadowing: true
misspell:
locale: GB
nolintlint:
allow-leading-space: false
allow-unused: false
Expand Down
17 changes: 17 additions & 0 deletions cmd/flintlock-metrics/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package main

import (
"os"

"github.com/sirupsen/logrus"

"github.com/weaveworks/flintlock/internal/command/metrics"
)

func main() {
app := metrics.NewApp(os.Stdout)

if err := app.Run(os.Args); err != nil {
logrus.Error(err)
}
}
7 changes: 7 additions & 0 deletions core/ports/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,19 @@ type MicroVMService interface {
Start(ctx context.Context, vm *models.MicroVM) error
// State returns the state of a microvm.
State(ctx context.Context, id string) (MicroVMState, error)
// Metrics returns with the metrics of a microvm.
Metrics(ctx context.Context, id models.VMID) (MachineMetrics, error)
}

// This state represents the state of the Firecracker MVM process itself
// The state for the entire Flintlock MVM is represented in models.MicroVMState.
type MicroVMState string

// MachineMetrics is a metrics interface for providers.
type MachineMetrics interface {
ToPrometheus() []byte
}

const (
MicroVMStateUnknown MicroVMState = "unknown"
MicroVMStatePending MicroVMState = "pending"
Expand Down
26 changes: 24 additions & 2 deletions docs/quick-start.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ and run: mdtoc -inplace docs/quick-start.md
<!-- toc -->
- [MacOS Users](#macos-users)
- [Configure network](#configure-network)
- [Install packages and start <code>libvirtd</code>](#install-packages-and-start-)
- [Install packages and start <code>libvirtd</code>](#install-packages-and-start-libvirtd)
- [Create kvm network](#create-kvm-network)
- [Create and connect tap device](#create-and-connect-tap-device)
- [Containerd](#containerd)
Expand All @@ -20,13 +20,15 @@ and run: mdtoc -inplace docs/quick-start.md
- [Set up Firecracker](#set-up-firecracker)
- [Set up and start flintlock](#set-up-and-start-flintlock)
- [Interacting with the service](#interacting-with-the-service)
- [hammertime](#hammertime)
- [grpc-client-cli](#grpc-client-cli)
- [Example](#example)
- [BloomRPC](#bloomrpc)
- [Import](#import)
- [Example](#example-1)
- [Start metrics exporter](#start-metrics-exporter)
- [Troubleshooting](#troubleshooting)
- [flintlockd fails to start with <code>failed to reconcile vmid</code>](#flintlockd-fails-to-start-with-)
- [flintlockd fails to start with <code>failed to reconcile vmid</code>](#flintlockd-fails-to-start-with-failed-to-reconcile-vmid)
<!-- /toc -->

## MacOS Users
Expand Down Expand Up @@ -380,6 +382,26 @@ Click the green `>` in the centre of the screen. The response should come immedi
[grpcurl]: https://github.com/fullstorydev/grpcurl
[bloomrpc]: https://github.com/uw-labs/bloomrpc

## Start metrics exporter

Flintlock has a metrics exporter called `flintlock-metrics`. It listens on an
HTTP port and serves Prometheus compatible output.

```
sudo ./bin/flintlock-metrics serve \
--containerd-socket=/run/containerd-dev/containerd.sock \
--http-endpoint=0.0.0.0:8000
```

Available endpoints:

* `/machine/uid/{uid}`: Metrics for a specific MicroVM.
* `/machine/{namespace}/{name}`: Metrics for all MicroVMs with given name and namespace.
* `/machine/{namespace}`: Metrics for all MicroVMs under a specific Namespace.
* `/machine`: Metrics for all MicroVMs from all Namespaces.

For testing/development, there is a minimal docker compose setup under `hack/scripts/monitoring/metrics`.

## Troubleshooting

### flintlockd fails to start with `failed to reconcile vmid`
Expand Down
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,11 @@ require (
)

require (
github.com/gorilla/mux v1.8.0
github.com/urfave/cli/v2 v2.3.0
github.com/weaveworks/flintlock/api v0.0.0-20211217111250-5f8d70c4a581
github.com/weaveworks/flintlock/client v0.0.0-00010101000000-000000000000
github.com/yitsushi/file-tailor v1.0.0
)

require (
Expand All @@ -59,6 +62,7 @@ require (
github.com/containerd/ttrpc v1.1.0 // indirect
github.com/containernetworking/cni v0.8.1 // indirect
github.com/containernetworking/plugins v0.9.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-openapi/analysis v0.19.10 // indirect
Expand Down Expand Up @@ -100,6 +104,7 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
Expand Down Expand Up @@ -570,6 +571,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
Expand Down Expand Up @@ -870,6 +873,7 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
Expand Down Expand Up @@ -951,7 +955,10 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
Expand All @@ -973,6 +980,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yitsushi/file-tailor v1.0.0 h1:f9tqOqmZtEGumL3vZCFMDWpT4M8K6O72Kg1Jl3N/af0=
github.com/yitsushi/file-tailor v1.0.0/go.mod h1:GXj00IZZ8qPqs0SARahDSJ+O3ee+WewXVorQ0YiGB98=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
12 changes: 12 additions & 0 deletions hack/scripts/monitoring/metrics/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
version: "3.9"
services:
prom:
image: prom/prometheus
ports:
- 9090
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
grafana:
image: grafana/grafana:8.3.3
ports:
- 3000:3000
14 changes: 14 additions & 0 deletions hack/scripts/monitoring/metrics/prometheus.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
global:
scrape_interval: 15s
evaluation_interval: 30s

scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['127.0.0.1:9090']

- job_name: flintlock
scrape_interval: 5s
metrics_path: '/machine'
static_configs:
- targets: ['192.168.100.35:8090']
7 changes: 5 additions & 2 deletions hack/scripts/payload/CreateMicroVM.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,11 @@
],
"interfaces": [
{
"guest_device_name": "eth1",
"type": 0
"device_id": "eth1",
"type": 1,
"address": {
"address": "192.168.100.30/32"
}
}
],
"metadata": {
Expand Down
39 changes: 39 additions & 0 deletions infrastructure/firecracker/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package firecracker

import (
"fmt"
"strings"
)

type MachineMetrics struct {
Namespace string `json:"Namespace"`
MachineName string `json:"MachineName"`
MachineUID string `json:"MachineUID"`
Data Metrics `json:"Data"`
}

type Metrics map[string]map[string]int64

func (mm MachineMetrics) ToPrometheus() []byte {
output := []string{}
labels := strings.Join(
[]string{
metricsLabel("namespace", mm.Namespace),
metricsLabel("name", mm.MachineName),
metricsLabel("uid", mm.MachineUID),
},
",",
)

for prefix, group := range mm.Data {
for key, value := range group {
output = append(output, fmt.Sprintf("%s_%s{%s} %d", prefix, key, labels, value))
}
}

return []byte(strings.Join(output, "\n"))
}

func metricsLabel(key, value string) string {
return fmt.Sprintf("%s=\"%s\"", key, value)
}
34 changes: 34 additions & 0 deletions infrastructure/firecracker/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@ package firecracker

import (
"context"
"encoding/json"
"fmt"
"os"
"syscall"
"time"

"github.com/sirupsen/logrus"
"github.com/spf13/afero"
tailor "github.com/yitsushi/file-tailor"

"github.com/weaveworks/flintlock/core/models"
"github.com/weaveworks/flintlock/core/ports"
Expand Down Expand Up @@ -148,3 +151,34 @@ func (p *fcProvider) State(ctx context.Context, id string) (ports.MicroVMState,

return ports.MicroVMStateRunning, nil
}

func (p *fcProvider) Metrics(ctx context.Context, vmid models.VMID) (ports.MachineMetrics, error) {
machineMetrics := MachineMetrics{
Namespace: vmid.Namespace(),
MachineName: vmid.Name(),
MachineUID: vmid.UID(),
Data: Metrics{},
}

vmState := NewState(vmid, p.config.StateRoot, p.fs)

file, err := os.Open(vmState.MetricsPath())
if err != nil {
return machineMetrics, fmt.Errorf("unable to open metrics file: %w", err)
}

defer file.Close()

content, err := tailor.Tail(file, 1)
if err != nil {
return machineMetrics, fmt.Errorf("unable to read the last line of the metrics file: %w", err)
}

// It can throw an error, but we don't care.
// For example the utc_timestamp_ms field is in the root of the metrics JSON,
// and it does not follow the map[string]string pattern, but we don't care
// about that value.
_ = json.Unmarshal(content, &machineMetrics.Data)

return machineMetrics, nil
}
15 changes: 15 additions & 0 deletions infrastructure/mock/ports.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 7e31a76

Please sign in to comment.