Skip to content

Commit

Permalink
Merge pull request #1612 from orbs-network/feature/metrics-json
Browse files Browse the repository at this point in the history
Metrics update names + structure (merge pending)
  • Loading branch information
noambergIL authored Aug 11, 2020
2 parents f31cb35 + 2e0c6b7 commit a3f7be2
Show file tree
Hide file tree
Showing 44 changed files with 586 additions and 454 deletions.
119 changes: 0 additions & 119 deletions .circleci/check-audit-node-deployment.js

This file was deleted.

41 changes: 5 additions & 36 deletions .circleci/check-e2e-network-liveness.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env node

const { waitUntilSync, getBlockHeight } = require('@orbs-network/orbs-nebula/lib/metrics');
const {eventuallyClosingBlocks} = require('./check-metrics');

const targetChainId = process.argv[2] || 42;
const optionalAlternativePollInterval = process.argv[3] || 60;
Expand All @@ -10,44 +10,13 @@ if (!targetChainId) {
process.exit(1);
}

async function eventuallyClosingBlocks({ chainId }) {
let publicApiEndpoint;

publicApiEndpoint = 'localhost:8080';

// First let's get the current blockheight and wait for it to close 5 more blocks
const currentBlockheight = await getBlockHeight(publicApiEndpoint);
console.log('Fetching current blockheight of the network: ', currentBlockheight);

try {
let minuteCounter = 0;

// This is here to avoid 10 minutes without output to the terminal on CircleCI.
setInterval(async () => {
minuteCounter++;
const sampleBlockheight = await getBlockHeight(publicApiEndpoint);
console.log(`${minuteCounter}m Network blockheight: ${sampleBlockheight}`);
}, 60 * 1000);

await waitUntilSync(publicApiEndpoint, currentBlockheight + 5, optionalAlternativePollInterval * 1000, 60 * 1000 * 60);

return {
ok: true,
chainId
};
} catch (err) {
console.log('Network is not advancing for vchain: ', chainId, ' with error: ', err);
return err;
}
}

(async () => {
const result = await eventuallyClosingBlocks({ targetChainId });
if (result.ok === true) {
const result = await eventuallyClosingBlocks('localhost:8080', optionalAlternativePollInterval );
if (result === true) {
console.log(`Blocks are being closed on localhost network (chain ${targetChainId}) !`);
process.exit(0);
} else {
console.error('Chain not closing blocks within the defined 15 minutes window, quiting..');
console.error('Chain not closing blocks within the defined window, quiting..');
process.exit(3);
}
})();
})();
72 changes: 72 additions & 0 deletions .circleci/check-metrics.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
const request = require("request-promise");

class Metrics {
constructor(endpoint) {
this._endpoint = endpoint;
this._metrics = {};
}

async refresh() {
try {
const body = await request(`http://${this._endpoint}/metrics`, {
timeout: 1000,
});
this._metrics = JSON.parse(body);
} catch (e) {
// Suppressed errors
console.error(`${e.message}: ${this._endpoint}`);
}
}

_get(names){
let curr = this._metrics.Payload;
for (let i = 0;i < names.length; i++) {
if (curr[names[i]]) {
curr = curr[names[i]];
} else {
return 0;
}
}
return curr;
}

getBlockHeight() { return this._get(["BlockStorage", "LastCommitted", "BlockHeight"]) || 0}
}

async function eventuallyClosingBlocks(endpoint, pollingIntervalSeconds) {
let metrics = new Metrics(endpoint);
await metrics.refresh();
pollingIntervalSeconds = pollingIntervalSeconds || 60;
if (pollingIntervalSeconds <= 0) {
pollingIntervalSeconds = 60;
}

// First let's get the current blockheight and wait for it to close 5 more blocks
const startBlockHeight = metrics.getBlockHeight();
console.log('Fetching current blockheight of the network: ', startBlockHeight);

for (let counter = 0; counter < 60; counter++) {
await sleep(pollingIntervalSeconds * 1000);
await metrics.refresh();
let currentBlockHeight = metrics.getBlockHeight();
if (currentBlockHeight >= startBlockHeight + 5) {
console.log(`got to ${currentBlockHeight} - yay!`);
return true;
}
if (counter % 5 === 0) {
console.log(`try #${counter + 1}: network blockheight: ${currentBlockHeight}`);
}
}
return false;
}

// general
function sleep(ms) {
return new Promise(resolve => {
setTimeout(resolve, ms)
})
}

module.exports = {
Metrics, eventuallyClosingBlocks
};
29 changes: 0 additions & 29 deletions .circleci/deploy-audit-node.sh

This file was deleted.

8 changes: 4 additions & 4 deletions Metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ Useful to app developers and node operators.

### Transactions per second (block storage)
Retrieved from:
* `TransactionPool.CommitRate.PerSecond` (since `v1.0.0`)
* `TransactionPool.CommitRate` (since `v2.0.1`) (was `TransactionPool.CommitRate.PerSecond` since `v1.0.0`)

Meaning:
* Amount of transactions from committed blocks that were moved from pending pool to committed pool.
Expand All @@ -55,7 +55,7 @@ Useful to app developers and node operators.

### Transactions per second (public API)
Retrieved from:
* `PublicAPI.Transactions.PerSecond` (since `v1.1.0`)
* `PublicA[i].Transactions` (since `v2.0.1`) (was `PublicAPI.Transactions.PerSecond` since `v1.1.0`)

Meaning:
* Indicates the number of transactions received by virtual chain public API.
Expand All @@ -64,7 +64,7 @@ Useful to app developers and node operators.

### Queries per second (public API)
Retrieved from:
* `PublicAPI.Queries.PerSecond` (since `v1.1.0`)
* `PublicApi.Queries` (since `v2.0.1`) (was `PublicAPI.Queries.PerSecond` since `v1.1.0`)

Meaning:
* Indicates the number of queries received by virtual chain public API. Queries differ from transactions because they only use read-only methods of the smart contract and does not alter the state, therefore the node can serve many more queries than it can process transactions.
Expand Down Expand Up @@ -100,7 +100,7 @@ Useful to node operators.

### Resources metrics
Retrieved from:
* `OS.Process.CPU.PerCent` (since `v1.0.0`)
* `OS.Process.CPU.Percent` (since `v2.0.1`) (was `OS.Process.CPU.PerCent` since `v1.0.0`)
* `OS.Process.Memory.Bytes` (since `v1.0.0`)
* `BlockStorage.FileSystemSize.Bytes` (since `v1.0.0`)

Expand Down
21 changes: 6 additions & 15 deletions bootstrap/gamma/harness.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
package gamma

import (
"encoding/json"
"fmt"
"github.com/orbs-network/orbs-client-sdk-go/codec"
orbsClient "github.com/orbs-network/orbs-client-sdk-go/orbs"
"github.com/orbs-network/orbs-network-go/instrumentation/metric"
"github.com/orbs-network/orbs-network-go/services/blockstorage"
"github.com/orbs-network/orbs-network-go/test"
"github.com/orbs-network/orbs-spec/types/go/primitives"
"github.com/orbs-network/orbs-spec/types/go/protocol"
"github.com/orbs-network/scribe/log"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"io/ioutil"
"net/http"
"strings"
"testing"
Expand All @@ -20,24 +20,16 @@ import (

const WAIT_FOR_BLOCK_TIMEOUT = 20 * time.Second

type metrics map[string]map[string]interface{}

func waitForBlock(endpoint string, targetBlockHeight primitives.BlockHeight) func() error {
return func() error {
res, err := http.Get(endpoint + "/metrics")
if err != nil {
return err
}

readBytes, err := ioutil.ReadAll(res.Body)
metricReader, err := metric.NewReader(endpoint + "/metrics")
if err != nil {
return err
}
m := make(metrics)
json.Unmarshal(readBytes, &m)

blockHeight := m["BlockStorage.BlockHeight"]["Value"].(float64)
if primitives.BlockHeight(blockHeight) < targetBlockHeight {
if blockHeight, found := metricReader.GetAsInt(blockstorage.MetricBlockHeight); !found {
return errors.Errorf("field value for key %s is not found", blockstorage.MetricBlockHeight)
} else if primitives.BlockHeight(blockHeight) < targetBlockHeight {
return errors.Errorf("block %d is less than target block %d", int(blockHeight), targetBlockHeight)
}

Expand All @@ -55,7 +47,6 @@ func RunOnRandomPort(t testing.TB, overrideConfig string) string {
}

func SendTransaction(t testing.TB, orbs *orbsClient.OrbsClient, sender *orbsClient.OrbsAccount, contractName string, method string, args ...interface{}) *codec.SendTransactionResponse {

tx, txId, err := orbs.CreateTransaction(sender.PublicKey, sender.PrivateKey, contractName, method, args...)
require.NoError(t, err, "failed creating tx %s.%s", contractName, method)
res, err := orbs.SendTransaction(tx)
Expand Down
4 changes: 2 additions & 2 deletions bootstrap/httpserver/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,8 @@ func (s *HttpServer) createRouter() *http.ServeMux {
s.registerHttpHandler(router, "/api/v1/get-transaction-receipt-proof", true, s.getTransactionReceiptProofHandler)
s.registerHttpHandler(router, "/api/v1/get-block", true, s.getBlockHandler)
s.registerHttpHandler(router, "/status", true, s.getStatus)
s.registerHttpHandler(router, "/metrics", true, s.dumpMetricsAsJSON)
s.registerHttpHandler(router, "/metrics.json", true, s.dumpMetricsAsJSON)
s.registerHttpHandler(router, "/metrics", true, s.getStatus)
s.registerHttpHandler(router, "/metrics.json", true, s.getStatus)
s.registerHttpHandler(router, "/metrics.prometheus", true, s.dumpMetricsAsPrometheus)
s.registerHttpHandler(router, "/robots.txt", false, s.robots)
s.registerHttpHandler(router, "/debug/logs/filter-on", false, s.filterOn)
Expand Down
Loading

0 comments on commit a3f7be2

Please sign in to comment.