ansible: metrics server

ansible/inventory.yml

@@ -14,6 +14,7 @@ hosts:
         ubuntu1604-x64-1: {ip: 138.197.224.240, alias: www}
         ubuntu1804-x64-1: {ip: 178.128.202.158, alias: gzemnid}
         ubuntu1804-x64-2: {ip: 45.55.45.227, alias: unofficial-builds}
+        ubuntu1804-x64-3: {ip: 157.245.7.159, alias: metrics}
 
     - joyent:
         smartos15-x64-1: {ip: 72.2.118.125, alias: backup}

ansible/playbooks/create-metrics.yml

@@ -0,0 +1,12 @@
+---
+
+#
+# sets up the host that runs metrics collection and storage
+#
+
+- hosts: infra-digitalocean-ubuntu1804-x64-3
+  roles:
+    - bootstrap
+    - package-upgrade
+    - baselayout
+    - metrics

ansible/roles/metrics/README.md

@@ -0,0 +1,3 @@
+* Copy secrets/build/infra/gcp-metrics-service-acct-key.json to ~nodejs/
+* As user `nodejs` run `gcloud auth activate-service-account --key-file gcp-metrics-service-acct-key.json`
+* Set up new SSH key with access to [email protected] and [email protected] under user `nodejs`

ansible/roles/metrics/files/process-cloudflare/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "process-cloudflare",
+  "version": "1.0.0",
+  "description": "",
+  "main": "process-cloudflare.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Rod <[email protected]> (http://r.va.gg/)",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "split2": "~3.1.1",
+    "strftime": "~0.10.0"
+  }
+}

ansible/roles/metrics/files/process-cloudflare/process-cloudflare.js

@@ -0,0 +1,155 @@
+#!/usr/bin/env node
+
+const { pipeline, Transform } = require('stream')
+const split2 = require('split2')
+const strftime = require('strftime').timezone(0)
+
+const jsonStream = new Transform({
+  readableObjectMode: true,
+  transform (chunk, encoding, callback) {
+    try {
+      this.push(JSON.parse(chunk.toString()))
+      callback()
+    } catch (e) {
+      callback(e)
+    }
+  }
+})
+
+const extensionRe = /\.(tar\.gz|tar\.xz|pkg|msi|exe|zip|7z)$/
+const uriRe = /(\/+(dist|download\/+release)\/+(node-latest\.tar\.gz|([^/]+)\/+((win-x64|win-x86|x64)?\/+?node\.exe|(x64\/)?node-+(v[0-9.]+)[.-]([^? ]+))))/
+const versionRe = /^v[0-9.]+$/
+
+function determineOS (path, file, fileType) {
+  if (/node\.exe$/.test(file)) {
+    return 'win'
+  } else if (/\/node-latest\.tar\.gz$/.test(path)) {
+    return 'src'
+  } else if (fileType == null) {
+    return ''
+  } else if (/msi$/.test(fileType) || /^win-/.test(fileType)) {
+    return 'win'
+  } else if (/^tar\..z$/.test(fileType)) {
+    return 'src'
+  } else if (/^headers\.tar\..z$/.test(fileType)) {
+    return 'headers'
+  } else if (/^linux-/.test(fileType)) {
+    return 'linux'
+  } else if (fileType === 'pkg' || /^darwin-/.test(fileType)) {
+    return 'osx'
+  } else if (/^sunos-/.test(fileType)) {
+    return 'sunos'
+  } else if (/^aix-/.test(fileType)) {
+    return 'aix'
+  } else {
+    return ''
+  }
+}
+
+function determineArch (fileType, winArch, os) {
+  if (fileType != null) {
+    if (fileType.indexOf('x64') >= 0 || fileType === 'pkg') {
+      // .pkg for Node.js <= 0.12 were universal so may be used for either x64 or x86
+      return 'x64'
+    } else if (fileType.indexOf('x86') >= 0) {
+      return 'x86'
+    } else if (fileType.indexOf('armv6') >= 0) {
+      return 'armv6l'
+    } else if (fileType.indexOf('armv7') >= 0) { // 4.1.0 had a misnamed binary, no 'l' in 'armv7l'
+      return 'armv7l'
+    } else if (fileType.indexOf('arm64') >= 0) {
+      return 'arm64'
+    } else if (fileType.indexOf('ppc64le') >= 0) {
+      return 'ppc64le'
+    } else if (fileType.indexOf('ppc64') >= 0) {
+      return 'ppc64'
+    } else if (fileType.indexOf('s390x') >= 0) {
+      return 's390x'
+    }
+  }
+
+  if (os === 'win') {
+    // we get here for older .msi files and node.exe files
+    if (winArch && winArch.indexOf('x64') >= 0) {
+      // could be 'x64' or 'win-x64'
+      return 'x64'
+    } else {
+      // could be 'win-x86' or ''
+      return 'x86'
+    }
+  }
+
+  return ''
+}
+
+const logTransformStream = new Transform({
+  writableObjectMode: true,
+  transform (chunk, encoding, callback) {
+    if (chunk.ClientRequestMethod !== 'GET' ||
+        chunk.EdgeResponseStatus < 200 ||
+        chunk.EdgeResponseStatus >= 300) {
+      return callback()
+    }
+
+    if (chunk.EdgeResponseBytes < 1024) { // unreasonably small for something we want to measure
+      return callback()
+    }
+
+    if (!extensionRe.test(chunk.ClientRequestPath)) { // not a file we care about
+      return callback()
+    }
+
+    const requestPath = chunk.ClientRequestPath.replace(/\/\/+/g, '/')
+    const uriMatch = requestPath.match(uriRe)
+    if (!uriMatch) { // what is this then?
+      return callback()
+    }
+
+    const path = uriMatch[1]
+    const pathVersion = uriMatch[4]
+    const file = uriMatch[5]
+    const winArch = uriMatch[6]
+    const fileVersion = uriMatch[8]
+    const fileType = uriMatch[9]
+
+    let version = ''
+    // version can come from the filename or the path, filename is best
+    // but it may not be there (e.g. node.exe) so fall back to path version
+    if (versionRe.test(fileVersion)) {
+      version = fileVersion
+    } else if (versionRe.test(pathVersion)) {
+      version = pathVersion
+    }
+
+    const os = determineOS(path, file, fileType)
+    const arch = determineArch(fileType, winArch, os)
+
+    const line = []
+    line.push(strftime('%Y-%m-%d', new Date(chunk.EdgeStartTimestamp / 1000 / 1000))) // date
+    line.push(chunk.ClientCountry.toUpperCase()) // country
+    line.push('') // state/province, derived from chunk.EdgeColoCode probably
+    line.push(chunk.ClientRequestPath) // URI
+    line.push(version) // version
+    line.push(os) // os
+    line.push(arch) // arch
+    line.push(chunk.EdgeResponseBytes)
+
+    this.push(`${line.join(',')}\n`)
+
+    callback()
+  }
+})
+
+pipeline(
+  process.stdin,
+  split2(),
+  jsonStream,
+  logTransformStream,
+  process.stdout,
+  (err) => {
+    if (err) {
+      console.error('ERROR', err)
+      process.exit(1)
+    }
+  }
+)

ansible/roles/metrics/files/process-cloudflare/summaries.js

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+
+// data example:
+//
+// day,country,region,path,version,os,arch,bytes
+// 2019-10-31,US,,/dist/v13.0.1/node-v13.0.1-linux-x64.tar.xz,v13.0.1,linux,x64,20102340
+//
+
+const { pipeline, Transform } = require('stream')
+const split2 = require('split2')
+
+const csvStream = new Transform({
+  readableObjectMode: true,
+  transform (chunk, encoding, callback) {
+    try {
+      const schunk = chunk.toString()
+      if (!schunk.startsWith('day,country')) { // ignore header
+        this.push(schunk.split(','))
+      }
+      callback()
+    } catch (e) {
+      callback(e)
+    }
+  }
+})
+
+const counts = { bytes: 0, total: 0 }
+function increment (type, key) {
+  if (!key) {
+    key = 'unknown'
+  }
+
+  if (!counts[type]) {
+    counts[type] = {}
+  }
+
+  if (counts[type][key] === undefined) {
+    counts[type][key] = 1
+  } else {
+    counts[type][key]++
+  }
+}
+
+function prepare () {
+  function sort (type) {
+    const ca = Object.entries(counts[type])
+    ca.sort((e1, e2) => e2[1] > e1[1] ? 1 : e2[1] < e1[1] ? -1 : 0)
+    counts[type] = ca.reduce((p, c) => {
+      p[c[0]] = c[1]
+      return p
+    }, {})
+  }
+
+  'country version os arch'.split(' ').forEach(sort)
+}
+
+const summaryStream = new Transform({
+  writableObjectMode: true,
+  transform (chunk, encoding, callback) {
+    const [date, country, region, path, version, os, arch, bytes] = chunk
+    increment('country', country)
+    increment('version', version)
+    increment('os', os)
+    increment('arch', arch)
+    counts.bytes += parseInt(bytes, 10)
+    counts.total++
+    callback()
+  }
+})
+
+pipeline(
+  process.stdin,
+  split2(),
+  csvStream,
+  summaryStream,
+  (err) => {
+    prepare()
+    console.log(JSON.stringify(counts, null, 2))
+    if (err) {
+      console.error('ERROR', err)
+      process.exit(1)
+    }
+  }
+)

ansible/roles/metrics/files/sync-backup-www-periodic.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Periodic log sync from backup-www.nodejs.org
+
+[Service]
+Type=oneshot
+User=nodejs
+ExecStart=/usr/bin/rsync -e "ssh" -aqz [email protected]:/var/log/nginx/ /home/nodejs/logs/backup-www.nodejs.org/
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/metrics/files/sync-backup-www-periodic.timer

@@ -0,0 +1,14 @@
+[Unit]
+Description=Schedule periodic backup-www sync
+RefuseManualStart=no
+RefuseManualStop=no
+
+[Timer]
+Persistent=true
+OnBootSec=300
+RandomizedDelaySec=300
+OnUnitActiveSec=3600
+Unit=sync-backup-www-periodic.service
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/metrics/files/sync-cloudflare-periodic.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Periodic log sync from cloudflare
+
+[Service]
+Type=oneshot
+User=nodejs
+ExecStart=/usr/bin/gsutil rsync -d -r gs://cloudflare-logs-nodejs/ /home/nodejs/logs/cloudflare/
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/metrics/files/sync-cloudflare-periodic.timer

@@ -0,0 +1,14 @@
+[Unit]
+Description=Schedule periodic cloudflare sync
+RefuseManualStart=no
+RefuseManualStop=no
+
+[Timer]
+Persistent=true
+OnBootSec=300
+RandomizedDelaySec=300
+OnUnitActiveSec=3600
+Unit=sync-cloudflare-periodic.service
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/metrics/files/sync-www-periodic.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Periodic log sync from nodejs.org
+
+[Service]
+Type=oneshot
+User=nodejs
+ExecStart=/usr/bin/rsync -e "ssh" -aqz [email protected]:/var/log/nginx/ /home/nodejs/logs/nodejs.org/
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/metrics/files/sync-www-periodic.timer

@@ -0,0 +1,14 @@
+[Unit]
+Description=Schedule periodic www sync
+RefuseManualStart=no
+RefuseManualStop=no
+
+[Timer]
+Persistent=true
+OnBootSec=300
+RandomizedDelaySec=300
+OnUnitActiveSec=3600
+Unit=sync-www-periodic.service
+
+[Install]
+WantedBy=multi-user.target