cmd/scaleway, env/linux-arm/scaleway: notes, configs, and tools for S…

…caleway

This is what I used to start the ARM scaleway servers.

I forgot to mail it out. I'd like to submit this before I tweak it
further.

Updates golang/go#8647

Change-Id: Icf789de4e3acae8084fd75151ef5ef02d8073b73
Reviewed-on: https://go-review.googlesource.com/10052
Reviewed-by: Andrew Gerrand <[email protected]>

cmd/scaleway/scaleway.go

@@ -0,0 +1,153 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The scaleway command creates ARM servers on Scaleway.com.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var (
+	token = flag.String("token", "", "API token")
+	org   = flag.String("org", "1f34701d-668b-441b-bf08-0b13544e99de", "Organization ID (default is [email protected]'s account)")
+	image = flag.String("image", "b9fcca88-fa85-4606-a2b2-3c8a7ff94fbd", "Disk image ID; default is the snapshot we made last")
+	num   = flag.Int("n", 20, "Number of servers to create")
+)
+
+func main() {
+	flag.Parse()
+	if *token == "" {
+		file := filepath.Join(os.Getenv("HOME"), "keys/go-scaleway.token")
+		slurp, err := ioutil.ReadFile(file)
+		if err != nil {
+			log.Fatalf("No --token specified and error reading backup token file: %v", err)
+		}
+		*token = strings.TrimSpace(string(slurp))
+	}
+
+	cl := &Client{Token: *token}
+	serverList, err := cl.Servers()
+	if err != nil {
+		log.Fatal(err)
+	}
+	servers := map[string]*Server{}
+	for _, s := range serverList {
+		servers[s.Name] = s
+	}
+
+	for i := 1; i <= *num; i++ {
+		name := fmt.Sprintf("go-build-%d", i)
+		_, ok := servers[name]
+		if !ok {
+			body, err := json.Marshal(createServerRequest{
+				Org:   *org,
+				Name:  name,
+				Image: *image,
+			})
+			if err != nil {
+				log.Fatal(err)
+			}
+			log.Printf("Doing req %q for token %q", body, *token)
+			req, err := http.NewRequest("POST", "https://api.scaleway.com/servers", bytes.NewReader(body))
+			if err != nil {
+				log.Fatal(err)
+			}
+			req.Header.Set("Content-Type", "application/json")
+			req.Header.Set("X-Auth-Token", *token)
+			res, err := http.DefaultClient.Do(req)
+			if err != nil {
+				log.Fatal(err)
+			}
+			log.Printf("Create of %v: %v", i, res.Status)
+			res.Body.Close()
+		}
+	}
+
+	serverList, err = cl.Servers()
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, s := range serverList {
+		if s.State == "stopped" {
+			log.Printf("Powering on %s = %v", s.ID, cl.PowerOn(s.ID))
+		}
+	}
+}
+
+type createServerRequest struct {
+	Org   string `json:"organization"`
+	Name  string `json:"name"`
+	Image string `json:"image"`
+}
+
+type Client struct {
+	Token string
+}
+
+func (c *Client) PowerOn(serverID string) error {
+	return c.serverAction(serverID, "poweron")
+}
+
+func (c *Client) serverAction(serverID, action string) error {
+	req, _ := http.NewRequest("POST", "https://api.scaleway.com/servers/"+serverID+"/action", strings.NewReader(fmt.Sprintf(`{"action":"%s"}`, action)))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-Auth-Token", c.Token)
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode/100 != 2 {
+		return fmt.Errorf("Error doing %q on %s: %v", action, serverID, res.Status)
+	}
+	return nil
+}
+
+func (c *Client) Servers() ([]*Server, error) {
+	req, _ := http.NewRequest("GET", "https://api.scaleway.com/servers", nil)
+	req.Header.Set("X-Auth-Token", c.Token)
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("Failed to get Server list: %v", res.Status)
+	}
+	var jres struct {
+		Servers []*Server `json:"servers"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&jres)
+	return jres.Servers, err
+}
+
+type Server struct {
+	ID        string   `json:"id"`
+	Name      string   `json:"name"`
+	PublicIP  *IP      `json:"public_ip"`
+	PrivateIP string   `json:"private_ip"`
+	Tags      []string `json:"tags"`
+	State     string   `json:"state"`
+	Image     *Image   `json:"image"`
+}
+
+type Image struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+type IP struct {
+	ID      string `json:"id"`
+	Address string `json:"address"`
+}

env/linux-arm/scaleway/Dockerfile

@@ -0,0 +1,46 @@
+# Copyright 2015 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM armbuild/ubuntu:trusty
+
+MAINTAINER golang-dev <[email protected]>
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends curl
+
+RUN echo "607573c55dc89d135c3c9c84bba6ba6095a37a1e  go.tar.gz" > /tmp/go.tar.gz.sha1
+RUN cd /tmp && \
+     curl --silent -o go.tar.gz http://dave.cheney.net/paste/go1.4.2.linux-arm~multiarch-armv7-1.tar.gz && \
+     sha1sum -c go.tar.gz.sha1 && \
+     tar -C /usr/local -zxvf go.tar.gz && \
+     rm -rf go.tar.gz
+
+RUN apt-get install -y --no-install-recommends ca-certificates
+
+RUN mkdir /usr/local/gomaster
+ENV GO_MASTER_VERSION d4bb72b4
+RUN curl https://go.googlesource.com/go/+archive/$GO_MASTER_VERSION.tar.gz  | tar -C /usr/local/gomaster -zxv
+ENV GOROOT /usr/local/gomaster
+RUN echo "devel $GO_MASTER_VERSION" > $GOROOT/VERSION
+
+RUN apt-get install -y --no-install-recommends gcc
+RUN apt-get install -y --no-install-recommends libc6-dev
+
+ENV GOROOT_BOOTSTRAP /usr/local/go
+RUN cd $GOROOT/src && ./make.bash
+
+RUN apt-get install -y --no-install-recommends git-core
+
+ENV GOPATH /gopath
+RUN mkdir /gopath
+RUN $GOROOT/bin/go get golang.org/x/build/cmd/buildlet
+
+ADD run-buildlet.sh /usr/local/bin/run-buildlet.sh
+
+# Environment variables to be passed to "docker run"
+# for linux-arm and linux-arm-arm5:
+ENV BUILDKEY_ARM="" BUILDKEY_ARM5=""
+
+ENTRYPOINT ["/usr/local/bin/run-buildlet.sh"]

env/linux-arm/scaleway/README

@@ -0,0 +1,35 @@
+To create a fresh builder host:  (should only be needed once)
+
+Create a new Scaleway server with type "Docker 1.5" (second tab).
+
+Make a tmpfs:
+# echo "tmpfs /tmpfs tmpfs" >> /etc/fstab
+# mkdir /tmpfs
+
+Make a 2GB swap file:
+# dd if=/dev/zero of=/swap count=2097152 obs=1024
+# mkswap /swap
+# chmod 0600 /swap
+# echo "/swap none swap loop 0 0" >> /etc/fstab
+
+Reboot.
+
+Should see swaps in "cat /proc/swaps" and tmpfs in "cat /proc/mounts" now.
+
+* Copy the contents of this directory to the server.
+
+* Go into that directory and:
+  # docker build -t buildlet .
+
+Add to /etc/rc.local:
+
+  (mkdir -p /tmpfs/buildertmp && docker run -e BUILDKEY_ARM=xxx -e BUILDKEY_ARM5=xxx -v=/tmpfs/buildertmp:/tmp --restart=always -d --name=buildlet buildlet) &
+  sleep 5
+
+.. before the exit 0
+
+Power it down (with ARCHIVE),
+
+Snapshot the disk.
+
+Start a bunch of them. (see golang.org/x/build/cmd/scaleway)

env/linux-arm/scaleway/run-buildlet.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Meant to be run under Docker.
+
+if [ "$BUILDKEY_ARM" == "" ]; then
+env
+    echo "ERROR: BUILDKEY_ARM not set. (using docker run?)" >&2
+    exit 1
+fi
+if [ "$BUILDKEY_ARM5" == "" ]; then
+env
+    echo "ERROR: BUILDKEY_ARM5 not set. (using docker run?)" >&2
+    exit 1
+fi
+
+
+set -e
+echo $BUILDKEY_ARM > /root/.gobuildkey-linux-arm
+echo $BUILDKEY_ARM5 > /root/.gobuildkey-linux-arm-arm5
+exec /gopath/bin/buildlet -reverse=linux-arm,linux-arm-arm5 -coordinator farmer.golang.org:443