x/website: add code and static files for website

All of the code and static assets that the website uses to run have been
copied to this repo. There was also a few lines of code added telling
the website where the doc directory, favicon.ico and robots.txt are.

go repo change-id: Ife6443c32673b38000b90dd2efb2985db37ab773
x/tools repo change-id: Ia979a8b06d1b4db47d25ffdfdf925ba8a0ac67de

Real new code additions:
- main.go
    * lines 89-95 added getFullPath method
    * lines 217-222 mapped paths to doc/, favicon.ico, robots.txt in vfs
- appinit.go
    * lines 147-153 added getFullPath method
    * lines 80-84 mapped paths to doc/, favicon.ico in vfs

Several files were copied from x/tools and go so paths (and corresponding
import paths) were changed as follows:

"x/tools/cmd/godoc/" --> "x/website/cmd/golangorg/"
"x/tools/godoc/static/" --> "x/website/content/static/"
"x/tools/godoc/" (without godoc/static/) --> "x/website/cmd/golangorg/godoc/"
"x/tools/internal/memcache" --> "x/website/internal/memcache"
"go/doc/" --> "x/website/content/doc/"
"go/favicon.ico" --> "x/website/favicon.ico"
"go/robots.txt" --> "x/website/robots.txt"

Updates golang/go#29206

Change-Id: I53985fc027f73e60c6946038f85133acf1ecb08c
Reviewed-on: https://go-review.googlesource.com/c/156321
Reviewed-by: Brad Fitzpatrick <[email protected]>

cmd/golangorg/Dockerfile.prod

@@ -0,0 +1,67 @@
+# Builder
+#########
+
+FROM golang:1.11 AS build
+
+RUN apt-get update && apt-get install -y \
+      zip # required for generate-index.bash
+
+# Check out the desired version of Go, both to build the godoc binary and serve
+# as the goroot for content serving.
+ARG GO_REF
+RUN test -n "$GO_REF" # GO_REF is required.
+RUN git clone --single-branch --depth=1 -b $GO_REF https://go.googlesource.com/go /goroot
+RUN cd /goroot/src && ./make.bash
+
+ENV GOROOT /goroot
+ENV PATH=/goroot/bin:$PATH
+
+RUN go version
+
+RUN go get -v -d \
+      golang.org/x/net/context \
+      google.golang.org/appengine \
+      cloud.google.com/go/datastore \
+      golang.org/x/build \
+      github.com/gomodule/redigo/redis
+
+COPY . /go/src/golang.org/x/tools
+
+WORKDIR /go/src/golang.org/x/website/cmd/golangorg
+RUN GODOC_DOCSET=/goroot ./generate-index.bash
+
+RUN go build -o /godoc -tags=golangorg golang.org/x/website/cmd/golangorg
+
+# Clean up goroot for the final image.
+RUN cd /goroot && git clean -xdf
+
+# Add build metadata.
+RUN cd /goroot && echo "go repo HEAD: $(git rev-parse HEAD)" >> /goroot/buildinfo
+RUN echo "requested go ref: ${GO_REF}" >> /goroot/buildinfo
+ARG TOOLS_HEAD
+RUN echo "x/tools HEAD: ${TOOLS_HEAD}" >> /goroot/buildinfo
+ARG TOOLS_CLEAN
+RUN echo "x/tools clean: ${TOOLS_CLEAN}" >> /goroot/buildinfo
+ARG DOCKER_TAG
+RUN echo "image: ${DOCKER_TAG}" >> /goroot/buildinfo
+ARG BUILD_ENV
+RUN echo "build env: ${BUILD_ENV}" >> /goroot/buildinfo
+
+RUN rm -rf /goroot/.git
+
+# Final image
+#############
+
+FROM gcr.io/distroless/base
+
+WORKDIR /app
+COPY --from=build /godoc /app/
+COPY --from=build /go/src/golang.org/x/website/cmd/golangorg/hg-git-mapping.bin /app/
+
+COPY --from=build /goroot /goroot
+ENV GOROOT /goroot
+
+COPY --from=build /go/src/golang.org/x/website/cmd/golangorg/index.split.* /app/
+ENV GODOC_INDEX_GLOB index.split.*
+
+CMD ["/app/godoc"]

cmd/golangorg/Makefile

@@ -0,0 +1,80 @@
+# Copyright 2018 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+.PHONY: usage
+
+GO_REF ?= release-branch.go1.11
+TOOLS_HEAD := $(shell git rev-parse HEAD)
+TOOLS_CLEAN := $(shell (git status --porcelain | grep -q .) && echo dirty || echo clean)
+ifeq ($(TOOLS_CLEAN),clean)
+	DOCKER_VERSION ?= $(TOOLS_HEAD)
+else
+	DOCKER_VERSION ?= $(TOOLS_HEAD)-dirty
+endif
+GCP_PROJECT := golang-org
+DOCKER_TAG := gcr.io/$(GCP_PROJECT)/godoc:$(DOCKER_VERSION)
+
+usage:
+	@echo "See Makefile and README.golangorg-app"
+	@exit 1
+
+cloud-build: Dockerfile.prod
+	gcloud builds submit \
+		--project=$(GCP_PROJECT) \
+		--config=cloudbuild.yaml \
+		--substitutions=_GO_REF=$(GO_REF),_TOOLS_HEAD=$(TOOLS_HEAD),_TOOLS_CLEAN=$(TOOLS_CLEAN),_DOCKER_TAG=$(DOCKER_TAG) \
+		../.. # source code
+
+docker-build: Dockerfile.prod
+	# NOTE(cbro): move up in directory to include entire tools repo.
+	# NOTE(cbro): any changes made to this command must also be made in cloudbuild.yaml.
+	cd ../..; docker build \
+		-f=cmd/godoc/Dockerfile.prod \
+		--build-arg=GO_REF=$(GO_REF) \
+		--build-arg=TOOLS_HEAD=$(TOOLS_HEAD) \
+		--build-arg=TOOLS_CLEAN=$(TOOLS_CLEAN) \
+		--build-arg=DOCKER_TAG=$(DOCKER_TAG) \
+		--build-arg=BUILD_ENV=local \
+		--tag=$(DOCKER_TAG) \
+		.
+
+docker-push: docker-build
+	docker push $(DOCKER_TAG)
+
+deploy:
+	gcloud -q app deploy app.prod.yaml \
+		--project=$(GCP_PROJECT) \
+		--no-promote \
+		--image-url=$(DOCKER_TAG)
+
+get-latest-url:
+	@gcloud app versions list \
+		--service=default \
+		--project=$(GCP_PROJECT) \
+		--sort-by=~version.createTime \
+		--format='value(version.versionUrl)' \
+		--limit 1 | cut -f1 # NOTE(cbro): gcloud prints out createTime as the second field.
+
+get-latest-id:
+	@gcloud app versions list \
+		--service=default \
+		--project=$(GCP_PROJECT) \
+		--sort-by=~version.createTime \
+		--format='value(version.id)' \
+		--limit 1 | cut -f1 # NOTE(cbro): gcloud prints out createTime as the second field.
+
+regtest:
+	go test -v \
+		-regtest.host=$(shell make get-latest-url) \
+		-run=Live
+
+publish: regtest
+	gcloud -q app services set-traffic default \
+		--splits=$(shell make get-latest-id)=1 \
+		--project=$(GCP_PROJECT)
+
+	@echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+	@echo Stop and/or delete old versions:
+	@echo "https://console.cloud.google.com/appengine/versions?project=$(GCP_PROJECT)&serviceId=default&versionssize=50"
+	@echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

cmd/golangorg/README.golangorgserver-app

@@ -0,0 +1,94 @@
+godoc on Google App Engine
+==========================
+
+Prerequisites
+-------------
+
+* Google Cloud SDK
+  https://cloud.google.com/sdk/
+
+* Redis
+
+* Go sources under $GOROOT
+
+* Godoc sources inside $GOPATH
+  (go get -d golang.org/x/website/cmd/golangorg)
+
+
+Running locally, in production mode
+-----------------------------------
+
+Build the app:
+
+  go build -tags golangorg
+
+Run the app:
+
+  ./golangorg
+
+godoc should come up at http://localhost:8080
+
+Use the PORT environment variable to change the port:
+
+    PORT=8081 ./golangorg
+
+Running locally, in production mode, using Docker
+-------------------------------------------------
+
+Build the app's Docker container:
+
+  make docker-build
+
+Make sure redis is running on port 6379:
+
+  $ echo PING | nc localhost 6379
+  +PONG
+  ^C
+
+Run the datastore emulator:
+
+  gcloud beta emulators datastore start --project golang-org
+
+In another terminal window, run the container:
+
+  $(gcloud beta emulators datastore env-init)
+
+  docker run --rm \
+    --net host \
+    --env GODOC_REDIS_ADDR=localhost:6379 \
+    --env DATASTORE_EMULATOR_HOST=$DATASTORE_EMULATOR_HOST \
+    --env DATASTORE_PROJECT_ID=$DATASTORE_PROJECT_ID \
+    gcr.io/golang-org/godoc
+
+godoc should come up at http://localhost:8080
+
+
+Deploying to golang.org
+-----------------------
+
+Make sure you're signed in to gcloud:
+
+  gcloud auth login
+
+Build the image, push it to gcr.io, and deploy to Flex:
+
+  make cloud-build deploy
+
+Point the load balancer to the newly deployed version:
+(This also runs regression tests)
+
+  make publish
+
+Stop and/or delete down any very old versions. (Stopped versions can be re-started.)
+Keep at least one older verson to roll back to, just in case.
+You can also migrate traffic to the new version via this UI.
+
+  https://console.cloud.google.com/appengine/versions?project=golang-org&serviceId=default&versionssize=50
+
+
+Troubleshooting
+---------------
+
+Ensure the Cloud SDK is on your PATH and you have the app-engine-go component
+installed (gcloud components install app-engine-go) and your components are
+up-to-date (gcloud components update)

cmd/golangorg/app.dev.yaml

@@ -0,0 +1,13 @@
+runtime: go
+api_version: go1
+instance_class: F4_1G
+
+handlers:
+- url: /s
+  script: _go_app
+  login: admin
+- url: /dl/init
+  script: _go_app
+  login: admin
+- url: /.*
+  script: _go_app

cmd/golangorg/app.prod.yaml

@@ -0,0 +1,16 @@
+runtime: custom
+env: flex
+
+env_variables:
+  GODOC_PROD: true
+  GODOC_ENFORCE_HOSTS: true
+  GODOC_REDIS_ADDR: 10.0.0.4:6379 # instance "gophercache"
+  GODOC_ANALYTICS: UA-11222381-2
+  DATASTORE_PROJECT_ID: golang-org
+
+network:
+  name: golang
+
+resources:
+  cpu: 4
+  memory_gb: 7.50