HBASE-18548 Move sources of website gen and check jobs into source co…

…ntrol

dev-support/jenkins-scripts/check-website-links.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# This script is designed to run as a Jenkins job, such as at
+# https://builds.apache.org/view/All/job/HBase%20Website%20Link%20Checker/
+#
+# It generates artifacts which the Jenkins job then can mail out and/or archive.
+#
+# We download a specific version of linklint because the original has bugs and
+# is not well maintained.
+#
+# See http://www.linklint.org/doc/inputs.html for linklint options
+
+# Clean up the workspace
+rm -rf *.zip > /dev/null
+rm -rf linklint > /dev/null
+rm -Rf link_report
+
+# This version of linklint fixes some bugs in the now-unmaintained 2.3.5 version
+wget http://ingo-karkat.de/downloads/tools/linklint/linklint-2.3.5_ingo_020.zip
+unzip linklint-2.3.5_ingo_020.zip
+chmod +x linklint/linklint.pl
+
+# Run the checker
+echo "Checking http://hbase.apache.org and saving report to link_report/"
+echo "Excluding /testapidocs/ because some tests use private classes not published in /apidocs/."
+# Check internal structure
+linklint/linklint.pl -http \
+                     -host hbase.apache.org \
+                     /@ \
+                     -skip /testapidocs/@ \
+                     -skip /testdevapidocs/@ \
+                     -net \
+                     -redirect \
+                     -no_query_string \
+                     -htmlonly \
+                     -timeout 30 \
+                     -delay 1 \
+                     -limit 100000 \
+                     -doc link_report
+
+# Detect whether we had errors and act accordingly
+if ! grep -q 'ERROR' link_report/index.html; then
+  echo "Errors found. Sending email."
+  exit 1
+else
+  echo "No errors found. Warnings might be present."
+fi

dev-support/jenkins-scripts/generate-hbase-website.sh

@@ -0,0 +1,152 @@
+#!/bin/bash
+
+# This script is meant to run as part of a Jenkins job such as
+# https://builds.apache.org/job/hbase_generate_website/
+#
+# It needs to be built on a Jenkins server with the label git-websites
+#
+# It expects to have the hbase repo cloned to the directory hbase
+#
+# If there is a build error, the Jenkins job is configured to send an email
+
+LOCAL_REPO=${WORKSPACE}/.m2/repo
+# Nuke the local maven repo each time, to start with a known environment
+rm -Rf "${LOCAL_REPO}"
+mkdir -p "${LOCAL_REPO}"
+
+# Clean any leftover files in case we are reusing the workspace
+rm -Rf -- *.patch *.patch.zip hbase/target target *.txt hbase-site
+
+# Set up the environment
+export JAVA_HOME=$JDK_1_8_LATEST__HOME
+export PATH=$JAVA_HOME/bin:$MAVEN_3_3_3_HOME/bin:$PATH
+export MAVEN_OPTS="-XX:MaxPermSize=256m -Dmaven.repo.local=${LOCAL_REPO}"
+
+# Verify the Maven version
+mvn -version
+
+# Save and print the SHA we are building
+CURRENT_HBASE_COMMIT="$(git log --pretty=format:%H -n1)"
+echo "Current HBase commit: $CURRENT_HBASE_COMMIT"
+
+# Clone the hbase-site repo manually so it doesn't trigger spurious
+# commits in Jenkins.
+git clone --depth 1 --branch asf-site https://git-wip-us.apache.org/repos/asf/hbase-site.git
+
+# Figure out the last commit we built the site from, and bail if the build
+# still represents the SHA of HBase master
+cd "${WORKSPACE}/hbase-site" || exit -1
+git log --pretty=%s | grep ${CURRENT_HBASE_COMMIT}
+PUSHED=$?
+echo "PUSHED is $PUSHED"
+
+if [ $PUSHED -eq 0 ]; then
+  echo "$CURRENT_HBASE_COMMIT is already mentioned in the hbase-site commit log. Not building."
+  exit 0
+else
+  echo "$CURRENT_HBASE_COMMIT is not yet mentioned in the hbase-site commit log. Assuming we don't have it yet. $PUSHED"
+fi
+
+# Go to the hbase directory so we can build the site
+cd "${WORKSPACE}/hbase" || exit -1
+
+# This will only be set for builds that are triggered by SCM change, not manual builds
+if [ "$CHANGE_ID" ]; then
+  echo -n " ($CHANGE_ID - $CHANGE_TITLE)"
+fi
+
+# Build and install HBase, then build the site
+echo "Building HBase"
+mvn \
+  -DskipTests \
+  -Dmaven.javadoc.skip=true \
+  --batch-mode \
+  -Dcheckstyle.skip=true \
+  -Dfindbugs.skip=true \
+  --log-file="${WORKSPACE}/hbase-build-log-${CURRENT_HBASE_COMMIT}.txt" \
+  clean install \
+&& mvn clean site \
+  --batch-mode \
+  -DskipTests \
+  --log-file="${WORKSPACE}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt"
+
+status=$?
+if [ $status -ne 0 ]; then
+  echo "Failure: mvn clean site"
+  exit $status
+fi
+
+# Stage the site
+echo "Staging HBase site"
+mvn \
+  --batch-mode \
+  --log-file="${WORKSPACE}/hbase-stage-log-${CURRENT_HBASE_COMMIT}.txt" \
+  site:stage
+status=$?
+if [ $status -ne 0 ] || [ ! -d target/staging ]; then
+  echo "Failure: mvn site:stage"
+  exit $status
+fi
+
+# Get ready to update the hbase-site repo with the new artifacts
+cd "${WORKSPACE}/hbase-site" || exit -1
+
+#Remove previously-generated files
+FILES_TO_REMOVE=("hbase-*"
+                 "apidocs"
+                 "devapidocs"
+                 "testapidocs"
+                 "testdevapidocs"
+                 "xref"
+                 "xref-test"
+                 "*book*"
+                 "*.html"
+                 "*.pdf*"
+                 "css"
+                 "js"
+                 "images")
+
+for FILE in "${FILES_TO_REMOVE[@]}"; do
+  echo "Removing ${WORKSPACE}/hbase-site/$FILE"
+  rm -Rf "${FILE}"
+done
+
+# Copy in the newly-built artifacts
+cp -au "${WORKSPACE}"/hbase/target/staging/* .
+
+# If the index.html is missing, bail because this is serious
+if [ ! -f index.html ]; then
+  echo "The index.html is missing. Aborting."
+  exit 1
+else
+  # Add all the changes
+  echo "Adding all the files we know about"
+  git add .
+  # Create the commit message and commit the changes
+  WEBSITE_COMMIT_MSG="Published site at $CURRENT_HBASE_COMMIT."
+  echo "WEBSITE_COMMIT_MSG: $WEBSITE_COMMIT_MSG"
+  git commit -m "${WEBSITE_COMMIT_MSG}" -a
+  # Dump a little report
+  echo "This commit changed these files (excluding Modified files):"
+  git diff --name-status --diff-filter=ADCRTXUB origin/asf-site
+  # Create a patch, which Jenkins can save as an artifact and can be examined for debugging
+  git format-patch --stdout origin/asf-site > "${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch"
+  echo "Change set saved to patch ${WORKSPACE}/${CURRENT_HBASE_COMMIT}.patch"
+  # Push the real commit
+  git push origin asf-site || (echo "Failed to push to asf-site. Website not updated." && exit -1)
+  # Create an empty commit to work around INFRA-10751
+  git commit --allow-empty -m "INFRA-10751 Empty commit"
+  # Push the empty commit
+  git push origin asf-site || (echo "Failed to push the empty commit to asf-site. Website may not update. Manually push an empty commit to fix this. (See INFRA-10751)" && exit -1)
+  echo "Pushed the changes to branch asf-site. Refresh http://hbase.apache.org/ to see the changes within a few minutes."
+  git fetch origin
+  git reset --hard origin/asf-site
+
+  # Zip up the patch so Jenkins can save it
+  cd "${WORKSPACE}" || exit -1
+  zip website.patch.zip "${CURRENT_HBASE_COMMIT}.patch"
+fi
+
+#echo "Dumping current environment:"
+#env
+

src/main/asciidoc/_chapters/appendix_contributing_to_documentation.adoc

@@ -138,23 +138,23 @@ When you are satisfied with your changes, follow the procedure in
 [[website_publish]]
 === Publishing the HBase Website and Documentation
 
-HBase uses the ASF's `gitpubsub` mechanism.
-. After generating the website and documentation
-artifacts using `mvn clean site site:stage`, check out the `asf-site` repository.
-
-. Remove previously-generated content using the following command:
-+
-----
-rm -rf rm -rf *apidocs* *book* *.html *.pdf* css js
-----
-+
-WARNING: Do not remove the `0.94/` directory. To regenerate them, you must check out
-the 0.94 branch and run `mvn clean site site:stage` from there, and then copy the
-artifacts to the 0.94/ directory of the `asf-site` branch.
-
-. Copy the contents of `target/staging` to the branch.
-
-. Add and commit your changes, and submit a patch for review.
+HBase uses the ASF's `gitpubsub` mechanism. A Jenkins job runs the
+`dev-support/jenkins-scripts/generate-hbase-website.sh` script, which runs the
+`mvn clean site site:stage` against the `master` branch of the `hbase`
+repository and commits the built artifacts to the `asf-site` branch of the
+`hbase-site` repository. When the commit is pushed, the website is redeployed
+automatically. If the script encounters an error, an email is sent to the
+developer mailing list. You can run the script manually or examine it to see the
+steps involved.
+
+[[website_check_links]]
+=== Checking the HBase Website for Broken Links
+
+A Jenkins job runs periodically to check HBase website for broken links, using
+the `dev-support/jenkins-scripts/check-website-links.sh` script. This script
+uses a tool called `linklint` to check for bad links and create a report. If
+broken links are found, an email is sent to the developer mailing list. You can
+run the script manually or examine it to see the steps involved.
 
 === HBase Reference Guide Style Guide and Cheat Sheet