From 0005c96c89acb6a6f7a18e3c1e2cc5a5927be746 Mon Sep 17 00:00:00 2001 From: Vitalii Parfonov Date: Thu, 25 Jun 2020 16:08:29 +0300 Subject: [PATCH 1/5] Initial contribution for workspace sync feature Signed-off-by: Vitalii Parfonov --- .gitignore | 7 + LICENSE | 277 ++++++++++++++++++++++++ README.md | 31 +++ build.sh | 54 +++++ common.sh | 42 ++++ dockerfiles/agent/Dockerfile | 65 ++++++ dockerfiles/agent/cron/backup-cron-job | 9 + dockerfiles/agent/scripts/backup.sh | 40 ++++ dockerfiles/agent/scripts/entrypoint.sh | 20 ++ dockerfiles/agent/scripts/restore.sh | 43 ++++ dockerfiles/storage/Dockerfile | 35 +++ dockerfiles/storage/entrypoint.sh | 25 +++ go.mod | 5 + go.sum | 2 + pkg/progress/model.go | 6 + pkg/progress/progress_watcher.go | 124 +++++++++++ watcher/watcher.go | 72 ++++++ 17 files changed, 857 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100755 build.sh create mode 100755 common.sh create mode 100644 dockerfiles/agent/Dockerfile create mode 100644 dockerfiles/agent/cron/backup-cron-job create mode 100755 dockerfiles/agent/scripts/backup.sh create mode 100755 dockerfiles/agent/scripts/entrypoint.sh create mode 100755 dockerfiles/agent/scripts/restore.sh create mode 100644 dockerfiles/storage/Dockerfile create mode 100755 dockerfiles/storage/entrypoint.sh create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/progress/model.go create mode 100644 pkg/progress/progress_watcher.go create mode 100644 watcher/watcher.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68afa7e --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# IDE's # +.theia/ +.idea/ +.vscode/ + +# Compiled # +dockerfiles/agent/scripts/watcher diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e48e096 --- /dev/null +++ b/LICENSE @@ -0,0 +1,277 @@ +Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + +"Contributor" means any person or entity that Distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. + +"Program" means the Contributions Distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. + +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. + +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. + +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. + +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + +3. REQUIREMENTS + +3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + +3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), +version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0cee204 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +## About Eclipse Che Workspace Data Sync + +This tooling provide ability to increase I/O performance for a developer workspaces. + +============== +## Workspace lifecycle: + +#### Workspaces Startup phase: +- Pods (workspace and data sync) are started in parallel +#### Startup data sync phase: +- Data flow goes from the persistent volume (rsync server) to the ephemeral volume (rsync client) +#### Normal workspace usage phase: +- Containers in the Workspace Pod have full R/W access to the ephemeral volume with restored project sources +- Container (rsync client) in the Workspace Pod have full R/W access to the persistent volume +- Data flow goes from rsync client to rsync server periodically by cron job +#### Workspace Shutdown phase +- Containers in the Workspace Pod stopped and data flushed to the persistent volume + +## Requirements + +- Golang `1.12` +- Docker + +## Build + +```sh +./build.sh -help +``` + +### License +Che is open sourced under the Eclipse Public License 2.0. diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..5daa3ae --- /dev/null +++ b/build.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright (c) 2012-2019 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +#include common scripts +. ./common.sh + +usage="$(basename "$0") -- script for compile Go source and build docker images + +usage: + -t,--tag tag name for docker images (default: latest) + -o,--organization organization name for docker images (default: che-incubator) + -h,--help show this help text" + +while [[ $# -gt 0 ]] +do + key="${1}" + case $key in + -t|--tag) + TAG="${2}" + shift + shift + ;; + -o|--organization) + ORGANIZATION="${2}" + shift + shift + ;; + -h|--help) + echo "$usage" + exit 0 + ;; + *) printf "illegal option: -%s\n" "$key" + echo "$usage" + exit 1 + ;; + esac +done + +#default +if [[ -z "$TAG" ]] +then TAG="latest" +fi +if [[ -z "$ORGANIZATION" ]] +then ORGANIZATION="che-incubator" +fi + +compile; +dockerBuild "$TAG" "${ORGANIZATION}"; diff --git a/common.sh b/common.sh new file mode 100755 index 0000000..b38fc18 --- /dev/null +++ b/common.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Copyright (c) 2012-2019 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# + +IMAGE_AGENT=workspace-data-sync-agent +IMAGE_STORAGE=workspace-data-sync-storage +TAG=$1 +ORGANIZATION=$2 + +function compile() { + echo "Compile file sync progress watcher binary from source code" + $(GOOS=linux go build -o ./dockerfiles/agent/scripts/watcher ./watcher/watcher.go) + if [ $? != 0 ]; then + echo "Failed to compile code" + exit 0 + fi + echo "Compilation successfully completed" +} + +function dockerBuild() { + printf "Build docker image %s/%s:%s \n" "$ORGANIZATION" "$IMAGE_AGENT" "$TAG"; + docker build -t "$ORGANIZATION/$IMAGE_AGENT:$TAG" ./dockerfiles/agent + if [ $? != 0 ]; then + printf "Failed build docker image %s/%s:%s \n" "$ORGANIZATION" "$IMAGE_AGENT" "$TAG"; + exit 0 + fi + + printf "Build docker image %s/%s:%s \n" "$ORGANIZATION" "$IMAGE_STORAGE" "$TAG"; + docker build -t $ORGANIZATION/$IMAGE_STORAGE:$TAG ./dockerfiles/storage + if [ $? != 0 ]; then + printf "Failed build docker image %s/%s:%s \n" "$ORGANIZATION" "$IMAGE_STORAGE" "$TAG"; + exit 0 + fi + + echo "Build images successfully completed" +} diff --git a/dockerfiles/agent/Dockerfile b/dockerfiles/agent/Dockerfile new file mode 100644 index 0000000..e59e735 --- /dev/null +++ b/dockerfiles/agent/Dockerfile @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Copyright (c) 2019-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +FROM alpine:3.11 + +ENV USER=user +ENV UID=12345 +ENV GID=23456 + +#cron task not work in openshift in case https://github.com/gliderlabs/docker-alpine/issues/381 +#so will used supercronic https://github.com/aptible/supercronic +ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.9/supercronic-linux-amd64 \ + SUPERCRONIC=supercronic-linux-amd64 \ + SUPERCRONIC_SHA1SUM=5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85 + +COPY cron/backup-cron-job /etc/crontabs/backup-cron-job +COPY scripts scripts + +# Add user that will be able to start machine-exec-binary but nothing more +# the result will be propagated then into scratch image +# See https://stackoverflow.com/a/55757473/12429735RUN +RUN addgroup --gid "$GID" "$USER" \ + && adduser \ + --disabled-password \ + --gecos "" \ + --home "$(pwd)" \ + --ingroup "$USER" \ + --no-create-home \ + --uid "$UID" \ + "$USER" \ + && mkdir /var/run/sshd && \ + # Change permissions to let any arbitrary user + for f in "/etc/passwd" "/var/run/sshd" "/scripts"; do \ + echo "Changing permissions on ${f}" && chgrp -R 0 ${f} && \ + chmod -R g+rwX ${f}; \ + done \ + # install needed software + && apk update \ + && apk upgrade \ + && apk add --no-cache \ + rsync \ + curl \ + openssh \ + ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/cache/apk/* \ + #install supercronic + && curl -fsSLO "$SUPERCRONIC_URL" \ + && echo "${SUPERCRONIC_SHA1SUM} ${SUPERCRONIC}" | sha1sum -c - \ + && chmod +x "$SUPERCRONIC" \ + && mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" \ + && ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic \ + # change permissions + && chmod +x /scripts/* \ + && chmod 0644 /etc/crontabs/backup-cron-job \ + && sed -i s/root:!/"root:*"/g /etc/shadow + +EXPOSE 4445 +ENTRYPOINT [ "/scripts/entrypoint.sh" ] diff --git a/dockerfiles/agent/cron/backup-cron-job b/dockerfiles/agent/cron/backup-cron-job new file mode 100644 index 0000000..d37c48b --- /dev/null +++ b/dockerfiles/agent/cron/backup-cron-job @@ -0,0 +1,9 @@ +# Copyright (c) 2019-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +# run backup.sh as cron task every 10 minutes +*/10 * * * * /scripts/backup.sh diff --git a/dockerfiles/agent/scripts/backup.sh b/dockerfiles/agent/scripts/backup.sh new file mode 100755 index 0000000..1ca3e6d --- /dev/null +++ b/dockerfiles/agent/scripts/backup.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# +# Copyright (c) 2019-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +USER_NAME="user" +##### SSH options ##### +SSH_OPTIONS="" +# Add SSH connection options +SSH_OPTIONS=" ${SSH_OPTIONS} -i /etc/ssh/private/rsync-via-ssh -l ${USER_NAME} -p ${RSYNC_PORT}" +# Disable password authentication since we use key-based auth +SSH_OPTIONS=" ${SSH_OPTIONS} -o PasswordAuthentication=no" +# Disable hosts fingerprint checking because it may fail due to +# starting different containers on the same ports after some period +SSH_OPTIONS=" ${SSH_OPTIONS} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +# Set SSH logging level to make it possible to investigate problems +#SSH_OPTIONS=" ${SSH_OPTIONS} -o LogLevel=${SSH_LOG_LEVEL}" + +##### Rsync options ##### +RSYNC_OPTIONS="" +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --quiet" +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --recursive" +# Throughput limit for rsync +#RSYNC_OPTIONS+=" ${RSYNC_OPTIONS} --bwlimit=${RSYNC_BACKUP_BWLIMIT}" +# Sync modification timestamps to optimise transfer of not modified files. +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --omit-dir-times " #--times" +# Do not remove partially transferred files if transfer is interrupted. Make subsequent transfers faster. +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --partial" +# Delete files/folders on receiving side if they are not present on sending side. +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --delete" +# Preserve sym links in a safe way +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --links --safe-links" +# Transition of ownership and permissions +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --no-o --no-g --no-perms" #--owner --group --numeric-ids + +rsync ${RSYNC_OPTIONS} --rsh="ssh ${SSH_OPTIONS}" ${CHE_PROJECTS_ROOT}/ async-storage:/var/lib/storage/data/${CHE_WORKSPACE_ID}/ diff --git a/dockerfiles/agent/scripts/entrypoint.sh b/dockerfiles/agent/scripts/entrypoint.sh new file mode 100755 index 0000000..567bb43 --- /dev/null +++ b/dockerfiles/agent/scripts/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# +# Copyright (c) 2019-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +export USER_ID=$(id -u) +export GROUP_ID=$(id -g) + +if ! whoami &> /dev/null; then + echo "${USER_NAME:-user}:x:${USER_ID}:0:${USER_NAME:-user} user:${HOME}:/bin/sh" >> /etc/passwd +fi +/scripts/watcher & +supercronic /etc/crontabs/backup-cron-job + +/usr/sbin/sshd -p 2222 +tail -f /dev/null diff --git a/dockerfiles/agent/scripts/restore.sh b/dockerfiles/agent/scripts/restore.sh new file mode 100755 index 0000000..833678e --- /dev/null +++ b/dockerfiles/agent/scripts/restore.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# +# Copyright (c) 2019-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +USER_NAME="user" +##### SSH options ##### +SSH_OPTIONS="" +# Add SSH connection options +SSH_OPTIONS=" ${SSH_OPTIONS} -i /etc/ssh/private/rsync-via-ssh -l ${USER_NAME} -p ${RSYNC_PORT}" +# Disable password authentication since we use key-based auth +SSH_OPTIONS=" ${SSH_OPTIONS} -o PasswordAuthentication=no" +# Disable hosts fingerprint checking because it may fail due to +# starting different containers on the same ports after some period +SSH_OPTIONS=" ${SSH_OPTIONS} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +# Set SSH logging level to make it possible to investigate problems +#SSH_OPTIONS+=" -o LogLevel=${SSH_LOG_LEVEL}" + +##### Rsync options ##### +RSYNC_OPTIONS=" --info=progress2 -ah --no-inc-recursive" +#RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --quiet" +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --recursive" +# Throughput limit for rsync +#RSYNC_OPTIONS+=" --bwlimit=${RSYNC_RESTORE_BWLIMIT}" +# Sync modification timestamps to optimise transfer of not modified files. +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --omit-dir-times " #--times" +# Delete files/folders on receiving side if they are not present on sending side. +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --delete" +# Preserve sym links in a safe way +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --links --safe-links" +# Transition of ownership and permissions +RSYNC_OPTIONS=" ${RSYNC_OPTIONS} --no-o --no-g --no-perms" #--owner --group --numeric-ids +######################### + +ssh -i /etc/ssh/private/rsync-via-ssh -q async-storage -p ${RSYNC_PORT} [[ -d /var/lib/storage/data/${CHE_WORKSPACE_ID} ]] +if [[ $? -eq 0 ]]; then + rsync ${RSYNC_OPTIONS} --rsh="ssh ${SSH_OPTIONS}" async-storage:/var/lib/storage/data/${CHE_WORKSPACE_ID}/ ${CHE_PROJECTS_ROOT}/ +fi + diff --git a/dockerfiles/storage/Dockerfile b/dockerfiles/storage/Dockerfile new file mode 100644 index 0000000..3873502 --- /dev/null +++ b/dockerfiles/storage/Dockerfile @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright (c) 2019-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +FROM alpine:3.11 + +COPY entrypoint.sh /usr/local/bin + +RUN mkdir /var/run/sshd /.ssh && \ + # Change permissions to let any arbitrary user + for f in "/etc/passwd" "/.ssh" "/var/run/sshd" ; do \ + echo "Changing permissions on ${f}" && chgrp -R 0 ${f} && \ + chmod -R g+rwX ${f}; \ + done \ + && apk update \ + && apk upgrade \ + && apk add --no-cache \ + rsync \ + openssh \ + ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/cache/apk/* \ + && chmod 0550 /.ssh \ + && sed -i s/root:!/"root:*"/g /etc/shadow \ + && chmod +x /usr/local/bin/entrypoint.sh + +EXPOSE 2222 +ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] + + diff --git a/dockerfiles/storage/entrypoint.sh b/dockerfiles/storage/entrypoint.sh new file mode 100755 index 0000000..9d5b8bf --- /dev/null +++ b/dockerfiles/storage/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# +# Copyright (c) 2019-2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 + +export USER_ID=$(id -u) +export GROUP_ID=$(id -g) + +# Add current (arbitrary) user `theia` to /etc/passwd +if ! whoami &> /dev/null; then + echo "${USER_NAME:-user}:x:${USER_ID}:0:${USER_NAME:-user} user:${HOME}:/bin/sh" >> /etc/passwd +fi + +# Grant access to projects volume in case of non root user with sudo rights +if [ "${USER_ID}" -ne 0 ] && command -v sudo >/dev/null 2>&1 && sudo -n true > /dev/null 2>&1; then + sudo chown "${USER_ID}:${GROUP_ID}" /projects +fi + +/usr/sbin/sshd -p 2222 +tail -f /dev/null + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a6595b1 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/che-incubator/workspace-data-sync + +go 1.12 + +require github.com/gorilla/websocket v1.4.2 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..85efffd --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/pkg/progress/model.go b/pkg/progress/model.go new file mode 100644 index 0000000..fa50711 --- /dev/null +++ b/pkg/progress/model.go @@ -0,0 +1,6 @@ +package progress + +type ProgressState struct { + State string `json:"state"` + Info string `json:"info"` +} diff --git a/pkg/progress/progress_watcher.go b/pkg/progress/progress_watcher.go new file mode 100644 index 0000000..b4260a7 --- /dev/null +++ b/pkg/progress/progress_watcher.go @@ -0,0 +1,124 @@ +package progress + +import ( + "bufio" + "fmt" + "log" + "os/exec" + "strconv" + "strings" + "time" +) + +type Watcher struct { + progressPercents int + exitCode int + buf *bufio.Reader + subscribers []func(ProgressState) + done chan bool + cmd *exec.Cmd +} + +func CmdWithProgressWatching(name string, arg ...string) (*Watcher, error) { + cmd := exec.Command(name, arg...) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + err = cmd.Start() + if err != nil { + return nil, err + } + + return &Watcher{ + progressPercents: 0, + buf: bufio.NewReader(stdout), + subscribers: []func(ProgressState){}, + done: make(chan bool), + cmd: cmd, + }, nil +} + +func (p *Watcher) Watch() { + // track also if exec is not finished to stop automatically + go func() { + err := p.cmd.Wait() + + exitCode := p.cmd.ProcessState.ExitCode() + p.stop(exitCode) + + if err != nil { + //TODO Make sure that it's right way to finish the process + log.Panic(err) + } + }() + + ticker := time.NewTicker(500 * time.Millisecond) + + for { + select { + case <-ticker.C: + { + line, err := p.buf.ReadSlice('\r') + if err != nil { + fmt.Printf("Failed to read the next line. Cause: %s", err) + //TODO continue or break here? Should we send ERROR + } + var upd bool + p.progressPercents, upd = parse(line, p.progressPercents) + if upd { + newState := ProgressState{ + State: "IN_PROGRESS", + Info: strconv.Itoa(p.progressPercents) + "%", + } + for _, s := range p.subscribers { + s(newState) + } + } + } + case <-p.done: + return + } + } +} + +func (p *Watcher) stop(code int) { + p.done <- false + var state ProgressState + if code == 0 { + state = ProgressState{ + State: "DONE", + Info: "100%", + } + } else if code > 0 { + state = ProgressState{ + State: "ERROR", + Info: fmt.Sprintf("Process exited with non-zero exit code: %s", code), + } + } + + for _, s := range p.subscribers { + s(state) + } +} + +func (p *Watcher) Notify(receiver func(progress ProgressState)) { + p.subscribers = append(p.subscribers, receiver) +} + +// parse parses percents in the specified output line +// returns new value and boolean that indicated that it's changed from previous state +func parse(line []byte, prev int) (int, bool) { + fields := strings.Fields(string(line)) + fmt.Println(fields) + if len(fields) > 1 && strings.Contains(fields[1], "%") { + nowStr := strings.Split(fields[1], "%")[0] + prog, _ := strconv.Atoi(nowStr) + if prog > prev { + return prog, true + } + } + return prev, false +} diff --git a/watcher/watcher.go b/watcher/watcher.go new file mode 100644 index 0000000..f75917f --- /dev/null +++ b/watcher/watcher.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "encoding/json" + "github.com/che-incubator/workspace-data-sync/pkg/progress" + "log" + "net/http" + "os" + "os/signal" + "time" + + "github.com/gorilla/websocket" +) + +var upgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { + return true + }, +} + +func main() { + pw, err := progress.CmdWithProgressWatching("/bin/sh", "/scripts/restore.sh") + if err != nil { + log.Fatal(err) + } + + go func() { + pw.Watch() + }() + + http.HandleFunc("/track", func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + pw.Notify(func(p progress.ProgressState) { + send(conn, p) + if err != nil { + log.Println(err) + return + } + }) + + }) + + server := &http.Server{Addr: ":4445"} + + go func() { + if err := server.ListenAndServe(); err != nil { + // handle err + } + }() + + // Setting up signal capturing + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt) + + // Waiting for SIGINT (pkill -2) + <-stop + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(ctx); err != nil { + log.Printf("Failed to stop server gracefully. Cause: %s", err) + } +} + +func send(conn *websocket.Conn, msg progress.ProgressState) { + sMsg, _ := json.Marshal(msg) + err := conn.WriteMessage(1, sMsg) + if err != nil { + log.Println(err) + } +} From 3553797ccee66bbf72e06f72ff3e662fb53f5d1c Mon Sep 17 00:00:00 2001 From: Vitalii Parfonov Date: Tue, 30 Jun 2020 15:39:31 +0300 Subject: [PATCH 2/5] Update pkg/progress/progress_watcher.go Co-authored-by: Serhii Leshchenko --- pkg/progress/progress_watcher.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/progress/progress_watcher.go b/pkg/progress/progress_watcher.go index b4260a7..df277e9 100644 --- a/pkg/progress/progress_watcher.go +++ b/pkg/progress/progress_watcher.go @@ -19,6 +19,7 @@ type Watcher struct { cmd *exec.Cmd } +// CmdWithProgressWatching starts cmd and returns watchers that allows to track the progress func CmdWithProgressWatching(name string, arg ...string) (*Watcher, error) { cmd := exec.Command(name, arg...) From a4389fb91923116955df6f0a58c4abce02d6e05e Mon Sep 17 00:00:00 2001 From: Vitalii Parfonov Date: Tue, 30 Jun 2020 15:52:16 +0300 Subject: [PATCH 3/5] Fix review comments Signed-off-by: Vitalii Parfonov --- README.md | 3 +-- build.sh | 16 ++++++---------- common.sh | 6 +++--- dockerfiles/agent/Dockerfile | 6 +++--- dockerfiles/storage/Dockerfile | 2 -- pkg/progress/progress_watcher.go | 6 +++--- 6 files changed, 16 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 0cee204..1cdb1b5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ ## About Eclipse Che Workspace Data Sync -This tooling provide ability to increase I/O performance for a developer workspaces. +This tooling provides the ability to increase I/O performance for a developer workspaces. -============== ## Workspace lifecycle: #### Workspaces Startup phase: diff --git a/build.sh b/build.sh index 5daa3ae..48d6048 100755 --- a/build.sh +++ b/build.sh @@ -14,9 +14,13 @@ usage="$(basename "$0") -- script for compile Go source and build docker images usage: -t,--tag tag name for docker images (default: latest) - -o,--organization organization name for docker images (default: che-incubator) + -o,--organization organization name for docker images (default: eclipse) -h,--help show this help text" +#default +ORGANIZATION="eclipse" +TAG="latest" + while [[ $# -gt 0 ]] do key="${1}" @@ -42,13 +46,5 @@ do esac done -#default -if [[ -z "$TAG" ]] -then TAG="latest" -fi -if [[ -z "$ORGANIZATION" ]] -then ORGANIZATION="che-incubator" -fi - compile; -dockerBuild "$TAG" "${ORGANIZATION}"; +dockerBuild "${TAG}" "${ORGANIZATION}"; diff --git a/common.sh b/common.sh index b38fc18..d5ae49b 100755 --- a/common.sh +++ b/common.sh @@ -8,8 +8,8 @@ # SPDX-License-Identifier: EPL-2.0 # -IMAGE_AGENT=workspace-data-sync-agent -IMAGE_STORAGE=workspace-data-sync-storage +IMAGE_AGENT=che-sidecar-workspace-data-sync +IMAGE_STORAGE=che-workspace-data-sync-storage TAG=$1 ORGANIZATION=$2 @@ -32,7 +32,7 @@ function dockerBuild() { fi printf "Build docker image %s/%s:%s \n" "$ORGANIZATION" "$IMAGE_STORAGE" "$TAG"; - docker build -t $ORGANIZATION/$IMAGE_STORAGE:$TAG ./dockerfiles/storage + docker build -t "$ORGANIZATION/$IMAGE_STORAGE:$TAG" ./dockerfiles/storage if [ $? != 0 ]; then printf "Failed build docker image %s/%s:%s \n" "$ORGANIZATION" "$IMAGE_STORAGE" "$TAG"; exit 0 diff --git a/dockerfiles/agent/Dockerfile b/dockerfiles/agent/Dockerfile index e59e735..d9cb5b0 100644 --- a/dockerfiles/agent/Dockerfile +++ b/dockerfiles/agent/Dockerfile @@ -9,9 +9,9 @@ FROM alpine:3.11 -ENV USER=user -ENV UID=12345 -ENV GID=23456 +ENV USER=user \ + UID=12345 \ + GID=23456 #cron task not work in openshift in case https://github.com/gliderlabs/docker-alpine/issues/381 #so will used supercronic https://github.com/aptible/supercronic diff --git a/dockerfiles/storage/Dockerfile b/dockerfiles/storage/Dockerfile index 3873502..9271f69 100644 --- a/dockerfiles/storage/Dockerfile +++ b/dockerfiles/storage/Dockerfile @@ -31,5 +31,3 @@ RUN mkdir /var/run/sshd /.ssh && \ EXPOSE 2222 ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] - - diff --git a/pkg/progress/progress_watcher.go b/pkg/progress/progress_watcher.go index b4260a7..775f41a 100644 --- a/pkg/progress/progress_watcher.go +++ b/pkg/progress/progress_watcher.go @@ -50,8 +50,7 @@ func (p *Watcher) Watch() { p.stop(exitCode) if err != nil { - //TODO Make sure that it's right way to finish the process - log.Panic(err) + log.Fatal(err) //fatal will call os.Exit(1) } }() @@ -64,7 +63,8 @@ func (p *Watcher) Watch() { line, err := p.buf.ReadSlice('\r') if err != nil { fmt.Printf("Failed to read the next line. Cause: %s", err) - //TODO continue or break here? Should we send ERROR + //Let's continue restore process. Probably not critical error for process itself + continue } var upd bool p.progressPercents, upd = parse(line, p.progressPercents) From 69f0fcf740d0f67a7cb109b04b0f6149f9f347f3 Mon Sep 17 00:00:00 2001 From: Vitalii Parfonov Date: Tue, 30 Jun 2020 15:58:00 +0300 Subject: [PATCH 4/5] Add license header Signed-off-by: Vitalii Parfonov --- pkg/progress/model.go | 12 ++++++++++++ pkg/progress/progress_watcher.go | 12 ++++++++++++ watcher/watcher.go | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/pkg/progress/model.go b/pkg/progress/model.go index fa50711..1b37636 100644 --- a/pkg/progress/model.go +++ b/pkg/progress/model.go @@ -1,3 +1,15 @@ +// +// Copyright (c) 2019-2020 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// + package progress type ProgressState struct { diff --git a/pkg/progress/progress_watcher.go b/pkg/progress/progress_watcher.go index 775f41a..cca268b 100644 --- a/pkg/progress/progress_watcher.go +++ b/pkg/progress/progress_watcher.go @@ -1,3 +1,15 @@ +// +// Copyright (c) 2019-2020 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// + package progress import ( diff --git a/watcher/watcher.go b/watcher/watcher.go index f75917f..519c0e9 100644 --- a/watcher/watcher.go +++ b/watcher/watcher.go @@ -1,3 +1,15 @@ +// +// Copyright (c) 2019-2020 Red Hat, Inc. +// This program and the accompanying materials are made +// available under the terms of the Eclipse Public License 2.0 +// which is available at https://www.eclipse.org/legal/epl-2.0/ +// +// SPDX-License-Identifier: EPL-2.0 +// +// Contributors: +// Red Hat, Inc. - initial API and implementation +// + package main import ( From 7ef57dce2edd1a7f2864a22022e478ad82563567 Mon Sep 17 00:00:00 2001 From: Vitalii Parfonov Date: Tue, 30 Jun 2020 18:00:32 +0300 Subject: [PATCH 5/5] Fix typo Signed-off-by: Vitalii Parfonov --- dockerfiles/agent/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/agent/Dockerfile b/dockerfiles/agent/Dockerfile index d9cb5b0..62be454 100644 --- a/dockerfiles/agent/Dockerfile +++ b/dockerfiles/agent/Dockerfile @@ -22,7 +22,7 @@ ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0. COPY cron/backup-cron-job /etc/crontabs/backup-cron-job COPY scripts scripts -# Add user that will be able to start machine-exec-binary but nothing more +# Add user that will be able to start watcher binary but nothing more # the result will be propagated then into scratch image # See https://stackoverflow.com/a/55757473/12429735RUN RUN addgroup --gid "$GID" "$USER" \