Merge branch 'development' into 803

.dockerignore

@@ -0,0 +1,5 @@
+.git
+.github
+.coveragerc
+.gitignore
+docker

.gitignore

@@ -2,4 +2,5 @@ __pycache__
 venv
 .DS_Store
 .vscode
+*.code-workspace
 src/plotman.egg-info

CHANGELOG.md

@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [unreleased]
+### Fixed
+- `plotman kill` doesn't leave any temporary files behind anymore.
+  ([#801](https://github.com/ericaltendorf/plotman/pull/801))
 ### Added
 - `plotman export` command to output summaries from plot logs in `.csv` format.
   ([#557](https://github.com/ericaltendorf/plotman/pull/557))
@@ -20,6 +23,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter).
   See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up.
   ([#797](https://github.com/ericaltendorf/plotman/pull/797))
+- Added argument `-f`/`--force` to `plotman kill` to skip confirmation before killing the job.
+  ([#801](https://github.com/ericaltendorf/plotman/pull/801))
+- Docker container support.
+  See the [docker configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Docker-Configuration) for help setting it up.
+  ([#783](https://github.com/ericaltendorf/plotman/pull/783))
 
 ## [0.4.1] - 2021-06-11
 ### Fixed

MANIFEST.in

@@ -9,3 +9,6 @@ include .coveragerc
 recursive-include src *.py
 recursive-include src/plotman/_tests/resources *
 recursive-include src/plotman/resources *
+recursive-exclude docker *
+exclude .dockerignore
+exclude build-docker-plotman.sh

build-docker-plotman.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+DOCKER_REGISTRY="<docker-registry>"
+PROJECT="chia-plotman"
+TAG="plotter"
+BASE_CONTAINER="ubuntu:20.04"
+CHIA_GIT_REFERENCE="1.1.7"
+
+# The UID/GID should match the 'chia' owner of the directories on the host system
+UID=10001
+GID=10001
+
+docker rmi ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
+
+docker build . \
+	--squash \
+	--build-arg BASE_CONTAINER=${BASE_CONTAINER} \
+	--build-arg CHIA_GIT_REFERENCE=${CHIA_GIT_REFERENCE} \
+	--build-arg UID=${UID} \
+	--build-arg GID=${GID} \
+	-f docker/Dockerfile \
+        -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
+
+docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG}

docker/Dockerfile

@@ -0,0 +1,56 @@
+# Build deployable artifacts
+ARG BASE_CONTAINER=ubuntu:20.04
+FROM ${BASE_CONTAINER} as plotman-builder
+
+ARG CHIA_GIT_REFERENCE
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 ansible tar bash ca-certificates git openssl unzip wget python3-pip sudo acl build-essential python3-dev python3.8-venv python3.8-distutils apt nfs-common python-is-python3
+
+RUN echo "cloning ${CHIA_GIT_REFERENCE}"
+RUN git clone --branch "${CHIA_GIT_REFERENCE}" https://github.com/Chia-Network/chia-blockchain.git \
+&& cd chia-blockchain \
+&& git submodule update --init mozilla-ca
+
+WORKDIR /chia-blockchain
+# Placeholder for patches
+RUN /bin/bash ./install.sh
+
+COPY . /plotman
+
+RUN ["/bin/bash", "-c", "source ./activate && pip install /plotman && deactivate"] 
+
+# Build deployment container
+FROM ${BASE_CONTAINER} as plotman
+
+ARG UID=10001
+ARG GID=10001
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+&& DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 python3.8-venv ca-certificates tzdata ssh rsync \
+&& apt-get clean all \
+&& rm -rf /var/lib/apt/lists
+
+COPY --from=plotman-builder /chia-blockchain /chia-blockchain
+
+RUN groupadd -g ${GID} chia
+RUN useradd -m -u ${UID} -g ${GID} chia
+
+RUN mkdir -p /data/chia/tmp \
+&& mkdir -p /data/chia/plots \
+&& mkdir -p /data/chia/logs
+
+VOLUME ["/data/chia/tmp","/data/chia/plots","/data/chia/logs"]
+
+RUN chown -R chia:chia /chia-blockchain \
+&& chown -R chia:chia /data/chia
+
+WORKDIR /chia-blockchain
+USER chia
+
+ENV VIRTUAL_ENV="/chia-blockchain/venv"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Kick off plots (assumes the environemnt is good to go)
+CMD ["/bin/bash", "-c", "plotman plot" ]
+# Alternative command to simply provide shell environment
+# CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ]

docker/sample.docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3"
+
+services:
+  chia_plotman:
+    restart: always
+    container_name: chia-plotman
+    image: ${DOCKER_IMAGE}
+    volumes:
+      - ${HOME}/.ssh:/home/chia/.ssh
+      - ${HOME}/.chia:/home/chia/.chia
+      - ${HOME}/.config:/home/chia/.config
+      - ${LOGS_DIR}:/data/chia/logs
+      - ${PLOTS_DIR}:/data/chia/plots
+      - ${PLOTS_TMP_DIR}:/data/chia/tmp
+      - /tmp:/tmp
+    logging:
+      options:
+        max-size: ${DOCKER_LOG_MAX_SIZE}
+        max-file: ${DOCKER_LOG_MAX_FILE}

docker/sample.env

@@ -0,0 +1,7 @@
+DOCKER_IMAGE=<docker-registry>/chia-plotman:plotter
+LOGS_DIR=/data/chia/logs
+PLOTS_DIR=/data/chia/plots
+PLOTS_TMP_DIR=/data/chia/tmp
+DOCKER_LOG_MAX_SIZE=4m
+DOCKER_LOG_MAX_FILE=10
+

src/plotman/job.py

@@ -7,6 +7,7 @@
 import random
 import re
 import sys
+import glob
 import time
 from datetime import datetime
 from enum import Enum, auto
@@ -555,13 +556,11 @@ def resume(self) -> None:
     def get_temp_files(self) -> typing.Set[str]:
         # Prevent duplicate file paths by using set.
         temp_files = set([])
-        for f in self.proc.open_files():
-            if any(
-                dir in f.path
-                for dir in [self.tmpdir, self.tmp2dir, self.dstdir]
-                if dir is not None
-            ):
-                temp_files.add(f.path)
+
+        for dir in [self.tmpdir, self.tmp2dir, self.dstdir]:
+            if dir is not None:
+                temp_files.update(glob.glob(os.path.join(dir, f"plot-*-{self.plot_id}.tmp")))
+
         return temp_files
 
     def cancel(self) -> None:

src/plotman/plotman.py

@@ -73,6 +73,7 @@ def parse_args(self) -> typing.Any:
         self.add_idprefix_arg(p_files)
 
         p_kill = sp.add_parser('kill', help='kill job (and cleanup temp files)')
+        p_kill.add_argument('-f', '--force', action='store_true', default=False, help="Don't ask for confirmation before killing the plot job")
         self.add_idprefix_arg(p_kill)
 
         p_suspend = sp.add_parser('suspend', help='suspend job')
@@ -309,15 +310,24 @@ def main() -> None:
                         job.suspend()
 
                         temp_files = job.get_temp_files()
+
                         print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id))
                         print('Will delete %d temp files' % len(temp_files))
-                        conf = input('Are you sure? ("y" to confirm): ')
+
+                        if args.force:
+                            conf = 'y'
+                        else:
+                            conf = input('Are you sure? ("y" to confirm): ')
+
                         if (conf != 'y'):
-                            print('canceled.  If you wish to resume the job, do so manually.')
+                            print('Canceled.  If you wish to resume the job, do so manually.')
                         else:
                             print('killing...')
+
                             job.cancel()
+
                             print('cleaning up temp files...')
+
                             for f in temp_files:
                                 os.remove(f)