feat: Cache DSM info (#378)

* feat: Cache location

* feat: Cache location

* feat: Add support URL

* feat: Cache location

* fix: Remove files

* feat: Reset filesystem

* fix: Exit when PID is missing

* fix: Counter file

* fix: Check flags

* docs: Readme

* feat: Cleanup files

* fix: Check flags

* fix: Check flags

* fix: Initialization

* fix: Initalization

* fix: Initialization

* fix: Cleanup temp

* fix: Initialize system

* feat: Config system

* feat: Configure system

* fix: Variables

* fix: Variables

* fix: Error handling

* style: Comments

* fix: Returnvalue

* fix: Returnvalue

* fix: Returnvalue

* fix: Returnvalue

* fix: Returnvalue

* docs: Multi-disk support

* feat: Use cached location

* fix: Swap order

readme.md

@@ -17,7 +17,6 @@ Virtual DSM in a docker container.
  - Multi-platform
  - KVM acceleration
  - GPU passthrough
- - Graceful shutdowns
  - Upgrades supported
 
 ## Usage
@@ -85,6 +84,32 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti
     ```
 
     Keep in mind that this will not affect any of your existing disks, it only applies to newly created disks.
+
+  * ### How do I add multiple disks?
+
+    To mount extra volumes, modify your compose file like this:
+
+    ```yaml
+    environment:
+      DISK2_SIZE: "32G"
+      DISK3_SIZE: "64G"
+    volumes:
+      - /home/example2:/storage2
+      - /home/example3:/storage3
+    ```
+
+    Additionally, it's also possible to passthrough raw disk devices like this:
+
+    ```yaml
+    environment:
+      DEVICE2: "/dev/vdc1"
+      DEVICE3: "/dev/vdc2"
+    devices:
+      - /dev/vdc1
+      - /dev/vdc2
+    ```    
+
+    Please beware that any pre-existing partitions and data on those devices will be wiped.
 
   * ### How do I increase the amount of CPU or RAM?
 

run/check.sh

@@ -2,49 +2,60 @@
 set -u
 
 [ ! -f "/run/qemu.pid" ] && echo "QEMU not running yet.." && exit 0
-[ -f "/run/qemu.counter" ] && echo "QEMU is shutting down.." && exit 1
+[ -f "/run/qemu.count" ] && echo "QEMU is shutting down.." && exit 1
 
-# Retrieve IP from guest VM for Docker healthcheck
-RESPONSE=$(curl -s -m 16 -S http://127.0.0.1:2210/read?command=10 2>&1)
+file="/run/dsm.url"
 
-if [[ ! "${RESPONSE}" =~ "\"success\"" ]] ; then
-  echo "Failed to connect to guest: $RESPONSE" && exit 1
-fi
+if [ ! -f  "$file" ]; then
 
-# Retrieve the HTTP port number
-if [[ ! "${RESPONSE}" =~ "\"http_port\"" ]] ; then
-  echo "Failed to parse response from guest: $RESPONSE" && exit 1
-fi
+  # Retrieve IP from guest VM for Docker healthcheck
+  RESPONSE=$(curl -s -m 16 -S http://127.0.0.1:2210/read?command=10 2>&1)
 
-rest=${RESPONSE#*http_port}
-rest=${rest#*:}
-rest=${rest%%,*}
-PORT=${rest%%\"*}
+  if [[ ! "${RESPONSE}" =~ "\"success\"" ]] ; then
+    echo "Failed to connect to guest: $RESPONSE" && exit 1
+  fi
 
-[ -z "${PORT}" ] && echo "Guest has not set a portnumber yet.." && exit 1
+  # Retrieve the HTTP port number
+  if [[ ! "${RESPONSE}" =~ "\"http_port\"" ]] ; then
+    echo "Failed to parse response from guest: $RESPONSE" && exit 1
+  fi
 
-# Retrieve the IP address
-if [[ ! "${RESPONSE}" =~ "eth0" ]] ; then
-  echo "Failed to parse response from guest: $RESPONSE" && exit 1
-fi
+  rest=${RESPONSE#*http_port}
+  rest=${rest#*:}
+  rest=${rest%%,*}
+  PORT=${rest%%\"*}
+
+  [ -z "${PORT}" ] && echo "Guest has not set a portnumber yet.." && exit 1
+
+  # Retrieve the IP address
+  if [[ ! "${RESPONSE}" =~ "eth0" ]] ; then
+    echo "Failed to parse response from guest: $RESPONSE" && exit 1
+  fi
 
-rest=${RESPONSE#*eth0}
-rest=${rest#*ip}
-rest=${rest#*:}
-rest=${rest#*\"}
-IP=${rest%%\"*}
+  rest=${RESPONSE#*eth0}
+  rest=${rest#*ip}
+  rest=${rest#*:}
+  rest=${rest#*\"}
+  IP=${rest%%\"*}
+
+  [ -z "${IP}" ] && echo "Guest has not received an IP yet.." && exit 1
+
+  echo "${IP}:${PORT}" > $file
+
+fi
 
-[ -z "${IP}" ] && echo "Guest has not received an IP yet.." && exit 1
+LOCATION=$(cat "$file")
 
-if ! curl -m 3 -ILfSs "http://${IP}:${PORT}/" > /dev/null; then
-  echo "Failed to reach ${IP}:${PORT}"
+if ! curl -m 20 -ILfSs "http://${LOCATION}/" > /dev/null; then
+  rm -f $file
+  echo "Failed to reach http://${LOCATION}"
   exit 1
 fi
 
-if [[ "$IP" == "20.20"* ]]; then
+if [[ "$LOCATION" == "20.20"* ]]; then
   echo "Healthcheck OK"
 else
-  echo "Healthcheck OK ( $IP )"
+  echo "Healthcheck OK ( ${LOCATION%:*} )"
 fi
 
 exit 0

run/config.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+KVM_ERR=""
+KVM_OPTS=""
+
+if [ -e /dev/kvm ] && sh -c 'echo -n > /dev/kvm' &> /dev/null; then
+  if ! grep -q -e vmx -e svm /proc/cpuinfo; then
+    KVM_ERR="(vmx/svm disabled)"
+  fi
+else
+  [ -e /dev/kvm ] && KVM_ERR="(no write access)" || KVM_ERR="(device file missing)"
+fi
+
+if [ -n "${KVM_ERR}" ]; then
+  if [ "$ARCH" == "amd64" ]; then
+    error "KVM acceleration not detected ${KVM_ERR}, see the FAQ about this."
+    [[ "${DEBUG}" != [Yy1]* ]] && exit 88
+  fi
+else
+  KVM_OPTS=",accel=kvm -enable-kvm -cpu host"
+fi
+
+DEF_OPTS="-nographic -nodefaults -boot strict=on -display none"
+RAM_OPTS=$(echo "-m ${RAM_SIZE}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
+CPU_OPTS="-smp ${CPU_CORES},sockets=1,dies=1,cores=${CPU_CORES},threads=1"
+MAC_OPTS="-machine type=q35,usb=off,dump-guest-core=off,hpet=off${KVM_OPTS}"
+EXTRA_OPTS="-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4"
+EXTRA_OPTS="$EXTRA_OPTS -object rng-random,id=objrng0,filename=/dev/urandom"
+EXTRA_OPTS="$EXTRA_OPTS -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pcie.0,addr=0x1c"
+
+if [[ "${GPU}" == [Yy1]* ]] && [[ "$ARCH" == "amd64" ]]; then
+  DEF_OPTS="-nodefaults -boot strict=on -display egl-headless,rendernode=/dev/dri/renderD128"
+  DEF_OPTS="${DEF_OPTS} -device virtio-vga,id=video0,max_outputs=1,bus=pcie.0,addr=0x1"
+fi
+
+ARGS="${DEF_OPTS} ${CPU_OPTS} ${RAM_OPTS} ${MAC_OPTS} ${MON_OPTS} ${SERIAL_OPTS} ${NET_OPTS} ${DISK_OPTS} ${EXTRA_OPTS} ${ARGUMENTS}"
+ARGS=$(echo "$ARGS" | sed 's/\t/ /g' | tr -s ' ')
+
+return 0

run/disk.sh

@@ -237,3 +237,5 @@ if [ -n "${DEVICE3}" ]; then
     -device scsi-hd,bus=hw-userdata6.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata6,id=userdata6,rotation_rate=${DISK_ROTATION},bootindex=8"
 
 fi
+
+return 0

run/gpu.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 set -Eeuo pipefail
 
+if [[ "${GPU}" != [Yy1]* ]] || [[ "$ARCH" != "amd64" ]]; then
+  return 0
+fi
+
 [ ! -d /dev/dri ] && mkdir -m 755 /dev/dri
 
 if [ ! -c /dev/dri/card0 ]; then
@@ -14,9 +18,6 @@ fi
 chmod 666 /dev/dri/card0
 chmod 666 /dev/dri/renderD128
 
-DEF_OPTS="-nodefaults -boot strict=on -display egl-headless,rendernode=/dev/dri/renderD128"
-DEF_OPTS="${DEF_OPTS} -device virtio-vga,id=video0,max_outputs=1,bus=pcie.0,addr=0x1"
-
 if ! apt-mark showinstall | grep -q "xserver-xorg-video-intel"; then
 
   info "Installing Intel GPU drivers..."
@@ -40,3 +41,5 @@ if ! apt-mark showinstall | grep -q "qemu-system-modules-opengl"; then
   apt-get -qq --no-install-recommends -y install qemu-system-modules-opengl > /dev/null
 
 fi
+
+return 0

run/install.sh

@@ -1,6 +1,23 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
+STORAGE="/storage"
+[ ! -d "$STORAGE" ] && error "Storage folder (${STORAGE}) not found!" && exit 13
+
+if [ -f "$STORAGE"/dsm.ver ]; then
+  BASE=$(cat "${STORAGE}/dsm.ver")
+else
+  # Fallback for old installs
+  BASE="DSM_VirtualDSM_42962"
+fi
+
+[ -n "$URL" ] && BASE=$(basename "$URL" .pat)
+
+if [[ -f "$STORAGE/$BASE.boot.img" ]] && [[ -f "$STORAGE/$BASE.system.img" ]]; then
+  # Previous installation found
+  return 0
+fi
+
 # Display wait message
 /run/server.sh 5000 install &
 
@@ -31,18 +48,20 @@ rm -f "$STORAGE"/"$BASE".agent
 rm -f "$STORAGE"/"$BASE".boot.img
 rm -f "$STORAGE"/"$BASE".system.img
 
-TMP="/tmp/dsm"
 MIN_SPACE=6442450944
 FS=$(stat -f -c %T "$STORAGE")
 
 if [[ "$FS" != "fat"* && "$FS" != "vfat"* && "$FS" != "exfat"* && \
         "$FS" != "ntfs"* && "$FS" != "fuse"* && "$FS" != "msdos"* ]]; then
   TMP="$STORAGE/tmp"
 else
+  TMP="/tmp/dsm"
   SPACE=$(df --output=avail -B 1 /tmp | tail -n 1)
   (( MIN_SPACE > SPACE )) && TMP="$STORAGE/tmp"
 fi
 
+rm -rf /tmp/dsm
+rm -rf "$STORAGE/tmp"
 rm -rf "$TMP" && mkdir -p "$TMP"
 
 # Check free diskspace

run/power.sh

@@ -3,14 +3,14 @@ set -Eeuo pipefail
 
 # Configure QEMU for graceful shutdown
 
-QEMU_MONPORT=7100
-QEMU_POWERDOWN_TIMEOUT=50
+QEMU_PORT=7100
+QEMU_TIMEOUT=50
 
-_QEMU_PID=/run/qemu.pid
-_QEMU_SHUTDOWN_COUNTER=/run/qemu.counter
+QEMU_PID=/run/qemu.pid
+QEMU_COUNT=/run/qemu.count
 
-rm -f "${_QEMU_PID}"
-rm -f "${_QEMU_SHUTDOWN_COUNTER}"
+rm -f "${QEMU_PID}"
+rm -f "${QEMU_COUNT}"
 
 _trap(){
     func="$1" ; shift
@@ -23,14 +23,14 @@ _graceful_shutdown() {
 
   set +e
 
-  [ ! -f "${_QEMU_PID}" ] && return
-  [ -f "${_QEMU_SHUTDOWN_COUNTER}" ] && return
+  [ ! -f "${QEMU_PID}" ] && exit 130
+  [ -f "${QEMU_COUNT}" ] && return
 
   echo && info "Received $1 signal, shutting down..."
-  echo 0 > "${_QEMU_SHUTDOWN_COUNTER}"
+  echo 0 > "${QEMU_COUNT}"
 
   # Don't send the powerdown signal because vDSM ignores ACPI signals
-  # echo 'system_powerdown' | nc -q 1 -w 1 localhost "${QEMU_MONPORT}" > /dev/null
+  # echo 'system_powerdown' | nc -q 1 -w 1 localhost "${QEMU_PORT}" > /dev/null
 
   # Send shutdown command to guest agent via serial port
   RESPONSE=$(curl -s -m 5 -S http://127.0.0.1:2210/read?command=6 2>&1)
@@ -39,30 +39,30 @@ _graceful_shutdown() {
 
     echo && error "Could not send shutdown command to the guest ($RESPONSE)"
 
-    kill -15 "$(cat "${_QEMU_PID}")"
+    kill -15 "$(cat "${QEMU_PID}")"
     pkill -f qemu-system-x86_64 || true
 
   fi
 
-  while [ "$(cat ${_QEMU_SHUTDOWN_COUNTER})" -lt "${QEMU_POWERDOWN_TIMEOUT}" ]; do
+  while [ "$(cat ${QEMU_COUNT})" -lt "${QEMU_TIMEOUT}" ]; do
 
     # Increase the counter
-    echo $(($(cat ${_QEMU_SHUTDOWN_COUNTER})+1)) > ${_QEMU_SHUTDOWN_COUNTER}
+    echo $(($(cat ${QEMU_COUNT})+1)) > ${QEMU_COUNT}
 
     # Try to connect to qemu
-    if echo 'info version'| nc -q 1 -w 1 localhost "${QEMU_MONPORT}" >/dev/null 2>&1 ; then
+    if echo 'info version'| nc -q 1 -w 1 localhost "${QEMU_PORT}" >/dev/null 2>&1 ; then
 
       sleep 1
 
-      CNT="$(cat ${_QEMU_SHUTDOWN_COUNTER})/${QEMU_POWERDOWN_TIMEOUT}"
+      CNT="$(cat ${QEMU_COUNT})/${QEMU_TIMEOUT}"
       [[ "${DEBUG}" == [Yy1]* ]] && info "Shutting down, waiting... (${CNT})"
 
     fi
 
   done
 
   echo && echo "❯ Quitting..."
-  echo 'quit' | nc -q 1 -w 1 localhost "${QEMU_MONPORT}" >/dev/null 2>&1 || true
+  echo 'quit' | nc -q 1 -w 1 localhost "${QEMU_PORT}" >/dev/null 2>&1 || true
 
   closeNetwork
 
@@ -71,4 +71,4 @@ _graceful_shutdown() {
 
 _trap _graceful_shutdown SIGTERM SIGHUP SIGINT SIGABRT SIGQUIT
 
-MON_OPTS="-monitor telnet:localhost:${QEMU_MONPORT},server,nowait,nodelay"
+MON_OPTS="-monitor telnet:localhost:${QEMU_PORT},server,nowait,nodelay"

run/print.sh

@@ -4,12 +4,13 @@ set -Eeuo pipefail
 info () { echo -e >&2 "\E[1;34m❯\E[1;36m $1\E[0m" ; }
 error () { echo -e >&2 "\E[1;31m❯ ERROR: $1\E[0m" ; }
 
-retry=true
+file="/run/dsm.url"
 
-while [ "$retry" = true ]
+while [ ! -f  "$file" ]
 do
 
   sleep 3
+  [ -f "$file" ] && continue
 
   # Retrieve IP from guest VM
 
@@ -46,14 +47,16 @@ do
 
   [ -z "${IP}" ] && continue
 
-  retry=false
+  echo "${IP}:${PORT}" > $file
 
 done
 
-if [[ "$IP" == "20.20"* ]]; then
-  MSG="port ${PORT}"
+LOCATION=$(cat "$file")
+
+if [[ "$LOCATION" == "20.20"* ]]; then
+  MSG="port ${LOCATION##*:}"
 else
-  MSG="http://${IP}:${PORT}"
+  MSG="http://${LOCATION}"
 fi
 
 echo "" >&2