Merge pull request #81 from Jayy001/Eeems-patch-1

Add ls/cat and fix extract on non-linux

.github/workflows/main.yml

@@ -32,6 +32,9 @@ jobs:
           execute_install_scripts: true
           packages: libfuse-dev
           version: 1.0
+      - name: Install brew packages
+        if: matrix.os == 'macos-latest'
+        run: brew install coreutils
       - name: Checkout the Git repository
         uses: actions/checkout@v4
       - name: ccache
@@ -52,9 +55,15 @@ jobs:
           cache-dependency-path: |
             **/requirements*.txt
       - name: Test codexctl
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.os != 'windows-latest'
         shell: bash
         run: make test
+      - name: Test codexctl
+        if: matrix.os == 'windows-latest'
+        shell: bash
+        run: make test
+        env:
+          VENV_BIN_ACTIVATE: .venv/Scripts/activate
       - name: Build codexctl
         if: matrix.os == 'windows-latest'
         shell: bash
@@ -76,7 +85,6 @@ jobs:
           path: compilation-report.xml
           if-no-files-found: warn
       - name: Test Built version
-        if: matrix.os == 'ubuntu-latest'
         shell: bash
         run: make test-executable
       - name: Move .ccache

Makefile

@@ -2,56 +2,69 @@
 FW_VERSION := 2.15.1.1189
 FW_DATA := wVbHkgKisg-
 IMG_SHA := fc7d145e18f14a1a3f435f2fd5ca5924fe8dfe59bf45605dc540deed59551ae4
+LS_DATA := ". .. lost+found bin boot dev etc home lib media mnt postinst proc run sbin sys tmp uboot-postinst uboot-version usr var"
+CAT_DATA := 20221026104022
 SHELL := /bin/bash
 
 ifeq ($(VENV_BIN_ACTIVATE),)
 VENV_BIN_ACTIVATE := .venv/bin/activate
 endif
 
-OBJ := $(shell find codexctl -type f)
+OBJ := $(shell find codexctl -type f | grep -v __pycache__)
 OBJ += $(shell find data -type f)
 OBJ += README.md
 
 $(VENV_BIN_ACTIVATE): requirements.remote.txt requirements.txt
 	@echo "[info] Setting up development virtual env in .venv"
 	python -m venv .venv
+	@set -e; \
+	. $(VENV_BIN_ACTIVATE); \
+	python -m pip install wheel
 	@echo "[info] Installing dependencies"
+	@set -e; \
 	. $(VENV_BIN_ACTIVATE); \
 	python -m pip install \
 	    --extra-index-url=https://wheels.eeems.codes/ \
 	    -r requirements.remote.txt
 
 .venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed: $(VENV_BIN_ACTIVATE) $(OBJ)
 	@echo "[info] Downloading remarkable update file"
+	@set -e; \
 	. $(VENV_BIN_ACTIVATE); \
 	python -m codexctl download --out .venv ${FW_VERSION}
 
 test: $(VENV_BIN_ACTIVATE) .venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed
 	@echo "[info] Running test"
+	@set -e; \
 	. $(VENV_BIN_ACTIVATE); \
-	python test.py
-	if [ -d .venv/mnt ] && mountpoint -q .venv/mnt; then \
-	    umount -ql .venv/mnt; \
-	fi
-	mkdir -p .venv/mnt
-	. $(VENV_BIN_ACTIVATE); \
+	python test.py; \
 	if [[ "linux" == "$$(python -c 'import sys;print(sys.platform)')" ]]; then \
+	  if [ -d .venv/mnt ] && mountpoint -q .venv/mnt; then \
+	    umount -ql .venv/mnt; \
+	  fi; \
+	  mkdir -p .venv/mnt; \
 	  python -m codexctl mount --out .venv/mnt ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed"; \
 	  mountpoint .venv/mnt; \
 	  umount -ql .venv/mnt; \
-	  python -m codexctl extract --out ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img" ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed"; \
-	  echo "${IMG_SHA}  .venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img" | sha256sum --check; \
-	  rm -f ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img"; \
-	fi
+	fi; \
+	python -m codexctl extract --out ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img" ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed"; \
+	echo "${IMG_SHA}  .venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img" | sha256sum --check; \
+	rm -f ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img"
 
 test-executable: .venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed
+	@set -e; \
 	. $(VENV_BIN_ACTIVATE); \
-	if [[ "linux" == "$$(python -c 'import sys;print(sys.platform)')" ]]; then \
-	  dist/codexctl.* extract --out ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img" ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed"; \
-	  echo "${IMG_SHA}  .venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img" | sha256sum --check; \
-	  rm -f ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img"; \
-	else \
-	  dist/codexctl.* list; \
+	dist/codexctl.* extract --out ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img" ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed"; \
+	echo "${IMG_SHA}  .venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img" | sha256sum --check; \
+	rm -f ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.img"; \
+	set -o pipefail; \
+	if ! diff --color <(dist/codexctl.* ls ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed" /) <(echo ${LS_DATA}) | cat -te; then \
+	  echo "codexctl ls failed test"; \
+	  exit 1; \
+	fi
+	if ! diff --color <(dist/codexctl.* cat ".venv/${FW_VERSION}_reMarkable2-${FW_DATA}.signed" /etc/version) <(echo ${CAT_DATA}) | cat -te; then \
+	  echo "codexctl cat failed test"; \
+	  exit 1; \
 	fi
 
 clean:
@@ -63,9 +76,11 @@ clean:
 
 executable: $(VENV_BIN_ACTIVATE)
 	@echo "[info] Installing Nuitka"
+	@set -e; \
 	. $(VENV_BIN_ACTIVATE); \
-	python -m pip install --extra-index-url=https://wheels.eeems.codes/ wheel nuitka
+	python -m pip install --extra-index-url=https://wheels.eeems.codes/ nuitka
 	@echo "[info] Building codexctl"
+	@set -e; \
 	. $(VENV_BIN_ACTIVATE); \
 	NUITKA_CACHE_DIR="$(realpath .)/.nuitka" \
 	python -m nuitka \

README.md

@@ -86,4 +86,6 @@ python codexctl.py --backup # Exports all files to local directory
 python codexctl.py --backup -l root -r FM --no-recursion --no-overwrite # Exports all files from FM directory to root folder on localhost
 python codexctl.py extract 3.8.0.1944_reMarkable2-7eGpAv7sYB.signed # Extracts contents to filesystem named "extracted"
 python codexctl.py mount extracted /opt/remarkable # Mounts extracted filesystem to /opt/remarkable
+python codexctl.py ls 3.8.0.1944_reMarkable2-7eGpAv7sYB.signed / # Lists the root directory of the update image
+python codexctl.py cat 3.8.0.1944_reMarkable2-7eGpAv7sYB.signed /etc/version # Outputs the contents of /etc/version from the update image
 ```

codexctl.py

@@ -3,9 +3,8 @@
 # nuitka-project: --warn-implicit-exceptions
 # nuitka-project: --onefile
 # nuitka-project: --lto=yes
-# nuitka-project-if: {OS} in ("Linux"):
-#    nuitka-project: --include-package=google
-#    nuitka-project: --noinclude-unittest-mode=allow
+# nuitka-project: --include-package=google
+# nuitka-project: --noinclude-unittest-mode=allow
 
 from codexctl import main
 

codexctl/__init__.py

@@ -1,4 +1,5 @@
 import argparse
+import errno
 import subprocess
 import re
 import threading
@@ -244,6 +245,35 @@ def is_rm():
         return f.read().strip().startswith("reMarkable")
 
 
+def get_update_image(file):
+    import ext4
+    from remarkable_update_image import UpdateImage
+    from remarkable_update_image import UpdateImageSignatureException
+
+    image = UpdateImage(file)
+    volume = ext4.Volume(image, offset=0)
+    try:
+        inode = volume.inode_at("/usr/share/update_engine/update-payload-key.pub.pem")
+        if inode is None:
+            raise FileNotFoundError()
+
+        inode.verify()
+        image.verify(inode.open().read())
+
+    except UpdateImageSignatureException:
+        warnings.warn("Signature doesn't match contents", RuntimeWarning)
+
+    except FileNotFoundError:
+        warnings.warn("Public key missing", RuntimeWarning)
+
+    except OSError as e:
+        if e.errno != errno.ENOTDIR:
+            raise
+        warnings.warn("Unable to open public key", RuntimeWarning)
+
+    return image, volume
+
+
 def do_status(args):
     if is_rm():
         if os.path.exists("/etc/remarkable.conf"):
@@ -557,19 +587,53 @@ def do_backup(args):
     )
 
 
+def do_ls(args):
+    image, volume = get_update_image(args.file)
+    try:
+        inode = volume.inode_at(args.target_path)
+        print(" ".join([x.name_str for x, _ in inode.opendir()]))
+
+    except FileNotFoundError:
+        print(f"cannot access '{args.target_path}': No such file or directory")
+        sys.exit(1)
+
+    except OSError as e:
+        print(f"cannot access '{args.target_path}': {os.strerror(e.errno)}")
+        sys.exit(e.errno)
+
+
+def do_cat(args):
+    image, volume = get_update_image(args.file)
+    try:
+        inode = volume.inode_at(args.target_path)
+        sys.stdout.buffer.write(inode.open().read())
+
+    except FileNotFoundError:
+        print(f"'{args.target_path}': No such file or directory")
+        sys.exit(1)
+
+    except OSError:
+        print(f"'{args.target_path}': {os.strerror(e.errno)}")
+        sys.exit(e.errno)
+
+
 def do_extract(args):
     if not args.out:
         args.out = os.getcwd() + "/extracted"
 
     logger.debug(f"Extracting {args.file} to {args.out}")
-    from remarkable_update_fuse import UpdateImage
-
-    image = UpdateImage(args.file)
+    image, volume = get_update_image(args.file)
+    image.seek(0)
     with open(args.out, "wb") as f:
         f.write(image.read())
 
 
 def do_mount(args):
+    if sys.platform != "linux":
+        raise NotImplementedError(
+            f"Mounting has not been implemented on {sys.platform}"
+        )
+
     if args.out is None:
         args.out = "/opt/remarkable/"
 
@@ -625,8 +689,11 @@ def main():
     subparsers.add_parser(
         "restore", help="Restores to previous version installed on device"
     )
-
     subparsers.add_parser("list", help="List all versions available for use")
+    ls = subparsers.add_parser("ls", help="List files inside an update image")
+    cat = subparsers.add_parser(
+        "cat", help="Cat the contents of a file inside an update image"
+    )
 
     install.add_argument("version", help="Version to install")
     install.add_argument(
@@ -658,6 +725,7 @@ def main():
         help="Remote directory to upload to. Defaults to root folder",
         default="",
     )
+
     backup.add_argument(
         "-r",
         "--remote",
@@ -670,7 +738,6 @@ def main():
         help="Local directory to backup to. Defaults to download folder",
         default="./",
     )
-
     backup.add_argument(
         "-nr",
         "--no-recursion",
@@ -681,6 +748,16 @@ def main():
         "-no-ow", "--no-overwrite", help="Disables overwrite", action="store_true"
     )
 
+    ls.add_argument("file", help="Path to update file to extract", default=None)
+    ls.add_argument(
+        "target_path", help="Path inside the image to list", default=None, type=str
+    )
+
+    cat.add_argument("file", help="Path to update file to cat", default=None)
+    cat.add_argument(
+        "target_path", help="Path inside the image to list", default=None, type=str
+    )
+
     args = parser.parse_args()
     level = "ERROR"
 
@@ -735,6 +812,12 @@ def main():
     elif choice == "mount":
         do_mount(args)
 
+    elif choice == "ls":
+        do_ls(args)
+
+    elif choice == "cat":
+        do_cat(args)
+
 
 if __name__ == "__main__":
     main()

requirements.txt

@@ -1,3 +1,4 @@
 requests==2.31.0
 loguru==0.7.2
-remarkable-update-fuse==1.0.8; sys_platform == 'linux'
+remarkable-update-image==1.0.1
+remarkable-update-fuse==1.1.1; sys_platform == 'linux'