tests: use gojq - part 1

tests/core/auto-refresh-backoff-after-reboot/check_auto_refresh_count.sh

@@ -4,4 +4,4 @@ LAST_CHANGE_ID=$1
 CHANGES_COUNT=$2
 
 #shellcheck disable=SC2086,SC2046
-test $(snap debug api /v2/changes?select=ready | jq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | length") == $CHANGES_COUNT
+test $(snap debug api /v2/changes?select=ready | gojq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | length") == $CHANGES_COUNT

tests/core/auto-refresh-backoff-after-reboot/task.yaml

@@ -21,15 +21,11 @@ prepare: |
         exit
     fi
 
-    # Needed by make-snap-installable, Install before switching to fakestore
-    snap install jq
-    snap install remarshal
-
     # Prevent refreshes until we have right snap revisions
     snap set system refresh.hold=forever
 
     # Record last change id before we start to avoid flakiness due to auto-refreshes in other tests
-    snap debug api /v2/changes?select=all | jq '.result | sort_by(.id|tonumber) | .[-1].id' > last-change-id
+    snap debug api /v2/changes?select=all | gojq '.result | sort_by(.id|tonumber) | .[-1].id' > last-change-id
     # Record current snap revision for reference
     readlink "/snap/$SNAP_NAME/current" > snap.rev
 
@@ -58,7 +54,7 @@ restore: |
     snap set system refresh.hold!
 
 debug: |
-    snap debug api /v2/changes?select=ready | jq "[.result[] | select(.kind == \"auto-refresh\")] | sort_by(.id|tonumber)"
+    snap debug api /v2/changes?select=ready | gojq "[.result[] | select(.kind == \"auto-refresh\")] | sort_by(.id|tonumber)"
 
 execute: |
     if [ "$TRUST_TEST_KEYS" = "false" ]; then
@@ -169,17 +165,18 @@ execute: |
         retry -n 50 --wait 1 "$(pwd)"/check_auto_refresh_count.sh "$LAST_CHANGE_ID" 2
 
         echo "Check auto-refresh behaviour matches expectations for backoff algorithm"
-        snap debug api /v2/changes?select=ready | jq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | sort_by(.id|tonumber)" > changes.json
+        snap debug api /v2/changes?select=ready | \
+            gojq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | sort_by(.id|tonumber)" > changes.json
 
         # 1st auto-refresh
-        jq '.[0].status' < changes.json | MATCH "Error"
-        jq '.[0].data."snap-names" | length' < changes.json | MATCH "1"
-        jq '.[0].data."snap-names"' < changes.json | MATCH "$SNAP_NAME"
-        jq '.[0].data."refresh-failed"' < changes.json | MATCH "$SNAP_NAME"
+        gojq '.[0].status' < changes.json | MATCH "Error"
+        gojq '.[0].data."snap-names" | length' < changes.json | MATCH "1"
+        gojq '.[0].data."snap-names"' < changes.json | MATCH "$SNAP_NAME"
+        gojq '.[0].data."refresh-failed"' < changes.json | MATCH "$SNAP_NAME"
 
         # 2nd auto-refresh
-        jq '.[1].status' < changes.json | MATCH "Done"
-        jq '.[1].data."snap-names" | length' < changes.json | MATCH "1"
-        jq '.[1].data."snap-names"' < changes.json | MATCH "$SNAP_NAME"
-        jq '.[1].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_NAME"
+        gojq '.[1].status' < changes.json | MATCH "Done"
+        gojq '.[1].data."snap-names" | length' < changes.json | MATCH "1"
+        gojq '.[1].data."snap-names"' < changes.json | MATCH "$SNAP_NAME"
+        gojq '.[1].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_NAME"
     fi

tests/core/snap-repair/task.yaml

@@ -8,10 +8,6 @@ environment:
     BLOB_DIR: $(pwd)/fake-store-blobdir
     STORE_ADDR: localhost:11028
 
-prepare: |
-    snap install jq
-    tests.cleanup defer snap remove jq
-
 restore: |
     if [ "$TRUST_TEST_KEYS" = "false" ]; then
         echo "This test needs test keys to be trusted"
@@ -93,7 +89,7 @@ execute: |
 
         # note that the value "2" here _must_ be a string, otherwise we can't 
         # sign it as all values must be strings or lists of strings, etc. 
-        jq '."repair-id" = "2"' < "$PWD/$REPAIR_JSON" > "$PWD/$REPAIR_JSON.tmp"
+        gojq '."repair-id" = "2"' < "$PWD/$REPAIR_JSON" > "$PWD/$REPAIR_JSON.tmp"
         mv "$PWD/$REPAIR_JSON.tmp" "$PWD/2-$REPAIR_JSON"
 
         fakestore new-repair --dir "$BLOB_DIR" retry.sh --repair-json="$PWD/2-$REPAIR_JSON"
@@ -108,7 +104,7 @@ execute: |
 
         echo "Add a new repair ID 2 revision that completes successfully"
 
-        jq '."revision" = "1"' < "$PWD/2-$REPAIR_JSON" > "$PWD/2-$REPAIR_JSON.tmp"
+        gojq '."revision" = "1"' < "$PWD/2-$REPAIR_JSON" > "$PWD/2-$REPAIR_JSON.tmp"
         mv "$PWD/2-$REPAIR_JSON.tmp" "$PWD/2-$REPAIR_JSON"
 
         fakestore new-repair --dir "$BLOB_DIR" "$REPAIR_SCRIPT" --repair-json="$PWD/2-$REPAIR_JSON"

tests/core/snapd-maintenance-msg/task.yaml

@@ -7,16 +7,12 @@ details: |
 systems: [ubuntu-core-20-64]
 
 prepare: |
-    snap install jq
-
     # make sure that the snapd daemon gives us time for comms before
     # closing the socket
     echo "SNAPD_SHUTDOWN_DELAY=1" >> /etc/environment
     systemctl restart snapd
 
 restore: |
-    snap remove jq
-
     # remove SNAPD_SHUTDOWN_DELAY from /etc/environment again
     #shellcheck disable=SC2005
     echo "$(grep -v 'SNAPD_SHUTDOWN_DELAY=1' /etc/environment)" > /etc/environment
@@ -31,15 +27,15 @@ execute: |
         # closed so we need to catch it in that timeframe.
         echo "Testing maintenance message for daemon restarts"
         snap install --dangerous "$SNAPD_SNAP" &
-        retry -n 20 --wait 0.5 sh -c 'snap debug api '/v2/changes?select=all' | jq ".maintenance" | MATCH "daemon is restarting"'
+        retry -n 20 --wait 0.5 sh -c 'snap debug api '/v2/changes?select=all' | gojq ".maintenance" | MATCH "daemon is restarting"'
         wait
 
         echo "Restoring the snapd snap"
         snap revert snapd
 
         echo "Testing maintenance message for system reboots"
         snap refresh core20 --channel=stable --amend &
-        retry -n 20 --wait 0.5 sh -c 'snap debug api '/v2/changes?select=all' | jq ".maintenance" | MATCH "system is restarting"'
+        retry -n 20 --wait 0.5 sh -c 'snap debug api '/v2/changes?select=all' | gojq ".maintenance" | MATCH "system is restarting"'
         wait
 
         REBOOT

tests/core/snapd-refresh-vs-services/task.yaml

@@ -42,11 +42,8 @@ environment:
   SNAPD_2_49_2_ARMHF: https://storage.googleapis.com/snapd-spread-tests/snaps/snapd_2.49.2_11586.snap
 
 prepare: |
-  # install http snap to download files, jq + remarshal to simplify the check if
-  # stable == 2.49.2 so we can skip that case automatically until a new version
-  # is released to stable
+  # install http snap to download files
   snap install test-snapd-curl --edge --devmode # devmode so it can save to any dir
-  snap install jq remarshal
   # save the current version of snapd for later
   INITIAL_REV=$(snap list snapd | tail -n +2 | awk '{print $3}')
   cp "/var/lib/snapd/snaps/snapd_$INITIAL_REV.snap" snapd-pr.snap
@@ -62,7 +59,7 @@ prepare: |
 execute: |
   # check if snapd 2.49.2 is the current latest/stable release as it simplifies
   # some of the logic below
-  if snap info snapd | yaml2json | jq -r '.channels."latest/stable"'  | grep -q -Po '2.49.2\s+'; then
+  if snap info snapd | gojq --yaml-input -r '.channels."latest/stable"'  | grep -q -Po '2.49.2\s+'; then
     # skip the stable variant of the test
     if [ "${SNAPD_VERSION_UNDER_TEST}" = "stable" ]; then
       echo "Skipping duplicated test case"

tests/core/uc20-recovery/task.yaml

@@ -34,22 +34,20 @@ execute: |
     if [ "$SPREAD_REBOOT" == "0" ]; then
         echo "In run mode"
 
-        snap install --edge jq
-
         MATCH 'snapd_recovery_mode=run' < /proc/cmdline
         # verify we are in run mode via the API
         snap debug api '/v2/system-info' > system-info
-        jq -r '.result["system-mode"]' < system-info | MATCH 'run'
+        gojq -r '.result["system-mode"]' < system-info | MATCH 'run'
 
         echo "Obtain available systems"
         snap debug api '/v2/systems' > systems.json
         # TODO:UC20: there is only one system for now
-        jq .result.systems[0].current < systems.json | MATCH 'true'
-        label="$(jq -r .result.systems[0].label < systems.json)"
+        gojq .result.systems[0].current < systems.json | MATCH 'true'
+        label="$(gojq -r .result.systems[0].label < systems.json)"
         test -n "$label"
         # make sure that the seed exists
         test -d "/var/lib/snapd/seed/systems/$label"
-        jq -r .result.systems[0].actions[].mode < systems.json | sort | tr '\n' ' ' | MATCH 'install recover run'
+        gojq -r .result.systems[0].actions[].mode < systems.json | sort | tr '\n' ' ' | MATCH 'install recover run'
 
         # keep a copy of the systems dump for later reference
         cp systems.json /writable/systems.json.run
@@ -66,7 +64,7 @@ execute: |
         test -e /host/ubuntu-data/systems.json.run
 
         snap debug api '/v2/systems' > systems.json
-        jq -r .result.systems[0].actions[].mode < systems.json | sort | tr '\n' ' ' | MATCH 'install run'
+        gojq -r .result.systems[0].actions[].mode < systems.json | sort | tr '\n' ' ' | MATCH 'install run'
 
         label="$(cat /host/ubuntu-data/systems.label)"
         test -n "$label"
@@ -79,11 +77,11 @@ execute: |
     elif [ "$SPREAD_REBOOT" == "2" ]; then
         echo "In run mode again"
         snap debug api '/v2/system-info' > system-info
-        jq -r '.result["system-mode"]' < system-info | MATCH 'run'
+        gojq -r '.result["system-mode"]' < system-info | MATCH 'run'
 
         # now go back to recover mode so we can test that a simple reboot 
         # works to transition us back to run mode
-        label="$(jq -r .result.systems[0].label < systems.json)"
+        label="$(gojq -r .result.systems[0].label < systems.json)"
         transition_to_recover_mode "$label"
     elif [ "$SPREAD_REBOOT" == "3" ]; then
         echo "In recover mode again"
@@ -107,5 +105,5 @@ execute: |
     elif [ "$SPREAD_REBOOT" == "4" ]; then
         echo "In run mode again again"
         snap debug api '/v2/system-info' > system-info
-        jq -r '.result["system-mode"]' < system-info | MATCH 'run'
+        gojq -r '.result["system-mode"]' < system-info | MATCH 'run'
     fi

tests/lib/nested.sh

@@ -299,7 +299,7 @@ nested_get_snap_rev_for_channel() {
          -H "Content-Type: application/json" \
          --data "{\"context\": [], \"actions\": [{\"action\": \"install\", \"name\": \"$SNAP\", \"channel\": \"$CHANNEL\", \"instance-key\": \"1\"}]}" \
          https://api.snapcraft.io/v2/snaps/refresh | \
-        jq '.results[0].snap.revision'
+        gojq '.results[0].snap.revision'
 }
 
 nested_is_nested_system() {

tests/lib/prepare-restore.sh

@@ -608,6 +608,15 @@ prepare_project() {
         disable_journald_rate_limiting
         disable_journald_start_limiting
     fi
+
+    # native jq replacement, but still with some incompatibilies, see
+    # https://github.com/itchyny/gojq
+    # major differences:
+    # - map keys are sorted by default
+    # - with --yaml-input, can parse YAML
+    GOBIN=$PROJECT_PATH/tests/bin \
+    CGO_ENABLED=0 \
+        go install github.com/itchyny/gojq/cmd/[email protected]
 }
 
 prepare_project_each() {

tests/lib/prepare.sh

@@ -97,9 +97,6 @@ ensure_jq() {
 }
 
 disable_refreshes() {
-    echo "Ensure jq is available"
-    ensure_jq
-
     echo "Modify state to make it look like the last refresh just happened"
     systemctl stop snapd.socket snapd.service
     "$TESTSTOOLS"/snapd-state prevent-autorefresh
@@ -108,13 +105,6 @@ disable_refreshes() {
     echo "Minimize risk of hitting refresh schedule"
     snap set core refresh.schedule=00:00-23:59
     snap refresh --time --abs-time | MATCH "last: 2[0-9]{3}"
-
-    echo "Ensure jq is gone"
-    snap remove --purge jq
-    snap remove --purge jq-core18
-    snap remove --purge jq-core20
-    snap remove --purge jq-core22
-    snap remove --purge test-snapd-jq-core24
 }
 
 setup_systemd_snapd_overrides() {

tests/lib/tools/snapd-state

@@ -19,7 +19,7 @@ print_state() {
         echo "snapd-state: jq-filter is a required parameter"
         exit 1
     fi
-    jq -r "$JQ_FILTER" < /var/lib/snapd/state.json
+    gojq -r "$JQ_FILTER" < /var/lib/snapd/state.json
 }
 
 check_state() {
@@ -56,17 +56,17 @@ change_snap_channel() {
         echo "snapd-state: snap and channel are required parameters"
         exit 1
     fi
-    jq ".data.snaps[\"$SNAP\"].channel = \"$CHANNEL\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
+    gojq ".data.snaps[\"$SNAP\"].channel = \"$CHANNEL\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
     mv /var/lib/snapd/state.json.new /var/lib/snapd/state.json
 }
 
 force_autorefresh() {
-    jq ".data[\"last-refresh\"] = \"2007-08-22T09:30:44.449455783+01:00\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
+    gojq ".data[\"last-refresh\"] = \"2007-08-22T09:30:44.449455783+01:00\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
     mv /var/lib/snapd/state.json.new /var/lib/snapd/state.json
 }
 
 prevent_autorefresh() {
-    jq ".data[\"last-refresh\"] = \"$(date +%Y-%m-%dT%H:%M:%S%:z)\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
+    gojq ".data[\"last-refresh\"] = \"$(date +%Y-%m-%dT%H:%M:%S%:z)\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
     mv /var/lib/snapd/state.json.new /var/lib/snapd/state.json
 }
 

tests/lib/tools/store-state

@@ -73,21 +73,9 @@ make_snap_installable(){
     local snap_id="${3:-}"
 
     if [ -n "$snap_id" ]; then
-        if ! command -v yaml2json; then
-            # FIXME: When fakestore is setup this snap cannot be installed
-            # TODO: Install remarshal in setup_fake_store
-            snap install remarshal
-        fi
-        if ! command -v jq; then
-            # FIXME: When fakestore is setup this snap cannot be installed
-            # TODO: Install jq in setup_fake_store or don't use snapped jq
-            SUFFIX="$(snaps.name snap-suffix)"
-            snap install "jq$SUFFIX"
-        fi
-
         # unsquash the snap to get its name
         unsquashfs -d /tmp/snap-squashfs "$snap_path" meta/snap.yaml
-        snap_name=$(yaml2json < /tmp/snap-squashfs/meta/snap.yaml | jq -r .name)
+        snap_name=$(gojq --yaml-input -r '.name' < /tmp/snap-squashfs/meta/snap.yaml)
         rm -rf /tmp/snap-squashfs
 
         cat >> /tmp/snap-decl.json << EOF
@@ -102,7 +90,7 @@ EOF
         if [ -n "$extra_decl_json_file" ]; then
             # then we need to combine the extra snap declaration json with the one
             # we just wrote
-            jq -s '.[0] * .[1]' <(cat /tmp/snap-decl.json) <(cat "$extra_decl_json_file") > /tmp/snap-decl.json.tmp
+            gojq -s '.[0] * .[1]' <(cat /tmp/snap-decl.json) <(cat "$extra_decl_json_file") > /tmp/snap-decl.json.tmp
             mv /tmp/snap-decl.json.tmp /tmp/snap-decl.json
         fi
 

tests/main/api-get-systems-label/task.yaml

@@ -8,27 +8,25 @@ systems:
   - ubuntu-core-2*
 
 execute: |
-  snap install --edge jq
-
   echo "Find what systems are available"
   snap debug api /v2/systems > systems
-  current_label=$(jq -r '.result.systems[0]["label"]' < systems)
+  current_label=$(gojq -r '.result.systems[0]["label"]' < systems)
 
   echo "Get details for a specific system"
   snap debug api "/v2/systems/$current_label" > current-system
   echo "Ensure the result contains a model assertion"
-  jq -r '.result.model.type' < current-system | MATCH model
-  jq -r '.result.model.series' < current-system | MATCH 16
-  jq -r '.result.model.base' < current-system | MATCH "core[0-9][0-9]"
+  gojq -r '.result.model.type' < current-system | MATCH model
+  gojq -r '.result.model.series' < current-system | MATCH 16
+  gojq -r '.result.model.base' < current-system | MATCH "core[0-9][0-9]"
   echo "Ensure the result looks like a systems reply"
-  jq -r '.result.brand.id' < current-system | MATCH "$(snap model --verbose|awk '/brand-id:/ {print $2}')"
-  jq -r '.result.brand.validation' < current-system | MATCH '(verified|unproven|starred)'
-  jq -r '.result.label' < current-system | MATCH "$current_label"
-  jq -r '.result.current' < current-system | MATCH '(true|false)'
+  gojq -r '.result.brand.id' < current-system | MATCH "$(snap model --verbose|awk '/brand-id:/ {print $2}')"
+  gojq -r '.result.brand.validation' < current-system | MATCH '(verified|unproven|starred)'
+  gojq -r '.result.label' < current-system | MATCH "$current_label"
+  gojq -r '.result.current' < current-system | MATCH '(true|false)'
   # we expect at least one current action to be available and
   # each action always has a mode
-  jq -r '.result.actions[0]' < current-system | MATCH 'mode'
+  gojq -r '.result.actions[0]' < current-system | MATCH 'mode'
   echo "Ensure the result contains the gadget volumes"
-  jq -r '.result.volumes' < current-system | MATCH bootloader
+  gojq -r '.result.volumes' < current-system | MATCH bootloader
   # internal fields are not exported
-  jq -r '.result.volumes' < current-system | NOMATCH VolumeName
+  gojq -r '.result.volumes' < current-system | NOMATCH VolumeName

tests/main/apparmor-prompting-flag-restart/task.yaml

@@ -14,7 +14,6 @@ systems:
   - ubuntu-core-*
 
 prepare: |
-    snap install jq
     # prerequisite for having a prompts handler service
     snap set system experimental.user-daemons=true
     "$TESTSTOOLS"/snaps-state install-local test-snapd-prompt-handler
@@ -93,9 +92,9 @@ execute: |
         echo "Check that snap CLI reports prompting flag set correctly"
         snap get system experimental.apparmor-prompting | MATCH "$1"
         echo "Check that /v2/snaps/system/conf reports prompting flag set correctly"
-        snap debug api /v2/snaps/system/conf | jq -r '.result.experimental."apparmor-prompting"' | MATCH "$1"
+        snap debug api /v2/snaps/system/conf | gojq -r '.result.experimental."apparmor-prompting"' | MATCH "$1"
         echo "Check that /v2/system-info reports prompting correctly"
-        snap debug api /v2/system-info | jq -r '.result.features."apparmor-prompting".enabled' | MATCH "$1"
+        snap debug api /v2/system-info | gojq -r '.result.features."apparmor-prompting".enabled' | MATCH "$1"
     }
 
     echo "Precondition check that snapd is active"
@@ -140,14 +139,18 @@ execute: |
 
     echo "Enable prompting via API request"
 
-    echo '{"experimental.apparmor-prompting": true}' | snap debug api -X PUT -H 'Content-Type: application/json' /v2/snaps/system/conf | jq -r '.status' | MATCH "Accepted" || reset_start_limit
+    echo '{"experimental.apparmor-prompting": true}' | \
+        snap debug api -X PUT -H 'Content-Type: application/json' /v2/snaps/system/conf | \
+        gojq -r '.status' | MATCH "Accepted" || reset_start_limit
 
     echo "Check that snapd restarted after prompting set to true via api"
     check_snapd_restarted
     check_prompting_setting "true"
 
     echo "Disable prompting via API request"
-    echo '{"experimental.apparmor-prompting": false}' | snap debug api -X PUT -H 'Content-Type: application/json' /v2/snaps/system/conf | jq -r '.status' | MATCH "Accepted" || reset_start_limit
+    echo '{"experimental.apparmor-prompting": false}' | \
+        snap debug api -X PUT -H 'Content-Type: application/json' /v2/snaps/system/conf | \
+        gojq -r '.status' | MATCH "Accepted" || reset_start_limit
 
     echo "Check that snapd restarted after prompting set to false via api"
     check_snapd_restarted