Revert "tests: use gojq - part 1 (canonical#14686)"

This reverts commit ede0457.

tests/core/auto-refresh-backoff-after-reboot/check_auto_refresh_count.sh

@@ -4,4 +4,4 @@ LAST_CHANGE_ID=$1
 CHANGES_COUNT=$2
 
 #shellcheck disable=SC2086,SC2046
-test $(snap debug api /v2/changes?select=ready | gojq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | length") == $CHANGES_COUNT
+test $(snap debug api /v2/changes?select=ready | jq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | length") == $CHANGES_COUNT

tests/core/auto-refresh-backoff-after-reboot/task.yaml

@@ -21,11 +21,15 @@ prepare: |
         exit
     fi
 
+    # Needed by make-snap-installable, Install before switching to fakestore
+    snap install jq
+    snap install remarshal
+
     # Prevent refreshes until we have right snap revisions
     snap set system refresh.hold=forever
 
     # Record last change id before we start to avoid flakiness due to auto-refreshes in other tests
-    snap debug api /v2/changes?select=all | gojq '.result | sort_by(.id|tonumber) | .[-1].id' > last-change-id
+    snap debug api /v2/changes?select=all | jq '.result | sort_by(.id|tonumber) | .[-1].id' > last-change-id
     # Record current snap revision for reference
     readlink "/snap/$SNAP_NAME/current" > snap.rev
 
@@ -54,7 +58,7 @@ restore: |
     snap set system refresh.hold!
 
 debug: |
-    snap debug api /v2/changes?select=ready | gojq "[.result[] | select(.kind == \"auto-refresh\")] | sort_by(.id|tonumber)"
+    snap debug api /v2/changes?select=ready | jq "[.result[] | select(.kind == \"auto-refresh\")] | sort_by(.id|tonumber)"
 
 execute: |
     if [ "$TRUST_TEST_KEYS" = "false" ]; then
@@ -165,18 +169,17 @@ execute: |
         retry -n 50 --wait 1 "$(pwd)"/check_auto_refresh_count.sh "$LAST_CHANGE_ID" 2
 
         echo "Check auto-refresh behaviour matches expectations for backoff algorithm"
-        snap debug api /v2/changes?select=ready | \
-            gojq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | sort_by(.id|tonumber)" > changes.json
+        snap debug api /v2/changes?select=ready | jq "[.result[] | select(.kind == \"auto-refresh\" and (.id|tonumber) > ($LAST_CHANGE_ID|tonumber))] | sort_by(.id|tonumber)" > changes.json
 
         # 1st auto-refresh
-        gojq '.[0].status' < changes.json | MATCH "Error"
-        gojq '.[0].data."snap-names" | length' < changes.json | MATCH "1"
-        gojq '.[0].data."snap-names"' < changes.json | MATCH "$SNAP_NAME"
-        gojq '.[0].data."refresh-failed"' < changes.json | MATCH "$SNAP_NAME"
+        jq '.[0].status' < changes.json | MATCH "Error"
+        jq '.[0].data."snap-names" | length' < changes.json | MATCH "1"
+        jq '.[0].data."snap-names"' < changes.json | MATCH "$SNAP_NAME"
+        jq '.[0].data."refresh-failed"' < changes.json | MATCH "$SNAP_NAME"
 
         # 2nd auto-refresh
-        gojq '.[1].status' < changes.json | MATCH "Done"
-        gojq '.[1].data."snap-names" | length' < changes.json | MATCH "1"
-        gojq '.[1].data."snap-names"' < changes.json | MATCH "$SNAP_NAME"
-        gojq '.[1].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_NAME"
+        jq '.[1].status' < changes.json | MATCH "Done"
+        jq '.[1].data."snap-names" | length' < changes.json | MATCH "1"
+        jq '.[1].data."snap-names"' < changes.json | MATCH "$SNAP_NAME"
+        jq '.[1].data."refresh-failed"' < changes.json | NOMATCH "$SNAP_NAME"
     fi

tests/core/snap-repair/task.yaml

@@ -8,6 +8,10 @@ environment:
     BLOB_DIR: $(pwd)/fake-store-blobdir
     STORE_ADDR: localhost:11028
 
+prepare: |
+    snap install jq
+    tests.cleanup defer snap remove jq
+
 restore: |
     if [ "$TRUST_TEST_KEYS" = "false" ]; then
         echo "This test needs test keys to be trusted"
@@ -89,7 +93,7 @@ execute: |
 
         # note that the value "2" here _must_ be a string, otherwise we can't 
         # sign it as all values must be strings or lists of strings, etc. 
-        gojq '."repair-id" = "2"' < "$PWD/$REPAIR_JSON" > "$PWD/$REPAIR_JSON.tmp"
+        jq '."repair-id" = "2"' < "$PWD/$REPAIR_JSON" > "$PWD/$REPAIR_JSON.tmp"
         mv "$PWD/$REPAIR_JSON.tmp" "$PWD/2-$REPAIR_JSON"
 
         fakestore new-repair --dir "$BLOB_DIR" retry.sh --repair-json="$PWD/2-$REPAIR_JSON"
@@ -104,7 +108,7 @@ execute: |
 
         echo "Add a new repair ID 2 revision that completes successfully"
 
-        gojq '."revision" = "1"' < "$PWD/2-$REPAIR_JSON" > "$PWD/2-$REPAIR_JSON.tmp"
+        jq '."revision" = "1"' < "$PWD/2-$REPAIR_JSON" > "$PWD/2-$REPAIR_JSON.tmp"
         mv "$PWD/2-$REPAIR_JSON.tmp" "$PWD/2-$REPAIR_JSON"
 
         fakestore new-repair --dir "$BLOB_DIR" "$REPAIR_SCRIPT" --repair-json="$PWD/2-$REPAIR_JSON"

tests/core/snapd-maintenance-msg/task.yaml

@@ -7,12 +7,16 @@ details: |
 systems: [ubuntu-core-20-64]
 
 prepare: |
+    snap install jq
+
     # make sure that the snapd daemon gives us time for comms before
     # closing the socket
     echo "SNAPD_SHUTDOWN_DELAY=1" >> /etc/environment
     systemctl restart snapd
 
 restore: |
+    snap remove jq
+
     # remove SNAPD_SHUTDOWN_DELAY from /etc/environment again
     #shellcheck disable=SC2005
     echo "$(grep -v 'SNAPD_SHUTDOWN_DELAY=1' /etc/environment)" > /etc/environment
@@ -27,15 +31,15 @@ execute: |
         # closed so we need to catch it in that timeframe.
         echo "Testing maintenance message for daemon restarts"
         snap install --dangerous "$SNAPD_SNAP" &
-        retry -n 20 --wait 0.5 sh -c 'snap debug api '/v2/changes?select=all' | gojq ".maintenance" | MATCH "daemon is restarting"'
+        retry -n 20 --wait 0.5 sh -c 'snap debug api '/v2/changes?select=all' | jq ".maintenance" | MATCH "daemon is restarting"'
         wait
 
         echo "Restoring the snapd snap"
         snap revert snapd
 
         echo "Testing maintenance message for system reboots"
         snap refresh core20 --channel=stable --amend &
-        retry -n 20 --wait 0.5 sh -c 'snap debug api '/v2/changes?select=all' | gojq ".maintenance" | MATCH "system is restarting"'
+        retry -n 20 --wait 0.5 sh -c 'snap debug api '/v2/changes?select=all' | jq ".maintenance" | MATCH "system is restarting"'
         wait
 
         REBOOT

tests/core/snapd-refresh-vs-services/task.yaml

@@ -42,8 +42,11 @@ environment:
   SNAPD_2_49_2_ARMHF: https://storage.googleapis.com/snapd-spread-tests/snaps/snapd_2.49.2_11586.snap
 
 prepare: |
-  # install http snap to download files
+  # install http snap to download files, jq + remarshal to simplify the check if
+  # stable == 2.49.2 so we can skip that case automatically until a new version
+  # is released to stable
   snap install test-snapd-curl --edge --devmode # devmode so it can save to any dir
+  snap install jq remarshal
   # save the current version of snapd for later
   INITIAL_REV=$(snap list snapd | tail -n +2 | awk '{print $3}')
   cp "/var/lib/snapd/snaps/snapd_$INITIAL_REV.snap" snapd-pr.snap
@@ -59,7 +62,7 @@ prepare: |
 execute: |
   # check if snapd 2.49.2 is the current latest/stable release as it simplifies
   # some of the logic below
-  if snap info snapd | gojq --yaml-input -r '.channels."latest/stable"'  | grep -q -Po '2.49.2\s+'; then
+  if snap info snapd | yaml2json | jq -r '.channels."latest/stable"'  | grep -q -Po '2.49.2\s+'; then
     # skip the stable variant of the test
     if [ "${SNAPD_VERSION_UNDER_TEST}" = "stable" ]; then
       echo "Skipping duplicated test case"

tests/core/uc20-recovery/task.yaml

@@ -34,20 +34,22 @@ execute: |
     if [ "$SPREAD_REBOOT" == "0" ]; then
         echo "In run mode"
 
+        snap install --edge jq
+
         MATCH 'snapd_recovery_mode=run' < /proc/cmdline
         # verify we are in run mode via the API
         snap debug api '/v2/system-info' > system-info
-        gojq -r '.result["system-mode"]' < system-info | MATCH 'run'
+        jq -r '.result["system-mode"]' < system-info | MATCH 'run'
 
         echo "Obtain available systems"
         snap debug api '/v2/systems' > systems.json
         # TODO:UC20: there is only one system for now
-        gojq .result.systems[0].current < systems.json | MATCH 'true'
-        label="$(gojq -r .result.systems[0].label < systems.json)"
+        jq .result.systems[0].current < systems.json | MATCH 'true'
+        label="$(jq -r .result.systems[0].label < systems.json)"
         test -n "$label"
         # make sure that the seed exists
         test -d "/var/lib/snapd/seed/systems/$label"
-        gojq -r .result.systems[0].actions[].mode < systems.json | sort | tr '\n' ' ' | MATCH 'install recover run'
+        jq -r .result.systems[0].actions[].mode < systems.json | sort | tr '\n' ' ' | MATCH 'install recover run'
 
         # keep a copy of the systems dump for later reference
         cp systems.json /writable/systems.json.run
@@ -64,7 +66,7 @@ execute: |
         test -e /host/ubuntu-data/systems.json.run
 
         snap debug api '/v2/systems' > systems.json
-        gojq -r .result.systems[0].actions[].mode < systems.json | sort | tr '\n' ' ' | MATCH 'install run'
+        jq -r .result.systems[0].actions[].mode < systems.json | sort | tr '\n' ' ' | MATCH 'install run'
 
         label="$(cat /host/ubuntu-data/systems.label)"
         test -n "$label"
@@ -77,11 +79,11 @@ execute: |
     elif [ "$SPREAD_REBOOT" == "2" ]; then
         echo "In run mode again"
         snap debug api '/v2/system-info' > system-info
-        gojq -r '.result["system-mode"]' < system-info | MATCH 'run'
+        jq -r '.result["system-mode"]' < system-info | MATCH 'run'
 
         # now go back to recover mode so we can test that a simple reboot 
         # works to transition us back to run mode
-        label="$(gojq -r .result.systems[0].label < systems.json)"
+        label="$(jq -r .result.systems[0].label < systems.json)"
         transition_to_recover_mode "$label"
     elif [ "$SPREAD_REBOOT" == "3" ]; then
         echo "In recover mode again"
@@ -105,5 +107,5 @@ execute: |
     elif [ "$SPREAD_REBOOT" == "4" ]; then
         echo "In run mode again again"
         snap debug api '/v2/system-info' > system-info
-        gojq -r '.result["system-mode"]' < system-info | MATCH 'run'
+        jq -r '.result["system-mode"]' < system-info | MATCH 'run'
     fi

tests/lib/nested.sh

@@ -299,7 +299,7 @@ nested_get_snap_rev_for_channel() {
          -H "Content-Type: application/json" \
          --data "{\"context\": [], \"actions\": [{\"action\": \"install\", \"name\": \"$SNAP\", \"channel\": \"$CHANNEL\", \"instance-key\": \"1\"}]}" \
          https://api.snapcraft.io/v2/snaps/refresh | \
-        gojq '.results[0].snap.revision'
+        jq '.results[0].snap.revision'
 }
 
 nested_is_nested_system() {

tests/lib/prepare-restore.sh

@@ -608,15 +608,6 @@ prepare_project() {
         disable_journald_rate_limiting
         disable_journald_start_limiting
     fi
-
-    # native jq replacement, but still with some incompatibilies, see
-    # https://github.com/itchyny/gojq
-    # major differences:
-    # - map keys are sorted by default
-    # - with --yaml-input, can parse YAML
-    GOBIN=$PROJECT_PATH/tests/bin \
-    CGO_ENABLED=0 \
-        go install github.com/itchyny/gojq/cmd/[email protected]
 }
 
 prepare_project_each() {

tests/lib/prepare.sh

@@ -97,6 +97,9 @@ ensure_jq() {
 }
 
 disable_refreshes() {
+    echo "Ensure jq is available"
+    ensure_jq
+
     echo "Modify state to make it look like the last refresh just happened"
     systemctl stop snapd.socket snapd.service
     "$TESTSTOOLS"/snapd-state prevent-autorefresh
@@ -105,6 +108,13 @@ disable_refreshes() {
     echo "Minimize risk of hitting refresh schedule"
     snap set core refresh.schedule=00:00-23:59
     snap refresh --time --abs-time | MATCH "last: 2[0-9]{3}"
+
+    echo "Ensure jq is gone"
+    snap remove --purge jq
+    snap remove --purge jq-core18
+    snap remove --purge jq-core20
+    snap remove --purge jq-core22
+    snap remove --purge test-snapd-jq-core24
 }
 
 setup_systemd_snapd_overrides() {

tests/lib/tools/snapd-state

@@ -19,7 +19,7 @@ print_state() {
         echo "snapd-state: jq-filter is a required parameter"
         exit 1
     fi
-    gojq -r "$JQ_FILTER" < /var/lib/snapd/state.json
+    jq -r "$JQ_FILTER" < /var/lib/snapd/state.json
 }
 
 check_state() {
@@ -56,17 +56,17 @@ change_snap_channel() {
         echo "snapd-state: snap and channel are required parameters"
         exit 1
     fi
-    gojq ".data.snaps[\"$SNAP\"].channel = \"$CHANNEL\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
+    jq ".data.snaps[\"$SNAP\"].channel = \"$CHANNEL\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
     mv /var/lib/snapd/state.json.new /var/lib/snapd/state.json
 }
 
 force_autorefresh() {
-    gojq ".data[\"last-refresh\"] = \"2007-08-22T09:30:44.449455783+01:00\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
+    jq ".data[\"last-refresh\"] = \"2007-08-22T09:30:44.449455783+01:00\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
     mv /var/lib/snapd/state.json.new /var/lib/snapd/state.json
 }
 
 prevent_autorefresh() {
-    gojq ".data[\"last-refresh\"] = \"$(date +%Y-%m-%dT%H:%M:%S%:z)\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
+    jq ".data[\"last-refresh\"] = \"$(date +%Y-%m-%dT%H:%M:%S%:z)\"" < /var/lib/snapd/state.json > /var/lib/snapd/state.json.new
     mv /var/lib/snapd/state.json.new /var/lib/snapd/state.json
 }
 

tests/lib/tools/store-state

@@ -73,9 +73,21 @@ make_snap_installable(){
     local snap_id="${3:-}"
 
     if [ -n "$snap_id" ]; then
+        if ! command -v yaml2json; then
+            # FIXME: When fakestore is setup this snap cannot be installed
+            # TODO: Install remarshal in setup_fake_store
+            snap install remarshal
+        fi
+        if ! command -v jq; then
+            # FIXME: When fakestore is setup this snap cannot be installed
+            # TODO: Install jq in setup_fake_store or don't use snapped jq
+            SUFFIX="$(snaps.name snap-suffix)"
+            snap install "jq$SUFFIX"
+        fi
+
         # unsquash the snap to get its name
         unsquashfs -d /tmp/snap-squashfs "$snap_path" meta/snap.yaml
-        snap_name=$(gojq --yaml-input -r '.name' < /tmp/snap-squashfs/meta/snap.yaml)
+        snap_name=$(yaml2json < /tmp/snap-squashfs/meta/snap.yaml | jq -r .name)
         rm -rf /tmp/snap-squashfs
 
         cat >> /tmp/snap-decl.json << EOF
@@ -90,7 +102,7 @@ EOF
         if [ -n "$extra_decl_json_file" ]; then
             # then we need to combine the extra snap declaration json with the one
             # we just wrote
-            gojq -s '.[0] * .[1]' <(cat /tmp/snap-decl.json) <(cat "$extra_decl_json_file") > /tmp/snap-decl.json.tmp
+            jq -s '.[0] * .[1]' <(cat /tmp/snap-decl.json) <(cat "$extra_decl_json_file") > /tmp/snap-decl.json.tmp
             mv /tmp/snap-decl.json.tmp /tmp/snap-decl.json
         fi
 

tests/main/api-get-systems-label/task.yaml

@@ -8,25 +8,27 @@ systems:
   - ubuntu-core-2*
 
 execute: |
+  snap install --edge jq
+
   echo "Find what systems are available"
   snap debug api /v2/systems > systems
-  current_label=$(gojq -r '.result.systems[0]["label"]' < systems)
+  current_label=$(jq -r '.result.systems[0]["label"]' < systems)
 
   echo "Get details for a specific system"
   snap debug api "/v2/systems/$current_label" > current-system
   echo "Ensure the result contains a model assertion"
-  gojq -r '.result.model.type' < current-system | MATCH model
-  gojq -r '.result.model.series' < current-system | MATCH 16
-  gojq -r '.result.model.base' < current-system | MATCH "core[0-9][0-9]"
+  jq -r '.result.model.type' < current-system | MATCH model
+  jq -r '.result.model.series' < current-system | MATCH 16
+  jq -r '.result.model.base' < current-system | MATCH "core[0-9][0-9]"
   echo "Ensure the result looks like a systems reply"
-  gojq -r '.result.brand.id' < current-system | MATCH "$(snap model --verbose|awk '/brand-id:/ {print $2}')"
-  gojq -r '.result.brand.validation' < current-system | MATCH '(verified|unproven|starred)'
-  gojq -r '.result.label' < current-system | MATCH "$current_label"
-  gojq -r '.result.current' < current-system | MATCH '(true|false)'
+  jq -r '.result.brand.id' < current-system | MATCH "$(snap model --verbose|awk '/brand-id:/ {print $2}')"
+  jq -r '.result.brand.validation' < current-system | MATCH '(verified|unproven|starred)'
+  jq -r '.result.label' < current-system | MATCH "$current_label"
+  jq -r '.result.current' < current-system | MATCH '(true|false)'
   # we expect at least one current action to be available and
   # each action always has a mode
-  gojq -r '.result.actions[0]' < current-system | MATCH 'mode'
+  jq -r '.result.actions[0]' < current-system | MATCH 'mode'
   echo "Ensure the result contains the gadget volumes"
-  gojq -r '.result.volumes' < current-system | MATCH bootloader
+  jq -r '.result.volumes' < current-system | MATCH bootloader
   # internal fields are not exported
-  gojq -r '.result.volumes' < current-system | NOMATCH VolumeName
+  jq -r '.result.volumes' < current-system | NOMATCH VolumeName

tests/main/apparmor-prompting-flag-restart/task.yaml

@@ -14,6 +14,7 @@ systems:
   - ubuntu-core-*
 
 prepare: |
+    snap install jq
     # prerequisite for having a prompts handler service
     snap set system experimental.user-daemons=true
     "$TESTSTOOLS"/snaps-state install-local test-snapd-prompt-handler
@@ -92,9 +93,9 @@ execute: |
         echo "Check that snap CLI reports prompting flag set correctly"
         snap get system experimental.apparmor-prompting | MATCH "$1"
         echo "Check that /v2/snaps/system/conf reports prompting flag set correctly"
-        snap debug api /v2/snaps/system/conf | gojq -r '.result.experimental."apparmor-prompting"' | MATCH "$1"
+        snap debug api /v2/snaps/system/conf | jq -r '.result.experimental."apparmor-prompting"' | MATCH "$1"
         echo "Check that /v2/system-info reports prompting correctly"
-        snap debug api /v2/system-info | gojq -r '.result.features."apparmor-prompting".enabled' | MATCH "$1"
+        snap debug api /v2/system-info | jq -r '.result.features."apparmor-prompting".enabled' | MATCH "$1"
     }
 
     echo "Precondition check that snapd is active"
@@ -139,18 +140,14 @@ execute: |
 
     echo "Enable prompting via API request"
 
-    echo '{"experimental.apparmor-prompting": true}' | \
-        snap debug api -X PUT -H 'Content-Type: application/json' /v2/snaps/system/conf | \
-        gojq -r '.status' | MATCH "Accepted" || reset_start_limit
+    echo '{"experimental.apparmor-prompting": true}' | snap debug api -X PUT -H 'Content-Type: application/json' /v2/snaps/system/conf | jq -r '.status' | MATCH "Accepted" || reset_start_limit
 
     echo "Check that snapd restarted after prompting set to true via api"
     check_snapd_restarted
     check_prompting_setting "true"
 
     echo "Disable prompting via API request"
-    echo '{"experimental.apparmor-prompting": false}' | \
-        snap debug api -X PUT -H 'Content-Type: application/json' /v2/snaps/system/conf | \
-        gojq -r '.status' | MATCH "Accepted" || reset_start_limit
+    echo '{"experimental.apparmor-prompting": false}' | snap debug api -X PUT -H 'Content-Type: application/json' /v2/snaps/system/conf | jq -r '.status' | MATCH "Accepted" || reset_start_limit
 
     echo "Check that snapd restarted after prompting set to false via api"
     check_snapd_restarted