NearNodeFlash · ajfloeder · Aug 14, 2024 · Aug 14, 2024
diff --git a/tools/collectNVLogs.sh b/tools/collectNVLogs.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2024 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# set -e
+# set -o xtrace
+
+shopt -s expand_aliases
+
+logSources=("FLASH" "MEMLOG" "REGS" "THRD_STACK" "SYS_STACK" "THRDS" "NVHDR" "RAM")
+switches=("/dev/switchtec0" "/dev/switchtec1")
+
+for switch in "${switches[@]}"; do
+    PAX_ID=$(switchtec fabric gfms-dump "$switch" | grep "^PAX ID:" | awk '{print $3}')
+    if ! (( PAX_ID >= 0 && PAX_ID <= 1 )); then
+        echo "$PAX_ID not in range 0-1"
+        exit 1
+    fi
+
+    for logSource in "${logSources[@]}"; do
+        echo pax"$PAX_ID"-"$logSource".log
+        switchtec log-dump "$switch" pax"$PAX_ID"-"$logSource".log --type="$logSource"
+    done
+done
diff --git a/tools/rabbit-s.sh b/tools/rabbit-s.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2020-2023 Hewlett Packard Enterprise Development LP
+# Copyright 2020-2024 Hewlett Packard Enterprise Development LP
 # Other additional copyright holders may be indicated within.
 #
 # The entirety of this work is licensed under the Apache License,
@@ -181,6 +181,46 @@ EOF
 EOF
         done
         ;;
+    hpecray-29)
+        for SESSION in "${SESSIONS[@]}"
+        do
+
+            # For this run, we want to enable some logging settings that we have been leaving off for a while.
+            # FABIOV - 0x82
+            #     log -m 0x82 -s 3
+            #     log -m 0x82 -s 3 -p on
+            # PTD - 0x53
+            #     log -m 0x53 -s 3
+            #     log -m 0x53 -s 3 -p on
+            # fabdbg -s pax
+            # fabdbg -s fio
+            # fabdbg -s gfms
+
+            $SSHPASS ssh root@$SYSTEM <<-EOF
+            screen -S $SESSION -X stuff "fabdbg -s pax\nfabdbg -s fio\nfabdbg -s gfms\nlog -m 0x82 -s 3\nlog -m 0x82 -s 3 -p on\nlog -m 0x53 -s 3\nlog -m 0x53 -s 3 -p on\n"
+EOF
+        done
+        ;;
+    hpecray-32)
+        for SESSION in "${SESSIONS[@]}"
+        do
+            # Enables medium severity and turns on logging for the PSC module
+            # log -m 0x54 -s 3
+            # log -m 0x54 -s 3 -p on
+
+            # Enables logs for the fabric debug modules
+            # fabdbg -s pax
+            # fabdbg -s fio
+            # fabdbg -s gfms
+
+            # Turn on the logging for all modules
+            # log -p on
+            $SSHPASS ssh root@$SYSTEM <<-EOF
+            screen -S $SESSION -X stuff "fabdbg -s pax\nfabdbg -s fio\nfabdbg -s gfms\nlog -m 0x54 -s 3\nlog -m 0x54 -s 3 -p on\nlog -p on\n"
+EOF
+        done
+        ;;
+
     quit-sessions)
         for SESSION in "${SESSIONS[@]}"
         do

diff --git a/tools/show-drive-ports.sh b/tools/show-drive-ports.sh
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+# Copyright 2024 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+usage() {
+    cat <<EOF
+Examine "lspci" status and report failing drive links
+Usage: $0 [-h] [-v]
+
+Arguments:
+  -v                verbose output
+  -h                display this help
+
+Examples:
+  $0 -v
+EOF
+}
+
+VERBOSE="false"
+while getopts ":vh" OPTION
+do
+    case "${OPTION}" in
+        'v')
+            VERBOSE="true"
+            ;;
+        'h')
+            usage
+            exit 0
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+drive_links="04 82"
+
+error_present="false"
+for pax in $drive_links;
+do
+    for drive in {0..8};
+    do
+        if [ $pax -eq 4  ] && [ $drive -eq 7 ]; then
+            continue
+        fi
+        if [ $pax -eq 82  ] && [ $drive -eq 4 ]; then
+            continue
+        fi
+
+        pci_result=$(lspci -s "$pax":"$drive".0 -vv | grep LnkSta:)
+        if [ $? -ne 0 ]; then
+            if [ $VERBOSE == "true" ]; then
+                printf "%d:%d.0 Error accessing drive\n" "$pax" "$drive";
+            fi
+            error_present="true"
+            continue
+        fi
+
+        access_error_result=$(echo "$pci_result" | grep "!!! Unknown header type")
+        if [ $? -eq 0 ]; then
+            if [ $VERBOSE == "true" ]; then
+                printf "%d:%d.0 Unable to retrieve link info\n" "$pax" "$drive"
+            fi
+            error_present="true"
+            continue
+        fi
+
+        width_zero_error_result=$(echo "$pci_result" | grep "Width x0")
+        if [ $? -eq 0 ]; then
+            if [ $VERBOSE == "true" ]; then
+                printf "%d:%d.0 Link down\n" "$pax" "$drive"
+            fi
+            error_present="true"
+            continue
+        fi
+
+        link_speed_error_result=$(echo "$pci_result" | grep 16GT)
+        if [ $? -ne 0 ]; then
+            if [ $VERBOSE == "true" ]; then
+                printf "%d:%d.0 Link speed too slow\n" "$pax" "$drive"
+            fi
+            error_present="true"
+            continue
+        fi
+    done;
+done
+
+if [ "$error_present" == "false" ]; then
+    printf "No errors\n"
+else
+    printf "ERRORS FOUND\n"
+fi
diff --git a/tools/switch.sh b/tools/switch.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2022-2023 Hewlett Packard Enterprise Development LP
+# Copyright 2022-2024 Hewlett Packard Enterprise Development LP
 # Other additional copyright holders may be indicated within.
 #
 # The entirety of this work is licensed under the Apache License,
@@ -28,7 +28,7 @@ usage() {
     cat <<EOF
 Run various switch command over all switches.
 
-Usage: $0 [-h] [-t] COMMAND [ARGS...]
+Usage: $0 [-hv] [-t] COMMAND [ARGS...]
 
 Commands:
     slot-info                            display slot status for each physical port
@@ -44,6 +44,7 @@ Commands:
 
 Arguments:
   -h                display this help
+  -v                verbose mode
   -t                time each command
 
 Examples:
@@ -88,6 +89,20 @@ execute() {
     done
 }
 
+getChassis() {
+    if [ "$VERBOSE" != "true" ]; then
+        CHASSIS="       "
+        return
+    fi
+
+    COMMAND=xhost-query.py
+    if command -v $COMMAND &> /dev/null; then
+        CHASSIS=$("$COMMAND" $(hostname) | cut -c -7)
+    else
+        CHASSIS="x****c*"
+    fi
+}
+
 getPAXID() {
     local SWITCH_NAME=$1
 
@@ -104,6 +119,18 @@ getPAXID() {
     fi
 }
 
+getPAXTemperature() {
+    local SWITCH_NAME=$1
+
+    # Make sure we can get the PAX ID
+    if [ ! "$(switchtec temp "$SWITCH_NAME")" ]; then
+        echo "Unable to retrieve PAX Temperature"
+        exit $?
+    fi
+
+    PAX_TEMPERATURE=$(switchtec temp "$SWITCH_NAME")
+}
+
 setDeviceName() {
     DRIVES=()
     getDriveList
@@ -118,6 +145,8 @@ setDeviceName() {
 displayDriveSlotStatus() {
     local SWITCH_NAME=$1
 
+    getChassis
+
     # Physical slot ids are set into the hardware. These are the mappings
     declare -a PAX0_DriveSlotFromPhysicalPort=(
         # Drives
@@ -203,6 +232,8 @@ displayDriveSlotStatus() {
 displayStatus() {
     local SWITCH_NAME=$1
 
+    getChassis
+
     # Physical slot ids are set into the hardware. These are the mappings
     declare -a PAX0_ConnectedEPToPhysicalPort=(
         # Drives
@@ -218,15 +249,15 @@ displayStatus() {
 
         # Other Links
         [0]="Interswitch Link           "
-        [24]="Rabbit,       x9000c?j7b0"
-        [32]="Compute 0,    x9000c?s0b0n0"
-        [34]="Compute 1,    x9000c?s0b1n0"
-        [36]="Compute 2,    x9000c?s1b0n0"
-        [38]="Compute 3,    x9000c?s1b1n0"
-        [40]="Compute 4,    x9000c?s2b0n0"
-        [42]="Compute 5,    x9000c?s2b1n0"
-        [44]="Compute 6,    x9000c?s3b0n0"
-        [46]="Compute 7,    x9000c?s3b1n0"
+        [24]="Rabbit,       ${CHASSIS}r7b0n0"
+        [32]="Compute 0,    ${CHASSIS}s0b0n0"
+        [34]="Compute 1,    ${CHASSIS}s0b1n0"
+        [36]="Compute 2,    ${CHASSIS}s1b0n0"
+        [38]="Compute 3,    ${CHASSIS}s1b1n0"
+        [40]="Compute 4,    ${CHASSIS}s2b0n0"
+        [42]="Compute 5,    ${CHASSIS}s2b1n0"
+        [44]="Compute 6,    ${CHASSIS}s3b0n0"
+        [46]="Compute 7,    ${CHASSIS}s3b1n0"
     )
     declare -a PAX1_ConnectedEPToPhysicalPort=(
         # Drives
@@ -242,23 +273,28 @@ displayStatus() {
 
         # Other Links
         [0]="Interswitch Link           "
-        [24]="Rabbit,       x9000c?j7b0"
-        [32]="Compute 8,    x9000c?s4b0n0"
-        [34]="Compute 9,    x9000c?s4b1n0"
-        [36]="Compute 10,   x9000c?s5b0n0"
-        [38]="Compute 11,   x9000c?s5b1n0"
-        [40]="Compute 12,   x9000c?s6b0n0"
-        [42]="Compute 13,   x9000c?s6b1n0"
-        [44]="Compute 14,   x9000c?s7b0n0"
-        [46]="Compute 15,   x9000c?s7b1n0"
+        [24]="Rabbit,       ${CHASSIS}r7b0n0"
+        [32]="Compute 8,    ${CHASSIS}s4b0n0"
+        [34]="Compute 9,    ${CHASSIS}s4b1n0"
+        [36]="Compute 10,   ${CHASSIS}s5b0n0"
+        [38]="Compute 11,   ${CHASSIS}s5b1n0"
+        [40]="Compute 12,   ${CHASSIS}s6b0n0"
+        [42]="Compute 13,   ${CHASSIS}s6b1n0"
+        [44]="Compute 14,   ${CHASSIS}s7b0n0"
+        [46]="Compute 15,   ${CHASSIS}s7b1n0"
     )
 
     getPAXID "$SWITCH_NAME"
 
     mapfile -t physicalPortIdStrings < <(switchtec status "$SWITCH_NAME" | grep "Phys Port ID:")
 
     local physicalPortString
-    printf "DEVICE: %s PAX_ID: %d\n\n" "$SWITCH_NAME" "$PAX_ID"
+    if [ "$VERBOSE" == "true" ]; then
+        getPAXTemperature "$SWITCH_NAME"
+        printf "DEVICE: %s PAX_ID: %d  TEMP: %s\n\n" "$SWITCH_NAME" "$PAX_ID" "$PAX_TEMPERATURE"
+    else
+        printf "DEVICE: %s PAX_ID: %d\n\n" "$SWITCH_NAME" "$PAX_ID"
+    fi
     printf "Switch Connection        \tStatus\n"
     printf "===========================\t======\n"
     for physicalPortString in "${physicalPortIdStrings[@]}";
@@ -285,13 +321,16 @@ displayStatus() {
 
 
 alias TIME=""
-while getopts "th:" OPTION
+while getopts "tvh:" OPTION
 do
     case "${OPTION}" in
         't')
             alias TIME=time
             export TIMEFORMAT='%3lR'
             ;;
+        'v')
+            export VERBOSE="true"
+            ;;
         'h',*)
             usage
             exit 0

diff --git a/tools/timestamp-log.sh b/tools/timestamp-log.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2024 Hewlett Packard Enterprise Development LP
+# Copyright 2020, 2021, 2022 Hewlett Packard Enterprise Development LP
 # Other additional copyright holders may be indicated within.
 #
 # The entirety of this work is licensed under the Apache License,