Enable windows logging

kubernetes-sigs · Apr 30, 2021 · 43949db · 43949db
1 parent 36bfaad
commit 43949db
Show file tree

Hide file tree

Showing 2 changed files with 126 additions and 6 deletions.
diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh
@@ -74,12 +74,14 @@ export KIND_EXPERIMENTAL_DOCKER_NETWORK="bridge"
 # Generate SSH key.
 AZURE_SSH_PUBLIC_KEY_FILE=${AZURE_SSH_PUBLIC_KEY_FILE:-""}
 if [ -z "${AZURE_SSH_PUBLIC_KEY_FILE}" ]; then
+    echo "generating sshkey for e2e"
     SSH_KEY_FILE=.sshkey
     rm -f "${SSH_KEY_FILE}" 2>/dev/null
     ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null
     AZURE_SSH_PUBLIC_KEY_FILE="${SSH_KEY_FILE}.pub"
 fi
 export AZURE_SSH_PUBLIC_KEY_B64=$(cat "${AZURE_SSH_PUBLIC_KEY_FILE}" | base64 | tr -d '\r\n')
+export AZURE_SSH_PUBLIC_KEY=$(cat "${AZURE_SSH_PUBLIC_KEY_FILE}" | tr -d '\r\n')
 
 cleanup() {
     ${REPO_ROOT}/hack/log/redact.sh || true

diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go
@@ -23,9 +23,12 @@ import (
 	"io/ioutil"
 	"net/http"
 	"path/filepath"
-	"sigs.k8s.io/cluster-api-provider-azure/azure"
 	"strings"
 
+	"sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4"
+
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
+
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-30/compute"
 	autorest "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
@@ -40,8 +43,7 @@ import (
 type AzureLogCollector struct{}
 
 // CollectMachineLog collects logs from a machine.
-func (k AzureLogCollector) CollectMachineLog(ctx context.Context,
-	managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error {
+func (k AzureLogCollector) CollectMachineLog(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error {
 	var errors []error
 
 	if err := collectLogsFromNode(ctx, managementClusterClient, m, outputPath); err != nil {
@@ -62,9 +64,11 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli
 	if err != nil {
 		return err
 	}
+
 	controlPlaneEndpoint := cluster.Spec.ControlPlaneEndpoint.Host
-	hostname := m.Spec.InfrastructureRef.Name
+	hostname := m.Status.Addresses[0].Address
 	port := e2eConfig.GetVariable(VMSSHPort)
+
 	execToPathFn := func(outputFileName, command string, args ...string) func() error {
 		return func() error {
 			f, err := fileOnHost(filepath.Join(outputPath, outputFileName))
@@ -78,7 +82,30 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli
 		}
 	}
 
-	return kinderrors.AggregateConcurrent([]func() error{
+	key := client.ObjectKey{
+		Namespace: m.Spec.InfrastructureRef.Namespace,
+		Name:      m.Spec.InfrastructureRef.Name,
+	}
+
+	azMachine := &v1alpha4.AzureMachine{}
+	if err := managementClusterClient.Get(ctx, key, azMachine); err != nil {
+		return err
+	}
+
+	if azMachine.Spec.OSDisk.OSType == azure.WindowsOS {
+		// if we initiate to many ssh connections they get dropped (default is 10) so split it up
+		var errors []error
+		errors = append(errors, kinderrors.AggregateConcurrent(windowsInfo(execToPathFn)))
+		errors = append(errors, kinderrors.AggregateConcurrent(windowsK8sLogs(execToPathFn)))
+		errors = append(errors, kinderrors.AggregateConcurrent(windowsNetworkLogs(execToPathFn)))
+		return kinderrors.NewAggregate(errors)
+	} else {
+		return kinderrors.AggregateConcurrent(linuxLogs(execToPathFn))
+	}
+}
+
+func linuxLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error {
+	return []func() error{
 		execToPathFn(
 			"journal.log",
 			"journalctl", "--no-pager", "--output=short-precise",
@@ -107,7 +134,98 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli
 			"cloud-init-output.log",
 			"cat", "/var/log/cloud-init-output.log",
 		),
-	})
+	}
+}
+
+func windowsK8sLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error {
+	return []func() error{
+		execToPathFn(
+			"hyperv-operation.log",
+			"Get-WinEvent", "-LogName Microsoft-Windows-Hyper-V-Compute-Operational | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Sort-Object TimeCreated",
+		),
+		execToPathFn(
+			"docker.log",
+			"get-eventlog", "-LogName Application -Source Docker | Select-Object Index, TimeGenerated, EntryType, Message | Sort-Object Index",
+		),
+		execToPathFn(
+			"containers.log",
+			"docker", "ps -a",
+		),
+		execToPathFn(
+			"kubelet.log",
+			`Get-ChildItem "C:\\var\\log\\kubelet\\*INFO*"  | ForEach-Object {  cat "$_" }`,
+		),
+	}
+}
+
+func windowsInfo(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error {
+	return []func() error{
+		execToPathFn(
+			"reboots.log",
+			"Get-WinEvent", `-ErrorAction Ignore -FilterHashtable @{logname = 'System'; id = 1074, 1076, 2004, 6005, 6006, 6008 } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`,
+		),
+		execToPathFn(
+			"crashes.log",
+			`Get-WinEvent -ErrorAction Ignore -FilterHashtable @{logname = "Application"; ProviderName = "Windows Error Reporting" } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`,
+		),
+		execToPathFn(
+			"scm.log",
+			"Get-WinEvent", `-FilterHashtable @{logname = 'System'; ProviderName = 'Service Control Manager' } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`,
+		),
+		execToPathFn(
+			"pagefile.log",
+			"Get-CimInstance", "win32_pagefileusage | Format-List *",
+		),
+		execToPathFn(
+			"cloudbase-init-unattend.log",
+			"get-content 'C:\\Program Files\\Cloudbase Solutions\\Cloudbase-Init\\log\\cloudbase-init-unattend.log'",
+		),
+		execToPathFn(
+			"cloudbase-init.log",
+			"get-content 'C:\\Program Files\\Cloudbase Solutions\\Cloudbase-Init\\log\\cloudbase-init.log'",
+		),
+	}
+}
+
+func windowsNetworkLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error {
+	return []func() error{
+		execToPathFn(
+			"network.log",
+			"Get-HnsNetwork | Select Name, Type, Id, AddressPrefix",
+		),
+		execToPathFn(
+			"network-detailed.log",
+			"Get-hnsnetwork | Convertto-json -Depth 20",
+		),
+		execToPathFn(
+			"network-individual-detailed.log",
+			"Get-hnsnetwork | % { Get-HnsNetwork -Id $_.ID -Detailed } | Convertto-json -Depth 20",
+		),
+		execToPathFn(
+			"hnsendpoints.log",
+			"Get-HnsEndpoint | Select IpAddress, MacAddress, IsRemoteEndpoint, State",
+		),
+		execToPathFn(
+			"hnsendpolicy-detailed.log",
+			"Get-hnspolicylist | Convertto-json -Depth 20",
+		),
+		execToPathFn(
+			"ips.log",
+			"ipconfig /allcompartments /all",
+		),
+		execToPathFn(
+			"ips.log",
+			"Get-NetIPAddress -IncludeAllCompartments",
+		),
+		execToPathFn(
+			"ips.log",
+			"Get-NetIPInterface -IncludeAllCompartments",
+		),
+		execToPathFn(
+			"hnsdiag.txt",
+			"hnsdiag list all -d",
+		),
+	}
 }
 
 // collectBootLog collects boot logs of the vm by using azure boot diagnostics