From a7a961e1c8c1f9477e535b2b6e1922ed82d4f022 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 29 Apr 2021 16:23:58 -0700 Subject: [PATCH] Enable windows logging --- scripts/ci-e2e.sh | 2 + test/e2e/azure_logcollector.go | 130 +++++++++++++++++++++++++++++++-- 2 files changed, 126 insertions(+), 6 deletions(-) diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 8cd71f17e31..10e64be0fe4 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -74,12 +74,14 @@ export KIND_EXPERIMENTAL_DOCKER_NETWORK="bridge" # Generate SSH key. AZURE_SSH_PUBLIC_KEY_FILE=${AZURE_SSH_PUBLIC_KEY_FILE:-""} if [ -z "${AZURE_SSH_PUBLIC_KEY_FILE}" ]; then + echo "generating sshkey for e2e" SSH_KEY_FILE=.sshkey rm -f "${SSH_KEY_FILE}" 2>/dev/null ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null AZURE_SSH_PUBLIC_KEY_FILE="${SSH_KEY_FILE}.pub" fi export AZURE_SSH_PUBLIC_KEY_B64=$(cat "${AZURE_SSH_PUBLIC_KEY_FILE}" | base64 | tr -d '\r\n') +export AZURE_SSH_PUBLIC_KEY=$(cat "${AZURE_SSH_PUBLIC_KEY_FILE}" | tr -d '\r\n') cleanup() { ${REPO_ROOT}/hack/log/redact.sh || true diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go index 00e9fa83b86..5276184aca0 100644 --- a/test/e2e/azure_logcollector.go +++ b/test/e2e/azure_logcollector.go @@ -23,9 +23,12 @@ import ( "io/ioutil" "net/http" "path/filepath" - "sigs.k8s.io/cluster-api-provider-azure/azure" "strings" + "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4" + + "sigs.k8s.io/cluster-api-provider-azure/azure" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-30/compute" autorest "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/azure/auth" @@ -40,8 +43,7 @@ import ( type AzureLogCollector struct{} // CollectMachineLog collects logs from a machine. -func (k AzureLogCollector) CollectMachineLog(ctx context.Context, - managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error { +func (k AzureLogCollector) CollectMachineLog(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine, outputPath string) error { var errors []error if err := collectLogsFromNode(ctx, managementClusterClient, m, outputPath); err != nil { @@ -62,9 +64,11 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli if err != nil { return err } + controlPlaneEndpoint := cluster.Spec.ControlPlaneEndpoint.Host - hostname := m.Spec.InfrastructureRef.Name + hostname := m.Status.Addresses[0].Address port := e2eConfig.GetVariable(VMSSHPort) + execToPathFn := func(outputFileName, command string, args ...string) func() error { return func() error { f, err := fileOnHost(filepath.Join(outputPath, outputFileName)) @@ -78,7 +82,30 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli } } - return kinderrors.AggregateConcurrent([]func() error{ + key := client.ObjectKey{ + Namespace: m.Spec.InfrastructureRef.Namespace, + Name: m.Spec.InfrastructureRef.Name, + } + + azMachine := &v1alpha4.AzureMachine{} + if err := managementClusterClient.Get(ctx, key, azMachine); err != nil { + return err + } + + if azMachine.Spec.OSDisk.OSType == azure.WindowsOS { + // if we initiate to many ssh connections they get dropped (default is 10) so split it up + var errors []error + errors = append(errors, kinderrors.AggregateConcurrent(windowsInfo(execToPathFn))) + errors = append(errors, kinderrors.AggregateConcurrent(windowsK8sLogs(execToPathFn))) + errors = append(errors, kinderrors.AggregateConcurrent(windowsNetworkLogs(execToPathFn))) + return kinderrors.NewAggregate(errors) + } else { + return kinderrors.AggregateConcurrent(linuxLogs(execToPathFn)) + } +} + +func linuxLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error { + return []func() error{ execToPathFn( "journal.log", "journalctl", "--no-pager", "--output=short-precise", @@ -107,7 +134,98 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli "cloud-init-output.log", "cat", "/var/log/cloud-init-output.log", ), - }) + } +} + +func windowsK8sLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error { + return []func() error{ + execToPathFn( + "hyperv-operation.log", + "Get-WinEvent", "-LogName Microsoft-Windows-Hyper-V-Compute-Operational | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Sort-Object TimeCreated", + ), + execToPathFn( + "docker.log", + "get-eventlog", "-LogName Application -Source Docker | Select-Object Index, TimeGenerated, EntryType, Message | Sort-Object Index", + ), + execToPathFn( + "containers.log", + "docker", "ps -a", + ), + execToPathFn( + "kubelet.log", + `Get-ChildItem "C:\\var\\log\\kubelet\\*INFO*" | ForEach-Object { cat "$_" }`, + ), + } +} + +func windowsInfo(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error { + return []func() error{ + execToPathFn( + "reboots.log", + "Get-WinEvent", `-ErrorAction Ignore -FilterHashtable @{logname = 'System'; id = 1074, 1076, 2004, 6005, 6006, 6008 } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`, + ), + execToPathFn( + "crashes.log", + `Get-WinEvent -ErrorAction Ignore -FilterHashtable @{logname = 'Application'; ProviderName = 'Windows Error Reporting' } | Select-Object -ErrorAction Ignore -Property TimeCreated, Id, LevelDisplayName, Message`, + ), + execToPathFn( + "scm.log", + "Get-WinEvent", `-FilterHashtable @{logname = 'System'; ProviderName = 'Service Control Manager' } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`, + ), + execToPathFn( + "pagefile.log", + "Get-CimInstance", "win32_pagefileusage | Format-List *", + ), + execToPathFn( + "cloudbase-init-unattend.log", + "get-content 'C:\\Program Files\\Cloudbase Solutions\\Cloudbase-Init\\log\\cloudbase-init-unattend.log'", + ), + execToPathFn( + "cloudbase-init.log", + "get-content 'C:\\Program Files\\Cloudbase Solutions\\Cloudbase-Init\\log\\cloudbase-init.log'", + ), + } +} + +func windowsNetworkLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error { + return []func() error{ + execToPathFn( + "network.log", + "Get-HnsNetwork | Select Name, Type, Id, AddressPrefix", + ), + execToPathFn( + "network-detailed.log", + "Get-hnsnetwork | Convertto-json -Depth 20", + ), + execToPathFn( + "network-individual-detailed.log", + "Get-hnsnetwork | % { Get-HnsNetwork -Id $_.ID -Detailed } | Convertto-json -Depth 20", + ), + execToPathFn( + "hnsendpoints.log", + "Get-HnsEndpoint | Select IpAddress, MacAddress, IsRemoteEndpoint, State", + ), + execToPathFn( + "hnsendpolicy-detailed.log", + "Get-hnspolicylist | Convertto-json -Depth 20", + ), + execToPathFn( + "ips.log", + "ipconfig /allcompartments /all", + ), + execToPathFn( + "ips.log", + "Get-NetIPAddress -IncludeAllCompartments", + ), + execToPathFn( + "ips.log", + "Get-NetIPInterface -IncludeAllCompartments", + ), + execToPathFn( + "hnsdiag.txt", + "hnsdiag list all -d", + ), + } } // collectBootLog collects boot logs of the vm by using azure boot diagnostics