diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go index 5276184aca0..13ad5a66e02 100644 --- a/test/e2e/azure_logcollector.go +++ b/test/e2e/azure_logcollector.go @@ -20,6 +20,7 @@ package e2e import ( "context" + "fmt" "io/ioutil" "net/http" "path/filepath" @@ -65,8 +66,22 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli return err } + fmt.Printf("INFO: Collecting logs for machine %s in cluster %s in namespace %s\n", m.GetName(), cluster.Name, cluster.Namespace) + isWindows, err := isNodeWindows(ctx, managementClusterClient, m) + if err != nil { + return err + } + controlPlaneEndpoint := cluster.Spec.ControlPlaneEndpoint.Host + hostname := m.Spec.InfrastructureRef.Name + if isWindows { + // Windows host name ends up being different than the infra machine name + // due to Windows name limitations in Azure so use ipaddress instead hostname := m.Status.Addresses[0].Address + hostname = m.Status.Addresses[0].Address + } + } + port := e2eConfig.GetVariable(VMSSHPort) execToPathFn := func(outputFileName, command string, args ...string) func() error { @@ -82,6 +97,19 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli } } + if isWindows { + // if we initiate to many ssh connections they get dropped (default is 10) so split it up + var errors []error + errors = append(errors, kinderrors.AggregateConcurrent(windowsInfo(execToPathFn))) + errors = append(errors, kinderrors.AggregateConcurrent(windowsK8sLogs(execToPathFn))) + errors = append(errors, kinderrors.AggregateConcurrent(windowsNetworkLogs(execToPathFn))) + return kinderrors.NewAggregate(errors) + } else { + return kinderrors.AggregateConcurrent(linuxLogs(execToPathFn)) + } +} + +func isNodeWindows(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine) (bool, error) { key := client.ObjectKey{ Namespace: m.Spec.InfrastructureRef.Namespace, Name: m.Spec.InfrastructureRef.Name, @@ -89,19 +117,13 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli azMachine := &v1alpha4.AzureMachine{} if err := managementClusterClient.Get(ctx, key, azMachine); err != nil { - return err + return false, err } if azMachine.Spec.OSDisk.OSType == azure.WindowsOS { - // if we initiate to many ssh connections they get dropped (default is 10) so split it up - var errors []error - errors = append(errors, kinderrors.AggregateConcurrent(windowsInfo(execToPathFn))) - errors = append(errors, kinderrors.AggregateConcurrent(windowsK8sLogs(execToPathFn))) - errors = append(errors, kinderrors.AggregateConcurrent(windowsNetworkLogs(execToPathFn))) - return kinderrors.NewAggregate(errors) - } else { - return kinderrors.AggregateConcurrent(linuxLogs(execToPathFn)) + return true, nil } + return false, nil } func linuxLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error { @@ -141,19 +163,23 @@ func windowsK8sLogs(execToPathFn func(outputFileName string, command string, arg return []func() error{ execToPathFn( "hyperv-operation.log", - "Get-WinEvent", "-LogName Microsoft-Windows-Hyper-V-Compute-Operational | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Sort-Object TimeCreated", + "Get-WinEvent", "-LogName Microsoft-Windows-Hyper-V-Compute-Operational | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Sort-Object TimeCreated | Format-Table -Wrap -Autosize", ), execToPathFn( "docker.log", - "get-eventlog", "-LogName Application -Source Docker | Select-Object Index, TimeGenerated, EntryType, Message | Sort-Object Index", + "get-eventlog", "-LogName Application -Source Docker | Select-Object Index, TimeGenerated, EntryType, Message | Sort-Object Index | Format-Table -Wrap -Autosize", ), execToPathFn( "containers.log", "docker", "ps -a", ), + execToPathFn( + "containers-hcs.log", + "hcsdiag", "list", + ), execToPathFn( "kubelet.log", - `Get-ChildItem "C:\\var\\log\\kubelet\\*INFO*" | ForEach-Object { cat "$_" }`, + `Get-ChildItem "C:\\var\\log\\kubelet\\" | ForEach-Object { write-output "$_" ;cat "c:\\var\\log\\kubelet\\$_" }`, ), } } @@ -162,15 +188,15 @@ func windowsInfo(execToPathFn func(outputFileName string, command string, args . return []func() error{ execToPathFn( "reboots.log", - "Get-WinEvent", `-ErrorAction Ignore -FilterHashtable @{logname = 'System'; id = 1074, 1076, 2004, 6005, 6006, 6008 } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`, - ), - execToPathFn( - "crashes.log", - `Get-WinEvent -ErrorAction Ignore -FilterHashtable @{logname = 'Application'; ProviderName = 'Windows Error Reporting' } | Select-Object -ErrorAction Ignore -Property TimeCreated, Id, LevelDisplayName, Message`, + "Get-WinEvent", `-ErrorAction Ignore -FilterHashtable @{logname = 'System'; id = 1074, 1076, 2004, 6005, 6006, 6008 } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Format-Table -Wrap -Autosize`, ), + //execToPathFn( + // "crashes.log", + // `Get-WinEvent -ErrorAction Ignore -FilterHashtable @{logname = 'Application'; ProviderName = 'Windows Error Reporting' } | Select-Object -ErrorAction Ignore -Property TimeCreated, Id, LevelDisplayName, Message | Format-Table -Wrap -Autosize`, + //), execToPathFn( "scm.log", - "Get-WinEvent", `-FilterHashtable @{logname = 'System'; ProviderName = 'Service Control Manager' } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`, + "Get-WinEvent", `-FilterHashtable @{logname = 'System'; ProviderName = 'Service Control Manager' } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Format-Table -Wrap -Autosize`, ), execToPathFn( "pagefile.log", @@ -184,6 +210,10 @@ func windowsInfo(execToPathFn func(outputFileName string, command string, args . "cloudbase-init.log", "get-content 'C:\\Program Files\\Cloudbase Solutions\\Cloudbase-Init\\log\\cloudbase-init.log'", ), + execToPathFn( + "services.log", + "get-service", + ), } } @@ -191,7 +221,7 @@ func windowsNetworkLogs(execToPathFn func(outputFileName string, command string, return []func() error{ execToPathFn( "network.log", - "Get-HnsNetwork | Select Name, Type, Id, AddressPrefix", + "Get-HnsNetwork | Select Name, Type, Id, AddressPrefix | Format-Table -Wrap -Autosize", ), execToPathFn( "network-detailed.log", @@ -210,7 +240,7 @@ func windowsNetworkLogs(execToPathFn func(outputFileName string, command string, "Get-hnspolicylist | Convertto-json -Depth 20", ), execToPathFn( - "ips.log", + "ipconfig.log", "ipconfig /allcompartments /all", ), execToPathFn( @@ -218,7 +248,7 @@ func windowsNetworkLogs(execToPathFn func(outputFileName string, command string, "Get-NetIPAddress -IncludeAllCompartments", ), execToPathFn( - "ips.log", + "interfaces.log", "Get-NetIPInterface -IncludeAllCompartments", ), execToPathFn(