Skip to content

Commit

Permalink
updates for the logging
Browse files Browse the repository at this point in the history
  • Loading branch information
jsturtevant committed Apr 30, 2021
1 parent 16535e8 commit acca60f
Showing 1 changed file with 51 additions and 21 deletions.
72 changes: 51 additions & 21 deletions test/e2e/azure_logcollector.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package e2e

import (
"context"
"fmt"
"io/ioutil"
"net/http"
"path/filepath"
Expand Down Expand Up @@ -65,8 +66,22 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli
return err
}

fmt.Printf("INFO: Collecting logs for machine %s in cluster %s in namespace %s\n", m.GetName(), cluster.Name, cluster.Namespace)
isWindows, err := isNodeWindows(ctx, managementClusterClient, m)
if err != nil {
return err
}

controlPlaneEndpoint := cluster.Spec.ControlPlaneEndpoint.Host
hostname := m.Spec.InfrastructureRef.Name
if isWindows {
// Windows host name ends up being different than the infra machine name
// due to Windows name limitations in Azure so use ipaddress instead
hostname := m.Status.Addresses[0].Address
hostname = m.Status.Addresses[0].Address
}
}

port := e2eConfig.GetVariable(VMSSHPort)

execToPathFn := func(outputFileName, command string, args ...string) func() error {
Expand All @@ -82,26 +97,33 @@ func collectLogsFromNode(ctx context.Context, managementClusterClient client.Cli
}
}

if isWindows {
// if we initiate to many ssh connections they get dropped (default is 10) so split it up
var errors []error
errors = append(errors, kinderrors.AggregateConcurrent(windowsInfo(execToPathFn)))
errors = append(errors, kinderrors.AggregateConcurrent(windowsK8sLogs(execToPathFn)))
errors = append(errors, kinderrors.AggregateConcurrent(windowsNetworkLogs(execToPathFn)))
return kinderrors.NewAggregate(errors)
} else {
return kinderrors.AggregateConcurrent(linuxLogs(execToPathFn))
}
}

func isNodeWindows(ctx context.Context, managementClusterClient client.Client, m *clusterv1.Machine) (bool, error) {
key := client.ObjectKey{
Namespace: m.Spec.InfrastructureRef.Namespace,
Name: m.Spec.InfrastructureRef.Name,
}

azMachine := &v1alpha4.AzureMachine{}
if err := managementClusterClient.Get(ctx, key, azMachine); err != nil {
return err
return false, err
}

if azMachine.Spec.OSDisk.OSType == azure.WindowsOS {
// if we initiate to many ssh connections they get dropped (default is 10) so split it up
var errors []error
errors = append(errors, kinderrors.AggregateConcurrent(windowsInfo(execToPathFn)))
errors = append(errors, kinderrors.AggregateConcurrent(windowsK8sLogs(execToPathFn)))
errors = append(errors, kinderrors.AggregateConcurrent(windowsNetworkLogs(execToPathFn)))
return kinderrors.NewAggregate(errors)
} else {
return kinderrors.AggregateConcurrent(linuxLogs(execToPathFn))
return true, nil
}
return false, nil
}

func linuxLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error {
Expand Down Expand Up @@ -141,19 +163,23 @@ func windowsK8sLogs(execToPathFn func(outputFileName string, command string, arg
return []func() error{
execToPathFn(
"hyperv-operation.log",
"Get-WinEvent", "-LogName Microsoft-Windows-Hyper-V-Compute-Operational | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Sort-Object TimeCreated",
"Get-WinEvent", "-LogName Microsoft-Windows-Hyper-V-Compute-Operational | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Sort-Object TimeCreated | Format-Table -Wrap -Autosize",
),
execToPathFn(
"docker.log",
"get-eventlog", "-LogName Application -Source Docker | Select-Object Index, TimeGenerated, EntryType, Message | Sort-Object Index",
"get-eventlog", "-LogName Application -Source Docker | Select-Object Index, TimeGenerated, EntryType, Message | Sort-Object Index | Format-Table -Wrap -Autosize",
),
execToPathFn(
"containers.log",
"docker", "ps -a",
),
execToPathFn(
"containers-hcs.log",
"hcsdiag", "list",
),
execToPathFn(
"kubelet.log",
`Get-ChildItem "C:\\var\\log\\kubelet\\*INFO*" | ForEach-Object { cat "$_" }`,
`Get-ChildItem "C:\\var\\log\\kubelet\\" | ForEach-Object { write-output "$_" ;cat "c:\\var\\log\\kubelet\\$_" }`,
),
}
}
Expand All @@ -162,15 +188,15 @@ func windowsInfo(execToPathFn func(outputFileName string, command string, args .
return []func() error{
execToPathFn(
"reboots.log",
"Get-WinEvent", `-ErrorAction Ignore -FilterHashtable @{logname = 'System'; id = 1074, 1076, 2004, 6005, 6006, 6008 } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`,
),
execToPathFn(
"crashes.log",
`Get-WinEvent -ErrorAction Ignore -FilterHashtable @{logname = 'Application'; ProviderName = 'Windows Error Reporting' } | Select-Object -ErrorAction Ignore -Property TimeCreated, Id, LevelDisplayName, Message`,
"Get-WinEvent", `-ErrorAction Ignore -FilterHashtable @{logname = 'System'; id = 1074, 1076, 2004, 6005, 6006, 6008 } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Format-Table -Wrap -Autosize`,
),
//execToPathFn(
// "crashes.log",
// `Get-WinEvent -ErrorAction Ignore -FilterHashtable @{logname = 'Application'; ProviderName = 'Windows Error Reporting' } | Select-Object -ErrorAction Ignore -Property TimeCreated, Id, LevelDisplayName, Message | Format-Table -Wrap -Autosize`,
//),
execToPathFn(
"scm.log",
"Get-WinEvent", `-FilterHashtable @{logname = 'System'; ProviderName = 'Service Control Manager' } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message`,
"Get-WinEvent", `-FilterHashtable @{logname = 'System'; ProviderName = 'Service Control Manager' } | Select-Object -Property TimeCreated, Id, LevelDisplayName, Message | Format-Table -Wrap -Autosize`,
),
execToPathFn(
"pagefile.log",
Expand All @@ -184,14 +210,18 @@ func windowsInfo(execToPathFn func(outputFileName string, command string, args .
"cloudbase-init.log",
"get-content 'C:\\Program Files\\Cloudbase Solutions\\Cloudbase-Init\\log\\cloudbase-init.log'",
),
execToPathFn(
"services.log",
"get-service",
),
}
}

func windowsNetworkLogs(execToPathFn func(outputFileName string, command string, args ...string) func() error) []func() error {
return []func() error{
execToPathFn(
"network.log",
"Get-HnsNetwork | Select Name, Type, Id, AddressPrefix",
"Get-HnsNetwork | Select Name, Type, Id, AddressPrefix | Format-Table -Wrap -Autosize",
),
execToPathFn(
"network-detailed.log",
Expand All @@ -210,15 +240,15 @@ func windowsNetworkLogs(execToPathFn func(outputFileName string, command string,
"Get-hnspolicylist | Convertto-json -Depth 20",
),
execToPathFn(
"ips.log",
"ipconfig.log",
"ipconfig /allcompartments /all",
),
execToPathFn(
"ips.log",
"Get-NetIPAddress -IncludeAllCompartments",
),
execToPathFn(
"ips.log",
"interfaces.log",
"Get-NetIPInterface -IncludeAllCompartments",
),
execToPathFn(
Expand Down

0 comments on commit acca60f

Please sign in to comment.