Debug closing of sockets

.github/workflows/test.yml

@@ -98,10 +98,6 @@ jobs:
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-      - name: Ports
-        run: |
-          sudo netstat -tulpn | grep LISTEN
-
       - name: CPU Info
         run: |
           cat /proc/cpuinfo

charts/plgd-hub/templates/device-provisioning-service/_helpers.tpl

@@ -61,7 +61,7 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 
 {{- define "plgd-hub.deviceprovisioningservice.labels" -}}
 helm.sh/chart: {{ include "plgd-hub.chart" . }}
-{{ include "plgd-hub.deviceprovisioningservice.selectorLabel" . }}
+{{ include "plgd-hub.deviceprovisioningservice.selectorLabels" . }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}

coap-gateway/test/test.go

@@ -72,26 +72,14 @@ func SetUp(t require.TestingT) (tearDown func()) {
 }
 
 func checkForClosedSockets(t require.TestingT, cfg service.Config) {
-	protocolClosed := make([]bool, len(cfg.APIs.COAP.Protocols))
-	// wait for all sockets to be closed - max 3 minutes = 900*200
-	for j := 0; j < 900; j++ {
-		allClosed := true
-		for i, protocol := range cfg.APIs.COAP.Protocols {
-			if protocolClosed[i] {
-				continue
-			}
-			protocolClosed[i] = test.IsListenSocketClosed(t, string(protocol), cfg.APIs.COAP.Addr)
-			if protocolClosed[i] {
-				continue
-			}
-			allClosed = false
-			break
-		}
-		if allClosed {
-			break
-		}
-		time.Sleep(time.Millisecond * 200)
+	sockets := make(test.ListenSockets, 0, len(cfg.APIs.COAP.Protocols))
+	for _, protocol := range cfg.APIs.COAP.Protocols {
+		sockets = append(sockets, test.ListenSocket{
+			Network: string(protocol),
+			Address: cfg.APIs.COAP.Addr,
+		})
 	}
+	sockets.CheckForClosedSockets(t)
 }
 
 // New creates test coap-gateway.

device-provisioning-service/service/acls_test.go

@@ -17,6 +17,8 @@ import (
 )
 
 func TestAclsTCP(t *testing.T) {
+	test.ListPorts()
+
 	defer test.ClearDB(t)
 	hubShutdown := hubTestService.SetUpServices(context.Background(), t, hubTestService.SetUpServicesCertificateAuthority|hubTestService.SetUpServicesResourceDirectory|hubTestService.SetUpServicesMachine2MachineOAuth|hubTestService.SetUpServicesOAuth|hubTestService.SetUpServicesId)
 	defer hubShutdown()
@@ -45,6 +47,8 @@ func TestAclsTCP(t *testing.T) {
 }
 
 func TestAclsUDP(t *testing.T) {
+	test.ListPorts()
+
 	defer test.ClearDB(t)
 	hubShutdown := hubTestService.SetUpServices(context.Background(), t, hubTestService.SetUpServicesCertificateAuthority|hubTestService.SetUpServicesResourceDirectory|hubTestService.SetUpServicesMachine2MachineOAuth|hubTestService.SetUpServicesOAuth|hubTestService.SetUpServicesId)
 	defer hubShutdown()

device-provisioning-service/test/test.go

@@ -2,7 +2,9 @@ package test
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"os/exec"
 	"sync"
 	"testing"
 	"time"
@@ -262,27 +264,30 @@ func New(t *testing.T, cfg service.Config, opts ...service.Option) func() {
 	return NewWithContext(context.Background(), t, cfg, opts...)
 }
 
+func ListPorts() {
+	cmd := exec.Command("netstat", "-tunpl")
+	out, err := cmd.Output()
+	if err != nil {
+		return
+	}
+	fmt.Printf("ports:\n%v", string(out))
+}
+
 func checkForClosedSockets(t require.TestingT, cfg service.Config) {
-	protocolClosed := make([]bool, len(cfg.APIs.COAP.Config.Protocols))
-	// wait for all sockets to be closed - max 3 minutes = 900*200
-	for j := 0; j < 900; j++ {
-		allClosed := true
-		for i, protocol := range cfg.APIs.COAP.Config.Protocols {
-			if protocolClosed[i] {
-				continue
-			}
-			protocolClosed[i] = hubTest.IsListenSocketClosed(t, string(protocol), cfg.APIs.COAP.Config.Addr)
-			if protocolClosed[i] {
-				continue
-			}
-			allClosed = false
-			break
-		}
-		if allClosed {
-			break
-		}
-		time.Sleep(time.Millisecond * 200)
+	sockets := make(hubTest.ListenSockets, 0, len(cfg.APIs.COAP.Protocols)+1)
+	for _, protocol := range cfg.APIs.COAP.Protocols {
+		sockets = append(sockets, hubTest.ListenSocket{
+			Network: string(protocol),
+			Address: cfg.APIs.COAP.Addr,
+		})
+	}
+	if cfg.APIs.HTTP.Enabled {
+		sockets = append(sockets, hubTest.ListenSocket{
+			Network: "tcp",
+			Address: cfg.APIs.HTTP.Config.Connection.Addr,
+		})
 	}
+	sockets.CheckForClosedSockets(t)
 }
 
 // New creates test dps-gateway.
@@ -309,6 +314,8 @@ func NewWithContext(ctx context.Context, t *testing.T, cfg service.Config, opts
 		require.NoError(t, err)
 
 		checkForClosedSockets(t, cfg)
+		// wait for all connections to be closed
+		time.Sleep(time.Millisecond * 500)
 	}
 }
 

test/test.go

@@ -999,25 +999,69 @@ func GenerateDeviceIDbyIdx(deviceIndex int) string {
 	return GenerateIDbyIdx("d", deviceIndex)
 }
 
-func IsListenSocketClosed(t require.TestingT, target string, addStr string) bool {
-	if strings.Contains(target, "udp") {
-		addr, err := net.ResolveUDPAddr(target, addStr)
-		require.NoError(t, err)
-		c, err := net.ListenUDP(target, addr)
+type ListenSocket struct {
+	Network string
+	Address string
+}
+
+func (ls *ListenSocket) IsClosed() (bool, error) {
+	if strings.Contains(ls.Network, "udp") {
+		addr, err := net.ResolveUDPAddr(ls.Network, ls.Address)
 		if err != nil {
-			return false
+			return false, err
+		}
+		c, err := net.ListenUDP(ls.Network, addr)
+		if err != nil {
+			fmt.Printf("ListenUDP error: %v\n", err)
+			return false, nil
 		}
 		err = c.Close()
-		require.NoError(t, err)
-		return true
+		if err != nil {
+			return false, err
+		}
+		return true, nil
 	}
-	addr, err := net.ResolveTCPAddr(target, addStr)
-	require.NoError(t, err)
-	c, err := net.ListenTCP(target, addr)
+
+	addr, err := net.ResolveTCPAddr(ls.Network, ls.Address)
 	if err != nil {
-		return false
+		return false, err
+	}
+	c, err := net.ListenTCP(ls.Network, addr)
+	if err != nil {
+		fmt.Printf("ListenTCP error: %v\n", err)
+		return false, nil
 	}
 	err = c.Close()
-	require.NoError(t, err)
-	return true
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+type ListenSockets []ListenSocket
+
+func (ls ListenSockets) CheckForClosedSockets(t require.TestingT) {
+	// wait for all sockets to be closed - max 3 minutes = 900*200
+	socketClosed := make([]bool, len(ls))
+	for j := 0; j < 900; j++ {
+		allClosed := true
+		for i, socket := range ls {
+			if socketClosed[i] {
+				continue
+			}
+			fmt.Printf("Check socket: %v\n", socket)
+			closed, err := socket.IsClosed()
+			require.NoError(t, err)
+			socketClosed[i] = closed
+			if socketClosed[i] {
+				fmt.Printf("Socket closed: %v\n", socket)
+				continue
+			}
+			allClosed = false
+		}
+		if allClosed {
+			break
+		}
+		time.Sleep(time.Millisecond * 200)
+	}
 }