From 5fa8d696d14ff04b442e404084d003d3886e7520 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 13 Feb 2024 17:45:10 -0800 Subject: [PATCH 1/5] [chore] try using actuated to build --- .github/workflows/build-and-test.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 430508af1fc..916b27430d7 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -137,8 +137,9 @@ jobs: unittest-matrix: strategy: matrix: + runner: [ubuntu-latest, actuated-arm64-8cpu-16gb] go-version: ["~1.22", "~1.21.5"] # 1.20 needs quotes otherwise it's interpreted as 1.2 - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} needs: [setup-environment] steps: - name: Checkout Repo @@ -155,14 +156,15 @@ jobs: path: | ~/go/bin ~/go/pkg/mod - key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + key: go-cache-${{ runner.os }}-${{ matrix.runner }}-${{ hashFiles('**/go.sum') }} - name: Cache Build uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 with: path: ~/.cache/go-build - key: unittest-${{ runner.os }}-go-build-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} + key: unittest-${{ runner.os }}-${{ matrix.runner }}-go-build-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} - name: Run Unit Tests - run: make gotest + run: | + make -j8 gotest unittest: if: always() runs-on: ubuntu-latest From c7617b38cc0c568742762f229f759bf98319b6e1 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 15 Feb 2024 13:03:34 -0800 Subject: [PATCH 2/5] work on TestNoDataLoss --- .github/workflows/build-and-test.yml | 11 +++++- .../internal/mock_exporter.go | 39 +++++++++++-------- .../memorylimiter_test.go | 6 ++- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 916b27430d7..cc8372916b5 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -137,11 +137,18 @@ jobs: unittest-matrix: strategy: matrix: - runner: [ubuntu-latest, actuated-arm64-8cpu-16gb] + runner: [ubuntu-latest, actuated-arm64-4cpu-4gb] go-version: ["~1.22", "~1.21.5"] # 1.20 needs quotes otherwise it's interpreted as 1.2 runs-on: ${{ matrix.runner }} needs: [setup-environment] steps: + - name: Set up arkade + uses: alexellis/setup-arkade@master + - name: Install vmmeter + run: | + sudo -E arkade oci install ghcr.io/openfaasltd/vmmeter:latest --path /usr/local/bin/ + - name: Run vmmeter + uses: self-actuated/vmmeter-action@master - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Setup Go @@ -164,7 +171,7 @@ jobs: key: unittest-${{ runner.os }}-${{ matrix.runner }}-go-build-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} - name: Run Unit Tests run: | - make -j8 gotest + make -j4 gotest unittest: if: always() runs-on: ubuntu-latest diff --git a/processor/memorylimiterprocessor/internal/mock_exporter.go b/processor/memorylimiterprocessor/internal/mock_exporter.go index 1a55c9299e9..fee7503dd26 100644 --- a/processor/memorylimiterprocessor/internal/mock_exporter.go +++ b/processor/memorylimiterprocessor/internal/mock_exporter.go @@ -8,14 +8,15 @@ import ( "sync/atomic" "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/pdata/plog" ) type MockExporter struct { - destAvailable int64 - acceptedLogCount int64 - deliveredLogCount int64 - Logs []plog.Logs + destAvailable atomic.Bool + acceptedLogCount atomic.Int64 + deliveredLogCount atomic.Int64 + Logs consumertest.LogsSink } var _ consumer.Logs = (*MockExporter)(nil) @@ -24,15 +25,15 @@ func (e *MockExporter) Capabilities() consumer.Capabilities { return consumer.Capabilities{} } -func (e *MockExporter) ConsumeLogs(_ context.Context, ld plog.Logs) error { - atomic.AddInt64(&e.acceptedLogCount, int64(ld.LogRecordCount())) +func (e *MockExporter) ConsumeLogs(ctx context.Context, ld plog.Logs) error { + e.acceptedLogCount.Add(int64(ld.LogRecordCount())) - if atomic.LoadInt64(&e.destAvailable) == 1 { + if e.destAvailable.Load() { // Destination is available, immediately deliver. - atomic.AddInt64(&e.deliveredLogCount, int64(ld.LogRecordCount())) + e.deliveredLogCount.Add(int64(ld.LogRecordCount())) } else { // Destination is not available. Queue the logs in the exporter. - e.Logs = append(e.Logs, ld) + return e.Logs.ConsumeLogs(ctx, ld) } return nil } @@ -40,28 +41,32 @@ func (e *MockExporter) ConsumeLogs(_ context.Context, ld plog.Logs) error { func (e *MockExporter) SetDestAvailable(available bool) { if available { // Pretend we delivered all queued accepted logs. - atomic.AddInt64(&e.deliveredLogCount, atomic.LoadInt64(&e.acceptedLogCount)) + e.deliveredLogCount.Add(int64(e.Logs.LogRecordCount())) // Get rid of the delivered logs so that memory can be collected. - e.Logs = nil + e.Logs.Reset() // Now mark destination available so that subsequent ConsumeLogs // don't queue the logs anymore. - atomic.StoreInt64(&e.destAvailable, 1) - + e.destAvailable.Store(true) } else { - atomic.StoreInt64(&e.destAvailable, 0) + e.destAvailable.Store(false) } } func (e *MockExporter) AcceptedLogCount() int { - return int(atomic.LoadInt64(&e.acceptedLogCount)) + return int(e.acceptedLogCount.Load()) } func (e *MockExporter) DeliveredLogCount() int { - return int(atomic.LoadInt64(&e.deliveredLogCount)) + return int(e.deliveredLogCount.Load()) } func NewMockExporter() *MockExporter { - return &MockExporter{} + return &MockExporter{ + destAvailable: atomic.Bool{}, + acceptedLogCount: atomic.Int64{}, + deliveredLogCount: atomic.Int64{}, + Logs: consumertest.LogsSink{}, + } } diff --git a/processor/memorylimiterprocessor/memorylimiter_test.go b/processor/memorylimiterprocessor/memorylimiter_test.go index 626bf877d7c..10bc8d7911c 100644 --- a/processor/memorylimiterprocessor/memorylimiter_test.go +++ b/processor/memorylimiterprocessor/memorylimiter_test.go @@ -99,8 +99,10 @@ func TestNoDataLoss(t *testing.T) { // And eventually the exporter must confirm that it delivered exact number of produced logs. require.Eventually(t, func() bool { - return receiver.ProduceCount == exporter.DeliveredLogCount() - }, 5*time.Second, 1*time.Millisecond) + d := exporter.DeliveredLogCount() + t.Logf("received: %d, expected: %d\n", d, receiver.ProduceCount) + return receiver.ProduceCount == d + }, 5*time.Second, 100*time.Millisecond) // Double check that the number of logs accepted by exporter matches the number of produced by receiver. assert.Equal(t, receiver.ProduceCount, exporter.AcceptedLogCount()) From 3de2501e577b2d5967278461e8d37b5050697eaa Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Fri, 23 Feb 2024 22:14:31 -0800 Subject: [PATCH 3/5] matrix run for contrib --- .github/workflows/contrib-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index d17bfb6f9a8..4c7ab929e78 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -17,10 +17,12 @@ permissions: jobs: contrib-tests-matrix: - runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.labels.*.name, 'Skip Contrib Tests') }} strategy: matrix: + runner: + - ubuntu-latest + - actuated-arm64-4cpu-4gb group: - receiver-0 - receiver-1 @@ -36,6 +38,7 @@ jobs: - cmd-0 - cmd-1 - other + runs-on: ${{ matrix.runner }} steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 From 742ad5dc334b98b99e9a3e25ce0f41d78636c867 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 27 Feb 2024 08:33:58 -0800 Subject: [PATCH 4/5] revert trying to build contrib with actuated --- .github/workflows/contrib-tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index 4c7ab929e78..d17bfb6f9a8 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -17,12 +17,10 @@ permissions: jobs: contrib-tests-matrix: + runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.labels.*.name, 'Skip Contrib Tests') }} strategy: matrix: - runner: - - ubuntu-latest - - actuated-arm64-4cpu-4gb group: - receiver-0 - receiver-1 @@ -38,7 +36,6 @@ jobs: - cmd-0 - cmd-1 - other - runs-on: ${{ matrix.runner }} steps: - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 From b7aeb32fd7df976d21a4ed492dd2d587c87a0dec Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Mon, 11 Mar 2024 11:58:46 -0700 Subject: [PATCH 5/5] provide fixed tag versions of actions --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 15f5393a26a..f2c16289b13 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -143,12 +143,12 @@ jobs: needs: [setup-environment] steps: - name: Set up arkade - uses: alexellis/setup-arkade@master + uses: alexellis/setup-arkade@v3 - name: Install vmmeter run: | sudo -E arkade oci install ghcr.io/openfaasltd/vmmeter:latest --path /usr/local/bin/ - name: Run vmmeter - uses: self-actuated/vmmeter-action@master + uses: self-actuated/vmmeter-action@v1 - name: Checkout Repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Setup Go