[Fleet] add DELETED file update task

Adds Kibana task for updating file status to DELETED when there are no
file chunks.

x-pack/plugins/fleet/common/constants/index.ts

@@ -25,6 +25,8 @@ export * from './authz';
 // setting in the future?
 export const SO_SEARCH_LIMIT = 10000;
 
+export const ES_SEARCH_LIMIT = 10000;
+
 export const FLEET_SERVER_INDICES_VERSION = 1;
 
 export const FLEET_SERVER_ARTIFACTS_INDEX = '.fleet-artifacts';

x-pack/plugins/fleet/server/constants/fleet_es_assets.ts

@@ -194,3 +194,7 @@ on_failure:
       field: error.message
       value:
         - 'failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}'`;
+
+// File storage indexes supporting endpoint Upload/download
+export const FILE_STORAGE_METADATA_INDEX = '.fleet-*-files';
+export const FILE_STORAGE_DATA_INDEX = '.fleet-*-file-data';

x-pack/plugins/fleet/server/plugin.ts

@@ -111,6 +111,7 @@ import { BulkActionsResolver } from './services/agents';
 import type { PackagePolicyService } from './services/package_policy_service';
 import { PackagePolicyServiceImpl } from './services/package_policy';
 import { registerFleetUsageLogger, startFleetUsageLogger } from './services/fleet_usage_logger';
+import { CheckDeletedFilesTask } from './tasks/check_deleted_files_task';
 
 export interface FleetSetupDeps {
   security: SecurityPluginSetup;
@@ -220,6 +221,7 @@ export class FleetPlugin
   private readonly fleetStatus$: BehaviorSubject<ServiceStatus>;
   private bulkActionsResolver?: BulkActionsResolver;
   private fleetUsageSender?: FleetUsageSender;
+  private checkDeletedFilesTask?: CheckDeletedFilesTask;
 
   private agentService?: AgentService;
   private packageService?: PackageService;
@@ -426,6 +428,11 @@ export class FleetPlugin
 
     this.telemetryEventsSender.setup(deps.telemetry);
     this.bulkActionsResolver = new BulkActionsResolver(deps.taskManager, core);
+    this.checkDeletedFilesTask = new CheckDeletedFilesTask({
+      core,
+      taskManager: deps.taskManager,
+      logFactory: this.initializerContext.logger,
+    });
   }
 
   public start(core: CoreStart, plugins: FleetStartDeps): FleetStartContract {
@@ -457,6 +464,7 @@ export class FleetPlugin
     this.telemetryEventsSender.start(plugins.telemetry, core);
     this.bulkActionsResolver?.start(plugins.taskManager);
     this.fleetUsageSender?.start(plugins.taskManager);
+    this.checkDeletedFilesTask?.start({ taskManager: plugins.taskManager });
     startFleetUsageLogger(plugins.taskManager);
 
     const logger = appContextService.getLogger();

x-pack/plugins/fleet/server/services/files/index.test.ts

@@ -0,0 +1,204 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClientMock } from '@kbn/core/server/mocks';
+import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
+
+import { ES_SEARCH_LIMIT } from '../../../common/constants';
+import {
+  FILE_STORAGE_DATA_INDEX,
+  FILE_STORAGE_METADATA_INDEX,
+} from '../../constants/fleet_es_assets';
+
+import { fileIdsWithoutChunksByIndex, getFilesByStatus, updateFilesStatus } from '.';
+
+const ENDPOINT_FILE_METADATA_INDEX = '.fleet-endpoint-files';
+const ENDPOINT_FILE_INDEX = '.fleet-endpoint-file-data';
+
+describe('files service', () => {
+  let esClientMock: ElasticsearchClientMock;
+  const abortController = new AbortController();
+
+  beforeEach(() => {
+    esClientMock = elasticsearchServiceMock.createElasticsearchClient();
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  describe('#getFilesByStatus()', () => {
+    it('should return expected values', async () => {
+      const status = 'READY';
+      esClientMock.search.mockResolvedValueOnce({
+        took: 5,
+        timed_out: false,
+        _shards: {
+          total: 1,
+          successful: 1,
+          skipped: 0,
+          failed: 0,
+        },
+        hits: {
+          hits: [
+            {
+              _index: ENDPOINT_FILE_METADATA_INDEX,
+              _id: 'someid1',
+            },
+            {
+              _index: ENDPOINT_FILE_METADATA_INDEX,
+              _id: 'someid2',
+            },
+          ],
+        },
+      });
+
+      const result = await getFilesByStatus(esClientMock, abortController, status);
+
+      expect(esClientMock.search).toBeCalledWith(
+        {
+          index: FILE_STORAGE_METADATA_INDEX,
+          body: {
+            size: ES_SEARCH_LIMIT,
+            query: {
+              term: {
+                'file.Status.keyword': status,
+              },
+            },
+            _source: false,
+          },
+          ignore_unavailable: true,
+        },
+        { signal: abortController.signal }
+      );
+      expect(result).toEqual([
+        { _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'someid1' },
+        { _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'someid2' },
+      ]);
+    });
+  });
+
+  describe('#fileIdsWithoutChunks()', () => {
+    it('should return expected values', async () => {
+      esClientMock.search.mockResolvedValueOnce({
+        took: 5,
+        timed_out: false,
+        _shards: {
+          total: 1,
+          successful: 1,
+          skipped: 0,
+          failed: 0,
+        },
+        hits: {
+          hits: [
+            {
+              _index: ENDPOINT_FILE_INDEX,
+              _id: 'keep1',
+              _source: {
+                bid: 'keep1',
+              },
+            },
+            {
+              _index: ENDPOINT_FILE_INDEX,
+              _id: 'keep2',
+              _source: {
+                bid: 'keep2',
+              },
+            },
+          ],
+        },
+      });
+
+      const files = [
+        { _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'keep1' },
+        { _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'keep2' },
+        { _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'delete1' },
+        { _index: ENDPOINT_FILE_METADATA_INDEX, _id: 'delete2' },
+      ];
+      const { fileIdsByIndex: deletedFileIdsByIndex, allFileIds: allDeletedFileIds } =
+        await fileIdsWithoutChunksByIndex(esClientMock, abortController, files);
+
+      expect(esClientMock.search).toBeCalledWith(
+        {
+          index: FILE_STORAGE_DATA_INDEX,
+          body: {
+            size: ES_SEARCH_LIMIT,
+            query: {
+              bool: {
+                must: [
+                  {
+                    terms: {
+                      'bid.keyword': Array.from(files.map((file) => file._id)),
+                    },
+                  },
+                  {
+                    term: {
+                      last: true,
+                    },
+                  },
+                ],
+              },
+            },
+            _source: ['bid'],
+          },
+        },
+        { signal: abortController.signal }
+      );
+      expect(deletedFileIdsByIndex).toEqual({
+        [ENDPOINT_FILE_METADATA_INDEX]: new Set(['delete1', 'delete2']),
+      });
+      expect(allDeletedFileIds).toEqual(new Set(['delete1', 'delete2']));
+    });
+  });
+
+  describe('#updateFilesStatus()', () => {
+    it('calls esClient.updateByQuery with expected values', () => {
+      const FAKE_INTEGRATION_METADATA_INDEX = '.fleet-someintegration-files';
+      const files = {
+        [ENDPOINT_FILE_METADATA_INDEX]: new Set(['delete1', 'delete2']),
+        [FAKE_INTEGRATION_METADATA_INDEX]: new Set(['delete2', 'delete3']),
+      };
+      const status = 'DELETED';
+      updateFilesStatus(esClientMock, abortController, files, status);
+
+      expect(esClientMock.updateByQuery).toHaveBeenNthCalledWith(
+        1,
+        {
+          index: ENDPOINT_FILE_METADATA_INDEX,
+          refresh: true,
+          query: {
+            ids: {
+              values: Array.from(files[ENDPOINT_FILE_METADATA_INDEX]),
+            },
+          },
+          script: {
+            source: `ctx._source.file.Status = '${status}'`,
+            lang: 'painless',
+          },
+        },
+        { signal: abortController.signal }
+      );
+      expect(esClientMock.updateByQuery).toHaveBeenNthCalledWith(
+        2,
+        {
+          index: FAKE_INTEGRATION_METADATA_INDEX,
+          refresh: true,
+          query: {
+            ids: {
+              values: Array.from(files[FAKE_INTEGRATION_METADATA_INDEX]),
+            },
+          },
+          script: {
+            source: `ctx._source.file.Status = '${status}'`,
+            lang: 'painless',
+          },
+        },
+        { signal: abortController.signal }
+      );
+    });
+  });
+});