Skip to content

Commit

Permalink
[Fleet] enabling diagnostics feature flag and changed query for files…
Browse files Browse the repository at this point in the history
… to use upload_id (#149575)

## Summary

Closes #141074

Enabled feature flag and tweaked implementation to find file by
`upload_id` rather than doc id.

How to test:
- Start local kibana, start Fleet Server, enroll Elastic Agent from
local (pull [these
changes](elastic/elastic-agent#1703) )
- Click on Request Diagnostics action on the Agent
- The diagnostics file should appear on Agent Details / Diagnostics tab.
- The action should be completed on Agent activity

<img width="1585" alt="image"
src="https://user-images.githubusercontent.com/90178898/214805187-2b1abe34-ba7e-4612-9fad-7ef1f5942f47.png">
<img width="745" alt="image"
src="https://user-images.githubusercontent.com/90178898/214805997-20fdaa01-e4c5-461c-b395-1b1e43117f8a.png">

The file metadata and binary can be queried from these indices:

```
GET .fleet-files-agent/_search

GET .fleet-file-data-agent/_search
```

Tweaked the implementation so that the pending actions are showing up as
soon as the `.fleet-actions` record is created (it can take several
minutes until the action result is ready)
Plus added a tooltip for error status

<img width="948" alt="image"
src="https://user-images.githubusercontent.com/90178898/214841337-eacbb1fc-4934-4d8b-9d52-8db4502d2493.png">



### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
  • Loading branch information
juliaElastic authored Jan 27, 2023
1 parent 0174705 commit 2d67837
Show file tree
Hide file tree
Showing 5 changed files with 45 additions and 17 deletions.
2 changes: 1 addition & 1 deletion x-pack/plugins/fleet/common/experimental_features.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export const allowedExperimentalValues = Object.freeze({
createPackagePolicyMultiPageLayout: true,
packageVerification: true,
showDevtoolsRequest: true,
diagnosticFileUploadEnabled: false,
diagnosticFileUploadEnabled: true,
experimentalDataStreamSettings: false,
displayAgentMetrics: true,
showIntegrationsSubcategories: false,
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugins/fleet/common/types/models/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ export interface AgentDiagnostics {
name: string;
createTime: string;
filePath: string;
status: 'READY' | 'AWAITING_UPLOAD' | 'DELETED' | 'IN_PROGRESS';
status: 'READY' | 'AWAITING_UPLOAD' | 'DELETED' | 'IN_PROGRESS' | 'FAILED';
actionId: string;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/

import type { EuiTableFieldDataColumnType } from '@elastic/eui';
import { EuiToolTip } from '@elastic/eui';
import {
EuiBasicTable,
EuiButton,
Expand Down Expand Up @@ -131,7 +132,10 @@ export const AgentDiagnosticsTab: React.FunctionComponent<AgentDiagnosticsProps>
</EuiText>
) : (
<EuiText color="subdued">
<EuiIcon type="alert" color="red" /> &nbsp;
<EuiToolTip content={`Diagnostics status: ${currentItem?.status}`}>
<EuiIcon type="alert" color="red" />
</EuiToolTip>
&nbsp;
{currentItem?.name}
</EuiText>
);
Expand Down
49 changes: 36 additions & 13 deletions x-pack/plugins/fleet/server/services/agents/uploads.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,23 @@ export async function getAgentUploads(
const getFile = async (fileId: string) => {
if (!fileId) return;
try {
const file = await esClient.get({
const fileResponse = await esClient.search({
index: FILE_STORAGE_METADATA_AGENT_INDEX,
id: fileId,
query: {
bool: {
filter: {
term: { upload_id: fileId },
},
},
},
});
if (fileResponse.hits.total === 0) {
appContextService.getLogger().debug(`No matches for upload_id ${fileId}`);
return;
}
return {
id: file._id,
...(file._source as any)?.file,
id: fileResponse.hits.hits[0]._id,
...(fileResponse.hits.hits[0]._source as any)?.file,
};
} catch (err) {
if (err.statusCode === 404) {
Expand All @@ -56,13 +66,13 @@ export async function getAgentUploads(

const results = [];
for (const action of actions) {
const file = await getFile(action.fileId);
const file = action.fileId ? await getFile(action.fileId) : undefined;
const fileName = file?.name ?? `${moment(action.timestamp!).format('YYYY-MM-DD HH:mm:ss')}.zip`;
const filePath = file ? agentRouteService.getAgentFileDownloadLink(file.id, file.name) : '';
const result = {
actionId: action.actionId,
id: file?.id ?? action.actionId,
status: file?.Status ?? 'IN_PROGRESS',
status: file?.Status ?? (action.error ? 'FAILED' : 'IN_PROGRESS'),
name: fileName,
createTime: action.timestamp!,
filePath,
Expand All @@ -76,7 +86,7 @@ export async function getAgentUploads(
async function _getRequestDiagnosticsActions(
esClient: ElasticsearchClient,
agentId: string
): Promise<Array<{ actionId: string; timestamp?: string; fileId: string }>> {
): Promise<Array<{ actionId: string; timestamp?: string; fileId?: string; error?: string }>> {
const agentActionRes = await esClient.search<any>({
index: AGENT_ACTIONS_INDEX,
ignore_unavailable: true,
Expand All @@ -99,14 +109,17 @@ async function _getRequestDiagnosticsActions(
},
});

const agentActionIds = agentActionRes.hits.hits.map((hit) => hit._source?.action_id as string);
const agentActions = agentActionRes.hits.hits.map((hit) => ({
actionId: hit._source?.action_id as string,
timestamp: hit._source?.['@timestamp'],
}));

if (agentActionIds.length === 0) {
if (agentActions.length === 0) {
return [];
}

try {
const actionResults = await esClient.search<any>({
const actionResultsRes = await esClient.search<any>({
index: AGENT_ACTIONS_RESULTS_INDEX,
ignore_unavailable: true,
size: SO_SEARCH_LIMIT,
Expand All @@ -115,7 +128,7 @@ async function _getRequestDiagnosticsActions(
must: [
{
terms: {
action_id: agentActionIds,
action_id: agentActions.map((action) => action.actionId),
},
},
{
Expand All @@ -127,11 +140,21 @@ async function _getRequestDiagnosticsActions(
},
},
});
return actionResults.hits.hits.map((hit) => ({
const actionResults = actionResultsRes.hits.hits.map((hit) => ({
actionId: hit._source?.action_id as string,
timestamp: hit._source?.['@timestamp'],
fileId: hit._source?.data?.file_id as string,
fileId: hit._source?.data?.upload_id as string,
error: hit._source?.error,
}));
return agentActions.map((action) => {
const actionResult = actionResults.find((result) => result.actionId === action.actionId);
return {
actionId: action.actionId,
timestamp: actionResult?.timestamp ?? action.timestamp,
fileId: actionResult?.fileId,
error: actionResult?.error,
};
});
} catch (err) {
if (err.statusCode === 404) {
// .fleet-actions-results does not yet exist
Expand Down
3 changes: 2 additions & 1 deletion x-pack/test/fleet_api_integration/apis/agents/uploads.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ export default function (providerContext: FtrProviderContext) {
agent_id: 'agent1',
'@timestamp': '2022-10-07T12:00:00.000Z',
data: {
file_id: 'file1',
upload_id: 'file1',
},
},
},
Expand All @@ -67,6 +67,7 @@ export default function (providerContext: FtrProviderContext) {
body: {
doc_as_upsert: true,
doc: {
upload_id: 'file1',
file: {
ChunkSize: 4194304,
extension: 'zip',
Expand Down

0 comments on commit 2d67837

Please sign in to comment.