Skip to content

Commit

Permalink
[saved objects] enable deletion of saved objects by type if configured (
Browse files Browse the repository at this point in the history
opensearch-project#6443)

* [saved objects] enable deletion of saved objects by type if configured

Adds the following settings:
```
migrations.delete.enabled
migrations.delete.types
```

`unknown` types already exist but the purpose of this type is for plugins
that are disabled. OpenSearch Dashboards gets confused when a plugin is not
defining a saved object type but the saved object exists. This can occur
when migrating from a non-OSD version and there exists non-compatiable
saved objects.

If OSD is failing to migrate an index because of a document, I can now
configure OSD to delete types of saved objects that I specified because
I know that these types should not be carried over.

resolves: opensearch-project#1040

Signed-off-by: Kawika Avilla <[email protected]>

* address comments

Signed-off-by: Kawika Avilla <[email protected]>

* Changeset file for PR opensearch-project#6443 created/updated

---------

Signed-off-by: Kawika Avilla <[email protected]>
Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
  • Loading branch information
kavilla and opensearch-changeset-bot[bot] authored May 5, 2024
1 parent b5121d3 commit 7a8c281
Show file tree
Hide file tree
Showing 13 changed files with 8,660 additions and 14 deletions.
2 changes: 2 additions & 0 deletions changelogs/fragments/6443.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
feat:
- Adds `migrations.delete` to delete saved objects by type during a migration ([#6443](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6443))
8 changes: 7 additions & 1 deletion config/opensearch_dashboards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,12 @@
# Set the value to true to enable workspace feature
# workspace.enabled: false

# Optional settings to specify saved object types to be deleted during migration.
# This feature can help address compatibility issues that may arise during the migration of saved objects, such as types defined by legacy applications.
# Please note, using this feature carries a risk. Deleting saved objects during migration could potentially lead to unintended data loss. Use with caution.
# migrations.delete.enabled: false
# migrations.delete.types: []

# Set the value to true to enable Ui Metric Collectors in Usage Collector
# This publishes the Application Usage and UI Metrics into the saved object, which can be accessed by /api/stats?extended=true&legacy=true&exclude_usage=false
# usageCollection.uiMetric.enabled: false
# usageCollection.uiMetric.enabled: false
222 changes: 222 additions & 0 deletions src/core/server/saved_objects/migrations/core/index_migrator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,228 @@ describe('IndexMigrator', () => {
});
});

test('deletes saved objects by type if configured', async () => {
const { client } = testOpts;

const deleteType = 'delete_type';

const rawConfig = configMock.create();
rawConfig.get.mockImplementation((path) => {
if (path === 'migrations.delete.enabled') {
return true;
}
if (path === 'migrations.delete.types') {
return [deleteType];
}
});
testOpts.opensearchDashboardsRawConfig = rawConfig;

testOpts.mappingProperties = { foo: { type: 'text' } as any };

withIndex(client, {
index: {
'.kibana_1': {
aliases: {},
mappings: {
properties: {
delete_type: { properties: { type: deleteType } },
},
},
},
},
});

await new IndexMigrator(testOpts).migrate();

expect(client.indices.create).toHaveBeenCalledWith({
body: {
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: {
foo: '625b32086eb1d1203564cf85062dd22e',
migrationVersion: '4a1746014a75ade3a714e1db5763276f',
namespace: '2f4316de49999235636386fe51dc06c1',
namespaces: '2f4316de49999235636386fe51dc06c1',
originId: '2f4316de49999235636386fe51dc06c1',
references: '7997cf5a56cc02bdc9c93361bde732b0',
type: '2f4316de49999235636386fe51dc06c1',
updated_at: '00da57df13e94e9d98437d13ace4bfe0',
},
},
properties: {
foo: { type: 'text' },
migrationVersion: { dynamic: 'true', type: 'object' },
namespace: { type: 'keyword' },
namespaces: { type: 'keyword' },
originId: { type: 'keyword' },
type: { type: 'keyword' },
updated_at: { type: 'date' },
references: {
type: 'nested',
properties: {
name: { type: 'keyword' },
type: { type: 'keyword' },
id: { type: 'keyword' },
},
},
},
},
settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
},
index: '.kibana_2',
});
});

test('retains saved objects by type if delete is not enabled', async () => {
const { client } = testOpts;

const deleteType = 'delete_type';

const rawConfig = configMock.create();
rawConfig.get.mockImplementation((path) => {
if (path === 'migrations.delete.enabled') {
return false;
}
if (path === 'migrations.delete.types') {
return [deleteType];
}
});
testOpts.opensearchDashboardsRawConfig = rawConfig;

testOpts.mappingProperties = { foo: { type: 'text' } as any };

withIndex(client, {
index: {
'.kibana_1': {
aliases: {},
mappings: {
properties: {
delete_type: { properties: { type: deleteType } },
},
},
},
},
});

await new IndexMigrator(testOpts).migrate();

expect(client.indices.create).toHaveBeenCalledWith({
body: {
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: {
foo: '625b32086eb1d1203564cf85062dd22e',
migrationVersion: '4a1746014a75ade3a714e1db5763276f',
namespace: '2f4316de49999235636386fe51dc06c1',
namespaces: '2f4316de49999235636386fe51dc06c1',
originId: '2f4316de49999235636386fe51dc06c1',
references: '7997cf5a56cc02bdc9c93361bde732b0',
type: '2f4316de49999235636386fe51dc06c1',
updated_at: '00da57df13e94e9d98437d13ace4bfe0',
},
},
properties: {
delete_type: { dynamic: false, properties: {} },
foo: { type: 'text' },
migrationVersion: { dynamic: 'true', type: 'object' },
namespace: { type: 'keyword' },
namespaces: { type: 'keyword' },
originId: { type: 'keyword' },
type: { type: 'keyword' },
updated_at: { type: 'date' },
references: {
type: 'nested',
properties: {
name: { type: 'keyword' },
type: { type: 'keyword' },
id: { type: 'keyword' },
},
},
},
},
settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
},
index: '.kibana_2',
});
});

test('retains saved objects by type if delete types does not exist', async () => {
const { client } = testOpts;

const deleteType = 'delete_type';
const retainType = 'retain_type';

const rawConfig = configMock.create();
rawConfig.get.mockImplementation((path) => {
if (path === 'migrations.delete.enabled') {
return true;
}
if (path === 'migrations.delete.types') {
return [deleteType];
}
});
testOpts.opensearchDashboardsRawConfig = rawConfig;

testOpts.mappingProperties = { foo: { type: 'text' } as any };

withIndex(client, {
index: {
'.kibana_1': {
aliases: {},
mappings: {
properties: {
retain_type: { properties: { type: retainType } },
},
},
},
},
});

await new IndexMigrator(testOpts).migrate();

expect(client.indices.create).toHaveBeenCalledWith({
body: {
mappings: {
dynamic: 'strict',
_meta: {
migrationMappingPropertyHashes: {
foo: '625b32086eb1d1203564cf85062dd22e',
migrationVersion: '4a1746014a75ade3a714e1db5763276f',
namespace: '2f4316de49999235636386fe51dc06c1',
namespaces: '2f4316de49999235636386fe51dc06c1',
originId: '2f4316de49999235636386fe51dc06c1',
references: '7997cf5a56cc02bdc9c93361bde732b0',
type: '2f4316de49999235636386fe51dc06c1',
updated_at: '00da57df13e94e9d98437d13ace4bfe0',
},
},
properties: {
retain_type: { dynamic: false, properties: {} },
foo: { type: 'text' },
migrationVersion: { dynamic: 'true', type: 'object' },
namespace: { type: 'keyword' },
namespaces: { type: 'keyword' },
originId: { type: 'keyword' },
type: { type: 'keyword' },
updated_at: { type: 'date' },
references: {
type: 'nested',
properties: {
name: { type: 'keyword' },
type: { type: 'keyword' },
id: { type: 'keyword' },
},
},
},
},
settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
},
index: '.kibana_2',
});
});

test('points the alias at the dest index', async () => {
const { client } = testOpts;

Expand Down
29 changes: 29 additions & 0 deletions src/core/server/saved_objects/migrations/core/index_migrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
* under the License.
*/

import { DeleteByQueryRequest } from '@opensearch-project/opensearch/api/types';
import { diffMappings } from './build_active_mappings';
import * as Index from './opensearch_index';
import { migrateRawDocs } from './migrate_raw_docs';
Expand Down Expand Up @@ -123,6 +124,7 @@ async function migrateIndex(context: Context): Promise<MigrationResult> {
const { client, alias, source, dest, log } = context;

await deleteIndexTemplates(context);
await deleteSavedObjectsByType(context);

log.info(`Creating index ${dest.indexName}.`);

Expand Down Expand Up @@ -171,6 +173,33 @@ async function deleteIndexTemplates({ client, log, obsoleteIndexTemplatePattern
return Promise.all(templateNames.map((name) => client.indices.deleteTemplate({ name: name! })));
}

/**
* Delete saved objects by type. If migrations.delete.types is specified,
* any saved objects that matches that type will be deleted.
*/
async function deleteSavedObjectsByType(context: Context) {
const { client, source, log, typesToDelete } = context;
if (!source.exists || !typesToDelete || typesToDelete.length === 0) {
return;
}

log.info(`Removing saved objects of types: ${typesToDelete.join(', ')}`);
const params = {
index: source.indexName,
body: {
query: {
bool: {
should: [...typesToDelete.map((type) => ({ term: { type } }))],
},
},
},
conflicts: 'proceed',
refresh: true,
} as DeleteByQueryRequest;
log.debug(`Delete by query params: ${JSON.stringify(params)}`);
return client.deleteByQuery(params);
}

/**
* Moves all docs from sourceIndex to destIndex, migrating each as necessary.
* This moves documents from the concrete index, rather than the alias, to prevent
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
* under the License.
*/

import { disableUnknownTypeMappingFields } from './migration_context';
import { disableUnknownTypeMappingFields, deleteTypeMappingsFields } from './migration_context';
import { configMock } from '../../../config/mocks';

describe('disableUnknownTypeMappingFields', () => {
const sourceMappings = {
Expand Down Expand Up @@ -97,3 +98,87 @@ describe('disableUnknownTypeMappingFields', () => {
});
});
});

describe('deleteTypeMappingsFields', () => {
it('should delete specified type mappings fields', () => {
const targetMappings = {
properties: {
type1: { type: 'text' },
type2: { type: 'keyword' },
type3: { type: 'boolean' },
},
} as const;

const rawConfig = configMock.create();
rawConfig.get.mockImplementation((path) => {
if (path === 'migrations.delete.enabled') {
return true;
}
if (path === 'migrations.delete.types') {
return ['type1', 'type3'];
}
});

const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);

expect(updatedMappings.properties).toEqual({
type2: { type: 'keyword' },
});
});

it('should not delete any type mappings fields if delete is not enabled', () => {
const targetMappings = {
properties: {
type1: { type: 'text' },
type2: { type: 'keyword' },
type3: { type: 'boolean' },
},
} as const;

const rawConfig = configMock.create();
rawConfig.get.mockImplementation((path) => {
if (path === 'migrations.delete.enabled') {
return false;
}
if (path === 'migrations.delete.types') {
return ['type1', 'type3'];
}
});

const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);

expect(updatedMappings.properties).toEqual({
type1: { type: 'text' },
type2: { type: 'keyword' },
type3: { type: 'boolean' },
});
});

it('should not delete any type mappings fields if delete types are not specified', () => {
const targetMappings = {
properties: {
type1: { type: 'text' },
type2: { type: 'keyword' },
type3: { type: 'boolean' },
},
} as const;

const rawConfig = configMock.create();
rawConfig.get.mockImplementation((path) => {
if (path === 'migrations.delete.enabled') {
return true;
}
if (path === 'migrations.delete.types') {
return [];
}
});

const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig);

expect(updatedMappings.properties).toEqual({
type1: { type: 'text' },
type2: { type: 'keyword' },
type3: { type: 'boolean' },
});
});
});
Loading

0 comments on commit 7a8c281

Please sign in to comment.