Skip to content

Commit

Permalink
Sm/levenshtein-missing-types (#1374)
Browse files Browse the repository at this point in the history
* feat: did you mean when types aren't in Registry

* refactor: nicer output formatting

* test: skip a test

* refactor: remove unused handler

* refactor: complexity, simplified naming

---------

Co-authored-by: Willie Ruemmele <[email protected]>
  • Loading branch information
mshanemc and WillieRuemmele authored Jul 29, 2024
1 parent 7870179 commit a66f50d
Show file tree
Hide file tree
Showing 9 changed files with 452 additions and 361 deletions.
13 changes: 13 additions & 0 deletions messages/sdr.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,3 +190,16 @@ If the type is available via Metadata API but not in the registry

- Open an issue <https://github.com/forcedotcom/cli/issues>
- Add the type via PR. Instructions: <https://github.com/forcedotcom/source-deploy-retrieve/blob/main/contributing/metadata.md>

# type_name_suggestions

Confirm the metadata type name is correct. Validate against the registry at:
<https://github.com/forcedotcom/source-deploy-retrieve/blob/main/src/registry/metadataRegistry.json>

If the type is not listed in the registry, check that it has Metadata API support via the Metadata Coverage Report:
<https://developer.salesforce.com/docs/metadata-coverage>

If the type is available via Metadata API but not in the registry

- Open an issue <https://github.com/forcedotcom/cli/issues>
- Add the type via PR. Instructions: <https://github.com/forcedotcom/source-deploy-retrieve/blob/main/contributing/metadata.md>
8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"node": ">=18.0.0"
},
"dependencies": {
"@salesforce/core": "^8.2.1",
"@salesforce/core": "^8.2.3",
"@salesforce/kit": "^3.1.6",
"@salesforce/ts-types": "^2.0.10",
"fast-levenshtein": "^3.0.0",
Expand All @@ -39,9 +39,9 @@
"proxy-agent": "^6.4.0"
},
"devDependencies": {
"@jsforce/jsforce-node": "^3.3.2",
"@jsforce/jsforce-node": "^3.3.3",
"@salesforce/cli-plugins-testkit": "^5.3.20",
"@salesforce/dev-scripts": "^10.2.2",
"@salesforce/dev-scripts": "^10.2.5",
"@types/deep-equal-in-any-order": "^1.0.1",
"@types/fast-levenshtein": "^0.0.4",
"@types/graceful-fs": "^4.1.9",
Expand All @@ -54,7 +54,7 @@
"mocha-snap": "^5.0.0",
"ts-node": "^10.9.2",
"ts-patch": "^3.2.1",
"typescript": "^5.5.3"
"typescript": "^5.5.4"
},
"scripts": {
"build": "wireit",
Expand Down
230 changes: 107 additions & 123 deletions src/collections/componentSetBuilder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,137 +70,132 @@ export class ComponentSetBuilder {
* @param options: options for creating a ComponentSet
*/

// eslint-disable-next-line complexity
public static async build(options: ComponentSetOptions): Promise<ComponentSet> {
const logger = Logger.childFromRoot('componentSetBuilder');
let componentSet: ComponentSet | undefined;

const { sourcepath, manifest, metadata, packagenames, apiversion, sourceapiversion, org, projectDir } = options;
const registryAccess = new RegistryAccess(undefined, projectDir);

try {
if (sourcepath) {
logger.debug(`Building ComponentSet from sourcepath: ${sourcepath.join(', ')}`);
const fsPaths = sourcepath.map(validateAndResolvePath);
componentSet = ComponentSet.fromSource({
fsPaths,
registry: registryAccess,
});
}
const { sourcepath, manifest, metadata, packagenames, org } = options;
const registry = new RegistryAccess(undefined, options.projectDir);

// Return empty ComponentSet and use packageNames in the connection via `.retrieve` options
if (packagenames) {
logger.debug(`Building ComponentSet for packagenames: ${packagenames.toString()}`);
componentSet ??= new ComponentSet(undefined, registryAccess);
}
if (sourcepath) {
logger.debug(`Building ComponentSet from sourcepath: ${sourcepath.join(', ')}`);
const fsPaths = sourcepath.map(validateAndResolvePath);
componentSet = ComponentSet.fromSource({
fsPaths,
registry,
});
}

// Resolve manifest with source in package directories.
if (manifest) {
logger.debug(`Building ComponentSet from manifest: ${manifest.manifestPath}`);
assertFileExists(manifest.manifestPath);

logger.debug(`Searching in packageDir: ${manifest.directoryPaths.join(', ')} for matching metadata`);
componentSet = await ComponentSet.fromManifest({
manifestPath: manifest.manifestPath,
resolveSourcePaths: manifest.directoryPaths,
forceAddWildcards: true,
destructivePre: manifest.destructiveChangesPre,
destructivePost: manifest.destructiveChangesPost,
registry: registryAccess,
});
}
// Return empty ComponentSet and use packageNames in the connection via `.retrieve` options
if (packagenames) {
logger.debug(`Building ComponentSet for packagenames: ${packagenames.toString()}`);
componentSet ??= new ComponentSet(undefined, registry);
}

// Resolve metadata entries with source in package directories.
if (metadata) {
logger.debug(`Building ComponentSet from metadata: ${metadata.metadataEntries.toString()}`);
const directoryPaths = metadata.directoryPaths;
componentSet ??= new ComponentSet(undefined, registryAccess);
const componentSetFilter = new ComponentSet(undefined, registryAccess);

// Build a Set of metadata entries
metadata.metadataEntries
.map(entryToTypeAndName(registryAccess))
.flatMap(typeAndNameToMetadataComponents({ directoryPaths, registry: registryAccess }))
.map(addToComponentSet(componentSet))
.map(addToComponentSet(componentSetFilter));

logger.debug(`Searching for matching metadata in directories: ${directoryPaths.join(', ')}`);

// add destructive changes if defined. Because these are deletes, all entries
// are resolved to SourceComponents
if (metadata.destructiveEntriesPre) {
metadata.destructiveEntriesPre
.map(entryToTypeAndName(registryAccess))
.map(assertNoWildcardInDestructiveEntries)
.flatMap(typeAndNameToMetadataComponents({ directoryPaths, registry: registryAccess }))
.map((mdComponent) => new SourceComponent({ type: mdComponent.type, name: mdComponent.fullName }))
.map(addToComponentSet(componentSet, DestructiveChangesType.PRE));
}
if (metadata.destructiveEntriesPost) {
metadata.destructiveEntriesPost
.map(entryToTypeAndName(registryAccess))
.map(assertNoWildcardInDestructiveEntries)
.flatMap(typeAndNameToMetadataComponents({ directoryPaths, registry: registryAccess }))
.map((mdComponent) => new SourceComponent({ type: mdComponent.type, name: mdComponent.fullName }))
.map(addToComponentSet(componentSet, DestructiveChangesType.POST));
}
// Resolve manifest with source in package directories.
if (manifest) {
logger.debug(`Building ComponentSet from manifest: ${manifest.manifestPath}`);
assertFileExists(manifest.manifestPath);

logger.debug(`Searching in packageDir: ${manifest.directoryPaths.join(', ')} for matching metadata`);
componentSet = await ComponentSet.fromManifest({
manifestPath: manifest.manifestPath,
resolveSourcePaths: manifest.directoryPaths,
forceAddWildcards: true,
destructivePre: manifest.destructiveChangesPre,
destructivePost: manifest.destructiveChangesPost,
registry,
});
}

const resolvedComponents = ComponentSet.fromSource({
fsPaths: directoryPaths,
include: componentSetFilter,
registry: registryAccess,
});

if (resolvedComponents.forceIgnoredPaths) {
// if useFsForceIgnore = true, then we won't be able to resolve a forceignored path,
// which we need to do to get the ignored source component
const resolver = new MetadataResolver(registryAccess, undefined, false);

for (const ignoredPath of resolvedComponents.forceIgnoredPaths ?? []) {
resolver.getComponentsFromPath(ignoredPath).map((ignored) => {
componentSet = componentSet?.filter(
(resolved) => !(resolved.fullName === ignored.name && resolved.type === ignored.type)
);
});
}
componentSet.forceIgnoredPaths = resolvedComponents.forceIgnoredPaths;
}
// Resolve metadata entries with source in package directories.
if (metadata) {
logger.debug(`Building ComponentSet from metadata: ${metadata.metadataEntries.toString()}`);
const directoryPaths = metadata.directoryPaths;
componentSet ??= new ComponentSet(undefined, registry);
const componentSetFilter = new ComponentSet(undefined, registry);

// Build a Set of metadata entries
metadata.metadataEntries
.map(entryToTypeAndName(registry))
.flatMap(typeAndNameToMetadataComponents({ directoryPaths, registry }))
.map(addToComponentSet(componentSet))
.map(addToComponentSet(componentSetFilter));

logger.debug(`Searching for matching metadata in directories: ${directoryPaths.join(', ')}`);

// add destructive changes if defined. Because these are deletes, all entries
// are resolved to SourceComponents
if (metadata.destructiveEntriesPre) {
metadata.destructiveEntriesPre
.map(entryToTypeAndName(registry))
.map(assertNoWildcardInDestructiveEntries)
.flatMap(typeAndNameToMetadataComponents({ directoryPaths, registry }))
.map((mdComponent) => new SourceComponent({ type: mdComponent.type, name: mdComponent.fullName }))
.map(addToComponentSet(componentSet, DestructiveChangesType.PRE));
}
if (metadata.destructiveEntriesPost) {
metadata.destructiveEntriesPost
.map(entryToTypeAndName(registry))
.map(assertNoWildcardInDestructiveEntries)
.flatMap(typeAndNameToMetadataComponents({ directoryPaths, registry }))
.map((mdComponent) => new SourceComponent({ type: mdComponent.type, name: mdComponent.fullName }))
.map(addToComponentSet(componentSet, DestructiveChangesType.POST));
}

resolvedComponents.toArray().map(addToComponentSet(componentSet));
const resolvedComponents = ComponentSet.fromSource({
fsPaths: directoryPaths,
include: componentSetFilter,
registry,
});

if (resolvedComponents.forceIgnoredPaths) {
// if useFsForceIgnore = true, then we won't be able to resolve a forceignored path,
// which we need to do to get the ignored source component
const resolver = new MetadataResolver(registry, undefined, false);

for (const ignoredPath of resolvedComponents.forceIgnoredPaths ?? []) {
resolver.getComponentsFromPath(ignoredPath).map((ignored) => {
componentSet = componentSet?.filter(
(resolved) => !(resolved.fullName === ignored.name && resolved.type === ignored.type)
);
});
}
componentSet.forceIgnoredPaths = resolvedComponents.forceIgnoredPaths;
}

// Resolve metadata entries with an org connection
if (org) {
componentSet ??= new ComponentSet(undefined, registryAccess);
resolvedComponents.toArray().map(addToComponentSet(componentSet));
}

logger.debug(
`Building ComponentSet from targetUsername: ${org.username} ${
metadata ? `filtered by metadata: ${metadata.metadataEntries.toString()}` : ''
}`
);
// Resolve metadata entries with an org connection
if (org) {
componentSet ??= new ComponentSet(undefined, registry);

const mdMap = metadata
? buildMapFromComponents(metadata.metadataEntries.map(entryToTypeAndName(registryAccess)))
: (new Map() as MetadataMap);
logger.debug(
`Building ComponentSet from targetUsername: ${org.username} ${
metadata ? `filtered by metadata: ${metadata.metadataEntries.toString()}` : ''
}`
);

const fromConnection = await ComponentSet.fromConnection({
usernameOrConnection: (await StateAggregator.getInstance()).aliases.getUsername(org.username) ?? org.username,
componentFilter: getOrgComponentFilter(org, mdMap, metadata),
metadataTypes: mdMap.size ? Array.from(mdMap.keys()) : undefined,
registry: registryAccess,
});
const mdMap = metadata
? buildMapFromComponents(metadata.metadataEntries.map(entryToTypeAndName(registry)))
: (new Map() as MetadataMap);

fromConnection.toArray().map(addToComponentSet(componentSet));
}
} catch (e) {
return componentSetBuilderErrorHandler(e);
const fromConnection = await ComponentSet.fromConnection({
usernameOrConnection: (await StateAggregator.getInstance()).aliases.getUsername(org.username) ?? org.username,
componentFilter: getOrgComponentFilter(org, mdMap, metadata),
metadataTypes: mdMap.size ? Array.from(mdMap.keys()) : undefined,
registry,
});

fromConnection.toArray().map(addToComponentSet(componentSet));
}

// there should have been a componentSet created by this point.
componentSet = assertComponentSetIsNotUndefined(componentSet);
componentSet.apiVersion ??= apiversion;
componentSet.sourceApiVersion ??= sourceapiversion;
componentSet.projectDirectory = projectDir;
componentSet.apiVersion ??= options.apiversion;
componentSet.sourceApiVersion ??= options.sourceapiversion;
componentSet.projectDirectory = options.projectDir;

logComponents(logger, componentSet);
return componentSet;
Expand All @@ -214,17 +209,6 @@ const addToComponentSet =
return cmp;
};

const componentSetBuilderErrorHandler = (e: unknown): never => {
if (e instanceof Error && e.message.includes('Missing metadata type definition in registry for id')) {
// to remain generic to catch missing metadata types regardless of parameters, split on '
// example message : Missing metadata type definition in registry for id 'NonExistentType'
const issueType = e.message.split("'")[1];
throw new SfError(`The specified metadata type is unsupported: [${issueType}]`);
} else {
throw e;
}
};

const validateAndResolvePath = (filepath: string): string => path.resolve(assertFileExists(filepath));

const assertFileExists = (filepath: string): string => {
Expand Down
57 changes: 57 additions & 0 deletions src/registry/levenshtein.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* Copyright (c) 2023, salesforce.com, inc.
* All rights reserved.
* Licensed under the BSD 3-Clause license.
* For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
*/

import { Messages } from '@salesforce/core/messages';
import * as Levenshtein from 'fast-levenshtein';
import { MetadataRegistry } from './types';

Messages.importMessagesDirectory(__dirname);
const messages = Messages.loadMessages('@salesforce/source-deploy-retrieve', 'sdr');

/** "did you mean" for Metadata type names */
export const getTypeSuggestions = (registry: MetadataRegistry, typeName: string): string[] => {
const scores = getScores(
Object.values(registry.types).map((t) => t.name),
typeName
);

const guesses = getLowestScores(scores).map((guess) => guess.registryKey);
return [
...(guesses.length
? [
`Did you mean one of the following types? [${guesses.join(',')}]`,
'', // Add a blank line for better readability
]
: []),
messages.getMessage('type_name_suggestions'),
];
};

export const getSuffixGuesses = (suffixes: string[], input: string): string[] => {
const scores = getScores(suffixes, input);
return getLowestScores(scores).map((g) => g.registryKey);
};

type LevenshteinScore = {
registryKey: string;
score: number;
};

const getScores = (choices: string[], input: string): LevenshteinScore[] =>
choices.map((registryKey) => ({
registryKey,
score: Levenshtein.get(input, registryKey, { useCollator: true }),
}));

/** Levenshtein uses positive integers for scores, find all scores that match the lowest score */
const getLowestScores = (scores: LevenshteinScore[]): LevenshteinScore[] => {
const sortedScores = scores.sort(levenshteinSorter);
const lowestScore = scores[0].score;
return sortedScores.filter((score) => score.score === lowestScore);
};

const levenshteinSorter = (a: LevenshteinScore, b: LevenshteinScore): number => a.score - b.score;
Loading

0 comments on commit a66f50d

Please sign in to comment.