Port pr v4.1.1 (#432)

* fix: export all public TS types and interfaces (#423)

* fix: making exports in the right format

@W-9727845@

* chore: removing yarn changes

* fix: xml nodes might not be arrays (#424)

Co-authored-by: Willie Ruemmele <[email protected]>

* fix: throw an error for unexpected child types (#426)

* fix: throw an error for unexpected child types

* fix: throw better error when transforming invalid child

* fix: ensure valid children during retrieve

Co-authored-by: Willie Ruemmele <[email protected]>

* fix: force fullName to CustomLabels (#427)

* fix: force fullName to CustomLabels

* chore: update prop name

* fix: backward support for toolbelt suffix differences

* Revert "fix: backward support for toolbelt suffix differences"

This reverts commit 9cf9505.

Co-authored-by: mshanemc <[email protected]>
Co-authored-by: Willie Ruemmele <[email protected]>

* fix: forceIgnore does not work for `SFDX: Retrieve Source in Manifest from Org` in vscode extension (#413)

* fix: forceIgnore does not work with retrieve manifest

* fix: matchingContent subfolder

* Wr/content type of undefined (#429)

* chore: initial pass on preventing improperly structured static resources

* chore: added error message

Co-authored-by: Shane McLaughlin <[email protected]>

* fix: support toolbelt suffixes (#428)

* fix: force fullName to CustomLabels

* chore: update prop name

* fix: backward support for toolbelt suffix differences

* Revert "fix: backward support for toolbelt suffix differences"

This reverts commit 9cf9505.

* fix: support toolbelt suffixes

* feat: support legacy suffix conversion

Co-authored-by: Mike Donnalley <[email protected]>
Co-authored-by: Willie Ruemmele <[email protected]>

Co-authored-by: vamsimundra <[email protected]>
Co-authored-by: Shane McLaughlin <[email protected]>
Co-authored-by: Willie Ruemmele <[email protected]>
Co-authored-by: Steve Hetzel <[email protected]>
Co-authored-by: Mike Donnalley <[email protected]>
Co-authored-by: Violet Yao <[email protected]>

src/client/index.ts

@@ -13,15 +13,43 @@ export {
 } from './metadataApiRetrieve';
 export { ToolingApi, ToolingDeployOptions, ToolingRetrieveOptions } from './toolingApi';
 export {
-  AsyncResult,
+  ComponentDeployment,
+  ComponentRetrieval,
+  ComponentDiagnostic,
   FileResponse,
+  SourceApiResult,
+  AsyncResult,
+  SourceDeployResult,
+  RequestStatus,
+  MetadataRequestStatus,
+  RetrieveFailure,
+  RetrieveSuccess,
+  SourceRetrieveResult,
   MetadataApiDeployStatus,
+  DeployDetails,
+  RunTestResult,
+  CodeCoverage,
+  LocationsNotCovered,
+  CodeCoverageWarnings,
+  Failures,
+  Successes,
+  DeployMessage,
+  RetrieveRequest,
+  RetrieveMessage,
+  FileProperties,
+  ComponentStatus,
   MetadataApiRetrieveStatus,
+  PackageOption,
+  PackageOptions,
   RetrieveOptions,
-  SourceDeployResult,
-  RetrieveMessage,
-  SourceRetrieveResult,
+  ContainerAsyncRequest,
   ToolingDeployStatus,
-  ComponentStatus,
-  FileProperties,
+  QueryResult,
+  ApexRecord,
+  VFRecord,
+  AuraRecord,
+  LWCRecord,
+  ToolingCreateResult,
+  AuraDefinition,
+  LightningComponentResource,
 } from './types';

src/client/metadataApiRetrieve.ts

@@ -216,6 +216,7 @@ export class MetadataApiRetrieve extends MetadataTransfer<
             type: 'merge',
             mergeWith: this.components.getSourceComponents(),
             defaultDirectory: pkg.outputDir,
+            forceIgnoredPaths: this.components.forceIgnoredPaths ?? new Set<string>(),
           }
         : {
             type: 'directory',
@@ -229,7 +230,9 @@ export class MetadataApiRetrieve extends MetadataTransfer<
         .getSourceComponents()
         .toArray();
       const convertResult = await converter.convert(zipComponents, 'source', outputConfig);
-      components.push(...convertResult.converted);
+      if (convertResult) {
+        components.push(...convertResult.converted);
+      }
     }
     return new ComponentSet(components, registry);
   }

src/client/metadataTransfer.ts

@@ -111,6 +111,10 @@ export abstract class MetadataTransfer<
       return result;
     } catch (e) {
       const error = new MetadataTransferError('md_request_fail', e.message);
+      if (error.stack && e.stack) {
+        // append the original stack to this new error
+        error.stack += `\nDUE TO:\n${e.stack}`;
+      }
       if (this.event.listenerCount('error') === 0) {
         throw error;
       }

src/collections/componentSet.ts

@@ -58,6 +58,7 @@ export class ComponentSet extends LazyCollection<MetadataComponent> {
    */
   public sourceApiVersion: string;
   public fullName?: string;
+  public forceIgnoredPaths?: Set<string>;
   private logger: Logger;
   private registry: RegistryAccess;
   private components = new Map<string, Map<string, SourceComponent>>();
@@ -137,6 +138,8 @@ export class ComponentSet extends LazyCollection<MetadataComponent> {
     buildComponents(fsPaths, false);
     buildComponents(fsDeletePaths, true);
 
+    set.forceIgnoredPaths = resolver.forceIgnoredPaths;
+
     return set;
   }
 
@@ -192,6 +195,7 @@ export class ComponentSet extends LazyCollection<MetadataComponent> {
         include: resolveIncludeSet,
         registry: options.registry,
       });
+      result.forceIgnoredPaths = components.forceIgnoredPaths;
       for (const component of components) {
         result.add(component);
       }

src/collections/index.ts

@@ -6,4 +6,10 @@
  */
 export { LazyCollection } from './lazyCollection';
 export { ComponentSet, DeploySetOptions, RetrieveSetOptions } from './componentSet';
-export { FromSourceOptions, FromManifestOptions } from './types';
+export {
+  PackageTypeMembers,
+  PackageManifestObject,
+  DestructiveChangesType,
+  FromSourceOptions,
+  FromManifestOptions,
+} from './types';

src/common/index.ts

@@ -4,7 +4,7 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-export { SourcePath, OptionalTreeRegistryOptions } from './types';
+export { SourcePath, TreeOptions, OptionalTreeRegistryOptions, RegistryOptions } from './types';
 export {
   DEFAULT_PACKAGE_ROOT_SFDX,
   META_XML_SUFFIX,

src/convert/convertContext.ts

@@ -11,6 +11,7 @@ import { JsToXml } from './streams';
 import { META_XML_SUFFIX, XML_NS_KEY, XML_NS_URL } from '../common';
 import { getString, JsonArray, JsonMap } from '@salesforce/ts-types';
 import { ComponentSet } from '../collections';
+import { normalizeToArray } from '../utils/collections';
 import { RecompositionStrategy } from '../registry/types';
 import { isEmpty } from '@salesforce/kit';
 
@@ -89,7 +90,8 @@ class RecompositionFinalizer extends ConvertTransactionFinalizer<RecompositionSt
         parent[groupName] = [];
       }
 
-      const group = parent[groupName] as JsonArray;
+      // it might be an object and not an array.  Example: custom object with a Field property containing a single field
+      const group = normalizeToArray(parent[groupName]) as JsonArray;
 
       group.push(childContents);
     }
@@ -306,8 +308,7 @@ class NonDecompositionFinalizer extends ConvertTransactionFinalizer<NonDecomposi
         parent[groupName] = [];
       }
 
-      const group = parent[groupName] as JsonArray;
-
+      const group = normalizeToArray(parent[groupName]) as JsonArray;
       group.push(child);
     }
 

src/convert/index.ts

@@ -6,10 +6,13 @@
  */
 export { MetadataConverter } from './metadataConverter';
 export {
-  ConvertResult,
-  ConvertOutputConfig,
-  SfdxFileFormat,
   WriteInfo,
   WriterFormat,
+  DirectoryConfig,
+  ZipConfig,
+  MergeConfig,
   MetadataTransformer,
+  SfdxFileFormat,
+  ConvertOutputConfig,
+  ConvertResult,
 } from './types';

src/convert/metadataConverter.ts

@@ -143,6 +143,7 @@ export class MetadataConverter {
             mergeSet.add(component.parent ?? component);
           }
           writer = new StandardWriter(output.defaultDirectory);
+          writer.forceIgnoredPaths = output.forceIgnoredPaths;
           break;
       }
 

src/convert/streams.ts

@@ -6,21 +6,21 @@
  */
 import { Archiver, create as createArchive } from 'archiver';
 import { createWriteStream } from 'fs';
-import { isAbsolute, join } from 'path';
+import { basename, dirname, isAbsolute, join } from 'path';
 import { pipeline as cbPipeline, Readable, Transform, Writable } from 'stream';
 import { promisify } from 'util';
 import { SourceComponent, MetadataResolver } from '../resolve';
 import { SfdxFileFormat, WriteInfo, WriterFormat } from './types';
 import { ensureFileExists } from '../utils/fileSystemHandler';
-import { SourcePath, XML_DECL } from '../common';
+import { META_XML_SUFFIX, SourcePath, XML_DECL } from '../common';
 import { ConvertContext } from './convertContext';
 import { MetadataTransformerFactory } from './transformers';
 import { JsonMap } from '@salesforce/ts-types';
 import { j2xParser } from 'fast-xml-parser';
 import { ComponentSet } from '../collections';
 import { LibraryError } from '../errors';
 import { RegistryAccess } from '../registry';
-import { Logger } from '@salesforce/core';
+import { fs, Logger } from '@salesforce/core';
 export const pipeline = promisify(cbPipeline);
 
 export class ComponentReader extends Readable {
@@ -126,6 +126,7 @@ export class ComponentConverter extends Transform {
 }
 
 export abstract class ComponentWriter extends Writable {
+  public forceIgnoredPaths?: Set<string> = new Set<string>();
   protected rootDestination?: SourcePath;
 
   constructor(rootDestination?: SourcePath) {
@@ -158,6 +159,19 @@ export class StandardWriter extends ComponentWriter {
           const fullDest = isAbsolute(info.output)
             ? info.output
             : join(this.rootDestination, info.output);
+          if (!fs.fileExistsSync(fullDest)) {
+            for (const ignoredPath of this.forceIgnoredPaths) {
+              if (
+                dirname(ignoredPath).includes(dirname(fullDest)) &&
+                basename(ignoredPath).includes(basename(fullDest))
+              ) {
+                return;
+              }
+            }
+          }
+          if (this.forceIgnoredPaths.has(fullDest)) {
+            return;
+          }
           // if there are children, resolve each file. o/w just pick one of the files to resolve
           if (toResolve.length === 0 || chunk.component.type.children) {
             // This is a workaround for a server side ListViews bug where

src/convert/transformers/decomposedMetadataTransformer.ts

@@ -14,6 +14,7 @@ import { SourcePath, META_XML_SUFFIX, XML_NS_URL, XML_NS_KEY } from '../../commo
 import { ComponentSet } from '../../collections';
 import { DecompositionState } from '../convertContext';
 import { DecompositionStrategy } from '../../registry';
+import { TypeInferenceError } from '../../errors';
 
 export class DecomposedMetadataTransformer extends BaseMetadataTransformer {
   public async toMetadataFormat(component: SourceComponent): Promise<WriteInfo[]> {
@@ -32,16 +33,10 @@ export class DecomposedMetadataTransformer extends BaseMetadataTransformer {
     } else {
       const { fullName } = component;
       this.context.recomposition.setState((state) => {
-        if (state[fullName]) {
-          for (const child of component.getChildren()) {
-            state[fullName].children.add(child);
-          }
-        } else {
-          state[fullName] = {
-            component,
-            children: new ComponentSet(component.getChildren(), this.registry),
-          };
+        if (!state[fullName]) {
+          state[fullName] = { component, children: new ComponentSet([], this.registry) };
         }
+        state[fullName].children = this.ensureValidChildren(component, state[fullName].children);
       });
     }
     // noop since the finalizer will push the writes to the component writer
@@ -53,11 +48,9 @@ export class DecomposedMetadataTransformer extends BaseMetadataTransformer {
     mergeWith?: SourceComponent
   ): Promise<WriteInfo[]> {
     const writeInfos: WriteInfo[] = [];
+    const childrenOfMergeComponent = mergeWith && this.ensureValidChildren(mergeWith);
     const { type, fullName: parentFullName } = component;
 
-    const childrenOfMergeComponent = mergeWith
-      ? new ComponentSet(mergeWith.getChildren(), this.registry)
-      : undefined;
     let parentXmlObject: JsonMap;
     const composedMetadata = await this.getComposedMetadataEntries(component);
 
@@ -147,6 +140,27 @@ export class DecomposedMetadataTransformer extends BaseMetadataTransformer {
     return writeInfos;
   }
 
+  // Ensures that the children of the provided SourceComponent are valid child
+  // types before adding them to the returned ComponentSet. Invalid child types
+  // can occur when projects are structured in an atypical way such as having
+  // ApexClasses or Layouts within a CustomObject folder.
+  private ensureValidChildren(component: SourceComponent, compSet?: ComponentSet): ComponentSet {
+    compSet = compSet || new ComponentSet([], this.registry);
+    const validChildTypes = Object.keys(component.type.children.types);
+    for (const child of component.getChildren()) {
+      // Ensure only valid child types are included with the parent.
+      if (!validChildTypes.includes(child.type?.id)) {
+        const filePath = child.xml || child.content;
+        throw new TypeInferenceError('error_unexpected_child_type', [
+          filePath,
+          component.type.name,
+        ]);
+      }
+      compSet.add(child);
+    }
+    return compSet;
+  }
+
   private async getComposedMetadataEntries(component: SourceComponent): Promise<[string, any][]> {
     const composedMetadata = (await component.parseXml())[component.type.name];
     return Object.entries(composedMetadata);

src/convert/transformers/defaultMetadataTransformer.ts

@@ -86,7 +86,7 @@ export class DefaultMetadataTransformer extends BaseMetadataTransformer {
       return mergeWith.xml;
     }
 
-    const { folderContentType, suffix } = component.type;
+    const { folderContentType, suffix, legacySuffix } = component.type;
     let xmlDestination = component.getPackageRelativePath(component.xml, targetFormat);
 
     // quirks:
@@ -117,7 +117,9 @@ export class DefaultMetadataTransformer extends BaseMetadataTransformer {
         );
       }
     }
-
+    if (legacySuffix && suffix && xmlDestination.includes(legacySuffix)) {
+      xmlDestination = xmlDestination.replace(legacySuffix, suffix);
+    }
     return xmlDestination;
   }
 }

src/convert/transformers/staticResourceMetadataTransformer.ts

@@ -141,7 +141,15 @@ export class StaticResourceMetadataTransformer extends BaseMetadataTransformer {
   }
 
   private async getContentType(component: SourceComponent): Promise<string> {
-    return ((await component.parseXml()).StaticResource as JsonMap).contentType as string;
+    try {
+      return ((await component.parseXml()).StaticResource as JsonMap).contentType as string;
+    } catch (e) {
+      if ((e as Error).message.includes("Cannot read property 'contentType' of undefined")) {
+        throw new LibraryError('error_static_resource_missing_resource_file', [
+          join('staticresources', component.name),
+        ]);
+      }
+    }
   }
 
   private getExtensionFromType(contentType: string): string {

src/convert/types.ts

@@ -64,6 +64,7 @@ export type MergeConfig = {
    * Location to store components that aren't merged.
    */
   defaultDirectory: SourcePath;
+  forceIgnoredPaths?: Set<string>;
 };
 
 /**

src/i18n/i18n.ts

@@ -30,6 +30,7 @@ export const messages = {
   error_convert_invalid_format: "Invalid conversion format '%s'",
   error_convert_not_implemented: '%s format conversion not yet implemented for type %s',
   error_could_not_infer_type: '%s: Could not infer a metadata type',
+  error_unexpected_child_type: 'Unexpected child metadata [%s] found for parent type [%s]',
   error_expected_source_files: "%s: Expected source files for type '%s'",
   error_failed_convert: 'Component conversion failed: %s',
   error_merge_metadata_target_unsupported:
@@ -60,6 +61,8 @@ export const messages = {
   error_invalid_package: 'The metadata pacakge was not initialized properly',
   error_static_resource_expected_archive_type:
     'A StaticResource directory must have a content type of application/zip or application/jar - found %s for %s',
+  error_static_resource_missing_resource_file:
+    'A StaticResource must have an associated .resource file, missing %s.resource-meta.xml',
   error_no_job_id:
     'The %s operation is missing a job ID. Initialize an operation with an ID, or start a new job.',
   tapi_deploy_component_limit_error: