fix(package): Resolve dependencies for Build (#1019)

* Resolve package dependencies in build

Resolve package dependencies at the start of build and also dynamically, as
dependencies are built. Take last validated package version created from the
same branch, for packages from the same repo.

Save resolved dependencies in artifact metadata and sfdx-project.json

Fixes #496

decision records/001-package-dependency-version-resolution.md

@@ -0,0 +1,17 @@
+# Package dependency version resolution
+•	Status: Accepted
+•	Issue: #496
+
+## Context and Problem Statement
+The Build command resolves package dependency versions, using `sfpowerkit:package:dependencies:list`, but there are several shortcomings with the current implementation:
+
+1. Executing sfpowerkit CLI in a node process
+1. The package versions are not filtered to the branch
+1. Dependencies on packages that are part of the same build are not guaranteed to pick up the version created in the build; it just fetches the LATEST version
+1. The resolved dependencies are not reflected in the artifact metadata or sfdx-project.json
+
+## Solution
+
+To address the first issue, the sfpowerkit command will be replaced with a direct API call and implementation within sfpowerscripts. This includes a fetcher for the Package2Version sObject.
+At the start of a build, the package dependencies will be resolved in one go, leaving dependencies on packages that are part of the same build to be resolved dynamically, as they are created.
+For dependencies on packages that are part of the same repo, the validated package versions are filtered to the current branch using git tags created by Publish.

packages/core/src/package/SfpPackageBuilder.ts

@@ -19,16 +19,26 @@ import CreateDataPackageImpl from './packageCreators/CreateDataPackageImpl';
 import ImpactedApexTestClassFetcher from '../apextest/ImpactedApexTestClassFetcher';
 import * as rimraf from 'rimraf';
 import PackageToComponent from './PackageToComponent';
+import lodash = require('lodash');
 import { EOL } from 'os';
 
 export default class SfpPackageBuilder {
+
     public static async buildPackageFromProjectDirectory(
         logger: Logger,
         projectDirectory: string,
         sfdx_package: string,
         params?: SfpPackageParams,
-        packageCreationParams?: PackageCreationParams
+        packageCreationParams?: PackageCreationParams,
+        projectConfig?: any
     ) {
+        if (!projectConfig) {
+            projectConfig = ProjectConfig.getSFDXProjectConfig(projectDirectory);
+        } else {
+            // Clone the projectConfig to prevent mutation
+            projectConfig = lodash.cloneDeep(projectConfig);
+        }
+
         let propertyFetchers: PropertyFetcher[] = [
             new AssignPermissionSetFetcher(),
             new DestructiveManifestPathFetcher(),
@@ -38,9 +48,9 @@ export default class SfpPackageBuilder {
         let startTime = Date.now;
         let sfpPackage: SfpPackage = new SfpPackage();
         sfpPackage.package_name = sfdx_package;
-        sfpPackage.projectConfig = ProjectConfig.getSFDXProjectConfig(projectDirectory);
+        sfpPackage.projectConfig = projectConfig;
         sfpPackage.apiVersion = sfpPackage.projectConfig.sourceApiVersion;
-        sfpPackage.packageDescriptor = ProjectConfig.getSFDXPackageDescriptor(projectDirectory, sfdx_package);
+        sfpPackage.packageDescriptor = ProjectConfig.getPackageDescriptorFromConfig(sfdx_package, sfpPackage.projectConfig);
         sfpPackage.projectDirectory = projectDirectory;
         sfpPackage.packageDirectory = sfpPackage.packageDescriptor.path;
 
@@ -61,6 +71,7 @@ export default class SfpPackageBuilder {
         sfpPackage.workingDirectory = await SfpPackageContentGenerator.generateSfpPackageDirectory(
             logger,
             sfpPackage.projectDirectory,
+            sfpPackage.projectConfig,
             sfpPackage.packageName,
             sfpPackage.packageDescriptor.path,
             sfpPackage.destructiveChangesPath,
@@ -105,7 +116,7 @@ export default class SfpPackageBuilder {
             sfpPackage.apexClassWithOutTestClasses = apexFetcher.getClassesOnlyExcludingTestsAndInterfaces();
 
             sfpPackage.isTriggerAllTests = this.isAllTestsToBeTriggered(sfpPackage,logger);
-            
+
             //Introspect Diff Package Created
             //On Failure.. remove diff and move on
             try {
@@ -191,7 +202,7 @@ export default class SfpPackageBuilder {
 
         let workingDirectory = path.join(sfpPackage.workingDirectory, 'diff');
         if (fs.existsSync(workingDirectory)) {
-            
+
             let changedComponents = new PackageToComponent(
                 sfpPackage.packageName,
                 path.join(workingDirectory, sfpPackage.packageDirectory)
@@ -258,7 +269,7 @@ export default class SfpPackageBuilder {
             if (pkgDescriptor['isOptimizedDeployment'] == null) return true;
             else return pkgDescriptor['isOptimizedDeployment'];
         }
-    
+
 }
 
 // Options while creating package

packages/core/src/package/coverage/PackageVersionCoverage.ts

@@ -1,25 +1,22 @@
 import { Connection } from '@salesforce/core';
 import SFPLogger, { Logger, LoggerLevel } from '../../logger/SFPLogger';
-import QueryHelper from '../../queryHelper/QueryHelper';
-
-const QUERY = `SELECT SubscriberPackageVersionId,Package2Id, Package2.Name,MajorVersion,MinorVersion,PatchVersion,BuildNumber, CodeCoverage, HasPassedCodeCoverageCheck, Name FROM Package2Version WHERE `;
+import Package2VersionFetcher from '../version/Package2VersionFetcher';
 
 export default class PackageVersionCoverage {
     public constructor(private connection: Connection, private logger: Logger) {}
 
-    public async getCoverage(versionId: string[]): Promise<PackageCoverage> {
-        let whereClause = `SubscriberPackageVersionId='${versionId}'`;
-
-        const records = await QueryHelper.query<any>(`${QUERY} ${whereClause}`, this.connection, true);
-        SFPLogger.log(`Fetched Records ${JSON.stringify(records)}`, LoggerLevel.TRACE, this.logger);
-        if (records[0]) {
+    public async getCoverage(versionId: string): Promise<PackageCoverage> {
+        const package2VersionFetcher = new Package2VersionFetcher(this.connection);
+        const package2Version = await package2VersionFetcher.fetchBySubscriberPackageVersionId(versionId);
+        SFPLogger.log(`Fetched Record ${JSON.stringify(package2Version)}`, LoggerLevel.TRACE, this.logger);
+        if (package2Version) {
             var packageCoverage = <PackageCoverage>{};
-            packageCoverage.HasPassedCodeCoverageCheck = records[0].HasPassedCodeCoverageCheck;
-            packageCoverage.coverage = records[0].CodeCoverage ? records[0].CodeCoverage.apexCodeCoveragePercentage : 0;
-            packageCoverage.packageId = records[0].Package2Id;
-            packageCoverage.packageName = records[0].Package2.Name;
-            packageCoverage.packageVersionId = records[0].SubscriberPackageVersionId;
-            packageCoverage.packageVersionNumber = `${records[0].MajorVersion}.${records[0].MinorVersion}.${records[0].PatchVersion}.${records[0].BuildNumber}`;
+            packageCoverage.HasPassedCodeCoverageCheck = package2Version.HasPassedCodeCoverageCheck;
+            packageCoverage.coverage = package2Version.CodeCoverage ? package2Version.CodeCoverage.apexCodeCoveragePercentage : 0;
+            packageCoverage.packageId = package2Version.Package2Id;
+            packageCoverage.packageName = package2Version.Package2.Name;
+            packageCoverage.packageVersionId = package2Version.SubscriberPackageVersionId;
+            packageCoverage.packageVersionNumber = `${package2Version.MajorVersion}.${package2Version.MinorVersion}.${package2Version.PatchVersion}.${package2Version.BuildNumber}`;
 
             SFPLogger.log(
                 `Successfully Retrieved the Apex Test Coverage of the package version`,

packages/core/src/package/diff/PackageComponentDiff.ts

@@ -139,7 +139,7 @@ export default class PackageComponentDiff {
             SFPLogger.log(`.forceignore not found, skipping..`, LoggerLevel.DEBUG, this.logger);
         }
         try {
-            let cleanedUpProjectManifest = ProjectConfig.cleanupMPDFromManifest(null, this.sfdxPackage);
+            let cleanedUpProjectManifest = ProjectConfig.cleanupMPDFromProjectDirectory(null, this.sfdxPackage);
             fs.writeJSONSync(path.join(outputFolder, 'sfdx-project.json'), cleanedUpProjectManifest, { spaces: 4 });
         } catch (error) {
             SFPLogger.log(`sfdx-project.json not found, skipping..`, LoggerLevel.DEBUG, this.logger);

packages/core/src/package/generators/SfpPackageContentGenerator.ts

@@ -13,6 +13,7 @@ export default class SfpPackageContentGenerator {
     public static async generateSfpPackageDirectory(
         logger: Logger,
         projectDirectory: string,
+        projectConfig: any,
         sfdx_package: string,
         packageDirectory: string,
         destructiveManifestFilePath?: string,
@@ -50,7 +51,7 @@ export default class SfpPackageContentGenerator {
         if (
             revisionFrom &&
             revisionTo &&
-            !ProjectConfig.getSFDXPackageDescriptor(projectDirectory, sfdx_package).aliasfy
+            !ProjectConfig.getPackageDescriptorFromConfig(sfdx_package, projectConfig).aliasfy
         ) {
             try {
                 let packageComponentDiffer: PackageComponentDiff = new PackageComponentDiff(
@@ -82,16 +83,16 @@ export default class SfpPackageContentGenerator {
             SfpPackageContentGenerator.copyConfigFilePath(configFilePath, artifactDirectory, rootDirectory, logger);
         }
 
-        SfpPackageContentGenerator.createPackageManifests(artifactDirectory, rootDirectory, sfdx_package);
+        SfpPackageContentGenerator.createPackageManifests(artifactDirectory, rootDirectory, projectConfig, sfdx_package);
 
         fs.copySync(path.join(rootDirectory, packageDirectory), path.join(artifactDirectory, packageDirectory));
 
         return artifactDirectory;
     }
 
-    private static createPackageManifests(artifactDirectory: string, projectDirectory: string, sfdx_package: string) {
+    private static createPackageManifests(artifactDirectory: string, projectDirectory: string, projectConfig: any, sfdx_package: string) {
         // Create pruned package manifest in source directory
-        let cleanedUpProjectManifest = ProjectConfig.cleanupMPDFromManifest(projectDirectory, sfdx_package);
+        let cleanedUpProjectManifest = ProjectConfig.cleanupMPDFromProjectConfig(projectConfig, sfdx_package);
 
         //Setup preDeployment Script Path
         if (fs.existsSync(path.join(artifactDirectory, 'scripts', `preDeployment`)))

packages/core/src/package/packageCreators/CreateUnlockedPackageImpl.ts

@@ -77,9 +77,6 @@ export default class CreateUnlockedPackageImpl extends CreatePackage {
             // Store original dependencies to artifact
             sfpPackage.dependencies = sfpPackage.packageDescriptor['dependencies'];
         } else if (!this.isOrgDependentPackage && !this.packageCreationParams.isSkipValidation) {
-            // With dependencies, so fetch it again
-            this.resolvePackageDependencies(sfpPackage.packageDescriptor, this.workingDirectory);
-            //Redo the fetch of the descriptor as the above command would have redone the dependencies
             sfpPackage.packageDescriptor = ProjectConfig.getSFDXPackageDescriptor(
                 this.workingDirectory,
                 this.sfpPackage.packageName
@@ -182,7 +179,7 @@ export default class CreateUnlockedPackageImpl extends CreatePackage {
             try {
                 SFPLogger.log('Fetching Version Number and Coverage details', LoggerLevel.INFO, this.logger);
 
-                let pkgInfoResult = await packageVersionCoverage.getCoverage([sfpPackage.package_version_id]);
+                let pkgInfoResult = await packageVersionCoverage.getCoverage(sfpPackage.package_version_id);
 
                 sfpPackage.isDependencyValidated = !this.packageCreationParams.isSkipValidation;
                 sfpPackage.package_version_number = pkgInfoResult.packageVersionNumber;
@@ -208,15 +205,4 @@ export default class CreateUnlockedPackageImpl extends CreatePackage {
         }
         return CreateUnlockedPackageImpl.packageTypeInfos;
     }
-
-    private resolvePackageDependencies(packageDescriptor: any, workingDirectory: string) {
-        SFPLogger.log('Resolving project dependencies', LoggerLevel.INFO, this.logger);
-
-        let resolveResult = child_process.execSync(
-            `sfdx sfpowerkit:package:dependencies:list -p ${packageDescriptor['path']} -v ${this.packageCreationParams.devHub} -w --usedependencyvalidatedpackages`,
-            { cwd: workingDirectory, encoding: 'utf8' }
-        );
-
-        SFPLogger.log(`Resolved Depenendecies: ${resolveResult}`, LoggerLevel.INFO, this.logger);
-    }
 }

packages/core/src/package/version/Package2VersionFetcher.ts

@@ -0,0 +1,74 @@
+import { Connection } from '@salesforce/core';
+import QueryHelper from '../../queryHelper/QueryHelper';
+import semver from 'semver';
+
+/**
+ * Fetcher for second-generation package version in Dev Hub
+ */
+export default class Package2VersionFetcher {
+
+    private readonly query: string =
+      'Select SubscriberPackageVersionId, Package2Id, Package2.Name, IsPasswordProtected, IsReleased, MajorVersion, MinorVersion, PatchVersion, BuildNumber, CodeCoverage, HasPassedCodeCoverageCheck from Package2Version ';
+
+    constructor(private conn: Connection) {}
+
+    /**
+     * Fetch Package2 versions by Package2 Id
+     * Sorts by semantic version, in descending order
+     * @param package2Id
+     * @param versionNumber
+     * @param isValidatedPackages
+     * @returns
+     */
+    async fetchByPackage2Id(package2Id: string, versionNumber?: string, isValidatedPackages?: boolean): Promise<Package2Version[]> {
+      let query = this.query;
+
+      let whereClause: string = `where Package2Id='${package2Id}'  `;
+
+      if (versionNumber) {
+        // TODO: validate version number
+        const versions = versionNumber.split(".");
+
+        if (versions[0]) whereClause += `and MajorVersion=${versions[0]} `;
+        if (versions[1]) whereClause += `and MinorVersion=${versions[1]} `;
+        if (versions[2]) whereClause += `and PatchVersion=${versions[2]} `;
+        if (versions[3]) whereClause += `and BuildNumber=${versions[3]} `;
+      }
+
+      if (isValidatedPackages) whereClause += `and ValidationSkipped = false `;
+
+      whereClause += `and IsDeprecated = false `;
+      query += whereClause
+
+      const records = await QueryHelper.query<Package2Version>(query, this.conn, true);
+      return records.sort((a,b) => {
+        const v1 = `${a.MajorVersion}.${a.MinorVersion}.${a.PatchVersion}-${a.BuildNumber}`;
+        const v2 = `${b.MajorVersion}.${b.MinorVersion}.${b.PatchVersion}-${b.BuildNumber}`;
+        return semver.rcompare(v1, v2);
+      });
+    }
+
+    async fetchBySubscriberPackageVersionId(subscriberPackageVersionId: string): Promise<Package2Version> {
+      let query = this.query;
+
+      let whereClause: string = `where SubscriberPackageVersionId='${subscriberPackageVersionId}'`;
+      query += whereClause;
+
+      const records = await QueryHelper.query<Package2Version>(query, this.conn, true);
+      return records[0];
+    }
+}
+
+export interface Package2Version {
+  SubscriberPackageVersionId: string;
+  Package2Id: string;
+  Package2: { Name: string };
+  IsPasswordProtected: boolean;
+  IsReleased: boolean;
+  MajorVersion: number;
+  MinorVersion: number;
+  PatchVersion: number;
+  BuildNumber: number;
+  CodeCoverage: { apexCodeCoveragePercentage: number };
+  HasPassedCodeCoverageCheck: boolean
+}

packages/core/src/project/ProjectConfig.ts

@@ -171,25 +171,34 @@ export default class ProjectConfig {
      * @param projectDirectory
      * @param sfdxPackage
      */
-    public static cleanupMPDFromManifest(projectDirectory: string, sfdxPackage: string): any {
-        let sfdxManifest = this.getSFDXProjectConfig(projectDirectory);
+    public static cleanupMPDFromProjectDirectory(projectDirectory: string, sfdxPackage: string): any {
+        const projectConfig = this.getSFDXProjectConfig(projectDirectory);
 
+        return ProjectConfig.cleanupMPDFromProjectConfig(projectConfig, sfdxPackage);
+    }
+
+    /**
+     * Returns pruned package manifest, containing sfdxPackage only
+     * @param projectConfig
+     * @param sfdxPackage
+     */
+     public static cleanupMPDFromProjectConfig(projectConfig: any, sfdxPackage: string): any {
         if (sfdxPackage) {
-            let i = sfdxManifest['packageDirectories'].length;
+            let i = projectConfig['packageDirectories'].length;
             while (i--) {
-                if (sfdxPackage != sfdxManifest['packageDirectories'][i]['package']) {
-                    sfdxManifest['packageDirectories'].splice(i, 1);
+                if (sfdxPackage != projectConfig['packageDirectories'][i]['package']) {
+                    projectConfig['packageDirectories'].splice(i, 1);
                 }
             }
         } else {
-            let i = sfdxManifest['packageDirectories'].length;
+            let i = projectConfig['packageDirectories'].length;
             while (i--) {
-                if (!fs.existsSync(sfdxManifest['packageDirectories'][i]['path'])) {
-                    sfdxManifest['packageDirectories'].splice(i, 1);
+                if (!fs.existsSync(projectConfig['packageDirectories'][i]['path'])) {
+                    projectConfig['packageDirectories'].splice(i, 1);
                 }
             }
         }
-        sfdxManifest['packageDirectories'][0]['default'] = true; //add default = true
-        return sfdxManifest;
+        projectConfig['packageDirectories'][0]['default'] = true; //add default = true
+        return projectConfig;
     }
 }

packages/core/src/scratchorg/pool/ClientSourceTracking.ts

@@ -167,7 +167,7 @@ export default class ClientSourceTracking {
 
     private cleanupSFDXProjectJsonTonOnePackage(projectDir: string, packageName: string) {
         try {
-            let cleanedUpProjectManifest = ProjectConfig.cleanupMPDFromManifest(projectDir, packageName);
+            let cleanedUpProjectManifest = ProjectConfig.cleanupMPDFromProjectDirectory(projectDir, packageName);
             fs.writeJSONSync(path.join(projectDir, 'sfdx-project.json'), cleanedUpProjectManifest, {
                 spaces: 4,
             });

packages/core/tests/package/SFPackageBuilder.test.ts

@@ -46,6 +46,7 @@ jest.mock('../../src/package/generators/SfpPackageContentGenerator', () => {
     class SfpPackageContentGenerator {
         static async generateSfpPackageDirectory(
             projectDirectory: string,
+            projectConfig: any,
             sfdx_package: string,
             packageDirectory: string,
             destructiveManifestFilePath?: string,
@@ -152,7 +153,7 @@ jest.mock('../../src/package/packageCreators/CreateSourcePackageImpl', () => {
             protected logger?: Logger,
             protected params?: SfpPackageParams
         ) {
-            
+
         }
 
         public async exec(): Promise<SfpPackage> {

packages/core/tests/project/ProjectConfig.test.ts

@@ -161,6 +161,6 @@ describe('Given a project directory or sfdx-project.json with multiple packages'
             packageAliases: { bi: '0x002232323232' },
         };
 
-        expect(ProjectConfig.cleanupMPDFromManifest(null, 'temp')).toStrictEqual(cleaned_sfdx_project);
+        expect(ProjectConfig.cleanupMPDFromProjectDirectory(null, 'temp')).toStrictEqual(cleaned_sfdx_project);
     });
 });