fix(core): filter branch in preparation for nx import

e2e/nx/src/import.test.ts

@@ -8,7 +8,7 @@ import {
   updateFile,
   e2eCwd,
 } from '@nx/e2e/utils';
-import { mkdirSync, rmdirSync } from 'fs';
+import { writeFileSync, mkdirSync, rmdirSync } from 'fs';
 import { execSync } from 'node:child_process';
 import { join } from 'path';
 
@@ -86,4 +86,54 @@ describe('Nx Import', () => {
     );
     runCLI(`vite:build created-vite-app`);
   });
+
+  it('should be able to import two directories from same repo', () => {
+    // Setup repo with two packages: a and b
+    const repoPath = join(tempImportE2ERoot, 'repo');
+    mkdirSync(repoPath, { recursive: true });
+    writeFileSync(join(repoPath, 'README.md'), `# Repo`);
+    execSync(`git init`, {
+      cwd: repoPath,
+    });
+    execSync(`git add .`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -am "initial commit"`, {
+      cwd: repoPath,
+    });
+    execSync(`git checkout -b main`, {
+      cwd: repoPath,
+    });
+    mkdirSync(join(repoPath, 'packages/a'), { recursive: true });
+    writeFileSync(join(repoPath, 'packages/a/README.md'), `# A`);
+    execSync(`git add packages/a`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -m "add package a"`, {
+      cwd: repoPath,
+    });
+    mkdirSync(join(repoPath, 'packages/b'), { recursive: true });
+    writeFileSync(join(repoPath, 'packages/b/README.md'), `# B`);
+    execSync(`git add packages/b`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -m "add package b"`, {
+      cwd: repoPath,
+    });
+
+    runCLI(
+      `import ${repoPath} packages/a --ref main --source packages/a --no-interactive`,
+      {
+        verbose: true,
+      }
+    );
+    runCLI(
+      `import ${repoPath} packages/b --ref main --source packages/b --no-interactive`,
+      {
+        verbose: true,
+      }
+    );
+
+    checkFilesExist('packages/a/README.md', 'packages/b/README.md');
+  });
 });

packages/nx/src/command-line/import/command-object.ts

@@ -28,6 +28,11 @@ export const yargsImportCommand: CommandModule = {
             type: 'string',
             description: 'The branch from the source repository to import',
           })
+          .option('depth', {
+            type: 'number',
+            description:
+              'The depth to clone the source repository (limit this for faster git clone)',
+          })
           .option('interactive', {
             type: 'boolean',
             description: 'Interactive mode',

packages/nx/src/command-line/import/import.ts

@@ -41,6 +41,10 @@ export interface ImportOptions {
    * The directory in the destination repo to import into
    */
   destination: string;
+  /**
+   * The depth to clone the source repository (limit this for faster clone times)
+   */
+  depth: number;
 
   verbose: boolean;
   interactive: boolean;
@@ -101,6 +105,7 @@ export async function importHandler(options: ImportOptions) {
   try {
     sourceGitClient = await cloneFromUpstream(sourceRemoteUrl, sourceRepoPath, {
       originName: importRemoteName,
+      depth: options.depth,
     });
   } catch (e) {
     spinner.fail(`Failed to clone ${sourceRemoteUrl} into ${sourceRepoPath}`);
@@ -149,6 +154,7 @@ export async function importHandler(options: ImportOptions) {
           name: 'destination',
           message: 'Where in this workspace should the code be imported into?',
           required: true,
+          initial: source ? source : undefined,
         },
       ])
     ).destination;

packages/nx/src/command-line/import/utils/prepare-source-repo.ts

@@ -25,108 +25,72 @@ export async function prepareSourceRepo(
     new: true,
     base: `${originName}/${ref}`,
   });
+
   spinner.succeed(`Created a ${tempImportBranch} branch based on ${ref}`);
   const relativeSourceDir = relative(
     gitClient.root,
     join(gitClient.root, source)
   );
 
+  if (relativeSourceDir !== '') {
+    spinner.start(
+      `Filtering git history to only include files in ${relativeSourceDir}`
+    );
+    await gitClient.filterBranch(relativeSourceDir, tempImportBranch);
+    spinner.succeed(
+      `Filtered git history to only include files in ${relativeSourceDir}`
+    );
+  }
+
   const destinationInSource = join(gitClient.root, relativeDestination);
   spinner.start(`Moving files and git history to ${destinationInSource}`);
-  if (relativeSourceDir === '') {
-    const files = await gitClient.getGitFiles('.');
-    try {
-      await rm(destinationInSource, {
-        recursive: true,
-      });
-    } catch {}
-    await mkdir(destinationInSource, { recursive: true });
-    const gitignores = new Set<string>();
-    for (const file of files) {
-      if (basename(file) === '.gitignore') {
-        gitignores.add(file);
-        continue;
-      }
-      spinner.start(
-        `Moving files and git history to ${destinationInSource}: ${file}`
-      );
-
-      const newPath = join(destinationInSource, file);
 
-      await mkdir(dirname(newPath), { recursive: true });
-      try {
-        await gitClient.move(file, newPath);
-      } catch {
-        await wait(100);
-        await gitClient.move(file, newPath);
-      }
+  // The result of filter-branch will contain only the files in the subdirectory at its root.
+  const files = await gitClient.getGitFiles('.');
+  try {
+    await rm(destinationInSource, {
+      recursive: true,
+    });
+  } catch {}
+  await mkdir(destinationInSource, { recursive: true });
+  const gitignores = new Set<string>();
+  for (const file of files) {
+    if (basename(file) === '.gitignore') {
+      gitignores.add(file);
+      continue;
     }
-
-    await gitClient.commit(
-      `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported`
+    spinner.start(
+      `Moving files and git history to ${destinationInSource}: ${file}`
     );
 
-    for (const gitignore of gitignores) {
-      await gitClient.move(gitignore, join(destinationInSource, gitignore));
-    }
-    await gitClient.amendCommit();
-    for (const gitignore of gitignores) {
-      await copyFile(
-        join(destinationInSource, gitignore),
-        join(gitClient.root, gitignore)
-      );
-    }
-  } else {
-    let needsSquash = false;
-    const needsMove = destinationInSource !== join(gitClient.root, source);
-    if (needsMove) {
-      try {
-        await rm(destinationInSource, {
-          recursive: true,
-        });
-        await gitClient.commit(
-          `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported`
-        );
-        needsSquash = true;
-      } catch {}
+    const newPath = join(destinationInSource, file);
 
-      await mkdir(destinationInSource, { recursive: true });
+    await mkdir(dirname(newPath), { recursive: true });
+    try {
+      await gitClient.move(file, newPath);
+    } catch {
+      await wait(100);
+      await gitClient.move(file, newPath);
     }
+  }
 
-    const files = await gitClient.getGitFiles('.');
-    for (const file of files) {
-      if (file === '.gitignore') {
-        continue;
-      }
-      spinner.start(
-        `Moving files and git history to ${destinationInSource}: ${file}`
-      );
+  await gitClient.commit(
+    `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported`
+  );
 
-      if (!relative(source, file).startsWith('..')) {
-        if (needsMove) {
-          const newPath = join(destinationInSource, file);
+  for (const gitignore of gitignores) {
+    await gitClient.move(gitignore, join(destinationInSource, gitignore));
+  }
 
-          await mkdir(dirname(newPath), { recursive: true });
-          try {
-            await gitClient.move(file, newPath);
-          } catch {
-            await wait(100);
-            await gitClient.move(file, newPath);
-          }
-        }
-      } else {
-        await rm(join(gitClient.root, file), {
-          recursive: true,
-        });
-      }
-    }
-    await gitClient.commit(
-      `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported`
+  await gitClient.amendCommit();
+
+  for (const gitignore of gitignores) {
+    await copyFile(
+      join(destinationInSource, gitignore),
+      join(gitClient.root, gitignore)
     );
-    if (needsSquash) {
-      await gitClient.squashLastTwoCommits();
-    }
   }
+
   spinner.succeed(
     `${sourceRemoteUrl} has been prepared to be imported into this workspace on a temporary branch: ${tempImportBranch} in ${gitClient.root}`
   );

packages/nx/src/utils/git-utils.ts

@@ -18,10 +18,14 @@ function execAsync(command: string, execOptions: ExecOptions) {
 export async function cloneFromUpstream(
   url: string,
   destination: string,
-  { originName } = { originName: 'origin' }
+  { originName, depth }: { originName: string; depth?: number } = {
+    originName: 'origin',
+  }
 ) {
   await execAsync(
-    `git clone ${url} ${destination} --depth 1 --origin ${originName}`,
+    `git clone ${url} ${destination} ${
+      depth ? `--depth ${depth}` : ''
+    } --origin ${originName}`,
     {
       cwd: dirname(destination),
     }
@@ -42,8 +46,8 @@ export class GitRepository {
       .trim();
   }
 
-  addFetchRemote(remoteName: string, branch: string) {
-    return this.execAsync(
+  async addFetchRemote(remoteName: string, branch: string) {
+    return await this.execAsync(
       `git config --add remote.${remoteName}.fetch "+refs/heads/${branch}:refs/remotes/${remoteName}/${branch}"`
     );
   }
@@ -79,22 +83,22 @@ export class GitRepository {
   }
 
   async reset(ref: string) {
-    return this.execAsync(`git reset ${ref} --hard`);
+    return await this.execAsync(`git reset ${ref} --hard`);
   }
 
   async squashLastTwoCommits() {
-    return this.execAsync(
+    return await this.execAsync(
       `git -c core.editor="node ${SQUASH_EDITOR}" rebase --interactive --no-autosquash HEAD~2`
     );
   }
 
   async mergeUnrelatedHistories(ref: string, message: string) {
-    return this.execAsync(
+    return await this.execAsync(
       `git merge ${ref} -X ours --allow-unrelated-histories -m "${message}"`
     );
   }
   async fetch(remote: string, ref?: string) {
-    return this.execAsync(`git fetch ${remote}${ref ? ` ${ref}` : ''}`);
+    return await this.execAsync(`git fetch ${remote}${ref ? ` ${ref}` : ''}`);
   }
 
   async checkout(
@@ -104,38 +108,42 @@ export class GitRepository {
       base: string;
     }
   ) {
-    return this.execAsync(
+    return await this.execAsync(
       `git checkout ${opts.new ? '-b ' : ' '}${branch}${
         opts.base ? ' ' + opts.base : ''
       }`
     );
   }
 
   async move(path: string, destination: string) {
-    return this.execAsync(`git mv "${path}" "${destination}"`);
+    return await this.execAsync(`git mv "${path}" "${destination}"`);
   }
 
   async push(ref: string, remoteName: string) {
-    return this.execAsync(`git push -u -f ${remoteName} ${ref}`);
+    return await this.execAsync(`git push -u -f ${remoteName} ${ref}`);
   }
 
   async commit(message: string) {
-    return this.execAsync(`git commit -am "${message}"`);
+    return await this.execAsync(`git commit -am "${message}"`);
   }
   async amendCommit() {
-    return this.execAsync(`git commit --amend -a --no-edit`);
+    return await this.execAsync(`git commit --amend -a --no-edit`);
   }
 
-  deleteGitRemote(name: string) {
-    return this.execAsync(`git remote rm ${name}`);
+  async deleteGitRemote(name: string) {
+    return await this.execAsync(`git remote rm ${name}`);
   }
 
-  deleteBranch(branch: string) {
-    return this.execAsync(`git branch -D ${branch}`);
+  async addGitRemote(name: string, url: string) {
+    return await this.execAsync(`git remote add ${name} ${url}`);
   }
 
-  addGitRemote(name: string, url: string) {
-    return this.execAsync(`git remote add ${name} ${url}`);
+  async filterBranch(subdirectory: string, branchName: string) {
+    // We need non-ASCII file names to not be quoted, or else filter-branch will exclude them.
+    await this.execAsync(`git config core.quotepath false`);
+    return await this.execAsync(
+      `git filter-branch --subdirectory-filter ${subdirectory} -- ${branchName}`
+    );
   }
 }
 
@@ -150,14 +158,6 @@ export function updateRebaseFile(contents: string): string {
   return lines.join('\n');
 }
 
-export function fetchGitRemote(
-  name: string,
-  branch: string,
-  execOptions: ExecSyncOptions
-) {
-  return execSync(`git fetch ${name} ${branch} --depth 1`, execOptions);
-}
-
 /**
  * This is currently duplicated in Nx Console. Please let @MaxKless know if you make changes here.
  */