From 96636870e759cef99a6c459a9533c6aad89520a5 Mon Sep 17 00:00:00 2001 From: Stephanie Roy Date: Fri, 14 Oct 2022 10:12:37 -0400 Subject: [PATCH 01/12] Use demo project as submodule --- .codeclimate.json | 2 +- .gitmodules | 3 + .vscode/settings.json | 2 +- CONTRIBUTING.md | 10 +- demo/.dvc/.gitignore | 3 - demo/.dvc/config | 4 - demo/.dvcignore | 5 - demo/.gitignore | 7 - demo/.vscode/extensions.json | 3 - demo/data/MNIST/.gitignore | 1 - demo/data/MNIST/raw.dvc | 5 - demo/dvc.lock | 34 ---- demo/dvc.yaml | 22 --- demo/params.yaml | 3 - demo/requirements.txt | 3 - demo/train.py | 163 ------------------ demo/training_metrics.json | 5 - extension/package.json | 2 +- .../resources/walkthrough/setup-project.md | 2 +- extension/src/fileSystem/index.test.ts | 2 +- extension/src/test/e2e/wdio.conf.ts | 2 +- .../test/suite/experiments/data/index.test.ts | 4 +- extension/src/test/util/index.ts | 2 +- scripts/virtualenv-install.ts | 2 +- vscode-dvc-demo | 1 + 25 files changed, 19 insertions(+), 273 deletions(-) create mode 100644 .gitmodules delete mode 100644 demo/.dvc/.gitignore delete mode 100644 demo/.dvc/config delete mode 100644 demo/.dvcignore delete mode 100644 demo/.gitignore delete mode 100644 demo/.vscode/extensions.json delete mode 100644 demo/data/MNIST/.gitignore delete mode 100644 demo/data/MNIST/raw.dvc delete mode 100644 demo/dvc.lock delete mode 100644 demo/dvc.yaml delete mode 100644 demo/params.yaml delete mode 100644 demo/requirements.txt delete mode 100644 demo/train.py delete mode 100644 demo/training_metrics.json create mode 160000 vscode-dvc-demo diff --git a/.codeclimate.json b/.codeclimate.json index 957e963d3b..e7ec942ad2 100644 --- a/.codeclimate.json +++ b/.codeclimate.json @@ -5,7 +5,7 @@ "**/*.test.*", "**/constants.ts", "**/contract.ts", - "demo/**", + "vscode-dvc-demo/**", "extension/src/test/**", "languageServer/src/test/**", "webview/src/react-table-config.d.ts", diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..38af2e4fa6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "vscode-dvc-demo"] + path = vscode-dvc-demo + url = https://github.com/iterative/vscode-dvc-demo diff --git a/.vscode/settings.json b/.vscode/settings.json index f35eed42cf..552c226125 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,7 @@ "prettier.configPath": ".prettierrc.json", "editor.tabSize": 2, "typescript.updateImportsOnFileMove.enabled": "prompt", - "npm.exclude": ["**/{demo,.wdio*}/**"], + "npm.exclude": ["**/{vscode-dvc-demo,.wdio*}/**"], "cSpell.words": [ "appdirs", "camelcase", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2fd12d67b0..1b5d0fa8c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,17 +58,17 @@ First, ensure that [Visual Studio Code](https://code.visualstudio.com) and ## The demo project -The [demo project](demo) is provided as a lightweight, convenient testbed to try -your changes to this extension. +The [demo project](vscode-dvc-demo) is provided as a lightweight, convenient +testbed to try your changes to this extension. > **Note**: It is not an exhaustive showcase of DVC's features. Testers are > encouraged to try other DVC repositories -- especially real-world cases! - Run `yarn setup:venv` from the _Terminal_ in the root of this repo to set up a - Python virtual environment for the demo project (in `demo/.env`). + Python virtual environment for the demo project (in `vscode-dvc-demo/.env`). -- Open the `./demo` project in a VS Code window, for example in the _Extension - Development Host_ (see previous section). +- Open the `./vscode-dvc-demo` project in a VS Code window, for example in the + _Extension Development Host_ (see previous section). - Pull the project data using this extension (either from the **DVC Tracked** panel in the _File Explorer_ or the **DVC panel** in _Source Control_) or by diff --git a/demo/.dvc/.gitignore b/demo/.dvc/.gitignore deleted file mode 100644 index 528f30c71c..0000000000 --- a/demo/.dvc/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/config.local -/tmp -/cache diff --git a/demo/.dvc/config b/demo/.dvc/config deleted file mode 100644 index fde9c420eb..0000000000 --- a/demo/.dvc/config +++ /dev/null @@ -1,4 +0,0 @@ -[core] - remote = storage -['remote "storage"'] - url = https://remote.dvc.org/mnist-vscode diff --git a/demo/.dvcignore b/demo/.dvcignore deleted file mode 100644 index 816e5ba9bf..0000000000 --- a/demo/.dvcignore +++ /dev/null @@ -1,5 +0,0 @@ -# Add patterns of files dvc should ignore, which could improve -# the performance. Learn more at -# https://dvc.org/doc/user-guide/dvcignore - -.env/* diff --git a/demo/.gitignore b/demo/.gitignore deleted file mode 100644 index 83dd11a9a8..0000000000 --- a/demo/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -/.vscode/settings.json -.env -/model.pt -/training_metrics -/misclassified.jpg -/predictions.json - diff --git a/demo/.vscode/extensions.json b/demo/.vscode/extensions.json deleted file mode 100644 index 328baf33d6..0000000000 --- a/demo/.vscode/extensions.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "recommendations": ["ms-python.python"] -} diff --git a/demo/data/MNIST/.gitignore b/demo/data/MNIST/.gitignore deleted file mode 100644 index 55ff1d08fa..0000000000 --- a/demo/data/MNIST/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/raw diff --git a/demo/data/MNIST/raw.dvc b/demo/data/MNIST/raw.dvc deleted file mode 100644 index c50a83a000..0000000000 --- a/demo/data/MNIST/raw.dvc +++ /dev/null @@ -1,5 +0,0 @@ -outs: -- md5: 8c257df187855c681f88bde92d721ccd.dir - size: 66544770 - nfiles: 8 - path: raw diff --git a/demo/dvc.lock b/demo/dvc.lock deleted file mode 100644 index 3bab9240d7..0000000000 --- a/demo/dvc.lock +++ /dev/null @@ -1,34 +0,0 @@ -schema: '2.0' -stages: - train: - cmd: python train.py - deps: - - path: data/MNIST - md5: 0aed307494600d178fbdc0d000d6db38.dir - size: 66544866 - nfiles: 10 - - path: train.py - md5: 90f29a92c178927514c7f4d61a984a8a - size: 4865 - params: - params.yaml: - epochs: 15 - lr: 0.003 - weight_decay: 0 - outs: - - path: misclassified.jpg - md5: 5edd7dd13144d95ef72e270570d9d338 - size: 32829 - - path: model.pt - md5: 38126781764ca9fb04496ca2c2173056 - size: 439383 - - path: predictions.json - md5: 0aa0e279e904572c28c1203ebfa41b29 - size: 310000 - - path: training_metrics - md5: 5abc8d18d6b57bd513acf95e55e2755a.dir - size: 947 - nfiles: 2 - - path: training_metrics.json - md5: 2a592caac70651d359b2331ff674beec - size: 69 diff --git a/demo/dvc.yaml b/demo/dvc.yaml deleted file mode 100644 index 434013cca2..0000000000 --- a/demo/dvc.yaml +++ /dev/null @@ -1,22 +0,0 @@ -stages: - train: - cmd: python train.py - deps: - - data/MNIST - - train.py - params: - - params.yaml: - outs: - - model.pt: - checkpoint: true - metrics: - - training_metrics.json: - persist: true - cache: false - plots: - - training_metrics - - misclassified.jpg - - predictions.json: - template: confusion - x: actual - y: predicted diff --git a/demo/params.yaml b/demo/params.yaml deleted file mode 100644 index a90973e184..0000000000 --- a/demo/params.yaml +++ /dev/null @@ -1,3 +0,0 @@ -lr: 0.003 -weight_decay: 0 -epochs: 15 diff --git a/demo/requirements.txt b/demo/requirements.txt deleted file mode 100644 index fa81e540e2..0000000000 --- a/demo/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -dvc[s3]==2.30.0 -torch==1.12.0 -torchvision==0.13.0 \ No newline at end of file diff --git a/demo/train.py b/demo/train.py deleted file mode 100644 index 03ba1f72f2..0000000000 --- a/demo/train.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Model training and evaluation.""" -import json -import os - -import numpy as np -import torch -import torch.nn.functional as F -import torchvision - -from dvclive import Live -from PIL import Image -from ruamel.yaml import YAML - - -def transform(dataset): - """Get inputs and targets from dataset.""" - x = dataset.data.reshape(len(dataset.data), 1, 28, 28)/255 - y = dataset.targets - return x, y - - -def train(model, x, y, lr, weight_decay): - """Train a single epoch.""" - model.train() - criterion = torch.nn.CrossEntropyLoss() - optimizer = torch.optim.Adam(model.parameters(), lr=lr, - weight_decay=weight_decay) - y_pred = model(x) - loss = criterion(y_pred, y) - optimizer.zero_grad() - loss.backward() - optimizer.step() - - -def predict(model, x): - """Get model prediction scores.""" - model.eval() - with torch.no_grad(): - y_pred = model(x) - return y_pred - - -def get_metrics(y, y_pred, y_pred_label): - """Get loss and accuracy metrics.""" - metrics = {} - criterion = torch.nn.CrossEntropyLoss() - metrics["loss"] = criterion(y_pred, y).item() - metrics["acc"] = (y_pred_label == y).sum().item()/len(y) - return metrics - - -def evaluate(model, x, y): - """Evaluate model and save metrics.""" - scores = predict(model, x) - _, labels = torch.max(scores, 1) - actual = [int(v) for v in y] - predicted = [int(v) for v in labels] - predictions = [{ - "actual": int(actual), - "predicted": int(predicted) - } for actual, predicted in zip(actual, predicted)] - with open("predictions.json", "w") as f: - json.dump(predictions, f) - - metrics = get_metrics(y, scores, labels) - - return metrics, predictions - -def get_confusion_image(predictions, dataset): - confusion = {} - for n, pred in enumerate(predictions): - actual = pred["actual"] - predicted = pred["predicted"] - image = np.array(dataset[n][0]) / 255 - confusion[(actual, predicted)] = image - - max_i, max_j = 0, 0 - for (i, j) in confusion: - if i > max_i: - max_i = i - if j > max_j: - max_j = j - - frame_size = 30 - image_shape = (28, 28) - incorrect_color = np.array((255, 100, 100), dtype="uint8") - label_color = np.array((100, 100, 240), dtype="uint8") - - out_matrix = np.ones(shape=((max_i+2) * frame_size, (max_j+2) * frame_size, 3), dtype="uint8") * 240 - - for i in range(max_i+1): - if (i, i) in confusion: - image = confusion[(i, i)] - xs = (i + 1) * frame_size + 1 - xe = (i + 2) * frame_size - 1 - ys = 1 - ye = frame_size - 1 - for c in range(3): - out_matrix[xs:xe, ys:ye, c] = (1 - image) * label_color[c] - out_matrix[ys:ye, xs:xe, c] = (1 - image) * label_color[c] - - for (i, j) in confusion: - image = confusion[(i, j)] - assert image.shape == image_shape - xs = (i + 1) * frame_size + 1 - xe = (i + 2) * frame_size - 1 - ys = (j + 1) * frame_size + 1 - ye = (j + 2) * frame_size - 1 - assert (xe-xs, ye-ys) == image_shape - if i != j: - for c in range(3): - out_matrix[xs:xe, ys:ye, c] = (1 - image) * incorrect_color[c] - - return out_matrix - - -def main(): - """Train model and evaluate on test data.""" - torch.manual_seed(473987) - - model = torch.nn.Sequential( - torch.nn.Flatten(), - torch.nn.Linear(28 * 28, 128), - torch.nn.ReLU(), - torch.nn.Dropout(0.1), - torch.nn.Linear(128, 64), - torch.nn.ReLU(), - torch.nn.Dropout(0.1), - torch.nn.Linear(64, 10), - ) - live = Live("training_metrics", report=None) - - # Load model. - if os.path.exists("model.pt"): - model.load_state_dict(torch.load("model.pt")) - - # Load params. - yaml = YAML(typ="safe") - with open("params.yaml") as f: - params = yaml.load(f) - - # Load train and test data. - mnist_train = torchvision.datasets.MNIST("data", download=True) - x_train, y_train = transform(mnist_train) - mnist_test = torchvision.datasets.MNIST("data", download=True, train=False) - x_test, y_test = transform(mnist_test) - - # Iterate over training epochs. - for epoch in range(params["epochs"]): - print(f"EPOCH: {epoch + 1} / {params['epochs']}") - train(model, x_train, y_train, params["lr"], params["weight_decay"]) - torch.save(model.state_dict(), "model.pt") - # Evaluate and checkpoint. - metrics, predictions = evaluate(model, x_test, y_test) - for k, v in metrics.items(): - live.log(k, v) - missclassified = get_confusion_image(predictions, mnist_test) - Image.fromarray(missclassified).save("misclassified.jpg") - live.next_step() - - -if __name__ == "__main__": - main() diff --git a/demo/training_metrics.json b/demo/training_metrics.json deleted file mode 100644 index 486cd24a07..0000000000 --- a/demo/training_metrics.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "step": 14, - "loss": 0.9596208930015564, - "acc": 0.7735 -} diff --git a/extension/package.json b/extension/package.json index 50604efbe4..60dc665eda 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1412,7 +1412,7 @@ }, "completionEvents": [ "onLink:https://github.com/iterative/example-get-started", - "onLink:https://github.com/iterative/vscode-dvc/tree/main/demo", + "onLink:https://github.com/iterative/vscode-dvc/tree/main/vscode-dvc-demo", "onLink:https://dvc.org/doc/dvclive" ] }, diff --git a/extension/resources/walkthrough/setup-project.md b/extension/resources/walkthrough/setup-project.md index 6e2aa1e31c..adc555065b 100644 --- a/extension/resources/walkthrough/setup-project.md +++ b/extension/resources/walkthrough/setup-project.md @@ -5,7 +5,7 @@ 💡 Check out the [DVC Get Started](https://github.com/iterative/example-get-started) or -[Extension Demo](https://github.com/iterative/vscode-dvc/tree/main/demo) +[Extension Demo](https://github.com/iterative/vscode-dvc/tree/main/vscode-dvc-demo) projects to quickly try the extension. To quickly setup a new DVC project run diff --git a/extension/src/fileSystem/index.test.ts b/extension/src/fileSystem/index.test.ts index 4edf680d52..cf6930d2fb 100644 --- a/extension/src/fileSystem/index.test.ts +++ b/extension/src/fileSystem/index.test.ts @@ -34,7 +34,7 @@ describe('findDvcRootPaths', () => { remove(mockDvcRoot) - expect([...dvcRoots]).toStrictEqual([dvcDemoPath, mockDvcRoot]) + expect([...dvcRoots]).toStrictEqual([mockDvcRoot, dvcDemoPath]) }) }) diff --git a/extension/src/test/e2e/wdio.conf.ts b/extension/src/test/e2e/wdio.conf.ts index a9c14490a6..d2628dc531 100644 --- a/extension/src/test/e2e/wdio.conf.ts +++ b/extension/src/test/e2e/wdio.conf.ts @@ -7,7 +7,7 @@ import { Logger } from '../../common/logger' const screenshotDir = join(__dirname, 'screenshots') const logsDir = join(__dirname, 'logs') const extensionPath = resolve(__dirname, '..', '..', '..') -const dvcDemoPath = resolve(extensionPath, '..', 'demo') +const dvcDemoPath = resolve(extensionPath, '..', 'vscode-dvc-demo') export const config: Options.Testrunner = { after: async function () { diff --git a/extension/src/test/suite/experiments/data/index.test.ts b/extension/src/test/suite/experiments/data/index.test.ts index 0d7579ee9a..a23f571003 100644 --- a/extension/src/test/suite/experiments/data/index.test.ts +++ b/extension/src/test/suite/experiments/data/index.test.ts @@ -1,4 +1,4 @@ -import { join, resolve, sep } from 'path' +import { join, sep } from 'path' import { afterEach, beforeEach, describe, it, suite } from 'mocha' import { EventEmitter, FileSystemWatcher, RelativePattern, Uri } from 'vscode' import { expect } from 'chai' @@ -159,7 +159,7 @@ suite('Experiments Data Test Suite', () => { it('should watch the .git directory for updates', async () => { const mockNow = getMockNow() - const gitRoot = resolve(dvcDemoPath, '..') + const gitRoot = dvcDemoPath const mockExecuteCommand = (command: CommandId) => { if (command === AvailableCommands.GIT_GET_REPOSITORY_ROOT) { diff --git a/extension/src/test/util/index.ts b/extension/src/test/util/index.ts index f9120d4c3a..ac8b2c8210 100644 --- a/extension/src/test/util/index.ts +++ b/extension/src/test/util/index.ts @@ -1,7 +1,7 @@ import { resolve } from 'path' import { Memento, Uri } from 'vscode' -const dvcRoot = resolve(__dirname, '..', '..', '..', '..', 'demo') +const dvcRoot = resolve(__dirname, '..', '..', '..', '..', 'vscode-dvc-demo') export const dvcDemoPath = Uri.file(dvcRoot).fsPath export const basePlotsUrl = Uri.file( resolve(__dirname, '..', 'fixtures', 'plotsDiff', 'staticImages') diff --git a/scripts/virtualenv-install.ts b/scripts/virtualenv-install.ts index fe9f5735e4..5445b38834 100644 --- a/scripts/virtualenv-install.ts +++ b/scripts/virtualenv-install.ts @@ -9,6 +9,6 @@ const importModuleAfterMockingVsCode = () => { const setupVenv = importModuleAfterMockingVsCode() -const cwd = resolve(__dirname, '..', 'demo') +const cwd = resolve(__dirname, '..', 'vscode-dvc-demo') setupVenv(cwd, '.env', '-r', join('.', 'requirements.txt')) diff --git a/vscode-dvc-demo b/vscode-dvc-demo new file mode 160000 index 0000000000..461d2a8807 --- /dev/null +++ b/vscode-dvc-demo @@ -0,0 +1 @@ +Subproject commit 461d2a88079aa49255193816417265bd995cf28a From b365b39dc61d91126874aa97e66f1045c9d3c8db Mon Sep 17 00:00:00 2001 From: Stephanie Roy Date: Mon, 17 Oct 2022 16:50:05 -0400 Subject: [PATCH 02/12] Get repo .git directory --- extension/src/cli/git/constants.ts | 42 ++++++++++++++++--- extension/src/cli/git/executor.ts | 1 + extension/src/experiments/data/constants.ts | 6 +-- extension/src/experiments/data/index.ts | 14 ++++--- extension/src/repository/data/index.ts | 8 ++-- .../test/suite/experiments/data/index.test.ts | 4 +- .../test/suite/repository/data/index.test.ts | 7 ++-- extension/src/test/util/index.ts | 11 +++++ 8 files changed, 69 insertions(+), 24 deletions(-) diff --git a/extension/src/cli/git/constants.ts b/extension/src/cli/git/constants.ts index 61c5575ad0..2f03958352 100644 --- a/extension/src/cli/git/constants.ts +++ b/extension/src/cli/git/constants.ts @@ -1,11 +1,41 @@ import { join } from 'path' +import fs from 'fs' -export const DOT_GIT = '.git' -export const DOT_GIT_HEAD = join(DOT_GIT, 'HEAD') -export const DOT_GIT_INDEX = join(DOT_GIT, 'index') -export const GIT_REFS = join(DOT_GIT, 'refs') -export const GIT_LOGS_REFS = join(DOT_GIT, 'logs', 'refs') -export const HEADS_GIT_REFS = join(GIT_REFS, 'heads') +export enum gitPath { + DOT_GIT = '.git', + DOT_GIT_HEAD = 'HEAD', + DOT_GIT_INDEX = 'index', + GIT_REFS = 'refs', + GIT_LOGS_REFS = 'logs/ref', + HEADS_GIT_REFS = 'heads' +} + +const getGitDirPath = (gitRoot: string) => { + const dotGitPath = join(gitRoot, gitPath.DOT_GIT) + + if (fs.lstatSync(dotGitPath).isFile()) { + const dotGitAsFileContent = fs.readFileSync(dotGitPath, 'utf8') + const gitDirPrefix = 'gitdir: ' + const gitDirLine = dotGitAsFileContent + .split(/\r?\n/) + .find(line => line.indexOf(gitDirPrefix) === 0) + return join(gitRoot, ...(gitDirLine?.slice(8).split('/') || [])) + } + return dotGitPath +} + +const gitRootGitDir: { [key: string]: string } = {} + +export const getGitPath = (gitRoot: string, path: gitPath | string) => { + const gitDir = gitRootGitDir[gitRoot] || getGitDirPath(gitRoot) + gitRootGitDir[gitRoot] = gitDir + + if (path === gitPath.DOT_GIT) { + return gitDir + } + + return join(gitDir, ...path.split('/')) +} export enum Command { ADD = 'add', diff --git a/extension/src/cli/git/executor.ts b/extension/src/cli/git/executor.ts index fe03052ab6..2303a9e1d2 100644 --- a/extension/src/cli/git/executor.ts +++ b/extension/src/cli/git/executor.ts @@ -49,6 +49,7 @@ export class GitExecutor extends GitCli { public async stageAll(cwd: string) { const gitRoot = await this.getGitRepositoryRoot(cwd) + const options = getOptions(gitRoot, Command.ADD, Flag.DOT) return this.executeProcess(options) diff --git a/extension/src/experiments/data/constants.ts b/extension/src/experiments/data/constants.ts index 0875cd0383..77f7a0435e 100644 --- a/extension/src/experiments/data/constants.ts +++ b/extension/src/experiments/data/constants.ts @@ -1,6 +1,6 @@ import { join } from 'path' -import { GIT_LOGS_REFS, GIT_REFS } from '../../cli/git/constants' +import { gitPath } from '../../cli/git/constants' -export const EXPERIMENTS_GIT_REFS = join(GIT_REFS, 'exps') -export const EXPERIMENTS_GIT_LOGS_REFS = join(GIT_LOGS_REFS, 'exps') +export const EXPERIMENTS_GIT_REFS = join(gitPath.GIT_REFS, 'exps') +export const EXPERIMENTS_GIT_LOGS_REFS = join(gitPath.GIT_LOGS_REFS, 'exps') export const EXPERIMENTS_GIT_REFS_EXEC = join(EXPERIMENTS_GIT_REFS, 'exec') diff --git a/extension/src/experiments/data/index.ts b/extension/src/experiments/data/index.ts index 6a4d1d6f97..36b5560bad 100644 --- a/extension/src/experiments/data/index.ts +++ b/extension/src/experiments/data/index.ts @@ -14,7 +14,7 @@ import { AvailableCommands, InternalCommands } from '../../commands/internal' import { ExperimentsOutput } from '../../cli/dvc/contract' import { BaseData } from '../../data' import { ExperimentFlag } from '../../cli/dvc/constants' -import { DOT_GIT, DOT_GIT_HEAD, HEADS_GIT_REFS } from '../../cli/git/constants' +import { getGitPath, gitPath } from '../../cli/git/constants' export const QUEUED_EXPERIMENT_PATH = join('.dvc', 'tmp', 'exps') @@ -80,16 +80,18 @@ export class ExperimentsData extends BaseData { AvailableCommands.GIT_GET_REPOSITORY_ROOT, this.dvcRoot ) + + const dotGitPath = getGitPath(gitRoot, gitPath.DOT_GIT_HEAD) const watchedRelPaths = [ - DOT_GIT_HEAD, - EXPERIMENTS_GIT_REFS, - EXPERIMENTS_GIT_LOGS_REFS, - HEADS_GIT_REFS + dotGitPath, + getGitPath(gitRoot, EXPERIMENTS_GIT_REFS), + getGitPath(gitRoot, EXPERIMENTS_GIT_LOGS_REFS), + getGitPath(gitRoot, gitPath.HEADS_GIT_REFS) ] this.dispose.track( createFileSystemWatcher( - getRelativePattern(join(gitRoot, DOT_GIT), '**'), + getRelativePattern(dotGitPath, '**'), (path: string) => { if (path.includes(EXPERIMENTS_GIT_REFS_EXEC)) { return diff --git a/extension/src/repository/data/index.ts b/extension/src/repository/data/index.ts index 26b09201e7..fb93aa4371 100644 --- a/extension/src/repository/data/index.ts +++ b/extension/src/repository/data/index.ts @@ -1,4 +1,3 @@ -import { join } from 'path' import { Event, EventEmitter } from 'vscode' import { AvailableCommands, InternalCommands } from '../../commands/internal' import { ProcessManager } from '../../processManager' @@ -12,7 +11,7 @@ import { EXPERIMENTS_GIT_REFS } from '../../experiments/data/constants' import { DeferredDisposable } from '../../class/deferred' -import { DOT_GIT } from '../../cli/git/constants' +import { getGitPath, gitPath } from '../../cli/git/constants' import { DataStatusOutput, DvcError } from '../../cli/dvc/contract' export type Data = { @@ -128,7 +127,10 @@ export class RepositoryData extends DeferredDisposable { this.dispose.track( createFileSystemWatcher( - getRelativePattern(join(gitRoot, DOT_GIT), '{HEAD,index}'), + getRelativePattern( + getGitPath(gitRoot, gitPath.DOT_GIT), + '{HEAD,index}' + ), (path: string) => { if (!path) { return diff --git a/extension/src/test/suite/experiments/data/index.test.ts b/extension/src/test/suite/experiments/data/index.test.ts index a23f571003..ef869f97f0 100644 --- a/extension/src/test/suite/experiments/data/index.test.ts +++ b/extension/src/test/suite/experiments/data/index.test.ts @@ -25,7 +25,7 @@ import { import { buildExperimentsData, buildExperimentsDataDependencies } from '../util' import { ExperimentFlag } from '../../../../cli/dvc/constants' import { EXPERIMENTS_GIT_LOGS_REFS } from '../../../../experiments/data/constants' -import { DOT_GIT_HEAD } from '../../../../cli/git/constants' +import { getGitPath, gitPath } from '../../../../cli/git/constants' suite('Experiments Data Test Suite', () => { const disposable = Disposable.fn() @@ -186,7 +186,7 @@ suite('Experiments Data Test Suite', () => { data.onDidUpdate(() => resolve(undefined)) ) - await Watcher.fireWatcher(join(gitRoot, DOT_GIT_HEAD)) + await Watcher.fireWatcher(getGitPath(gitRoot, gitPath.DOT_GIT_HEAD)) await dataUpdatedEvent expect(managedUpdateSpy).to.be.called diff --git a/extension/src/test/suite/repository/data/index.test.ts b/extension/src/test/suite/repository/data/index.test.ts index 664ac6cd64..b20a34f2a2 100644 --- a/extension/src/test/suite/repository/data/index.test.ts +++ b/extension/src/test/suite/repository/data/index.test.ts @@ -1,4 +1,3 @@ -import { join, resolve } from 'path' import { afterEach, beforeEach, describe, it, suite } from 'mocha' import { expect } from 'chai' import { restore, spy, stub } from 'sinon' @@ -13,7 +12,7 @@ import { CommandId, InternalCommands } from '../../../../commands/internal' -import { DOT_GIT_HEAD } from '../../../../cli/git/constants' +import { getGitPath, gitPath } from '../../../../cli/git/constants' suite('Repository Data Test Suite', () => { const disposable = Disposable.fn() @@ -42,7 +41,7 @@ suite('Repository Data Test Suite', () => { }) it('should watch the .git index and HEAD for updates', async () => { - const gitRoot = resolve(dvcDemoPath, '..') + const gitRoot = dvcDemoPath const mockExecuteCommand = (command: CommandId) => { if (command === AvailableCommands.GIT_GET_REPOSITORY_ROOT) { @@ -67,7 +66,7 @@ suite('Repository Data Test Suite', () => { data.onDidUpdate(() => resolve(undefined)) ) - await fireWatcher(join(gitRoot, DOT_GIT_HEAD)) + await fireWatcher(getGitPath(gitRoot, gitPath.DOT_GIT_HEAD)) await dataUpdatedEvent expect(managedUpdateSpy).to.be.called diff --git a/extension/src/test/util/index.ts b/extension/src/test/util/index.ts index ac8b2c8210..9eb98d1c4b 100644 --- a/extension/src/test/util/index.ts +++ b/extension/src/test/util/index.ts @@ -2,7 +2,18 @@ import { resolve } from 'path' import { Memento, Uri } from 'vscode' const dvcRoot = resolve(__dirname, '..', '..', '..', '..', 'vscode-dvc-demo') +const dvcDemoGitRoot = resolve( + __dirname, + '..', + '..', + '..', + '..', + '.git', + 'modules', + 'vscode-dvc-demo' +) export const dvcDemoPath = Uri.file(dvcRoot).fsPath +export const dvcDemoGitPath = Uri.file(dvcDemoGitRoot).fsPath export const basePlotsUrl = Uri.file( resolve(__dirname, '..', 'fixtures', 'plotsDiff', 'staticImages') ).fsPath From b5eb47cd680a01ceb8b58e4a62b91dc47cc4e618 Mon Sep 17 00:00:00 2001 From: Stephanie Roy Date: Tue, 18 Oct 2022 14:31:41 -0400 Subject: [PATCH 03/12] Change more demo paths --- .../experiments/model/filterBy/tree.test.ts | 16 +++--- .../src/experiments/model/sortBy/tree.test.ts | 6 +-- extension/src/experiments/model/tree.test.ts | 22 ++++---- extension/src/fileSystem/watcher.test.ts | 52 ++++++++++++------- extension/src/test/runTest.ts | 2 +- 5 files changed, 55 insertions(+), 43 deletions(-) diff --git a/extension/src/experiments/model/filterBy/tree.test.ts b/extension/src/experiments/model/filterBy/tree.test.ts index a61907f22e..cffa133774 100644 --- a/extension/src/experiments/model/filterBy/tree.test.ts +++ b/extension/src/experiments/model/filterBy/tree.test.ts @@ -59,7 +59,7 @@ describe('ExperimentsFilterByTree', () => { mockedExperiments, mockedInternalCommands ) - mockedGetDvcRoots.mockReturnValueOnce(['demo']) + mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo']) mockedGetFilters.mockReturnValueOnce([]) const rootElements = await experimentsFilterByTree.getChildren() expect(rootElements).toStrictEqual([]) @@ -77,7 +77,7 @@ describe('ExperimentsFilterByTree', () => { value: '90000' } ] - const dvcRoots = ['demo'] + const dvcRoots = ['vsocde-dvc-demo'] mockedGetDvcRoots.mockReturnValueOnce(dvcRoots) mockedGetFilters.mockReturnValueOnce(mockedFilters) mockedGetFilters.mockReturnValueOnce(mockedFilters) @@ -86,7 +86,7 @@ describe('ExperimentsFilterByTree', () => { expect(filters).toStrictEqual([ { description: '== 90000', - dvcRoot: 'demo', + dvcRoot: 'vsocde-dvc-demo', id: buildMetricOrParamPath( ColumnType.PARAMS, 'params.yaml', @@ -102,7 +102,7 @@ describe('ExperimentsFilterByTree', () => { mockedExperiments, mockedInternalCommands ) - const dvcRoots = ['demo', 'other'] + const dvcRoots = ['vsocde-dvc-demo', 'other'] mockedGetDvcRoots.mockReturnValueOnce(dvcRoots) mockedGetFilters.mockReturnValueOnce([ { @@ -134,7 +134,7 @@ describe('ExperimentsFilterByTree', () => { mockedExperiments, mockedInternalCommands ) - const dvcRoots = ['demo', 'and', 'another'] + const dvcRoots = ['vsocde-dvc-demo', 'and', 'another'] mockedGetDvcRoots.mockReturnValueOnce(dvcRoots) mockedGetFilters.mockReturnValueOnce(mockedFilters) mockedGetFilters.mockReturnValueOnce([]) @@ -142,12 +142,12 @@ describe('ExperimentsFilterByTree', () => { await experimentsFilterByTree.getChildren() mockedGetFilters.mockReturnValueOnce(mockedFilters) - const filters = await experimentsFilterByTree.getChildren('demo') + const filters = await experimentsFilterByTree.getChildren('vsocde-dvc-demo') expect(filters).toStrictEqual([ { description: '== 90000', - dvcRoot: 'demo', + dvcRoot: 'vsocde-dvc-demo', id: buildMetricOrParamPath( ColumnType.PARAMS, 'params.yml', @@ -157,7 +157,7 @@ describe('ExperimentsFilterByTree', () => { }, { description: '< 1', - dvcRoot: 'demo', + dvcRoot: 'vsocde-dvc-demo', id: buildMetricOrParamPath(ColumnType.METRICS, 'logs.json', 'metric<1'), label: buildMetricOrParamPath(ColumnType.METRICS, 'logs.json', 'metric') } diff --git a/extension/src/experiments/model/sortBy/tree.test.ts b/extension/src/experiments/model/sortBy/tree.test.ts index 9b6ff9ac14..d71af5d2a3 100644 --- a/extension/src/experiments/model/sortBy/tree.test.ts +++ b/extension/src/experiments/model/sortBy/tree.test.ts @@ -58,7 +58,7 @@ beforeEach(() => { }) describe('ExperimentsSortByTree', () => { - const dvcRoot = 'demo' + const dvcRoot = 'vsocde-dvc-demo' const examplePath = buildMetricOrParamPath(ColumnType.PARAMS, 'test') const exampleSortDefinition: SortDefinition = { descending: true, @@ -113,8 +113,8 @@ describe('ExperimentsSortByTree', () => { mockedInternalCommands ) expect(await experimentsSortByTree.getChildren(undefined)).toStrictEqual([ - dvcRoot, - 'demo2' + 'demo2', + dvcRoot ]) }) diff --git a/extension/src/experiments/model/tree.test.ts b/extension/src/experiments/model/tree.test.ts index ed3a576481..e5635272b3 100644 --- a/extension/src/experiments/model/tree.test.ts +++ b/extension/src/experiments/model/tree.test.ts @@ -73,7 +73,7 @@ describe('ExperimentsTree', () => { mockedExperiments, mockedResourceLocator ) - mockedGetDvcRoots.mockReturnValueOnce(['demo', 'second/repo']) + mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo', 'second/repo']) mockedGetExperiments.mockReturnValueOnce([]) mockedGetExperiments.mockReturnValueOnce([]) @@ -87,7 +87,7 @@ describe('ExperimentsTree', () => { mockedExperiments, mockedResourceLocator ) - mockedGetDvcRoots.mockReturnValueOnce(['demo']) + mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo']) mockedGetExperiments.mockReturnValueOnce([]) const rootElements = await experimentsTree.getChildren() @@ -96,7 +96,7 @@ describe('ExperimentsTree', () => { }) it('should return an array of root elements when at least one experiment exists in one of the repositories', async () => { - const dvcRoots = ['demo', 'and/mock', 'other/repo'] + const dvcRoots = ['vsocde-dvc-demo', 'and/mock', 'other/repo'] const experimentsTree = new ExperimentsTree( mockedExperiments, mockedResourceLocator @@ -402,13 +402,13 @@ describe('ExperimentsTree', () => { mockedExperiments, mockedResourceLocator ) - mockedGetDvcRoots.mockReturnValueOnce(['demo', 'other']) + mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo', 'other']) mockedGetExperiments.mockReturnValueOnce([]) mockedGetExperiments.mockReturnValueOnce([]) await experimentsTree.getChildren() - const treeItem = experimentsTree.getTreeItem('demo') + const treeItem = experimentsTree.getTreeItem('vsocde-dvc-demo') expect(treeItem).toStrictEqual({ ...mockedItem }) }) @@ -428,7 +428,7 @@ describe('ExperimentsTree', () => { const treeItem = experimentsTree.getTreeItem({ collapsibleState: 0, description: undefined, - dvcRoot: 'demo', + dvcRoot: 'vsocde-dvc-demo', iconPath: mockedClockResource, id: 'f0778b3', label: 'f0778b3', @@ -463,7 +463,7 @@ describe('ExperimentsTree', () => { const treeItem = experimentsTree.getTreeItem({ collapsibleState: 0, description: undefined, - dvcRoot: 'demo', + dvcRoot: 'vsocde-dvc-demo', iconPath: new ThemeIcon('loading~spin'), id: 'workspace', label: 'workspace', @@ -496,7 +496,7 @@ describe('ExperimentsTree', () => { const treeItem = experimentsTree.getTreeItem({ collapsibleState: 1, description: undefined, - dvcRoot: 'demo', + dvcRoot: 'vsocde-dvc-demo', iconPath: new ThemeIcon('loading~spin'), id: 'f0778b3', label: 'f0778b3', @@ -529,7 +529,7 @@ describe('ExperimentsTree', () => { const treeItem = experimentsTree.getTreeItem({ collapsibleState: 0, description: undefined, - dvcRoot: 'demo', + dvcRoot: 'vsocde-dvc-demo', iconPath: new ThemeIcon('circle-filled'), id: 'f0778b3', label: 'f0778b3', @@ -556,12 +556,12 @@ describe('ExperimentsTree', () => { mockedExperiments, mockedResourceLocator ) - mockedGetDvcRoots.mockReturnValueOnce(['demo']) + mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo']) const treeItem = experimentsTree.getTreeItem({ collapsibleState: 1, description: undefined, - dvcRoot: 'demo', + dvcRoot: 'vsocde-dvc-demo', iconPath: new ThemeIcon('circle-filled'), id: 'f0998a3', label: 'f0998a3', diff --git a/extension/src/fileSystem/watcher.test.ts b/extension/src/fileSystem/watcher.test.ts index 9886fb5bd3..a24fd02e0d 100644 --- a/extension/src/fileSystem/watcher.test.ts +++ b/extension/src/fileSystem/watcher.test.ts @@ -25,58 +25,70 @@ beforeEach(() => { describe('ignoredDotDirectories', () => { it('should match all paths under .dvc directories', () => { expect( - ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/.dvc/tmp') + ignoredDotDirectories.test( + '/Users/robot/vscode-dvc/vsocde-dvc-demo/.dvc/tmp' + ) + ).toBe(true) + expect( + ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\.dvc\\tmp') ).toBe(true) - expect(ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\.dvc\\tmp')).toBe( - true - ) }) it('should match all paths under .env directories', () => { expect( - ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/.env/bin') + ignoredDotDirectories.test( + '/Users/robot/vscode-dvc/vsocde-dvc-demo/.env/bin' + ) + ).toBe(true) + expect( + ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\.env\\bin') ).toBe(true) - expect(ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\.env\\bin')).toBe( - true - ) }) it('should match all paths under .venv directories', () => { expect( ignoredDotDirectories.test( - '/Users/robot/vscode-dvc/demo/.venv/bin/python' + '/Users/robot/vscode-dvc/vsocde-dvc-demo/.venv/bin/python' ) ).toBe(true) expect( - ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\.venv\\bin\\python') + ignoredDotDirectories.test( + 'C:\\vscode-dvc\\vsocde-dvc-demo\\.venv\\bin\\python' + ) ).toBe(true) }) it('should not match dot files', () => { expect( - ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/.gitignore') + ignoredDotDirectories.test( + '/Users/robot/vscode-dvc/vsocde-dvc-demo/.gitignore' + ) + ).toBe(false) + expect( + ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\.gitignore') ).toBe(false) - expect(ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\.gitignore')).toBe( - false - ) }) it('should not match normal directories', () => { expect( - ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/data/MNIST') + ignoredDotDirectories.test( + '/Users/robot/vscode-dvc/vsocde-dvc-demo/data/MNIST' + ) ).toBe(false) expect( - ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\data\\MNIST') + ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\data\\MNIST') ).toBe(false) }) it('should not match normal files', () => { expect( - ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/train.py') + ignoredDotDirectories.test( + '/Users/robot/vscode-dvc/vsocde-dvc-demo/train.py' + ) + ).toBe(false) + expect( + ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\train.py') ).toBe(false) - expect(ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\train.py')).toBe( - false - ) }) it('should not match .dvc files', () => { diff --git a/extension/src/test/runTest.ts b/extension/src/test/runTest.ts index 0ed87ee3b0..2fcb1f009f 100644 --- a/extension/src/test/runTest.ts +++ b/extension/src/test/runTest.ts @@ -12,7 +12,7 @@ async function main() { const vscodeExecutablePath = await downloadAndUnzipVSCode('insiders') - const workspacePath = resolve(__dirname, '../../../demo') + const workspacePath = resolve(__dirname, '../../../vsocde-dvc-demo') await runTests({ extensionDevelopmentPath, From 22257803c9584fabc5886831b4cb6d9cfe03969e Mon Sep 17 00:00:00 2001 From: Stephanie Roy Date: Tue, 18 Oct 2022 15:07:21 -0400 Subject: [PATCH 04/12] Cleanup --- extension/src/cli/git/constants.ts | 1 + extension/src/cli/git/executor.ts | 1 - extension/src/test/util/index.ts | 12 +----------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/extension/src/cli/git/constants.ts b/extension/src/cli/git/constants.ts index 2f03958352..88426ab9d1 100644 --- a/extension/src/cli/git/constants.ts +++ b/extension/src/cli/git/constants.ts @@ -10,6 +10,7 @@ export enum gitPath { HEADS_GIT_REFS = 'heads' } +// .git inside a submodule is a file with the following content: `gitdir: ../.git/modules/vscode-dvc-demo` const getGitDirPath = (gitRoot: string) => { const dotGitPath = join(gitRoot, gitPath.DOT_GIT) diff --git a/extension/src/cli/git/executor.ts b/extension/src/cli/git/executor.ts index 2303a9e1d2..fe03052ab6 100644 --- a/extension/src/cli/git/executor.ts +++ b/extension/src/cli/git/executor.ts @@ -49,7 +49,6 @@ export class GitExecutor extends GitCli { public async stageAll(cwd: string) { const gitRoot = await this.getGitRepositoryRoot(cwd) - const options = getOptions(gitRoot, Command.ADD, Flag.DOT) return this.executeProcess(options) diff --git a/extension/src/test/util/index.ts b/extension/src/test/util/index.ts index 7153cf488a..4e5ff5c18c 100644 --- a/extension/src/test/util/index.ts +++ b/extension/src/test/util/index.ts @@ -2,18 +2,8 @@ import { resolve } from 'path' import { Memento, Uri, workspace, WorkspaceFolder } from 'vscode' const dvcRoot = resolve(__dirname, '..', '..', '..', '..', 'vscode-dvc-demo') -const dvcDemoGitRoot = resolve( - __dirname, - '..', - '..', - '..', - '..', - '.git', - 'modules', - 'vscode-dvc-demo' -) + export const dvcDemoPath = Uri.file(dvcRoot).fsPath -export const dvcDemoGitPath = Uri.file(dvcDemoGitRoot).fsPath export const basePlotsUrl = Uri.file( resolve(__dirname, '..', 'fixtures', 'plotsDiff', 'staticImages') ).fsPath From 59c04c718220e56e1becc13c4d54920fcfe66ae3 Mon Sep 17 00:00:00 2001 From: Stephanie Roy Date: Tue, 18 Oct 2022 16:16:36 -0400 Subject: [PATCH 05/12] Add demo project init postinstall --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6f4848f915..ce455c6740 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "build": "yarn turbo run package", "install-frozen-lockfile": "./scripts/install-frozen-lockfile.sh", "dev-server": "yarn turbo run dev --parallel", - "postinstall": "husky install", + "postinstall": "husky install && git submodule init && git submodule update", "storybook": "yarn workspace dvc-vscode-webview storybook", "build-storybook": "yarn turbo run build-storybook --filter=dvc-vscode-webview", "chromatic": "yarn workspace dvc-vscode-webview chromatic", From 58e540a881bf709548ab15a2ab714a2791b307d7 Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Wed, 19 Oct 2022 10:52:08 +1100 Subject: [PATCH 06/12] rename vscode-dvc-demo back to demo --- .codeclimate.json | 2 +- .gitmodules | 4 +- .vscode/settings.json | 2 +- CONTRIBUTING.md | 10 ++-- vscode-dvc-demo => demo | 0 extension/package.json | 2 +- .../resources/walkthrough/setup-project.md | 4 +- extension/src/cli/git/constants.ts | 2 +- .../experiments/model/filterBy/tree.test.ts | 16 +++--- .../src/experiments/model/sortBy/tree.test.ts | 6 +-- extension/src/experiments/model/tree.test.ts | 22 ++++---- extension/src/fileSystem/index.test.ts | 2 +- extension/src/fileSystem/watcher.test.ts | 52 +++++++------------ extension/src/test/e2e/wdio.conf.ts | 2 +- extension/src/test/runTest.ts | 2 +- extension/src/test/util/index.ts | 2 +- scripts/virtualenv-install.ts | 2 +- 17 files changed, 60 insertions(+), 72 deletions(-) rename vscode-dvc-demo => demo (100%) diff --git a/.codeclimate.json b/.codeclimate.json index e7ec942ad2..957e963d3b 100644 --- a/.codeclimate.json +++ b/.codeclimate.json @@ -5,7 +5,7 @@ "**/*.test.*", "**/constants.ts", "**/contract.ts", - "vscode-dvc-demo/**", + "demo/**", "extension/src/test/**", "languageServer/src/test/**", "webview/src/react-table-config.d.ts", diff --git a/.gitmodules b/.gitmodules index 38af2e4fa6..ae8e88b6ed 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "vscode-dvc-demo"] - path = vscode-dvc-demo +[submodule "demo"] + path = demo url = https://github.com/iterative/vscode-dvc-demo diff --git a/.vscode/settings.json b/.vscode/settings.json index 552c226125..f35eed42cf 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,7 @@ "prettier.configPath": ".prettierrc.json", "editor.tabSize": 2, "typescript.updateImportsOnFileMove.enabled": "prompt", - "npm.exclude": ["**/{vscode-dvc-demo,.wdio*}/**"], + "npm.exclude": ["**/{demo,.wdio*}/**"], "cSpell.words": [ "appdirs", "camelcase", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b5d0fa8c9..2fd12d67b0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,17 +58,17 @@ First, ensure that [Visual Studio Code](https://code.visualstudio.com) and ## The demo project -The [demo project](vscode-dvc-demo) is provided as a lightweight, convenient -testbed to try your changes to this extension. +The [demo project](demo) is provided as a lightweight, convenient testbed to try +your changes to this extension. > **Note**: It is not an exhaustive showcase of DVC's features. Testers are > encouraged to try other DVC repositories -- especially real-world cases! - Run `yarn setup:venv` from the _Terminal_ in the root of this repo to set up a - Python virtual environment for the demo project (in `vscode-dvc-demo/.env`). + Python virtual environment for the demo project (in `demo/.env`). -- Open the `./vscode-dvc-demo` project in a VS Code window, for example in the - _Extension Development Host_ (see previous section). +- Open the `./demo` project in a VS Code window, for example in the _Extension + Development Host_ (see previous section). - Pull the project data using this extension (either from the **DVC Tracked** panel in the _File Explorer_ or the **DVC panel** in _Source Control_) or by diff --git a/vscode-dvc-demo b/demo similarity index 100% rename from vscode-dvc-demo rename to demo diff --git a/extension/package.json b/extension/package.json index 2fe1db0d66..2193cb4940 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1412,7 +1412,7 @@ }, "completionEvents": [ "onLink:https://github.com/iterative/example-get-started", - "onLink:https://github.com/iterative/vscode-dvc/tree/main/vscode-dvc-demo", + "onLink:https://github.com/iterative/vscode-dvc-demo", "onLink:https://dvc.org/doc/dvclive" ] }, diff --git a/extension/resources/walkthrough/setup-project.md b/extension/resources/walkthrough/setup-project.md index adc555065b..281c33bb39 100644 --- a/extension/resources/walkthrough/setup-project.md +++ b/extension/resources/walkthrough/setup-project.md @@ -5,8 +5,8 @@ 💡 Check out the [DVC Get Started](https://github.com/iterative/example-get-started) or -[Extension Demo](https://github.com/iterative/vscode-dvc/tree/main/vscode-dvc-demo) -projects to quickly try the extension. +[Extension Demo](https://github.com/iterative/vscode-dvc-demo) projects to +quickly try the extension. To quickly setup a new DVC project run [`dvc exp init -i`](https://dvc.org/doc/command-reference/exp/init#example-interactive-mode) diff --git a/extension/src/cli/git/constants.ts b/extension/src/cli/git/constants.ts index 88426ab9d1..50fa2ceceb 100644 --- a/extension/src/cli/git/constants.ts +++ b/extension/src/cli/git/constants.ts @@ -10,7 +10,7 @@ export enum gitPath { HEADS_GIT_REFS = 'heads' } -// .git inside a submodule is a file with the following content: `gitdir: ../.git/modules/vscode-dvc-demo` +// .git inside a submodule is a file with the following content: `gitdir: ../.git/modules/demo` const getGitDirPath = (gitRoot: string) => { const dotGitPath = join(gitRoot, gitPath.DOT_GIT) diff --git a/extension/src/experiments/model/filterBy/tree.test.ts b/extension/src/experiments/model/filterBy/tree.test.ts index cffa133774..a61907f22e 100644 --- a/extension/src/experiments/model/filterBy/tree.test.ts +++ b/extension/src/experiments/model/filterBy/tree.test.ts @@ -59,7 +59,7 @@ describe('ExperimentsFilterByTree', () => { mockedExperiments, mockedInternalCommands ) - mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo']) + mockedGetDvcRoots.mockReturnValueOnce(['demo']) mockedGetFilters.mockReturnValueOnce([]) const rootElements = await experimentsFilterByTree.getChildren() expect(rootElements).toStrictEqual([]) @@ -77,7 +77,7 @@ describe('ExperimentsFilterByTree', () => { value: '90000' } ] - const dvcRoots = ['vsocde-dvc-demo'] + const dvcRoots = ['demo'] mockedGetDvcRoots.mockReturnValueOnce(dvcRoots) mockedGetFilters.mockReturnValueOnce(mockedFilters) mockedGetFilters.mockReturnValueOnce(mockedFilters) @@ -86,7 +86,7 @@ describe('ExperimentsFilterByTree', () => { expect(filters).toStrictEqual([ { description: '== 90000', - dvcRoot: 'vsocde-dvc-demo', + dvcRoot: 'demo', id: buildMetricOrParamPath( ColumnType.PARAMS, 'params.yaml', @@ -102,7 +102,7 @@ describe('ExperimentsFilterByTree', () => { mockedExperiments, mockedInternalCommands ) - const dvcRoots = ['vsocde-dvc-demo', 'other'] + const dvcRoots = ['demo', 'other'] mockedGetDvcRoots.mockReturnValueOnce(dvcRoots) mockedGetFilters.mockReturnValueOnce([ { @@ -134,7 +134,7 @@ describe('ExperimentsFilterByTree', () => { mockedExperiments, mockedInternalCommands ) - const dvcRoots = ['vsocde-dvc-demo', 'and', 'another'] + const dvcRoots = ['demo', 'and', 'another'] mockedGetDvcRoots.mockReturnValueOnce(dvcRoots) mockedGetFilters.mockReturnValueOnce(mockedFilters) mockedGetFilters.mockReturnValueOnce([]) @@ -142,12 +142,12 @@ describe('ExperimentsFilterByTree', () => { await experimentsFilterByTree.getChildren() mockedGetFilters.mockReturnValueOnce(mockedFilters) - const filters = await experimentsFilterByTree.getChildren('vsocde-dvc-demo') + const filters = await experimentsFilterByTree.getChildren('demo') expect(filters).toStrictEqual([ { description: '== 90000', - dvcRoot: 'vsocde-dvc-demo', + dvcRoot: 'demo', id: buildMetricOrParamPath( ColumnType.PARAMS, 'params.yml', @@ -157,7 +157,7 @@ describe('ExperimentsFilterByTree', () => { }, { description: '< 1', - dvcRoot: 'vsocde-dvc-demo', + dvcRoot: 'demo', id: buildMetricOrParamPath(ColumnType.METRICS, 'logs.json', 'metric<1'), label: buildMetricOrParamPath(ColumnType.METRICS, 'logs.json', 'metric') } diff --git a/extension/src/experiments/model/sortBy/tree.test.ts b/extension/src/experiments/model/sortBy/tree.test.ts index d71af5d2a3..9b6ff9ac14 100644 --- a/extension/src/experiments/model/sortBy/tree.test.ts +++ b/extension/src/experiments/model/sortBy/tree.test.ts @@ -58,7 +58,7 @@ beforeEach(() => { }) describe('ExperimentsSortByTree', () => { - const dvcRoot = 'vsocde-dvc-demo' + const dvcRoot = 'demo' const examplePath = buildMetricOrParamPath(ColumnType.PARAMS, 'test') const exampleSortDefinition: SortDefinition = { descending: true, @@ -113,8 +113,8 @@ describe('ExperimentsSortByTree', () => { mockedInternalCommands ) expect(await experimentsSortByTree.getChildren(undefined)).toStrictEqual([ - 'demo2', - dvcRoot + dvcRoot, + 'demo2' ]) }) diff --git a/extension/src/experiments/model/tree.test.ts b/extension/src/experiments/model/tree.test.ts index e5635272b3..ed3a576481 100644 --- a/extension/src/experiments/model/tree.test.ts +++ b/extension/src/experiments/model/tree.test.ts @@ -73,7 +73,7 @@ describe('ExperimentsTree', () => { mockedExperiments, mockedResourceLocator ) - mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo', 'second/repo']) + mockedGetDvcRoots.mockReturnValueOnce(['demo', 'second/repo']) mockedGetExperiments.mockReturnValueOnce([]) mockedGetExperiments.mockReturnValueOnce([]) @@ -87,7 +87,7 @@ describe('ExperimentsTree', () => { mockedExperiments, mockedResourceLocator ) - mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo']) + mockedGetDvcRoots.mockReturnValueOnce(['demo']) mockedGetExperiments.mockReturnValueOnce([]) const rootElements = await experimentsTree.getChildren() @@ -96,7 +96,7 @@ describe('ExperimentsTree', () => { }) it('should return an array of root elements when at least one experiment exists in one of the repositories', async () => { - const dvcRoots = ['vsocde-dvc-demo', 'and/mock', 'other/repo'] + const dvcRoots = ['demo', 'and/mock', 'other/repo'] const experimentsTree = new ExperimentsTree( mockedExperiments, mockedResourceLocator @@ -402,13 +402,13 @@ describe('ExperimentsTree', () => { mockedExperiments, mockedResourceLocator ) - mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo', 'other']) + mockedGetDvcRoots.mockReturnValueOnce(['demo', 'other']) mockedGetExperiments.mockReturnValueOnce([]) mockedGetExperiments.mockReturnValueOnce([]) await experimentsTree.getChildren() - const treeItem = experimentsTree.getTreeItem('vsocde-dvc-demo') + const treeItem = experimentsTree.getTreeItem('demo') expect(treeItem).toStrictEqual({ ...mockedItem }) }) @@ -428,7 +428,7 @@ describe('ExperimentsTree', () => { const treeItem = experimentsTree.getTreeItem({ collapsibleState: 0, description: undefined, - dvcRoot: 'vsocde-dvc-demo', + dvcRoot: 'demo', iconPath: mockedClockResource, id: 'f0778b3', label: 'f0778b3', @@ -463,7 +463,7 @@ describe('ExperimentsTree', () => { const treeItem = experimentsTree.getTreeItem({ collapsibleState: 0, description: undefined, - dvcRoot: 'vsocde-dvc-demo', + dvcRoot: 'demo', iconPath: new ThemeIcon('loading~spin'), id: 'workspace', label: 'workspace', @@ -496,7 +496,7 @@ describe('ExperimentsTree', () => { const treeItem = experimentsTree.getTreeItem({ collapsibleState: 1, description: undefined, - dvcRoot: 'vsocde-dvc-demo', + dvcRoot: 'demo', iconPath: new ThemeIcon('loading~spin'), id: 'f0778b3', label: 'f0778b3', @@ -529,7 +529,7 @@ describe('ExperimentsTree', () => { const treeItem = experimentsTree.getTreeItem({ collapsibleState: 0, description: undefined, - dvcRoot: 'vsocde-dvc-demo', + dvcRoot: 'demo', iconPath: new ThemeIcon('circle-filled'), id: 'f0778b3', label: 'f0778b3', @@ -556,12 +556,12 @@ describe('ExperimentsTree', () => { mockedExperiments, mockedResourceLocator ) - mockedGetDvcRoots.mockReturnValueOnce(['vsocde-dvc-demo']) + mockedGetDvcRoots.mockReturnValueOnce(['demo']) const treeItem = experimentsTree.getTreeItem({ collapsibleState: 1, description: undefined, - dvcRoot: 'vsocde-dvc-demo', + dvcRoot: 'demo', iconPath: new ThemeIcon('circle-filled'), id: 'f0998a3', label: 'f0998a3', diff --git a/extension/src/fileSystem/index.test.ts b/extension/src/fileSystem/index.test.ts index cf6930d2fb..4edf680d52 100644 --- a/extension/src/fileSystem/index.test.ts +++ b/extension/src/fileSystem/index.test.ts @@ -34,7 +34,7 @@ describe('findDvcRootPaths', () => { remove(mockDvcRoot) - expect([...dvcRoots]).toStrictEqual([mockDvcRoot, dvcDemoPath]) + expect([...dvcRoots]).toStrictEqual([dvcDemoPath, mockDvcRoot]) }) }) diff --git a/extension/src/fileSystem/watcher.test.ts b/extension/src/fileSystem/watcher.test.ts index a24fd02e0d..9886fb5bd3 100644 --- a/extension/src/fileSystem/watcher.test.ts +++ b/extension/src/fileSystem/watcher.test.ts @@ -25,70 +25,58 @@ beforeEach(() => { describe('ignoredDotDirectories', () => { it('should match all paths under .dvc directories', () => { expect( - ignoredDotDirectories.test( - '/Users/robot/vscode-dvc/vsocde-dvc-demo/.dvc/tmp' - ) - ).toBe(true) - expect( - ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\.dvc\\tmp') + ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/.dvc/tmp') ).toBe(true) + expect(ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\.dvc\\tmp')).toBe( + true + ) }) it('should match all paths under .env directories', () => { expect( - ignoredDotDirectories.test( - '/Users/robot/vscode-dvc/vsocde-dvc-demo/.env/bin' - ) - ).toBe(true) - expect( - ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\.env\\bin') + ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/.env/bin') ).toBe(true) + expect(ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\.env\\bin')).toBe( + true + ) }) it('should match all paths under .venv directories', () => { expect( ignoredDotDirectories.test( - '/Users/robot/vscode-dvc/vsocde-dvc-demo/.venv/bin/python' + '/Users/robot/vscode-dvc/demo/.venv/bin/python' ) ).toBe(true) expect( - ignoredDotDirectories.test( - 'C:\\vscode-dvc\\vsocde-dvc-demo\\.venv\\bin\\python' - ) + ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\.venv\\bin\\python') ).toBe(true) }) it('should not match dot files', () => { expect( - ignoredDotDirectories.test( - '/Users/robot/vscode-dvc/vsocde-dvc-demo/.gitignore' - ) - ).toBe(false) - expect( - ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\.gitignore') + ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/.gitignore') ).toBe(false) + expect(ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\.gitignore')).toBe( + false + ) }) it('should not match normal directories', () => { expect( - ignoredDotDirectories.test( - '/Users/robot/vscode-dvc/vsocde-dvc-demo/data/MNIST' - ) + ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/data/MNIST') ).toBe(false) expect( - ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\data\\MNIST') + ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\data\\MNIST') ).toBe(false) }) it('should not match normal files', () => { expect( - ignoredDotDirectories.test( - '/Users/robot/vscode-dvc/vsocde-dvc-demo/train.py' - ) - ).toBe(false) - expect( - ignoredDotDirectories.test('C:\\vscode-dvc\\vsocde-dvc-demo\\train.py') + ignoredDotDirectories.test('/Users/robot/vscode-dvc/demo/train.py') ).toBe(false) + expect(ignoredDotDirectories.test('C:\\vscode-dvc\\demo\\train.py')).toBe( + false + ) }) it('should not match .dvc files', () => { diff --git a/extension/src/test/e2e/wdio.conf.ts b/extension/src/test/e2e/wdio.conf.ts index 907871fd2c..b40ba1daa8 100644 --- a/extension/src/test/e2e/wdio.conf.ts +++ b/extension/src/test/e2e/wdio.conf.ts @@ -7,7 +7,7 @@ import { Logger } from '../../common/logger' const screenshotDir = join(__dirname, 'screenshots') const logsDir = join(__dirname, 'logs') const extensionPath = resolve(__dirname, '..', '..', '..') -const dvcDemoPath = resolve(extensionPath, '..', 'vscode-dvc-demo') +const dvcDemoPath = resolve(extensionPath, '..', 'demo') export const config: Options.Testrunner = { after: async function () { diff --git a/extension/src/test/runTest.ts b/extension/src/test/runTest.ts index 2fcb1f009f..0ed87ee3b0 100644 --- a/extension/src/test/runTest.ts +++ b/extension/src/test/runTest.ts @@ -12,7 +12,7 @@ async function main() { const vscodeExecutablePath = await downloadAndUnzipVSCode('insiders') - const workspacePath = resolve(__dirname, '../../../vsocde-dvc-demo') + const workspacePath = resolve(__dirname, '../../../demo') await runTests({ extensionDevelopmentPath, diff --git a/extension/src/test/util/index.ts b/extension/src/test/util/index.ts index 4e5ff5c18c..4cf4cfda33 100644 --- a/extension/src/test/util/index.ts +++ b/extension/src/test/util/index.ts @@ -1,7 +1,7 @@ import { resolve } from 'path' import { Memento, Uri, workspace, WorkspaceFolder } from 'vscode' -const dvcRoot = resolve(__dirname, '..', '..', '..', '..', 'vscode-dvc-demo') +const dvcRoot = resolve(__dirname, '..', '..', '..', '..', 'demo') export const dvcDemoPath = Uri.file(dvcRoot).fsPath export const basePlotsUrl = Uri.file( diff --git a/scripts/virtualenv-install.ts b/scripts/virtualenv-install.ts index 5445b38834..fe9f5735e4 100644 --- a/scripts/virtualenv-install.ts +++ b/scripts/virtualenv-install.ts @@ -9,6 +9,6 @@ const importModuleAfterMockingVsCode = () => { const setupVenv = importModuleAfterMockingVsCode() -const cwd = resolve(__dirname, '..', 'vscode-dvc-demo') +const cwd = resolve(__dirname, '..', 'demo') setupVenv(cwd, '.env', '-r', join('.', 'requirements.txt')) From 2b5d84b72f59374fe31e07a7810edec989b68b3d Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Wed, 19 Oct 2022 10:52:28 +1100 Subject: [PATCH 07/12] checkout submodule in CI --- .github/workflows/continuous-integration.yml | 1 + .github/workflows/cross-platform-test.yml | 1 + .github/workflows/dvc-cli-output-test.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index db473115af..b56bad9e5e 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,6 +18,7 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + submodules: true - name: Setup Python environment uses: actions/setup-python@v4 diff --git a/.github/workflows/cross-platform-test.yml b/.github/workflows/cross-platform-test.yml index 62b243d851..7e95c1578f 100644 --- a/.github/workflows/cross-platform-test.yml +++ b/.github/workflows/cross-platform-test.yml @@ -21,6 +21,7 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + submodules: true - name: Setup Node.js environment uses: actions/setup-node@v3 diff --git a/.github/workflows/dvc-cli-output-test.yml b/.github/workflows/dvc-cli-output-test.yml index 0c168f4a8f..a1b43604c6 100644 --- a/.github/workflows/dvc-cli-output-test.yml +++ b/.github/workflows/dvc-cli-output-test.yml @@ -16,6 +16,7 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + submodules: true - name: Setup Node.js environment uses: actions/setup-node@v3 From 27f6cdf7a010467fe5c5c6b8a897ff8f41774d61 Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Wed, 19 Oct 2022 11:53:41 +1100 Subject: [PATCH 08/12] move root finding out of constants file and add test --- extension/src/cli/git/constants.ts | 45 ++++-------------- extension/src/experiments/data/index.ts | 11 +++-- extension/src/fileSystem/index.ts | 30 ++++++++++++ extension/src/repository/data/index.ts | 3 +- .../test/suite/experiments/data/index.test.ts | 3 +- .../src/test/suite/fileSystem/index.test.ts | 46 +++++++++++++++++++ .../test/suite/repository/data/index.test.ts | 3 +- extension/src/test/util/index.ts | 1 - 8 files changed, 96 insertions(+), 46 deletions(-) create mode 100644 extension/src/test/suite/fileSystem/index.test.ts diff --git a/extension/src/cli/git/constants.ts b/extension/src/cli/git/constants.ts index 50fa2ceceb..364b6b4807 100644 --- a/extension/src/cli/git/constants.ts +++ b/extension/src/cli/git/constants.ts @@ -1,42 +1,13 @@ import { join } from 'path' -import fs from 'fs' -export enum gitPath { - DOT_GIT = '.git', - DOT_GIT_HEAD = 'HEAD', - DOT_GIT_INDEX = 'index', - GIT_REFS = 'refs', - GIT_LOGS_REFS = 'logs/ref', - HEADS_GIT_REFS = 'heads' -} - -// .git inside a submodule is a file with the following content: `gitdir: ../.git/modules/demo` -const getGitDirPath = (gitRoot: string) => { - const dotGitPath = join(gitRoot, gitPath.DOT_GIT) - - if (fs.lstatSync(dotGitPath).isFile()) { - const dotGitAsFileContent = fs.readFileSync(dotGitPath, 'utf8') - const gitDirPrefix = 'gitdir: ' - const gitDirLine = dotGitAsFileContent - .split(/\r?\n/) - .find(line => line.indexOf(gitDirPrefix) === 0) - return join(gitRoot, ...(gitDirLine?.slice(8).split('/') || [])) - } - return dotGitPath -} - -const gitRootGitDir: { [key: string]: string } = {} - -export const getGitPath = (gitRoot: string, path: gitPath | string) => { - const gitDir = gitRootGitDir[gitRoot] || getGitDirPath(gitRoot) - gitRootGitDir[gitRoot] = gitDir - - if (path === gitPath.DOT_GIT) { - return gitDir - } - - return join(gitDir, ...path.split('/')) -} +export const gitPath = { + DOT_GIT: '.git', + DOT_GIT_HEAD: 'HEAD', + DOT_GIT_INDEX: 'index', + GIT_LOGS_REFS: join('logs', 'ref'), + GIT_REFS: 'refs', + HEADS_GIT_REFS: 'heads' +} as const export enum Command { ADD = 'add', diff --git a/extension/src/experiments/data/index.ts b/extension/src/experiments/data/index.ts index 36b5560bad..051d367f87 100644 --- a/extension/src/experiments/data/index.ts +++ b/extension/src/experiments/data/index.ts @@ -14,7 +14,8 @@ import { AvailableCommands, InternalCommands } from '../../commands/internal' import { ExperimentsOutput } from '../../cli/dvc/contract' import { BaseData } from '../../data' import { ExperimentFlag } from '../../cli/dvc/constants' -import { getGitPath, gitPath } from '../../cli/git/constants' +import { gitPath } from '../../cli/git/constants' +import { getGitPath } from '../../fileSystem' export const QUEUED_EXPERIMENT_PATH = join('.dvc', 'tmp', 'exps') @@ -83,10 +84,10 @@ export class ExperimentsData extends BaseData { const dotGitPath = getGitPath(gitRoot, gitPath.DOT_GIT_HEAD) const watchedRelPaths = [ - dotGitPath, - getGitPath(gitRoot, EXPERIMENTS_GIT_REFS), - getGitPath(gitRoot, EXPERIMENTS_GIT_LOGS_REFS), - getGitPath(gitRoot, gitPath.HEADS_GIT_REFS) + gitPath.DOT_GIT_HEAD, + EXPERIMENTS_GIT_REFS, + EXPERIMENTS_GIT_LOGS_REFS, + gitPath.HEADS_GIT_REFS ] this.dispose.track( diff --git a/extension/src/fileSystem/index.ts b/extension/src/fileSystem/index.ts index 6eb7f0cd89..efbb3e4da1 100644 --- a/extension/src/fileSystem/index.ts +++ b/extension/src/fileSystem/index.ts @@ -13,6 +13,7 @@ import { Uri } from 'vscode' import { standardizePath } from './path' import { definedAndNonEmpty } from '../util/array' import { Logger } from '../common/logger' +import { gitPath } from '../cli/git/constants' export const exists = (path: string): boolean => existsSync(path) @@ -63,6 +64,35 @@ export const findAbsoluteDvcRootPath = async ( return [absoluteRoot] } +// .git inside a submodule is a file with the following content: `gitdir: ../.git/modules/demo` +const findDotGitDir = (gitRoot: string) => { + const dotGitPath = join(gitRoot, gitPath.DOT_GIT) + + const isSubmodule = lstatSync(dotGitPath).isFile() + if (isSubmodule) { + const dotGitAsFileContent = readFileSync(dotGitPath, 'utf8') + const gitDirPrefix = 'gitdir: ' + const gitDirLine = dotGitAsFileContent + .split(/\r?\n/) + .find(line => line.indexOf(gitDirPrefix) === 0) + return resolve(gitRoot, ...(gitDirLine?.slice(8).split('/') || [])) + } + return dotGitPath +} + +const gitRootGitDir: { [key: string]: string } = {} + +export const getGitPath = (gitRoot: string, path: string) => { + const gitDir = gitRootGitDir[gitRoot] || findDotGitDir(gitRoot) + gitRootGitDir[gitRoot] = gitDir + + if (path === gitPath.DOT_GIT) { + return gitDir + } + + return join(gitDir, path) +} + export const isSameOrChild = (root: string, path: string) => { const rel = relative(root, path) return !rel.startsWith('..') diff --git a/extension/src/repository/data/index.ts b/extension/src/repository/data/index.ts index fb93aa4371..ae22c759c1 100644 --- a/extension/src/repository/data/index.ts +++ b/extension/src/repository/data/index.ts @@ -11,8 +11,9 @@ import { EXPERIMENTS_GIT_REFS } from '../../experiments/data/constants' import { DeferredDisposable } from '../../class/deferred' -import { getGitPath, gitPath } from '../../cli/git/constants' +import { gitPath } from '../../cli/git/constants' import { DataStatusOutput, DvcError } from '../../cli/dvc/contract' +import { getGitPath } from '../../fileSystem' export type Data = { dataStatus: DataStatusOutput | DvcError diff --git a/extension/src/test/suite/experiments/data/index.test.ts b/extension/src/test/suite/experiments/data/index.test.ts index 2db9936466..2f9f84e7fa 100644 --- a/extension/src/test/suite/experiments/data/index.test.ts +++ b/extension/src/test/suite/experiments/data/index.test.ts @@ -25,7 +25,8 @@ import { import { buildExperimentsData, buildExperimentsDataDependencies } from '../util' import { ExperimentFlag } from '../../../../cli/dvc/constants' import { EXPERIMENTS_GIT_LOGS_REFS } from '../../../../experiments/data/constants' -import { getGitPath, gitPath } from '../../../../cli/git/constants' +import { gitPath } from '../../../../cli/git/constants' +import { getGitPath } from '../../../../fileSystem' suite('Experiments Data Test Suite', () => { const disposable = Disposable.fn() diff --git a/extension/src/test/suite/fileSystem/index.test.ts b/extension/src/test/suite/fileSystem/index.test.ts new file mode 100644 index 0000000000..6191afad75 --- /dev/null +++ b/extension/src/test/suite/fileSystem/index.test.ts @@ -0,0 +1,46 @@ +import { resolve } from 'path' +import { afterEach, beforeEach, describe, it, suite } from 'mocha' +import { expect } from 'chai' +import { Disposable } from '@hediet/std/disposable' +import { restore } from 'sinon' +import { dvcDemoPath } from '../../util' +import { getGitPath } from '../../../fileSystem' +import { gitPath } from '../../../cli/git/constants' +import { GitReader } from '../../../cli/git/reader' +import { standardizePath } from '../../../fileSystem/path' + +suite('File System Watcher Test Suite', () => { + const disposable = Disposable.fn() + + beforeEach(() => { + restore() + }) + + afterEach(() => { + disposable.dispose() + }) + + describe('getGitPath', () => { + it('should get the expected path for the demo repository (submodule)', async () => { + const reader = disposable.track(new GitReader()) + const root = await reader.getGitRepositoryRoot(__dirname) + const submoduleDotGit = standardizePath( + resolve(root, gitPath.DOT_GIT, 'modules', 'demo') + ) + + const dotGitPath = getGitPath(dvcDemoPath, gitPath.DOT_GIT) + + expect(dotGitPath).to.equal(submoduleDotGit) + }) + + it('should get the expected paths for this project', async () => { + const reader = disposable.track(new GitReader()) + const root = await reader.getGitRepositoryRoot(__dirname) + const rootDotGit = standardizePath(resolve(root, gitPath.DOT_GIT)) + + const dotGitPath = getGitPath(root, gitPath.DOT_GIT) + + expect(dotGitPath).to.equal(rootDotGit) + }) + }) +}) diff --git a/extension/src/test/suite/repository/data/index.test.ts b/extension/src/test/suite/repository/data/index.test.ts index b20a34f2a2..302c54282c 100644 --- a/extension/src/test/suite/repository/data/index.test.ts +++ b/extension/src/test/suite/repository/data/index.test.ts @@ -12,7 +12,8 @@ import { CommandId, InternalCommands } from '../../../../commands/internal' -import { getGitPath, gitPath } from '../../../../cli/git/constants' +import { gitPath } from '../../../../cli/git/constants' +import { getGitPath } from '../../../../fileSystem' suite('Repository Data Test Suite', () => { const disposable = Disposable.fn() diff --git a/extension/src/test/util/index.ts b/extension/src/test/util/index.ts index 4cf4cfda33..5354b58a04 100644 --- a/extension/src/test/util/index.ts +++ b/extension/src/test/util/index.ts @@ -2,7 +2,6 @@ import { resolve } from 'path' import { Memento, Uri, workspace, WorkspaceFolder } from 'vscode' const dvcRoot = resolve(__dirname, '..', '..', '..', '..', 'demo') - export const dvcDemoPath = Uri.file(dvcRoot).fsPath export const basePlotsUrl = Uri.file( resolve(__dirname, '..', 'fixtures', 'plotsDiff', 'staticImages') From 4a7d62e7c2dceb76df80387c0aacaa9538954003 Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Wed, 19 Oct 2022 12:11:50 +1100 Subject: [PATCH 09/12] add further tests --- extension/src/cli/git/index.ts | 5 +++-- .../src/test/suite/fileSystem/index.test.ts | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/extension/src/cli/git/index.ts b/extension/src/cli/git/index.ts index d54c33a51d..d8be852428 100644 --- a/extension/src/cli/git/index.ts +++ b/extension/src/cli/git/index.ts @@ -1,11 +1,12 @@ import { Command, Flag } from './constants' import { getOptions } from './options' import { Cli } from '..' +import { standardizePath } from '../../fileSystem/path' export class GitCli extends Cli { - public getGitRepositoryRoot(cwd: string) { + public async getGitRepositoryRoot(cwd: string) { const options = getOptions(cwd, Command.REV_PARSE, Flag.SHOW_TOPLEVEL) - return this.executeProcess(options) + return standardizePath(await this.executeProcess(options)) as string } } diff --git a/extension/src/test/suite/fileSystem/index.test.ts b/extension/src/test/suite/fileSystem/index.test.ts index 6191afad75..68e1b64400 100644 --- a/extension/src/test/suite/fileSystem/index.test.ts +++ b/extension/src/test/suite/fileSystem/index.test.ts @@ -1,4 +1,4 @@ -import { resolve } from 'path' +import { join, resolve } from 'path' import { afterEach, beforeEach, describe, it, suite } from 'mocha' import { expect } from 'chai' import { Disposable } from '@hediet/std/disposable' @@ -8,6 +8,7 @@ import { getGitPath } from '../../../fileSystem' import { gitPath } from '../../../cli/git/constants' import { GitReader } from '../../../cli/git/reader' import { standardizePath } from '../../../fileSystem/path' +import { EXPERIMENTS_GIT_REFS } from '../../../experiments/data/constants' suite('File System Watcher Test Suite', () => { const disposable = Disposable.fn() @@ -21,26 +22,32 @@ suite('File System Watcher Test Suite', () => { }) describe('getGitPath', () => { - it('should get the expected path for the demo repository (submodule)', async () => { + it('should get the expected paths for the demo repository (submodule)', async () => { const reader = disposable.track(new GitReader()) const root = await reader.getGitRepositoryRoot(__dirname) const submoduleDotGit = standardizePath( resolve(root, gitPath.DOT_GIT, 'modules', 'demo') - ) + ) as string const dotGitPath = getGitPath(dvcDemoPath, gitPath.DOT_GIT) - expect(dotGitPath).to.equal(submoduleDotGit) + + const expRefPaths = getGitPath(dvcDemoPath, EXPERIMENTS_GIT_REFS) + expect(expRefPaths).to.equal(join(submoduleDotGit, EXPERIMENTS_GIT_REFS)) }) it('should get the expected paths for this project', async () => { const reader = disposable.track(new GitReader()) const root = await reader.getGitRepositoryRoot(__dirname) - const rootDotGit = standardizePath(resolve(root, gitPath.DOT_GIT)) + const rootDotGit = standardizePath( + resolve(root, gitPath.DOT_GIT) + ) as string const dotGitPath = getGitPath(root, gitPath.DOT_GIT) - expect(dotGitPath).to.equal(rootDotGit) + + const expRefPaths = getGitPath(rootDotGit, EXPERIMENTS_GIT_REFS) + expect(expRefPaths).to.equal(join(rootDotGit, EXPERIMENTS_GIT_REFS)) }) }) }) From 2d9a1c3822041f84a2d2fd4eb1404f473f66cedb Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Wed, 19 Oct 2022 12:32:58 +1100 Subject: [PATCH 10/12] fix tests --- extension/src/cli/git/executor.test.ts | 6 +++--- extension/src/test/suite/fileSystem/index.test.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extension/src/cli/git/executor.test.ts b/extension/src/cli/git/executor.test.ts index 5f5101c6c2..7e1b129e70 100644 --- a/extension/src/cli/git/executor.test.ts +++ b/extension/src/cli/git/executor.test.ts @@ -4,6 +4,7 @@ import { GitExecutor } from './executor' import { createProcess } from '../../processExecution' import { CliResult, CliStarted } from '..' import { getMockedProcess } from '../../test/util/jest' +import { standardizePath } from '../../fileSystem/path' jest.mock('vscode') jest.mock('@hediet/std/disposable') @@ -38,9 +39,10 @@ describe('GitExecutor', () => { } as unknown as EventEmitter }) + const cwd = standardizePath(__dirname) as string + describe('pushBranch', () => { it('should call createProcess with the correct parameters to push a branch', async () => { - const cwd = __dirname const branchName = 'my-branch' mockedCreateProcess.mockReturnValueOnce( getMockedProcess( @@ -57,7 +59,6 @@ describe('GitExecutor', () => { }) it('should call createProcess with the correct parameters to push the current branch', async () => { - const cwd = __dirname mockedCreateProcess.mockReturnValueOnce( getMockedProcess('Everything up-to-date') ) @@ -73,7 +74,6 @@ describe('GitExecutor', () => { describe('stageAndCommit', () => { it('should call createProcess with the correct parameters to stage all files and then commit', async () => { - const cwd = __dirname const message = 'best experiment' mockedCreateProcess.mockReturnValueOnce(getMockedProcess(cwd)) mockedCreateProcess diff --git a/extension/src/test/suite/fileSystem/index.test.ts b/extension/src/test/suite/fileSystem/index.test.ts index 68e1b64400..3bba05f86e 100644 --- a/extension/src/test/suite/fileSystem/index.test.ts +++ b/extension/src/test/suite/fileSystem/index.test.ts @@ -46,7 +46,7 @@ suite('File System Watcher Test Suite', () => { const dotGitPath = getGitPath(root, gitPath.DOT_GIT) expect(dotGitPath).to.equal(rootDotGit) - const expRefPaths = getGitPath(rootDotGit, EXPERIMENTS_GIT_REFS) + const expRefPaths = getGitPath(root, EXPERIMENTS_GIT_REFS) expect(expRefPaths).to.equal(join(rootDotGit, EXPERIMENTS_GIT_REFS)) }) }) From f8b4fbffecbcdc710772a7003a15bb64b132274b Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Wed, 19 Oct 2022 12:49:01 +1100 Subject: [PATCH 11/12] watch correct path for experiments --- extension/src/experiments/data/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/src/experiments/data/index.ts b/extension/src/experiments/data/index.ts index 051d367f87..8ebc383d7c 100644 --- a/extension/src/experiments/data/index.ts +++ b/extension/src/experiments/data/index.ts @@ -82,7 +82,7 @@ export class ExperimentsData extends BaseData { this.dvcRoot ) - const dotGitPath = getGitPath(gitRoot, gitPath.DOT_GIT_HEAD) + const dotGitPath = getGitPath(gitRoot, gitPath.DOT_GIT) const watchedRelPaths = [ gitPath.DOT_GIT_HEAD, EXPERIMENTS_GIT_REFS, From 32c95e14400b3ce70b23cced40e910eac6ddfc62 Mon Sep 17 00:00:00 2001 From: Matt Seddon Date: Wed, 19 Oct 2022 13:16:45 +1100 Subject: [PATCH 12/12] fix string in suite statement --- extension/src/test/suite/fileSystem/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/src/test/suite/fileSystem/index.test.ts b/extension/src/test/suite/fileSystem/index.test.ts index 3bba05f86e..6397ec9821 100644 --- a/extension/src/test/suite/fileSystem/index.test.ts +++ b/extension/src/test/suite/fileSystem/index.test.ts @@ -10,7 +10,7 @@ import { GitReader } from '../../../cli/git/reader' import { standardizePath } from '../../../fileSystem/path' import { EXPERIMENTS_GIT_REFS } from '../../../experiments/data/constants' -suite('File System Watcher Test Suite', () => { +suite('File System Test Suite', () => { const disposable = Disposable.fn() beforeEach(() => {