diff --git a/content/docs/sidebar.json b/content/docs/sidebar.json index d09a17fe76..254cc8c4b3 100644 --- a/content/docs/sidebar.json +++ b/content/docs/sidebar.json @@ -514,6 +514,22 @@ } ] }, + { + "label": "Studio", + "slug": "studio", + "source": "studio/index.md", + "icon": "studio", + "children": [ + "get-started", + { + "label": "Create a View", + "slug": "create-view" + }, + "visualize-experiments", + "run-experiments", + "teams" + ] + }, { "label": "Dvclive", "slug": "dvclive", diff --git a/content/docs/studio/create-view.md b/content/docs/studio/create-view.md new file mode 100644 index 0000000000..93de45d2cc --- /dev/null +++ b/content/docs/studio/create-view.md @@ -0,0 +1,67 @@ +# Create a View + +To create a new view for your repository, follow these steps. + +1. Sign into your [DVC Studio](https://studio.iterative.ai/) dashboard. + ![](https://static.iterative.ai/img/studio/login_home.png) + +2. Click on `Add a View`. All the organizations that you have access to will be + listed. ![](https://static.iterative.ai/img/studio/create_view.png) + +3. Open the organization whose repository you want to connect to. You can also + use the search bar to search directly for the repo which you want to connect + to. ![](https://static.iterative.ai/img/studio/select_repo.png) + +4. Select the Git repository for which you want to create a view. For this, + hover over the required repository and click `Connect`. + +5. Once you click on `Connect`, you will be able to specify advanced settings + for the connection. + ![](https://static.iterative.ai/img/studio/view_settings.png) + +> In most scenarios, you can skip the advanced settings. Refer to the +> [Advanced Settings](#advanced-settings) section below for more details. + +You should now see that a view has been added in your dashboard. + +![](https://static.iterative.ai/img/studio/view_added.png) + +If your project requires any of the advanced settings, then remember to +configure them by opening the View settings. Otherwise, your view may not work +as expected. To go to View settings, click on the +![](https://static.iterative.ai/img/studio/view_open_settings_icon.png) icon in +the view. In the menu that opens up (highlighted in yellow in the screenshot +below), click on `Settings`. + +![](https://static.iterative.ai/img/studio/view_open_settings.png) + +## Advanced Settings + +You will need to specify advanced settings in the following scenarios. + +> These settings can be skipped when creating the view -- they can be edited +> later. + +- **Monorepo:** If you have connected to a + [monorepo](https://en.wikipedia.org/wiki/Monorepo), then specify the full path + to the root directory of the project that you want to connect. + +- **Data remotes:** The metrics and parameters that you want to include in the + view may be present in a data remote (cloud storage or another + location outside of the Git repo). If you want to include such data in your + views, then you will have to grant DVC Studio access to the data remote. + +- **Custom metrics and parameters:** DVC Studio automatically detects metrics, + plots, and parameters files specified in the project's `dvc.yaml`. If you want + to connect custom files, you can add them by clicking the `Add file` button. + Enter the full file path, and specify whether the file is for `Metrics` or + `Parameters`. + +## Share a view + +You can share your views on the web. Simply click on the button labelled +`Private` next to the name of the view. In the menu that pops up, turn on +`Share to Web`. ![](https://static.iterative.ai/img/studio/view_share.png) + +You can change a shared (public) view and make it private by turning off +`Share to web`. diff --git a/content/docs/studio/get-started.md b/content/docs/studio/get-started.md new file mode 100644 index 0000000000..3057f3ee6f --- /dev/null +++ b/content/docs/studio/get-started.md @@ -0,0 +1,64 @@ +# Get Started with DVC Studio + +Here, we will walk you through a tutorial to use DVC Studio for collaboration on +your ML projects. You will need access to a Github, Gitlab or Bitbucket account +which has access to the Git repositories you want to connect. DVC Studio creates +views from repositories when you connect to them. + +## What is a view? + +A _view_ is an interactive representation of the experiments run in your ML +project. DVC Studio automatically identifies datasets, metrics and +hyperparameters in your ML experiments. You can also add custom files with the +required data (more on this later). Using this data, DVC Studio creates a +summary view. This includes tables of all the metrics across all your +experiments. You can also generate plots and compare experiments here. + +## DVC Studio Views page + +In your browser, open . Sign in with your Github, +Gitlab, or Bitbucket account. + +![](https://static.iterative.ai/img/studio/login_home.png) _When you first +login, an example view is already created for you to explore, and you can add +more views._ + +When you first login, you will find that there already exists a view connecting +to an example DVC project. Use this view to explore the features that DVC Studio +has to offer. + +DVC Studio automatically identifies datasets, metrics and hyperparameters in +your ML experiments. Each view on the dashboard displays the metrics. In the +figure above, you can see that `avg_prec` and `roc_auc` metrics are displayed. + +## Components of a view + +You can dive deep into all the experiments committed to the repo. For this, open +the view by clicking the view name (in this case, `example-get-started`). + +A table will be generated as shown below. This includes metrics, hyperparameters +and information about the datasets. All the data is flattened and neatly +presented for you to evaluate and compare the experiments. + +![](https://static.iterative.ai/img/studio/view_components.png) + +This tabular display has the following components: + +- The branches in your Git repository. +- All commits in each branch. Each commit, corresponding to a single row in the + table, represents an experiment. +- Values of all the metrics, files and parameters in the given commits; + corresponding to the table columns. +- Various buttons for performing actions: + - **Filters:** Filter commits + - **Columns:** Select columns to display + - **Show plots:** Show plots for the selected commits + - **Compare:** Compare different experiments + - **Run:** Run experiments by selecting any one commit (Refer + [here](/doc/studio/run-experiments) for how to run experiments) + - **Trends:** Generate trend charts to show metric evolution over time + - **Delta mode:** Toggle between absolute values and difference from the first + row + +You can connect to additional repositories and add more views as needed. You'll +find out how to do this in the next section. diff --git a/content/docs/studio/index.md b/content/docs/studio/index.md new file mode 100644 index 0000000000..b3e63b51c4 --- /dev/null +++ b/content/docs/studio/index.md @@ -0,0 +1,25 @@ +# DVC Studio + +[`DVC Studio`](https://studio.iterative.ai/) is a comprehensive and interactive +collaboration tool for your Machine Learning projects. + +Using the power of leading open-source tools DVC, CML and Git, it enables you to +seamlessly manage data and models, run and track experiments, and visualize and +share results. + +Project website: https://studio.iterative.ai + +![](https://static.iterative.ai/img/studio/main.png) _DVC Studio experiments +dashboard_ + +Use DVC, CML and Studio to: + +- **Visualize, collaborate, and do everything that a regular ML tracking tool + does.** + +- **Keep your code, data and model connected at all times.** + +- **Use the power of Git to track and preserve all your experiments.** + +- **Run [CI/CD](https://en.wikipedia.org/wiki/CI/CD) for your ML projects on + cloud resources of your choice without any new tools.** diff --git a/content/docs/studio/run-experiments.md b/content/docs/studio/run-experiments.md new file mode 100644 index 0000000000..ee60c25a2e --- /dev/null +++ b/content/docs/studio/run-experiments.md @@ -0,0 +1,28 @@ +# Run Experiments + +You can also run your ML experiments directly from DVC Studio. Your regular +CI/CD set-up is used to run the experiments. For instance, +[CML](https://dvc.org/doc/cml) Github Actions can be used to run ML experiments +on each new commit. + +> Note that you cannot run experiments on the demo view (`example-get-started`) +> that is provided to you by default. Once you create views for your ML project +> repositories, you can follow the instructions given below and run experiments +> directly from DVC Studio. + +To run experiments from DVC Studio, select the commit that you want to use and +click the `Run` button. A form will let you to specify all the changes that you +want to make to your experiment input files/dirs and parameters. + +drawing + +The default values in this form are extracted from your selected commit. For +instance, in the given example, `max_features` can be increased to 4000, +`ngrams` can be changed to 2, and so on. Enter your commit message and +description, select the branch to commit to (either the base branch or a new +branch), and click on `Commit changes`. + +Now, the CI/CD pipeline that you have set up (eg, Github Actions) will be +invoked to run the experiment. If you refresh the view (reload the page), the +experiment (i.e. the commit that you just pushed) along with its results +(metrics) will be included in the list of all commits. diff --git a/content/docs/studio/teams.md b/content/docs/studio/teams.md new file mode 100644 index 0000000000..4b18b28af1 --- /dev/null +++ b/content/docs/studio/teams.md @@ -0,0 +1,72 @@ +# Teams + +So far, you have seen how you can create personal and public views. In this +section, we will illustrate how you can create teams. In DVC Studio, you can +define teams with one or more team members. The team members are also called +collaborators, and you can assign different roles to them. The views that you +create in your team's page will be accessible to all members of the team. + +## Roles + +Team members can have different roles. + +- **View.** Users with the View access cannot edit team settings or create new + views within the team's page. They have read-only access to the views created + by other team members. +- **Edit.** In addition to accessing all the team's views, users with the Edit + role can also create new views for the team, edit the views' settings, and run + experiments. +- **Admin.** Admin users have full access to the team's views and settings. They + can do everything that viewers and editors can do. Additionally, they can add + (invite) and remove collaborators as well as change team settings such as + cloud credentials (data remotes). + +## Create a team + +To create a team, click on the drop down next to `Personal`. All the teams that +you have created so far will be listed within "Teams" in the drop down menu. If +you have not created any team so far, this list will be empty. Now, click on +`Create a team`. ![](https://static.iterative.ai/img/studio/team_create.png) + +You will be asked to enter the URL namespace for your team. Enter a unique name. +The URL for your team will be formed using this name. +![](https://static.iterative.ai/img/studio/team_enter_name.png) + +Then, click the `Create team` button on the top right corner. + +## Invite collaborators + +The next step is to invite the team members (or collaborators) for your team. +You can also choose to skip adding collaborators at this point. For this, click +on `Skip and Close` on the top right corner. You will be able to add +collaborators by accessing team settings later. + +If you wish to add collaborators now, enter their email addresses. An email +invite will be sent to each invitee, and they will have to join using their +Github, Gitlab or Bitbucket account. + +You can add multiple collaborators. Each collaborator can be assigned the Admin, +Edit, or View role. ![](https://static.iterative.ai/img/studio/team_roles.png) + +Once you have added the people that you wish to add to your team, click on +`Send Invites and Close` on the top right corner. + +## Manage your team and its views + +Once you have created the team, the team's page opens up. + +![](https://static.iterative.ai/img/studio/team_page.png) + +On this page, you can perform three types of tasks: + +- **Add a view.** Click on the `View` menu item to add views to the team's page. + The process for adding a view is the same as that for adding personal views + ([instructions](/doc/studio/create-view)). However, the views that you create + within the team will be accessible to all members (collaborators) of the team. + +- **Edit collaborators.** You can click on the `Team` menu item to edit the + collaborators in the team. + +- **Change settings.** Finally, you can click on the `Settings` menu item to + change the team name, add credentials for the data remotes, and delete the + team. ![](https://static.iterative.ai/img/studio/team_settings.png) diff --git a/content/docs/studio/visualize-experiments.md b/content/docs/studio/visualize-experiments.md new file mode 100644 index 0000000000..b3f2dc87bd --- /dev/null +++ b/content/docs/studio/visualize-experiments.md @@ -0,0 +1,40 @@ +# Visualize and Compare Experiments + +You can visualize and compare experiments using a variety of tools. + +## Generate metric plots + +You can visualize certain metrics of machine learning experiments as plots. +Usual plot examples are AUC curves, loss functions, and confusion matrices, +among others. This type of metrics files are created by users, or generated by +user data processing code, and can be defined in dvc.yaml (plots field) for +tracking (optional). Refer to the +[DVC plots documentation](/doc/command-reference/plots) for details on how plots +are generated. + +To generate metric plots, select one or more experiments (represented by the +commits), and click on the 'Show plots' button. + +drawing + +The plots will appear below the tabular display. If you have selected more than +one experiment, you can use the plots to compare them. + +drawing + +## Generate trend charts + +Click on the Trends button to generate a plot of how the metrics changed over +the course of the different experiments. For each metric, the trend charts show +how the metric changed from one commit to another. You can include one or more +branches in the trend chart. + +drawing + +## Compare experiments + +To compare different experiments, select two experiments (represented by the +commits), and click on the `Compare` button. The metrics, parameters and files +in the selected experiments will be displayed side by side for easy comparison. + +drawing diff --git a/src/components/Documentation/Layout/SidebarMenu/index.tsx b/src/components/Documentation/Layout/SidebarMenu/index.tsx index a70e5b074b..d48eac9390 100644 --- a/src/components/Documentation/Layout/SidebarMenu/index.tsx +++ b/src/components/Documentation/Layout/SidebarMenu/index.tsx @@ -11,6 +11,7 @@ import Link from '../../../Link' import { ReactComponent as ExternalLinkIcon } from './external-link-icon.svg' import { ReactComponent as HouseIcon } from './house.svg' import { ReactComponent as CMLIcon } from './cml_bw_logo.svg' +import { ReactComponent as StudioIcon } from './studio_gray_icon.svg' import { structure, @@ -25,7 +26,8 @@ import styles from './styles.module.css' // Use the key string here as the "icon" field in sidebar.json const ICONS: { [key: string]: React.FC<{ className?: string }> } = { house: HouseIcon, - cml: CMLIcon + cml: CMLIcon, + studio: StudioIcon } interface ISidebarMenuItemProps { diff --git a/src/components/Documentation/Layout/SidebarMenu/studio_gray_icon.svg b/src/components/Documentation/Layout/SidebarMenu/studio_gray_icon.svg new file mode 100644 index 0000000000..3cad3360e6 --- /dev/null +++ b/src/components/Documentation/Layout/SidebarMenu/studio_gray_icon.svg @@ -0,0 +1,457 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/components/Documentation/Layout/SidebarMenu/studio_icon.svg b/src/components/Documentation/Layout/SidebarMenu/studio_icon.svg new file mode 100644 index 0000000000..af633ed015 --- /dev/null +++ b/src/components/Documentation/Layout/SidebarMenu/studio_icon.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + +