Add support for starter projects in DevWorkspaces

docs/additional-configuration.adoc

@@ -94,6 +94,30 @@ spec:
       controller.devfile.io/project-clone: disable
 ----
 
+### Configuring sparse checkout for projects
+The project-level attribute `sparseCheckout` can be used to enable a sparse checkout for a given project. The value of this attribute should be a list of paths within the project that should be included in the sparse checkout, separated by spaces. For example, the project
+
+[source,yaml]
+----
+kind: DevWorkspace
+apiVersion: workspace.devfile.io/v1alpha2
+metadata:
+  name: my-workspace
+spec:
+  template:
+    projects:
+      - name: devworkspace-operator
+        attributes:
+          sparseCheckout: "docs"
+        git:
+          remotes:
+            origin: "https://github.com/devfile/devworkspace-operator.git"
+----
+
+will clone the DevWorkspace Operator sparsely, so only the `docs` directory is present.
+
+For more information on sparse checkouts, see documentation for [git sparse-checkout](https://git-scm.com/docs/git-sparse-checkout)
+
 ## Automatically mounting volumes, configmaps, and secrets
 Existing configmaps, secrets, and persistent volume claims on the cluster can be configured by applying the appropriate labels. To mark a resource for mounting to workspaces, apply the **label**
 [source,yaml]

pkg/constants/attributes.go

@@ -141,4 +141,8 @@ const (
 	//       container:
 	//         image: ...
 	ContainerOverridesAttribute = "container-overrides"
+
+	// StarterProjectAttribute is an attribute applied to the top-level attributes in a DevWorkspace to specify which
+	// starterProject in the workspace should be cloned.
+	StarterProjectAttribute = "controller.devfile.io/use-starter-project"
 )

pkg/library/projects/clone.go

@@ -42,7 +42,11 @@ type Options struct {
 }
 
 func GetProjectCloneInitContainer(workspace *dw.DevWorkspaceTemplateSpec, options Options, proxyConfig *controllerv1alpha1.Proxy) (*corev1.Container, error) {
-	if len(workspace.Projects) == 0 {
+	starterProject, err := GetStarterProject(workspace)
+	if err != nil {
+		return nil, err
+	}
+	if len(workspace.Projects) == 0 && starterProject == nil {
 		return nil, nil
 	}
 	if workspace.Attributes.GetString(constants.ProjectCloneAttribute, nil) == constants.ProjectCloneDisable {
@@ -93,6 +97,24 @@ func GetProjectCloneInitContainer(workspace *dw.DevWorkspaceTemplateSpec, option
 	}, nil
 }
 
+func GetStarterProject(workspace *dw.DevWorkspaceTemplateSpec) (*dw.StarterProject, error) {
+	if !workspace.Attributes.Exists(constants.StarterProjectAttribute) {
+		return nil, nil
+	}
+	var err error
+	selectedStarterProject := workspace.Attributes.GetString(constants.StarterProjectAttribute, &err)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s attribute on workspace: %w", constants.StarterProjectAttribute, err)
+	}
+	for _, starterProject := range workspace.StarterProjects {
+		if starterProject.Name == selectedStarterProject {
+			starterProject := starterProject
+			return &starterProject, nil
+		}
+	}
+	return nil, fmt.Errorf("selected starter project %s not found in workspace starterProjects", selectedStarterProject)
+}
+
 func hasContainerComponents(workspace *dw.DevWorkspaceTemplateSpec) bool {
 	for _, component := range workspace.Components {
 		if component.Container != nil {

project-clone/internal/devfile.go

@@ -21,11 +21,17 @@ import (
 	"os"
 
 	dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
+	"github.com/devfile/api/v2/pkg/attributes"
 	"sigs.k8s.io/yaml"
 
 	"github.com/devfile/devworkspace-operator/pkg/provision/metadata"
 )
 
+const (
+	ProjectSparseCheckout = "sparseCheckout"
+	ProjectSubDir         = "subDir"
+)
+
 // GetClonePath gets the correct clonePath for a project, given the semantics in devfile/api
 func GetClonePath(project *dw.Project) string {
 	if project.ClonePath != "" {
@@ -55,3 +61,23 @@ func ReadFlattenedDevWorkspace() (*dw.DevWorkspaceTemplateSpec, error) {
 	log.Printf("Read DevWorkspace at %s", flattenedDevWorkspacePath)
 	return dwts, nil
 }
+
+// StarterProjectToRegularProject converts a starter project defined in a DevWorkspace to a standard Project for
+// easier handling
+func StarterProjectToRegularProject(starterProject *dw.StarterProject) dw.Project {
+	// Note: starter project does not allow for specifying clonePath
+	project := dw.Project{
+		Name:          starterProject.Name,
+		Attributes:    starterProject.Attributes,
+		ProjectSource: starterProject.ProjectSource,
+	}
+
+	if starterProject.SubDir != "" {
+		if project.Attributes == nil {
+			project.Attributes = attributes.Attributes{}
+		}
+		project.Attributes.PutString(ProjectSubDir, starterProject.SubDir)
+	}
+
+	return project
+}

project-clone/internal/git/operations.go

@@ -23,6 +23,7 @@ import (
 	"github.com/go-git/go-git/v5"
 	gitConfig "github.com/go-git/go-git/v5/config"
 
+	"github.com/devfile/devworkspace-operator/project-clone/internal"
 	"github.com/devfile/devworkspace-operator/project-clone/internal/shell"
 )
 
@@ -62,16 +63,41 @@ func CloneProject(project *dw.Project, projectPath string) error {
 		}
 	}
 
-	// Delegate to standard git binary because git.PlainClone takes a lot of memory for large repos
-	err := shell.GitCloneProject(defaultRemoteURL, defaultRemoteName, projectPath)
-	if err != nil {
-		return fmt.Errorf("failed to git clone from %s: %s", defaultRemoteURL, err)
+	if project.Attributes.Exists(internal.ProjectSparseCheckout) {
+		if err := shell.GitSparseCloneProject(defaultRemoteURL, defaultRemoteName, projectPath); err != nil {
+			return fmt.Errorf("failed to sparsely git clone from %s: %s", defaultRemoteURL, err)
+		}
+	} else {
+		// Delegate to standard git binary because git.PlainClone takes a lot of memory for large repos
+		err := shell.GitCloneProject(defaultRemoteURL, defaultRemoteName, projectPath)
+		if err != nil {
+			return fmt.Errorf("failed to git clone from %s: %s", defaultRemoteURL, err)
+		}
+
 	}
 
 	log.Printf("Cloned project %s to %s", project.Name, projectPath)
 	return nil
 }
 
+func SetupSparseCheckout(project *dw.Project, projectPath string) error {
+	log.Printf("Setting up sparse checkout for project %s", project.Name)
+
+	var err error
+	sparseCheckoutDir := project.Attributes.GetString(internal.ProjectSparseCheckout, &err)
+	if err != nil {
+		return fmt.Errorf("failed to read %s attribute on project %s", internal.ProjectSparseCheckout, project.Name)
+	}
+	if sparseCheckoutDir == "" {
+		return nil
+	}
+	if err := shell.GitSetupSparseCheckout(projectPath, sparseCheckoutDir); err != nil {
+		return fmt.Errorf("error running sparse-checkout set: %w", err)
+	}
+
+	return nil
+}
+
 // SetupRemotes sets up a git remote in repo for each remote in project.Git.Remotes
 func SetupRemotes(repo *git.Repository, project *dw.Project, projectPath string) error {
 	log.Printf("Setting up remotes for project %s", project.Name)
@@ -93,7 +119,7 @@ func SetupRemotes(repo *git.Repository, project *dw.Project, projectPath string)
 }
 
 // CheckoutReference sets the current HEAD in repo to point at the revision and remote referenced by checkoutFrom
-func CheckoutReference(repo *git.Repository, project *dw.Project, projectPath string) error {
+func CheckoutReference(project *dw.Project, projectPath string) error {
 	checkoutFrom := project.Git.CheckoutFrom
 	if checkoutFrom == nil || checkoutFrom.Revision == "" {
 		return nil

project-clone/internal/git/setup.go

@@ -49,24 +49,32 @@ func doInitialGitClone(project *dw.Project) error {
 	if err != nil {
 		return fmt.Errorf("failed to clone project: %s", err)
 	}
+
+	if project.Attributes.Exists(internal.ProjectSparseCheckout) {
+		if err := SetupSparseCheckout(project, tmpClonePath); err != nil {
+			return fmt.Errorf("failed to set up sparse checkout on project %s: %w", project.Name, err)
+		}
+	}
+
 	repo, err := internal.OpenRepo(tmpClonePath)
 	if err != nil {
 		return fmt.Errorf("failed to open existing project in filesystem: %s", err)
 	} else if repo == nil {
 		return fmt.Errorf("unexpected error while setting up remotes for project: git repository not present")
 	}
+
 	if err := SetupRemotes(repo, project, tmpClonePath); err != nil {
 		return fmt.Errorf("failed to set up remotes for project: %s", err)
 	}
-	if err := CheckoutReference(repo, project, tmpClonePath); err != nil {
+
+	if err := CheckoutReference(project, tmpClonePath); err != nil {
 		return fmt.Errorf("failed to checkout revision: %s", err)
 	}
 
-	projectPath := path.Join(internal.ProjectsRoot, internal.GetClonePath(project))
-	log.Printf("Moving cloned project %s from temporary dir %s to %s", project.Name, tmpClonePath, projectPath)
-	if err := os.Rename(tmpClonePath, projectPath); err != nil {
-		return fmt.Errorf("failed to move cloned project to PROJECTS_ROOT: %w", err)
+	if err := copyProjectFromTmpDir(project, tmpClonePath); err != nil {
+		return err
 	}
+
 	return nil
 }
 
@@ -83,3 +91,27 @@ func setupRemotesForExistingProject(project *dw.Project) error {
 	}
 	return nil
 }
+
+func copyProjectFromTmpDir(project *dw.Project, tmpClonePath string) error {
+	if project.Attributes.Exists(internal.ProjectSubDir) {
+		// Only want one directory from the project
+		var err error
+		subDirSubPath := project.Attributes.GetString(internal.ProjectSubDir, &err)
+		if err != nil {
+			return fmt.Errorf("failed to process subDir on project: %w", err)
+		}
+		subDirPath := path.Join(tmpClonePath, subDirSubPath)
+		projectPath := path.Join(internal.ProjectsRoot, internal.GetClonePath(project))
+		log.Printf("Moving subdirectory %s in project %s from temporary directory to %s", subDirSubPath, project.Name, projectPath)
+		if err := os.Rename(subDirPath, projectPath); err != nil {
+			return fmt.Errorf("failed to move subdirectory of cloned project to %s: %w", internal.ProjectsRoot, err)
+		}
+	} else {
+		projectPath := path.Join(internal.ProjectsRoot, internal.GetClonePath(project))
+		log.Printf("Moving cloned project %s from temporary directory %s to %s", project.Name, tmpClonePath, projectPath)
+		if err := os.Rename(tmpClonePath, projectPath); err != nil {
+			return fmt.Errorf("failed to move cloned project to %s: %w", internal.ProjectsRoot, err)
+		}
+	}
+	return nil
+}