Add suport for gists #1740

docs/modules/ROOT/pages/running/run-from-github.adoc

@@ -1,25 +1,64 @@
 [[run-from-github]]
 = Run from GitHub
 
-It is possible to run integrations from GitHub with a dedicated URL
-syntax:
+It is possible to run integrations from a GitHub repository or Gist with dedicated URL syntax:
 
-```
+== Repository
+
+.Syntax
+[source]
+----
 kamel run github:$user/$repo/$path?branch=$branch
-```
+----
 
 As example, running the following command
 
-```
+
+[source]
+----
 kamel run github:apache/camel-k/examples/Sample.java
-```
+----
 
 is equivalent to:
 
-```
+[source]
+----
 kamel run https://raw.githubusercontent.com/apache/camel-k/master/examples/Sample.java
-```
+----
 
 but does not require to type the full GitHub RAW URL.
 
-Declaring the branch query param is not required and defaults to `master` if not explicit set.
+Declaring the branch query param is not required and defaults to `master` if not explicit set.
+
+== Gist
+
+.Syntax
+[source]
+----
+kamel run https://gist.github.com/${user-id}/${gist-id}
+kamel run gist:${gist-id}
+----
+
+camel-k will add any file that is part of the Gist as a source.
+
+As example, assuming there are two files listed as part of a Gist, beans.yaml and routes.yaml, then the following command
+
+
+[source]
+----
+kamel run gist:${gist-id}
+----
+
+is equivalent to:
+
+[source]
+----
+kamel run \
+    https://gist.githubusercontent.com/${user-id}/${gist-id}/raw/${...}/beans.yaml \
+    https://gist.githubusercontent.com/${user-id}/${gist-id}/raw/${...}/routes.yaml
+----
+
+[NOTE]
+====
+GitHub applies rate limiting to its APIs and as Authenticated requests get a higher rate limit, the camel-k cli honour the env var GITHUB_TOKEN and if it is found, then it is used for GitHub authentication.
+====

e2e/common/run_test.go

@@ -44,24 +44,6 @@ func TestRunSimpleExamples(t *testing.T) {
 			Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
 		})
 
-		t.Run("run java from GitHub", func(t *testing.T) {
-			RegisterTestingT(t)
-			Expect(Kamel("run", "-n", ns, "github:apache/camel-k/e2e/common/files/Java.java").Execute()).Should(BeNil())
-			Eventually(IntegrationPodPhase(ns, "java"), TestTimeoutMedium).Should(Equal(v1.PodRunning))
-			Eventually(IntegrationCondition(ns, "java", camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
-			Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
-			Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
-		})
-
-		t.Run("run java from GitHub (RAW)", func(t *testing.T) {
-			RegisterTestingT(t)
-			Expect(Kamel("run", "-n", ns, "https://raw.githubusercontent.com/apache/camel-k/master/e2e/common/files/Java.java").Execute()).Should(BeNil())
-			Eventually(IntegrationPodPhase(ns, "java"), TestTimeoutMedium).Should(Equal(v1.PodRunning))
-			Eventually(IntegrationCondition(ns, "java", camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
-			Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
-			Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
-		})
-
 		t.Run("run java with properties", func(t *testing.T) {
 			RegisterTestingT(t)
 			Expect(Kamel("run", "-n", ns, "files/Prop.java", "--property-file", "files/prop.properties").Execute()).Should(BeNil())
@@ -138,3 +120,49 @@ func TestRunSimpleExamples(t *testing.T) {
 
 	})
 }
+
+func TestRunExamplesFromGitHUB(t *testing.T) {
+	WithNewTestNamespace(t, func(ns string) {
+		Expect(Kamel("install", "-n", ns).Execute()).Should(BeNil())
+
+		t.Run("run java from GitHub", func(t *testing.T) {
+			RegisterTestingT(t)
+			Expect(Kamel("run", "-n", ns, "github:apache/camel-k/e2e/common/files/Java.java").Execute()).Should(BeNil())
+			Eventually(IntegrationPodPhase(ns, "java"), TestTimeoutMedium).Should(Equal(v1.PodRunning))
+			Eventually(IntegrationCondition(ns, "java", camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
+			Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+			Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
+		})
+
+		t.Run("run java from GitHub (RAW)", func(t *testing.T) {
+			RegisterTestingT(t)
+			Expect(Kamel("run", "-n", ns, "https://raw.githubusercontent.com/apache/camel-k/master/e2e/common/files/Java.java").Execute()).Should(BeNil())
+			Eventually(IntegrationPodPhase(ns, "java"), TestTimeoutMedium).Should(Equal(v1.PodRunning))
+			Eventually(IntegrationCondition(ns, "java", camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
+			Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+			Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
+		})
+
+		t.Run("run from GitHub Gist (ID)", func(t *testing.T) {
+			name := "github-gist-id"
+			RegisterTestingT(t)
+			Expect(Kamel("run", "-n", ns, "--name", name, "gist:e2c3f9a5fd0d9e79b21b04809786f17a").Execute()).Should(BeNil())
+			Eventually(IntegrationPodPhase(ns, name), TestTimeoutMedium).Should(Equal(v1.PodRunning))
+			Eventually(IntegrationCondition(ns, name, camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
+			Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+			Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Tick!"))
+			Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
+		})
+
+		t.Run("run from GitHub Gist (URL)", func(t *testing.T) {
+			name := "github-gist-url"
+			RegisterTestingT(t)
+			Expect(Kamel("run", "-n", ns, "--name", name, "https://gist.github.com/lburgazzoli/e2c3f9a5fd0d9e79b21b04809786f17a").Execute()).Should(BeNil())
+			Eventually(IntegrationPodPhase(ns, name), TestTimeoutMedium).Should(Equal(v1.PodRunning))
+			Eventually(IntegrationCondition(ns, name, camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
+			Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+			Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Tick!"))
+			Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
+		})
+	})
+}

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/fatih/structs v1.1.0
 	github.com/gertd/go-pluralize v0.1.1
 	github.com/go-logr/logr v0.1.0
+	github.com/google/go-github/v32 v32.1.0
 	github.com/google/uuid v1.1.1
 	github.com/jpillora/backoff v1.0.0
 	github.com/magiconair/properties v1.8.1
@@ -35,6 +36,7 @@ require (
 	github.com/stoewer/go-strcase v1.0.2
 	github.com/stretchr/testify v1.5.1
 	go.uber.org/multierr v1.5.0
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	gopkg.in/inf.v0 v0.9.1
 	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.18.9

go.sum

@@ -585,11 +585,15 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-github/v27 v27.0.6/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
 github.com/google/go-github/v29 v29.0.3/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
+github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
+github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk=
 github.com/google/go-licenses v0.0.0-20200227160636-0fa8c766a591/go.mod h1:JWeTIGPLQ9gF618ZOdlUitd1gRR/l99WOkHOlmR/UVA=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
 github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no=

pkg/cmd/modeline.go

@@ -47,18 +47,17 @@ var (
 
 	// file options must be considered relative to the source files they belong to
 	fileOptions = map[string]bool{
-		"source":        true,
 		"resource":      true,
 		"config":        true,
 		"open-api":      true,
 		"property-file": true,
 	}
 )
 
+// NewKamelWithModelineCommand ---
 func NewKamelWithModelineCommand(ctx context.Context, osArgs []string) (*cobra.Command, []string, error) {
-	processed := make(map[string]bool)
 	originalFlags := osArgs[1:]
-	rootCmd, flags, err := createKamelWithModelineCommand(ctx, append([]string(nil), originalFlags...), processed)
+	rootCmd, flags, err := createKamelWithModelineCommand(ctx, originalFlags)
 	if err != nil {
 		fmt.Printf("Error: %s\n", err.Error())
 		return rootCmd, flags, err
@@ -75,7 +74,7 @@ func NewKamelWithModelineCommand(ctx context.Context, osArgs []string) (*cobra.C
 	return rootCmd, flags, nil
 }
 
-func createKamelWithModelineCommand(ctx context.Context, args []string, processedFiles map[string]bool) (*cobra.Command, []string, error) {
+func createKamelWithModelineCommand(ctx context.Context, args []string) (*cobra.Command, []string, error) {
 	rootCmd, err := NewKamelCommand(ctx)
 	if err != nil {
 		return nil, nil, err
@@ -99,44 +98,20 @@ func createKamelWithModelineCommand(ctx context.Context, args []string, processe
 
 	fg := target.Flags()
 
-	sources, err := fg.GetStringArray(runCmdSourcesArgs)
+	additionalSources, err := fg.GetStringArray(runCmdSourcesArgs)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	var files = append([]string(nil), fg.Args()...)
-	files = append(files, sources...)
+	files := make([]string, 0, len(fg.Args())+len(additionalSources))
+	files = append(files, fg.Args()...)
+	files = append(files, additionalSources...)
 
-	opts := make([]modeline.Option, 0)
-	for _, f := range files {
-		if processedFiles[f] {
-			continue
-		}
-		baseDir := filepath.Dir(f)
-		content, _, err := loadData(f, false, false)
-		if err != nil {
-			return nil, nil, errors.Wrapf(err, "cannot read file %s", f)
-		}
-		ops, err := modeline.Parse(f, content)
-		if err != nil {
-			return nil, nil, errors.Wrapf(err, "cannot process file %s", f)
-		}
-		for i, o := range ops {
-			if disallowedOptions[o.Name] {
-				return nil, nil, fmt.Errorf("option %q is disallowed in modeline", o.Name)
-			}
-
-			if fileOptions[o.Name] && isLocal(f) {
-				refPath := o.Value
-				if !filepath.IsAbs(refPath) {
-					full := path.Join(baseDir, refPath)
-					o.Value = full
-					ops[i] = o
-				}
-			}
-		}
-		opts = append(opts, ops...)
+	opts, err := extractModelineOptions(ctx, files)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "cannot read sources")
 	}
+
 	// filter out in place non-run options
 	nOpts := 0
 	for _, o := range opts {
@@ -145,23 +120,9 @@ func createKamelWithModelineCommand(ctx context.Context, args []string, processe
 			nOpts++
 		}
 	}
-	opts = opts[:nOpts]
 
-	// No new options, returning a new command with computed args
-	if len(opts) == 0 {
-		// Recreating the command as it's dirty
-		rootCmd, err = NewKamelCommand(ctx)
-		if err != nil {
-			return nil, nil, err
-		}
-		rootCmd.SetArgs(args)
-		return rootCmd, args, nil
-	}
+	opts = opts[:nOpts]
 
-	// New options added, recomputing
-	for _, f := range files {
-		processedFiles[f] = true
-	}
 	for _, o := range opts {
 		prefix := "-"
 		if len(o.Name) > 1 {
@@ -175,5 +136,46 @@ func createKamelWithModelineCommand(ctx context.Context, args []string, processe
 		}
 	}
 
-	return createKamelWithModelineCommand(ctx, args, processedFiles)
+	// Recreating the command as it's dirty
+	rootCmd, err = NewKamelCommand(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+	rootCmd.SetArgs(args)
+
+	return rootCmd, args, nil
+}
+
+func extractModelineOptions(ctx context.Context, sources []string) ([]modeline.Option, error) {
+	opts := make([]modeline.Option, 0)
+
+	resolvedSources, err := ResolveSources(ctx, sources, false)
+	if err != nil {
+		return opts, errors.Wrap(err, "cannot read sources")
+	}
+
+	for _, resolvedSource := range resolvedSources {
+		ops, err := modeline.Parse(resolvedSource.Location, resolvedSource.Content)
+		if err != nil {
+			return opts, errors.Wrapf(err, "cannot process file %s", resolvedSource.Location)
+		}
+		for i, o := range ops {
+			if disallowedOptions[o.Name] {
+				return opts, fmt.Errorf("option %q is disallowed in modeline", o.Name)
+			}
+
+			if fileOptions[o.Name] && resolvedSource.Local {
+				baseDir := filepath.Dir(resolvedSource.Origin)
+				refPath := o.Value
+				if !filepath.IsAbs(refPath) {
+					full := path.Join(baseDir, refPath)
+					o.Value = full
+					ops[i] = o
+				}
+			}
+		}
+		opts = append(opts, ops...)
+	}
+
+	return opts, nil
 }

pkg/cmd/modeline_test.go

@@ -82,23 +82,23 @@ func TestModelineRunMultipleFiles(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	file := `
-		// camel-k: source=ext.groovy
+		// camel-k: dependency=mvn:org.my/lib1:3.0
 	`
 	fileName := path.Join(dir, "simple.groovy")
 	err = ioutil.WriteFile(fileName, []byte(file), 0777)
 	assert.NoError(t, err)
 
 	file2 := `
-		// camel-k: dependency=mvn:org.my/lib:3.0
+		// camel-k: dependency=mvn:org.my/lib2:3.0
 	`
 	fileName2 := path.Join(dir, "ext.groovy")
 	err = ioutil.WriteFile(fileName2, []byte(file2), 0777)
 	assert.NoError(t, err)
 
-	cmd, flags, err := NewKamelWithModelineCommand(context.TODO(), []string{"kamel", "run", fileName})
+	cmd, flags, err := NewKamelWithModelineCommand(context.TODO(), []string{"kamel", "run", fileName, fileName2})
 	assert.NoError(t, err)
 	assert.NotNil(t, cmd)
-	assert.Equal(t, []string{"run", fileName, "--source=" + fileName2, "--dependency=mvn:org.my/lib:3.0"}, flags)
+	assert.Equal(t, []string{"run", fileName, fileName2, "--dependency=mvn:org.my/lib1:3.0", "--dependency=mvn:org.my/lib2:3.0"}, flags)
 }
 
 func TestModelineRunPropertyFiles(t *testing.T) {