Python: Support OpenTelemetry (#7015)

* feat(sdk/python): support OpenTelemetry Signed-off-by: Helder Correia <[email protected]> * Add change log Signed-off-by: Helder Correia <[email protected]> * Simplify managing dev dependencies Signed-off-by: Helder Correia <[email protected]> --------- Signed-off-by: Helder Correia <[email protected]>
dagger · Apr 11, 2024 · d3a11cf · d3a11cf
1 parent 4d3e6ff
commit d3a11cf
Show file tree

Hide file tree

Showing 14 changed files with 363 additions and 615 deletions.
diff --git a/ci/sdk_python.go b/ci/sdk_python.go
@@ -14,6 +14,7 @@ import (
 // TODO: use dev module (this is just the mage port)
 
 const (
+	pythonSubdir           = "sdk/python"
 	pythonGeneratedAPIPath = "sdk/python/src/dagger/client/gen.py"
 	pythonDefaultVersion   = "3.11"
 )
@@ -101,7 +102,7 @@ func (t PythonSDK) Test(ctx context.Context) error {
 }
 
 // Regenerate the Python SDK API
-func (t PythonSDK) Generate(ctx context.Context) (*Directory, error) {
+func (t PythonSDK) Generate(ctx context.Context) (*dagger.Directory, error) {
 	installer, err := t.Dagger.installer(ctx, "sdk-python-generate")
 	if err != nil {
 		return nil, err
@@ -111,13 +112,13 @@ func (t PythonSDK) Generate(ctx context.Context) (*Directory, error) {
 		return nil, err
 	}
 	generated := t.pythonBase(pythonDefaultVersion, true).
-		WithWorkdir("/sdk/python/codegen").
+		// codegen lock file has a relative `-e .` path
+		WithWorkdir("./codegen").
 		WithExec([]string{"pip", "install", "-r", "requirements.lock"}).
-		WithWorkdir("/").
 		WithMountedFile("/schema.json", introspection).
-		WithExec([]string{"python", "-m", "codegen", "generate", "-i", "/schema.json", "-o", pythonGeneratedAPIPath}).
-		WithExec([]string{"black", pythonGeneratedAPIPath}).
-		File(pythonGeneratedAPIPath)
+		WithExec([]string{"python", "-m", "codegen", "generate", "-i", "/schema.json", "-o", "gen.py"}).
+		WithExec([]string{"black", "gen.py"}).
+		File("gen.py")
 	return dag.Directory().WithFile(pythonGeneratedAPIPath, generated), nil
 }
 
@@ -154,7 +155,7 @@ func (t PythonSDK) Publish(
 }
 
 // Bump the Python SDK's Engine dependency
-func (t PythonSDK) Bump(ctx context.Context, version string) (*Directory, error) {
+func (t PythonSDK) Bump(ctx context.Context, version string) (*dagger.Directory, error) {
 	// trim leading v from version
 	version = strings.TrimPrefix(version, "v")
 	engineReference := fmt.Sprintf("# Code generated by dagger. DO NOT EDIT.\n\nCLI_VERSION = %q\n", version)
@@ -164,12 +165,15 @@ func (t PythonSDK) Bump(ctx context.Context, version string) (*Directory, error)
 	return dag.Directory().WithNewFile("sdk/python/src/dagger/_engine/_version.py", engineReference), nil
 }
 
-// pythonBaseEnv returns a general python environment, without source files.
-func (t PythonSDK) pythonBaseEnv(version string) *Container {
+// pythonBase returns a python container with the Python SDK source files
+// added and dependencies installed.
+func (t PythonSDK) pythonBase(version string, install bool) *Container {
+	src := t.Dagger.Source.Directory(pythonSubdir)
+
 	pipx := dag.HTTP("https://github.com/pypa/pipx/releases/download/1.2.0/pipx.pyz")
 	venv := "/opt/venv"
 
-	return dag.Container().
+	base := dag.Container().
 		From(fmt.Sprintf("python:%s-slim", version)).
 		WithEnvVariable("PIPX_BIN_DIR", "/usr/local/bin").
 		WithMountedCache("/root/.cache/pip", dag.CacheVolume("pip_cache_"+version)).
@@ -182,38 +186,20 @@ func (t PythonSDK) pythonBaseEnv(version string) *Container {
 		WithEnvVariable(
 			"PATH",
 			"$VIRTUAL_ENV/bin:$PATH",
-			ContainerWithEnvVariableOpts{
+			dagger.ContainerWithEnvVariableOpts{
 				Expand: true,
 			},
 		).
-		WithEnvVariable("HATCH_ENV_TYPE_VIRTUAL_PATH", venv)
-}
-
-// pythonBase returns a python container with the Python SDK source files
-// added and dependencies installed.
-func (t PythonSDK) pythonBase(version string, install bool) *Container {
-	var (
-		appDir = "sdk/python"
-	)
-
-	src := t.Dagger.Source.Directory(appDir)
-
-	// Mirror the same dir structure from the repo because of the
-	// relative paths in ruff (for docs linting).
-	mountPath := fmt.Sprintf("/%s", appDir)
-
-	reqPath := fmt.Sprintf("%s/requirements", appDir)
-	reqFile := fmt.Sprintf("%s.txt", reqPath)
-
-	base := t.pythonBaseEnv(version).
-		WithDirectory(reqPath, src.Directory("requirements")).
-		WithFile(reqFile, src.File("requirements.txt")).
-		WithExec([]string{"pip", "install", "-r", reqFile}).
-		WithWorkdir(mountPath)
+		WithEnvVariable("HATCH_ENV_TYPE_VIRTUAL_PATH", venv).
+		// Mirror the same dir structure from the repo because of the
+		// relative paths in ruff (for docs linting).
+		WithWorkdir(pythonSubdir).
+		WithMountedFile("requirements.txt", src.File("requirements.txt")).
+		WithExec([]string{"pip", "install", "-r", "requirements.txt"})
 
 	if install {
 		base = base.
-			WithMountedDirectory(mountPath, src).
+			WithMountedDirectory("", src).
 			WithExec([]string{"pip", "install", "--no-deps", "."})
 	}
 

diff --git a/core/integration/module_python_test.go b/core/integration/module_python_test.go
@@ -464,7 +464,8 @@ func TestModulePythonAltRuntime(t *testing.T) {
 
 	base := goGitBase(t, c).
 		WithMountedDirectory("/work/runtime", c.Host().Directory(runtimeSrcPath)).
-		WithMountedDirectory("/work/extended", c.Host().Directory(extSrcPath))
+		WithMountedDirectory("/work/extended", c.Host().Directory(extSrcPath)).
+		WithExec([]string{"sed", "-i", "s#../../../../../sdk/python/##", "/work/extended/dagger.json"})
 
 	t.Run("git dependency", func(t *testing.T) {
 		out, err := base.

diff --git a/core/integration/testdata/modules/python/extended/dagger.json b/core/integration/testdata/modules/python/extended/dagger.json
@@ -2,13 +2,13 @@
   "name": "ext-python-sdk",
   "sdk": "python",
   "exclude": [
-      ".venv",
-      "sdk"
+    ".venv",
+    "sdk"
   ],
   "dependencies": [
     {
       "name": "python-sdk",
-      "source": "../runtime"
+      "source": "../../../../../../sdk/python/runtime"
     }
   ],
   "source": ".",