Allow resolving enso:// URIs in Data.read and other places (#9225)

- Implements the core parts of #9048 - Currently the path resolution is done by resolving each segment, one by one - requiring as many API calls as there are segments in the path. - This should be replaced in a followup PR, once enso-org/cloud-v2#899 is implemented.
enso-org · Mar 2, 2024 · 39af372 · 39af372
1 parent ade7d42
commit 39af372
Show file tree

Hide file tree

Showing 14 changed files with 146 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -620,6 +620,7 @@
 - [Allow `copy_to` and `move_to` to work between local and S3 files.][9054]
 - [Adjusted expression handling and new `Simple_Expression` type.][9128]
 - [Allow reading Data Links configured locally or in the Cloud.][9215]
+- [Allow using `enso://` paths in `Data.read` and other places.][9225]
 - [Update the XML methods and add more capabilities to document.][9233]
 
 [debug-shortcuts]:
@@ -897,6 +898,7 @@
 [9054]: https://github.com/enso-org/enso/pull/9054
 [9128]: https://github.com/enso-org/enso/pull/9128
 [9215]: https://github.com/enso-org/enso/pull/9215
+[9225]: https://github.com/enso-org/enso/pull/9225
 [9233]: https://github.com/enso-org/enso/pull/9233
 
 #### Enso Compiler

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Data_Link.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Data_Link.enso
@@ -75,8 +75,7 @@ parse_secure_value (json : Text | JS_Object) -> Text | Enso_Secret =
             case get_required_field "type" json of
                 "secret" ->
                     secret_path = get_required_field "secretPath" json
-                    _ = secret_path
-                    Unimplemented.throw "Reading secrets from a path is not implemented yet, see: https://github.com/enso-org/enso/issues/9048"
+                    Enso_Secret.get secret_path
                 other -> Error.throw (Illegal_State.Error "Unexpected value inside of a data-link: "+other+".")
 
 ## PRIVATE

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso
@@ -1,6 +1,7 @@
 import project.Any.Any
 import project.Enso_Cloud.Errors.Enso_Cloud_Error
-import project.Enso_Cloud.Utils
+import project.Enso_Cloud.Internal.Enso_Path.Enso_Path
+import project.Enso_Cloud.Internal.Utils
 import project.Data.Index_Sub_Range.Index_Sub_Range
 import project.Data.Json.JS_Object
 import project.Data.Numbers.Integer
@@ -36,6 +37,31 @@ from project.System.File.Generic.File_Write_Strategy import generic_copy
 from project.System.File_Format import Auto_Detect, Bytes, File_Format, Plain_Text_Format
 
 type Enso_File
+    ## Resolves an `enso://` path and returns the corresponding `Enso_File`
+       instance.
+
+       Arguments:
+       - path: The `enso://` path to a file or directory.
+
+       ? Enso Cloud Paths
+
+         The paths consist of the organization (user) name followed by a path to
+         the file/directory delimited by `/`.
+         For example `enso://my_org/some_dir/some-file.txt`.
+
+       ! Work in progress - only existing resources
+
+         Currently the API is only able to resolve paths to existing files or
+         directories. This is a temporary limitation and it will be improved in
+         the future, alongside with implementing the capabilities to write new
+         files.
+    new : Text -> Enso_File ! Not_Found
+    new (path : Text) =
+        parsed = Enso_Path.parse path
+        parent = parsed.resolve_parent
+        if parsed.asset_name == Nothing then parent else
+            parent / parsed.asset_name
+
     ## PRIVATE
        Represents a file or folder within the Enso cloud.
     Value name:Text id:Text organization:Text asset_type:Enso_Asset_Type

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_Secret.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_Secret.enso
@@ -1,7 +1,8 @@
 import project.Data.Base_64.Base_64
 import project.Enso_Cloud.Enso_File.Enso_Asset_Type
 import project.Enso_Cloud.Enso_File.Enso_File
-import project.Enso_Cloud.Utils
+import project.Enso_Cloud.Internal.Enso_Path.Enso_Path
+import project.Enso_Cloud.Internal.Utils
 import project.Data.Json.JS_Object
 import project.Data.Map.Map
 import project.Data.Text.Text
@@ -70,12 +71,25 @@ type Enso_Secret
     ## Get a Secret if it exists.
 
        Arguments:
-       - name: The name of the secret
-       - parent: The parent folder for the secret. If `Nothing` then will check
-         in the current working directory.
+       - name: The name of the secret, or an `enso://` path.
+       - parent: The parent folder for the secret, if resolving by name.
+         If `Nothing` then will search in the current working directory.
+         If an `enso://` path is provided, the parent argument is not considered
+         and must be `Nothing`.
     get : Text -> Enso_File | Nothing -> Enso_Secret ! Not_Found
     get name:Text parent:(Enso_File | Nothing)=Nothing =
-        Enso_Secret.list parent . find s-> s.name == name
+        is_path = name.starts_with "enso://"
+        case is_path of
+            True ->
+                if parent != Nothing then Error.throw (Illegal_Argument.Error "Parent argument must be Nothing when resolving by `enso://` path.") else
+                    parsed_path = Enso_Path.parse name
+                    case parsed_path.asset_name of
+                        Nothing ->
+                            Error.throw (Illegal_Argument.Error "The secret path must consist of at least user name and the secret name.")
+                        parsed_name ->
+                            Enso_Secret.get parsed_name parsed_path.resolve_parent
+            False ->
+                Enso_Secret.list parent . find s-> s.name == name
 
     ## GROUP Metadata
        Checks if a Secret exists.

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_User.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_User.enso
@@ -1,6 +1,6 @@
 import project.Enso_Cloud.Enso_File.Enso_Asset_Type
 import project.Enso_Cloud.Enso_File.Enso_File
-import project.Enso_Cloud.Utils
+import project.Enso_Cloud.Internal.Utils
 import project.Data.Json.JS_Object
 import project.Data.Text.Text
 import project.Data.Vector.Vector

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_Path.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_Path.enso
@@ -0,0 +1,37 @@
+private
+
+import project.Data.Index_Sub_Range.Index_Sub_Range
+import project.Data.Text.Text
+import project.Data.Vector.Vector
+import project.Enso_Cloud.Enso_File.Enso_File
+import project.Enso_Cloud.Enso_User.Enso_User
+import project.Error.Error
+import project.Errors.Illegal_Argument.Illegal_Argument
+import project.Errors.Unimplemented.Unimplemented
+import project.Nothing.Nothing
+from project.Data.Text.Extensions import all
+
+## PRIVATE
+   UNSTABLE
+   This is a temporary helper for resolving `enso://` paths.
+   It will be replaced once the backend resolver is implemented: https://github.com/enso-org/cloud-v2/issues/899
+type Enso_Path
+    ## PRIVATE
+    Value (organization_name : Text) (path_segments : Vector Text) (asset_name : Text | Nothing)
+
+    ## PRIVATE
+    parse (path : Text) -> Enso_Path =
+        prefix = "enso://"
+        if path.starts_with prefix . not then Error.throw (Illegal_Argument.Error "Invalid path - it should start with `enso://`.") else
+            raw_segments = path.drop prefix.length . split "/"
+            if raw_segments.is_empty then Error.throw (Illegal_Argument.Error "Invalid path - it should contain at least one segment.") else
+                organization_name = raw_segments.first
+                segments = raw_segments.drop 1 . filter s-> s.is_empty.not
+                if organization_name != Enso_User.current.name then Error.throw (Unimplemented.throw "Currently only resolving paths for the current user is supported.") else
+                    if segments.is_empty then Enso_Path.Value organization_name [] Nothing else
+                        asset_name = segments.last
+                        Enso_Path.Value organization_name (segments.drop (Index_Sub_Range.Last 1)) asset_name
+
+    ## PRIVATE
+    resolve_parent self =
+        self.path_segments.fold Enso_File.root (/)
diff --git a/.../Base/0.0.0-dev/src/Enso_Cloud/Utils.enso → ....0-dev/src/Enso_Cloud/Internal/Utils.enso b/.../Base/0.0.0-dev/src/Enso_Cloud/Utils.enso → ....0-dev/src/Enso_Cloud/Internal/Utils.enso
diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Header.enso
@@ -1,6 +1,5 @@
 import project.Enso_Cloud.Enso_Secret.Derived_Secret_Value
 import project.Enso_Cloud.Enso_Secret.Enso_Secret
-import project.Enso_Cloud.Utils as Cloud_Utils
 import project.Data.Numbers.Integer
 import project.Data.Text.Encoding.Encoding
 import project.Data.Text.Text

diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/URI.enso
@@ -1,6 +1,5 @@
 import project.Enso_Cloud.Enso_Secret.Enso_Secret
 import project.Enso_Cloud.Enso_Secret.Enso_Secret_Error
-import project.Enso_Cloud.Utils as Cloud_Utils
 import project.Data.Json.JS_Object
 import project.Data.Numbers.Integer
 import project.Data.Ordering.Comparable

diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoPathFileSystemSPI.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoPathFileSystemSPI.java
@@ -0,0 +1,26 @@
+package org.enso.base.enso_cloud;
+
+import org.enso.base.file_system.FileSystemSPI;
+
+/**
+ * Registers the `enso://` protocol for resolving file paths.
+ *
+ * <p>See `Enso_File.new` for more information on path resolution.
+ */
+@org.openide.util.lookup.ServiceProvider(service = FileSystemSPI.class)
+public class EnsoPathFileSystemSPI extends FileSystemSPI {
+  @Override
+  protected String getModuleName() {
+    return "Standard.Base.Enso_Cloud.Enso_File";
+  }
+
+  @Override
+  protected String getTypeName() {
+    return "Enso_File";
+  }
+
+  @Override
+  protected String getProtocol() {
+    return "enso";
+  }
+}
diff --git a/test/AWS_Tests/data/credentials-with-secrets.datalink b/test/AWS_Tests/data/credentials-with-secrets.datalink
@@ -7,11 +7,11 @@
     "subType": "access_key",
     "accessKeyId": {
       "type": "secret",
-      "secretPath": "enso://datalink-secret-AWS-keyid"
+      "secretPath": "enso://USERNAME/datalink-secret-AWS-keyid"
     },
     "secretAccessKey": {
       "type": "secret",
-      "secretPath": "enso://datalink-secret-AWS-secretkey"
+      "secretPath": "enso://USERNAME/datalink-secret-AWS-secretkey"
     }
   }
 }
diff --git a/test/AWS_Tests/src/S3_Spec.enso b/test/AWS_Tests/src/S3_Spec.enso
@@ -549,15 +549,16 @@ add_specs suite_builder =
             # It reads the datalink description and then reads the actual S3 file contents:
             (enso_project.data / "simple.datalink") . read . should_equal "Hello WORLD!"
 
-        #group_builder.specify "should be able to read a data link with custom credentials and secrets" pending=cloud_setup.pending <| cloud_setup.with_prepared_environment <|
-        group_builder.specify "should be able to read a data link with custom credentials and secrets" pending="TODO: reading secrets from path" <| cloud_setup.with_prepared_environment <|
+        group_builder.specify "should be able to read a data link with custom credentials and secrets" pending=cloud_setup.pending <| cloud_setup.with_prepared_environment <|
+            transformed_datalink_file = replace_username_in_datalink (enso_project.data / "credentials-with-secrets.datalink")
+
             secret_key_id = Enso_Secret.create "datalink-secret-AWS-keyid" (Environment.get "AWS_ACCESS_KEY_ID")
             secret_key_id.should_succeed
             Panic.with_finalizer secret_key_id.delete <|
                 secret_key_value = Enso_Secret.create "datalink-secret-AWS-secretkey" (Environment.get "AWS_SECRET_ACCESS_KEY")
                 secret_key_value.should_succeed
                 Panic.with_finalizer secret_key_value.delete <| with_retries <|
-                    (enso_project.data / "credentials-with-secrets.datalink") . read . should_equal "Hello WORLD!"
+                    transformed_datalink_file.read . should_equal "Hello WORLD!"
 
         group_builder.specify "should be able to read a data link with a custom file format set" <|
             r = (enso_project.data / "format-delimited.datalink") . read
@@ -570,3 +571,12 @@ main filter=Nothing =
     suite = Test.build suite_builder->
         add_specs suite_builder
     suite.run_with_filter filter
+
+## Reads the datalink as plain text and replaces the placeholder username with
+   actual one. It then writes the new contents to a temporary file and returns
+   it.
+replace_username_in_datalink base_file =
+    content = base_file.read Plain_Text
+    new_content = content.replace "USERNAME" Enso_User.current.name
+    temp_file = File.create_temporary_file prefix=base_file.name suffix=base_file.extension
+    new_content.write temp_file . if_not_error temp_file
diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Enso_File_Spec.enso
@@ -73,6 +73,10 @@ add_specs suite_builder setup:Cloud_Tests_Setup = setup.with_prepared_environmen
             f.is_directory . should_be_false
             f.exists . should_be_true
 
+        group_builder.specify "should be able to find a file by path" <|
+            File.new "enso://"+Enso_User.current.name+"/" . should_equal Enso_File.root
+            File.new "enso://"+Enso_User.current.name+"/test_file.json" . should_equal (Enso_File.root / "test_file.json")
+
         group_builder.specify "should not find nonexistent files" <|
             f = Enso_File.root / "nonexistent_file.json"
             f.should_fail_with Not_Found
@@ -120,6 +124,13 @@ add_specs suite_builder setup:Cloud_Tests_Setup = setup.with_prepared_environmen
 
             test_file.read_bytes . should_equal expected_file_text.utf_8
 
+        group_builder.specify "should be able to read the file by path using Data.read" <|
+            Data.read "enso://"+Enso_User.current.name+"/test_file.json" . should_equal [1, 2, 3, "foo"]
+            Data.read "enso://"+Enso_User.current.name+"/test-directory/another.txt" . should_equal "Hello Another!"
+
+            r = Data.read "enso://"+Enso_User.current.name+"/test-directory/nonexistent-directory/some-file.txt"
+            r.should_fail_with Not_Found
+
         group_builder.specify "should be able to open a file as input stream" <|
             test_file = Enso_File.root / "test_file.json"
             test_file.exists . should_be_true

diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Secrets_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Secrets_Spec.enso
@@ -43,14 +43,20 @@ add_specs suite_builder setup:Cloud_Tests_Setup = setup.with_prepared_environmen
             with_retries <|
                 Enso_Secret.list . should_not_contain my_secret
 
-        group_builder.specify "should allow to get a secret by name" <|
+        group_builder.specify "should allow to get a secret by name or path" <|
             created_secret = Enso_Secret.create "my_test_secret-2" "my_secret_value"
             created_secret.should_succeed
             Panic.with_finalizer created_secret.delete <|
                 with_retries <|
                     fetched_secret = Enso_Secret.get "my_test_secret-2"
                     fetched_secret . should_equal created_secret
 
+                    path_secret = Enso_Secret.get "enso://"+Enso_User.current.name+"/my_test_secret-2"
+                    path_secret . should_equal created_secret
+
+        group_builder.specify "does not allow both parent and path in Enso_Secret.get" <|
+            Enso_Secret.get "enso://"+Enso_User.current.name+"/SOME-SECRET" parent=Enso_File.root . should_fail_with Illegal_Argument
+
         group_builder.specify "should fail to create a secret if it already exists" <|
             created_secret = Enso_Secret.create "my_test_secret-3" "my_secret_value"
             created_secret.should_succeed