diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso index 2123bd0f5814..09445020675f 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data.enso @@ -174,6 +174,7 @@ read_text path=(Missing_Argument.throw "path") (encoding : Encoding = Encoding.d example_list_files = Data.list Examples.data_dir name_filter="**.md" recursive=True @directory Folder_Browse +@name_filter File_Format.name_filter_widget list : Text | File -> Text -> Boolean -> Vector File list (directory:(Text | File)=enso_project.root) (name_filter:Text="") recursive:Boolean=False = file_obj = File.new directory diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Extensions.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Extensions.enso index e0fe19b5b3ec..7f85781c8734 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Extensions.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Text/Extensions.enso @@ -335,7 +335,7 @@ Text.find_all self pattern:Text|Regex=".*" case_sensitivity:Case_Sensitivity=..S # Evaluates to true "CONTACT@enso.org".match regex Case_Sensitivity.Insensitive Text.match : Text|Regex -> Case_Sensitivity -> Boolean ! Regex_Syntax_Error | Illegal_Argument -Text.match self pattern:Text|Regex=".*" case_sensitivity:Case_Sensitivity=..Sensitive = +Text.match self pattern:Text|Regex=".*" case_sensitivity:Case_Sensitivity=..Sensitive -> Boolean = case_insensitive = case_sensitivity.is_case_insensitive_in_memory compiled_pattern = Regex.compile pattern case_insensitive=case_insensitive compiled_pattern.matches self diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML/XML_Format.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML/XML_Format.enso index c583e385396c..a450a1763475 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML/XML_Format.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/XML/XML_Format.enso @@ -9,6 +9,7 @@ import project.Network.URI.URI import project.Nothing.Nothing import project.System.File.File import project.System.File.Generic.Writable_File.Writable_File +import project.System.File_Format.File_Name_Pattern import project.System.File_Format_Metadata.File_Format_Metadata import project.System.Input_Stream.Input_Stream from project.Data.Text.Extensions import all @@ -50,6 +51,10 @@ type XML_Format get_dropdown_options : Vector Option get_dropdown_options = [Option "XML" (Meta.get_qualified_type_name XML_Format)] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "XML" ["*.xml"]] + ## PRIVATE Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso index b3df00218cc7..29536ef11c11 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso @@ -791,6 +791,7 @@ type File example_list_md_files = Examples.data_dir.list name_filter="**.{txt,md}" recursive=True + @name_filter File_Format.name_filter_widget list : Text -> Boolean -> Vector File list self name_filter:Text="" recursive:Boolean=False = if self.is_directory.not then Error.throw (Illegal_Argument.Error "Cannot `list` a non-directory.") else @@ -799,7 +800,7 @@ type File "" -> all_files _ -> used_filter = if recursive.not || name_filter.contains "**" then name_filter else - (if name_filter.starts_with "*" then "*" else "**/") + name_filter + (if name_filter.starts_with "*" then "*" else "{**/,}") + name_filter matcher = File_Utils.matchPath "glob:"+used_filter all_files.filter file-> pathStr = self.relativize file . path diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso index 85a0522c7752..1c7f17e5e1e7 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/System/File_Format.enso @@ -19,6 +19,7 @@ import project.Metadata.Widget import project.Network.URI.URI import project.Nothing.Nothing import project.Panic.Panic +import project.Runtime import project.System.File.File import project.System.File.Generic.Writable_File.Writable_File import project.System.File_Format_Metadata.File_Format_Metadata @@ -79,6 +80,12 @@ type Auto_Detect get_dropdown_options : Vector Option get_dropdown_options = [Option "Auto Detect" (Meta.get_qualified_type_name Auto_Detect)] + ## PRIVATE + Returns the union of name patterns of all currently loaded formats, + since `Auto_Detect` should be able to read any of the loaded formats. + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "All known formats" File_Format.all_known_name_patterns] + ## Interface for all file formats. type File_Format ## PRIVATE @@ -118,12 +125,50 @@ type File_Format _ = [stream, metadata] Unimplemented.throw "This is an interface only." + ## PRIVATE + A static method on each format that returns a vector of options that can + be displayed in format selectors that allow choosing this file format. + + A single format instance can provide multiple options to choose, or none + at all. + get_dropdown_options : Vector Option + get_dropdown_options = Unimplemented.throw "This is an interface only." + + ## PRIVATE + A static method on each format that returns a vector of name pattern + options that can be displayed in the `name_filter_widget`. + get_name_patterns -> Vector File_Name_Pattern = Unimplemented.throw "This is an interface only." + + ## PRIVATE + Returns a list of all name patterns of all known file formats. + all_known_name_patterns -> Vector Text = + format_types.flat_map .get_name_patterns . flat_map .patterns . distinct + ## PRIVATE default_widget : Widget default_widget = options = ([Auto_Detect]+format_types).flat_map .get_dropdown_options Single_Choice display=Display.Always values=options + ## PRIVATE + Builds a widget intended to be used for `name_filter` of `File.list` and + its siblings that allows to filter file names by file format. + name_filter_widget -> Widget = + known_patterns = File_Format.all.flat_map .get_name_patterns + options = [Option "Any file" '""'] + known_patterns.map file_name_pattern-> + value = (_combine_patterns file_name_pattern.patterns) . pretty + Option file_name_pattern.display_name value + Single_Choice display=Display.When_Modified values=options + +## Combines a set of file name patterns into a single pattern that will match any of them. + It is compatible with the `name_filter` format of `File.list`. +private _combine_patterns (patterns : Vector Text) -> Text = + Runtime.assert (patterns.length > 0) + Runtime.assert message="The name patterns cannot contain {a,b} patterns to be mergeable." <| + contains_cases_regex = ".*[{},].*" + patterns.all (p-> p.match contains_cases_regex . not) + patterns.join prefix="{" separator="," suffix="}" + ## A file format for plain text files. type Plain_Text_Format ## A file format for plain text files with the specified encoding. @@ -168,6 +213,10 @@ type Plain_Text_Format get_dropdown_options : Vector Option get_dropdown_options = [Option "Plain Text" "..Plain_Text"] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "Plain Text" ["*.txt"], File_Name_Pattern.Value ".log files as Plain Text" ["*.log"]] + ## PRIVATE Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any @@ -214,6 +263,10 @@ type Bytes get_dropdown_options : Vector Option get_dropdown_options = [Option "Bytes" (Meta.get_qualified_type_name Bytes)] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value ".dat Binary Data" ["*.dat"]] + ## PRIVATE Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any @@ -260,6 +313,10 @@ type JSON_Format get_dropdown_options : Vector Option get_dropdown_options = [Option "JSON" (Meta.get_qualified_type_name JSON_Format)] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "JSON" ["*.json", "*.geojson"]] + ## PRIVATE Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any @@ -296,3 +353,16 @@ parse_boolean_with_infer (field_name : Text) (value : Boolean | Text | Nothing) "true" -> True "false" -> False _ -> Error.throw (Illegal_Argument.Error ("The field `"+field_name+"` must be a boolean or the string `infer`.")) + +## PRIVATE +type File_Name_Pattern + ## PRIVATE + Represents a single file pattern entry. + It may still contain multiple patterns that are related to a single type + of file. + + Each pattern should comply with the format expected by `name_filter` in + `File.list`, however, the patterns should not use the `{a,b}` syntax, + as it will be used by the `File_Format` to merge patterns and nesting it + would not be allowed. + Value display_name:Text (patterns : Vector Text) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso index 4230b1a2eabc..52ec194238a5 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/SQLite_Format.enso @@ -2,6 +2,7 @@ from Standard.Base import all import Standard.Base.Errors.Common.Type_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.System.File.Generic.Writable_File.Writable_File +import Standard.Base.System.File_Format.File_Name_Pattern import Standard.Base.System.File_Format_Metadata.File_Format_Metadata import Standard.Base.System.Input_Stream.Input_Stream from Standard.Base.Metadata.Choice import Option @@ -53,6 +54,10 @@ type SQLite_Format get_dropdown_options : Vector Option get_dropdown_options = [Option "SQLite" "..SQLite"] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "SQLite Database" ["*.sqlite", "*.db"]] + ## PRIVATE Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any diff --git a/distribution/lib/Standard/Image/0.0.0-dev/src/Image_File_Format.enso b/distribution/lib/Standard/Image/0.0.0-dev/src/Image_File_Format.enso index 42617183301c..d401b2f9bd2a 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/src/Image_File_Format.enso +++ b/distribution/lib/Standard/Image/0.0.0-dev/src/Image_File_Format.enso @@ -2,6 +2,7 @@ from Standard.Base import all import Standard.Base.Errors.Common.Type_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.System.File.Generic.Writable_File.Writable_File +import Standard.Base.System.File_Format.File_Name_Pattern import Standard.Base.System.File_Format_Metadata.File_Format_Metadata import Standard.Base.System.Input_Stream.Input_Stream from Standard.Base.Metadata.Choice import Option @@ -39,6 +40,11 @@ type Image_File_Format get_dropdown_options : Vector Option get_dropdown_options = [Option "Image" "..Image"] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + patterns = supported.map ext-> "*" + ext + [File_Name_Pattern.Value "Image" patterns] + ## PRIVATE Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Delimited/Delimited_Format.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Delimited/Delimited_Format.enso index d230dbefba44..510f2ddf9f36 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Delimited/Delimited_Format.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Delimited/Delimited_Format.enso @@ -3,6 +3,7 @@ import Standard.Base.Errors.Common.Type_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Network.HTTP.Response.Response import Standard.Base.System.File.Generic.Writable_File.Writable_File +import Standard.Base.System.File_Format.File_Name_Pattern import Standard.Base.System.File_Format_Metadata.File_Format_Metadata import Standard.Base.System.Input_Stream.Input_Stream from Standard.Base.Metadata.Choice import Option @@ -96,6 +97,10 @@ type Delimited_Format get_dropdown_options : Vector Option get_dropdown_options = [Option "Delimited" "..Delimited"] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "CSV" ["*.csv"], File_Name_Pattern.Value "Tab Delimited" ["*.tsv", "*.tab"], File_Name_Pattern.Value "Delimited Flat Files" ["*.csv", "*.tsv", "*.tab"]] + ## PRIVATE ADVANCED Implements the `File.read` for this `File_Format` diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Format.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Format.enso index da91e089839b..7981a2e2fc29 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Format.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Excel/Excel_Format.enso @@ -4,6 +4,7 @@ import Standard.Base.Errors.Common.Type_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Metadata.Display import Standard.Base.System.File.Generic.Writable_File.Writable_File +import Standard.Base.System.File_Format.File_Name_Pattern import Standard.Base.System.File_Format_Metadata.File_Format_Metadata import Standard.Base.System.Input_Stream.Input_Stream from Standard.Base.Metadata.Choice import Option @@ -106,6 +107,10 @@ type Excel_Format range = Option "Excel Range" "..Range" [workbook, sheet, range] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "Excel" ["*.xls", "*.xlsx", "*.xlsm", "*.xlt"]] + ## PRIVATE ADVANCED Implements the `File.read` for this `File_Format` diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Tableau_Format.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Tableau_Format.enso index afd9084189c0..6db83c5f3fab 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Tableau_Format.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Tableau_Format.enso @@ -2,6 +2,7 @@ from Standard.Base import all import Standard.Base.Errors.Common.Type_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.System.File.Generic.Writable_File.Writable_File +import Standard.Base.System.File_Format.File_Name_Pattern import Standard.Base.System.File_Format_Metadata.File_Format_Metadata import Standard.Base.System.Input_Stream.Input_Stream from Standard.Base.Metadata.Choice import Option @@ -43,6 +44,10 @@ type Tableau_Format get_dropdown_options : Vector Option get_dropdown_options = [Option "Tableau Hyper" "..Hyper_File"] + ## PRIVATE + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "Tableau Hyper" ["*.hyper"]] + ## PRIVATE Implements the `File.read` for this `File_Format` read : File -> Problem_Behavior -> Any diff --git a/test/Base_Tests/src/System/File_Read_Spec.enso b/test/Base_Tests/src/System/File_Read_Spec.enso index d9b9629cdba0..e4186e6f8ce3 100644 --- a/test/Base_Tests/src/System/File_Read_Spec.enso +++ b/test/Base_Tests/src/System/File_Read_Spec.enso @@ -79,8 +79,13 @@ add_specs suite_builder = r1.should_fail_with File_Error r1.catch.should_be_a File_Error.Corrupted_Format + suite_builder.group "File Format" group_builder-> + group_builder.specify "should provide a list of all supported file format name patterns" <| + patterns = File_Format.all_known_name_patterns + patterns.should_contain "*.txt" + patterns.should_contain "*.json" + main filter=Nothing = suite = Test.build suite_builder-> add_specs suite_builder suite.run_with_filter filter - diff --git a/test/Base_Tests/src/System/File_Spec.enso b/test/Base_Tests/src/System/File_Spec.enso index 478cae48b210..f6d6ff9d7452 100644 --- a/test/Base_Tests/src/System/File_Spec.enso +++ b/test/Base_Tests/src/System/File_Spec.enso @@ -933,6 +933,14 @@ add_specs suite_builder = filtered1b = root.list name_filter="*.txt" recursive=True . map .to_text filtered1b.sort.should_equal (resolve ["sample.txt", "subdirectory/a.txt", "subdirectory/nested/b.txt"]) + # It should also work if more complicated pattern is used + filtered1c = root.list name_filter="{*.txt,foobarbaz}" recursive=True . map .to_text + filtered1c.sort.should_equal (resolve ["sample.txt", "subdirectory/a.txt", "subdirectory/nested/b.txt"]) + + # And correctly match file 'starts with' condition in recursive mode + filtered1d = root.list name_filter="a*.txt" recursive=True . map .to_text + filtered1d.sort.should_equal (resolve ["subdirectory/a.txt"]) + filtered2 = root.list name_filter="*/*/*" recursive=True . map .to_text filtered2.should_equal (resolve ["subdirectory/nested/b.txt"]) diff --git a/test/Visualization_Tests/src/Widgets/File_Format_Widgets_Spec.enso b/test/Visualization_Tests/src/Widgets/File_Format_Widgets_Spec.enso index d00b98242dc5..ef043869bfc9 100644 --- a/test/Visualization_Tests/src/Widgets/File_Format_Widgets_Spec.enso +++ b/test/Visualization_Tests/src/Widgets/File_Format_Widgets_Spec.enso @@ -6,6 +6,7 @@ import Standard.Base.Metadata.Display from Standard.Table import all from Standard.Database import all +from Standard.Image import all import Standard.Visualization.Widgets @@ -14,7 +15,7 @@ from Standard.Test import all add_specs suite_builder = suite_builder.group "Widgets for Data.read" group_builder-> - group_builder.specify "should work and return basic formats" <| + group_builder.specify "should provide a list of loaded file formats" <| result = Widgets.get_widget_json Data .read ["format"] result.should_contain "Auto Detect" result.should_contain "Plain Text" @@ -22,6 +23,15 @@ add_specs suite_builder = result.should_contain "Excel Sheet" result.should_contain "SQLite" + group_builder.specify "should provide a list of available file name patterns" <| + result = Widgets.get_widget_json Data .list ["name_filter"] + result.should_contain "*.txt" + result.should_contain "*.xls" + result.should_contain "*.csv" + result.should_contain "*.png" + result.should_contain "Any file" + result.should_contain "All known formats" + main filter=Nothing = suite = Test.build suite_builder-> add_specs suite_builder