diff --git a/app/gui/CHANGELOG.md b/app/gui/CHANGELOG.md index c7fbcde0de7a4..ddace656c2d3c 100644 --- a/app/gui/CHANGELOG.md +++ b/app/gui/CHANGELOG.md @@ -28,6 +28,8 @@ ensured it fails with a dataflow error on out of bounds access instead of an internal Java exception.][3232] - [Implemented the `Table.select_columns` operation.][3230] +- [Implemented the `Table.remove_columns` and `Table.reorder_columns` + operations.][3240] [3153]: https://github.com/enso-org/enso/pull/3153 [3166]: https://github.com/enso-org/enso/pull/3166 @@ -40,6 +42,7 @@ [3231]: https://github.com/enso-org/enso/pull/3231 [3232]: https://github.com/enso-org/enso/pull/3232 [3230]: https://github.com/enso-org/enso/pull/3230 +[3240]: https://github.com/enso-org/enso/pull/3240 # Enso 2.0.0-alpha.18 (2021-10-12) diff --git a/distribution/lib/Standard/Database/0.2.32-SNAPSHOT/src/Data/Table.enso b/distribution/lib/Standard/Database/0.2.32-SNAPSHOT/src/Data/Table.enso index ed8ca08bd1781..b0e864bcfa444 100644 --- a/distribution/lib/Standard/Database/0.2.32-SNAPSHOT/src/Data/Table.enso +++ b/distribution/lib/Standard/Database/0.2.32-SNAPSHOT/src/Data/Table.enso @@ -8,12 +8,13 @@ import Standard.Table.Data.Table as Materialized_Table import Standard.Table.Internal.Java_Exports import Standard.Table.Internal.Table_Helpers -from Standard.Database.Data.Column as Column_Module import all +from Standard.Database.Data.Column as Column_Module import Column, Aggregate_Column from Standard.Database.Data.Internal.IR import Internal_Column -from Standard.Table.Data.Order_Rule as Order_Rule_Module import Order_Rule from Standard.Table.Data.Table import No_Such_Column_Error -from Standard.Table.Data.Column_Selector as Column_Selector_Module import all -from Standard.Base.Error.Problem_Behavior as Problem_Behavior_Module import all +from Standard.Table.Data.Order_Rule as Order_Rule_Module import Order_Rule +from Standard.Table.Data.Column_Selector as Column_Selector_Module import Column_Selector, By_Index +from Standard.Base.Error.Problem_Behavior as Problem_Behavior_Module import Problem_Behavior, Report_Warning +import Standard.Table.Data.Position import Standard.Base.Error.Warnings polyglot java import java.sql.JDBCType @@ -112,7 +113,7 @@ type Table > Example Select columns by name. - table.select_columns (By_Name ["bar", "foo"] (Matching.Exact True)) + table.select_columns (By_Name.new ["bar", "foo"]) ## TODO [RW] default arguments do not work on atoms, once this is fixed, the above should be replaced with just `Matching.Exact`. @@ -138,6 +139,119 @@ type Table new_columns = Table_Helpers.select_columns internal_columns=this.internal_columns selector=columns reorder=reorder on_problems=on_problems warnings=warnings this.updated_columns new_columns + ## Returns a new table with the chosen set of columns, as specified by the + `columns`, removed from the input table. Any unmatched input columns will + be kept in the output. Columns are returned in the same order as in the + input. + + Arguments: + - columns: Criteria specifying which columns should be removed. + - on_problems: Specifies how to handle problems if they occur, reporting + them as warnings by default. + + The following problems can occur: + - If a column in columns is not in the input table, a + `Missing_Input_Columns`. + - If duplicate columns, names or indices are provided, a + `Duplicate_Column_Selectors`. + - If a column index is out of range, a `Column_Indexes_Out_Of_Range`. + - If two distinct indices would refer to the same column, a + `Input_Indices_Already_Matched`, indicating that the additional + indices will not introduce additional columns. + - If there are no columns in the output table, a `No_Output_Columns`. + - warnings: A `Warning_System` instance specifying how to handle + warnings. This is a temporary workaround to allow for testing the + warning mechanism. Once the proper warning system is implemented, this + argument will become obsolete and will be removed. No user code should + use this argument, as it will be removed in the future. + + > Example + Remove columns with given names. + + table.remove_columns (By_Name.new ["bar", "foo"]) + + ## TODO [RW] default arguments do not work on atoms, once this is fixed, + the above should be replaced with just `Matching.Exact`. + See: https://github.com/enso-org/enso/issues/1600 + + > Example + Remove columns matching a regular expression. + + table.remove_columns (By_Name ["foo.+", "b.*"] (Matching.Regex case_senitivity=Matching.Case_Insensitive)) + + > Example + Remove the first two columns and the last column. + + table.remove_columns (By_Index [-1, 0, 1]) + + > Example + Remove columns with the same names as the ones provided. + + table.remove_columns (By_Column [column1, column2]) + remove_columns : Column_Selector -> Problem_Behavior -> Warnings.Warning_System -> Table + remove_columns (columns = By_Index [0]) (on_problems = Report_Warning) (warnings = Warnings.default) = + new_columns = Table_Helpers.remove_columns internal_columns=this.internal_columns selector=columns on_problems=on_problems warnings=warnings + this.updated_columns new_columns + + ## Returns a new table with the specified selection of columns moved to + either the start or the end in the specified order. + + Arguments: + - columns: Criteria specifying which columns should be reordered and + specifying their order. + - position: Specifies how to place the selected columns in relation to + the remaining columns which were not matched by `columns` (if any). + - on_problems: Specifies how to handle problems if they occur, reporting + them as warnings by default. + + The following problems can occur: + - If a column in columns is not in the input table, a + `Missing_Input_Columns`. + - If duplicate columns, names or indices are provided, a + `Duplicate_Column_Selectors`. + - If a column index is out of range, a `Column_Indexes_Out_Of_Range`. + - If two distinct indices would refer to the same column, a + `Input_Indices_Already_Matched`, indicating that the additional + indices will not introduce additional columns. + - warnings: A `Warning_System` instance specifying how to handle + warnings. This is a temporary workaround to allow for testing the + warning mechanism. Once the proper warning system is implemented, this + argument will become obsolete and will be removed. No user code should + use this argument, as it will be removed in the future. + + > Example + Move a column with a specified name to back. + + table.reorder_columns (By_Name.new ["foo"]) position=After_Other_Columns + + ## TODO [RW] default arguments do not work on atoms, once this is fixed, + the above should be replaced with just `Matching.Exact`. + See: https://github.com/enso-org/enso/issues/1600 + + > Example + Move columns matching a regular expression to front, keeping columns matching "foo.+" before columns matching "b.*". + + table.reorder_columns (By_Name ["foo.+", "b.*"] (Matching.Regex case_senitivity=Matching.Case_Insensitive)) + + > Example + Swap the first two columns. + + table.reorder_columns (By_Index [1, 0]) position=Before_Other_Columns + + > Example + Move the first column to back. + + table.reorder_columns (By_Index [0]) position=After_Other_Columns + + > Example + Move the columns with names matching the provided columns to the front. + + table.reorder_columns (By_Column [column1, column2]) + reorder_columns : Column_Selector -> Position.Position -> Problem_Behavior -> Warnings.Warning_System -> Table + reorder_columns (columns = By_Index [0]) (position = Position.Before_Other_Columns) (on_problems = Report_Warning) (warnings = Warnings.default) = + new_columns = Table_Helpers.reorder_columns internal_columns=this.internal_columns selector=columns position=position on_problems=on_problems warnings=warnings + this.updated_columns new_columns + ## PRIVATE Resolves the column name to a column within this table. diff --git a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Column_Selector.enso b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Column_Selector.enso index 31e0a629c3ddf..5f426f9163759 100644 --- a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Column_Selector.enso +++ b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Column_Selector.enso @@ -11,14 +11,14 @@ type Column_Selector The `matching_strategy` can be used to specify if the names should be matched exactly or should be treated as regular expressions. It also allows to specify if the matching should be case-sensitive. - type By_Name (names : Vector Text) (matching_strategy : Matching_Strategy = Exact True) + type By_Name (names : Vector Text) (matching_strategy : Matching_Strategy = Exact.new) ## Selects columns by their index. The index of the first column in the table is 0. If the provided index is negative, it counts from the end of the table (e.g. -1 refers to the last column in the table). - type By_Index (indexes : Vector Number) + type By_Index (indexes : Vector Integer) ## Selects columns having exactly the same names as the columns provided in the input. @@ -27,3 +27,8 @@ type Column_Selector this approach can be used to match columns with the same names as a set of columns of some other table, for example, when preparing for a join. type By_Column (columns : Vector Column) + +## UNSTABLE + A temporary workaround to allow the By_Name constructor to work with default arguments. +By_Name.new : Vector Text -> Matching_Strategy -> By_Name +By_Name.new names (matching_strategy = Exact.new) = By_Name names matching_strategy diff --git a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Matching.enso b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Matching.enso index 9b40bc92cf335..14817f8125b33 100644 --- a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Matching.enso +++ b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Matching.enso @@ -2,8 +2,8 @@ from Standard.Base import all import Standard.Base.Data.Locale import Standard.Base.Data.Text.Regex as Regex_Module -from Standard.Base.Error.Problem_Behavior as Problem_Behavior_Module import all -from Standard.Base.Error.Warnings import all +from Standard.Base.Error.Problem_Behavior as Problem_Behavior_Module import Problem_Behavior, Report_Warning +from Standard.Base.Error.Warnings import Warning_System ## Strategy for matching names. type Matching_Strategy @@ -19,6 +19,16 @@ type Matching_Strategy A name is matched if its name matches the provided regular expression. type Regex (case_sensitivity : (True | Case_Insensitive) = True) +## UNSTABLE + A temporary workaround to allow the Exact constructor to work with default arguments. +Exact.new : (True | Case_Insensitive) -> Exact +Exact.new (case_sensitivity = True) = Exact case_sensitivity + +## UNSTABLE + A temporary workaround to allow the Regex constructor to work with default arguments. +Regex.new : (True | Case_Insensitive) -> Regex +Regex.new (case_sensitivity = True) = Regex case_sensitivity + ## UNSTABLE Specifies that the operation should ignore case. diff --git a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Position.enso b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Position.enso new file mode 100644 index 0000000000000..d01eb6926afda --- /dev/null +++ b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Position.enso @@ -0,0 +1,10 @@ +from Standard.Base import all + +type Position + ## UNSTABLE + Selected columns will be moved to the front of the output table. + type Before_Other_Columns + + ## UNSTABLE + Selected columns will be moved to the back of the output table. + type After_Other_Columns diff --git a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Table.enso b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Table.enso index 7ecc03cb1d6a5..c1e378c9b1740 100644 --- a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Table.enso +++ b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Data/Table.enso @@ -10,8 +10,9 @@ import Standard.Table.Io.Format import Standard.Table.Internal.Table_Helpers from Standard.Table.Data.Order_Rule as Order_Rule_Module import Order_Rule -from Standard.Table.Data.Column_Selector as Column_Selector_Module import all -from Standard.Base.Error.Problem_Behavior as Problem_Behavior_Module import all +from Standard.Table.Data.Column_Selector as Column_Selector_Module import Column_Selector, By_Index +from Standard.Base.Error.Problem_Behavior as Problem_Behavior_Module import Problem_Behavior, Report_Warning +import Standard.Table.Data.Position import Standard.Base.Error.Warnings polyglot java import org.enso.table.data.table.Table as Java_Table @@ -267,7 +268,7 @@ type Table > Example Select columns by name. - table.select_columns (By_Name ["bar", "foo"] (Matching.Exact True)) + table.select_columns (By_Name.new ["bar", "foo"]) ## TODO [RW] default arguments do not work on atoms, once this is fixed, the above should be replaced with just `Matching.Exact`. @@ -292,6 +293,119 @@ type Table new_columns = Table_Helpers.select_columns internal_columns=this.columns selector=columns reorder=reorder on_problems=on_problems warnings=warnings here.new new_columns + ## Returns a new table with the chosen set of columns, as specified by the + `columns`, removed from the input table. Any unmatched input columns will + be kept in the output. Columns are returned in the same order as in the + input. + + Arguments: + - columns: Criteria specifying which columns should be removed. + - on_problems: Specifies how to handle problems if they occur, reporting + them as warnings by default. + + The following problems can occur: + - If a column in columns is not in the input table, a + `Missing_Input_Columns`. + - If duplicate columns, names or indices are provided, a + `Duplicate_Column_Selectors`. + - If a column index is out of range, a `Column_Indexes_Out_Of_Range`. + - If two distinct indices would refer to the same column, a + `Input_Indices_Already_Matched`, indicating that the additional + indices will not introduce additional columns. + - If there are no columns in the output table, a `No_Output_Columns`. + - warnings: A `Warning_System` instance specifying how to handle + warnings. This is a temporary workaround to allow for testing the + warning mechanism. Once the proper warning system is implemented, this + argument will become obsolete and will be removed. No user code should + use this argument, as it will be removed in the future. + + > Example + Remove columns with given names. + + table.remove_columns (By_Name.new ["bar", "foo"]) + + ## TODO [RW] default arguments do not work on atoms, once this is fixed, + the above should be replaced with just `Matching.Exact`. + See: https://github.com/enso-org/enso/issues/1600 + + > Example + Remove columns matching a regular expression. + + table.remove_columns (By_Name ["foo.+", "b.*"] (Matching.Regex case_senitivity=Matching.Case_Insensitive)) + + > Example + Remove the first two columns and the last column. + + table.remove_columns (By_Index [-1, 0, 1]) + + > Example + Remove columns with the same names as the ones provided. + + table.remove_columns (By_Column [column1, column2]) + remove_columns : Column_Selector -> Problem_Behavior -> Warnings.Warning_System -> Table + remove_columns (columns = By_Index [0]) (on_problems = Report_Warning) (warnings = Warnings.default) = + new_columns = Table_Helpers.remove_columns internal_columns=this.columns selector=columns on_problems=on_problems warnings=warnings + here.new new_columns + + ## Returns a new table with the specified selection of columns moved to + either the start or the end in the specified order. + + Arguments: + - columns: Criteria specifying which columns should be reordered and + specifying their order. + - position: Specifies how to place the selected columns in relation to + the remaining columns which were not matched by `columns` (if any). + - on_problems: Specifies how to handle problems if they occur, reporting + them as warnings by default. + + The following problems can occur: + - If a column in columns is not in the input table, a + `Missing_Input_Columns`. + - If duplicate columns, names or indices are provided, a + `Duplicate_Column_Selectors`. + - If a column index is out of range, a `Column_Indexes_Out_Of_Range`. + - If two distinct indices would refer to the same column, a + `Input_Indices_Already_Matched`, indicating that the additional + indices will not introduce additional columns. + - warnings: A `Warning_System` instance specifying how to handle + warnings. This is a temporary workaround to allow for testing the + warning mechanism. Once the proper warning system is implemented, this + argument will become obsolete and will be removed. No user code should + use this argument, as it will be removed in the future. + + > Example + Move a column with a specified name to back. + + table.reorder_columns (By_Name.new ["foo"]) position=After_Other_Columns + + ## TODO [RW] default arguments do not work on atoms, once this is fixed, + the above should be replaced with just `Matching.Exact`. + See: https://github.com/enso-org/enso/issues/1600 + + > Example + Move columns matching a regular expression to front, keeping columns matching "foo.+" before columns matching "b.*". + + table.reorder_columns (By_Name ["foo.+", "b.*"] (Matching.Regex case_senitivity=Matching.Case_Insensitive)) + + > Example + Swap the first two columns. + + table.reorder_columns (By_Index [1, 0]) position=Before_Other_Columns + + > Example + Move the first column to back. + + table.reorder_columns (By_Index [0]) position=After_Other_Columns + + > Example + Move the columns with names matching the provided columns to the front. + + table.reorder_columns (By_Column [column1, column2]) + reorder_columns : Column_Selector -> Position.Position -> Problem_Behavior -> Warnings.Warning_System -> Table + reorder_columns (columns = By_Index [0]) (position = Position.Before_Other_Columns) (on_problems = Report_Warning) (warnings = Warnings.default) = + new_columns = Table_Helpers.reorder_columns internal_columns=this.columns selector=columns position=position on_problems=on_problems warnings=warnings + here.new new_columns + ## ALIAS Filter Rows ALIAS Mask Columns diff --git a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Internal/Table_Helpers.enso b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Internal/Table_Helpers.enso index 112b452bfbab3..d6c142f85dbeb 100644 --- a/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Internal/Table_Helpers.enso +++ b/distribution/lib/Standard/Table/0.2.32-SNAPSHOT/src/Internal/Table_Helpers.enso @@ -2,9 +2,10 @@ from Standard.Base import all import Standard.Base.Error.Warnings import Standard.Table.Data.Matching -from Standard.Base.Error.Problem_Behavior as Problem_Behavior_Module import all -from Standard.Table.Data.Column_Selector as Column_Selector_Module import all -from Standard.Table.Error as Error_Module import all +from Standard.Table.Data.Column_Selector as Column_Selector_Module import Column_Selector, By_Name, By_Index, By_Column +from Standard.Base.Error.Problem_Behavior as Problem_Behavior_Module import Problem_Behavior, Report_Warning +import Standard.Table.Data.Position +from Standard.Table.Error as Error_Module import Missing_Input_Columns, Column_Indexes_Out_Of_Range, No_Output_Columns, Duplicate_Column_Selectors, Input_Indices_Already_Matched ## PRIVATE A helper function encapsulating shared code for `select_columns` @@ -42,6 +43,73 @@ select_columns internal_columns selector reorder on_problems warnings = problems = if result.is_empty then [No_Output_Columns] else [] on_problems.attach_problems_after result problems warnings +## PRIVATE + A helper function encapsulating shared code for `remove_columns` + implementations of various Table variants. See the documentation for the + Table type for details. + + It takes a list of columns and returns the columns which should be kept. It + is the responsibility of each implementation to reconstruct a proper table + from the resulting list of columns. + + Arguments: + - internal_columns: A list of all columns in a table. + - selector: Column selection criteria. + - on_problems: Specifies the behavior when a problem occurs during the + operation. By default, a warning is issued, but the operation proceeds. + If set to `Report_Error`, the operation fails with a dataflow error. + If set to `Ignore`, the operation proceeds without errors or warnings. + - warnings: A Warning_System instance specifying how to handle warnings. This + is a temporary workaround to allow for testing the warning mechanism. Once + the proper warning system is implemented, this argument will become + obsolete and will be removed. No user code should use this argument, as it + will be removed in the future. +remove_columns : Vector -> Column_Selector -> Problem_Behavior -> Warnings.Warning_System -> Vector +remove_columns internal_columns selector on_problems warnings = + selection = here.select_columns_helper internal_columns selector reorder=False on_problems warnings + selected_names = Map.from_vector (selection.map column-> [column.name, True]) + result = internal_columns.filter column-> + should_be_removed = selected_names.get_or_else column.name False + should_be_removed.not + issues = if result.is_empty then [No_Output_Columns] else [] + on_problems.attach_problems_after result issues warnings + +## PRIVATE + A helper function encapsulating shared code for `reorder_columns` + implementations of various Table variants. See the documentation for the + Table type for details. + + It takes a list of columns and returns the columns which should be kept. It + is the responsibility of each implementation to reconstruct a proper table + from the resulting list of columns. + + Arguments: + - internal_columns: A list of all columns in a table. + - selector: A selector specifying which columns should be moved and the order + in which they should appear in the result. + - position: Specifies how to place the selected columns in relation to the + columns which were not matched by the `selector` (if any). + - on_problems: Specifies the behavior when a problem occurs during the + operation. By default, a warning is issued, but the operation proceeds. + If set to `Report_Error`, the operation fails with a dataflow error. + If set to `Ignore`, the operation proceeds without errors or warnings. + - warnings: A Warning_System instance specifying how to handle warnings. This + is a temporary workaround to allow for testing the warning mechanism. Once + the proper warning system is implemented, this argument will become + obsolete and will be removed. No user code should use this argument, as it + will be removed in the future. +reorder_columns : Vector -> Column_Selector -> Position.Position -> Problem_Behavior -> Warnings.Warning_System -> Vector +reorder_columns internal_columns selector position on_problems warnings = + selection = here.select_columns_helper internal_columns selector reorder=True on_problems warnings + selected_names = Map.from_vector (selection.map column-> [column.name, True]) + other_columns = internal_columns.filter column-> + is_selected = selected_names.get_or_else column.name False + is_selected.not + result = case position of + Position.Before_Other_Columns -> selection + other_columns + Position.After_Other_Columns -> other_columns + selection + result + ## PRIVATE A helper function which selects columns from the table based on the provided selection criteria. diff --git a/engine/runner/src/main/scala/org/enso/runner/Main.scala b/engine/runner/src/main/scala/org/enso/runner/Main.scala index ce81dd7172cdf..9431bea170e8d 100644 --- a/engine/runner/src/main/scala/org/enso/runner/Main.scala +++ b/engine/runner/src/main/scala/org/enso/runner/Main.scala @@ -841,7 +841,7 @@ object Main { /** Default log level to use if the LOG_LEVEL option is not provided. */ - val defaultLogLevel: LogLevel = LogLevel.Warning + val defaultLogLevel: LogLevel = LogLevel.Error /** Main entry point for the CLI program. * diff --git a/engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java b/engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java index b47b3d38e6a50..13129a1b007ba 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/OptionsHelper.java @@ -2,11 +2,7 @@ import com.oracle.truffle.api.TruffleFile; import com.oracle.truffle.api.TruffleLanguage; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import org.enso.polyglot.RuntimeOptions; public class OptionsHelper { diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java index 809573bafe76d..e0a2996b19788 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java @@ -5,13 +5,6 @@ import com.oracle.truffle.api.TruffleLanguage; import com.oracle.truffle.api.TruffleLanguage.Env; import com.oracle.truffle.api.TruffleLogger; -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.util.Optional; -import java.util.UUID; import org.enso.compiler.Compiler; import org.enso.compiler.PackageRepository; import org.enso.compiler.data.CompilerConfig; @@ -34,6 +27,10 @@ import org.enso.polyglot.RuntimeOptions; import scala.jdk.javaapi.OptionConverters; +import java.io.*; +import java.util.Optional; +import java.util.UUID; + /** * The language context is the internal state of the language that is associated with each thread in * a running Enso program. @@ -111,17 +108,17 @@ public void initialize() { Optional projectRoot = OptionsHelper.getProjectRoot(environment); Optional> projectPackage = - projectRoot.flatMap( - file -> { - var result = packageManager.fromDirectory(file); - if (result.isEmpty()) { - var projectName = file.getName(); - throw new ProjectLoadingFailure(projectName); - } - return ScalaConversions.asJava(result); - }); - - var languageHome = + projectRoot.map( + file -> + packageManager + .loadPackage(file) + .fold( + err -> { + throw new ProjectLoadingFailure(file.getName(), err); + }, + res -> res)); + + Optional languageHome = OptionsHelper.getLanguageHomeOverride(environment).or(() -> Optional.ofNullable(home)); var resourceManager = new org.enso.distribution.locking.ResourceManager(lockManager); diff --git a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala index 7db6c461fc191..b476714835e5e 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala @@ -1,5 +1,8 @@ package org.enso.compiler +import java.io.StringReader +import java.util.logging.Level + import com.oracle.truffle.api.TruffleLogger import com.oracle.truffle.api.source.Source import org.enso.compiler.codegen.{AstToIr, IrToTruffle, RuntimeStubsGenerator} @@ -24,8 +27,6 @@ import org.enso.polyglot.{LanguageInfo, RuntimeOptions} import org.enso.syntax.text.Parser.IDMap import org.enso.syntax.text.{AST, Parser} -import java.io.StringReader -import java.util.logging.Level import scala.jdk.OptionConverters._ /** This class encapsulates the static transformation processes that take place @@ -54,8 +55,13 @@ class Compiler( new SerializationManager(this) private val logger: TruffleLogger = context.getLogger(getClass) - /** Lazy-initializes the IR for the builtins module. - */ + /** Run the initialization sequence. */ + def initialize(): Unit = { + initializeBuiltinsIr() + packageRepository.initialize().left.foreach(reportPackageError) + } + + /** Lazy-initializes the IR for the builtins module. */ def initializeBuiltinsIr(): Unit = { if (!builtins.isIrInitialized) { logger.log( @@ -97,7 +103,7 @@ class Compiler( * @return the list of modules imported by `module` */ def runImportsResolution(module: Module): List[Module] = { - initializeBuiltinsIr() + initialize() importResolver.mapImports(module) } @@ -169,7 +175,7 @@ class Compiler( * @param module the scope from which docs are generated */ def generateDocs(module: Module): Module = { - initializeBuiltinsIr() + initialize() parseModule(module, isGenDocs = true) module } @@ -179,7 +185,7 @@ class Compiler( generateCode: Boolean, shouldCompileDependencies: Boolean ): Unit = { - initializeBuiltinsIr() + initialize() modules.foreach(m => parseModule(m)) var requiredModules = modules.flatMap(runImportsAndExportsResolution) @@ -667,12 +673,24 @@ class Compiler( } } + /** Report the errors encountered when initializing the package repository. + * + * @param err the package repository error + */ + private def reportPackageError(err: PackageRepository.Error): Unit = { + context.getOut.println( + s"In package description ${org.enso.pkg.Package.configFileName}:" + ) + context.getOut.println("Compiler encountered warnings:") + context.getOut.println(err.toString) + } + /** Reports diagnostics from multiple modules. * * @param diagnostics the mapping between modules and existing diagnostics. * @return whether any errors were encountered. */ - def reportDiagnostics( + private def reportDiagnostics( diagnostics: List[(Module, List[IR.Diagnostic])] ): Boolean = { val results = diagnostics.map { case (mod, diags) => @@ -693,7 +711,7 @@ class Compiler( * @param source the original source code. * @return whether any errors were encountered. */ - def reportDiagnostics( + private def reportDiagnostics( diagnostics: List[IR.Diagnostic], source: Source ): Boolean = { diff --git a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala index 280eadf349348..ad43d3d44cd48 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala @@ -15,14 +15,28 @@ import org.enso.librarymanager.{ ResolvingLibraryProvider } import org.enso.logger.masking.MaskedPath -import org.enso.pkg.{Package, PackageManager, QualifiedName} - +import org.enso.pkg.{ + ComponentGroup, + ComponentGroups, + ExtendedComponentGroup, + Package, + PackageManager, + QualifiedName +} import java.nio.file.Path + import scala.util.Try /** Manages loaded packages and modules. */ trait PackageRepository { + /** Initialize the package repository. + * + * @return `Right` if the package repository initialized successfully, + * and a `Left` containing an error otherwise. + */ + def initialize(): Either[PackageRepository.Error, Unit] + /** Informs the repository that it should populate the top scope with modules * belonging to a given package. * @@ -51,6 +65,9 @@ trait PackageRepository { */ def freezeModuleMap: PackageRepository.FrozenModuleMap + /** Get the loaded library components. */ + def getComponents: PackageRepository.ComponentsMap + /** Get a loaded module by its qualified name. */ def getLoadedModule(qualifiedName: String): Option[Module] @@ -80,6 +97,7 @@ object PackageRepository { type ModuleMap = collection.concurrent.Map[String, Module] type FrozenModuleMap = Map[String, Module] + type ComponentsMap = Map[LibraryName, ComponentGroups] /** A trait representing errors reported by this system */ sealed trait Error @@ -143,7 +161,7 @@ object PackageRepository { * unsynchronized threads read this map, every element it contains is * already fully processed. */ - val loadedPackages + private val loadedPackages : collection.concurrent.Map[LibraryName, Option[Package[TruffleFile]]] = { val builtinsName = LibraryName(Builtins.NAMESPACE, Builtins.PACKAGE_NAME) collection.concurrent.TrieMap(builtinsName -> None) @@ -157,15 +175,30 @@ object PackageRepository { * Strings, and constantly converting them into [[QualifiedName]]s would * add more overhead than is probably necessary. */ - val loadedModules: collection.concurrent.Map[String, Module] = + private val loadedModules: collection.concurrent.Map[String, Module] = collection.concurrent.TrieMap(Builtins.MODULE_NAME -> builtins.getModule) + /** The mapping containing loaded component groups. + * + * The component mapping is added to the collection after ensuring that the + * corresponding library was loaded. + */ + private val loadedComponents + : collection.concurrent.TrieMap[LibraryName, ComponentGroups] = { + val builtinsName = LibraryName(Builtins.NAMESPACE, Builtins.PACKAGE_NAME) + collection.concurrent.TrieMap(builtinsName -> ComponentGroups.empty) + } + /** @inheritdoc */ override def getModuleMap: ModuleMap = loadedModules /** @inheritdoc */ override def freezeModuleMap: FrozenModuleMap = loadedModules.toMap + /** @inheritdoc */ + override def getComponents: ComponentsMap = + loadedComponents.readOnlySnapshot().toMap + /** @inheritdoc */ override def registerMainProjectPackage( libraryName: LibraryName, @@ -215,7 +248,7 @@ object PackageRepository { libraryName: LibraryName, libraryVersion: LibraryVersion, root: Path - ): Either[Error, Unit] = Try { + ): Either[Error, Package[TruffleFile]] = Try { logger.debug( s"Loading library $libraryName from " + s"[${MaskedPath(root).applyMasking()}]." @@ -230,8 +263,91 @@ object PackageRepository { pkg = pkg, isLibrary = true ) + pkg }.toEither.left.map { error => Error.PackageLoadingError(error.getMessage) } + /** @inheritdoc */ + override def initialize(): Either[Error, Unit] = this.synchronized { + val unprocessedPackages = + loadedPackages.keySet + .diff(loadedComponents.keySet) + .flatMap(loadedPackages(_)) + unprocessedPackages.foldLeft[Either[Error, Unit]](Right(())) { + (accumulator, pkg) => + for { + _ <- accumulator + _ <- resolveComponentGroups(pkg) + } yield () + } + } + + private def resolveComponentGroups( + pkg: Package[TruffleFile] + ): Either[Error, Unit] = { + if (loadedComponents.contains(pkg.libraryName)) Right(()) + else { + pkg.config.componentGroups match { + case Left(err) => + Left(Error.PackageLoadingError(err.getMessage())) + case Right(componentGroups) => + logger.debug( + s"Resolving component groups of package [${pkg.name}]." + ) + + registerComponentGroups(pkg.libraryName, componentGroups.newGroups) + componentGroups.extendedGroups + .foldLeft[Either[Error, Unit]](Right(())) { + (accumulator, componentGroup) => + for { + _ <- accumulator + extendedLibraryName = componentGroup.module.libraryName + _ <- ensurePackageIsLoaded(extendedLibraryName) + pkgOpt = loadedPackages(extendedLibraryName) + _ <- pkgOpt.fold[Either[Error, Unit]](Right(()))( + resolveComponentGroups + ) + _ = registerExtendedComponentGroup( + pkg.libraryName, + componentGroup + ) + } yield () + } + } + } + } + + /** Register the list of component groups defined by a library. + * + * @param library the library name + * @param newGroups the list of component groups that the library defines + */ + private def registerComponentGroups( + library: LibraryName, + newGroups: List[ComponentGroup] + ): Unit = + loadedComponents.updateWith(library) { + case Some(groups) => + Some(groups.copy(newGroups = groups.newGroups ::: newGroups)) + case None => + Some(ComponentGroups(newGroups, List())) + } + + /** Register a component group extended by a library. + * + * @param library the library name + * @param group the extended component group + */ + private def registerExtendedComponentGroup( + library: LibraryName, + group: ExtendedComponentGroup + ): Unit = + loadedComponents.updateWith(library) { + case Some(groups) => + Some(groups.copy(extendedGroups = groups.extendedGroups :+ group)) + case None => + Some(ComponentGroups(List(), List(group))) + } + /** @inheritdoc */ override def ensurePackageIsLoaded( libraryName: LibraryName @@ -242,7 +358,7 @@ object PackageRepository { val resolvedLibrary = libraryProvider.findLibrary(libraryName) resolvedLibrary match { case Left(error) => - logger.error(s"Resolution failed with [$error].", error) + logger.warn(s"Resolution failed with [$error].", error) case Right(resolved) => logger.info( s"Found library ${resolved.name} @ ${resolved.version} " + @@ -259,6 +375,7 @@ object PackageRepository { .flatMap { library => loadPackage(library.name, library.version, library.location) } + .flatMap(resolveComponentGroups) .left .map { case ResolvingLibraryProvider.Error.NotResolved(details) => diff --git a/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala b/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala new file mode 100644 index 0000000000000..31700db54c14e --- /dev/null +++ b/engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala @@ -0,0 +1,248 @@ +package org.enso.interpreter.test.instrument + +import java.io.{ByteArrayOutputStream, File} +import java.nio.file.{Files, Path, Paths} +import java.util.UUID +import java.util.concurrent.{LinkedBlockingQueue, TimeUnit} + +import org.enso.distribution.FileSystem +import org.enso.distribution.locking.ThreadSafeFileLockManager +import org.enso.editions.LibraryName +import org.enso.interpreter.runtime +import org.enso.interpreter.test.Metadata +import org.enso.pkg.{ + Component, + ComponentGroups, + ExtendedComponentGroup, + ModuleName, + ModuleReference, + Package, + PackageManager +} +import org.enso.polyglot._ +import org.enso.polyglot.runtime.Runtime.Api +import org.enso.testkit.OsSpec +import org.graalvm.polyglot.Context +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} + +@scala.annotation.nowarn("msg=multiarg infix syntax") +class RuntimeComponentsTest + extends AnyFlatSpec + with Matchers + with BeforeAndAfterEach + with BeforeAndAfterAll + with OsSpec { + + final val ContextPathSeparator: String = File.pathSeparator + + var context: TestContext = _ + + class TestContext(packageName: String) { + + val messageQueue: LinkedBlockingQueue[Api.Response] = + new LinkedBlockingQueue() + + val tmpDir: Path = Files.createTempDirectory("enso-test-packages") + sys.addShutdownHook(FileSystem.removeDirectoryIfExists(tmpDir)) + val distributionHome: File = + Paths.get("../../distribution/component").toFile.getAbsoluteFile + val lockManager = new ThreadSafeFileLockManager(tmpDir.resolve("locks")) + val runtimeServerEmulator = + new RuntimeServerEmulator(messageQueue, lockManager) + + val pkg: Package[File] = { + val componentGroups = + ComponentGroups( + newGroups = List(), + extendedGroups = List( + ExtendedComponentGroup( + module = ModuleReference( + LibraryName("Standard", "Base"), + ModuleName("Group2") + ), + color = None, + icon = None, + exports = List(Component("foo", None)) + ) + ) + ) + PackageManager.Default.create( + root = tmpDir.toFile, + name = packageName, + namespace = "Enso_Test", + componentGroups = componentGroups + ) + } + + val out: ByteArrayOutputStream = new ByteArrayOutputStream() + val executionContext = new PolyglotContext( + Context + .newBuilder(LanguageInfo.ID) + .allowExperimentalOptions(true) + .allowAllAccess(true) + .option(RuntimeOptions.PROJECT_ROOT, pkg.root.getAbsolutePath) + .option( + RuntimeOptions.LANGUAGE_HOME_OVERRIDE, + distributionHome.toString + ) + .option(RuntimeOptions.LOG_LEVEL, "WARNING") + .option(RuntimeOptions.INTERPRETER_SEQUENTIAL_COMMAND_EXECUTION, "true") + .option(RuntimeServerInfo.ENABLE_OPTION, "true") + .option(RuntimeOptions.INTERACTIVE_MODE, "true") + .option(RuntimeOptions.DISABLE_IR_CACHES, "true") + .out(out) + .serverTransport(runtimeServerEmulator.makeServerTransport) + .build() + ) + executionContext.context.initialize(LanguageInfo.ID) + + val languageContext = executionContext.context + .getBindings(LanguageInfo.ID) + .invokeMember(MethodNames.TopScope.LEAK_CONTEXT) + .asHostObject[runtime.Context] + + def toPackagesPath(paths: String*): String = + paths.mkString(File.pathSeparator) + + def writeMain(contents: String): File = + Files.write(pkg.mainFile.toPath, contents.getBytes).toFile + + def writeFile(file: File, contents: String): File = + Files.write(file.toPath, contents.getBytes).toFile + + def writeInSrcDir(moduleName: String, contents: String): File = { + val file = new File(pkg.sourceDir, s"$moduleName.enso") + Files.write(file.toPath, contents.getBytes).toFile + } + + def send(msg: Api.Request): Unit = runtimeServerEmulator.sendToRuntime(msg) + + def receiveOne: Option[Api.Response] = { + Option(messageQueue.poll()) + } + + def receive: Option[Api.Response] = { + Option(messageQueue.poll(3, TimeUnit.SECONDS)) + } + + def receive(timeout: Long): Option[Api.Response] = { + Option(messageQueue.poll(timeout, TimeUnit.SECONDS)) + } + + def receiveN(n: Int): List[Api.Response] = { + Iterator.continually(receive).take(n).flatten.toList + } + + def receiveN(n: Int, timeout: Long): List[Api.Response] = { + Iterator.continually(receive(timeout)).take(n).flatten.toList + } + + def receiveAllUntil( + msg: Api.Response, + timeout: Long + ): List[Api.Response] = { + val receivedUntil = Iterator + .continually(receive(timeout)) + .takeWhile(received => received.isDefined && !received.contains(msg)) + .flatten + .toList + receivedUntil :+ msg + } + + def consumeOut: List[String] = { + val result = out.toString + out.reset() + result.linesIterator.toList + } + + def executionComplete(contextId: UUID): Api.Response = + Api.Response(Api.ExecutionComplete(contextId)) + + } + + override protected def beforeEach(): Unit = { + context = new TestContext("Test") + val Some(Api.Response(_, Api.InitializedNotification())) = context.receive + } + + it should "load library extended by the component group" in { + val contextId = UUID.randomUUID() + val requestId = UUID.randomUUID() + val moduleName = "Enso_Test.Test.Main" + + val metadata = new Metadata + + val code = + """main = "Hello World!" + |""".stripMargin.linesIterator.mkString("\n") + val contents = metadata.appendToCode(code) + val mainFile = context.writeMain(contents) + + // create context + context.send(Api.Request(requestId, Api.CreateContextRequest(contextId))) + context.receive shouldEqual Some( + Api.Response(requestId, Api.CreateContextResponse(contextId)) + ) + + // open file + context.send( + Api.Request(Api.OpenFileNotification(mainFile, contents)) + ) + context.receiveOne shouldEqual None + + // push main + context.send( + Api.Request( + requestId, + Api.PushContextRequest( + contextId, + Api.StackItem.ExplicitCall( + Api.MethodPointer(moduleName, "Main", "main"), + None, + Vector() + ) + ) + ) + ) + val responses = + context.receiveAllUntil( + context.executionComplete(contextId), + timeout = 60 + ) + // sanity check + responses should contain allOf ( + Api.Response(requestId, Api.PushContextResponse(contextId)), + context.executionComplete(contextId) + ) + + // check LibraryLoaded notifications + val contentRootNotifications = responses.collect { + case Api.Response( + None, + Api.LibraryLoaded(namespace, name, version, _) + ) => + (namespace, name, version) + } + + val libraryVersion = buildinfo.Info.stdLibVersion + contentRootNotifications should contain( + ("Standard", "Base", libraryVersion) + ) + + // check the registered component groups + val components = context.languageContext.getPackageRepository.getComponents + val expectedComponents = Map( + LibraryName("Enso_Test", "Test") -> + context.pkg.config.componentGroups + .getOrElse(fail("Unexpected config value.")), + LibraryName("Standard", "Base") -> ComponentGroups.empty, + LibraryName("Standard", "Builtins") -> ComponentGroups.empty + ) + components should contain theSameElementsAs expectedComponents + + context.consumeOut shouldEqual List() + } + +} diff --git a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ProjectLoadingFailure.scala b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ProjectLoadingFailure.scala index 4866eb6eeb4a4..d416958e1263a 100644 --- a/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ProjectLoadingFailure.scala +++ b/lib/scala/library-manager/src/main/scala/org/enso/librarymanager/ProjectLoadingFailure.scala @@ -1,7 +1,9 @@ package org.enso.librarymanager -class ProjectLoadingFailure(name: String) +class ProjectLoadingFailure(name: String, cause: Throwable) extends RuntimeException( s"The runtime was run in context of a project [$name], " + - s"but the project's package could not be loaded." + s"but the project's package could not be loaded (caused by: " + + s"${cause.getMessage}).", + cause ) diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/ComponentGroup.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/ComponentGroup.scala index 2016bcbc97db3..eb3444a9bd2a9 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/ComponentGroup.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/ComponentGroup.scala @@ -218,16 +218,14 @@ object Shortcut { */ case class ModuleReference( libraryName: LibraryName, - moduleName: Option[ModuleName] + moduleName: ModuleName ) object ModuleReference { private def toModuleString(moduleReference: ModuleReference): String = { - val libraryName = - s"${moduleReference.libraryName.namespace}${LibraryName.separator}${moduleReference.libraryName.name}" - moduleReference.moduleName.fold(libraryName) { moduleName => - s"$libraryName${LibraryName.separator}${moduleName.name}" - } + s"${moduleReference.libraryName.namespace}${LibraryName.separator}" + + s"${moduleReference.libraryName.name}${LibraryName.separator}" + + moduleReference.moduleName.name } /** [[Encoder]] instance for the [[ModuleReference]]. */ @@ -239,11 +237,11 @@ object ModuleReference { implicit val decoder: Decoder[ModuleReference] = { json => json.as[String].flatMap { moduleString => moduleString.split(LibraryName.separator).toList match { - case namespace :: name :: module => + case namespace :: name :: module :: modules => Right( ModuleReference( LibraryName(namespace, name), - ModuleName.fromComponents(module) + ModuleName.fromComponents(module, modules) ) ) case _ => @@ -268,8 +266,8 @@ object ModuleReference { case class ModuleName(name: String) object ModuleName { - def fromComponents(items: List[String]): Option[ModuleName] = - Option.unless(items.isEmpty)(ModuleName(items.mkString("."))) + def fromComponents(item: String, items: List[String]): ModuleName = + ModuleName((item :: items).mkString(LibraryName.separator.toString)) /** [[Encoder]] instance for the [[ModuleName]]. */ implicit val encoder: Encoder[ModuleName] = { moduleName => diff --git a/lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala b/lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala index 6b860f46631a3..fbaae5d0d756b 100644 --- a/lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala +++ b/lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala @@ -132,7 +132,7 @@ class ConfigSpec ExtendedComponentGroup( module = ModuleReference( LibraryName("Standard", "Base"), - Some(ModuleName("Group 2")) + ModuleName("Group 2") ), color = None, icon = None, diff --git a/test/Table_Tests/src/Common_Table_Spec.enso b/test/Table_Tests/src/Common_Table_Spec.enso index 6592ec003472a..d305be60089cc 100644 --- a/test/Table_Tests/src/Common_Table_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Spec.enso @@ -7,6 +7,7 @@ import Standard.Base.Error.Warnings import Standard.Table.Data.Matching from Standard.Table.Error as Table_Errors import all from Standard.Table.Data.Column_Selector as Column_Selector_Module import all +from Standard.Table.Data.Position as Position_Module import all ## A common test suite for shared operations on the Table API. @@ -24,22 +25,22 @@ from Standard.Table.Data.Column_Selector as Column_Selector_Module import all TODO [RW] the Any in return type of the builder should ideally be replaced with the Table interface, once that is supported. spec : Text -> (Vector -> Any) -> Boolean -> Nothing spec prefix table_builder supports_case_sensitive_columns = - Test.group prefix+"Select" <| - table = - col1 = ["foo", Integer, [1,2,3]] - col2 = ["bar", Integer, [4,5,6]] - col3 = ["Baz", Integer, [7,8,9]] - col4 = ["foo_1", Integer, [10,11,12]] - col5 = ["foo_2", Integer, [13,14,15]] - col6 = ["ab.+123", Integer, [16,17,18]] - col7 = ["abcd123", Integer, [19,20,21]] - table_builder [col1, col2, col3, col4, col5, col6, col7] - - expect_column_names names table = - table.columns . map .name . should_equal names frames_to_skip=2 + table = + col1 = ["foo", Integer, [1,2,3]] + col2 = ["bar", Integer, [4,5,6]] + col3 = ["Baz", Integer, [7,8,9]] + col4 = ["foo_1", Integer, [10,11,12]] + col5 = ["foo_2", Integer, [13,14,15]] + col6 = ["ab.+123", Integer, [16,17,18]] + col7 = ["abcd123", Integer, [19,20,21]] + table_builder [col1, col2, col3, col4, col5, col6, col7] + expect_column_names names table = + table.columns . map .name . should_equal names frames_to_skip=2 + + Test.group prefix+"Table.select_columns" <| Test.specify "should work as shown in the doc examples" <| - expect_column_names ["foo", "bar"] <| table.select_columns (By_Name ["bar", "foo"] (Matching.Exact True)) + expect_column_names ["foo", "bar"] <| table.select_columns (By_Name.new ["bar", "foo"]) expect_column_names ["bar", "Baz", "foo_1", "foo_2"] <| table.select_columns (By_Name ["foo.+", "b.*"] (Matching.Regex Matching.Case_Insensitive)) expect_column_names ["abcd123", "foo", "bar"] <| table.select_columns (By_Index [-1, 0, 1]) reorder=True @@ -48,17 +49,17 @@ spec prefix table_builder supports_case_sensitive_columns = expect_column_names ["Baz", "foo_1"] <| table.select_columns (By_Column [column1, column2]) Test.specify "should allow to reorder columns if asked to" <| - table_2 = table.select_columns (By_Name ["bar", "foo"] (Matching.Exact True)) reorder=True + table_2 = table.select_columns (By_Name.new ["bar", "foo"]) reorder=True expect_column_names ["bar", "foo"] table_2 table_2 . at "bar" . to_vector . should_equal [4,5,6] table_2 . at "foo" . to_vector . should_equal [1,2,3] Test.specify "should correctly handle regex matching" <| - expect_column_names ["foo"] <| table.select_columns (By_Name ["foo"] (Matching.Regex True)) - expect_column_names ["ab.+123", "abcd123"] <| table.select_columns (By_Name ["a.*"] (Matching.Regex True)) - expect_column_names ["ab.+123", "abcd123"] <| table.select_columns (By_Name ["ab.+123"] (Matching.Regex True)) - expect_column_names ["ab.+123"] <| table.select_columns (By_Name ["ab.+123"] (Matching.Exact True)) - expect_column_names ["abcd123"] <| table.select_columns (By_Name ["abcd123"] (Matching.Regex True)) + expect_column_names ["foo"] <| table.select_columns (By_Name ["foo"] Matching.Regex.new) + expect_column_names ["ab.+123", "abcd123"] <| table.select_columns (By_Name ["a.*"] Matching.Regex.new) + expect_column_names ["ab.+123", "abcd123"] <| table.select_columns (By_Name ["ab.+123"] Matching.Regex.new) + expect_column_names ["ab.+123"] <| table.select_columns (By_Name.new ["ab.+123"]) + expect_column_names ["abcd123"] <| table.select_columns (By_Name ["abcd123"] Matching.Regex.new) Test.specify "should allow negative indices" <| expect_column_names ["foo", "bar", "foo_2"] <| table.select_columns (By_Index [-3, 0, 1]) @@ -73,8 +74,8 @@ spec prefix table_builder supports_case_sensitive_columns = expect_column_names ["bar", "Bar"] <| table.select_columns (By_Name ["bar"] (Matching.Exact Matching.Case_Insensitive)) Test.specify "should correctly handle regexes matching multiple names" <| - expect_column_names ["foo", "bar", "foo_1", "foo_2"] <| table.select_columns (By_Name ["b.*", "f.+"] (Matching.Regex True)) - expect_column_names ["bar", "foo", "foo_1", "foo_2"] <| table.select_columns (By_Name ["b.*", "f.+"] (Matching.Regex True)) reorder=True + expect_column_names ["foo", "bar", "foo_1", "foo_2"] <| table.select_columns (By_Name ["b.*", "f.+"] Matching.Regex.new) + expect_column_names ["bar", "foo", "foo_1", "foo_2"] <| table.select_columns (By_Name ["b.*", "f.+"] Matching.Regex.new) reorder=True Test.specify "should correctly handle problems: out of bounds indices" <| selector = By_Index [1, 0, 100, -200, 300] @@ -98,7 +99,7 @@ spec prefix table_builder supports_case_sensitive_columns = Problems.test_problem_handling action problems tester Test.specify "should correctly handle problems: duplicate names" <| - selector = By_Name ["foo", "foo"] (Matching.Exact True) + selector = By_Name.new ["foo", "foo"] action = table.select_columns selector warnings=_ on_problems=_ tester = expect_column_names ["foo"] problems = [Duplicate_Column_Selectors ["foo"]] @@ -106,7 +107,7 @@ spec prefix table_builder supports_case_sensitive_columns = Test.specify "should correctly handle problems: unmatched names" <| weird_name = '.*?-!@#!"' - selector = By_Name ["foo", "hmm", weird_name] (Matching.Exact True) + selector = By_Name.new ["foo", "hmm", weird_name] action = table.select_columns selector warnings=_ on_problems=_ tester = expect_column_names ["foo"] problems = [Missing_Input_Columns ["hmm", weird_name]] @@ -133,14 +134,14 @@ spec prefix table_builder supports_case_sensitive_columns = Problems.test_problem_handling action problems tester Test.specify "should correctly handle problems: no columns in the output" <| - selector = By_Name [] (Matching.Exact True) + selector = By_Name.new [] action = table.select_columns selector warnings=_ on_problems=_ tester = expect_column_names [] problems = [No_Output_Columns] Problems.test_problem_handling action problems tester Test.specify "should correctly handle multiple problems" <| - selector = By_Name ["hmmm"] (Matching.Exact True) + selector = By_Name.new ["hmmm"] action = table.select_columns selector warnings=_ on_problems=_ tester = expect_column_names [] problems = [Missing_Input_Columns ["hmmm"], No_Output_Columns] @@ -150,3 +151,208 @@ spec prefix table_builder supports_case_sensitive_columns = problems_2 = [Column_Indexes_Out_Of_Range [100], Duplicate_Column_Selectors [0], Input_Indices_Already_Matched [-7]] tester_2 = expect_column_names ["foo"] Problems.test_problem_handling action_2 problems_2 tester_2 + + Test.group prefix+"Table.remove_columns" <| + Test.specify "should work as shown in the doc examples" <| + expect_column_names ["Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] <| table.remove_columns (By_Name.new ["bar", "foo"]) + expect_column_names ["foo", "ab.+123", "abcd123"] <| table.remove_columns (By_Name ["foo.+", "b.*"] (Matching.Regex Matching.Case_Insensitive)) + expect_column_names ["Baz", "foo_1", "foo_2", "ab.+123"] <| table.remove_columns (By_Index [-1, 0, 1]) + + column1 = table.at "foo_1" + column2 = table.at "Baz" + expect_column_names ["foo", "bar", "foo_2", "ab.+123", "abcd123"] <| table.remove_columns (By_Column [column1, column2]) + + Test.specify "should correctly handle regex matching" <| + last_ones = table.columns.tail.map .name + expect_column_names last_ones <| table.remove_columns (By_Name ["foo"] Matching.Regex.new) + first_ones = ["foo", "bar", "Baz", "foo_1", "foo_2"] + expect_column_names first_ones <| table.remove_columns (By_Name ["a.*"] Matching.Regex.new) + expect_column_names first_ones <| table.remove_columns (By_Name ["ab.+123"] Matching.Regex.new) + expect_column_names first_ones+["abcd123"] <| table.remove_columns (By_Name.new ["ab.+123"]) + expect_column_names first_ones+["ab.+123"] <| table.remove_columns (By_Name ["abcd123"] Matching.Regex.new) + + Test.specify "should allow negative indices" <| + expect_column_names ["Baz", "foo_1", "ab.+123"] <| table.remove_columns (By_Index [-1, -3, 0, 1]) + + if supports_case_sensitive_columns then + Test.specify "should correctly handle exact matches matching multiple names due to case insensitivity" <| + table = + col1 = ["foo", Integer, [1,2,3]] + col2 = ["bar", Integer, [4,5,6]] + col3 = ["Bar", Integer, [7,8,9]] + table_builder [col1, col2, col3] + expect_column_names ["foo"] <| table.remove_columns (By_Name ["bar"] (Matching.Exact Matching.Case_Insensitive)) + + Test.specify "should correctly handle regexes matching multiple names" <| + expect_column_names ["Baz", "ab.+123", "abcd123"] <| table.remove_columns (By_Name ["b.*", "f.+"] Matching.Regex.new) + + Test.specify "should correctly handle problems: out of bounds indices" <| + selector = By_Index [1, 0, 100, -200, 300] + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names ["Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + problems = [Column_Indexes_Out_Of_Range [100, -200, 300]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: duplicate indices" <| + selector = By_Index [0, 0, 0] + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + problems = [Duplicate_Column_Selectors [0, 0]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: aliased indices" <| + selector = By_Index [0, -7, -6, 1] + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names ["Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + problems = [Input_Indices_Already_Matched [-7, 1]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: duplicate names" <| + selector = By_Name.new ["foo", "foo"] + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + problems = [Duplicate_Column_Selectors ["foo"]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: unmatched names" <| + weird_name = '.*?-!@#!"' + selector = By_Name.new ["foo", "hmm", weird_name] + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + problems = [Missing_Input_Columns ["hmm", weird_name]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: duplicate columns" <| + foo = table.at "foo" + selector = By_Column [foo, foo] + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + problems = [Duplicate_Column_Selectors ["foo"]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: unmatched columns" <| + table_2 = table_builder [["foo", Integer, [0,0,0]], ["weird_column", Integer, [0,0,0]]] + foo = table_2.at "foo" + weird_column = table_2.at "weird_column" + bar = table.at "bar" + + selector = By_Column [bar, weird_column, foo] + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names ["Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + problems = [Missing_Input_Columns ["weird_column"]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: no columns in the output" <| + selector = By_Name [".*"] Matching.Regex.new + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names [] + problems = [No_Output_Columns] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle multiple problems" <| + selector = By_Name [".*", "hmmm"] Matching.Regex.new + action = table.remove_columns selector warnings=_ on_problems=_ + tester = expect_column_names [] + problems = [Missing_Input_Columns ["hmmm"], No_Output_Columns] + Problems.test_problem_handling action problems tester + + action_2 = table.remove_columns (By_Index [0, -7, 0, 100]) warnings=_ on_problems=_ + problems_2 = [Column_Indexes_Out_Of_Range [100], Duplicate_Column_Selectors [0], Input_Indices_Already_Matched [-7]] + tester_2 = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + Problems.test_problem_handling action_2 problems_2 tester_2 + + Test.group prefix+"Table.reorder_columns" <| + Test.specify "should work as shown in the doc examples" <| + expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo"] <| table.reorder_columns (By_Name.new ["foo"]) position=After_Other_Columns + expect_column_names ["foo_1", "foo_2", "bar", "Baz", "foo", "ab.+123", "abcd123"] <| table.reorder_columns (By_Name ["foo.+", "b.*"] (Matching.Regex Matching.Case_Insensitive)) + expect_column_names ["bar", "foo", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] <| table.reorder_columns (By_Index [1, 0]) position=Before_Other_Columns + expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo"] <| table.reorder_columns (By_Index [0]) position=After_Other_Columns + + column1 = table.at "foo_1" + column2 = table.at "Baz" + expect_column_names ["foo_1", "Baz", "foo", "bar", "foo_2", "ab.+123", "abcd123"] <| table.reorder_columns (By_Column [column1, column2]) + + Test.specify "should correctly handle regex matching" <| + expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo"] <| table.reorder_columns (By_Name ["foo"] Matching.Regex.new) position=After_Other_Columns + rest = ["foo", "bar", "Baz", "foo_1", "foo_2"] + expect_column_names ["ab.+123", "abcd123"]+rest <| table.reorder_columns (By_Name ["a.*"] Matching.Regex.new) + expect_column_names ["ab.+123", "abcd123"]+rest <| table.reorder_columns (By_Name ["ab.+123"] Matching.Regex.new) + expect_column_names ["ab.+123"]+rest+["abcd123"] <| table.reorder_columns (By_Name.new ["ab.+123"]) + expect_column_names ["abcd123"]+rest+["ab.+123"] <| table.reorder_columns (By_Name ["abcd123"] Matching.Regex.new) + + Test.specify "should allow negative indices" <| + expect_column_names ["abcd123", "foo_2", "foo", "bar", "Baz", "foo_1", "ab.+123"] <| table.reorder_columns (By_Index [-1, -3, 0, 1]) + + if supports_case_sensitive_columns then + Test.specify "should correctly handle exact matches matching multiple names due to case insensitivity" <| + table = + col1 = ["foo", Integer, [1,2,3]] + col2 = ["bar", Integer, [4,5,6]] + col3 = ["Bar", Integer, [7,8,9]] + table_builder [col1, col2, col3] + expect_column_names ["bar", "Bar", "foo"] <| table.reorder_columns (By_Name ["bar"] (Matching.Exact Matching.Case_Insensitive)) + + Test.specify "should correctly handle regexes matching multiple names" <| + expect_column_names ["bar", "foo", "foo_1", "foo_2", "Baz", "ab.+123", "abcd123"] <| table.reorder_columns (By_Name ["b.*", "f.+"] Matching.Regex.new) + + Test.specify "should correctly handle problems: out of bounds indices" <| + selector = By_Index [1, 0, 100, -200, 300] + action = table.reorder_columns selector warnings=_ on_problems=_ + tester = expect_column_names ["bar", "foo", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123"] + problems = [Column_Indexes_Out_Of_Range [100, -200, 300]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: duplicate indices" <| + selector = By_Index [0, 0, 0] + action = table.reorder_columns selector position=After_Other_Columns warnings=_ on_problems=_ + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo"] + problems = [Duplicate_Column_Selectors [0, 0]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: aliased indices" <| + selector = By_Index [0, -7, -6, 1] + action = table.reorder_columns selector position=After_Other_Columns warnings=_ on_problems=_ + tester = expect_column_names ["Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo", "bar"] + problems = [Input_Indices_Already_Matched [-7, 1]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: duplicate names" <| + selector = By_Name.new ["foo", "foo"] + action = table.reorder_columns selector position=After_Other_Columns warnings=_ on_problems=_ + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo"] + problems = [Duplicate_Column_Selectors ["foo"]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: unmatched names" <| + weird_name = '.*?-!@#!"' + selector = By_Name.new ["foo", "hmm", weird_name] + action = table.reorder_columns selector position=After_Other_Columns warnings=_ on_problems=_ + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo"] + problems = [Missing_Input_Columns ["hmm", weird_name]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: duplicate columns" <| + foo = table.at "foo" + selector = By_Column [foo, foo] + action = table.reorder_columns selector position=After_Other_Columns warnings=_ on_problems=_ + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo"] + problems = [Duplicate_Column_Selectors ["foo"]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle problems: unmatched columns" <| + table_2 = table_builder [["foo", Integer, [0,0,0]], ["weird_column", Integer, [0,0,0]]] + foo = table_2.at "foo" + weird_column = table_2.at "weird_column" + bar = table.at "bar" + + selector = By_Column [bar, weird_column, foo] + action = table.reorder_columns selector position=After_Other_Columns warnings=_ on_problems=_ + tester = expect_column_names ["Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "bar", "foo"] + problems = [Missing_Input_Columns ["weird_column"]] + Problems.test_problem_handling action problems tester + + Test.specify "should correctly handle multiple problems" <| + action = table.reorder_columns (By_Index [0, -7, 0, 100]) position=After_Other_Columns warnings=_ on_problems=_ + problems = [Column_Indexes_Out_Of_Range [100], Duplicate_Column_Selectors [0], Input_Indices_Already_Matched [-7]] + tester = expect_column_names ["bar", "Baz", "foo_1", "foo_2", "ab.+123", "abcd123", "foo"] + Problems.test_problem_handling action problems tester