diff --git a/cli/cli.js b/cli/cli.js index 703b91233..7ba6ab044 100644 --- a/cli/cli.js +++ b/cli/cli.js @@ -140,7 +140,9 @@ function copyRedistributables(options, outputPath) { copyScalaFeature('core') } else if (options.target == 'TypeScript') { copyFiles('TypeScript/', outputPath) - } + } else if (options.target == 'Snowpark') { + copyFiles('Snowpark/', outputPath) + } } function copyRecursiveSync(src, dest) { diff --git a/docs/snowpark/snowpark-backend-types.md b/docs/snowpark/snowpark-backend-types.md new file mode 100644 index 000000000..039db2b7c --- /dev/null +++ b/docs/snowpark/snowpark-backend-types.md @@ -0,0 +1,41 @@ +--- +id: snowpark-backend-types +--- + +# Type mappings + +**TODO** + + When generating code using DataFrame operations types are mapped using the following criteria: + +| Elm/Morphir-IR type | Generated Scala type | Expected Snowflake type | +|-----------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------------------------| +| `Int` | `Column`\* | [INT](https://docs.snowflake.com/en/sql-reference/data-types-numeric#int-integer-bigint-smallint-tinyint-byteint) | +| `Float` | `Column`\* | [DOUBLE](https://docs.snowflake.com/en/sql-reference/data-types-numeric#double-double-precision-real) | +| `Bool` | `Column`\* | [BOOLEAN](https://docs.snowflake.com/en/sql-reference/data-types-logical#boolean) | +| `String` | `Column`\* | [VARCHAR](https://docs.snowflake.com/en/sql-reference/data-types-text) | +| Custom types without parameters | `Column`\* | [VARCHAR](https://docs.snowflake.com/en/sql-reference/data-types-text) | +| Custom types with parameters | `Column`\* | [OBJECT](https://docs.snowflake.com/en/sql-reference/data-types-semistructured#object) | +| Type alias | *As aliased* | *As aliased* | +| Record | Columns wrapper or **Column** | N/A | +| List of Record representing table | `DataFrame`\*\* | N/A | + +\* Snowpark [Column](https://docs.snowflake.com/developer-guide/snowpark/reference/scala/com/snowflake/snowpark/Column.html). + +\*\* Snowpark [DataFrame](https://docs.snowflake.com/developer-guide/snowpark/reference/scala/com/snowflake/snowpark/DataFrame.html). + + +When generating the code using Scala expressions the type conversion generates: + +| Elm/Morphir-IR type | Generated Scala type | +|-----------------------------------|--------------------------------| +| `Int` | `Int` | +| `Float` | `Double` | +| `Bool` | `Boolean` | +| `String` | `String` | +| Custom types without parameters | `String` | +| Custom types with parameters | **NOT IMPLEMENTED** | +| Type alias | *As aliased* | +| Record | Columns wrapper or **Column** | +| List of Record representing table | `DataFrame` | +| Complex Record | Scala case class | diff --git a/docs/snowpark/snowpark-backend-using-and-customize.md b/docs/snowpark/snowpark-backend-using-and-customize.md new file mode 100644 index 000000000..17feaaca3 --- /dev/null +++ b/docs/snowpark/snowpark-backend-using-and-customize.md @@ -0,0 +1,232 @@ +--- +id: snowpark-backend-using-and-customize +--- + + +## Using the Snowpark backend and customizing the output + +### Using the backend + +The backend is selected by specifying `Snowpark` as the target. For example: + +```bash +$ morphir-elm gen -t Snowpark -o outputDir +``` + +### Identifying not supported cases + +The output directory where the Scala code was generated contains a file called `GenerationReport.md`. This is a markdown file which contains the following information: + +- Generation issues by function: Elements not converted listed by function +- Listing of functions generated using the DataFrame operations strategy +- Listing of functions generated using the Scala expressions strategy +- Listing of types identified as `DataFrames` + +An example of this report: + +```markdown +# Generation report + + +## Generation issues + + +### MyModel:Basic:getTasksEstimationInSeconds22 + +- Call to function not generated: Morphir.SDK:List:range + +## Functions generated using DataFrame operations strategy + +- `MyModel:Basic:addThreeNumbers` +- `MyModel:Basic:checkLastName` +- `MyModel:Basic:classifyDepartment` +- `MyModel:Basic:getEmployeeWithLastName` +- `MyModel:Basic:getEmployeesInList` +- `MyModel:Basic:getEmployeesWithLastName` +- `MyModel:Basic:toSpString` + +## Functions generated using Scala strategy + +- MyModel:Basic:avgSalaries + +## Types identified as DataFrames + +- MyModel:Basic:department +- MyModel:Basic:employee +``` + +### Customizing the output using decorations + +The Snowpark backend supports a way to apply two customizations using [Morphir decorations](https://morphir.finos.org/docs/decorations-users-guide) . + +Two customizations could be applied : "Inline function call" or "Cache result" . + +#### Inlining functions + +The **Inline element** decoration allows the user to inline a function definition. For example given the following code: + +```elm +toSpString : CardinalDirection -> String +toSpString direction = + case direction of + North -> + "Norte" + South -> + "Sur" + East -> + "Este" + West -> + "Oeste" + +spanishDirections : List Directions -> List { translation : String } +spanishDirections directions = + directions + |> List.map (\dir -> { translation = toSpString dir.direction }) +``` + +The generated code for these definitions is: + +```scala +def toSpString( + direction: com.snowflake.snowpark.Column +)( + implicit sfSession: com.snowflake.snowpark.Session +): com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.when( + (direction) === (myModel.Basic.CardinalDirection.West), + com.snowflake.snowpark.functions.lit("Oeste") + ).when( + (direction) === (myModel.Basic.CardinalDirection.East), + com.snowflake.snowpark.functions.lit("Este") + ).when( + (direction) === (myModel.Basic.CardinalDirection.South), + com.snowflake.snowpark.functions.lit("Sur") + ).when( + (direction) === (myModel.Basic.CardinalDirection.North), + com.snowflake.snowpark.functions.lit("Norte") + ) + +def spanishDirections( + directions: com.snowflake.snowpark.DataFrame +)( + implicit sfSession: com.snowflake.snowpark.Session +): com.snowflake.snowpark.DataFrame = { + val directionsColumns: myModel.Basic.Directions = new myModel.Basic.DirectionsWrapper(directions) + + directions.select(myModel.Basic.toSpString(directionsColumns.direction).as("translation")) +} +``` + +By adding the `Inline element` decoration we can inline the `toSpString` function call inside `spanishDirections`. + +![inline decoration](sp_inline_decoration.png) + +Regenerating the code with these decorations shows that the definition was inlined in the place where the function was invoked. + +```scala + def spanishDirections( + directions: com.snowflake.snowpark.DataFrame + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.DataFrame = { + val directionsColumns: myModel.Basic.Directions = new myModel.Basic.DirectionsWrapper(directions) + + directions.select(com.snowflake.snowpark.functions.when( + (directionsColumns.direction) === (myModel.Basic.CardinalDirection.West), + com.snowflake.snowpark.functions.lit("Oeste") + ).when( + (directionsColumns.direction) === (myModel.Basic.CardinalDirection.East), + com.snowflake.snowpark.functions.lit("Este") + ).when( + (directionsColumns.direction) === (myModel.Basic.CardinalDirection.South), + com.snowflake.snowpark.functions.lit("Sur") + ).when( + (directionsColumns.direction) === (myModel.Basic.CardinalDirection.North), + com.snowflake.snowpark.functions.lit("Norte") + ).as("translation")) + } +``` + +#### Cache result + +Another customization allows the user to specify that caching code need to be generated for a specific function. + +For example given this function: + +```elm +getEmployeesWithLastName : String -> List Employee -> List Employee +getEmployeesWithLastName lastName employees = + employees + |> List.filter (\employee -> employee.lastName == lastName) +``` + +The code generated for this function looks like this: + +```scala + def getEmployeeWithLastName( + lastName: com.snowflake.snowpark.Column + )( + employees: com.snowflake.snowpark.DataFrame + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.DataFrame = { + val employeesColumns: myModel.Basic.Employee = new myModel.Basic.EmployeeWrapper(employees) + + employees.filter(myModel.Basic.checkLastName(lastName)(employeesColumns)) + } +``` + +We can add caching to this function with a decoration like this: + +![inline decoration](sp_caching_decoration.png) + +Regenerating the code show a different output. + +```scala +def getEmployeesWithLastName( + lastName: com.snowflake.snowpark.Column +)( + employees: com.snowflake.snowpark.DataFrame +)( + implicit sfSession: com.snowflake.snowpark.Session +): com.snowflake.snowpark.DataFrame = + getEmployeesWithLastNameCache.getOrElseUpdate( + (lastName, employees, sfSession), + { + val employeesColumns: myModel.Basic.Employee = new myModel.Basic.EmployeeWrapper(employees) + + employees.filter((employeesColumns.lastName) === (lastName)) + }.cacheResult + ) +``` + +Notice that this code use the [cacheResult](https://docs.snowflake.com/en/developer-guide/snowpark/scala/working-with-dataframes#caching-a-dataframe) mechanism. + +### Configuring the project for using Snowpark decorations + +Some modifications to the `morphir.json` are required to add these decorations. Here is an example of the configuration that needs to be added to this file: + +```json +{ + "name": "MyModel", + "sourceDirectory": "src", + "decorations": { + "snowparkgendecorations": { + "displayName" : "Snowpark generation customization", + "entryPoint": "SnowparkGenCustomization:Decorations:GenerationCustomization", + "ir": "out/decorations/morphir-ir.json", + "storageLocation": "spdecorations.json" + } + } +} +``` + +Every time code is generated with this Snowpark backend a `decorations` directory is created in the output directory. This directory contains the `morphir-ir.json` file to use the customizations. + +### Using decoration when generating code + +The `-dec` command line parameter is used to specify the decorations generated with the Morphir UI . For example given that the `spdecorations.json` name is used in the `storageLocation` section we can write: + +```bash +$ morphir-elm gen -t Snowpark -o output -dec spdecorations.json +``` diff --git a/docs/snowpark/snowpark-backend-value-mapping.md b/docs/snowpark/snowpark-backend-value-mapping.md new file mode 100644 index 000000000..30b142129 --- /dev/null +++ b/docs/snowpark/snowpark-backend-value-mapping.md @@ -0,0 +1,398 @@ +--- +id: snowpark-backend-value-mappings +--- + + +# Value mappings for DataFrame operations + +## Literals + +*Source* + +```elm +10 +"Hello" +True +``` + +*Target* + +```Scala +com.snowflake.snowpark.functions.lit(10) +com.snowflake.snowpark.functions.lit("Hello") +com.snowflake.snowpark.functions.lit(true) +``` + +## Field access values + +Field access expressions like `a.b` are converted depending on the value that is being accessed. For example the following function contains an access to the `lastName` field of the `Employee` record: + +*Source* + +```elm +checkLastName : String -> Employee -> Bool +checkLastName name employee = + if employee.lastName == name then + True + else + False +``` + +*Target* + +```scala + def checkLastName( + name: com.snowflake.snowpark.Column + )( + employee: myModel.Basic.Employee + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.when( + (employee.lastName) === (name), + com.snowflake.snowpark.functions.lit(true) + ).otherwise(com.snowflake.snowpark.functions.lit(false)) +``` + +As presented above the `employee.lastName` expression in Elm was generated as `employee.lastName`. + +## Variable values + +Variable access like `myValue` are almost always converted to variable accesses in Scala. There are few exceptions like access to global elements where identifiers are generated fully qualified. + +## Constructor call values + +Constructor invocations are generated depending of the current strategy being used by the backend. Some cases include: + +### Custom type without parameters + +*Source* + +```elm +type CardinalDirection + = North + | South + | East + | West + +myFunc : CardinalDirection +myFunc = + let + direction = North + in + direction +``` + +*Target* + +```scala +object CardinalDirection{ + + def East: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.lit("East") + + def North: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.lit("North") + + def South: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.lit("South") + + def West: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.lit("West") + +} + +def myFunc: com.snowflake.snowpark.Column = { + val direction = myModel.Basic.CardinalDirection.North + + direction +} +``` + +Notice that the call to construct `North` was replaced by an access to the helper object `CardinalDirection` . + + +### Custom type with parameters + +In this case the convention is use a JSON object to represent values. For example: + +*Source* + +```elm +type TimeRange = + Zero + | Seconds Int + | MinutesAndSeconds Int Int + +createMinutesAndSecs : Int -> Int -> TimeRange +createMinutesAndSecs min sec = + MinutesAndSeconds min sec +``` + +*Target* + +```scala + def createMinutesAndSecs( + min: com.snowflake.snowpark.Column + )( + sec: com.snowflake.snowpark.Column + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.object_construct( + com.snowflake.snowpark.functions.lit("__tag"), + com.snowflake.snowpark.functions.lit("MinutesAndSeconds"), + com.snowflake.snowpark.functions.lit("field0"), + min, + com.snowflake.snowpark.functions.lit("field1"), + sec + ) +``` + +### Returning records from functions + +In the case that a function returns a record, an [array](https://docs.snowflake.com/en/sql-reference/data-types-semistructured#array) is created to store the data. + +*Source* + +```elm +type alias EmpRes = + { code : Int + , name : String + } + +applyProcToEmployee : Employee -> Maybe EmpRes +applyProcToEmployee employee = + if employee.lastName == "Solo" then + Just <| EmpRes 1010 employee.firstName + else + Nothing +``` + +*Target* + +```scala + def applyProcToEmployee( + employee: myModel.Basic.Employee + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.when( + (employee.lastName) === (com.snowflake.snowpark.functions.lit("Solo")), + com.snowflake.snowpark.functions.array_construct( + com.snowflake.snowpark.functions.lit(1010), + employee.firstName + ) + ).otherwise(com.snowflake.snowpark.functions.lit(null)) +``` + +## List literal values + +Literal lists are converted a Scala [Seq](https://www.scala-lang.org/api/2.12.x/scala/collection/Seq.html) . + +*Source* + +```elm +someNames : List String +someNames = [ "Solo", "Jones" ] +``` + +*Target* + +```scala + def someNames: Seq[com.snowflake.snowpark.Column] = + Seq( + com.snowflake.snowpark.functions.lit("Solo"), + com.snowflake.snowpark.functions.lit("Jones") + ) +``` + +Sometimes that mapping function for an specific builtin function like `List.member` maybe change the conversion of literal lists. + +## Case/of values + +[Case/of](https://guide.elm-lang.org/types/pattern_matching) expressions are converted to a series of [when/otherwise](https://docs.snowflake.com/developer-guide/snowpark/reference/scala/com/snowflake/snowpark/CaseExpr.html#when(condition:com.snowflake.snowpark.Column,value:com.snowflake.snowpark.Column):com.snowflake.snowpark.CaseExpr) expressions. + +*Source* + +```elm +type CardinalDirection + = North + | South + | East + | West + +toSpString : CardinalDirection -> String +toSpString direction = + case direction of + North -> + "Norte" + South -> + "Sur" + East -> + "Este" + West -> + "Oeste" +``` + +*Target* + +```scala + def toSpString( + direction: com.snowflake.snowpark.Column + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.when( + (direction) === (myModel.Basic.CardinalDirection.West), + com.snowflake.snowpark.functions.lit("Oeste") + ).when( + (direction) === (myModel.Basic.CardinalDirection.East), + com.snowflake.snowpark.functions.lit("Este") + ).when( + (direction) === (myModel.Basic.CardinalDirection.South), + com.snowflake.snowpark.functions.lit("Sur") + ).when( + (direction) === (myModel.Basic.CardinalDirection.North), + com.snowflake.snowpark.functions.lit("Norte") + ) +``` + +## If/then/else values + +`If/then/else` expressions are converted to a series of [when/otherwise](https://docs.snowflake.com/developer-guide/snowpark/reference/scala/com/snowflake/snowpark/CaseExpr.html#when(condition:com.snowflake.snowpark.Column,value:com.snowflake.snowpark.Column):com.snowflake.snowpark.CaseExpr) expressions. + +*Source* + +```elm +if employee.lastName == name then + True +else + False +``` + +*Target* + +```scala + def checkLastName( + name: com.snowflake.snowpark.Column + )( + employee: myModel.Basic.Employee + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.when( + (employee.lastName) === (name), + com.snowflake.snowpark.functions.lit(true) + ).otherwise(com.snowflake.snowpark.functions.lit(false)) +``` + +## Let definition values + +`Let` expressions are converted to a sequence of Scala `val` declarations: + +*Source* + +```elm +myFunc2: Int -> Int +myFunc2 x = + let + y = x + 1 + z = y + 1 + in + x + y + z +``` + +*Target* + +```scala + def myFunc2( + x: com.snowflake.snowpark.Column + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = { + val y = (x) + (com.snowflake.snowpark.functions.lit(1)) + + val z = (y) + (com.snowflake.snowpark.functions.lit(1)) + + ((x) + (y)) + (z) + } +``` + +## Tuple values + +Tuples are treated as Snowflake [arrays](https://docs.snowflake.com/en/sql-reference/data-types-semistructured#array) . For example: + +*Source* + +```elm +let + y = x + 1 + z = y + 1 +in +(x, y, z) +``` + +*Target* + +```scala +{ + val y = (x) + (com.snowflake.snowpark.functions.lit(1)) + + val z = (y) + (com.snowflake.snowpark.functions.lit(1)) + + com.snowflake.snowpark.functions.array_construct( + x, + y, + z + ) +} +``` + +## Record values + +**TODO** + +## User defined function invocation values + +User defined functions are generated as Scala methods . This backend define the functions using [multiple parameter lists](https://docs.scala-lang.org/tour/multiple-parameter-lists.html) . + +```elm +aValue = addThreeNumbers 10 20 30 + +addThreeNumbers : Int -> Int -> Int -> Int +addThreeNumbers x y z = + x + y + z + +addTwo : Int -> Int -> Int +addTwo = addThreeNumbers 2 +``` + +*Target* + +```scala + def aValue: TypeNotConverted = + myModel.Basic.addThreeNumbers(com.snowflake.snowpark.functions.lit(10))(com.snowflake.snowpark.functions.lit(20))(com.snowflake.snowpark.functions.lit(30)) + + def addThreeNumbers( + x: com.snowflake.snowpark.Column + )( + y: com.snowflake.snowpark.Column + )( + z: com.snowflake.snowpark.Column + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = + ((x) + (y)) + (z) + + def addTwo: com.snowflake.snowpark.Column => com.snowflake.snowpark.Column => com.snowflake.snowpark.Column = + myModel.Basic.addThreeNumbers(com.snowflake.snowpark.functions.lit(2)) +``` + +## Builtin function invocation values + +Builtin functions are converted using different strategies depending on each case. The following document contains a description of this case. + +# Value mappings for plain Scala operations + +**TODO** diff --git a/docs/snowpark/snowpark-backend.md b/docs/snowpark/snowpark-backend.md new file mode 100644 index 000000000..5a9e4f3bb --- /dev/null +++ b/docs/snowpark/snowpark-backend.md @@ -0,0 +1,525 @@ +--- +id: snowpark-backend +--- + +# Snowpark Backend + +**TODO** + +**Snowpark** backend uses Scala as its JVM language. + +## Generation conventions and strategies + +The **Snowpark** backend supports two basic code generation strategies: + +- Generating code that manipulates DataFrame expressions +- Generating "plain" Scala code + +The backend uses a series of conventions for deciding which strategy is used to convert the code of a function. The conventions apply to types and function definitions. + +### Type definition conventions + +Type definitions in the input **Morphir IR** are classified according to the following conventions: + +#### Records that represent tables + +Records are classified as "representing a table definition" according to the types of its members. A DataFrame compatible type is one of the following: + +- A basic datatype + - Int + - Float + - Bool + - String +- A [custom type](https://guide.elm-lang.org/types/custom_types.html) +- A [Maybe](https://package.elm-lang.org/packages/elm/core/latest/Maybe) type used with a DataFrame compatible type +- An [alias](https://guide.elm-lang.org/types/type_aliases) of a DataFrame compatible type + +An example of these kinds of records is the following: + +```elm +type alias Employee = + { firstName : String + , lastName : String + } +``` + + +The **Snowpark** backend generates the following code for each type definiton: + +```scala + trait Employee { + + def firstName: com.snowflake.snowpark.Column + + def lastName: com.snowflake.snowpark.Column + + } + + object Employee extends Employee{ + + def firstName: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.col("firstName") + + def lastName: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.col("lastName") + + val schema = com.snowflake.snowpark.types.StructType( + com.snowflake.snowpark.types.StructField( + "FirstName", + com.snowflake.snowpark.types.StringType, + false + ), + com.snowflake.snowpark.types.StructField( + "LastName", + com.snowflake.snowpark.types.StringType, + false + ) + ) + + def createEmptyDataFrame( + session: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.DataFrame = + emptyDataFrameCache.getOrElseUpdate( + true, + session.createDataFrame( + Seq( + + ), + schema + ) + ) + + val emptyDataFrameCache: scala.collection.mutable.HashMap[Boolean, com.snowflake.snowpark.DataFrame] = new scala.collection.mutable.HashMap( + + ) + + } + + class EmployeeWrapper( + df: com.snowflake.snowpark.DataFrame + ) extends Employee{ + + def firstName: com.snowflake.snowpark.Column = + df("firstName") + + def lastName: com.snowflake.snowpark.Column = + df("lastName") + + } +``` + +This code includes: + +- A [trait](https://docs.scala-lang.org/tour/traits.html) with the definitions of the columns +- A [singleton object](https://docs.scala-lang.org/tour/singleton-objects.html) implementing the trait with the column definitions and an utility method to create an empty DataFrame for the current record +- A column wrapper class implementing the previous trait and giving access to the specific columns of a DataFrame + +#### Records representing Scala classes + +Records that contain fields that are not compatible with table column definitions are classified as "complex" and are generated as [Scala case classes](https://docs.scala-lang.org/tour/case-classes.html). + +Examples of these types are: + +- Lists +- Other record definitions +- Functions + +For example: + +```elm +type alias DataFromCompany + = + { employees : List Employee + , departments: List Department + } +``` + +The backend generates the following class for this record definition: + +```scala + case class DataFromCompany( + employees: com.snowflake.snowpark.DataFrame, + departments: com.snowflake.snowpark.DataFrame + ){} +``` + +#### Types representing DataFrames + +This backend considers lists of "records representing tables" as a [**Snowpark** DataFrame](https://docs.snowflake.com/en/developer-guide/snowpark/scala/working-with-dataframes) . For example: + + +```elm +getEmployeesWithLastName : String -> List Employee -> List Employee +getEmployeesWithLastName lastName employees = + employees + |> List.filter (\employee -> employee.lastName == lastName) +``` + +In this case references to `List Employee` are converted to DataFrames: + +```scala + def getEmployeesWithLastName( + lastName: com.snowflake.snowpark.Column + )( + employees: com.snowflake.snowpark.DataFrame + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.DataFrame = { + val employeesColumns: myModel.Basic.Employee = new myModel.Basic.EmployeeWrapper(employees) + + employees.filter((employeesColumns.lastName) === (lastName)) + } +``` + +### Custom types + +The **Snowpark** backend uses two conventions to deal with [custom types](https://guide.elm-lang.org/types/custom_types.html) used as a field of a *DataFrame record*. These conventions depend of the presence of parameters for type constructors. + +#### 1. Convention for custom types without data + +Custom types that define constructors without parameters are treated as a `String` (or `CHAR`, `VARCHAR`) column. + +For example: + +```elm +type CardinalDirection + = North + | South + | East + | West + +type alias Directions = + { + id : Int, + direction : CardinalDirection + } + +northDirections : List Directions -> List Directions +northDirections dirs = + dirs + |> List.filter (\e -> e.direction == North) +``` + +In this case the backend assumes that the code stored in a `Directions` table has a column of type `VARCHAR` or `CHAR` with text with the name of field. For example: + +| ID | DIRECTION | +|-----|------------| +| 10 | 'North' | +| 23 | 'East' | +| 43 | 'South' | + +As a convenience the backend generates a Scala object with the definition of the possible values: + +```Scala +object CardinalDirection{ + +def East: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.lit("East") + +def North: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.lit("North") + +def Sourth: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.lit("South") + +def West: com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.lit("West") + +} +``` +Notice the use of [lit](https://docs.snowflake.com/developer-guide/snowpark/reference/scala/com/snowflake/snowpark/functions$.html#lit(literal:Any):com.snowflake.snowpark.Column) to indicate that we expect a literal string value for each constructor. + +This class is used where the value of the possible constructors is used. For example for the definition of `northDirections` above the comparison with `North` is generated as follows: + +```Scala + def northDirections( + dirs: com.snowflake.snowpark.DataFrame + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.DataFrame = { + val dirsColumns: myModel.Basic.Directions = new myModel.Basic.DirectionsWrapper(dirs) + + dirs.filter((dirsColumns.direction) === (myModel.Basic.CardinalDirection.North)) + } +``` + +#### 2. Convention for custom types with data + +In the case that a custom type has constructors with parameters this backend assumes that values of this type are stored in a [OBJECT column](https://docs.snowflake.com/en/sql-reference/data-types-semistructured#object) . + +The encoding of column is defined as follows: + +- Values are encoded as a `JSON` object +- A special property of this object called `__tag` is used to determine which variant is used in the current value +- All the parameters in order are stored in properties called `field0`, `field1`, `field2` ... `fieldN` + +Given the following custom type definition: + +```elm +type TimeRange = + Zero + | Seconds Int + | MinutesAndSeconds Int Int + +type alias TasksEstimations = + { + taskId : Int, + estimation : TimeRange + } +``` + +The data for `TaskEstimations estimation` is expected to be stored in a table using an `OBJECT` column: + +| TASKID | ESTIMATION | +|--------|-------------------------------------------------------------------| +| 10 | `{ "__tag": "Zero" }` | +| 20 | `{ "__tag": "MinutesAndSeconds", "field0": 10, "field1": 20 }` | +| 30 | `{ "__tag": "Seconds", "field0": 2 }` | + +Pattern matching operations that manipulate values of this type are generated as operations that process JSON expressions following this convention. + +For example: + +```elm +getTasksEstimationInSeconds : List TasksEstimations -> List { seconds : Int } +getTasksEstimationInSeconds tasks = + tasks + |> List.map (\t -> + let + seconds = + case t.estimation of + Zero -> + 0 + Seconds s -> + s + MinutesAndSeconds mins secs -> + mins*60 + secs + in + { seconds = seconds }) + +``` + +This code is generated as: + +```scala + def getTasksEstimationInSeconds( + tasks: com.snowflake.snowpark.DataFrame + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.DataFrame = { + val tasksColumns: myModel.Basic.TasksEstimations = new myModel.Basic.TasksEstimationsWrapper(tasks) + + tasks.select(com.snowflake.snowpark.functions.when( + (tasksColumns.estimation("__tag")) === (com.snowflake.snowpark.functions.lit("Zero")), + com.snowflake.snowpark.functions.lit(0) + ).when( + (tasksColumns.estimation("__tag")) === (com.snowflake.snowpark.functions.lit("Seconds")), + tasksColumns.estimation("field0") + ).when( + (tasksColumns.estimation("__tag")) === (com.snowflake.snowpark.functions.lit("MinutesAndSeconds")), + ((tasksColumns.estimation("field0")) * (com.snowflake.snowpark.functions.lit(60))) + (tasksColumns.estimation("field1")) + ).as("seconds")) + } + +``` + +### Function definition conventions + +These conventions are based on the input and return types of a function. There are strategies: using DataFrame expressions or using Scala expressions. The following sections have more details. + +#### Code generation using DataFrame expressions manipulation + +For functions that receive or return DataFrames, simple types, or records the generation strategy is to generate DataFrame expressions for example: + +Given the following functions: + +```elm +checkLastName : String -> Employee -> Bool +checkLastName name employee = + if employee.lastName == name then + True + else + False + +getEmployeeWithLastName : String -> List Employee -> List Employee +getEmployeeWithLastName lastName employees = + employees + |> List.filter (\e -> checkLastName lastName e) +``` + +In this case the backend generates the following code: + +```scala + def checkLastName( + name: com.snowflake.snowpark.Column + )( + employee: myModel.Basic.Employee + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.when( + (employee.lastName) === (name), + com.snowflake.snowpark.functions.lit(true) + ).otherwise(com.snowflake.snowpark.functions.lit(false)) + + def getEmployeeWithLastName( + lastName: com.snowflake.snowpark.Column + )( + employees: com.snowflake.snowpark.DataFrame + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.DataFrame = { + val employeesColumns: myModel.Basic.Employee = new myModel.Basic.EmployeeWrapper(employees) + + employees.filter(myModel.Basic.checkLastName(lastName)(employeesColumns)) + } +``` + +Notice that language constructs are converted to DataFrame expression. For example in the way the `if` expression was converted to a combination of [`when`](https://docs.snowflake.com/ko/developer-guide/snowpark/reference/scala/com/snowflake/snowpark/functions$.html#when(condition:com.snowflake.snowpark.Column,value:com.snowflake.snowpark.Column):com.snowflake.snowpark.CaseExpr) and [`otherwise`](https://docs.snowflake.com/ko/developer-guide/snowpark/reference/scala/com/snowflake/snowpark/CaseExpr.html#otherwise(value:com.snowflake.snowpark.Column):com.snowflake.snowpark.Column) calls. + +#### Conventions for functions from values to lists of records + +If the function doesn't receive a DataFrame but produces lists of records, the strategy for code generation changes . In this case the convention assumes that an array of semi-structured objects is being created instead of a DataFrame. + +For example: + +```elm +type alias Department = { + name : String + } + +type Buildings + = B1 + | B2 + | B3 + +type alias DeptBuildingClassification = + { + deptName : String, + building : Buildings + } + +classifyDepartment : Department -> List DeptBuildingClassification +classifyDepartment dept = + if List.member dept.name ["HR", "IT"] then + [ { deptName = dept.name + , building = B1 } ] + else if String.startsWith "DEVEL" dept.name then + [ { deptName = dept.name + , building = B2 } + , { deptName = dept.name + , building = B3 } ] + else + [ ] + +getBuildings : List Department -> List DeptBuildingClassification +getBuildings depts = + depts + |> List.concatMap (\e -> classifyDepartment e) +``` + +Notice the function `classifyDepartment` which generates a list of records but does not receive a list of records. In this case the code is generated as: + +```scala + def classifyDepartment( + dept: myModel.Basic.Department + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.Column = + com.snowflake.snowpark.functions.when( + dept.name.in(Seq( + com.snowflake.snowpark.functions.lit("HR"), + com.snowflake.snowpark.functions.lit("IT") + )), + com.snowflake.snowpark.functions.array_construct(com.snowflake.snowpark.functions.array_construct( + dept.name, + myModel.Basic.Buildings.B1 + )) + ).otherwise(com.snowflake.snowpark.functions.when( + com.snowflake.snowpark.functions.startswith( + dept.name, + com.snowflake.snowpark.functions.lit("DEVEL") + ), + com.snowflake.snowpark.functions.array_construct( + com.snowflake.snowpark.functions.array_construct( + dept.name, + myModel.Basic.Buildings.B2 + ), + com.snowflake.snowpark.functions.array_construct( + dept.name, + myModel.Basic.Buildings.B3 + ) + ) + ).otherwise(com.snowflake.snowpark.functions.array_construct( + + ))) + + def getBuildings( + depts: com.snowflake.snowpark.DataFrame + )( + implicit sfSession: com.snowflake.snowpark.Session + ): com.snowflake.snowpark.DataFrame = { + val deptsColumns: myModel.Basic.Department = new myModel.Basic.DepartmentWrapper(depts) + + depts.select(myModel.Basic.classifyDepartment(myModel.Basic.Department).as("result")).flatten(com.snowflake.snowpark.functions.col("result")).select( + com.snowflake.snowpark.functions.as_char(com.snowflake.snowpark.functions.col("value")(0)).as("deptName"), + com.snowflake.snowpark.functions.col("value")(1).as("building") + ) + } +``` + +#### Code generation using Scala expressions + +When a function receives *"complex"* types as parameters the strategy is changed to use a Scala expression approach. + +For example: + +```elm +type alias EmployeeSal = + { firstName : String + , lastName : String + , salary : Float + } + + +type alias DataForCompany + = + { employees : List EmployeeSal + , departments: List Department + } + +avgSalaries : DataForCompany -> Float +avgSalaries companyData = + let + sum = companyData.employees + |> List.map (\e -> e.salary) + |> List.sum + count = companyData.employees + |> List.length + in + sum / (toFloat count) +``` + +In this case code for `avgSalaries` is going to perform a Scala division operation with the result of two DataFrame operations: + +```Scala + def avgSalaries( + companyData: myModel.Basic.DataForCompany + )( + implicit sfSession: com.snowflake.snowpark.Session + ): Double = { + val sum = companyData.employees.select(myModel.Basic.EmployeeSal.salary.as("result")).select(com.snowflake.snowpark.functions.coalesce( + com.snowflake.snowpark.functions.sum(com.snowflake.snowpark.functions.col("result")), + com.snowflake.snowpark.functions.lit(0) + )).first.get.getDouble(0) + + val count = companyData.employees.count + + (sum) / (count.toDouble) + } +``` + +Code generation for this strategy is meant to be used for code that manipulates the result of performing DataFrame operations . At this moment its coverage is very limited. + + + diff --git a/docs/snowpark/sp_caching_decoration.png b/docs/snowpark/sp_caching_decoration.png new file mode 100644 index 000000000..d197a069e Binary files /dev/null and b/docs/snowpark/sp_caching_decoration.png differ diff --git a/docs/snowpark/sp_inline_decoration.png b/docs/snowpark/sp_inline_decoration.png new file mode 100644 index 000000000..6043f6e68 Binary files /dev/null and b/docs/snowpark/sp_inline_decoration.png differ diff --git a/redistributable/Snowpark/decorations/elm.json b/redistributable/Snowpark/decorations/elm.json new file mode 100644 index 000000000..ce2a08dc7 --- /dev/null +++ b/redistributable/Snowpark/decorations/elm.json @@ -0,0 +1,24 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/browser": "1.0.2", + "elm/core": "1.0.5", + "elm/html": "1.0.0" + }, + "indirect": { + "elm/json": "1.1.3", + "elm/time": "1.0.0", + "elm/url": "1.0.0", + "elm/virtual-dom": "1.0.3" + } + }, + "test-dependencies": { + "direct": {}, + "indirect": {} + } +} diff --git a/redistributable/Snowpark/decorations/morphir.json b/redistributable/Snowpark/decorations/morphir.json new file mode 100644 index 000000000..437165aec --- /dev/null +++ b/redistributable/Snowpark/decorations/morphir.json @@ -0,0 +1,4 @@ +{ + "name": "SnowparkGenCustomization", + "sourceDirectory": "src" +} diff --git a/redistributable/Snowpark/decorations/src/SnowparkGenCustomization/Decorations.elm b/redistributable/Snowpark/decorations/src/SnowparkGenCustomization/Decorations.elm new file mode 100644 index 000000000..fc1345efe --- /dev/null +++ b/redistributable/Snowpark/decorations/src/SnowparkGenCustomization/Decorations.elm @@ -0,0 +1,5 @@ +module SnowparkGenCustomization.Decorations exposing (..) + +type GenerationCustomization = + InlineElement + | CacheResult diff --git a/src/Morphir/Snowpark/FunctionMappingsForPlainScala.elm b/src/Morphir/Snowpark/FunctionMappingsForPlainScala.elm index 994c9d136..0e11d6f48 100644 --- a/src/Morphir/Snowpark/FunctionMappingsForPlainScala.elm +++ b/src/Morphir/Snowpark/FunctionMappingsForPlainScala.elm @@ -18,10 +18,12 @@ import Morphir.Snowpark.MapFunctionsMapping exposing (FunctionMappingTable , mapUncurriedFunctionCall , dataFrameMappings , dataFrameMappings) -import Morphir.Snowpark.MappingContext exposing (ValueMappingContext, isUnionTypeWithoutParams) +import Morphir.Snowpark.MappingContext exposing (ValueMappingContext, isUnionTypeWithoutParams, isCandidateForDataFrame) import Morphir.Snowpark.Operatorsmaps exposing (mapOperator) import Morphir.Snowpark.PatternMatchMapping exposing (PatternMatchValues) -import Morphir.Snowpark.ReferenceUtils exposing (errorValueAndIssue, mapLiteralToPlainLiteral) +import Morphir.Snowpark.ReferenceUtils exposing (errorValueAndIssue, curryCall, mapLiteralToPlainLiteral) +import Morphir.IR.Value as Value +import Morphir.IR.Value as Value mapValueForPlainScala : TypedValue -> ValueMappingContext -> ValueGenerationResult mapValueForPlainScala value ctx = @@ -95,6 +97,7 @@ specificMappings = , ( basicsFunctionName [ "negate" ], mapNegateFunction ) , ( basicsFunctionName [ "max" ], mapMaxMinFunction "max" ) , ( basicsFunctionName [ "min" ], mapMaxMinFunction "min" ) + , ( basicsFunctionName [ "to", "float"], mapToFloatFunctionCall ) ] |> Dict.fromList @@ -118,27 +121,46 @@ mapListRangeFunction ( _, args ) mapValue ctx = _ -> errorValueAndIssue "List range scenario not supported" + mapListMapFunction : (TypedValue, List (TypedValue)) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult -mapListMapFunction ( _, args ) mapValue ctx = +mapListMapFunction (( _, args ) as call) mapValue ctx = case args of [ action, collection ] -> - let - (mappedStart, startIssues) = mapMapPredicate action mapValue ctx - (mappedEnd, endIssues) = mapValue collection ctx - in - (Scala.Apply (Scala.Select mappedEnd "map") [ Scala.ArgValue Nothing mappedStart], startIssues ++ endIssues) + if isCandidateForDataFrame (Value.valueAttribute collection) ctx.typesContextInfo then + MapDfOperations.mapValue (curryCall call) ctx + else + let + (mappedStart, startIssues) = mapMapPredicate action mapValue ctx + (mappedEnd, endIssues) = mapValue collection ctx + in + (Scala.Apply (Scala.Select mappedEnd "map") [ Scala.ArgValue Nothing mappedStart], startIssues ++ endIssues) _ -> errorValueAndIssue "List map scenario not supported" mapListSumFunction : (TypedValue, List (TypedValue)) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult -mapListSumFunction ( _, args ) mapValue ctx = +mapListSumFunction (( _, args ) as call) mapValue ctx = + let + collectionComesFromDataFrameProjection : TypedValue -> Bool + collectionComesFromDataFrameProjection collection = + case collection of + ValueIR.Apply + _ + (ValueIR.Apply _ (Value.Reference _ ( [ [ "morphir" ], [ "s", "d", "k" ] ], [ [ "list" ] ], [ "map" ] )) _) + innerCollection -> + isCandidateForDataFrame (ValueIR.valueAttribute innerCollection) ctx.typesContextInfo + _ -> + False + in case args of [ collection ] -> - let - (mappedCollection, collectionIssues) = mapValue collection ctx - in - ( Scala.Apply (Scala.Select mappedCollection "reduce") [ Scala.ArgValue Nothing (Scala.BinOp (Scala.Wildcard) "+" (Scala.Wildcard)) ] - , collectionIssues) + if collectionComesFromDataFrameProjection collection then + MapDfOperations.mapValue (curryCall call) ctx + else + let + (mappedCollection, collectionIssues) = mapValue collection ctx + in + ( Scala.Apply (Scala.Select mappedCollection "reduce") [ Scala.ArgValue Nothing (Scala.BinOp (Scala.Wildcard) "+" (Scala.Wildcard)) ] + , collectionIssues) _ -> errorValueAndIssue "List sum scenario not supported" @@ -207,8 +229,6 @@ mapMapPredicate action mapValue ctx = _ -> mapValue action ctx - - mapForOperatorCall : Name.Name -> TypedValue -> TypedValue -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult mapForOperatorCall optname left right mapValue ctx = let @@ -220,4 +240,15 @@ mapForOperatorCall optname left right mapValue ctx = in (Scala.BinOp leftValue operatorname rightValue , leftValueIssues ++ rightValueIssues ) - \ No newline at end of file + + +mapToFloatFunctionCall : (TypedValue, List TypedValue) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult +mapToFloatFunctionCall ( _, args ) mapValue ctx = + case args of + [ value ] -> + let + (mappedValue, valueIssues) = mapValue value ctx + in + (Scala.Select mappedValue "toDouble", valueIssues) + _ -> + errorValueAndIssue "`toFloat` scenario not supported" diff --git a/src/Morphir/Snowpark/LetMapping.elm b/src/Morphir/Snowpark/LetMapping.elm index 74160c504..d3c7b8af4 100644 --- a/src/Morphir/Snowpark/LetMapping.elm +++ b/src/Morphir/Snowpark/LetMapping.elm @@ -1,4 +1,4 @@ -module Morphir.Snowpark.LetMapping exposing (mapLetDefinition) +module Morphir.Snowpark.LetMapping exposing (mapLetDefinition, collectNestedLetDeclarations) import List import Morphir.IR.Name as Name diff --git a/src/Morphir/Snowpark/MapFunctionsMapping.elm b/src/Morphir/Snowpark/MapFunctionsMapping.elm index 780782ca0..3f9265e7e 100644 --- a/src/Morphir/Snowpark/MapFunctionsMapping.elm +++ b/src/Morphir/Snowpark/MapFunctionsMapping.elm @@ -35,6 +35,10 @@ import Morphir.Snowpark.ReferenceUtils exposing (errorValueAndIssue import Morphir.Snowpark.UserDefinedFunctionMapping exposing (tryToConvertUserFunctionCall) import Morphir.Snowpark.Utils exposing (collectMaybeList) import Morphir.Snowpark.Operatorsmaps exposing (mapOperator) +import Morphir.Snowpark.LetMapping exposing (collectNestedLetDeclarations) +import Morphir.Snowpark.MappingContext exposing (addLocalDefinitions) +import Morphir.IR.Type as Type +import Morphir.IR.Name as Name type alias MappingFunctionType = (TypedValue, List TypedValue) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult @@ -65,6 +69,7 @@ dataFrameMappings = , ( listFunctionName [ "concat", "map"], mapListConcatMapFunction ) , ( listFunctionName [ "concat" ], mapListConcatFunction ) , ( listFunctionName [ "sum" ], mapListSumFunction ) + , ( listFunctionName [ "length" ], mapListLengthFunction ) , ( maybeFunctionName [ "just" ], mapJustFunction ) , ( maybeFunctionName [ "map" ], mapMaybeMapFunction ) , ( maybeFunctionName [ "with", "default" ], mapMaybeWithDefaultFunction ) @@ -91,11 +96,14 @@ dataFrameMappings = , ( basicsFunctionName ["floor"] , mapFloorFunctionCall ) , ( basicsFunctionName ["min"] , mapMinMaxFunctionCall ("min", "<")) , ( basicsFunctionName ["max"] , mapMinMaxFunctionCall ("max", ">")) + , ( basicsFunctionName ["to", "float"] , mapToFloatFunctionCall ) , ( stringsFunctionName ["concat"] , mapStringConcatFunctionCall ) , ( stringsFunctionName ["to", "upper"] , mapStringCaseCall ("String.toUpper", "upper") ) , ( stringsFunctionName ["to", "lower"] , mapStringCaseCall ("String.toLower", "lower") ) , ( stringsFunctionName ["reverse"] , mapStringReverse ) , ( stringsFunctionName ["replace"] , mapStringReplace ) + , ( stringsFunctionName ["starts", "with"] , mapStartsEndsWith ("String.statsWith", "startswith") ) + , ( stringsFunctionName ["ends", "with"] , mapStartsEndsWith ("String.endsWith", "endswith") ) , ( ([["morphir"], ["s","d","k"]], [["aggregate"]], ["aggregate"]), mapAggregateFunction ) ] |> Dict.fromList @@ -177,7 +185,7 @@ mapListMemberFunction ( _, args ) mapValue ctx = in ( Scala.Apply (Scala.Select variable "in") [ Scala.ArgValue Nothing applySequence ], issues) _ -> - errorValueAndIssue "List.member scenario not converted" + errorValueAndIssue "`List.member` scenario not converted" mapListConcatMapFunction : (TypedValue, List TypedValue) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult mapListConcatMapFunction ( _, args ) mapValue ctx = @@ -333,6 +341,21 @@ mapListSumFunction ( _, args ) mapValue ctx = _ -> errorValueAndIssue "List sum scenario not supported" +mapListLengthFunction : (TypedValue, List TypedValue) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult +mapListLengthFunction ( _, args ) mapValue ctx = + case args of + [ elements ] -> + if isCandidateForDataFrame (ValueIR.valueAttribute elements) ctx.typesContextInfo then + let + (mappedCollection, collectionIssues) = + mapValue elements ctx + in + (Scala.Select mappedCollection "count", collectionIssues) + else + errorValueAndIssue "`List.length` scenario not supported" + _ -> + errorValueAndIssue "`List.length` scenario not supported" + generateForListSum : TypedValue -> ValueMappingContext -> Constants.MapValueType -> ValueGenerationResult generateForListSum collection ctx mapValue = case collection of @@ -393,7 +416,16 @@ generateForListFilter predicate sourceRelation ctx mapValue = (Scala.Apply (Scala.Ref (scalaPathToModule functionName) (functionName |> FQName.getLocalName |> Name.toCamelCase)) [Scala.ArgValue Nothing typeRefExpr], [])) _ -> - errorValueAndIssue "Unsupported filter function scenario2" + errorValueAndIssue "Unsupported filter function with referenced function" + (ValueIR.Apply ((Type.Function _ fromTpe toType) as tpe) _ _) as call -> + let + newLambda = + (ValueIR.Lambda + tpe + (ValueIR.AsPattern fromTpe (ValueIR.WildcardPattern fromTpe) [ "_t" ]) + (ValueIR.Apply toType call (ValueIR.Variable fromTpe [ "_t" ]))) + in + generateForListFilter newLambda sourceRelation ctx mapValue _ -> errorValueAndIssue "Unsupported filter function scenario" else @@ -529,17 +561,7 @@ processLambdaWithRecordBody functionExpr ctx mapValue = ValueIR.Lambda (TypeIR.Function _ _ returnType) (ValueIR.AsPattern _ _ _) (ValueIR.Record _ fields) -> if isAnonymousRecordWithSimpleTypes returnType ctx.typesContextInfo || isTypeRefToRecordWithSimpleTypes returnType ctx.typesContextInfo then - Just (fields - |> getFieldsInCorrectOrder returnType ctx - |> List.map (\(fieldName, value) -> - (Name.toCamelCase fieldName, (mapValue value ctx))) - |> List.map (\(fieldName, (value, issues)) -> - ((Scala.Apply - (Scala.Select value "as") - [ Scala.ArgValue Nothing (Scala.Literal (Scala.StringLit fieldName))]) - , issues)) - |> List.map (\(value, issues) -> (Scala.ArgValue Nothing value, issues)) - |> List.unzip) + processMapRecordFields fields returnType ctx mapValue else Nothing @@ -550,6 +572,9 @@ processLambdaWithRecordBody functionExpr ctx mapValue = mapValue expr ctx in Just ([Scala.ArgValue Nothing mappedBody], [ mappedBodyIssues ]) + else if isAnonymousRecordWithSimpleTypes returnType ctx.typesContextInfo + || isTypeRefToRecordWithSimpleTypes returnType ctx.typesContextInfo then + processLambdaBodyOfNonRecordLambda expr ctx mapValue else Nothing ValueIR.FieldFunction _ _ -> @@ -561,6 +586,52 @@ processLambdaWithRecordBody functionExpr ctx mapValue = _ -> Nothing + +processMapRecordFields : Dict (Name.Name) TypedValue -> Type () -> ValueMappingContext -> (Constants.MapValueType) -> Maybe ( List Scala.ArgValue, List (List GenerationIssue) ) +processMapRecordFields fields returnType ctx mapValue = + Just (fields + |> getFieldsInCorrectOrder returnType ctx + |> List.map (\(fieldName, value) -> + (Name.toCamelCase fieldName, (mapValue value ctx))) + |> List.map (\(fieldName, (value, issues)) -> + ((Scala.Apply + (Scala.Select value "as") + [ Scala.ArgValue Nothing (Scala.Literal (Scala.StringLit fieldName))]) + , issues)) + |> List.map (\(value, issues) -> (Scala.ArgValue Nothing value, issues)) + |> List.unzip) + +processLambdaBodyOfNonRecordLambda : TypedValue -> ValueMappingContext -> Constants.MapValueType -> Maybe ((List Scala.ArgValue, List (List GenerationIssue))) +processLambdaBodyOfNonRecordLambda body ctx mapValue = + case body of + (ValueIR.LetDefinition _ _ _ _) as topDefinition -> + let + (letDecls, letBodyExpr) = + collectNestedLetDeclarations topDefinition [] + (newCtx, resultIssues) = + letDecls + |> List.foldr + (\(name, def) (currentCtx, currentIssues) -> + let + (mappedDecl, issues) = + mapValue def.body currentCtx + newIssues = + currentIssues ++ issues + in + ( addReplacementForIdentifier name mappedDecl currentCtx + , newIssues)) + (ctx ,[]) + in + case letBodyExpr of + ValueIR.Record recordType fields -> + processMapRecordFields fields recordType newCtx mapValue + |> Maybe.map (\(args, issuesLst) -> (args, issuesLst ++ [resultIssues])) + _ -> + Nothing + + _ -> + Nothing + getFieldsInCorrectOrder : Type () -> ValueMappingContext -> Dict Name.Name TypedValue -> List (Name.Name, TypedValue) getFieldsInCorrectOrder originalType ctx fields = case originalType of @@ -736,6 +807,17 @@ mapMinMaxFunctionCall (morphirName, operator) ( _, args ) mapValue ctx = _ -> errorValueAndIssue ("`" ++ morphirName ++ " scenario not supported") +mapToFloatFunctionCall : (TypedValue, List TypedValue) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult +mapToFloatFunctionCall ( _, args ) mapValue ctx = + case args of + [ value ] -> + let + (mappedValue, valueIssues) = mapValue value ctx + in + (Constants.applySnowparkFunc "as_double" [ mappedValue ], valueIssues) + _ -> + errorValueAndIssue "`toFloat` scenario not supported" + mapFloorFunctionCall : (TypedValue, List TypedValue) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult mapFloorFunctionCall ( _, args ) mapValue ctx = case args of @@ -759,6 +841,18 @@ mapStringCaseCall (morphirName, spName) ( _, args ) mapValue ctx = _ -> errorValueAndIssue ("`" ++ morphirName ++ "` scenario not supported") +mapStartsEndsWith : (String, String) -> (TypedValue, List TypedValue) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult +mapStartsEndsWith (morphirName, spName) ( _, args ) mapValue ctx = + case args of + [ prefixOrSuffix, str ] -> + let + (mappedStr, strIssues) = mapValue str ctx + (mappedPrefixOrSuffix, prefixOrIssues) = mapValue prefixOrSuffix ctx + issues = strIssues ++ prefixOrIssues + in + (Constants.applySnowparkFunc spName [ mappedStr, mappedPrefixOrSuffix ], issues) + _ -> + errorValueAndIssue ("`" ++ morphirName ++ "` scenario not supported") mapStringReverse : (TypedValue, List TypedValue) -> Constants.MapValueType -> ValueMappingContext -> ValueGenerationResult mapStringReverse ( _, args ) mapValue ctx = diff --git a/src/Morphir/Snowpark/PatternMatchMapping.elm b/src/Morphir/Snowpark/PatternMatchMapping.elm index e18ebfee5..6ed1793a8 100644 --- a/src/Morphir/Snowpark/PatternMatchMapping.elm +++ b/src/Morphir/Snowpark/PatternMatchMapping.elm @@ -294,9 +294,9 @@ checkUnionWithParams expr cases ctx = ((WildcardPattern _, wildCardResult)::restReversed) -> (collectMaybeList checkConstructorForUnionOfWithParams restReversed) |> Maybe.map (\parts -> (UnionTypesWithParams parts (Just wildCardResult))) - ((ConstructorPattern _ _ [], _)::_) as constructorCases -> + ((ConstructorPattern _ _ _, _)::_) as constructorCases -> (collectMaybeList checkConstructorForUnionOfWithParams constructorCases) - |> Maybe.map (\parts -> (UnionTypesWithParams parts Nothing)) + |> Maybe.map (\parts -> (UnionTypesWithParams (List.reverse parts) Nothing)) _ -> Nothing else diff --git a/src/Morphir/Snowpark/ReferenceUtils.elm b/src/Morphir/Snowpark/ReferenceUtils.elm index e8e000305..790787221 100644 --- a/src/Morphir/Snowpark/ReferenceUtils.elm +++ b/src/Morphir/Snowpark/ReferenceUtils.elm @@ -8,13 +8,14 @@ module Morphir.Snowpark.ReferenceUtils exposing ( , getListTypeParameter , getFunctionInputTypes , mapLiteralToPlainLiteral - , errorValueAndIssue) + , errorValueAndIssue + , curryCall) import Morphir.IR.Name as Name import Morphir.IR.FQName as FQName import Morphir.IR.Literal exposing (Literal(..)) import Morphir.IR.Type as IrType -import Morphir.IR.Value as Value exposing (Value(..)) +import Morphir.IR.Value as Value exposing (Value(..), TypedValue) import Morphir.Scala.AST as Scala import Morphir.Snowpark.Constants as Constants import Morphir.Snowpark.MappingContext exposing (MappingContextInfo, isRecordWithSimpleTypes) @@ -101,4 +102,9 @@ getFunctionInputTypes tpe = errorValueAndIssue : GenerationIssue -> (Scala.Value, List GenerationIssue) errorValueAndIssue issue = - (Scala.Literal (Scala.StringLit issue), [ issue ]) \ No newline at end of file + (Scala.Literal (Scala.StringLit issue), [ issue ]) + +curryCall : (TypedValue, List (TypedValue)) -> TypedValue +curryCall (func, args) = + args + |> List.foldl (\arg current -> Value.Apply (Value.valueAttribute arg) current arg) func \ No newline at end of file diff --git a/src/Morphir/Snowpark/TypeRefMapping.elm b/src/Morphir/Snowpark/TypeRefMapping.elm index fbd5b5aeb..c706b53da 100644 --- a/src/Morphir/Snowpark/TypeRefMapping.elm +++ b/src/Morphir/Snowpark/TypeRefMapping.elm @@ -89,7 +89,7 @@ checkForBasicTypeToScala tpe ctx = Reference _ ([ [ "morphir" ], [ "s", "d", "k" ] ],[ [ "basics" ] ], [ "double" ]) _ -> Just <| Scala.TypeVar "Double" Reference _ ([ [ "morphir" ], [ "s", "d", "k" ] ],[ [ "basics" ] ], [ "bool" ]) _ -> - Just <| Scala.TypeVar "Bool" + Just <| Scala.TypeVar "Boolean" Reference _ ([ [ "morphir" ], [ "s", "d", "k" ] ], [ [ "string" ] ], [ "string" ]) _ -> Just <| Scala.TypeVar "String" Reference _ fullName [] -> @@ -204,6 +204,8 @@ mapTypeReferenceToBuiltinTypes tpe ctx = Scala.TypeVar "Double" Type.Reference _ ( [ [ "morphir" ], [ "s", "d", "k" ] ], [ [ "basics" ] ], [ "int" ] ) [] -> Scala.TypeVar "Int" + Type.Reference _ ( [ [ "morphir" ], [ "s", "d", "k" ] ], [ [ "string" ] ], [ "string" ] ) [] -> + Scala.TypeVar "String" Type.Reference _ fullTypeName [] -> if isTypeAlias fullTypeName ctx then resolveTypeAlias fullTypeName ctx diff --git a/src/Morphir/Snowpark/Utils.elm b/src/Morphir/Snowpark/Utils.elm index 98481d719..997fd57b0 100644 --- a/src/Morphir/Snowpark/Utils.elm +++ b/src/Morphir/Snowpark/Utils.elm @@ -26,4 +26,4 @@ collectMaybeListAux action aList current = |> Maybe.map (\newFirst -> collectMaybeListAux action rest (newFirst::current)) |> Maybe.withDefault Nothing [] -> - Just (List.reverse current) \ No newline at end of file + Just (List.reverse current)