Merge pull request #269 from nfdi4plants/python

Python 🚧
nfdi4plants · Nov 28, 2023 · 67c0f90 · 67c0f90
2 parents 53a3f12 + 2214396
commit 67c0f90
Show file tree

Hide file tree

Showing 50 changed files with 927 additions and 438 deletions.
diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json
@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "fable": {
-      "version": "4.5.0",
+      "version": "4.6.0",
       "commands": [
         "fable"
       ]

diff --git a/.gitignore b/.gitignore
@@ -355,14 +355,10 @@ paket-files/
 # Custom ignored files
 dist/
 tmp/
-/tests/FileSystem/FileSystem.Tests/js
-/tests/ISA/ISA.Json.Tests/js
-/tests/ISA/ISA.Spreadsheet.Tests/js
-/tests/ISA/ISA.Tests/js
 /tests/JavaScript/ARCtrl
 /tests/Python/ARCtrl
-/tests/ARCtrl/js
-/tests/FileSystem/js
-
-/tests/UI/cypress/screenshots/Tests/Dictionary.cy.js
 /tests/UI/ARCtrl
+/tests/**/js
+/tests/**/py
+/tests/UI/cypress/screenshots/Tests/Dictionary.cy.js
+/tests/ISA/ISA.Tests/py
diff --git a/build/TestTasks.fs b/build/TestTasks.fs
@@ -105,6 +105,16 @@ module private Helper =
 
         createProcess npmPath
 
+    let python =
+        let path =
+            match ProcessUtils.tryFindFileOnPath "python" with
+            | Some path -> path
+            | None ->
+                "py was not found in path. Please install it and make sure it's available from your path."
+                |> failwith
+
+        createProcess path
+
     let run proc arg dir =
         proc arg dir
         |> Proc.run
@@ -144,6 +154,22 @@ module RunTests =
             run npx $"mocha {path}/js --timeout 20000" ""
     }
 
+    /// <summary>
+    /// Until we reach full Py compatibility we use these paths to check only compatible projects
+    /// </summary>
+    let testProjectsPy = 
+        [
+            "tests/ISA/ISA.Tests"
+        ]
+
+    let runTestsPy = BuildTask.create "runTestsPy" [clean; build] {
+        for path in testProjectsPy do
+            //transpile py files from fsharp code
+            run dotnet $"fable {path} -o {path}/py --lang python" ""
+            // run pyxpecto in target path to execute tests in python
+            run python $"{path}/py/main.py" ""
+    }
+
     let runTestsDotnet = BuildTask.create "runTestsDotnet" [clean; build] {
         let dotnetRun = run dotnet "run"
         testProjects

diff --git a/build/release_package.json b/build/release_package.json
@@ -6,17 +6,17 @@
   "main": "index.js",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/nfdi4plants/arcAPI.git"
+    "url": "git+https://github.com/nfdi4plants/ARCtrl.git"
   },
   "author": "Heinrich Lukas Weil <[email protected]> (https://github.com/HLWeil)",
   "contributors": [
     "Kevin Frey <[email protected]> (https://github.com/Freymaurer)"
   ],
   "license": "ISC",
   "bugs": {
-    "url": "https://github.com/nfdi4plants/arcAPI/issues"
+    "url": "https://github.com/nfdi4plants/ARCtrl/issues"
   },
-  "homepage": "https://github.com/nfdi4plants/arcAPI#readme",
+  "homepage": "https://github.com/nfdi4plants/ARCtrl#readme",
   "dependencies": {
     "fable-library": "^1.1.1",
     "isomorphic-fetch": "^3.0.0"

diff --git a/docs/ARC.md b/docs/ARC.md
@@ -15,8 +15,8 @@ ARCtrl aims to provide an easy solution to create and manipulate ARCs in memory.
 
 ```fsharp
 // F#
-#r "nuget: FsSpreadsheet.ExcelIO, 4.1.0"
-#r "nuget: ARCtrl, 1.0.0-alpha9"
+#r "nuget: FsSpreadsheet.ExcelIO, 5.0.2"
+#r "nuget: ARCtrl, 1.0.0-beta.8"
 
 open ARCtrl
 
@@ -52,77 +52,28 @@ open FsSpreadsheet.ExcelIO
 
 let arcRootPath = @"path/where/you/want/the/NewTestARC"
 
-// From ARCtrl.NET
-let fulfillWriteContract basePath (c : Contract) =
-    let ensureDirectory (filePath : string) =
-        let file = new System.IO.FileInfo(filePath);
-        file.Directory.Create()
-    match c.DTO with
-    | Some (DTO.Spreadsheet wb) ->
-        let path = System.IO.Path.Combine(basePath, c.Path)
-        ensureDirectory path
-        FsWorkbook.toFile path (wb :?> FsWorkbook)
-    | Some (DTO.Text t) ->
-        let path = System.IO.Path.Combine(basePath, c.Path)
-        ensureDirectory path
-        System.IO.File.WriteAllText(path,t)
-    | None -> 
-        let path = System.IO.Path.Combine(basePath, c.Path)
-        ensureDirectory path
-        System.IO.File.Create(path).Close()
-    | _ -> 
-        printfn "Contract %s is not an ISA contract" c.Path
-
 // From ARCtrl.NET
 let write (arcPath: string) (arc:ARC) =
     arc.GetWriteContracts()
-    |> Array.iter (fulfillWriteContract arcPath)
+    // `Contracts.fulfillWriteContract` from Contracts.fsx docs
+    |> Array.iter (Contracts.fulfillWriteContract arcPath)
 
 write arcRootPath arc
 ```
 
 ```js
 // JavaScript
-import {ARC} from "@nfdi4plants/arctrl";
-import {Xlsx} from "fsspreadsheet";
-import fs from "fs";
-import path from "path";
-
-const arcRootPath = "C:/Users/Kevin/Desktop/NewTestARCJS"
-
-async function fulfillWriteContract (basePath, contract) {
-    function ensureDirectory (filePath) {
-        let dirPath = path.dirname(filePath)
-        if (!fs.existsSync(dirPath)){
-            fs.mkdirSync(dirPath, { recursive: true });
-        }
-    }
-    const p = path.join(basePath,contract.Path)
-    if (contract.Operation = "CREATE") {
-        if (contract.DTO == undefined) {
-            ensureDirectory(p)
-            fs.writeFileSync(p, "")
-        } else if (contract.DTOType == "ISA_Assay" || contract.DTOType == "ISA_Assay" || contract.DTOType == "ISA_Investigation") {
-            ensureDirectory(p)
-            await Xlsx.toFile(p, contract.DTO)
-            console.log("ISA", p)
-        } else if (contract.DTOType == "PlainText") {
-            ensureDirectory(p)
-            fs.writeFileSync(p, contract.DTO)
-        } else {
-            console.log("Warning: The given contract is not a correct ARC write contract: ", contract)
-        }
-    }
-}
+import {fulfillWriteContract} from "./Contracts.js";
 
 async function write(arcPath, arc)  {
     let contracts = arc.GetWriteContracts()
-    contracts.forEach(async contract => {
+    for (const contract of contracts) {
+        // from Contracts.js docs
         await fulfillWriteContract(arcPath,contract)
-    });
+    };
 }
 
-await write(arcRootPath,arc)
+await write(arcRootPath, arc)
 ```
 
 ## Read
@@ -136,7 +87,6 @@ Setup will be placed on top, with the actual read below.
 open System.IO
 
 // Setup
-
 let normalizePathSeparators (str:string) = str.Replace("\\","/")
 
 let getAllFilePaths (basePath: string) =
@@ -149,44 +99,80 @@ let getAllFilePaths (basePath: string) =
     )
     |> Array.ofSeq
 
-// from ARCtrl.NET
-// https://github.com/nfdi4plants/ARCtrl.NET/blob/ba3d2fabe007d9ca2c8e07b62d02ddc5264306d0/src/ARCtrl.NET/Contract.fs#L7
-let fulfillReadContract basePath (c : Contract) =
-    match c.DTOType with
-    | Some DTOType.ISA_Assay 
-    | Some DTOType.ISA_Investigation 
-    | Some DTOType.ISA_Study ->
-        let path = System.IO.Path.Combine(basePath, c.Path)
-        let wb = FsWorkbook.fromXlsxFile path |> box |> DTO.Spreadsheet
-        {c with DTO = Some wb}
-    | Some DTOType.PlainText ->
-        let path = System.IO.Path.Combine(basePath, c.Path)
-        let text = System.IO.File.ReadAllText(path) |> DTO.Text
-        {c with DTO = Some text}
-    | _ -> 
-        printfn "Contract %s is not an ISA contract" c.Path 
-        c
-
 // put it all together
 let readARC(basePath: string) =
+    // Get all file paths for a given ARC folder
     let allFilePaths = getAllFilePaths basePath
     // Initiates an ARC from FileSystem but no ISA info.
     let arcRead = ARC.fromFilePaths allFilePaths
     // Read contracts will tell us what we need to read from disc.
     let readContracts = arcRead.GetReadContracts()
     let fulfilledContracts = 
         readContracts 
-        |> Array.map (fulfillReadContract basePath)
+        // `Contracts.fulfillReadContract` from Contracts.fsx docs
+        |> Array.map (Contracts.fulfillReadContract basePath) 
     arcRead.SetISAFromContracts(fulfilledContracts)
     arcRead 
 
 // execution
-
 readARC arcRootPath
 ```
 
 ```js
+// JavaScript
+import {ARC} from "@nfdi4plants/arctrl";
+import {fulfillWriteContract, fulfillReadContract} from "./Contracts.js";
+
+// Setup
+function normalizePathSeparators (str) {
+  const normalizedPath = path.normalize(str)
+  return normalizedPath.replace(/\\/g, '/');
+}
+
+export function getAllFilePaths(basePath) {
+  const filesList = []
+  function loop (dir) {
+      const files = fs.readdirSync(dir);
+      for (const file of files) {
+          const filePath = path.join(dir, file);
+
+          if (fs.statSync(filePath).isDirectory()) {
+              // If it's a directory, recursively call the function on that directory
+              loop(filePath);
+          } else {
+              // If it's a file, calculate the relative path and add it to the list
+              const relativePath = path.relative(basePath, filePath);
+              const normalizePath = normalizePathSeparators(relativePath)
+              filesList.push(normalizePath);
+          }
+      }
+  }
+  loop(basePath)
+  return filesList;
+}
+
+// put it all together
+async function read(basePath) {
+    let allFilePaths = getAllFilePaths(basePath)
+    // Initiates an ARC from FileSystem but no ISA info.
+    let arc = ARC.fromFilePaths(allFilePaths)
+    // Read contracts will tell us what we need to read from disc.
+    let readContracts = arc.GetReadContracts()
+    console.log(readContracts)
+    let fcontracts = await Promise.all(
+        readContracts.map(async (contract) => {
+            let content = await fulfillReadContract(basePath, contract)
+            contract.DTO = content
+            return (contract) 
+        })
+    )
+    arc.SetISAFromContracts(fcontracts)
+    return arc
+}
+
+// execution
 
+await read(arcRootPath).then(arc => console.log(arc))
 ```
 
 [1]: <https://www.nuget.org/packages/ARCtrl.NET> "ARCtrl.NET Nuget"
diff --git a/docs/Contracts.md b/docs/Contracts.md
@@ -49,9 +49,16 @@ Handling contracts can be generalized in a few functions.
 
 ## WRITE contracts
 
+Here we will showcase minimal contract handling functions. These will not handle all edge cases but are sufficient to get started with!
+
+Beginning with the WRITE contracts we may check the contract objects for the `Operation` field (This step is omitted in the .NET implementation). 
+Next we need to know how to handle our DTO. In .NET this is implemented as Discriminate Union type, on which we can match. This is not possible in JS. Instead we just have any of the allowed DTO objects as value for the DTO field. Therefore, we must match on `DTOType` in js. 
+
+In both languages we must specify how spreadsheet objects and plain text objects are correctly handled.
+
 ```fsharp
-#r "nuget: FsSpreadsheet.ExcelIO, 4.1.0"
-#r "nuget: ARCtrl, 1.0.0-alpha9"
+#r "nuget: FsSpreadsheet.ExcelIO, 5.0.2"
+#r "nuget: ARCtrl, 1.0.0-beta.8"
 
 open ARCtrl
 open ARCtrl.Contract
@@ -113,3 +120,52 @@ export async function fulfillWriteContract (basePath, contract) {
 
 
 ## READ contracts
+
+READ contracts follow the same logic, with one difference. ARCtrl will give you READ contracts with None/Null values for the `DTO` field. But with the given `Path` and `DTOType` we can correctly read in the required DTO and set it on the contract. Afterwards, we return it to the correct follow-up api call in ARCtrl (this step is shown in [ARC docs](./ARC.md)).
+
+```fsharp
+// from ARCtrl.NET
+// https://github.com/nfdi4plants/ARCtrl.NET/blob/ba3d2fabe007d9ca2c8e07b62d02ddc5264306d0/src/ARCtrl.NET/Contract.fs#L7
+let fulfillReadContract basePath (c : Contract) =
+    match c.DTOType with
+    | Some DTOType.ISA_Assay 
+    | Some DTOType.ISA_Investigation 
+    | Some DTOType.ISA_Study ->
+        let path = System.IO.Path.Combine(basePath, c.Path)
+        let wb = FsWorkbook.fromXlsxFile path |> box |> DTO.Spreadsheet
+        {c with DTO = Some wb}
+    | Some DTOType.PlainText ->
+        let path: string = System.IO.Path.Combine(basePath, c.Path)
+        let text = System.IO.File.ReadAllText(path) |> DTO.Text
+        {c with DTO = Some text}
+    | _ -> 
+        printfn "Contract %s is not an ISA contract" c.Path 
+        c
+```
+
+```js
+export async function fulfillReadContract (basePath, contract) {
+  async function fulfill() {
+      const normalizedPath = normalizePathSeparators(path.join(basePath, contract.Path))
+      switch (contract.DTOType) {
+          case "ISA_Assay":
+          case "ISA_Study":
+          case "ISA_Investigation":
+              let fswb = await Xlsx.fromXlsxFile(normalizedPath)
+              return fswb
+              break;
+          case "PlainText":
+              let content = fs.readFile(normalizedPath)
+              return content
+              break;
+          default:
+              console.log(`Handling of ${contract.DTOType} in a READ contract is not yet implemented`)
+      }
+  }
+  if (contract.Operation == "READ") {
+      return await fulfill()
+  } else {
+      console.error(`Error (fulfillReadContract): "${contract}" is not a READ contract`)
+  }
+}
+```
diff --git a/docs/GettingStarted.md b/docs/GettingStarted.md
@@ -15,8 +15,8 @@ Thats it! 🎉
 For any documentation we assume using ARCtrl from a .fsx file. Verify correct setup by creating a `ARC.fsx` file with the following content
 
 ```fsharp
-#r "nuget: FsSpreadsheet.ExcelIO, 4.1.0"
-#r "nuget: ARCtrl, 1.0.0-alpha9"
+#r "nuget: FsSpreadsheet.ExcelIO, 5.0.2"
+#r "nuget: ARCtrl, 1.0.0-beta.8"
 
 open ARCtrl
 

diff --git a/docs/Tables.md b/docs/Tables.md
@@ -0,0 +1,16 @@
+# Tables
+
+🔗 The script files for this documentation can be found here:
+- [JavaScript](/scripts_js/Tables.js)
+- [F#](/scripts_fsharp/Tables.fsx)
+
+The Tables shown in xlsx files in ISA files are modelled in ARCtrl as `ArcTable` object. These objects are assigned as `ArcTables` interface in both `ArcAssay` and `ArcStudy`.
+
+`ArcTable` has the following fields:
+- `Name`: Unique (per `ArcTables`) identifier.
+- `Headers`: Array of `CompositeHeader`.
+- `Values` (might be changed in the future.): A Dictionary of key value pairs. Keys are of type `int * int`, representing `columnIndex * rowIndex` of the cell. The value is of type `CompositeCell`.
+
+`ArcTable`s are built on the system, that each "column" (built from a `CompositeHeader` and all `CompositeCell`s of the same column index) create a `CompositeColumn`, which can be translated to all types of columns in xlsx format. Each `CompositeColumn` may translate to 1, 3 or 4 xlsx columns, depending on the column type. 
+
+🔗 Check out the code examples for a minimal table manipulation example!