diff --git a/nbdev_test.py b/nbdev_test.py index 8751f23c..4ff3a090 100644 --- a/nbdev_test.py +++ b/nbdev_test.py @@ -1,10 +1,14 @@ import subprocess +import os from nbdev.doclinks import nbglob files_to_test = nbglob() +files_to_skip = ['extended_version.ipynb'] processes = [] for file in files_to_test: + if os.path.basename(file) in files_to_skip: + continue command = f"nbdev_test --path {file} --do_print" try: # Redirect stderr to stdout diff --git a/nbs/extended_version.ipynb b/nbs/extended_version.ipynb new file mode 100644 index 00000000..89f2a2da --- /dev/null +++ b/nbs/extended_version.ipynb @@ -0,0 +1,231 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Advanced IE functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Rgxlog has been enhanced with additional advanced IE functions. \n", + "To utilize these functions, specific installations are required prior to usage.
\n", + "\n", + "Rust: To download and utilize the Rust-based ie functions, execute the following code:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "enum-spanner-rs was not found on your system\n", + "installing package. this might take up to 10 minutes...\n", + "info: syncing channel updates for '1.34-x86_64-unknown-linux-gnu'\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " 1.34-x86_64-unknown-linux-gnu unchanged - rustc 1.34.2 (6c2484dc3 2019-05-13)\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "info: checking for self-update\n", + " Updating git repository `https://github.com/NNRepos/enum-spanner-rs`\n", + " Installing enum-spanner-rs v0.1.0 (https://github.com/NNRepos/enum-spanner-rs#4c8ab5b3)\n", + "error: binary `enum-spanner-rs` already exists in destination as part of `enum-spanner-rs v0.1.0 (https://github.com/NNRepos/enum-spanner-rs#4c8ab5b3)`\n", + "Add --force to overwrite\n", + "installation completed\n" + ] + } + ], + "source": [ + "#| output: false\n", + "from rgxlog.ie_func.rust_spanner_regex import download_and_install_rust_regex\n", + "download_and_install_rust_regex()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Wrapping shell-based functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Rgxlog's `rgx_string` ie function is a good example of running an external shell as part of rgxlog code,
\n", + "`rgx_string` is a rust-based ie function, we can use it only after we installed the rust package.
\n", + "This time we won't remove the built-in function - we'll just show the implementation:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "def rgx(text, regex_pattern, out_type: str):\n", + " \"\"\"\n", + " An IE function which runs regex using rust's `enum-spanner-rs` and yields tuples of strings/spans (not both).\n", + "\n", + " @param text: the string on which regex is run.\n", + " @param regex_pattern: the pattern to run.\n", + " @param out_type: string/span - decides which one will be returned.\n", + " @return: a tuple of strings/spans.\n", + " \"\"\"\n", + " with tempfile.TemporaryDirectory() as temp_dir:\n", + " rgx_temp_file_name = os.path.join(temp_dir, TEMP_FILE_NAME)\n", + " with open(rgx_temp_file_name, \"w+\") as f:\n", + " f.write(text)\n", + "\n", + " if out_type == \"string\":\n", + " rust_regex_args = rf\"{REGEX_EXE_PATH} {regex_pattern} {rgx_temp_file_name}\"\n", + " format_function = _format_spanner_string_output\n", + " elif out_type == \"span\":\n", + " rust_regex_args = rf\"{REGEX_EXE_PATH} {regex_pattern} {rgx_temp_file_name} --bytes-offset\"\n", + " format_function = _format_spanner_span_output\n", + " else:\n", + " assert False, \"illegal out_type\"\n", + "\n", + " regex_output = format_function(run_cli_command(rust_regex_args, stderr=True))\n", + "\n", + " for out in regex_output:\n", + " yield out\n", + "\n", + "def rgx_string(text, regex_pattern):\n", + " \"\"\"\n", + " @param text: The input text for the regex operation.\n", + " @param regex_pattern: the pattern of the regex operation.\n", + " @return: tuples of strings that represents the results.\n", + " \"\"\"\n", + " return rgx(text, regex_pattern, \"string\")\n", + "\n", + "RGX_STRING = dict(ie_function=rgx_string,\n", + " ie_function_name='rgx_string',\n", + " in_rel=RUST_RGX_IN_TYPES,\n", + " out_rel=rgx_string_out_type)\n", + "\n", + "# another version of these functions exists (rgx_from_file), it can be seen in the source code\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`run_cli_command` is an STDLIB function used in rgxlog, which basically runs a command using python's `Popen`.\n", + "\n", + "in order to denote regex groups, use `(?Ppattern)`. the output is in alphabetical order.\n", + "Let's run the ie function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import rgxlog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "printing results for query 'string_rel(X, Y)':\n", + " X | Y\n", + "-----+-----\n", + " a | cc\n", + " a | c\n", + " z | c\n", + "\n" + ] + } + ], + "source": [ + "%%rgxlog\n", + "text = \"zcacc\"\n", + "pattern = \"(?P[^c]+)(?P[c]+)\"\n", + "string_rel(X,Y) <- rgx_string(text, pattern) -> (X,Y)\n", + "?string_rel(X,Y)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Similarly, to use nlp-based ie functions you need to first install nlp:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from rgxlog.ie_func.nlp import download_and_install_nlp\n", + "download_and_install_nlp()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "printing results for query 'tokens(Token, Span)':\n", + " Token | Span\n", + "---------+----------\n", + " Hello | [0, 5)\n", + " world | [6, 11)\n", + " . | [11, 12)\n", + " Hello | [13, 18)\n", + " world | [19, 24)\n", + " again | [25, 30)\n", + " . | [30, 31)\n", + "\n" + ] + } + ], + "source": [ + "%%rgxlog\n", + "sentence = \"Hello world. Hello world again.\"\n", + "tokens(X, Y) <- Tokenize(sentence) -> (X, Y)\n", + "?tokens(Token, Span)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/nbs/ie_func/04b_nlp.ipynb b/nbs/ie_func/04b_nlp.ipynb index a3af75b7..8dcb4e67 100644 --- a/nbs/ie_func/04b_nlp.ipynb +++ b/nbs/ie_func/04b_nlp.ipynb @@ -59,16 +59,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/nbdev_spanner_workbench/spanner_workbench/src/rgxlog_interpreter/src/rgxlog/stdlib/stanford-corenlp-4.1.0\n", - "/nbdev_spanner_workbench/spanner_workbench/src/rgxlog_interpreter/src/rgxlog/stdlib/stanford-corenlp-4.1.0\n" - ] - } - ], + "outputs": [], "source": [ "#| export\n", "JAVA_MIN_VERSION = 1.8\n", @@ -171,12 +162,14 @@ "outputs": [], "source": [ "#| export\n", - "#| eval: false\n", - "try:\n", - " _run_installation()\n", - " CoreNLPEngine = StanfordCoreNLP(NLP_DIR_PATH)\n", - "except:\n", - " logger.error(\"Installation NLP failed\")" + "CoreNLPEngine = None\n", + "def download_and_install_nlp():\n", + " global CoreNLPEngine\n", + " try:\n", + " _run_installation()\n", + " CoreNLPEngine = StanfordCoreNLP(NLP_DIR_PATH)\n", + " except:\n", + " logger.error(\"Installation NLP failed\")" ] }, { diff --git a/nbs/ie_func/04d_rust_spanner_regex.ipynb b/nbs/ie_func/04d_rust_spanner_regex.ipynb index 0e5a8073..07f96091 100644 --- a/nbs/ie_func/04d_rust_spanner_regex.ipynb +++ b/nbs/ie_func/04d_rust_spanner_regex.ipynb @@ -224,7 +224,19 @@ "source": [ "#| export\n", "#| hide\n", - "def _download_and_install_rust_regex() -> None:\n", + "def _is_installed_package() -> bool:\n", + " return Path(REGEX_EXE_PATH).is_file()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "#| hide\n", + "def download_and_install_rust_regex() -> None:\n", " # don't use \"cargo -V\" because it starts downloading stuff sometimes\n", " with Popen([WHICH_WORD, \"cargo\"], stdout=PIPE, stderr=PIPE) as cargo:\n", " errcode = cargo.wait(SHORT_TIMEOUT)\n", @@ -251,18 +263,6 @@ " logger.warning(\"installation completed\")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#| export\n", - "#| hide\n", - "def _is_installed_package() -> bool:\n", - " return Path(REGEX_EXE_PATH).is_file()" - ] - }, { "cell_type": "code", "execution_count": null, @@ -478,23 +478,6 @@ " in_rel=RUST_RGX_IN_TYPES,\n", " out_rel=rgx_string_out_type)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#| hide\n", - "#| export\n", - "#| eval: false\n", - "# the package is installed when this module is imported, exclude the scenraio of github runner\n", - "try:\n", - " if not _is_installed_package() and 'runner' not in os.getcwd():\n", - " _download_and_install_rust_regex()\n", - "except:\n", - " logger.error(f\"cargo or rustup are not installed in $PATH. please install rust: {DOWNLOAD_RUST_URL}\")" - ] } ], "metadata": { diff --git a/nbs/introduction.ipynb b/nbs/introduction.ipynb index 8fea219e..dd80a7a3 100644 --- a/nbs/introduction.ipynb +++ b/nbs/introduction.ipynb @@ -85,152 +85,153 @@ "execution_count": null, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "fatal: destination path 'spanner_workbench' already exists and is not an empty directory.\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "Processing c:\\users\\kloay\\desktop\\prs\\open_introduction_in_colab\\nbs\\spanner_workbench\n", - " Preparing metadata (setup.py): started\n", - " Preparing metadata (setup.py): finished with status 'done'\n", - "Requirement already satisfied: nbdev in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (2.3.12)\n", - "Requirement already satisfied: pandas in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (2.0.1)\n", - "Requirement already satisfied: notebook in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (6.5.4)\n", - "Requirement already satisfied: pyDatalog in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (0.17.4)\n", - "Requirement already satisfied: tabulate in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (0.9.0)\n", - "Requirement already satisfied: lark-parser>=0.9.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (0.12.0)\n", - "Requirement already satisfied: ipython>=7.18.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (8.13.0)\n", - "Requirement already satisfied: setuptools>=50.2.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (65.5.0)\n", - "Requirement already satisfied: networkx>=2.5 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (3.1)\n", - "Requirement already satisfied: docopt>=0.6.2 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (0.6.2)\n", - "Requirement already satisfied: jsonpath-ng in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (1.5.3)\n", - "Requirement already satisfied: psutil in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (5.9.5)\n", - "Requirement already satisfied: install-jdk in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (1.1.0)\n", - "Requirement already satisfied: parse in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (1.19.1)\n", - "Requirement already satisfied: spanner-nlp in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (0.0.6)\n", - "Requirement already satisfied: pytest in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (7.3.1)\n", - "Requirement already satisfied: Jinja2 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (3.1.2)\n", - "Requirement already satisfied: pycodestyle in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (2.11.0)\n", - "Requirement already satisfied: mypy in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-workbench==0.0.1) (1.5.1)\n", - "Requirement already satisfied: backcall in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (0.2.0)\n", - "Requirement already satisfied: decorator in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (5.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (0.18.2)\n", - "Requirement already satisfied: matplotlib-inline in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (0.1.6)\n", - "Requirement already satisfied: pickleshare in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (0.7.5)\n", - "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (3.0.38)\n", - "Requirement already satisfied: pygments>=2.4.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (2.15.1)\n", - "Requirement already satisfied: stack-data in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (0.6.2)\n", - "Requirement already satisfied: traitlets>=5 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (5.9.0)\n", - "Requirement already satisfied: colorama in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipython>=7.18.1->spanner-workbench==0.0.1) (0.4.6)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from Jinja2->spanner-workbench==0.0.1) (2.1.2)\n", - "Requirement already satisfied: ply in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonpath-ng->spanner-workbench==0.0.1) (3.11)\n", - "Requirement already satisfied: six in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonpath-ng->spanner-workbench==0.0.1) (1.16.0)\n", - "Requirement already satisfied: typing-extensions>=4.1.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from mypy->spanner-workbench==0.0.1) (4.7.1)\n", - "Requirement already satisfied: mypy-extensions>=1.0.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from mypy->spanner-workbench==0.0.1) (1.0.0)\n", - "Requirement already satisfied: fastcore>=1.5.27 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbdev->spanner-workbench==0.0.1) (1.5.29)\n", - "Requirement already satisfied: execnb>=0.1.4 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbdev->spanner-workbench==0.0.1) (0.1.5)\n", - "Requirement already satisfied: astunparse in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbdev->spanner-workbench==0.0.1) (1.6.3)\n", - "Requirement already satisfied: ghapi>=1.0.3 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbdev->spanner-workbench==0.0.1) (1.0.3)\n", - "Requirement already satisfied: watchdog in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbdev->spanner-workbench==0.0.1) (3.0.0)\n", - "Requirement already satisfied: asttokens in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbdev->spanner-workbench==0.0.1) (2.2.1)\n", - "Requirement already satisfied: PyYAML in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbdev->spanner-workbench==0.0.1) (6.0)\n", - "Requirement already satisfied: tornado>=6.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (6.3.1)\n", - "Requirement already satisfied: pyzmq>=17 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (25.0.2)\n", - "Requirement already satisfied: argon2-cffi in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (21.3.0)\n", - "Requirement already satisfied: jupyter-core>=4.6.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (5.3.0)\n", - "Requirement already satisfied: jupyter-client>=5.3.4 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (8.2.0)\n", - "Requirement already satisfied: ipython-genutils in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (0.2.0)\n", - "Requirement already satisfied: nbformat in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (5.8.0)\n", - "Requirement already satisfied: nbconvert>=5 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (7.3.1)\n", - "Requirement already satisfied: nest-asyncio>=1.5 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (1.5.6)\n", - "Requirement already satisfied: ipykernel in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (6.22.0)\n", - "Requirement already satisfied: Send2Trash>=1.8.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (1.8.2)\n", - "Requirement already satisfied: terminado>=0.8.3 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (0.17.1)\n", - "Requirement already satisfied: prometheus-client in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (0.16.0)\n", - "Requirement already satisfied: nbclassic>=0.4.7 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from notebook->spanner-workbench==0.0.1) (0.5.6)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->spanner-workbench==0.0.1) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->spanner-workbench==0.0.1) (2023.3)\n", - "Requirement already satisfied: tzdata>=2022.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->spanner-workbench==0.0.1) (2023.3)\n", - "Requirement already satisfied: numpy>=1.21.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas->spanner-workbench==0.0.1) (1.24.3)\n", - "Requirement already satisfied: iniconfig in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pytest->spanner-workbench==0.0.1) (2.0.0)\n", - "Requirement already satisfied: packaging in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pytest->spanner-workbench==0.0.1) (23.1)\n", - "Requirement already satisfied: pluggy<2.0,>=0.12 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pytest->spanner-workbench==0.0.1) (1.0.0)\n", - "Requirement already satisfied: requests in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spanner-nlp->spanner-workbench==0.0.1) (2.30.0)\n", - "Requirement already satisfied: pip in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from fastcore>=1.5.27->nbdev->spanner-workbench==0.0.1) (23.1.2)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jedi>=0.16->ipython>=7.18.1->spanner-workbench==0.0.1) (0.8.3)\n", - "Requirement already satisfied: platformdirs>=2.5 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-core>=4.6.1->notebook->spanner-workbench==0.0.1) (3.10.0)\n", - "Requirement already satisfied: pywin32>=300 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-core>=4.6.1->notebook->spanner-workbench==0.0.1) (306)\n", - "Requirement already satisfied: jupyter-server>=1.8 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (2.5.0)\n", - "Requirement already satisfied: notebook-shim>=0.2.3 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (0.2.3)\n", - "Requirement already satisfied: beautifulsoup4 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbconvert>=5->notebook->spanner-workbench==0.0.1) (4.12.2)\n", - "Requirement already satisfied: bleach in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbconvert>=5->notebook->spanner-workbench==0.0.1) (6.0.0)\n", - "Requirement already satisfied: defusedxml in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbconvert>=5->notebook->spanner-workbench==0.0.1) (0.7.1)\n", - "Requirement already satisfied: jupyterlab-pygments in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbconvert>=5->notebook->spanner-workbench==0.0.1) (0.2.2)\n", - "Requirement already satisfied: mistune<3,>=2.0.3 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbconvert>=5->notebook->spanner-workbench==0.0.1) (2.0.5)\n", - "Requirement already satisfied: nbclient>=0.5.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbconvert>=5->notebook->spanner-workbench==0.0.1) (0.7.4)\n", - "Requirement already satisfied: pandocfilters>=1.4.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbconvert>=5->notebook->spanner-workbench==0.0.1) (1.5.0)\n", - "Requirement already satisfied: tinycss2 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbconvert>=5->notebook->spanner-workbench==0.0.1) (1.2.1)\n", - "Requirement already satisfied: fastjsonschema in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbformat->notebook->spanner-workbench==0.0.1) (2.16.3)\n", - "Requirement already satisfied: jsonschema>=2.6 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from nbformat->notebook->spanner-workbench==0.0.1) (4.17.3)\n", - "Requirement already satisfied: wcwidth in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=7.18.1->spanner-workbench==0.0.1) (0.2.6)\n", - "Requirement already satisfied: pywinpty>=1.1.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from terminado>=0.8.3->notebook->spanner-workbench==0.0.1) (2.0.10)\n", - "Requirement already satisfied: argon2-cffi-bindings in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from argon2-cffi->notebook->spanner-workbench==0.0.1) (21.2.0)\n", - "Requirement already satisfied: wheel<1.0,>=0.23.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from astunparse->nbdev->spanner-workbench==0.0.1) (0.40.0)\n", - "Requirement already satisfied: comm>=0.1.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipykernel->notebook->spanner-workbench==0.0.1) (0.1.3)\n", - "Requirement already satisfied: debugpy>=1.6.5 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ipykernel->notebook->spanner-workbench==0.0.1) (1.6.7)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests->spanner-nlp->spanner-workbench==0.0.1) (3.1.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests->spanner-nlp->spanner-workbench==0.0.1) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests->spanner-nlp->spanner-workbench==0.0.1) (2.0.2)\n", - "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests->spanner-nlp->spanner-workbench==0.0.1) (2023.5.7)\n", - "Requirement already satisfied: executing>=1.2.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stack-data->ipython>=7.18.1->spanner-workbench==0.0.1) (1.2.0)\n", - "Requirement already satisfied: pure-eval in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from stack-data->ipython>=7.18.1->spanner-workbench==0.0.1) (0.2.2)\n", - "Requirement already satisfied: attrs>=17.4.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonschema>=2.6->nbformat->notebook->spanner-workbench==0.0.1) (23.1.0)\n", - "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonschema>=2.6->nbformat->notebook->spanner-workbench==0.0.1) (0.19.3)\n", - "Requirement already satisfied: anyio>=3.1.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (3.6.2)\n", - "Requirement already satisfied: jupyter-events>=0.4.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (0.6.3)\n", - "Requirement already satisfied: jupyter-server-terminals in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (0.4.4)\n", - "Requirement already satisfied: websocket-client in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (1.5.1)\n", - "Requirement already satisfied: cffi>=1.0.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from argon2-cffi-bindings->argon2-cffi->notebook->spanner-workbench==0.0.1) (1.15.1)\n", - "Requirement already satisfied: soupsieve>1.2 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from beautifulsoup4->nbconvert>=5->notebook->spanner-workbench==0.0.1) (2.4.1)\n", - "Requirement already satisfied: webencodings in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from bleach->nbconvert>=5->notebook->spanner-workbench==0.0.1) (0.5.1)\n", - "Requirement already satisfied: sniffio>=1.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from anyio>=3.1.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (1.3.0)\n", - "Requirement already satisfied: pycparser in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook->spanner-workbench==0.0.1) (2.21)\n", - "Requirement already satisfied: python-json-logger>=2.0.4 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-events>=0.4.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (2.0.7)\n", - "Requirement already satisfied: rfc3339-validator in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-events>=0.4.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (0.1.4)\n", - "Requirement already satisfied: rfc3986-validator>=0.1.1 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jupyter-events>=0.4.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook->spanner-workbench==0.0.1) (0.1.1)\n", - "Requirement already satisfied: fqdn in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonschema>=2.6->nbformat->notebook->spanner-workbench==0.0.1) (1.5.1)\n", - "Requirement already satisfied: isoduration in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonschema>=2.6->nbformat->notebook->spanner-workbench==0.0.1) (20.11.0)\n", - "Requirement already satisfied: jsonpointer>1.13 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonschema>=2.6->nbformat->notebook->spanner-workbench==0.0.1) (2.3)\n", - "Requirement already satisfied: uri-template in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonschema>=2.6->nbformat->notebook->spanner-workbench==0.0.1) (1.2.0)\n", - "Requirement already satisfied: webcolors>=1.11 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jsonschema>=2.6->nbformat->notebook->spanner-workbench==0.0.1) (1.13)\n", - "Requirement already satisfied: arrow>=0.15.0 in c:\\users\\kloay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from isoduration->jsonschema>=2.6->nbformat->notebook->spanner-workbench==0.0.1) (1.2.3)\n", - "Building wheels for collected packages: spanner-workbench\n", - " Building wheel for spanner-workbench (setup.py): started\n", - " Building wheel for spanner-workbench (setup.py): finished with status 'done'\n", - " Created wheel for spanner-workbench: filename=spanner_workbench-0.0.1-py3-none-any.whl size=88848 sha256=6198f438d040931e69cbffef5c06c7de66aa2eb2f389a831d51e5c2c665832a7\n", - " Stored in directory: C:\\Users\\kloay\\AppData\\Local\\Temp\\pip-ephem-wheel-cache-has0f_2n\\wheels\\40\\17\\67\\f6f19c1c93229a6358a65e59b2e3247268dfeea4c32c58d650\n", - "Successfully built spanner-workbench\n", - "Installing collected packages: spanner-workbench\n", - " Attempting uninstall: spanner-workbench\n", - " Found existing installation: spanner-workbench 0.0.1\n", - " Uninstalling spanner-workbench-0.0.1:\n", - " Successfully uninstalled spanner-workbench-0.0.1\n", - "Successfully installed spanner-workbench-0.0.1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "[notice] A new release of pip is available: 23.1.2 -> 23.3\n", - "[notice] To update, run: python.exe -m pip install --upgrade pip\n" + "fatal: destination path 'spanner_workbench' already exists and is not an empty directory.\n", + "Processing /edit_intro_based_on_reviews/nbs/spanner_workbench\n", + " Preparing metadata (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25hRequirement already satisfied: nbdev in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (2.3.13)\n", + "Requirement already satisfied: pandas in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (2.0.3)\n", + "Requirement already satisfied: notebook in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (7.0.6)\n", + "Requirement already satisfied: pyDatalog in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (0.17.4)\n", + "Requirement already satisfied: tabulate in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (0.9.0)\n", + "Requirement already satisfied: lark-parser>=0.9.0 in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (0.12.0)\n", + "Requirement already satisfied: ipython>=7.18.1 in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (8.12.3)\n", + "Requirement already satisfied: setuptools>=50.2.0 in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (65.6.3)\n", + "Requirement already satisfied: networkx>=2.5 in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (3.1)\n", + "Requirement already satisfied: docopt>=0.6.2 in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (0.6.2)\n", + "Requirement already satisfied: jsonpath-ng in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (1.6.0)\n", + "Requirement already satisfied: psutil in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (5.9.6)\n", + "Requirement already satisfied: install-jdk in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (1.1.0)\n", + "Requirement already satisfied: parse in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (1.19.1)\n", + "Requirement already satisfied: spanner-nlp in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (0.0.6)\n", + "Requirement already satisfied: pytest in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (7.4.3)\n", + "Requirement already satisfied: Jinja2 in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (3.1.2)\n", + "Requirement already satisfied: pycodestyle in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (2.11.1)\n", + "Requirement already satisfied: mypy in /miniconda/lib/python3.8/site-packages (from rgxlog==0.0.1) (1.6.1)\n", + "Requirement already satisfied: pickleshare in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (0.7.5)\n", + "Requirement already satisfied: traitlets>=5 in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (5.12.0)\n", + "Requirement already satisfied: matplotlib-inline in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (0.1.6)\n", + "Requirement already satisfied: backcall in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (0.2.0)\n", + "Requirement already satisfied: typing-extensions in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (4.8.0)\n", + "Requirement already satisfied: decorator in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (0.19.1)\n", + "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (3.0.39)\n", + "Requirement already satisfied: pexpect>4.3 in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (4.8.0)\n", + "Requirement already satisfied: pygments>=2.4.0 in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (2.16.1)\n", + "Requirement already satisfied: stack-data in /miniconda/lib/python3.8/site-packages (from ipython>=7.18.1->rgxlog==0.0.1) (0.6.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /miniconda/lib/python3.8/site-packages (from Jinja2->rgxlog==0.0.1) (2.1.3)\n", + "Requirement already satisfied: ply in /miniconda/lib/python3.8/site-packages (from jsonpath-ng->rgxlog==0.0.1) (3.11)\n", + "Requirement already satisfied: mypy-extensions>=1.0.0 in /miniconda/lib/python3.8/site-packages (from mypy->rgxlog==0.0.1) (1.0.0)\n", + "Requirement already satisfied: tomli>=1.1.0 in /miniconda/lib/python3.8/site-packages (from mypy->rgxlog==0.0.1) (2.0.1)\n", + "Requirement already satisfied: watchdog in /miniconda/lib/python3.8/site-packages (from nbdev->rgxlog==0.0.1) (3.0.0)\n", + "Requirement already satisfied: execnb>=0.1.4 in /miniconda/lib/python3.8/site-packages (from nbdev->rgxlog==0.0.1) (0.1.5)\n", + "Requirement already satisfied: asttokens in /miniconda/lib/python3.8/site-packages (from nbdev->rgxlog==0.0.1) (2.4.1)\n", + "Requirement already satisfied: PyYAML in /miniconda/lib/python3.8/site-packages (from nbdev->rgxlog==0.0.1) (6.0.1)\n", + "Requirement already satisfied: ghapi>=1.0.3 in /miniconda/lib/python3.8/site-packages (from nbdev->rgxlog==0.0.1) (1.0.4)\n", + "Requirement already satisfied: fastcore>=1.5.27 in /miniconda/lib/python3.8/site-packages (from nbdev->rgxlog==0.0.1) (1.5.29)\n", + "Requirement already satisfied: ipywidgets<=8.0.4 in /miniconda/lib/python3.8/site-packages (from nbdev->rgxlog==0.0.1) (8.0.4)\n", + "Requirement already satisfied: astunparse in /miniconda/lib/python3.8/site-packages (from nbdev->rgxlog==0.0.1) (1.6.3)\n", + "Requirement already satisfied: jupyterlab<5,>=4.0.2 in /miniconda/lib/python3.8/site-packages (from notebook->rgxlog==0.0.1) (4.0.7)\n", + "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /miniconda/lib/python3.8/site-packages (from notebook->rgxlog==0.0.1) (2.9.1)\n", + "Requirement already satisfied: jupyterlab-server<3,>=2.22.1 in /miniconda/lib/python3.8/site-packages (from notebook->rgxlog==0.0.1) (2.25.0)\n", + "Requirement already satisfied: notebook-shim<0.3,>=0.2 in /miniconda/lib/python3.8/site-packages (from notebook->rgxlog==0.0.1) (0.2.3)\n", + "Requirement already satisfied: tornado>=6.2.0 in /miniconda/lib/python3.8/site-packages (from notebook->rgxlog==0.0.1) (6.3.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /miniconda/lib/python3.8/site-packages (from pandas->rgxlog==0.0.1) (2.8.2)\n", + "Requirement already satisfied: numpy>=1.20.3 in /miniconda/lib/python3.8/site-packages (from pandas->rgxlog==0.0.1) (1.24.4)\n", + "Requirement already satisfied: tzdata>=2022.1 in /miniconda/lib/python3.8/site-packages (from pandas->rgxlog==0.0.1) (2023.3)\n", + "Requirement already satisfied: pytz>=2020.1 in /miniconda/lib/python3.8/site-packages (from pandas->rgxlog==0.0.1) (2023.3.post1)\n", + "Requirement already satisfied: pluggy<2.0,>=0.12 in /miniconda/lib/python3.8/site-packages (from pytest->rgxlog==0.0.1) (1.0.0)\n", + "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /miniconda/lib/python3.8/site-packages (from pytest->rgxlog==0.0.1) (1.1.3)\n", + "Requirement already satisfied: packaging in /miniconda/lib/python3.8/site-packages (from pytest->rgxlog==0.0.1) (22.0)\n", + "Requirement already satisfied: iniconfig in /miniconda/lib/python3.8/site-packages (from pytest->rgxlog==0.0.1) (2.0.0)\n", + "Requirement already satisfied: requests in /miniconda/lib/python3.8/site-packages (from spanner-nlp->rgxlog==0.0.1) (2.31.0)\n", + "Requirement already satisfied: pip in /miniconda/lib/python3.8/site-packages (from fastcore>=1.5.27->nbdev->rgxlog==0.0.1) (22.3.1)\n", + "Requirement already satisfied: ipykernel>=4.5.1 in /miniconda/lib/python3.8/site-packages (from ipywidgets<=8.0.4->nbdev->rgxlog==0.0.1) (6.26.0)\n", + "Requirement already satisfied: jupyterlab-widgets~=3.0 in /miniconda/lib/python3.8/site-packages (from ipywidgets<=8.0.4->nbdev->rgxlog==0.0.1) (3.0.9)\n", + "Requirement already satisfied: widgetsnbextension~=4.0 in /miniconda/lib/python3.8/site-packages (from ipywidgets<=8.0.4->nbdev->rgxlog==0.0.1) (4.0.9)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /miniconda/lib/python3.8/site-packages (from jedi>=0.16->ipython>=7.18.1->rgxlog==0.0.1) (0.8.3)\n", + "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (5.4.0)\n", + "Requirement already satisfied: nbformat>=5.3.0 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (5.9.2)\n", + "Requirement already satisfied: anyio>=3.1.0 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (4.0.0)\n", + "Requirement already satisfied: jupyter-client>=7.4.4 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (8.5.0)\n", + "Requirement already satisfied: pyzmq>=24 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (25.1.1)\n", + "Requirement already satisfied: overrides in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (7.4.0)\n", + "Requirement already satisfied: websocket-client in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (1.6.4)\n", + "Requirement already satisfied: jupyter-server-terminals in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.4.4)\n", + "Requirement already satisfied: prometheus-client in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.17.1)\n", + "Requirement already satisfied: nbconvert>=6.4.4 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (7.9.2)\n", + "Requirement already satisfied: send2trash>=1.8.2 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (1.8.2)\n", + "Requirement already satisfied: jupyter-events>=0.6.0 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.8.0)\n", + "Requirement already satisfied: argon2-cffi in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (23.1.0)\n", + "Requirement already satisfied: terminado>=0.8.3 in /miniconda/lib/python3.8/site-packages (from jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.17.1)\n", + "Requirement already satisfied: importlib-resources>=1.4 in /miniconda/lib/python3.8/site-packages (from jupyterlab<5,>=4.0.2->notebook->rgxlog==0.0.1) (6.1.0)\n", + "Requirement already satisfied: importlib-metadata>=4.8.3 in /miniconda/lib/python3.8/site-packages (from jupyterlab<5,>=4.0.2->notebook->rgxlog==0.0.1) (6.8.0)\n", + "Requirement already satisfied: async-lru>=1.0.0 in /miniconda/lib/python3.8/site-packages (from jupyterlab<5,>=4.0.2->notebook->rgxlog==0.0.1) (2.0.4)\n", + "Requirement already satisfied: jupyter-lsp>=2.0.0 in /miniconda/lib/python3.8/site-packages (from jupyterlab<5,>=4.0.2->notebook->rgxlog==0.0.1) (2.2.0)\n", + "Requirement already satisfied: json5>=0.9.0 in /miniconda/lib/python3.8/site-packages (from jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (0.9.14)\n", + "Requirement already satisfied: jsonschema>=4.18.0 in /miniconda/lib/python3.8/site-packages (from jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (4.19.1)\n", + "Requirement already satisfied: babel>=2.10 in /miniconda/lib/python3.8/site-packages (from jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (2.13.1)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /miniconda/lib/python3.8/site-packages (from pexpect>4.3->ipython>=7.18.1->rgxlog==0.0.1) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /miniconda/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=7.18.1->rgxlog==0.0.1) (0.2.8)\n", + "Requirement already satisfied: six>=1.5 in /miniconda/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->rgxlog==0.0.1) (1.16.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /miniconda/lib/python3.8/site-packages (from requests->spanner-nlp->rgxlog==0.0.1) (2.0.4)\n", + "Requirement already satisfied: idna<4,>=2.5 in /miniconda/lib/python3.8/site-packages (from requests->spanner-nlp->rgxlog==0.0.1) (3.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /miniconda/lib/python3.8/site-packages (from requests->spanner-nlp->rgxlog==0.0.1) (2022.12.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /miniconda/lib/python3.8/site-packages (from requests->spanner-nlp->rgxlog==0.0.1) (1.25.8)\n", + "Requirement already satisfied: wheel<1.0,>=0.23.0 in /miniconda/lib/python3.8/site-packages (from astunparse->nbdev->rgxlog==0.0.1) (0.37.1)\n", + "Requirement already satisfied: executing>=1.2.0 in /miniconda/lib/python3.8/site-packages (from stack-data->ipython>=7.18.1->rgxlog==0.0.1) (2.0.1)\n", + "Requirement already satisfied: pure-eval in /miniconda/lib/python3.8/site-packages (from stack-data->ipython>=7.18.1->rgxlog==0.0.1) (0.2.2)\n", + "Requirement already satisfied: sniffio>=1.1 in /miniconda/lib/python3.8/site-packages (from anyio>=3.1.0->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (1.3.0)\n", + "Requirement already satisfied: zipp>=0.5 in /miniconda/lib/python3.8/site-packages (from importlib-metadata>=4.8.3->jupyterlab<5,>=4.0.2->notebook->rgxlog==0.0.1) (3.17.0)\n", + "Requirement already satisfied: comm>=0.1.1 in /miniconda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets<=8.0.4->nbdev->rgxlog==0.0.1) (0.1.4)\n", + "Requirement already satisfied: debugpy>=1.6.5 in /miniconda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets<=8.0.4->nbdev->rgxlog==0.0.1) (1.8.0)\n", + "Requirement already satisfied: nest-asyncio in /miniconda/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets<=8.0.4->nbdev->rgxlog==0.0.1) (1.5.8)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (0.10.6)\n", + "Requirement already satisfied: referencing>=0.28.4 in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (0.30.2)\n", + "Requirement already satisfied: pkgutil-resolve-name>=1.3.10 in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (1.3.10)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (2023.7.1)\n", + "Requirement already satisfied: attrs>=22.2.0 in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (23.1.0)\n", + "Requirement already satisfied: platformdirs>=2.5 in /miniconda/lib/python3.8/site-packages (from jupyter-core!=5.0.*,>=4.12->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (3.11.0)\n", + "Requirement already satisfied: rfc3339-validator in /miniconda/lib/python3.8/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.1.4)\n", + "Requirement already satisfied: python-json-logger>=2.0.4 in /miniconda/lib/python3.8/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (2.0.7)\n", + "Requirement already satisfied: rfc3986-validator>=0.1.1 in /miniconda/lib/python3.8/site-packages (from jupyter-events>=0.6.0->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.1.1)\n", + "Requirement already satisfied: bleach!=5.0.0 in /miniconda/lib/python3.8/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (6.1.0)\n", + "Requirement already satisfied: jupyterlab-pygments in /miniconda/lib/python3.8/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.2.2)\n", + "Requirement already satisfied: defusedxml in /miniconda/lib/python3.8/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.7.1)\n", + "Requirement already satisfied: tinycss2 in /miniconda/lib/python3.8/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (1.2.1)\n", + "Requirement already satisfied: mistune<4,>=2.0.3 in /miniconda/lib/python3.8/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (3.0.2)\n", + "Requirement already satisfied: beautifulsoup4 in /miniconda/lib/python3.8/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (4.12.2)\n", + "Requirement already satisfied: nbclient>=0.5.0 in /miniconda/lib/python3.8/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.8.0)\n", + "Requirement already satisfied: pandocfilters>=1.4.1 in /miniconda/lib/python3.8/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (1.5.0)\n", + "Requirement already satisfied: fastjsonschema in /miniconda/lib/python3.8/site-packages (from nbformat>=5.3.0->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (2.18.1)\n", + "Requirement already satisfied: argon2-cffi-bindings in /miniconda/lib/python3.8/site-packages (from argon2-cffi->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (21.2.0)\n", + "Requirement already satisfied: webencodings in /miniconda/lib/python3.8/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (0.5.1)\n", + "Requirement already satisfied: uri-template in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (1.3.0)\n", + "Requirement already satisfied: fqdn in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (1.5.1)\n", + "Requirement already satisfied: isoduration in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (20.11.0)\n", + "Requirement already satisfied: jsonpointer>1.13 in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (2.1)\n", + "Requirement already satisfied: webcolors>=1.11 in /miniconda/lib/python3.8/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (1.13)\n", + "Requirement already satisfied: cffi>=1.0.1 in /miniconda/lib/python3.8/site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (1.15.1)\n", + "Requirement already satisfied: soupsieve>1.2 in /miniconda/lib/python3.8/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (2.5)\n", + "Requirement already satisfied: pycparser in /miniconda/lib/python3.8/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<3,>=2.4.0->notebook->rgxlog==0.0.1) (2.21)\n", + "Requirement already satisfied: arrow>=0.15.0 in /miniconda/lib/python3.8/site-packages (from isoduration->jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (1.3.0)\n", + "Requirement already satisfied: types-python-dateutil>=2.8.10 in /miniconda/lib/python3.8/site-packages (from arrow>=0.15.0->isoduration->jsonschema>=4.18.0->jupyterlab-server<3,>=2.22.1->notebook->rgxlog==0.0.1) (2.8.19.14)\n", + "Building wheels for collected packages: rgxlog\n", + " Building wheel for rgxlog (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for rgxlog: filename=rgxlog-0.0.1-py3-none-any.whl size=81151 sha256=b8979051a6250843826d263504502adebb3c94fae655aa360d6ec6a77c5cfefb\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-ni2euz_q/wheels/0c/c3/f1/614798b4fea9e9e48774f9b2b64aa9d1ea29cd772d6f305203\n", + "Successfully built rgxlog\n", + "Installing collected packages: rgxlog\n", + " Attempting uninstall: rgxlog\n", + " Found existing installation: rgxlog 0.0.1\n", + " Uninstalling rgxlog-0.0.1:\n", + " Successfully uninstalled rgxlog-0.0.1\n", + "Successfully installed rgxlog-0.0.1\n", + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n", + "\u001b[0m" ] } ], @@ -253,16 +254,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Installation NLP failed\n", - "cargo or rustup are not installed in $PATH. please install rust: https://rustup.rs/\n" - ] - } - ], + "outputs": [], "source": [ "#| output: false\n", "import rgxlog # or `load_ext rgxlog`" @@ -463,6 +455,14 @@ "new scores(str, int)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Whenever a relation is defined, a corresponding empty table is created in the database.
\n", + "The table has the same name as the relation, and its number of columns is equal to the number of variables in the relation." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -520,8 +520,20 @@ "goals(\"kronovi\", 10)\n", "goals(\"kronovi\", 10) <- False # 'goals' relation is now empty\n", "goals(\"kronovi\", 10) <- False # this statement does nothing\n", - "```\n", - "\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When adding or removing facts from a relation, the relation's corresponding table in the database gets updated respectively" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "# Rules\n", "Datalog allows you to deduce new tuples for a relation.\n", "RGXLog includes this feature as well:" @@ -803,7 +815,11 @@ "metadata": {}, "source": [ "# Queries\n", - "Querying is very simple in RGXLog. You can query by using constant values, local variables and free variables:" + "A query is essentially a way to retrieve specific information from a dataset.
\n", + "querying in rgxlog uses the same synatx and semantics as DataLog.
\n", + "Under said semantics, we try to find all instantiations of free variables that satisfy the queried relation.\n", + "\n", + "You can query by using constant values, local variables and free variables:" ] }, { @@ -905,7 +921,8 @@ "```\n", "\n", "returns an empty tuple. This is because of the fact that bob is alice's grandfather is true,\n", - "but we did not use any free variables to construct the tuple of the query's relation, that is why we get a single empty tuple as a result\n", + "our query has no free variables, which means it asks a specific factual question about the dataset. If the query is true, it means the specified condition exists in the dataset. If false, it means the condition does not exist.\n", + "And this is why if we have a query with no free variables, we get an empty set of instantiations if its true and no such set if its false.\n", "\n", "A good example for using free variables to construct a relation is the query:\n", "\n", @@ -916,6 +933,15 @@ "which finds all of george's grandchildren (`X`) and constructs a tuple for each one." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How Rules and Queries are saved in the database?\n", + "\n", + "Unlike facts, which are immediately stored in the database upon their creation, rules are not computed and added to the database upon declaration. Instead, the logic of a rule is saved separately and is only evaluated when needed (lazy evaluation). When a query is made, the engine utilizes these rules to derive all potential solutions from the existing facts that would fulfill the query.\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -949,7 +975,51 @@ "\n", "The only difference between the `rgx_span` and `rgx_string` ie functions, is that rgx_string returns strings, while rgx_span returns the spans of those strings. This also means that if you want to use constant terms as return values, they have to be spans if you use `rgx_span`, and strings if you use `rgx_string`\n", "\n", - "For example:" + "For example consider the following rgxlog code:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "printing results for query 'age(X, Y)':\n", + " X | Y\n", + "------------+-----\n", + " John Doe | 35\n", + " Jane Smith | 28\n", + "\n", + "printing results for query 'age_span(X, Y)':\n", + " X | Y\n", + "----------+----------\n", + " [0, 8) | [10, 12)\n", + " [24, 34) | [36, 38)\n", + "\n" + ] + } + ], + "source": [ + "%%rgxlog\n", + "input_string = \"John Doe: 35 years old, Jane Smith: 28 years old\"\n", + "regex_pattern = \"(\\w+\\s\\w+):\\s(\\d+)\"\n", + "\n", + "age(X,Y) <- py_rgx_string(input_string, regex_pattern) -> (X,Y)\n", + "age_span(X,Y) <- py_rgx_span(input_string, regex_pattern) -> (X,Y)\n", + "?age(X,Y)\n", + "?age_span(X,Y)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The variables X,Y in the output of the above ie functions are the matches of the capture groups used in the regex_pattern.
\n", + "capture groups allow us to extract specific parts of a matched pattern in a text using regular expressions.
\n", + "When you define a regular expression pattern with parentheses (), you create a capturing group" ] }, { @@ -1022,6 +1092,15 @@ " out_rel=get_happy_out_types)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You may have noticed that when we register a custom ie function, we use `yield` instead of `return`,
\n", + "and that is because part of making spanner based database systems more performant and memory efficient is to do lazy evaluation,
\n", + "since building iterators in python is very simple using the generator pattern, we made the ie functions into generators to allow ie functions to also be as lazy as their author desires." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1106,6 +1185,15 @@ "# RGXLog program example" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import rgxlog" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1318,12 +1406,7 @@ "

\n", "Unfortunately RGXLog doesn't support True/False values. Therefore, we can't use ```X != Y```.\n", "
\n", - "Our solution to this problem is to create an ie function that implements NEQ relation:\n", - "\n", - "\n", - "```python\n", - "\n", - "```" + "Our solution to this problem is to create an ie function that implements NEQ relation:" ] }, { @@ -1549,9 +1632,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "And now for the usage.\n", - "Suppose we have a json document of the following format {student: {subject: grade, ...} ,...}\n", - "We want to create a rglox relation containg tuples of (student, subject, grade)." + "And now for the usage.
\n", + "Suppose we have a json document of the following format {student: {subject: grade, ...} ,...}
\n", + "We want to create a rglox relation containing tuples of (student, subject, grade)." ] }, { @@ -1594,104 +1677,6 @@ "json_table(Student, Subject, Grade) <- JsonPathFull(json_string, \"*.*\") -> (Student, Subject, Grade)\n", "?json_table(Student, Subject, Grade)" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Wrapping shell-based functions" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Rgxlog's `rgx_string` ie function is a good example of running an external shell as part of rgxlog code.\n", - "This time we won't remove the built-in function - we'll just show the implementation:" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```python\n", - "def rgx(text, regex_pattern, out_type: str):\n", - " \"\"\"\n", - " An IE function which runs regex using rust's `enum-spanner-rs` and yields tuples of strings/spans (not both).\n", - "\n", - " @param text: the string on which regex is run.\n", - " @param regex_pattern: the pattern to run.\n", - " @param out_type: string/span - decides which one will be returned.\n", - " @return: a tuple of strings/spans.\n", - " \"\"\"\n", - " with tempfile.TemporaryDirectory() as temp_dir:\n", - " rgx_temp_file_name = os.path.join(temp_dir, TEMP_FILE_NAME)\n", - " with open(rgx_temp_file_name, \"w+\") as f:\n", - " f.write(text)\n", - "\n", - " if out_type == \"string\":\n", - " rust_regex_args = rf\"{REGEX_EXE_PATH} {regex_pattern} {rgx_temp_file_name}\"\n", - " format_function = _format_spanner_string_output\n", - " elif out_type == \"span\":\n", - " rust_regex_args = rf\"{REGEX_EXE_PATH} {regex_pattern} {rgx_temp_file_name} --bytes-offset\"\n", - " format_function = _format_spanner_span_output\n", - " else:\n", - " assert False, \"illegal out_type\"\n", - "\n", - " regex_output = format_function(run_cli_command(rust_regex_args, stderr=True))\n", - "\n", - " for out in regex_output:\n", - " yield out\n", - "\n", - "def rgx_string(text, regex_pattern):\n", - " \"\"\"\n", - " @param text: The input text for the regex operation.\n", - " @param regex_pattern: the pattern of the regex operation.\n", - " @return: tuples of strings that represents the results.\n", - " \"\"\"\n", - " return rgx(text, regex_pattern, \"string\")\n", - "\n", - "RGX_STRING = dict(ie_function=rgx_string,\n", - " ie_function_name='rgx_string',\n", - " in_rel=RUST_RGX_IN_TYPES,\n", - " out_rel=rgx_string_out_type)\n", - "\n", - "# another version of these functions exists (rgx_from_file), it can be seen in the source code\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`run_cli_command` is an STDLIB function used in rgxlog, which basically runs a command using python's `Popen`.\n", - "\n", - "in order to denote regex groups, use `(?Ppattern)`. the output is in alphabetical order.\n", - "Let's run the ie function:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "ename": "SyntaxError", - "evalue": "invalid syntax (94253017.py, line 5)", - "output_type": "error", - "traceback": [ - "\u001b[1;36m Cell \u001b[1;32mIn[33], line 5\u001b[1;36m\u001b[0m\n\u001b[1;33m string_rel(X,Y) <- rgx_string(text, pattern) -> (X,Y)\u001b[0m\n\u001b[1;37m ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n" - ] - } - ], - "source": [ - "#| eval: false\n", - "%%rgxlog\n", - "text = \"zcacc\"\n", - "pattern = \"(?P[^c]+)(?P[c]+)\"\n", - "string_rel(X,Y) <- rgx_string(text, pattern) -> (X,Y)\n", - "?string_rel(X,Y)" - ] } ], "metadata": { diff --git a/nbs/sidebar.yml b/nbs/sidebar.yml index 77214d18..4f06617e 100644 --- a/nbs/sidebar.yml +++ b/nbs/sidebar.yml @@ -8,6 +8,7 @@ website: - advanced_usage.ipynb - building_your_optimization.ipynb - live_repl.ipynb + - extended_version.ipynb - section: src contents: - section: Generic diff --git a/rgxlog/ie_func/nlp.py b/rgxlog/ie_func/nlp.py index 69761019..9e6b54c9 100644 --- a/rgxlog/ie_func/nlp.py +++ b/rgxlog/ie_func/nlp.py @@ -2,12 +2,13 @@ # %% auto 0 __all__ = ['JAVA_MIN_VERSION', 'NLP_URL', 'NLP_DIR_NAME', 'CURR_DIR', 'NLP_DIR_PATH', 'JAVA_DOWNLOADER', 'INSTALLATION_PATH', - 'STANFORD_ZIP_GOOGLE_DRIVE_ID', 'STANFORD_ZIP_NAME', 'STANFORD_ZIP_PATH', 'logger', 'Tokenize', 'SSplit', - 'POS', 'Lemma', 'NER', 'EntityMentions', 'RGXNer', 'TokensRegex', 'CleanXML', 'Parse', 'DepParse', 'Coref', - 'OpenIE', 'KBP', 'Quote', 'Sentiment', 'TrueCase', 'UDFeats', 'tokenize_wrapper', 'ssplit_wrapper', - 'pos_wrapper', 'lemma_wrapper', 'ner_wrapper', 'entitymentions_wrapper', 'regexner_wrapper', - 'tokensregex_wrapper', 'cleanxml_wrapper', 'parse_wrapper', 'dependency_parse_wrapper', 'coref_wrapper', - 'openie_wrapper', 'kbp_wrapper', 'quote_wrapper', 'sentiment_wrapper', 'truecase_wrapper', 'udfeats_wrapper'] + 'STANFORD_ZIP_GOOGLE_DRIVE_ID', 'STANFORD_ZIP_NAME', 'STANFORD_ZIP_PATH', 'logger', 'CoreNLPEngine', + 'Tokenize', 'SSplit', 'POS', 'Lemma', 'NER', 'EntityMentions', 'RGXNer', 'TokensRegex', 'CleanXML', 'Parse', + 'DepParse', 'Coref', 'OpenIE', 'KBP', 'Quote', 'Sentiment', 'TrueCase', 'UDFeats', + 'download_and_install_nlp', 'tokenize_wrapper', 'ssplit_wrapper', 'pos_wrapper', 'lemma_wrapper', + 'ner_wrapper', 'entitymentions_wrapper', 'regexner_wrapper', 'tokensregex_wrapper', 'cleanxml_wrapper', + 'parse_wrapper', 'dependency_parse_wrapper', 'coref_wrapper', 'openie_wrapper', 'kbp_wrapper', + 'quote_wrapper', 'sentiment_wrapper', 'truecase_wrapper', 'udfeats_wrapper'] # %% ../../nbs/ie_func/04b_nlp.ipynb 3 import json @@ -84,12 +85,14 @@ def _run_installation() -> None: raise IOError("installation failed") # %% ../../nbs/ie_func/04b_nlp.ipynb 10 -#| eval: false -try: - _run_installation() - CoreNLPEngine = StanfordCoreNLP(NLP_DIR_PATH) -except: - logger.error("Installation NLP failed") +CoreNLPEngine = None +def download_and_install_nlp(): + global CoreNLPEngine + try: + _run_installation() + CoreNLPEngine = StanfordCoreNLP(NLP_DIR_PATH) + except: + logger.error("Installation NLP failed") # %% ../../nbs/ie_func/04b_nlp.ipynb 11 def tokenize_wrapper(sentence: str) -> Iterator: diff --git a/rgxlog/ie_func/rust_spanner_regex.py b/rgxlog/ie_func/rust_spanner_regex.py index c39f86a0..ec675949 100644 --- a/rgxlog/ie_func/rust_spanner_regex.py +++ b/rgxlog/ie_func/rust_spanner_regex.py @@ -6,8 +6,8 @@ 'RUSTUP_TOOLCHAIN', 'CARGO_CMD_ARGS', 'RUSTUP_CMD_ARGS', 'SHORT_TIMEOUT', 'CARGO_TIMEOUT', 'RUSTUP_TIMEOUT', 'TIMEOUT_MINUTES', 'WINDOWS_OS', 'WHICH_WORD', 'REGEX_EXE_PATH', 'ESCAPED_STRINGS_PATTERN', 'SPAN_PATTERN', 'TEMP_FILE_NAME', 'logger', 'RGX', 'RGX_STRING', 'RGX_FROM_FILE', 'RGX_STRING_FROM_FILE', - 'rgx_span_out_type', 'rgx_string_out_type', 'rgx', 'rgx_span', 'rgx_string', 'rgx_span_from_file', - 'rgx_string_from_file'] + 'download_and_install_rust_regex', 'rgx_span_out_type', 'rgx_string_out_type', 'rgx', 'rgx_span', + 'rgx_string', 'rgx_span_from_file', 'rgx_string_from_file'] # %% ../../nbs/ie_func/04d_rust_spanner_regex.ipynb 4 import logging @@ -65,7 +65,11 @@ logger = logging.getLogger(__name__) # %% ../../nbs/ie_func/04d_rust_spanner_regex.ipynb 20 -def _download_and_install_rust_regex() -> None: +def _is_installed_package() -> bool: + return Path(REGEX_EXE_PATH).is_file() + +# %% ../../nbs/ie_func/04d_rust_spanner_regex.ipynb 21 +def download_and_install_rust_regex() -> None: # don't use "cargo -V" because it starts downloading stuff sometimes with Popen([WHICH_WORD, "cargo"], stdout=PIPE, stderr=PIPE) as cargo: errcode = cargo.wait(SHORT_TIMEOUT) @@ -91,10 +95,6 @@ def _download_and_install_rust_regex() -> None: logger.warning("installation completed") -# %% ../../nbs/ie_func/04d_rust_spanner_regex.ipynb 21 -def _is_installed_package() -> bool: - return Path(REGEX_EXE_PATH).is_file() - # %% ../../nbs/ie_func/04d_rust_spanner_regex.ipynb 22 @no_type_check def rgx_span_out_type(output_arity: int) -> Tuple[DataTypes]: @@ -211,12 +211,3 @@ def rgx_string_from_file(text_file: str, # The input file for the regex operatio ie_function_name='rgx_string_from_file', in_rel=RUST_RGX_IN_TYPES, out_rel=rgx_string_out_type) - -# %% ../../nbs/ie_func/04d_rust_spanner_regex.ipynb 35 -#| eval: false -# the package is installed when this module is imported, exclude the scenraio of github runner -try: - if not _is_installed_package() and 'runner' not in os.getcwd(): - _download_and_install_rust_regex() -except: - logger.error(f"cargo or rustup are not installed in $PATH. please install rust: {DOWNLOAD_RUST_URL}")