Pytorch Geometric support (deprecates KGCN)

.bazelversion

@@ -1 +1 @@
-4.0.0
+5.1.1

.github/ISSUE_TEMPLATE/bug-report.md

@@ -14,7 +14,7 @@ Please replace every line in curly brackets ( { like this } ) with appropriate a
 
 1. OS (where TypeDB server runs): { e.g. Mac OS 10, Windows 10, Ubuntu 16.4, etc. }
 2. TypeDB version (and platform): { e.g. TypeDB 2.1.0, or TypeDB Cluster 2.1.1 on Google Cloud }
-3. TypeDB KGLIB and client-python version: { e.g. KGLIB 0.1 and client-python 1.4 }
+3. TypeDB, typedb-ml and client-python version: { e.g. typedb-ml 0.2 and client-python 1.4 }
 4. Python version: { e.g. 2.7, 3.6, etc. }
 5. Other environment details:
 

.gitignore

@@ -35,6 +35,6 @@ tmp/
 __pycache__/
 
 # Data input/output directories
-kglib/kgcn_tensorflow/examples/diagnosis/events/
+examples/diagnosis/events/
 
 *.egg-info/

.grabl/automation.yml

@@ -24,8 +24,6 @@ config:
   dependencies:
     dependencies: [build]
     typedb-client-python: [build, release]
-    typedb-common: [build, release]
-    typedb: [build, release]
 
 build:
   quality:
@@ -37,47 +35,56 @@ build:
     build:
       image: vaticle-ubuntu-20.04
       command: |
-        pyenv install -s 3.6.3
-        pyenv global 3.6.3 system
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
         bazel build //...
         bazel run @vaticle_dependencies//tool/checkstyle:test-coverage
         bazel test $(bazel query 'kind(checkstyle_test, //...)') --test_output=errors
     test-markdown-link-health:
       image: vaticle-ubuntu-20.04
       command: |
         find . -name \*.md | xargs -L1 npx [email protected] -v
-    test-kgcn-data-loader:
+    test-pytorch-geometric:
       image: vaticle-ubuntu-20.04
       timeout: "10m"
       command: |
-        pyenv install -s 3.6.3
-        pyenv global 3.6.3 system
-        bazel test //kglib/utils/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
-    test-utils:
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
+        bazel test //typedb_ml/pytorch_geometric/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
+    test-typedb:
       image: vaticle-ubuntu-20.04
       timeout: "10m"
       command: |
-        pyenv install -s 3.6.3
-        pyenv global 3.6.3 system
-        bazel test //kglib/kgcn_data_loader/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
-    test-kgcn-tensorflow:
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
+        bazel test //typedb_ml/typedb/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
+    test-networkx:
       image: vaticle-ubuntu-20.04
       timeout: "10m"
       command: |
-        pyenv install -s 3.6.3
-        pyenv global 3.6.3 system
-        bazel test //kglib/kgcn_tensorflow/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
+        bazel test //typedb_ml/networkx/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
+    test-examples:
+      image: vaticle-ubuntu-20.04
+      timeout: "10m"
+      command: |
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
+        bazel test //examples/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
     test-end-to-end:
       image: vaticle-ubuntu-20.04
-      timeout: "30m"
+      timeout: "10m"
       command: |
-        pyenv install -s 3.6.3
-        pyenv global 3.6.3 system
-        bazel test //kglib/tests/end_to_end:diagnosis --test_output=streamed --spawn_strategy=standalone --action_env=PATH
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
+        bazel test //tests/end_to_end:diagnosis --test_output=streamed --spawn_strategy=standalone --action_env=PATH
     deploy-pip-snapshot:
       image: vaticle-ubuntu-20.04
-      dependencies: [build, test-kgcn-data-loader, test-utils, test-kgcn-tensorflow, test-end-to-end]
+      dependencies: [build, test-pytorch-geometric, test-typedb, test-networkx, test-examples, test-end-to-end]
       command: |
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
         export DEPLOY_PIP_USERNAME=$REPO_VATICLE_USERNAME
         export DEPLOY_PIP_PASSWORD=$REPO_VATICLE_PASSWORD
         bazel run --define version=$(git rev-parse HEAD) //:deploy-pip -- snapshot
@@ -89,15 +96,17 @@ build:
         branch: master
       type: foreground
       command: |
-        pyenv global 3.6.10
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2
         pip3 install -U pip
         sudo unlink /usr/bin/python3
         sudo ln -s $(which python3) /usr/bin/python3
-        sudo ln -s /usr/share/pyshared/lsb_release.py /opt/pyenv/versions/3.6.10/lib/python3.6/site-packages/lsb_release.py
-        bazel run //test:typedb-extractor -- typedb-all-linux
+        sudo ln -s /usr/share/pyshared/lsb_release.py /opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/lsb_release.py
+        bazel run //tests/end_to_end:typedb-extractor-linux -- typedb-all-linux
         ./typedb-all-linux/typedb server &
-        pip install --extra-index-url https://repo.vaticle.com/repository/pypi-snapshot/simple typedb-kglib==0.0.0-$GRABL_COMMIT
-        cd kglib/tests/deployment/ && python -m unittest kgcn.diagnosis && export TEST_SUCCESS=0 ||
+        pip install -r requirements.txt
+        pip install --extra-index-url https://repo.vaticle.com/repository/pypi-snapshot/simple typedb-ml==0.0.0-$GRABL_COMMIT
+        python -m examples.diagnosis.diagnosis "./typedb-all-linux" && export TEST_SUCCESS=0 ||
         export TEST_SUCCESS=1
         kill $(jps | awk '/TypeDBServer/ {print $1}')
         exit $TEST_SUCCESS
@@ -108,27 +117,27 @@ release:
   validation:
     validate-dependencies:
       image: vaticle-ubuntu-20.04
-      command: bazel test //:release-validate-deps --test_output=streamed
+      command: bazel test //:release-validate-python-deps --test_output=streamed
   deployment:
     deploy-github:
       image: vaticle-ubuntu-20.04
       command: |
-        pyenv install -s 3.6.10
-        pyenv global 3.6.10 system
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
         pip3 install -U pip
         pip install certifi
         export ARTIFACT_USERNAME=$REPO_VATICLE_USERNAME
         export ARTIFACT_PASSWORD=$REPO_VATICLE_PASSWORD
         bazel run @vaticle_dependencies//distribution/artifact:create-netrc
-        export RELEASE_NOTES_TOKEN=$REPO_GITHUB_TOKEN
-        bazel run @vaticle_dependencies//tool/release:create-notes -- kglib $(cat VERSION) ./RELEASE_TEMPLATE.md
+        export NOTES_CREATE_TOKEN=$REPO_GITHUB_TOKEN
+        bazel run @vaticle_dependencies//tool/release/notes:create -- $GRABL_OWNER $GRABL_REPO $GRABL_COMMIT $(cat VERSION) ./RELEASE_TEMPLATE.md
         export DEPLOY_GITHUB_TOKEN=$REPO_GITHUB_TOKEN
         bazel run --define version=$(cat VERSION) //:deploy-github -- $GRABL_COMMIT
     deploy-pip-release:
       image: vaticle-ubuntu-20.04
       command: |
-        pyenv install -s 3.6.10
-        pyenv global 3.6.10 system
+        pyenv install -s 3.7.2
+        pyenv global 3.7.2 system
         pip3 install -U pip
         export ARTIFACT_USERNAME=$REPO_VATICLE_USERNAME
         export ARTIFACT_PASSWORD=$REPO_VATICLE_PASSWORD

BUILD

@@ -23,27 +23,25 @@ exports_files(["requirements.txt", "RELEASE_TEMPLATE.md"])
 
 load("@rules_python//python:defs.bzl", "py_library", "py_test")
 
-load("@vaticle_kglib_pip//:requirements.bzl",
-       vaticle_kglib_requirement = "requirement")
+load("@vaticle_typedb_ml_pip//:requirements.bzl", vaticle_typedb_ml_requirement = "requirement")
 
 load("@vaticle_bazel_distribution//github:rules.bzl", "deploy_github")
 load("@vaticle_bazel_distribution//pip:rules.bzl", "assemble_pip", "deploy_pip")
-load("@vaticle_kglib_pip//:requirements.bzl",
-       vaticle_kglib_requirement = "requirement")
+load("@vaticle_typedb_ml_pip//:requirements.bzl", vaticle_typedb_ml_requirement = "requirement")
 
 load("@vaticle_dependencies//distribution:deployment.bzl", "deployment")
 load("//:deployment.bzl", github_deployment = "deployment")
-load("@vaticle_dependencies//tool/release/deps:rules.bzl", "release_validate_deps")
+load("@vaticle_dependencies//tool/release/deps:rules.bzl", "release_validate_python_deps")
 
 load("@vaticle_dependencies//tool/checkstyle:rules.bzl", "checkstyle_test")
 
 assemble_pip(
     name = "assemble-pip",
-    target = "//kglib:kglib",
-    package_name = "vaticle-kglib",
+    target = "//typedb_ml:typedb-ml",
+    package_name = "typedb-ml",
     classifiers = [
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
         "Intended Audience :: Developers",
@@ -54,11 +52,11 @@ assemble_pip(
         "Topic :: Software Development :: Libraries",
         "Topic :: Software Development :: Libraries :: Python Modules"
     ],
-    url = "https://github.com/vaticle/kglib",
+    url = "https://github.com/vaticle/typedb-ml",
     author = "Vaticle",
     author_email = "[email protected]",
     license = "Apache-2.0",
-    requirements_file = "//:requirements.txt",
+    requirements_file = "//:install_requires.txt",
     keywords = ["machine learning", "logical reasoning", "knowledege graph", "typedb", "database", "graph",
                 "knowledgebase", "knowledge-engineering"],
 
@@ -73,14 +71,12 @@ deploy_pip(
     release = deployment["pypi.release"],
 )
 
-release_validate_deps(
-    name = "release-validate-deps",
-    refs = "@vaticle_kglib_workspace_refs//:refs.json",
+release_validate_python_deps(
+    name = "release-validate-python-deps",
+    requirements = "//:requirements.txt",
     tagged_deps = [
-        "@vaticle_typedb",
-        "@vaticle_typedb_client_python",
+        "typedb-client",
     ],
-    tags = ["manual"]
 )
 
 checkstyle_test(
@@ -89,6 +85,13 @@ checkstyle_test(
         "*",
         ".grabl/*",
     ]),
+    exclude = glob([
+        "*.md"
+    ]) + [
+        ".bazelversion",
+        "LICENSE",
+        "VERSION",
+    ],
     license_type = "apache-header",
 )
 

README.md

@@ -1,68 +1,71 @@
-[![GitHub release](https://img.shields.io/github/release/vaticle/kglib.svg)](https://github.com/vaticle/typedb/releases/latest)
+[![GitHub release](https://img.shields.io/github/release/vaticle/typedb-ml.svg)](https://github.com/vaticle/typedb/releases/latest)
 [![Discord](https://img.shields.io/discord/665254494820368395?color=7389D8&label=chat&logo=discord&logoColor=ffffff)](https://vaticle.com/discord)
 [![Discussion Forum](https://img.shields.io/discourse/https/forum.vaticle.com/topics.svg)](https://forum.vaticle.com)
 [![Stack Overflow](https://img.shields.io/badge/stackoverflow-typedb-796de3.svg)](https://stackoverflow.com/questions/tagged/typedb)
 [![Stack Overflow](https://img.shields.io/badge/stackoverflow-typeql-3dce8c.svg)](https://stackoverflow.com/questions/tagged/typeql)
 
-# TypeDB KGLIB (Knowledge Graph Library)
+# TypeDB ML
+_Previously known as KGLIB._
 
-**KGLIB provides tools to enable machine learning with [TypeDB](https://github.com/vaticle/typedb).**
+**TypeDB ML provides tools to enable graph algorithms and machine learning with [TypeDB](https://github.com/vaticle/typedb).**
 
-This library is under development and will henceforth be transformed into primarily infrastructure tools and integrations between TypeDB and machine learning libraries.
+There are integrations for [NetworkX](https://networkx.org) and for [PyTorch Geometric (PyG)](https://github.com/pyg-team/pytorch_geometric).
 
-## Machine Learning Pipeline
+[NetworkX](https://networkx.org) integration allows you to use a [large library of algorithms](https://networkx.org/documentation/stable/reference/algorithms/index.html) over graph data exported from TypeDB.
 
-![Flow Diagram](kglib/kgcn_tensorflow/.images/knowledge_graph_machine_learning.png)
+[PyTorch Geometric (PyG)](https://github.com/pyg-team/pytorch_geometric) integration gives you a toolbox to build Graph Neural Networks (GNNs) for your TypeDB data, with an example included for link prediction (or: binary relation prediction, in TypeDB terms). The structure of the GNNs are totally customisable, with network components for popular topics such as graph attention and graph transformers built-in.  
 
-The pipeline provided helps by allowing us to extract subgraphs from TypeDB. Each subgraph is a training example, which are sent to the learner in batches. Algorithms using this approach are scalable since they do not need to hold the whole graph in memory for training.
+## Features
 
-The pipeline is as follows:
-1. Extract data from `TypeDB` into Python [NetworkX](https://networkx.org) in-memory subgraphs by specifying multiple [TypeQL](https://github.com/vaticle/typeql) queries.
-2. Encode the nodes and edges of the NetworkX graphs
-3. Either (a) transform the encoded values into features, ready for input into a graph/geometric learning pipeline (for example the upcoming PyTorch implementation); or (b) Embed the encoded values according to the Types present in your database (TensorFlow only, PyTorch coming soon). This type-centric embedding is crucial to extracting the context explicitly captured in TypeDB's Type System. 
-4. Feed the features to a learning algorithm (see below)
-5. Optionally, store the predictions made by the learner in TypeDB. These predictions can then be queried using TypeQL. This means we can trivially run more learning tasks over the knowledge base, including the newly made predictions. This is knowledge graph completion.
-
-## Learning Algorithms 
-This repo contains one algorithmic implementation: [*Knowledge Graph Convolutional Network* (KGCN)](kglib/kgcn_tensorflow). This is a generic method for relation predication over any TypeDB database. There is a [full worked example](kglib/kgcn_tensorflow/examples/diagnosis) and an explanation of how the approach works.
-
-You are encouraged to use the tools available in KGLIB to interface TypeDB to your own algorithmic implementations, or to use/leverage prebuilt implementations available from popular libraries such as [PyTorch Geometric](https://github.com/rusty1s/pytorch_geometric) or [Graph Nets](https://github.com/deepmind/graph_nets) (TensorFlow/Sonnet). 
+### NetworkX
+- Declare the graph structure of your queries, with optional sampling functions.
+- Query a TypeDB instance and combine many results across many queries into a single graph (`build_graph_from_queries`).
+### PyTorch Geometric
+- A `DataSet` object to lazily load graphs from a TypeDB instance. Each graph is converted to a PyG `Data` object.
+- It's most natural to work with `HeteroData` objects since all data in TypeDB has a type. This conversion is available by default in PyG, but TypeDB-ML provides `store_concepts_by_type` to map concepts by type so that they can be re-associated after learning is finished.
+- A `FeatureEncoder` to orchestrate encoders to generate features for graphs.
+- Encoders for Continuous and Categorical values to apply encodings/embedding spaces to the types and attribute values present in TypeDB data.
+- A [full example for link prediction](examples/diagnosis)
+### Other
+- Example usage of Tensorboard for PyG `HeteroData`
 
 ## Resources
-You may find the following resources useful: 
-- [Strongly Typed Data for Machine Learning](https://www.youtube.com/watch?v=qhUyurWMiSQ) (YouTube)
-- [How Can We Complete a Knowledge Graph?](https://www.youtube.com/watch?v=nYDi1_UaFtU) (YouTube)
+You may find the following resources useful, particularly to understand why TypeDB-ML started: 
+- [Strongly Typed Data for Machine Learning](https://www.youtube.com/watch?v=qhUyurWMiSQ) (YouTube, 2021)
+- [How Can We Complete a Knowledge Graph?](https://www.youtube.com/watch?v=nYDi1_UaFtU) (YouTube, 2018)
 
 ## Quickstart
 
-### Requirements
+### Install
+
+- Python >= 3.7.x
 
-- Python >= 3.6, <= 3.7.x (TensorFlow 1.14.0 doesn't support later Python versions).
+- Grab the `requirements.txt` file from [here](requirements.txt) and install the requirements with `pip install requirements.txt`. This is due to some intricacies installing PyG's dependencies, see [here](https://github.com/pyg-team/pytorch_geometric/issues/861) for details.
 
-- KGLIB installed via pip: `pip install typedb-kglib`. 
+- Installed TypeDB-ML: `pip install typedb-ml`. 
 
-- [TypeDB 2.1.1](https://github.com/vaticle/typedb/releases) running in the background.
+- [TypeDB 2.11.1](https://github.com/vaticle/typedb/releases) running in the background.
 
-- `typedb-client-python` 2.1.0 ([PyPi](https://pypi.org/project/typedb-client/), [GitHub release](https://github.com/vaticle/typedb-client-python/releases)). This should be installed automatically when you `pip install typedb-kglib`.
+- `typedb-client-python` 2.11.x ([PyPi](https://pypi.org/project/typedb-client/), [GitHub release](https://github.com/vaticle/typedb-client-python/releases)). This should be installed automatically when you `pip install typedb-ml`.
 
 ### Run the Example
 
-Take a look at [*Knowledge Graph Convolutional Networks* (KGCNs)](kglib/kgcn_tensorflow) to see a walkthrough of how to use the library.
+Take a look at the [PyTorch Geometric heterogeneous link prediction example](examples/diagnosis) to see how to use TypeDB-ML to build a GNN on TypeDB data.
 
 ### Building from source
 
-It's expected that you will use Pip to install, but should you need to make your own changes to the library, and import it into your project, you can build from source as follows.
+It's expected that you will use Pip to install, but should you need to make your own changes to the library, and import it into your project, you can build from source as follows:
 
-Clone KGLIB:
+Clone TypeDB-ML:
 
 ```
-git clone [email protected]:vaticle/kglib.git
+git clone [email protected]:vaticle/typedb-ml.git
 ```
 
 Go into the project directory:
 
 ```
-cd kglib
+cd typedb-ml
 ```
 
 Build all targets:
@@ -71,10 +74,10 @@ Build all targets:
 bazel build //...
 ```
 
-Run all tests. Requires Python 3.6+ on your `PATH`. Test dependencies are for Linux since that is the CI environment: 
+Run all tests. Requires Python 3.7+ on your `PATH`. Test dependencies are for Linux since that is the CI environment: 
 
 ```
-bazel test //kglib/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
+bazel test //typedb_ml/... --test_output=streamed --spawn_strategy=standalone --action_env=PATH
 ```
 
 Build the pip distribution. Outputs to `bazel-bin`:
@@ -85,6 +88,7 @@ bazel build //:assemble-pip
 
 ## Development
 
-To follow the development conversation, please join the [Vaticle Discord](https://discord.com/invite/grakn), and join the `#kglib` channel. Alternatively, start a new topic on the [Vaticle Discussion Forum](https://forum.vaticle.com).
+To follow the development conversation, please join the [Vaticle Discord](https://discord.com/invite/vaticle), and join the `#typedb-ml` channel. Alternatively, start a new topic on the [Vaticle Discussion Forum](https://forum.vaticle.com).
 
-KGLIB requires that you have migrated your data into a [TypeDB](https://github.com/vaticle/typedb) or TypeDB Cluster instance. There is an [official examples repo](https://github.com/vaticle/examples) for how to go about this, and information available on [migration in the docs](https://docs.vaticle.com/docs/examples/phone-calls-migration-python). Alternatively, there are fantastic community-led projects growing in the [TypeDB OSI](https://typedb.org) to facilitate fast and easy data loading.
+TypeDB-ML requires that you have migrated your data into a [TypeDB](https://github.com/vaticle/typedb) or TypeDB 
+Cluster instance. There is an [official examples repo](https://github.com/vaticle/examples) for how to go about this, and information available on [migration in the docs](https://docs.vaticle.com/docs/examples/phone-calls-migration-python). Alternatively, there are fantastic community-led projects growing in the [TypeDB OSI](https://typedb.org) to facilitate fast and easy data loading, for example [TypeDB Loader](https://github.com/typedb-osi/typedb-loader).

VERSION

@@ -1 +1 @@
-0.2.2
+0.3.0