diff --git a/ci/github_ci.sh b/ci/github_ci.sh index 0a9f637c26..cc48d79bb7 100644 --- a/ci/github_ci.sh +++ b/ci/github_ci.sh @@ -76,7 +76,7 @@ else cp -r ../../learn/examples/* ./ cp -r ../../demo/movie . if [[ "$WITH_PROCEDURE" == "OFF" ]]; then - rm -rf test_algo.py test_sampling.py test_train.py + rm -rf test_algo.py test_sampling.py test_train.py test_algo_v2.py fi pytest ./ # codecov diff --git a/cpplint/check_all.sh b/cpplint/check_all.sh index 4dfe833581..2a7e3ccfad 100755 --- a/cpplint/check_all.sh +++ b/cpplint/check_all.sh @@ -2,4 +2,4 @@ set -e dir=$(dirname "$(readlink -f "$0")") cd $dir/../ -find src test include toolkits procedures -name "*.cpp" -o -name "*.h" | grep -v "/lmdb/" | xargs python3 ./cpplint/cpplint.py --quiet +find src test include toolkits procedures -name "*.cpp" -o -name "*.h" | grep -v "/lmdb/" | grep -v "test/test_procedures/" | xargs python3 ./cpplint/cpplint.py --quiet diff --git a/deps/tugraph-db-client-java b/deps/tugraph-db-client-java index 9583bbe341..377026e45a 160000 --- a/deps/tugraph-db-client-java +++ b/deps/tugraph-db-client-java @@ -1 +1 @@ -Subproject commit 9583bbe341254c488116b1e9418065fc8392f4fc +Subproject commit 377026e45ae7f2eeac73536ca3cf5882788c43ca diff --git a/deps/tugraph-web b/deps/tugraph-web index 8253763309..a9ea45e20e 160000 --- a/deps/tugraph-web +++ b/deps/tugraph-web @@ -1 +1 @@ -Subproject commit 8253763309ad93cffcadffa3f67484a86c19bff2 +Subproject commit a9ea45e20eb224f019805ca085afcc95ee7a0cc2 diff --git a/docs/en-US/source/5.developer-manual/6.interface/1.query/1.cypher.md b/docs/en-US/source/5.developer-manual/6.interface/1.query/1.cypher.md index 2a398bc3a5..a3203b50f2 100644 --- a/docs/en-US/source/5.developer-manual/6.interface/1.query/1.cypher.md +++ b/docs/en-US/source/5.developer-manual/6.interface/1.query/1.cypher.md @@ -2492,78 +2492,78 @@ Clauses support progress list： ### 5.2.Whole List Of Procedures -| Name | Description | Signature | -|---------------------------------------|----------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| db.vertexLabels | list all vertex labels | db.vertexLabels() :: (label::STRING) | -| db.edgeLabels | list all edge labels | db.edgeLabels() :: (edgeLabels::STRING) | -| db.indexes | list all indexes | db.indexes() :: (label::STRING,field::STRING,label_type:STRING,unique::BOOLEAN,pair_unique::BOOLEAN) | -| db.listLabelIndexes | list indexes by label | db.listLabelIndexes(label_name:STRING,label_type:STRING) :: (label::STRING,field::STRING,unique::BOOLEAN,pair_unique::BOOLEAN) | -| db.warmup | warm up the DB | db.warmup() :: (time_used::STRING) | -| db.createVertexLabel | create a vertex label | db.createVertexLabel(label_name::STRING,field_specs::LIST) :: (::VOID) | -| db.createLabel | create a vertex/edge label | db.createLabel(label_type::STRING,label_name::STRING,extra::STRING,field_specs::LIST) :: () | -| db.getLabelSchema | get the schema of label | db.getLabelSchema(label_type::STRING,label_name::STRING) :: (name::STRING,type::STRING,optional::BOOLEAN) | -| db.getVertexSchema | get the schema of vertex label | db.getVertexSchema(label::STRING) :: (schema::MAP) | -| db.getEdgeSchema | get the schema of edge label | db.getEdgeSchema(label::STRING) :: (schema::MAP) | -| db.deleteLabel | delete vertex/edge label | db.deleteLabel(label_type::STRING,label_name::STRING) :: (::VOID) | -| db.alterLabelDelFields | delete some fields of a label on a subgraph | db.alterLabelDelFields(label_type::STRING,label_name::STRING,del_fields::LIST) :: (record_affected::INTEGER) | -| db.alterLabelAddFields | add some fields of a label on a subgraph | db.alterLabelAddFields(label_type::STRING,label_name::STRING,add_field_spec_values::LIST) :: (record_affected::INTEGER) | -| db.alterLabelModFields | modify some fields of a label on a subgraph | db.alterLabelModFields(label_type::STRING,label_name::STRING,mod_field_specs::LIST) :: (record_affected::INTEGER) | -| db.createEdgeLabel | create a edge label | db.createEdgeLabel(type_name::STRING,field_specs::LIST) :: (::VOID) | -| db.addIndex | add an index | db.addIndex(label_name::STRING,field_name::STRING,unique::BOOLEAN) :: (::VOID) | -| db.addEdgeIndex | add an index | db.addEdgeIndex(label_name::STRING,field_name::STRING,unique::BOOLEAN,pair_unique::BOOLEAN) :: (::VOID) | -| db.deleteIndex | delete an index | db.deleteIndex(label_name::STRING,field_name::STRING) :: (::VOID) | -| db.backup | backup the db | db.backup(destination::STRING) :: () | -| dbms.procedures | list all procedures | dbms.procedures() :: (name::STRING,signature::STRING) | -| dbms.security.changePassword | change current user password | dbms.security.changePassword(current_password::STRING,new_password::STRING) :: (::VOID) | -| dbms.security.changeUserPassword | change user password | dbms.security.changeUserPassword(user_name::STRING,new_password::STRING) :: (::VOID) | -| dbms.security.createUser | create an account | dbms.security.createUser(user_name::STRING,password::STRING) :: (::VOID) | -| dbms.security.deleteUser | delete an account | dbms.security.deleteUser(user_name::STRING) :: (::VOID) | -| dbms.security.listUsers | list all accounts | dbms.security.listUsers() :: (user_name::STRING,user_info::MAP) | -| dbms.security.showCurrentUser | get current user name | dbms.security.showCurrentUser() :: (current_user::STRING) | -| dbms.security.getUserPermissions | get the permissions of a specified user | dbms.security.getUserPermissions(user_name::STRING) :: (user_info::MAP) | -| dbms.graph.createGraph | create a subgraph | dbms.graph.createGraph(graph_name::STRING, description::STRING, max_size_GB::INTEGER) :: (::VOID) | -| dbms.graph.modGraph | modify the config of a subgraph | dbms.graph.modGraph(graph_name::STRING,config::MAP) :: (::VOID) | -| dbms.graph.deleteGraph | delete a subgraph | dbms.graph.deleteGraph(graph_name::STRING) :: (::VOID) | -| dbms.graph.listGraphs | list all subgraphs | dbms.graph.listGraphs() :: (graph_name::STRING,configuration::MAP) | -| dbms.graph.getGraphInfo | get the information of a specified graph | dbms.graph.getGraphInfo(graph_name::STRING)::(graph_name::STRING,configuration::MAP) | -| dbms.security.addAllowedHosts | add to the trust list | dbms.security.addAllowedHosts(hosts::LIST) :: (num_new::INTEGER) | -| dbms.security.deleteAllowedHosts | remove from the trust list | dbms.security.deleteAllowedHosts(hosts::LIST) :: (record_affected::INTEGER) | -| dbms.security.listAllowedHosts | list the trust list | dbms.security.listAllowedHosts() :: (host::STRING) | -| dbms.config.update | update the configuration | dbms.config.update(updates::MAP) :: (message::STRING) | -| dbms.config.list | list the configuration | dbms.config.list() :: (name::STRING,value::ANY) | -| algo.shortestPath | get a shortest path between two vertexes | algo.shortestPath(startNode::NODE,endNode::NODE,config::MAP) :: (nodeCount::INTEGER,totalCost::FLOAT) | -| algo.allShortestPaths | get all the shortest paths between two vertexes | algo.allShortestPaths(startNode::NODE,endNode::NODE,config::MAP) :: (nodeIds::LIST,relationshipIds::LIST,cost::LIST) | -| algo.native.extract | get the field values of a list of vertexes or edges specified id | algo.native.extract(id::ANY,config::MAP) :: (value::ANY) | -| db.flushDB | flush the db | db.flushDB() :: (::VOID) | -| dbms.security.listRoles | list all roles | dbms.security.listRoles() :: (role_name::STRING,role_info::MAP) | -| dbms.security.createRole | create a role | dbms.security.createRole(role_name::STRING,desc::STRING) :: (::VOID) | -| dbms.security.deleteRole | delete a role | dbms.security.deleteRole(role_name::STRING) :: (::VOID) | -| dbms.security.getRoleInfo | get the role information | dbms.security.getRoleInfo(role::STRING) :: (role_info::MAP) | -| dbms.security.disableRole | enable/disable the role | dbms.security.disableRole(role::STRING,disable::BOOLEAN) :: (::VOID) | -| dbms.security.modRoleDesc | modify the description of a role | dbms.security.modRoleDesc(role::STRING,description::STRING) :: (::VOID) | -| dbms.security.rebuildRoleAccessLevel | rebuild the user subgraph access rights | dbms.security.rebuildRoleAccessLevel(role::STRING,access_level::MAP) :: (::VOID) | -| dbms.security.modRoleAccessLevel | modify the user subgraph access rights | dbms.security.modRoleAccessLevel(role::STRING,access_level::MAP) :: (::VOID) | -| dbms.security.modRoleFieldAccessLevel | modify the user property access rights | dbms.security.modRoleFieldAccessLevel(role::STRING,graph::STRING,label::STRING,field::STRING,label_type::STRING,field_access_level::STRING) :: (::VOID) | -| dbms.security.getUserInfo | get the user information | dbms.security.getUserInfo(user::STRING) :: (user_info::MAP) | -| dbms.security.getUserMemoryUsage | get the memory usage for a user | dbms.security.getUserMemoryUsage(user::STRING) :: (memory_usage::INTEGER) | -| dbms.security.disableUser | enable/disable the user | dbms.security.disableUser(user::STRING,disable::BOOLEAN) :: (::VOID) | -| dbms.security.setCurrentDesc | set the current user description | dbms.security.setCurrentDesc(description::STRING) :: (::VOID) | -| dbms.security.setUserDesc | set user description | dbms.security.setUserDesc(user::STRING,description::STRING) :: (::VOID) | -| dbms.security.setUserMemoryLimit | set user memory limit | dbms.security.setUserMemoryLimit(user::STRING,memorylimit::INTEGER) :: (::VOID) | -| dbms.security.deleteUserRoles | delete roles from the user | dbms.security.deleteUserRoles(user::STRING,roles::LIST) :: (::VOID) | -| dbms.security.rebuildUserRoles | rebuild the relationship between the user and the role | dbms.security.rebuildUserRoles(user::STRING,roles::LIST) :: (::VOID) | -| dbms.security.addUserRoles | add the user roles | dbms.security.addUserRoles(user::STRING,roles::LIST) :: (::VOID) | -| db.plugin.loadPlugin | load a plugin | db.plugin.loadPlugin(plugin_type::STRING,plugin_name::STRING,plugin_content::STRING,code_type::STRING,plugin_description::STRING,read_only::BOOLEAN, version::STRING) :: (::VOID) | -| db.plugin.deletePlugin | unload a plugin | db.plugin.deletePlugin(plugin_type::STRING,plugin_name::STRING) :: (::VOID) | -| db.plugin.listPlugin | list all plugins | db.plugin.listPlugin(plugin_type::STRING,plugin_version::STRING) :: (plugin_description::LIST) | -| db.plugin.getPluginInfo | get the information of a specified plugin | db.plugin.getPluginInfo(plugin_type::STRING,plugin_name::STRING,show_code::BOOLEAN)::(plugin_description::MAP) | -| db.plugin.callPlugin | execute the plugins | db.plugin.callPlugin(plugin_type::STRING,plugin_name::STRING,param::STRING,timeout::DOUBLE,in_process::BOOLEAN) :: (success::BOOLEAN,result::STRING) | -| db.importor.dataImportor | import vertex/edge data | db.importor.dataImportor(description::STRING,content::STRING,continue_on_error::BOOLEAN,thread_nums::INTEGER,delimiter::STRING) :: (::VOID) | -| db.importor.schemaImportor | import vertex/edge schema | db.importor.schemaImportor(description::STRING) :: (::VOID) | -| dbms.meta.count | get the total number of vertex and edge | db.dbms.meta.count() :: (type::STRING, number::INTEGER) | -| dbms.meta.countDetail | get the number of vertex and edge for each label | db.dbms.meta.countDetail() :: (is_vertex::BOOLEAN, label::STRING, count::INTEGER) | -| dbms.meta.refreshCount | recount the number of vertex and edge, stop writing during the count | db.dbms.meta.refreshCount() :: (::VOID) | -| dbms.task.listTasks | list running tasks | dbms.task.listTasks()::(tasks::LIST) | -| dbms.task.terminateTask | terminate task | dbms.task.terminateTask(task_id::STRING)::(::VOID) | -| dbms.ha.clusterInfo | get cluster info in HA mode | dbms.ha.clusterInfo() :: (cluster_info::LIST, is_master::BOOLEAN) | -| db.dropDB | empty the db | db.dropDB() :: (::VOID) | +| Name | Description | Signature | +|---------------------------------------|----------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| db.vertexLabels | list all vertex labels | db.vertexLabels() :: (label::STRING) | +| db.edgeLabels | list all edge labels | db.edgeLabels() :: (edgeLabels::STRING) | +| db.indexes | list all indexes | db.indexes() :: (label::STRING,field::STRING,label_type:STRING,unique::BOOLEAN,pair_unique::BOOLEAN) | +| db.listLabelIndexes | list indexes by label | db.listLabelIndexes(label_name:STRING,label_type:STRING) :: (label::STRING,field::STRING,unique::BOOLEAN,pair_unique::BOOLEAN) | +| db.warmup | warm up the DB | db.warmup() :: (time_used::STRING) | +| db.createVertexLabel | create a vertex label | db.createVertexLabel(label_name::STRING,field_specs::LIST) :: (::VOID) | +| db.createLabel | create a vertex/edge label | db.createLabel(label_type::STRING,label_name::STRING,extra::STRING,field_specs::LIST) :: () | +| db.getLabelSchema | get the schema of label | db.getLabelSchema(label_type::STRING,label_name::STRING) :: (name::STRING,type::STRING,optional::BOOLEAN) | +| db.getVertexSchema | get the schema of vertex label | db.getVertexSchema(label::STRING) :: (schema::MAP) | +| db.getEdgeSchema | get the schema of edge label | db.getEdgeSchema(label::STRING) :: (schema::MAP) | +| db.deleteLabel | delete vertex/edge label | db.deleteLabel(label_type::STRING,label_name::STRING) :: (::VOID) | +| db.alterLabelDelFields | delete some fields of a label on a subgraph | db.alterLabelDelFields(label_type::STRING,label_name::STRING,del_fields::LIST) :: (record_affected::INTEGER) | +| db.alterLabelAddFields | add some fields of a label on a subgraph | db.alterLabelAddFields(label_type::STRING,label_name::STRING,add_field_spec_values::LIST) :: (record_affected::INTEGER) | +| db.alterLabelModFields | modify some fields of a label on a subgraph | db.alterLabelModFields(label_type::STRING,label_name::STRING,mod_field_specs::LIST) :: (record_affected::INTEGER) | +| db.createEdgeLabel | create a edge label | db.createEdgeLabel(type_name::STRING,field_specs::LIST) :: (::VOID) | +| db.addIndex | add an index | db.addIndex(label_name::STRING,field_name::STRING,unique::BOOLEAN) :: (::VOID) | +| db.addEdgeIndex | add an index | db.addEdgeIndex(label_name::STRING,field_name::STRING,unique::BOOLEAN,pair_unique::BOOLEAN) :: (::VOID) | +| db.deleteIndex | delete an index | db.deleteIndex(label_name::STRING,field_name::STRING) :: (::VOID) | +| db.backup | backup the db | db.backup(destination::STRING) :: () | +| dbms.procedures | list all procedures | dbms.procedures() :: (name::STRING,signature::STRING) | +| dbms.security.changePassword | change current user password | dbms.security.changePassword(current_password::STRING,new_password::STRING) :: (::VOID) | +| dbms.security.changeUserPassword | change user password | dbms.security.changeUserPassword(user_name::STRING,new_password::STRING) :: (::VOID) | +| dbms.security.createUser | create an account | dbms.security.createUser(user_name::STRING,password::STRING) :: (::VOID) | +| dbms.security.deleteUser | delete an account | dbms.security.deleteUser(user_name::STRING) :: (::VOID) | +| dbms.security.listUsers | list all accounts | dbms.security.listUsers() :: (user_name::STRING,user_info::MAP) | +| dbms.security.showCurrentUser | get current user name | dbms.security.showCurrentUser() :: (current_user::STRING) | +| dbms.security.getUserPermissions | get the permissions of a specified user | dbms.security.getUserPermissions(user_name::STRING) :: (user_info::MAP) | +| dbms.graph.createGraph | create a subgraph | dbms.graph.createGraph(graph_name::STRING, description::STRING, max_size_GB::INTEGER) :: (::VOID) | +| dbms.graph.modGraph | modify the config of a subgraph | dbms.graph.modGraph(graph_name::STRING,config::MAP) :: (::VOID) | +| dbms.graph.deleteGraph | delete a subgraph | dbms.graph.deleteGraph(graph_name::STRING) :: (::VOID) | +| dbms.graph.listGraphs | list all subgraphs | dbms.graph.listGraphs() :: (graph_name::STRING,configuration::MAP) | +| dbms.graph.getGraphInfo | get the information of a specified graph | dbms.graph.getGraphInfo(graph_name::STRING)::(graph_name::STRING,configuration::MAP) | +| dbms.security.addAllowedHosts | add to the trust list | dbms.security.addAllowedHosts(hosts::LIST) :: (num_new::INTEGER) | +| dbms.security.deleteAllowedHosts | remove from the trust list | dbms.security.deleteAllowedHosts(hosts::LIST) :: (record_affected::INTEGER) | +| dbms.security.listAllowedHosts | list the trust list | dbms.security.listAllowedHosts() :: (host::STRING) | +| dbms.config.update | update the configuration | dbms.config.update(updates::MAP) :: (message::STRING) | +| dbms.config.list | list the configuration | dbms.config.list() :: (name::STRING,value::ANY) | +| algo.shortestPath | get a shortest path between two vertexes | algo.shortestPath(startNode::NODE,endNode::NODE,config::MAP) :: (nodeCount::INTEGER,totalCost::FLOAT) | +| algo.allShortestPaths | get all the shortest paths between two vertexes | algo.allShortestPaths(startNode::NODE,endNode::NODE,config::MAP) :: (nodeIds::LIST,relationshipIds::LIST,cost::LIST) | +| algo.native.extract | get the field values of a list of vertexes or edges specified id | algo.native.extract(id::ANY,config::MAP) :: (value::ANY) | +| db.flushDB | flush the db | db.flushDB() :: (::VOID) | +| dbms.security.listRoles | list all roles | dbms.security.listRoles() :: (role_name::STRING,role_info::MAP) | +| dbms.security.createRole | create a role | dbms.security.createRole(role_name::STRING,desc::STRING) :: (::VOID) | +| dbms.security.deleteRole | delete a role | dbms.security.deleteRole(role_name::STRING) :: (::VOID) | +| dbms.security.getRoleInfo | get the role information | dbms.security.getRoleInfo(role::STRING) :: (role_info::MAP) | +| dbms.security.disableRole | enable/disable the role | dbms.security.disableRole(role::STRING,disable::BOOLEAN) :: (::VOID) | +| dbms.security.modRoleDesc | modify the description of a role | dbms.security.modRoleDesc(role::STRING,description::STRING) :: (::VOID) | +| dbms.security.rebuildRoleAccessLevel | rebuild the user subgraph access rights | dbms.security.rebuildRoleAccessLevel(role::STRING,access_level::MAP) :: (::VOID) | +| dbms.security.modRoleAccessLevel | modify the user subgraph access rights | dbms.security.modRoleAccessLevel(role::STRING,access_level::MAP) :: (::VOID) | +| dbms.security.modRoleFieldAccessLevel | modify the user property access rights | dbms.security.modRoleFieldAccessLevel(role::STRING,graph::STRING,label::STRING,field::STRING,label_type::STRING,field_access_level::STRING) :: (::VOID) | +| dbms.security.getUserInfo | get the user information | dbms.security.getUserInfo(user::STRING) :: (user_info::MAP) | +| dbms.security.getUserMemoryUsage | get the memory usage for a user | dbms.security.getUserMemoryUsage(user::STRING) :: (memory_usage::INTEGER) | +| dbms.security.disableUser | enable/disable the user | dbms.security.disableUser(user::STRING,disable::BOOLEAN) :: (::VOID) | +| dbms.security.setCurrentDesc | set the current user description | dbms.security.setCurrentDesc(description::STRING) :: (::VOID) | +| dbms.security.setUserDesc | set user description | dbms.security.setUserDesc(user::STRING,description::STRING) :: (::VOID) | +| dbms.security.setUserMemoryLimit | set user memory limit | dbms.security.setUserMemoryLimit(user::STRING,memorylimit::INTEGER) :: (::VOID) | +| dbms.security.deleteUserRoles | delete roles from the user | dbms.security.deleteUserRoles(user::STRING,roles::LIST) :: (::VOID) | +| dbms.security.rebuildUserRoles | rebuild the relationship between the user and the role | dbms.security.rebuildUserRoles(user::STRING,roles::LIST) :: (::VOID) | +| dbms.security.addUserRoles | add the user roles | dbms.security.addUserRoles(user::STRING,roles::LIST) :: (::VOID) | +| db.plugin.loadPlugin | load a plugin | db.plugin.loadPlugin(plugin_type::STRING,plugin_name::STRING,plugin_content::STRING or MAP,code_type::STRING,plugin_description::STRING,read_only::BOOLEAN, version::STRING) :: (::VOID) | +| db.plugin.deletePlugin | unload a plugin | db.plugin.deletePlugin(plugin_type::STRING,plugin_name::STRING) :: (::VOID) | +| db.plugin.listPlugin | list all plugins | db.plugin.listPlugin(plugin_type::STRING,plugin_version::STRING) :: (plugin_description::LIST) | +| db.plugin.getPluginInfo | get the information of a specified plugin | db.plugin.getPluginInfo(plugin_type::STRING,plugin_name::STRING,show_code::BOOLEAN)::(plugin_description::MAP) | +| db.plugin.callPlugin | execute the plugins | db.plugin.callPlugin(plugin_type::STRING,plugin_name::STRING,param::STRING,timeout::DOUBLE,in_process::BOOLEAN) :: (success::BOOLEAN,result::STRING) | +| db.importor.dataImportor | import vertex/edge data | db.importor.dataImportor(description::STRING,content::STRING,continue_on_error::BOOLEAN,thread_nums::INTEGER,delimiter::STRING) :: (::VOID) | +| db.importor.schemaImportor | import vertex/edge schema | db.importor.schemaImportor(description::STRING) :: (::VOID) | +| dbms.meta.count | get the total number of vertex and edge | db.dbms.meta.count() :: (type::STRING, number::INTEGER) | +| dbms.meta.countDetail | get the number of vertex and edge for each label | db.dbms.meta.countDetail() :: (is_vertex::BOOLEAN, label::STRING, count::INTEGER) | +| dbms.meta.refreshCount | recount the number of vertex and edge, stop writing during the count | db.dbms.meta.refreshCount() :: (::VOID) | +| dbms.task.listTasks | list running tasks | dbms.task.listTasks()::(tasks::LIST) | +| dbms.task.terminateTask | terminate task | dbms.task.terminateTask(task_id::STRING)::(::VOID) | +| dbms.ha.clusterInfo | get cluster info in HA mode | dbms.ha.clusterInfo() :: (cluster_info::LIST, is_master::BOOLEAN) | +| db.dropDB | empty the db | db.dropDB() :: (::VOID) | diff --git a/docs/en-US/source/5.developer-manual/6.interface/3.procedure/1.procedure.md b/docs/en-US/source/5.developer-manual/6.interface/3.procedure/1.procedure.md index 47c814d627..83ba668355 100644 --- a/docs/en-US/source/5.developer-manual/6.interface/3.procedure/1.procedure.md +++ b/docs/en-US/source/5.developer-manual/6.interface/3.procedure/1.procedure.md @@ -213,26 +213,31 @@ extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { extern "C" LGAPI bool ProcessInTxn(Transaction &txn, const std::string &request, - std::string &response) { + Result &response) { int64_t limit; try { json input = json::parse(request); limit = input["limit"].get(); } catch (std::exception &e) { - response = std::string("error parsing json: ") + e.what(); + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); return false; } - Result result({{"node", LGraphType::NODE}, - {"salt", LGraphType::FLOAT}, - }); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"salt", LGraphType::FLOAT} + }); for (size_t i = 0; i < limit; i++) { - auto r = result.MutableRecord(); + auto r = response.MutableRecord(); auto vit = txn.GetVertexIterator(i); r->Insert("node", vit); r->Insert("salt", FieldData::Float(20.23*float(i))); } - response = result.Dump(); return true; } ``` diff --git a/docs/zh-CN/source/5.developer-manual/6.interface/1.query/1.cypher.md b/docs/zh-CN/source/5.developer-manual/6.interface/1.query/1.cypher.md index 588b3ad1d4..dc6a25ef0c 100644 --- a/docs/zh-CN/source/5.developer-manual/6.interface/1.query/1.cypher.md +++ b/docs/zh-CN/source/5.developer-manual/6.interface/1.query/1.cypher.md @@ -2269,83 +2269,83 @@ TuGraph查询语言与OpenCypher的不同点如下： ### 5.2.内置procedures完整列表 -| Name | Description | Signature | -|---------------------------------------|------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| db.subgraph | 列出点的子图 | db.subgraph(vids::LIST) :: (subgraph::STRING) | -| db.vertexLabels | 列出所有Vertex Label | db.vertexLabels() :: (label::STRING) | -| db.edgeLabels | 列出所有Edge Label | db.edgeLabels() :: (edgeLabels::STRING) | -| db.indexes | 列出所有索引 | db.indexes() :: (label::STRING,field::STRING,label_type:STRING,unique::BOOLEAN,pair_unique::BOOLEAN) | -| db.listLabelIndexes | 列出所有与某个Label相关的索引 | db.listLabelIndexes(label_name:STRING,label_type:STRING) :: (label::STRING,field::STRING,unique::BOOLEAN,pair_unique::BOOLEAN) | -| db.warmup | 预热数据 | db.warmup() :: (time_used::STRING) | -| db.createVertexLabel | 创建Vertex Label | db.createVertexLabel(label_name::STRING,field_specs::LIST) :: (::VOID) | -| db.createLabel | 创建Vertex/Edge Label | db.createLabel(label_type::STRING,label_name::STRING,extra::STRING,field_specs::LIST) :: () | -| db.getLabelSchema | 列出label schema | db.getLabelSchema(label_type::STRING,label_name::STRING) :: (name::STRING,type::STRING,optional::BOOLEAN) | -| db.getVertexSchema | 列出点的 schema | db.getVertexSchema(label::STRING) :: (schema::MAP) | -| db.getEdgeSchema | 列出边的 schema | db.getEdgeSchema(label::STRING) :: (schema::MAP) | -| db.deleteLabel | 删除Vertex/Edge Label | db.deleteLabel(label_type::STRING,label_name::STRING) :: (::VOID) | -| db.alterLabelDelFields | 修改label删除属性 | db.alterLabelDelFields(label_type::STRING,label_name::STRING,del_fields::LIST) :: (record_affected::INTEGER) | -| db.alterLabelAddFields | 修改label添加field | db.alterLabelAddFields(label_type::STRING,label_name::STRING,add_field_spec_values::LIST) :: (record_affected::INTEGER) | -| db.alterLabelModFields | 修改label field | db.alterLabelModFields(label_type::STRING,label_name::STRING,mod_field_specs::LIST) :: (record_affected::INTEGER) | -| db.createEdgeLabel | 创建Edge Label | db.createEdgeLabel(type_name::STRING,field_specs::LIST) :: (::VOID) | -| db.addIndex | 创建索引 | db.addIndex(label_name::STRING,field_name::STRING,unique::BOOLEAN) :: (::VOID) | -| db.addEdgeIndex | 创建索引 | db.addEdgeIndex(label_name::STRING,field_name::STRING,unique::BOOLEAN,pair_unique::BOOLEAN) :: (::VOID) | -| db.deleteIndex | 删除索引 | db.deleteIndex(label_name::STRING,field_name::STRING) :: (::VOID) | -| db.backup | 备份数据 | db.backup(destination::STRING) :: () | -| dbms.procedures | 列出所有procedures | dbms.procedures() :: (name::STRING,signature::STRING) | -| dbms.security.changePassword | 更改当前用户的密码 | dbms.security.changePassword(current_password::STRING,new_password::STRING) :: (::VOID) | -| dbms.security.changeUserPassword | 更改指定用户的密码 | dbms.security.changeUserPassword(user_name::STRING,new_password::STRING) :: (::VOID) | -| dbms.security.createUser | 创建用户 | dbms.security.createUser(user_name::STRING,password::STRING) :: (::VOID) | -| dbms.security.deleteUser | 删除用户 | dbms.security.deleteUser(user_name::STRING) :: (::VOID) | -| dbms.security.listUsers | 列出所有用户 | dbms.security.listUsers() :: (user_name::STRING,user_info::MAP) | -| dbms.security.showCurrentUser | 列出当前用户信息 | dbms.security.showCurrentUser() :: (current_user::STRING) | -| dbms.security.getUserPermissions | 列出指定用户的权限 | dbms.security.getUserPermissions(user_name::STRING) :: (user_info::MAP) | -| dbms.graph.createGraph | 创建子图 | dbms.graph.createGraph(graph_name::STRING, description::STRING, max_size_GB::INTEGER) :: (::VOID) | -| dbms.graph.modGraph | 修改子图属性 | dbms.graph.modGraph(graph_name::STRING,config::MAP) :: (::VOID) | -| dbms.graph.deleteGraph | 删除子图 | dbms.graph.deleteGraph(graph_name::STRING) :: (::VOID) | -| dbms.graph.listGraphs | 列出所有子图 | dbms.graph.listGraphs() :: (graph_name::STRING,configuration::MAP) | -| dbms.graph.getGraphInfo | 列出指定子图的信息 | dbms.graph.getGraphInfo(graph_name::STRING)::(graph_name::STRING,configuration::MAP) | -| dbms.security.addAllowedHosts | 添加ip到信任列表 | dbms.security.addAllowedHosts(hosts::LIST) :: (num_new::INTEGER) | -| dbms.security.deleteAllowedHosts | 从信任列表删除ip | dbms.security.deleteAllowedHosts(hosts::LIST) :: (record_affected::INTEGER) | -| dbms.security.listAllowedHosts | 列出信任列表中的主机ip | dbms.security.listAllowedHosts() :: (host::STRING) | -| dbms.config.update | 更新TuGraph配置 | dbms.config.update(updates::MAP) :: (message::STRING) | -| dbms.config.list | 列出TuGraph配置 | dbms.config.list() :: (name::STRING,value::ANY) | -| algo.shortestPath | 查询两个点间的最短路径 | algo.shortestPath(startNode::NODE,endNode::NODE,config::MAP) :: (nodeCount::INTEGER,totalCost::FLOAT) | -| algo.allShortestPaths | 查询两个点间的所有最短路径 | algo.allShortestPaths(startNode::NODE,endNode::NODE,config::MAP) :: (nodeIds::LIST,relationshipIds::LIST,cost::LIST) | -| algo.native.extract | 查询指定VertexId/EdgeUid（列表）指定field的值（列表） | algo.native.extract(id::ANY,config::MAP) :: (value::ANY) | -| db.flushDB | 刷新db | db.flushDB() :: (::VOID) | -| dbms.security.listRoles | 列出所有角色 | dbms.security.listRoles() :: (role_name::STRING,role_info::MAP) | -| dbms.security.createRole | 创建角色 | dbms.security.createRole(role_name::STRING,desc::STRING) :: (::VOID) | -| dbms.security.deleteRole | 删除角色 | dbms.security.deleteRole(role_name::STRING) :: (::VOID) | -| dbms.security.getRoleInfo | 获取角色详细信息 | dbms.security.getRoleInfo(role::STRING) :: (role_info::MAP) | -| dbms.security.disableRole | 禁用/启用角色 | dbms.security.disableRole(role::STRING,disable::BOOLEAN) :: (::VOID) | -| dbms.security.modRoleDesc | 修改角色描述信息 | dbms.security.modRoleDesc(role::STRING,description::STRING) :: (::VOID) | -| dbms.security.rebuildRoleAccessLevel | 删除角色权限并重建 | dbms.security.rebuildRoleAccessLevel(role::STRING,access_level::MAP) :: (::VOID) | -| dbms.security.modRoleAccessLevel | 修改角色对指定图的访问权限 | dbms.security.modRoleAccessLevel(role::STRING,access_level::MAP) :: (::VOID) | -| dbms.security.modRoleFieldAccessLevel | 修改角色对指定属性的访问权限 | dbms.security.modRoleFieldAccessLevel(role::STRING,graph::STRING,label::STRING,field::STRING,label_type::STRING,field_access_level::STRING) :: (::VOID) | -| dbms.security.getUserInfo | 获取用户详细信息 | dbms.security.getUserInfo(user::STRING) :: (user_info::MAP) | -| dbms.security.disableUser | 禁用/启用用户 | dbms.security.disableUser(user::STRING,disable::BOOLEAN) :: (::VOID) | -| dbms.security.setCurrentDesc | 设置当前用户描述信息 | dbms.security.setCurrentDesc(description::STRING) :: (::VOID) | -| dbms.security.setUserDesc | 设置用户描述信息 | dbms.security.setUserDesc(user::STRING,description::STRING) :: (::VOID) | -| dbms.security.getUserMemoryUsage | 获取用户内存用量 | dbms.security.getUserMemoryUsage(user::STRING) :: (memory_usage::INTEGER) | -| dbms.security.setUserMemoryLimit | 设置用户内存限制 | dbms.security.setUserMemoryLimit(user::STRING,memorylimit::INTEGER) :: (::VOID) | -| dbms.security.deleteUserRoles | 删除用户与角色的联系 | dbms.security.deleteUserRoles(user::STRING,roles::LIST) :: (::VOID) | -| dbms.security.rebuildUserRoles | 清空用户角色的关系并重建 | dbms.security.rebuildUserRoles(user::STRING,roles::LIST) :: (::VOID) | -| dbms.security.addUserRoles | 新增用户与角色的联系 | dbms.security.addUserRoles(user::STRING,roles::LIST) :: (::VOID) | -| db.plugin.loadPlugin | 装载plugin | db.plugin.loadPlugin(plugin_type::STRING,plugin_name::STRING,plugin_content::STRING,code_type::STRING,plugin_description::STRING,read_only::BOOLEAN,version::STRING) :: (::VOID) | -| db.plugin.deletePlugin | 删除plugin | db.plugin.deletePlugin(plugin_type::STRING,plugin_name::STRING) :: (::VOID) | -| db.plugin.listPlugin | 列出已装载的plugin | db.plugin.listPlugin(plugin_type::STRING,plugin_version::STRING) :: (plugin_description::LIST) | -| db.plugin.getPluginInfo | 获取plugin的详细信息 | db.plugin.getPluginInfo(plugin_type::STRING,plugin_name::STRING,show_code::BOOLEAN)::(plugin_description::MAP) | -| db.plugin.callPlugin | 执行plugin | db.plugin.callPlugin(plugin_type::STRING,plugin_name::STRING,param::STRING,timeout::DOUBLE,in_process::BOOLEAN) :: (success::BOOLEAN,result::STRING) | -| db.importor.dataImportor | 导入点或边数据 | db.importor.dataImportor(description::STRING,content::STRING,continue_on_error::BOOLEAN,thread_nums::INTEGER,delimiter::STRING) :: (::VOID) | -| db.importor.schemaImportor | 导入点或边schema | db.importor.schemaImportor(description::STRING) :: (::VOID) | -| db.addFullTextIndex | 添加全文索引 | db.addFullTextIndex(is_vertex::BOOLEAN, label_name::STRING, field_name::STRING) :: (::VOID) | -| db.deleteFullTextIndex | 删除全文索引 | db.deleteFullTextIndex(is_vertex::BOOLEAN, label_name::STRING, field_name::STRING) :: (::VOID) | -| db.rebuildFullTextIndex | 重建全文索引 | db.rebuildFullTextIndex(vertex_labels::STRING, edge_labels::STRING) :: (::VOID) | -| db.fullTextIndexes | 查看全文索引 | db.fullTextIndexes() :: (is_vertex::BOOLEAN, label::STRING, field::STRING) | -| dbms.meta.count | 查看点边总数 | db.dbms.meta.count() :: (type::STRING, number::INTEGER) | -| dbms.meta.countDetail | 查看点边总数详情 | db.dbms.meta.countDetail() :: (is_vertex::BOOLEAN, label::STRING, count::INTEGER) | -| dbms.meta.refreshCount | 重新统计点边数量，统计期间停写。 | db.dbms.meta.refreshCount() :: (::VOID) | -| dbms.task.listTasks | 查询正在执行的任务 | dbms.task.listTasks()::(tasks::LIST) | -| dbms.task.terminateTask | 中止任务 | dbms.task.terminateTask(task_id::STRING)::(::VOID) | -| dbms.ha.clusterInfo | HA模式下查看集群状态 | dbms.ha.clusterInfo() :: (cluster_info::LIST, is_master::BOOLEAN) | -| db.dropDB | 清空数据库 | db.dropDB() :: (::VOID) | +| Name | Description | Signature | +|---------------------------------------|------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| db.subgraph | 列出点的子图 | db.subgraph(vids::LIST) :: (subgraph::STRING) | +| db.vertexLabels | 列出所有Vertex Label | db.vertexLabels() :: (label::STRING) | +| db.edgeLabels | 列出所有Edge Label | db.edgeLabels() :: (edgeLabels::STRING) | +| db.indexes | 列出所有索引 | db.indexes() :: (label::STRING,field::STRING,label_type:STRING,unique::BOOLEAN,pair_unique::BOOLEAN) | +| db.listLabelIndexes | 列出所有与某个Label相关的索引 | db.listLabelIndexes(label_name:STRING,label_type:STRING) :: (label::STRING,field::STRING,unique::BOOLEAN,pair_unique::BOOLEAN) | +| db.warmup | 预热数据 | db.warmup() :: (time_used::STRING) | +| db.createVertexLabel | 创建Vertex Label | db.createVertexLabel(label_name::STRING,field_specs::LIST) :: (::VOID) | +| db.createLabel | 创建Vertex/Edge Label | db.createLabel(label_type::STRING,label_name::STRING,extra::STRING,field_specs::LIST) :: () | +| db.getLabelSchema | 列出label schema | db.getLabelSchema(label_type::STRING,label_name::STRING) :: (name::STRING,type::STRING,optional::BOOLEAN) | +| db.getVertexSchema | 列出点的 schema | db.getVertexSchema(label::STRING) :: (schema::MAP) | +| db.getEdgeSchema | 列出边的 schema | db.getEdgeSchema(label::STRING) :: (schema::MAP) | +| db.deleteLabel | 删除Vertex/Edge Label | db.deleteLabel(label_type::STRING,label_name::STRING) :: (::VOID) | +| db.alterLabelDelFields | 修改label删除属性 | db.alterLabelDelFields(label_type::STRING,label_name::STRING,del_fields::LIST) :: (record_affected::INTEGER) | +| db.alterLabelAddFields | 修改label添加field | db.alterLabelAddFields(label_type::STRING,label_name::STRING,add_field_spec_values::LIST) :: (record_affected::INTEGER) | +| db.alterLabelModFields | 修改label field | db.alterLabelModFields(label_type::STRING,label_name::STRING,mod_field_specs::LIST) :: (record_affected::INTEGER) | +| db.createEdgeLabel | 创建Edge Label | db.createEdgeLabel(type_name::STRING,field_specs::LIST) :: (::VOID) | +| db.addIndex | 创建索引 | db.addIndex(label_name::STRING,field_name::STRING,unique::BOOLEAN) :: (::VOID) | +| db.addEdgeIndex | 创建索引 | db.addEdgeIndex(label_name::STRING,field_name::STRING,unique::BOOLEAN,pair_unique::BOOLEAN) :: (::VOID) | +| db.deleteIndex | 删除索引 | db.deleteIndex(label_name::STRING,field_name::STRING) :: (::VOID) | +| db.backup | 备份数据 | db.backup(destination::STRING) :: () | +| dbms.procedures | 列出所有procedures | dbms.procedures() :: (name::STRING,signature::STRING) | +| dbms.security.changePassword | 更改当前用户的密码 | dbms.security.changePassword(current_password::STRING,new_password::STRING) :: (::VOID) | +| dbms.security.changeUserPassword | 更改指定用户的密码 | dbms.security.changeUserPassword(user_name::STRING,new_password::STRING) :: (::VOID) | +| dbms.security.createUser | 创建用户 | dbms.security.createUser(user_name::STRING,password::STRING) :: (::VOID) | +| dbms.security.deleteUser | 删除用户 | dbms.security.deleteUser(user_name::STRING) :: (::VOID) | +| dbms.security.listUsers | 列出所有用户 | dbms.security.listUsers() :: (user_name::STRING,user_info::MAP) | +| dbms.security.showCurrentUser | 列出当前用户信息 | dbms.security.showCurrentUser() :: (current_user::STRING) | +| dbms.security.getUserPermissions | 列出指定用户的权限 | dbms.security.getUserPermissions(user_name::STRING) :: (user_info::MAP) | +| dbms.graph.createGraph | 创建子图 | dbms.graph.createGraph(graph_name::STRING, description::STRING, max_size_GB::INTEGER) :: (::VOID) | +| dbms.graph.modGraph | 修改子图属性 | dbms.graph.modGraph(graph_name::STRING,config::MAP) :: (::VOID) | +| dbms.graph.deleteGraph | 删除子图 | dbms.graph.deleteGraph(graph_name::STRING) :: (::VOID) | +| dbms.graph.listGraphs | 列出所有子图 | dbms.graph.listGraphs() :: (graph_name::STRING,configuration::MAP) | +| dbms.graph.getGraphInfo | 列出指定子图的信息 | dbms.graph.getGraphInfo(graph_name::STRING)::(graph_name::STRING,configuration::MAP) | +| dbms.security.addAllowedHosts | 添加ip到信任列表 | dbms.security.addAllowedHosts(hosts::LIST) :: (num_new::INTEGER) | +| dbms.security.deleteAllowedHosts | 从信任列表删除ip | dbms.security.deleteAllowedHosts(hosts::LIST) :: (record_affected::INTEGER) | +| dbms.security.listAllowedHosts | 列出信任列表中的主机ip | dbms.security.listAllowedHosts() :: (host::STRING) | +| dbms.config.update | 更新TuGraph配置 | dbms.config.update(updates::MAP) :: (message::STRING) | +| dbms.config.list | 列出TuGraph配置 | dbms.config.list() :: (name::STRING,value::ANY) | +| algo.shortestPath | 查询两个点间的最短路径 | algo.shortestPath(startNode::NODE,endNode::NODE,config::MAP) :: (nodeCount::INTEGER,totalCost::FLOAT) | +| algo.allShortestPaths | 查询两个点间的所有最短路径 | algo.allShortestPaths(startNode::NODE,endNode::NODE,config::MAP) :: (nodeIds::LIST,relationshipIds::LIST,cost::LIST) | +| algo.native.extract | 查询指定VertexId/EdgeUid（列表）指定field的值（列表） | algo.native.extract(id::ANY,config::MAP) :: (value::ANY) | +| db.flushDB | 刷新db | db.flushDB() :: (::VOID) | +| dbms.security.listRoles | 列出所有角色 | dbms.security.listRoles() :: (role_name::STRING,role_info::MAP) | +| dbms.security.createRole | 创建角色 | dbms.security.createRole(role_name::STRING,desc::STRING) :: (::VOID) | +| dbms.security.deleteRole | 删除角色 | dbms.security.deleteRole(role_name::STRING) :: (::VOID) | +| dbms.security.getRoleInfo | 获取角色详细信息 | dbms.security.getRoleInfo(role::STRING) :: (role_info::MAP) | +| dbms.security.disableRole | 禁用/启用角色 | dbms.security.disableRole(role::STRING,disable::BOOLEAN) :: (::VOID) | +| dbms.security.modRoleDesc | 修改角色描述信息 | dbms.security.modRoleDesc(role::STRING,description::STRING) :: (::VOID) | +| dbms.security.rebuildRoleAccessLevel | 删除角色权限并重建 | dbms.security.rebuildRoleAccessLevel(role::STRING,access_level::MAP) :: (::VOID) | +| dbms.security.modRoleAccessLevel | 修改角色对指定图的访问权限 | dbms.security.modRoleAccessLevel(role::STRING,access_level::MAP) :: (::VOID) | +| dbms.security.modRoleFieldAccessLevel | 修改角色对指定属性的访问权限 | dbms.security.modRoleFieldAccessLevel(role::STRING,graph::STRING,label::STRING,field::STRING,label_type::STRING,field_access_level::STRING) :: (::VOID) | +| dbms.security.getUserInfo | 获取用户详细信息 | dbms.security.getUserInfo(user::STRING) :: (user_info::MAP) | +| dbms.security.disableUser | 禁用/启用用户 | dbms.security.disableUser(user::STRING,disable::BOOLEAN) :: (::VOID) | +| dbms.security.setCurrentDesc | 设置当前用户描述信息 | dbms.security.setCurrentDesc(description::STRING) :: (::VOID) | +| dbms.security.setUserDesc | 设置用户描述信息 | dbms.security.setUserDesc(user::STRING,description::STRING) :: (::VOID) | +| dbms.security.getUserMemoryUsage | 获取用户内存用量 | dbms.security.getUserMemoryUsage(user::STRING) :: (memory_usage::INTEGER) | +| dbms.security.setUserMemoryLimit | 设置用户内存限制 | dbms.security.setUserMemoryLimit(user::STRING,memorylimit::INTEGER) :: (::VOID) | +| dbms.security.deleteUserRoles | 删除用户与角色的联系 | dbms.security.deleteUserRoles(user::STRING,roles::LIST) :: (::VOID) | +| dbms.security.rebuildUserRoles | 清空用户角色的关系并重建 | dbms.security.rebuildUserRoles(user::STRING,roles::LIST) :: (::VOID) | +| dbms.security.addUserRoles | 新增用户与角色的联系 | dbms.security.addUserRoles(user::STRING,roles::LIST) :: (::VOID) | +| db.plugin.loadPlugin | 装载plugin | db.plugin.loadPlugin(plugin_type::STRING,plugin_name::STRING,plugin_content::STRING or MAP,code_type::STRING,plugin_description::STRING,read_only::BOOLEAN,version::STRING) :: (::VOID) | +| db.plugin.deletePlugin | 删除plugin | db.plugin.deletePlugin(plugin_type::STRING,plugin_name::STRING) :: (::VOID) | +| db.plugin.listPlugin | 列出已装载的plugin | db.plugin.listPlugin(plugin_type::STRING,plugin_version::STRING) :: (plugin_description::LIST) | +| db.plugin.getPluginInfo | 获取plugin的详细信息 | db.plugin.getPluginInfo(plugin_type::STRING,plugin_name::STRING,show_code::BOOLEAN)::(plugin_description::MAP) | +| db.plugin.callPlugin | 执行plugin | db.plugin.callPlugin(plugin_type::STRING,plugin_name::STRING,param::STRING,timeout::DOUBLE,in_process::BOOLEAN) :: (success::BOOLEAN,result::STRING) | +| db.importor.dataImportor | 导入点或边数据 | db.importor.dataImportor(description::STRING,content::STRING,continue_on_error::BOOLEAN,thread_nums::INTEGER,delimiter::STRING) :: (::VOID) | +| db.importor.schemaImportor | 导入点或边schema | db.importor.schemaImportor(description::STRING) :: (::VOID) | +| db.addFullTextIndex | 添加全文索引 | db.addFullTextIndex(is_vertex::BOOLEAN, label_name::STRING, field_name::STRING) :: (::VOID) | +| db.deleteFullTextIndex | 删除全文索引 | db.deleteFullTextIndex(is_vertex::BOOLEAN, label_name::STRING, field_name::STRING) :: (::VOID) | +| db.rebuildFullTextIndex | 重建全文索引 | db.rebuildFullTextIndex(vertex_labels::STRING, edge_labels::STRING) :: (::VOID) | +| db.fullTextIndexes | 查看全文索引 | db.fullTextIndexes() :: (is_vertex::BOOLEAN, label::STRING, field::STRING) | +| dbms.meta.count | 查看点边总数 | db.dbms.meta.count() :: (type::STRING, number::INTEGER) | +| dbms.meta.countDetail | 查看点边总数详情 | db.dbms.meta.countDetail() :: (is_vertex::BOOLEAN, label::STRING, count::INTEGER) | +| dbms.meta.refreshCount | 重新统计点边数量，统计期间停写。 | db.dbms.meta.refreshCount() :: (::VOID) | +| dbms.task.listTasks | 查询正在执行的任务 | dbms.task.listTasks()::(tasks::LIST) | +| dbms.task.terminateTask | 中止任务 | dbms.task.terminateTask(task_id::STRING)::(::VOID) | +| dbms.ha.clusterInfo | HA模式下查看集群状态 | dbms.ha.clusterInfo() :: (cluster_info::LIST, is_master::BOOLEAN) | +| db.dropDB | 清空数据库 | db.dropDB() :: (::VOID) | diff --git a/docs/zh-CN/source/5.developer-manual/6.interface/3.procedure/1.procedure.md b/docs/zh-CN/source/5.developer-manual/6.interface/3.procedure/1.procedure.md index 47d0207278..1748f19794 100644 --- a/docs/zh-CN/source/5.developer-manual/6.interface/3.procedure/1.procedure.md +++ b/docs/zh-CN/source/5.developer-manual/6.interface/3.procedure/1.procedure.md @@ -217,26 +217,31 @@ extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { extern "C" LGAPI bool ProcessInTxn(Transaction &txn, const std::string &request, - std::string &response) { + Result &response) { int64_t limit; try { json input = json::parse(request); limit = input["limit"].get(); } catch (std::exception &e) { - response = std::string("error parsing json: ") + e.what(); + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); return false; } - Result result({{"node", LGraphType::NODE}, - {"salt", LGraphType::FLOAT}, - }); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"salt", LGraphType::FLOAT} + }); for (size_t i = 0; i < limit; i++) { - auto r = result.MutableRecord(); + auto r = response.MutableRecord(); auto vit = txn.GetVertexIterator(i); r->Insert("node", vit); r->Insert("salt", FieldData::Float(20.23*float(i))); } - response = result.Dump(); return true; } ``` diff --git a/include/lgraph/lgraph.h b/include/lgraph/lgraph.h index 25d6126e58..455363be03 100644 --- a/include/lgraph/lgraph.h +++ b/include/lgraph/lgraph.h @@ -33,12 +33,16 @@ #undef BOOL #endif +namespace lgraph_api { +class Result; +} + namespace lgraph_api { typedef bool GetSignature(SigSpec &sig_spec); typedef bool Process(lgraph_api::GraphDB &db, const std::string &input, std::string &output); typedef bool ProcessInTxn(lgraph_api::Transaction& txn, const std::string &input, - std::string &output); + lgraph_api::Result &output); } /* diff --git a/include/lgraph/lgraph_result.h b/include/lgraph/lgraph_result.h index 52587bb3a5..65488e78c0 100644 --- a/include/lgraph/lgraph_result.h +++ b/include/lgraph/lgraph_result.h @@ -235,6 +235,25 @@ class Result { */ Record *MutableRecord(); + /** + * @brief This function attempts to reserve enough memory for the result vector to hold + * the specified number of elements. + * + */ + void Reserve(size_t n); + + /** + * @brief This function will resize the vector to the specified number of elements + * + */ + void Resize(size_t n); + + /** + * @brief Provides access to the data contained in the vector. + * + */ + Record* At(size_t n); + /** * @brief return header of the table. * diff --git a/include/lgraph/lgraph_rpc_client.h b/include/lgraph/lgraph_rpc_client.h index efa7297f03..c5a8c10a36 100644 --- a/include/lgraph/lgraph_rpc_client.h +++ b/include/lgraph/lgraph_rpc_client.h @@ -73,7 +73,8 @@ class RpcClient { * @brief Load a user-defined procedure * * @param [out] result The result. - * @param [in] source_file the source_file contain procedure code. + * @param [in] source_files the source_file list contain procedure code(only + * for code_type cpp) * @param [in] procedure_type the procedure type, currently supported CPP and PY. * @param [in] procedure_name procedure name. * @param [in] code_type code type, currently supported PY, SO, CPP, ZIP. @@ -84,7 +85,7 @@ class RpcClient { * * @returns True if it succeeds, false if it fails. */ - bool LoadProcedure(std::string& result, const std::string& source_file, + bool LoadProcedure(std::string& result, const std::vector& source_files, const std::string& procedure_type, const std::string& procedure_name, const std::string& code_type, const std::string& procedure_description, bool read_only, const std::string& version = "v1", @@ -441,6 +442,27 @@ class RpcClient { bool read_only, const std::string& version = "v1", const std::string& graph = "default"); + /** + * @brief Load a built-in procedure + * + * @param [out] result The result. + * @param [in] source_files the source_file list contain procedure code(only +* for code_type cpp) + * @param [in] procedure_type the procedure type, currently supported CPP and PY. + * @param [in] procedure_name procedure name. + * @param [in] code_type code type, currently supported PY, SO, CPP, ZIP. + * @param [in] procedure_description procedure description. + * @param [in] read_only procedure is read only or not. + * @param [in] version (Optional) the version of procedure. + * @param [in] graph (Optional) the graph to query. + * @returns True if it succeeds, false if it fails. + */ + bool LoadProcedure(std::string& result, const std::vector& source_files, + const std::string& procedure_type, const std::string& procedure_name, + const std::string& code_type, const std::string& procedure_description, + bool read_only, const std::string& version = "v1", + const std::string& graph = "default"); + /** * @brief List user-defined procedures * diff --git a/include/lgraph/lgraph_utils.h b/include/lgraph/lgraph_utils.h index 549fc881f5..3202dc4505 100644 --- a/include/lgraph/lgraph_utils.h +++ b/include/lgraph/lgraph_utils.h @@ -142,4 +142,12 @@ void parse_from_json(std::vector& value, const char* key, json& input) } } +/** + * \brief Parse vid from the node passed in by cypher. For V2 procedure. + * + * \param[in] node_string node + * \return vid + */ +size_t GetVidFromNodeString(const std::string& node_string); + } // namespace lgraph_api diff --git a/include/lgraph/olap_base.h b/include/lgraph/olap_base.h index 3fc289357e..46dd017bdc 100644 --- a/include/lgraph/olap_base.h +++ b/include/lgraph/olap_base.h @@ -978,7 +978,6 @@ class OlapBase { * @param vertices The vertex id (in the Graph) to lock/unlock. * */ - void set_num_vertices(size_t vertices) { if (this->num_vertices_ == 0) { this->num_vertices_ = vertices; @@ -1041,6 +1040,7 @@ class OlapBase { num_threads = omp_get_num_threads(); } }; + // TODO(niyan.zy): move ThreadState to Construct ThreadState **thread_state; thread_state = new ThreadState *[num_threads]; for (int t_i = 0; t_i < num_threads; t_i++) { @@ -1166,6 +1166,7 @@ class OlapBase { num_threads = omp_get_num_threads(); } }; + // TODO(niyan.zy): move ThreadState to Construct ThreadState **thread_state; thread_state = new ThreadState *[num_threads]; for (int t_i = 0; t_i < num_threads; t_i++) { diff --git a/include/lgraph/olap_on_db.h b/include/lgraph/olap_on_db.h index 3b208a1e1f..14ee33fb2b 100644 --- a/include/lgraph/olap_on_db.h +++ b/include/lgraph/olap_on_db.h @@ -9,8 +9,8 @@ /** * @file olap_on_db.h - * @brief TuGraph OLAP interface. To implement a plugin that perform graph analytics on TuGraph, user - * can load a Snapshot from the database, and then use the Gather-Apply-Scatter style interface + * @brief TuGraph OLAP interface. To implement a plugin that perform graph analytics on TuGraph, + * user can load a Snapshot from the database, and then use the Gather-Apply-Scatter style interface * to do the computation. */ @@ -61,7 +61,7 @@ static constexpr size_t SNAPSHOT_IN_EDGES = 1ul << 6; */ template class OlapOnDB : public OlapBase { - GraphDB &db_; + GraphDB *db_; Transaction &txn_; ParallelVector original_vids_; cuckoohash_map vid_map_; @@ -106,29 +106,33 @@ class OlapOnDB : public OlapBase { vid_map_.reserve(this->num_vertices_); auto task_ctx = GetThreadContext(); auto worker = Worker::SharedWorker(); - if ((flags_ & SNAPSHOT_PARALLEL) && txn_.IsReadOnly()) { + if (flags_ & SNAPSHOT_PARALLEL) { // parallel generation worker->Delegate([&]() { int num_threads = 0; #pragma omp parallel { - if (omp_get_thread_num() == 0) { - num_threads = omp_get_num_threads(); - } + if (omp_get_thread_num() == 0) { + num_threads = omp_get_num_threads(); + } }; std::vector partition_offset(num_threads + 1, 0); std::vector out_edges_partition_offset(num_threads + 1, 0); + std::vector vits; + std::vector out_eits; + for (int i = 0; i < num_threads; i++) { + vits.template emplace_back(txn_.GetVertexIterator()); + out_eits.template emplace_back(vits[i].GetOutEdgeIterator()); + } #pragma omp parallel { ParallelVector local_original_vids(this->num_vertices_); ParallelVector local_out_index(this->num_vertices_ + 1); ParallelVector> local_out_edges(MAX_NUM_EDGES); - auto txn = db_.ForkTxn(txn_); int thread_id = omp_get_thread_num(); - int num_threads = omp_get_num_threads(); - auto vit = txn.GetVertexIterator(); + auto &vit = vits[thread_id]; for (size_t start = 64 * thread_id; start < this->num_vertices_; start += 64 * num_threads) { if (ShouldKillThisTask(task_ctx)) break; @@ -150,8 +154,8 @@ class OlapOnDB : public OlapBase { if (ShouldKillThisTask(task_ctx)) goto SNAPSHOT_PHASE1_ABORT; if (thread_id == 0) { - for (int thread_id = 0; thread_id < num_threads; thread_id++) { - partition_offset[thread_id + 1] += partition_offset[thread_id]; + for (int tid = 0; tid < num_threads; tid++) { + partition_offset[tid + 1] += partition_offset[tid]; } this->num_vertices_ = partition_offset[num_threads]; original_vids_.Resize(this->num_vertices_); @@ -179,11 +183,13 @@ class OlapOnDB : public OlapBase { if (vi % 64 == 0 && ShouldKillThisTask(task_ctx)) break; size_t original_vid = original_vids_[vi]; vit.Goto(original_vid); - for (auto eit = vit.GetOutEdgeIterator(); eit.IsValid(); eit.Next()) { - size_t dst = eit.GetDst(); + auto &out_eit = out_eits[thread_id]; + out_eit.Goto(EdgeUid{(int64_t)vi, 0, 0, 0, 0}, true); + for ( ; out_eit.IsValid(); out_eit.Next()) { + size_t dst = out_eit.GetDst(); EdgeData edata; if (vid_map_.contains(dst) && - (!out_edge_filter_ || out_edge_filter_(eit, edata))) { + (!out_edge_filter_ || out_edge_filter_(out_eit, edata))) { AdjUnit out_edge; out_edge.neighbour = dst; if (!std::is_same::value) { @@ -202,9 +208,9 @@ class OlapOnDB : public OlapBase { if (ShouldKillThisTask(task_ctx)) goto SNAPSHOT_PHASE1_ABORT; if (thread_id == 0) { - for (int thread_id = 0; thread_id < num_threads; thread_id++) { - out_edges_partition_offset[thread_id + 1] += - out_edges_partition_offset[thread_id]; + for (int tid = 0; tid < num_threads; tid++) { + out_edges_partition_offset[tid + 1] += + out_edges_partition_offset[tid]; } this->num_edges_ = out_edges_partition_offset[num_threads]; this->out_edges_.Resize(this->num_edges_); @@ -477,9 +483,8 @@ class OlapOnDB : public OlapBase { auto task_ctx = GetThreadContext(); auto worker = Worker::SharedWorker(); - // Read from TuGraph - if ((flags_ & SNAPSHOT_PARALLEL) && txn_.IsReadOnly()) { + if (flags_ & SNAPSHOT_PARALLEL) { this->out_index_.Resize(this->num_vertices_ + 1, (size_t)0); worker->Delegate([&]() { int num_threads = 0; @@ -491,15 +496,19 @@ class OlapOnDB : public OlapBase { }; std::vector out_edges_partition_offset(num_threads + 1, 0); + std::vector vits; + std::vector out_eits; + for (int i = 0; i < num_threads; i++) { + vits.template emplace_back(txn_.GetVertexIterator()); + out_eits.template emplace_back(vits[i].GetOutEdgeIterator()); + } #pragma omp parallel { ParallelVector local_out_index(this->num_vertices_); ParallelVector> local_out_edges(MAX_NUM_EDGES); - auto txn = db_.ForkTxn(txn_); int thread_id = omp_get_thread_num(); - int num_threads = omp_get_num_threads(); - auto vit = txn.GetVertexIterator(); + auto &vit = vits[thread_id]; size_t partition_size = this->num_vertices_ / num_threads; size_t partition_begin = partition_size * thread_id; @@ -512,7 +521,9 @@ class OlapOnDB : public OlapBase { for (size_t vi = partition_begin; vi < partition_end; vi++) { if (vi % 64 == 0 && ShouldKillThisTask(task_ctx)) break; vit.Goto(vi); - for (auto eit = vit.GetOutEdgeIterator(); eit.IsValid(); eit.Next()) { + auto &eit = out_eits[thread_id]; + eit.Goto(EdgeUid{(int64_t)vi, 0, 0, 0, 0}, true); + for ( ; eit.IsValid(); eit.Next()) { size_t dst = eit.GetDst(); EdgeData edata; if (!out_edge_filter_ || out_edge_filter_(eit, edata)) { @@ -797,7 +808,7 @@ class OlapOnDB : public OlapBase { this->in_degree_.Resize(this->num_vertices_, (size_t)0); } - if ((flags_ & SNAPSHOT_PARALLEL) && txn_.IsReadOnly()) { + if (flags_ & SNAPSHOT_PARALLEL) { worker->Delegate([&]() { int num_threads = 0; #pragma omp parallel @@ -809,12 +820,18 @@ class OlapOnDB : public OlapBase { std::vector out_edges_partition_offset(num_threads + 1, 0); std::vector in_edges_partition_offset(num_threads + 1, 0); + std::vector vits; + std::vector out_eits; + std::vector in_eits; + for (int i = 0; i < num_threads; i++) { + vits.template emplace_back(txn_.GetVertexIterator()); + out_eits.template emplace_back(vits[i].GetOutEdgeIterator()); + in_eits.template emplace_back(vits[i].GetInEdgeIterator()); + } #pragma omp parallel { - auto txn = db_.ForkTxn(txn_); int thread_id = omp_get_thread_num(); - int num_threads = omp_get_num_threads(); - auto vit = txn.GetVertexIterator(); + auto &vit = vits[thread_id]; size_t partition_size = this->num_vertices_ / num_threads; size_t partition_begin = partition_size * thread_id; @@ -885,21 +902,25 @@ class OlapOnDB : public OlapBase { if (vi % 64 == 0 && ShouldKillThisTask(task_ctx)) break; if (!vit.Goto(vi)) continue; size_t pos = this->out_index_[vi]; - for (auto eit = vit.GetOutEdgeIterator(); eit.IsValid(); eit.Next()) { - size_t dst = eit.GetDst(); + auto &out_eit = out_eits[thread_id]; + out_eit.Goto(EdgeUid{(int64_t)vi, 0, 0, 0, 0}, true); + for (; out_eit.IsValid(); out_eit.Next()) { + size_t dst = out_eit.GetDst(); this->out_edges_[pos].neighbour = dst; pos++; } + auto &in_eit = in_eits[thread_id]; + in_eit.Goto(EdgeUid{0, (int64_t)vi, 0, 0, 0}, true); if (flags_ & SNAPSHOT_UNDIRECTED) { - for (auto eit = vit.GetInEdgeIterator(); eit.IsValid(); eit.Next()) { - size_t src = eit.GetSrc(); + for (; in_eit.IsValid(); in_eit.Next()) { + size_t src = in_eit.GetSrc(); this->out_edges_[pos].neighbour = src; pos++; } } else { pos = this->in_index_[vi]; - for (auto eit = vit.GetInEdgeIterator(); eit.IsValid(); eit.Next()) { - size_t src = eit.GetSrc(); + for (; in_eit.IsValid(); in_eit.Next()) { + size_t src = in_eit.GetSrc(); this->in_edges_[pos].neighbour = src; pos++; } @@ -914,38 +935,42 @@ class OlapOnDB : public OlapBase { {}; } }); -// } else { -// auto vit = txn_.GetVertexIterator(); -// for (size_t vid = 0; vid < this->num_vertices_; vid++) { -// if (!vit.Goto(vid)) continue; -// for (auto eit = vit.GetOutEdgeIterator(); eit.IsValid(); eit.Next()) { -// size_t dst = eit.GetDst(); -// AdjUnit out_edge; -// out_edge.neighbour = dst; -// this->out_edges_.Append(out_edge, false); -// } -// if (flags_ & SNAPSHOT_UNDIRECTED) { -// for (auto eit = vit.GetInEdgeIterator(); eit.IsValid(); eit.Next()) { -// size_t src = eit.GetSrc(); -// AdjUnit in_edge; -// in_edge.neighbour = src; -// this->out_edges_.Append(in_edge, false); -// } -// } else { -// for (auto eit = vit.GetInEdgeIterator(); eit.IsValid(); eit.Next()) { -// size_t src = eit.GetSrc(); -// AdjUnit in_edge; -// in_edge.neighbour = src; -// this->in_edges_.Append(in_edge, false); -// } -// this->in_index_[vid + 1] = this->in_edges_.Size(); -// this->in_degree_[vid] = this->in_index_[vid + 1] - this->in_index_[vid]; -// } -// this->out_index_[vid + 1] = this->out_edges_.Size(); -// this->out_degree_[vid] = this->out_index_[vid + 1] - this->out_index_[vid]; -// } -// this->num_edges_ = this->out_edges_.Size(); -// this->out_edges_.Resize(this->num_edges_); + // } else { + // auto vit = txn_.GetVertexIterator(); + // for (size_t vid = 0; vid < this->num_vertices_; vid++) { + // if (!vit.Goto(vid)) continue; + // for (auto eit = vit.GetOutEdgeIterator(); eit.IsValid(); eit.Next()) { + // size_t dst = eit.GetDst(); + // AdjUnit out_edge; + // out_edge.neighbour = dst; + // this->out_edges_.Append(out_edge, false); + // } + // if (flags_ & SNAPSHOT_UNDIRECTED) { + // for (auto eit = vit.GetInEdgeIterator(); eit.IsValid(); + // eit.Next()) { + // size_t src = eit.GetSrc(); + // AdjUnit in_edge; + // in_edge.neighbour = src; + // this->out_edges_.Append(in_edge, false); + // } + // } else { + // for (auto eit = vit.GetInEdgeIterator(); eit.IsValid(); + // eit.Next()) { + // size_t src = eit.GetSrc(); + // AdjUnit in_edge; + // in_edge.neighbour = src; + // this->in_edges_.Append(in_edge, false); + // } + // this->in_index_[vid + 1] = this->in_edges_.Size(); + // this->in_degree_[vid] = this->in_index_[vid + 1] - + // this->in_index_[vid]; + // } + // this->out_index_[vid + 1] = this->out_edges_.Size(); + // this->out_degree_[vid] = this->out_index_[vid + 1] - + // this->out_index_[vid]; + // } + // this->num_edges_ = this->out_edges_.Size(); + // this->out_edges_.Resize(this->num_edges_); } this->lock_array_.Resize(this->num_vertices_); this->lock_array_.Fill(false); @@ -953,7 +978,37 @@ class OlapOnDB : public OlapBase { public: /** - * @brief Generate a graph with LightningGraph. + * @brief Generate a graph with LightningGraph. For V1/V2 Procedures + */ + OlapOnDB(GraphDB* db, Transaction &txn, size_t flags = 0, + std::function vertex_filter = nullptr, + std::function out_edge_filter = nullptr) + : db_(db), + txn_(txn), + flags_(flags), + vertex_filter_(vertex_filter), + out_edge_filter_(out_edge_filter) { + if (txn.GetNumVertices() == 0) { + throw std::runtime_error("The graph cannot be empty"); + } + if (vertex_filter != nullptr) { + flags_ |= SNAPSHOT_IDMAPPING; + } + Init(txn.GetNumVertices()); + + if (flags_ & SNAPSHOT_IDMAPPING) { + Construct(); + } else { + if ((out_edge_filter == nullptr) && (flags_ & SNAPSHOT_PARALLEL) && txn_.IsReadOnly()) { + ConstructWithDegree(); + } else { + ConstructWithVid(); + } + } + } + + /** + * @brief Generate a graph with LightningGraph. For V1 Procedures * * @exception std::runtime_error Raised when a runtime error condition * occurs. @@ -966,9 +1021,7 @@ class OlapOnDB : public OlapBase { * @param [in,out] out_edge_filter (Optional) A function filtering out * edges. * - * Note that the transaction must be read- - * only if SNAPSHOT_PARALLEL is specified - * (actually read-write transactions are + * Note: read-write transactions are * not recommended here for safety, e.g. * some vertices might be removed causing * inconsistencies of the analysis, and @@ -986,42 +1039,19 @@ class OlapOnDB : public OlapBase { OlapOnDB(GraphDB &db, Transaction &txn, size_t flags = 0, std::function vertex_filter = nullptr, std::function out_edge_filter = nullptr) - : db_(db), - txn_(txn), - flags_(flags), - vertex_filter_(vertex_filter), - out_edge_filter_(out_edge_filter) { - if (txn.GetNumVertices() == 0) { - throw std::runtime_error("The graph cannot be empty"); - } - if (vertex_filter != nullptr) { - flags_ |= SNAPSHOT_IDMAPPING; - } - Init(txn.GetNumVertices()); - - if (flags_ & SNAPSHOT_IDMAPPING) { - Construct(); - } else { - if ((out_edge_filter == nullptr) && (flags_ & SNAPSHOT_PARALLEL) && txn_.IsReadOnly()) { - ConstructWithDegree(); - } else { - ConstructWithVid(); - } - } - } + : OlapOnDB(&db, txn, flags, vertex_filter, out_edge_filter) {} // Filter subgraphs based on a set of triples of point labels, edge labels, and point labels OlapOnDB(GraphDB &db, Transaction &txn, std::vector> label_list, - size_t flags = 0): db_(db), - txn_(txn), - flags_(flags) { + size_t flags = 0) + : db_(&db), txn_(txn), flags_(flags) { if (txn.GetNumVertices() == 0) { throw std::runtime_error("The graph cannot be empty"); } flags_ |= SNAPSHOT_IDMAPPING; Init(txn.GetNumVertices()); std::vector> label_id_list; - for (auto& labels : label_list) { + for (auto &labels : label_list) { std::vector label_id; label_id.push_back(txn.GetVertexLabelId(labels[0])); label_id.push_back(txn.GetEdgeLabelId(labels[1])); @@ -1038,9 +1068,9 @@ class OlapOnDB : public OlapBase { int num_threads = 0; #pragma omp parallel { - if (omp_get_thread_num() == 0) { - num_threads = omp_get_num_threads(); - } + if (omp_get_thread_num() == 0) { + num_threads = omp_get_num_threads(); + } }; std::vector partition_offset(num_threads + 1, 0); @@ -1053,7 +1083,7 @@ class OlapOnDB : public OlapBase { local_out_index.Append(0, false); - auto txn = db_.ForkTxn(txn_); + auto txn = db_->ForkTxn(txn_); int thread_id = omp_get_thread_num(); int num_threads = omp_get_num_threads(); auto vit = txn.GetVertexIterator(); @@ -1073,8 +1103,8 @@ class OlapOnDB : public OlapBase { while (eit.IsValid()) { auto dst = eit.GetDst(); vit.Goto(dst); - if (size_t(eit.GetLabelId()) == labels[1] - && vit.GetLabelId() == labels[2]) { + if (size_t(eit.GetLabelId()) == labels[1] && + vit.GetLabelId() == labels[2]) { keepVertex = true; AdjUnit out_edge; out_edge.neighbour = dst; @@ -1088,8 +1118,8 @@ class OlapOnDB : public OlapBase { while (eit.IsValid()) { auto src = eit.GetSrc(); vit.Goto(src); - if (size_t(eit.GetLabelId()) == labels[1] - && vit.GetLabelId() == labels[0]) { + if (size_t(eit.GetLabelId()) == labels[1] && + vit.GetLabelId() == labels[0]) { keepVertex = true; break; } @@ -1377,12 +1407,30 @@ class OlapOnDB : public OlapBase { if (ShouldKillThisTask(task_ctx)) throw std::runtime_error("Task killed"); } + /** + * @brief Generate a graph without LightningGraph. For V2 Procedures + * + * @exception std::runtime_error Raised when a runtime error condition + * occurs. + * + * @param [in,out] txn The transaction. + * @param flags (Optional) The generation flags. + * @param [in,out] vertex_filter (Optional) A function filtering + * vertices. + * @param [in,out] out_edge_filter (Optional) A function filtering out + * edges. + **/ + OlapOnDB(Transaction &txn, size_t flags = 0, + std::function vertex_filter = nullptr, + std::function out_edge_filter = nullptr) + : OlapOnDB(nullptr, txn, flags, vertex_filter, out_edge_filter) {} + OlapOnDB() = delete; OlapOnDB(const OlapOnDB &rhs) = delete; OlapOnDB(OlapOnDB &&rhs) = default; - OlapOnDB& operator=(OlapOnDB &&rhs) { + OlapOnDB &operator=(OlapOnDB &&rhs) { printf("OlapOnDB assigment\n"); return *this; } @@ -1401,12 +1449,12 @@ class OlapOnDB : public OlapBase { std::function extract) { auto task_ctx = GetThreadContext(); ParallelVector a(this->num_vertices_, this->num_vertices_); - if (txn_.IsReadOnly()) { + if (txn_.IsReadOnly() && db_ != nullptr) { auto worker = Worker::SharedWorker(); worker->Delegate([&]() { #pragma omp parallel { - auto txn = db_.ForkTxn(txn_); + auto txn = db_->ForkTxn(txn_); int thread_id = omp_get_thread_num(); int num_threads = omp_get_num_threads(); size_t start = this->num_vertices_ / num_threads * thread_id; @@ -1456,13 +1504,12 @@ class OlapOnDB : public OlapBase { * */ template - void WriteToFile(ParallelVector& vertex_data, - const std::string& output_file) { + void WriteToFile(ParallelVector &vertex_data, const std::string &output_file) { fma_common::OutputFmaStream fout; fout.Open(output_file, 64 << 20); for (size_t i = 0; i < this->num_vertices_; ++i) { - std::string line = fma_common::StringFormatter::Format( - "{} {}\n", OriginalVid(i), vertex_data[i]); + std::string line = + fma_common::StringFormatter::Format("{} {}\n", OriginalVid(i), vertex_data[i]); fout.Write(line.c_str(), line.size()); } } @@ -1475,9 +1522,11 @@ class OlapOnDB : public OlapBase { * */ template - void WriteToGraphDB(ParallelVector& vertex_data, - const std::string& vertex_field) { - auto write_txn = db_.CreateWriteTxn(); + void WriteToGraphDB(ParallelVector &vertex_data, const std::string &vertex_field) { + if (db_ == nullptr) { + throw std::runtime_error("can't write to graph because db is null"); + } + auto write_txn = db_->CreateWriteTxn(); auto vit = write_txn.GetVertexIterator(); for (size_t i = 0; i < this->num_vertices_; i++) { FieldData local_distance(std::to_string(vertex_data[i])); @@ -1531,7 +1580,7 @@ class OlapOnDB : public OlapBase { */ template std::function edge_convert_default = - [] (OutEdgeIterator &eit, EdgeData &edge_data) -> bool { + [](OutEdgeIterator &eit, EdgeData &edge_data) -> bool { edge_data = 1; return true; }; @@ -1543,7 +1592,7 @@ std::function edge_convert_default = */ template std::function edge_convert_weight = - [](OutEdgeIterator &eit, EdgeData &edge_data) -> bool { + [](OutEdgeIterator &eit, EdgeData &edge_data) -> bool { edge_data = eit.GetField("weight").real(); return true; }; diff --git a/procedures/CMakeLists.txt b/procedures/CMakeLists.txt index e0b9146bb1..cd137d1b20 100644 --- a/procedures/CMakeLists.txt +++ b/procedures/CMakeLists.txt @@ -39,6 +39,13 @@ function(add_extension APP) LIBRARY_OUTPUT_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/algo") endfunction() +function(add_v2procedure APP) + add_library(${APP}_v2 SHARED algo_cpp/${APP}_core.cpp algo_cpp/${APP}_procedure_v2.cpp) + target_link_libraries(${APP}_v2 ${Boost_LIBRARIES} lgraph libgomp.a crypto) + set_target_properties( ${APP}_v2 PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/algo") +endfunction() + add_standalone(apsp) add_standalone(bfs) add_standalone(pagerank) @@ -126,3 +133,10 @@ add_extension(sssp) add_extension(wcc) add_extension(lcc) add_extension(lpa) + +add_v2procedure(bfs) +add_v2procedure(lcc) +add_v2procedure(wcc) +add_v2procedure(sssp) +add_v2procedure(lpa) +add_v2procedure(pagerank) diff --git a/procedures/algo_cpp/bfs_procedure_v2.cpp b/procedures/algo_cpp/bfs_procedure_v2.cpp new file mode 100644 index 0000000000..118cf6dad1 --- /dev/null +++ b/procedures/algo_cpp/bfs_procedure_v2.cpp @@ -0,0 +1,88 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#include +#include "lgraph/lgraph.h" +#include "lgraph/lgraph_types.h" +#include "lgraph/lgraph_result.h" +#include "tools/json.hpp" +#include "./algo.h" + +using json = nlohmann::json; +using namespace lgraph_api; +using namespace lgraph_api::olap; + +extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { + sig_spec.input_list = { + {.name = "root", .index = 0, .type = LGraphType::NODE}, + }; + sig_spec.result_list = { + {.name = "node", .index = 0, .type = LGraphType::NODE}, + {.name = "parent", .index = 1, .type = LGraphType::NODE} + }; + return true; +} + +extern "C" LGAPI bool ProcessInTxn(Transaction &txn, + const std::string &request, + Result &response) { + double start_time = get_time(); + std::string root_node; + LOG_DEBUG() << "input: " << request; + try { + json input = json::parse(request); + parse_from_json(root_node, "root", input); + } catch (std::exception &e) { + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); + return false; + } + + OlapOnDB graph(txn, SNAPSHOT_PARALLEL); + auto prepare_cost = get_time() - start_time; + start_time = get_time(); + auto parent = graph.AllocVertexArray(); + size_t root_vid = graph.MappedVid(GetVidFromNodeString(root_node)); + auto discovered_vertices = BFSCore(graph, root_vid, parent); + auto core_cost = get_time() - start_time; + + start_time = get_time(); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"parent", LGraphType::NODE}, + }); + + response.Resize(discovered_vertices); + size_t idx = 0; + for (size_t vid = 0; vid < parent.Size(); vid++) { + if (parent[vid] == (size_t)-1) { + continue; + } + auto r = response.At(idx); + r->InsertVertexByID("node", graph.OriginalVid(vid)); + r->InsertVertexByID("parent", graph.OriginalVid(parent[vid])); + idx++; + } + auto output_cost = get_time() - start_time; + + LOG_DEBUG() << "prepare_cost: " << prepare_cost << " (s)\n" + << "core_cost: " << core_cost << " (s)\n" + << "output_cost: " << output_cost << " (s)\n" + << "response.Size: " << response.Size(); + return true; +} diff --git a/procedures/algo_cpp/lcc_procedure_v2.cpp b/procedures/algo_cpp/lcc_procedure_v2.cpp new file mode 100644 index 0000000000..e5d6acd5d4 --- /dev/null +++ b/procedures/algo_cpp/lcc_procedure_v2.cpp @@ -0,0 +1,70 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#include +#include "lgraph/lgraph.h" +#include "lgraph/lgraph_types.h" +#include "lgraph/lgraph_result.h" +#include "tools/json.hpp" +#include "./algo.h" + +using json = nlohmann::json; +using namespace lgraph_api; +using namespace lgraph_api::olap; + +extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { + sig_spec.input_list = {}; + sig_spec.result_list = { + {.name = "node", .index = 0, .type = LGraphType::NODE}, + {.name = "score", .index = 1, .type = LGraphType::DOUBLE} + }; + return true; +} + +extern "C" LGAPI bool ProcessInTxn(Transaction &txn, + const std::string &request, + Result &response) { + double start_time = get_time(); + LOG_DEBUG() << "input: " << request; + + OlapOnDB graph(txn, SNAPSHOT_PARALLEL | SNAPSHOT_UNDIRECTED); + auto prepare_cost = get_time() - start_time; + start_time = get_time(); + auto score = graph.AllocVertexArray(); + LCCCore(graph, score); + auto core_cost = get_time() - start_time; + + start_time = get_time(); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"score", LGraphType::DOUBLE}, + }); + + response.Resize(score.Size()); + graph.ProcessVertexInRange( + [&] (size_t vid) { + auto r = response.At(vid); + r->InsertVertexByID("node", graph.OriginalVid(vid)); + r->Insert("score", FieldData::Double(score[vid])); + return 0; + }, + 0, score.Size()); + auto output_cost = get_time() - start_time; + + LOG_DEBUG() << "prepare_cost: " << prepare_cost << " (s)\n" + << "core_cost: " << core_cost << " (s)\n" + << "output_cost: " << output_cost << " (s)\n" + << "response.Size: " << response.Size(); + return true; +} diff --git a/procedures/algo_cpp/lpa_procedure_v2.cpp b/procedures/algo_cpp/lpa_procedure_v2.cpp new file mode 100644 index 0000000000..c3555824b6 --- /dev/null +++ b/procedures/algo_cpp/lpa_procedure_v2.cpp @@ -0,0 +1,85 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#include +#include "lgraph/lgraph.h" +#include "lgraph/lgraph_types.h" +#include "lgraph/lgraph_result.h" +#include "tools/json.hpp" +#include "./algo.h" + +using json = nlohmann::json; +using namespace lgraph_api; +using namespace lgraph_api::olap; + +extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { + sig_spec.input_list = { + {.name = "num_iteration", .index = 0, .type = LGraphType::INTEGER}, + }; + sig_spec.result_list = { + {.name = "node", .index = 0, .type = LGraphType::NODE}, + {.name = "label", .index = 1, .type = LGraphType::INTEGER} + }; + return true; +} + +extern "C" LGAPI bool ProcessInTxn(Transaction &txn, + const std::string &request, + Result &response) { + double start_time = get_time(); + int64_t num_iteration; + LOG_DEBUG() << "input: " << request; + try { + json input = json::parse(request); + parse_from_json(num_iteration, "num_iteration", input); + } catch (std::exception &e) { + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); + return false; + } + + OlapOnDB graph(txn, SNAPSHOT_PARALLEL | SNAPSHOT_UNDIRECTED); + auto prepare_cost = get_time() - start_time; + start_time = get_time(); + auto labels = graph.AllocVertexArray(); + LPACore(graph, labels, num_iteration, true); + auto core_cost = get_time() - start_time; + + start_time = get_time(); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"label", LGraphType::INTEGER}, + }); + + response.Resize(labels.Size()); + graph.ProcessVertexInRange( + [&] (size_t vid) { + auto r = response.At(vid); + r->InsertVertexByID("node", graph.OriginalVid(vid)); + r->Insert("label", FieldData::Int64(labels[vid])); + return 0; + }, + 0, labels.Size()); + auto output_cost = get_time() - start_time; + + LOG_DEBUG() << "prepare_cost: " << prepare_cost << " (s)\n" + << "core_cost: " << core_cost << " (s)\n" + << "output_cost: " << output_cost << " (s)\n" + << "response.Size: " << response.Size(); + return true; +} diff --git a/procedures/algo_cpp/pagerank_procedure_v2.cpp b/procedures/algo_cpp/pagerank_procedure_v2.cpp new file mode 100644 index 0000000000..6a403dee42 --- /dev/null +++ b/procedures/algo_cpp/pagerank_procedure_v2.cpp @@ -0,0 +1,89 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +*/ + +#include +#include "lgraph/lgraph.h" +#include "lgraph/lgraph_types.h" +#include "lgraph/lgraph_result.h" +#include "tools/json.hpp" +#include "./algo.h" + +using json = nlohmann::json; +using namespace lgraph_api; +using namespace lgraph_api::olap; + +extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { + sig_spec.input_list = { + {.name = "num_iteration", .index = 0, .type = LGraphType::INTEGER}, + }; + sig_spec.result_list = { + {.name = "node", .index = 0, .type = LGraphType::NODE}, + {.name = "weight", .index = 1, .type = LGraphType::DOUBLE} + }; + return true; +} + +extern "C" LGAPI bool ProcessInTxn(Transaction &txn, + const std::string &request, + Result &response) { + double start_time = get_time(); + int64_t num_iteration; + LOG_DEBUG() << "input: " << request; + try { + json input = json::parse(request); + num_iteration = input["num_iteration"].get(); + } catch (std::exception &e) { + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); + return false; + } + + OlapOnDB graph(txn, SNAPSHOT_PARALLEL); + auto prepare_cost = get_time() - start_time; + start_time = get_time(); + auto pr = graph.AllocVertexArray(); + PageRankCore(graph, num_iteration, pr); + size_t max_pr_vi = graph.ProcessVertexInRange( + [&](size_t vi) { return vi; }, 0, graph.NumVertices(), 0, + [&](size_t a, size_t b) { return pr[a] > pr[b] ? a : b; }); + LOG_INFO() << FMA_FMT("max_pr: pr[{}] = {}", max_pr_vi, pr[max_pr_vi]); + auto core_cost = get_time() - start_time; + + start_time = get_time(); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"weight", LGraphType::DOUBLE}, + }); + + response.Resize(pr.Size()); + graph.ProcessVertexInRange( + [&] (size_t vid) { + auto r = response.At(vid); + r->InsertVertexByID("node", graph.OriginalVid(vid)); + r->Insert("weight", FieldData::Double(pr[vid])); + return 0; + }, + 0, pr.Size()); + auto output_cost = get_time() - start_time; + + LOG_DEBUG() << "prepare_cost: " << prepare_cost << " (s)\n" + << "core_cost: " << core_cost << " (s)\n" + << "output_cost: " << output_cost << " (s)\n" + << "response.Size: " << response.Size(); + return true; +} diff --git a/procedures/algo_cpp/sssp_procedure_v2.cpp b/procedures/algo_cpp/sssp_procedure_v2.cpp new file mode 100644 index 0000000000..f4944a572d --- /dev/null +++ b/procedures/algo_cpp/sssp_procedure_v2.cpp @@ -0,0 +1,88 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#include +#include "lgraph/lgraph.h" +#include "lgraph/lgraph_types.h" +#include "lgraph/lgraph_result.h" +#include "tools/json.hpp" +#include "./algo.h" + +using json = nlohmann::json; +using namespace lgraph_api; +using namespace lgraph_api::olap; + +extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { + sig_spec.input_list = { + {.name = "root", .index = 0, .type = LGraphType::NODE}, + }; + sig_spec.result_list = { + {.name = "node", .index = 0, .type = LGraphType::NODE}, + {.name = "distance", .index = 1, .type = LGraphType::DOUBLE} + }; + return true; +} + +extern "C" LGAPI bool ProcessInTxn(Transaction &txn, + const std::string &request, + Result &response) { + double start_time = get_time(); + std::string root_node; + LOG_DEBUG() << "input: " << request; + try { + json input = json::parse(request); + parse_from_json(root_node, "root", input); + } catch (std::exception &e) { + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); + return false; + } + + std::function edge_filter = edge_convert_default; + + OlapOnDB graph(txn, SNAPSHOT_PARALLEL, nullptr, edge_filter); + auto prepare_cost = get_time() - start_time; + start_time = get_time(); + auto dis = graph.AllocVertexArray(); + auto root_vid = graph.MappedVid(GetVidFromNodeString(root_node)); + SSSPCore(graph, root_vid, dis); + auto core_cost = get_time() - start_time; + + start_time = get_time(); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"distance", LGraphType::DOUBLE}, + }); + + response.Resize(dis.Size()); + graph.ProcessVertexInRange( + [&] (size_t vid) { + auto r = response.At(vid); + r->InsertVertexByID("node", graph.OriginalVid(vid)); + r->Insert("distance", FieldData::Double(dis[vid])); + return 0; + }, + 0, dis.Size()); + auto output_cost = get_time() - start_time; + + LOG_DEBUG() << "prepare_cost: " << prepare_cost << " (s)\n" + << "core_cost: " << core_cost << " (s)\n" + << "output_cost: " << output_cost << " (s)\n" + << "response.Size: " << response.Size(); + return true; +} diff --git a/procedures/algo_cpp/wcc_procedure_v2.cpp b/procedures/algo_cpp/wcc_procedure_v2.cpp new file mode 100644 index 0000000000..94ae1ae6cb --- /dev/null +++ b/procedures/algo_cpp/wcc_procedure_v2.cpp @@ -0,0 +1,70 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#include +#include "lgraph/lgraph.h" +#include "lgraph/lgraph_types.h" +#include "lgraph/lgraph_result.h" +#include "tools/json.hpp" +#include "./algo.h" + +using json = nlohmann::json; +using namespace lgraph_api; +using namespace lgraph_api::olap; + +extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { + sig_spec.input_list = {}; + sig_spec.result_list = { + {.name = "node", .index = 0, .type = LGraphType::NODE}, + {.name = "label", .index = 1, .type = LGraphType::INTEGER} + }; + return true; +} + +extern "C" LGAPI bool ProcessInTxn(Transaction &txn, + const std::string &request, + Result &response) { + double start_time = get_time(); + LOG_DEBUG() << "input: " << request; + + OlapOnDB graph(txn, SNAPSHOT_PARALLEL | SNAPSHOT_UNDIRECTED); + auto prepare_cost = get_time() - start_time; + start_time = get_time(); + auto labels = graph.AllocVertexArray(); + WCCCore(graph, labels); + auto core_cost = get_time() - start_time; + + start_time = get_time(); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"label", LGraphType::INTEGER}, + }); + + response.Resize(labels.Size()); + graph.ProcessVertexInRange( + [&] (size_t vid) { + auto r = response.At(vid); + r->InsertVertexByID("node", graph.OriginalVid(vid)); + r->Insert("label", FieldData::Int64(labels[vid])); + return 0; + }, + 0, labels.Size()); + auto output_cost = get_time() - start_time; + + LOG_DEBUG() << "prepare_cost: " << prepare_cost << " (s)\n" + << "core_cost: " << core_cost << " (s)\n" + << "output_cost: " << output_cost << " (s)\n" + << "response.Size: " << response.Size(); + return true; +} diff --git a/procedures/demo/v2_pagerank.cpp b/procedures/demo/v2_pagerank.cpp index 8a0c42374b..98da566043 100644 --- a/procedures/demo/v2_pagerank.cpp +++ b/procedures/demo/v2_pagerank.cpp @@ -10,7 +10,7 @@ * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -*/ + */ #include @@ -21,8 +21,74 @@ #include "tools/json.hpp" using json = nlohmann::json; - using namespace lgraph_api; +using namespace lgraph_api::olap; + +#define __ADD_DANGLING__ + +void PageRankCore(OlapBase& graph, int num_iterations, ParallelVector& curr) { + auto all_vertices = graph.AllocVertexSubset(); + all_vertices.Fill(); + auto next = graph.AllocVertexArray(); + size_t num_vertices = graph.NumVertices(); + + double one_over_n = (double)1 / num_vertices; + double delta = 1; + double dangling = graph.ProcessVertexActive( + [&](size_t vi) { + curr[vi] = one_over_n; + if (graph.OutDegree(vi) > 0) { + curr[vi] /= graph.OutDegree(vi); + return 0.0; + } +#ifdef __ADD_DANGLING__ + return one_over_n; +#else + return 0.0; +#endif + }, + all_vertices); + dangling /= num_vertices; + + double d = (double)0.85; + for (int ii = 0; ii < num_iterations; ii++) { + printf("delta(%d)=%lf\n", ii, delta); + next.Fill((double)0); + delta = graph.ProcessVertexActive( + [&](size_t vi) { + double sum = 0; + for (auto& edge : graph.InEdges(vi)) { + size_t src = edge.neighbour; + sum += curr[src]; + } + next[vi] = sum; + next[vi] = (1 - d) * one_over_n + d * next[vi] + d * dangling; + if (ii == num_iterations - 1) { + return (double)0; + } else { + if (graph.OutDegree(vi) > 0) { + next[vi] /= graph.OutDegree(vi); + return fabs(next[vi] - curr[vi]) * graph.OutDegree(vi); + } else { + return fabs(next[vi] - curr[vi]); + } + } + }, + all_vertices); + curr.Swap(next); + +#ifdef __ADD_DANGLING__ + dangling = graph.ProcessVertexActive( + [&](size_t vi) { + if (graph.OutDegree(vi) == 0) + return curr[vi]; + return 0.0; + }, + all_vertices); + dangling /= num_vertices; +#endif + } +} extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { sig_spec.input_list = { @@ -30,36 +96,61 @@ extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { }; sig_spec.result_list = { {.name = "node", .index = 0, .type = LGraphType::NODE}, - {.name = "weight", .index = 1, .type = LGraphType::FLOAT} + {.name = "weight", .index = 1, .type = LGraphType::DOUBLE} }; return true; } extern "C" LGAPI bool ProcessInTxn(Transaction &txn, const std::string &request, - std::string &response) { + Result &response) { + double start_time = get_time(); int64_t num_iteration; + LOG_DEBUG() << "input: " << request; try { json input = json::parse(request); num_iteration = input["num_iteration"].get(); } catch (std::exception &e) { - response = std::string("error parsing json: ") + e.what(); + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); return false; } - // handle the page rank algo in dummy mode - // ... - Result result({{"node", LGraphType::NODE}, - {"weight", LGraphType::FLOAT}, - }); + OlapOnDB graph(txn, SNAPSHOT_PARALLEL); + auto prepare_cost = get_time() - start_time; + start_time = get_time(); + auto pr = graph.AllocVertexArray(); + PageRankCore(graph, num_iteration, pr); + size_t max_pr_vi = graph.ProcessVertexInRange( + [&](size_t vi) { return vi; }, 0, graph.NumVertices(), 0, + [&](size_t a, size_t b) { return pr[a] > pr[b] ? a : b; }); + LOG_INFO() << FMA_FMT("max_pr: pr[{}] = {}", max_pr_vi, pr[max_pr_vi]); + auto core_cost = get_time() - start_time; + start_time = get_time(); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"weight", LGraphType::DOUBLE}, + }); - for (size_t i = 0; i < 2; i++) { - auto r = result.MutableRecord(); - auto vit = txn.GetVertexIterator(i); - r->Insert("node", vit); - r->Insert("weight", FieldData::Float(float(i) + 0.1*float(i))); - } - response = result.Dump(); + response.Resize(pr.Size()); + graph.ProcessVertexInRange( + [&] (size_t vid) { + auto r = response.At(vid); + r->InsertVertexByID("node", graph.OriginalVid(vid)); + r->Insert("weight", FieldData::Double(pr[vid])); + return 0; + }, + 0, pr.Size()); + auto output_cost = get_time() - start_time; + + LOG_DEBUG() << "prepare_cost: " << prepare_cost << " (s)\n" + << "core_cost: " << core_cost << " (s)\n" + << "output_cost: " << output_cost << " (s)\n" + << "response.Size: " << response.Size(); return true; } diff --git a/src/client/cpp/restful/rest_client.cpp b/src/client/cpp/restful/rest_client.cpp index 8fa9cb0896..d9cdd069c4 100644 --- a/src/client/cpp/restful/rest_client.cpp +++ b/src/client/cpp/restful/rest_client.cpp @@ -1051,8 +1051,43 @@ bool RestClient::LoadPlugin(const std::string& db, lgraph_api::PluginCodeType ty body[RestStrings::READONLY] = json::value::boolean(plugin_info.read_only); body[RestStrings::DESC] = json::value::string(_TU(plugin_info.desc)); body[RestStrings::VERSION] = json::value::string(_TU(plugin_info.version)); - body[RestStrings::CODE] = json::value::string(_TU(lgraph_api::base64::Encode(code))); + body[RestStrings::CODE] = json::value::array( + std::vector{json::value::string(_TU(lgraph_api::base64::Encode(code)))}); body[RestStrings::CODE_TYPE] = json::value::string(_TU(lgraph_api::PluginCodeTypeStr(type))); + if (type == lgraph_api::PluginCodeType::CPP) { + body[RestStrings::FILENAMES] = json::value::array( + std::vector{json::value::string(_TU(plugin_info.name + ".cpp"))}); + } else { + body[RestStrings::FILENAMES] = json::value::array( + std::vector{json::value::string(_TU(plugin_info.name))}); + } + + DoGraphPost(db, + FMA_FMT("/{}_plugin", type == lgraph_api::PluginCodeType::PY ? "python" : "cpp"), + body, false); + LOG_DEBUG() << "[RestClient] " << __func__ << " succeeded"; + return true; +} + +bool RestClient::LoadPlugin(const std::string& db, lgraph_api::PluginCodeType type, + const PluginDesc& plugin_info, const std::vector& codes, + const std::vector& filenames) { + json::value body; + body[RestStrings::NAME] = json::value::string(_TU(plugin_info.name)); + body[RestStrings::READONLY] = json::value::boolean(plugin_info.read_only); + body[RestStrings::DESC] = json::value::string(_TU(plugin_info.desc)); + body[RestStrings::VERSION] = json::value::string(_TU(plugin_info.version)); + std::vector code_base64; + for (auto& code : codes) { + code_base64.push_back(json::value::string(_TU(lgraph_api::base64::Encode(code)))); + } + body[RestStrings::CODE] = json::value::array(code_base64); + body[RestStrings::CODE_TYPE] = json::value::string(_TU(lgraph_api::PluginCodeTypeStr(type))); + std::vector names_json; + for (auto f : filenames) { + names_json.push_back(json::value::string(_TU(f))); + } + body[RestStrings::FILENAMES] = json::value::array(names_json); DoGraphPost(db, FMA_FMT("/{}_plugin", type == lgraph_api::PluginCodeType::PY ? "python" : "cpp"), body, false); diff --git a/src/client/cpp/restful/rest_client.h b/src/client/cpp/restful/rest_client.h index e32ee348d1..4b8de9282d 100644 --- a/src/client/cpp/restful/rest_client.h +++ b/src/client/cpp/restful/rest_client.h @@ -259,6 +259,10 @@ class RestClient { bool LoadPlugin(const std::string& db, lgraph_api::PluginCodeType type, const PluginDesc& plugin_info, const std::string& code); + bool LoadPlugin(const std::string& db, lgraph_api::PluginCodeType type, + const PluginDesc& plugin_info, const std::vector& codes, + const std::vector& filenames); + std::vector GetPlugin(const std::string& db, bool is_cpp); lgraph::PluginCode GetPluginDetail(const std::string& db, const std::string& name, bool is_cpp); diff --git a/src/client/cpp/rpc/lgraph_rpc_client.cpp b/src/client/cpp/rpc/lgraph_rpc_client.cpp index f867e5be57..59ad431f7d 100644 --- a/src/client/cpp/rpc/lgraph_rpc_client.cpp +++ b/src/client/cpp/rpc/lgraph_rpc_client.cpp @@ -283,18 +283,24 @@ std::string RpcClient::RpcSingleClient::GraphQueryResponseExtractor(const GraphQ return ""; } -bool RpcClient::RpcSingleClient::LoadProcedure(std::string& result, const std::string& source_file, - const std::string& procedure_type, - const std::string& procedure_name, - const std::string& code_type, - const std::string& procedure_description, - bool read_only, const std::string& version, - const std::string& graph) { +bool RpcClient::RpcSingleClient::LoadProcedure(std::string& result, + const std::vector& source_files, + const std::string& procedure_type, const std::string& procedure_name, + const std::string& code_type, const std::string& procedure_description, + bool read_only, const std::string& version, const std::string& graph) { + if (source_files.size() > 1 && code_type != "CPP") { + result = "Only cpp files support uploading multiple files"; + return false; + } try { - std::string content; - if (!FieldSpecSerializer::FileReader(source_file, content)) { - std::swap(content, result); - return false; + std::vector content; + content.reserve(source_files.size()); + for (auto &file_path : source_files) { + content.emplace_back(""); + if (!FieldSpecSerializer::FileReader(file_path, content.back())) { + std::swap(content.back(), result); + return false; + } } LGraphRequest req; req.set_is_write_op(true); @@ -318,7 +324,12 @@ bool RpcClient::RpcSingleClient::LoadProcedure(std::string& result, const std::s loadPluginRequest->set_name(procedure_name); loadPluginRequest->set_desc(procedure_description); loadPluginRequest->set_read_only(read_only); - loadPluginRequest->set_code(content); + for (auto& file_name : source_files) { + loadPluginRequest->add_file_name(fma_common::Split(file_name, "/").back()); + } + for (auto& code : content) { + loadPluginRequest->add_code(code); + } HandleRequest(&req); } catch (std::exception& e) { result = e.what(); @@ -685,14 +696,24 @@ bool RpcClient::LoadProcedure(std::string &result, const std::string &source_fil bool read_only, const std::string& version, const std::string &graph) { + return RpcClient::LoadProcedure(result, std::vector{source_file}, + procedure_type, procedure_name, code_type, + procedure_description, read_only, version, + graph); +} + +bool RpcClient::LoadProcedure(std::string& result, const std::vector& source_files, + const std::string& procedure_type, const std::string& procedure_name, + const std::string& code_type, const std::string& procedure_description, + bool read_only, const std::string& version, const std::string& graph) { if (client_type == SINGLE_CONNECTION) { - return base_client->LoadProcedure(result, source_file, procedure_type, procedure_name, + return base_client->LoadProcedure(result, source_files, procedure_type, procedure_name, code_type, procedure_description, read_only, version, graph); } auto fun = [&]{ bool succeed = GetClient(false)-> - LoadProcedure(result, source_file, procedure_type, procedure_name, + LoadProcedure(result, source_files, procedure_type, procedure_name, code_type, procedure_description, read_only, version, graph); if (succeed) { RefreshUserDefinedProcedure(); diff --git a/src/client/python/TuGraphClient/TuGraphClient.py b/src/client/python/TuGraphClient/TuGraphClient.py index 208417e5f4..f89ad77005 100644 --- a/src/client/python/TuGraphClient/TuGraphClient.py +++ b/src/client/python/TuGraphClient/TuGraphClient.py @@ -220,11 +220,12 @@ async def load_plugin(self, name, desc, file_type, file_path, read_only, version content = f.read() data = {} data['name'] = name - data['code_base64'] = base64.b64encode(content).decode() + data['code_base64'] = [base64.b64encode(content).decode()] data['description'] = desc data['read_only'] = read_only data['code_type'] = file_type data['version'] = version + data['file_name'] = [file_path.split('/')[-1]] if file_type == 'cpp' or file_type == 'zip' or file_type == 'so': url = 'cpp_plugin' elif file_type == 'py': diff --git a/src/client/python/rpc/client.cpp b/src/client/python/rpc/client.cpp index f609710ff2..975377060b 100644 --- a/src/client/python/rpc/client.cpp +++ b/src/client/python/rpc/client.cpp @@ -85,9 +85,33 @@ void register_liblgraph_client_python(pybind11::module& m) { pybind11::arg("json_format") = true, pybind11::arg("timeout") = 0, pybind11::return_value_policy::move); - c.def("loadProcedure", &LGraphPythonClient::LoadProcedure, + c.def("loadProcedure", [] (LGraphPythonClient& self, pybind11::object source_files, + pybind11::object procedure_type, + pybind11::object procedure_name, + pybind11::object code_type, + pybind11::object procedure_description, + pybind11::object read_only, pybind11::object version, + pybind11::object graph) { + if (pybind11::isinstance(source_files)) { + return self.LoadProcedure( + source_files.cast(), procedure_type.cast(), + procedure_name.cast(), code_type.cast(), + procedure_description.cast(), read_only.cast(), + version.cast(), graph.cast()); + } else if (pybind11::isinstance(source_files)) { + return self.LoadProcedure( + source_files.cast>(), + procedure_type.cast(), + procedure_name.cast(), code_type.cast(), + procedure_description.cast(), read_only.cast(), + version.cast(), graph.cast()); + } else { + throw std::invalid_argument("Invalid argument types for LoadProcedure"); + } + }, "Load a user-defined procedure\n" - "source_file [in] the source_file contain procedure code\n" + "source_files [in] Source file or source file list containing code (only cpp " + "files support uploading multiple files)\n" "procedure_type [in] the procedure type, currently supported CPP and PY\n" "procedure_name [in] procedure name\n" "code_type [in] code type, currently supported PY, SO, CPP, ZIP\n" @@ -95,7 +119,7 @@ void register_liblgraph_client_python(pybind11::module& m) { "read_only [in] procedure is read only or not\n" "version [in] procedure version, currently v1 or v2" "graph [in] the graph to query.\n", - pybind11::arg("source_file"), pybind11::arg("procedure_type"), + pybind11::arg("source_files"), pybind11::arg("procedure_type"), pybind11::arg("procedure_name"), pybind11::arg("code_type"), pybind11::arg("procedure_description"), pybind11::arg("read_only"), diff --git a/src/client/python/rpc/lgraph_python_client.h b/src/client/python/rpc/lgraph_python_client.h index 3c01a1fc9a..b7069df49d 100644 --- a/src/client/python/rpc/lgraph_python_client.h +++ b/src/client/python/rpc/lgraph_python_client.h @@ -48,6 +48,26 @@ class LGraphPythonClient { return {ret, result}; } + std::pair LoadProcedure(const std::vector& source_files, + const std::string& procedure_type, + const std::string& procedure_name, + const std::string& code_type, + const std::string& procedure_description, + bool read_only, const std::string& version = "v1", + const std::string& graph = "default") { + std::string result; + bool ret; + try { + ret = client->LoadProcedure(result, source_files, procedure_type, + procedure_name, code_type, + procedure_description, read_only, version, graph); + } catch (lgraph::RpcException &e) { + ret = false; + result = e.what(); + } + return {ret, result}; + } + std::pair CallProcedure(const std::string& procedure_type, const std::string& procedure_name, const std::string& param, diff --git a/src/core/defs.h b/src/core/defs.h index 33991b14ec..e0ed4c2c85 100644 --- a/src/core/defs.h +++ b/src/core/defs.h @@ -175,6 +175,7 @@ static const char* const PLUGIN_CODE_TYPE_CPP = "cpp"; static const char* const PLUGIN_CODE_TYPE_SO = "so"; static const char* const PLUGIN_CODE_TYPE_ZIP = "zip"; static const char* const PLUGIN_CODE_TYPE_PY = "py"; +static const char* const PLUGIN_CODE_DELIMITER = "\n---PLUGIN---FILE---BOUNDARY---\n"; typedef ::lgraph_api::PluginCodeType CodeType; } // namespace plugin diff --git a/src/cypher/procedure/procedure.cpp b/src/cypher/procedure/procedure.cpp index cf7a136f91..8ef2bc0f20 100644 --- a/src/cypher/procedure/procedure.cpp +++ b/src/cypher/procedure/procedure.cpp @@ -1659,8 +1659,6 @@ void BuiltinProcedure::DbPluginLoadPlugin(RTContext *ctx, const Record *record, "plugin_type type should be string") CYPHER_ARG_CHECK(args[1].type == parser::Expression::STRING, "plugin_name type should be string") - CYPHER_ARG_CHECK(args[2].type == parser::Expression::STRING, - "plugin_content type should be string") CYPHER_ARG_CHECK(args[3].type == parser::Expression::STRING, "code_type type should be string") CYPHER_ARG_CHECK(args[4].type == parser::Expression::STRING, "plugin_description type should be string") @@ -1675,11 +1673,32 @@ void BuiltinProcedure::DbPluginLoadPlugin(RTContext *ctx, const Record *record, auto code_type_it = ValidPluginCodeType.find(args[3].String()); CYPHER_ARG_CHECK(code_type_it != ValidPluginCodeType.end(), "unknown plugin_type, one of ('PY', 'SO', 'CPP', 'ZIP')"); + bool success = false; fma_common::encrypt::Base64 base64; - std::string content = base64.Decode(args[2].String()); - bool success = - db.LoadPlugin(plugin_type_it->second, ctx->user_, args[1].String(), content, - code_type_it->second, args[4].String(), args[5].Bool(), args[6].String()); + if (args[2].type == parser::Expression::STRING) { + std::string content = base64.Decode(args[2].String()); + success = + db.LoadPlugin(plugin_type_it->second, ctx->user_, args[1].String(), + std::vector{content}, std::vector{}, + code_type_it->second, args[4].String(), args[5].Bool(), args[6].String()); + } else if (args[2].type == parser::Expression::MAP) { + std::vector filenames; + std::vector codes; + for (auto &kv : args[2].Map()) { + if (kv.first[0] == '`' && kv.first.back() == '`') { + filenames.push_back(kv.first.substr(1, kv.first.size() - 2)); + } else { + filenames.push_back(kv.first); + } + codes.push_back(base64.Decode(kv.second.String())); + } + success = + db.LoadPlugin(plugin_type_it->second, ctx->user_, args[1].String(), codes, filenames, + code_type_it->second, args[4].String(), args[5].Bool(), args[6].String()); + } else { + throw lgraph::ReminderException("plugin_content should be string or map"); + } + if (!success) { throw lgraph::PluginExistException(args[1].String()); } diff --git a/src/cypher/procedure/procedure.h b/src/cypher/procedure/procedure.h index 9944e12705..fff4ed5389 100644 --- a/src/cypher/procedure/procedure.h +++ b/src/cypher/procedure/procedure.h @@ -880,7 +880,7 @@ static std::vector global_procedures = { Procedure("db.plugin.loadPlugin", BuiltinProcedure::DbPluginLoadPlugin, Procedure::SIG_SPEC{{"plugin_type", {0, lgraph_api::LGraphType::STRING}}, {"plugin_name", {1, lgraph_api::LGraphType::STRING}}, - {"plugin_content", {2, lgraph_api::LGraphType::STRING}}, + {"plugin_content", {2, lgraph_api::LGraphType::ANY}}, {"code_type", {3, lgraph_api::LGraphType::STRING}}, {"plugin_description", {4, lgraph_api::LGraphType::STRING}}, {"read_only", {5, lgraph_api::LGraphType::BOOLEAN}}, diff --git a/src/cypher/procedure/utils.h b/src/cypher/procedure/utils.h index 6e0926d80e..9b8cfb9066 100644 --- a/src/cypher/procedure/utils.h +++ b/src/cypher/procedure/utils.h @@ -84,6 +84,8 @@ class PluginAdapter { public: NodeBuffer() { buffer_.reserve(128); } ~NodeBuffer() = default; + void Reserve(int64_t size) { buffer_.reserve(size); } + int64_t Capacity() { return buffer_.capacity(); } DISABLE_COPY(NodeBuffer); DISABLE_MOVE(NodeBuffer); Node& AllocNode(NodeID id, const std::string& alias) { @@ -198,13 +200,24 @@ class PluginAdapter { } request.append("}"); - std::string response; - ctx->ac_db_->CallPlugin(ctx->txn_.get(), type_, "A_DUMMY_TOKEN_FOR_CPP_PLUGIN", name_, - request, 0, false, response); + lgraph_api::Result api_result; + bool ret = ctx->ac_db_->CallV2Plugin(ctx->txn_.get(), + type_, "A_DUMMY_TOKEN_FOR_CPP_PLUGIN", name_, + request, 0, false, api_result); + if (!ret) { + throw lgraph::CypherException("Plugin return false, errMsg: " + api_result.Dump()); + } try { - lgraph_api::Result api_result; - api_result.Load(response); + results->reserve(results->size() + api_result.Size()); + int64_t node_num_in_result = 0; + for (const auto& result : sig_spec_->result_list) { + if (result.type == lgraph_api::LGraphType::NODE) { + node_num_in_result += api_result.Size(); + } + } + node_buffer_.Reserve(node_num_in_result); + for (int64_t i = 0; i < api_result.Size(); i++) { const auto& rview = api_result.RecordView(i); Record r; @@ -255,7 +268,7 @@ class PluginAdapter { results->emplace_back(std::move(r)); } } catch (std::exception& e) { - response = std::string("error parsing json: ") + e.what(); + LOG_WARN() << "error parsing json: " << e.what(); return false; } return true; diff --git a/src/db/db.cpp b/src/db/db.cpp index 6bc71084fc..3e31f25d1c 100644 --- a/src/db/db.cpp +++ b/src/db/db.cpp @@ -58,12 +58,14 @@ lgraph::Transaction lgraph::AccessControlledDB::ForkTxn(Transaction& txn) { } bool lgraph::AccessControlledDB::LoadPlugin(plugin::Type plugin_type, const std::string& user, - const std::string& name, const std::string& code, + const std::string& name, + const std::vector& code, + const std::vector& filename, plugin::CodeType code_type, const std::string& desc, bool is_read_only, const std::string& version) { CheckAdmin(); - return graph_->GetPluginManager()->LoadPluginFromCode(plugin_type, user, name, code, code_type, - desc, is_read_only, version); + return graph_->GetPluginManager()->LoadPluginFromCode(plugin_type, user, name, code, filename, + code_type, desc, is_read_only, version); } bool lgraph::AccessControlledDB::DelPlugin(plugin::Type plugin_type, const std::string& user, @@ -92,6 +94,26 @@ bool lgraph::AccessControlledDB::CallPlugin(lgraph_api::Transaction* txn, output); } +bool lgraph::AccessControlledDB::CallV2Plugin(lgraph_api::Transaction* txn, + plugin::Type plugin_type, const std::string& user, + const std::string& name, const std::string& request, + double timeout_seconds, bool in_process, + Result& output) { + auto pm = graph_->GetPluginManager(); + bool is_readonly = pm->IsReadOnlyPlugin(plugin_type, user, name); + if (access_level_ < AccessLevel::WRITE && !is_readonly) + THROW_CODE(Unauthorized, "Write permission needed to call this plugin."); + return pm->CallV2(txn, + plugin_type, + user, + this, + name, + request, + timeout_seconds, + in_process, + output); +} + std::vector lgraph::AccessControlledDB::ListPlugins(plugin::Type plugin_type, const std::string& user) { return graph_->GetPluginManager()->ListPlugins(plugin_type, user); diff --git a/src/db/db.h b/src/db/db.h index 38e6f96b3f..24baee0fcc 100644 --- a/src/db/db.h +++ b/src/db/db.h @@ -50,8 +50,9 @@ class AccessControlledDB { Transaction ForkTxn(Transaction& txn); bool LoadPlugin(plugin::Type plugin_type, const std::string& token, const std::string& name, - const std::string& code, plugin::CodeType code_type, const std::string& desc, - bool is_read_only, const std::string& version); + const std::vector& code, const std::vector& filename, + plugin::CodeType code_type, const std::string& desc, bool is_read_only, + const std::string& version); bool DelPlugin(plugin::Type plugin_type, const std::string& token, const std::string& name); @@ -60,6 +61,10 @@ class AccessControlledDB { const std::string& request, double timeout_seconds, bool in_process, std::string& output); + bool CallV2Plugin(lgraph_api::Transaction* txn, plugin::Type plugin_type, + const std::string& user, const std::string& name, const std::string& request, + double timeout_seconds, bool in_process, Result& output); + std::vector ListPlugins(plugin::Type plugin_type, const std::string& token); bool GetPluginCode(plugin::Type plugin_type, const std::string& token, const std::string& name, diff --git a/src/http/http_server.cpp b/src/http/http_server.cpp index 923401144f..11f7cd007b 100644 --- a/src/http/http_server.cpp +++ b/src/http/http_server.cpp @@ -750,19 +750,26 @@ void HttpService::DoUploadProcedure(const brpc::Controller* cntl, std::string& r } preq->set_type(type); - std::string procedureName, content, description, codeType; + std::string procedureName, description, codeType; + std::vector content, filenames; bool readonly; GET_FIELD_OR_THROW_BAD_REQUEST(params, std::string, "procedureName", procedureName); - GET_FIELD_OR_THROW_BAD_REQUEST(params, std::string, "content", content); GET_FIELD_OR_THROW_BAD_REQUEST(params, std::string, "description", description); GET_FIELD_OR_THROW_BAD_REQUEST(params, std::string, "codeType", codeType); GET_FIELD_OR_THROW_BAD_REQUEST(params, bool, "readonly", readonly); + GET_FIELD_OR_THROW_BAD_REQUEST(params, std::vector, "content", content); + GET_FIELD_OR_THROW_BAD_REQUEST(params, std::vector, "file_name", filenames); LoadPluginRequest* req = preq->mutable_load_plugin_request(); req->set_name(procedureName); req->set_read_only(readonly); - std::vector decoded = utility::conversions::from_base64(content); - req->set_code(std::string(decoded.begin(), decoded.end())); + for (auto &code : content) { + std::vector decoded = utility::conversions::from_base64(code); + req->add_code(std::string(decoded.begin(), decoded.end())); + } + for (const auto& filename : filenames) { + req->add_file_name(filename); + } req->set_desc(description); lgraph::LoadPluginRequest::CodeType _codeType; _GET_PLUGIN_REQUEST_CODE_TYPE(codeType, _codeType); diff --git a/src/lgraph_api/lgraph_result.cpp b/src/lgraph_api/lgraph_result.cpp index 346ccd9ce3..4cdb69c9a2 100644 --- a/src/lgraph_api/lgraph_result.cpp +++ b/src/lgraph_api/lgraph_result.cpp @@ -299,6 +299,19 @@ Record *Result::MutableRecord() { return &result[row_count_]; } +void Result::Reserve(size_t n) { + result.reserve(n); +} + +void Result::Resize(size_t n) { + result.resize(n, Record(header)); + row_count_ = (int64_t)(n - 1); +} + +Record* Result::At(size_t n) { + return &result.at(n); +} + void Result::ResetHeader(const std::vector> &new_header) { result.clear(); header = new_header; diff --git a/src/lgraph_api/lgraph_utils.cpp b/src/lgraph_api/lgraph_utils.cpp index e7e20f0886..388343e59b 100644 --- a/src/lgraph_api/lgraph_utils.cpp +++ b/src/lgraph_api/lgraph_utils.cpp @@ -183,4 +183,11 @@ void dealloc_buffer(void* buffer, size_t bytes) { } #endif } + +size_t GetVidFromNodeString(const std::string& node_string) { + // node_string format: V[{id}] + auto id_begin = node_string.find('['); + return strtoll(node_string.c_str() + id_begin + 1, NULL, 10); +} + } // namespace lgraph_api diff --git a/src/plugin/cpp_plugin.cpp b/src/plugin/cpp_plugin.cpp index 64cf78954b..baaca301cf 100644 --- a/src/plugin/cpp_plugin.cpp +++ b/src/plugin/cpp_plugin.cpp @@ -89,13 +89,41 @@ void CppPluginManagerImpl::DoCall(lgraph_api::Transaction* txn, r = procedure(db, request, output); } else if (info.func_txn && txn != nullptr) { PluginFuncInTxn * procedure = info.func_txn; - r = procedure(*txn, request, output); + Result re; + r = procedure(*txn, request, re); + output = re.Dump(); } CloseDynamicLib(info); if (!r) THROW_CODE(InputError, "Plugin returned false. Output: {}.", output); } +void CppPluginManagerImpl::DoCallV2(lgraph_api::Transaction* txn, + const std::string& user, + AccessControlledDB* db_with_access_control, + const std::string name, const PluginInfoBase* pinfo, + const std::string& request, + double timeout, bool in_process, Result& output) { + if (timeout > 0) { + // TODO: schedule a timer event to kill this task // NOLINT + } + + // TODO: support in_process // NOLINT + bool r = false; + DynamicLibinfo info; + OpenDynamicLib(pinfo, info); + if (info.func) { + CloseDynamicLib(info); + THROW_CODE(InputError, "Only support the V2 version procedure"); + } else if (info.func_txn && txn != nullptr) { + PluginFuncInTxn * procedure = info.func_txn; + r = procedure(*txn, request, output); + } + CloseDynamicLib(info); + if (!r) THROW_CODE(InputError, + FMA_FMT("Plugin returned false. Output: {}.", output.Dump(false))); +} + void CppPluginManagerImpl::LoadPlugin(const std::string& user, const std::string& name, PluginInfoBase* pinfo) { using namespace lgraph::dll; diff --git a/src/plugin/cpp_plugin.h b/src/plugin/cpp_plugin.h index 13a05b7cb8..8de5d92ce5 100644 --- a/src/plugin/cpp_plugin.h +++ b/src/plugin/cpp_plugin.h @@ -159,5 +159,11 @@ class CppPluginManagerImpl : public PluginManagerImplBase { AccessControlledDB* db_with_access_control, const std::string name, const PluginInfoBase* pinfo, const std::string& request, double timeout, bool in_process, std::string& output) override; + + void DoCallV2(lgraph_api::Transaction* txn, + const std::string& user, + AccessControlledDB* db_with_access_control, + const std::string name, const PluginInfoBase* pinfo, const std::string& request, + double timeout, bool in_process, Result& output) override; }; } // namespace lgraph diff --git a/src/plugin/plugin_manager.cpp b/src/plugin/plugin_manager.cpp index 527852d8b2..43c68fa307 100644 --- a/src/plugin/plugin_manager.cpp +++ b/src/plugin/plugin_manager.cpp @@ -337,21 +337,28 @@ std::string lgraph::SingleLanguagePluginManager::CompilePluginFromZip(const std: return ReadWholeFile(plugin_file, "plugin binary file"); } -std::string lgraph::SingleLanguagePluginManager::CompilePluginFromCpp(const std::string& name, - const std::string& file) { +std::string lgraph::SingleLanguagePluginManager::CompilePluginFromCpp( + const std::string& name, const std::string& all_codes) { #ifdef _WIN32 #endif std::string base_dir = impl_->GetPluginDir(); auto& fs = fma_common::FileSystem::GetFileSystem(base_dir); std::string tmp_dir = GenUniqueTempDir(base_dir, name); - std::string file_path = tmp_dir + fs.PathSeparater() + name + ".cpp"; + std::string file_path = tmp_dir + fs.PathSeparater() + "/"; std::string plugin_path = tmp_dir + fs.PathSeparater() + name + ".so"; AutoCleanDir tmp_dir_cleaner(tmp_dir); // compile - WriteWholeFile(file_path, file, "plugin source file"); + std::vector filename; + std::vector code; + SplitCode(code, filename, all_codes); + std::string source_files = ""; + for (size_t i = 0; i < filename.size(); i++) { + WriteWholeFile(file_path + filename[i], code[i], "plugin source file-" + std::to_string(i)); + source_files += FMA_FMT(" {}/{}", file_path, filename[i]); + } std::string exec_dir = fma_common::FileSystem::GetExecutablePath().Dir(); std::string CFLAGS = FMA_FMT("-I{}/../../include -I/usr/local/include", exec_dir); std::string LDFLAGS = FMA_FMT("-llgraph -L{}/ -L/usr/local/lib64/", exec_dir); @@ -361,24 +368,24 @@ std::string lgraph::SingleLanguagePluginManager::CompilePluginFromCpp(const std: "g++ -fno-gnu-unique " " -fPIC -g --std=c++17 {} -Wl,-z,nodelete " " -rdynamic -O3 -fopenmp -o {} {} {} -shared ", - CFLAGS, plugin_path, file_path, LDFLAGS); + CFLAGS, plugin_path, source_files, LDFLAGS); #else std::string cmd = FMA_FMT( "g++ -fno-gnu-unique -fPIC -g " " --std=c++17 {} -rdynamic -O3 -fopenmp -o {} {} {} -shared", - CFLAGS, plugin_path, file_path, LDFLAGS); + CFLAGS, plugin_path, source_files, LDFLAGS); #endif #elif __APPLE__ std::string cmd = FMA_FMT( "clang++ -stdlib=libc++ " " -fPIC -g --std=c++17 {} -rdynamic -O3 -Xpreprocessor -fopenmp -o " "{} {} {} -shared", - CFLAGS, plugin_path, file_path, LDFLAGS); + CFLAGS, plugin_path, source_files, LDFLAGS); #else std::string cmd = FMA_FMT( "clang++ -stdlib=libc++ " " -fPIC -g --std=c++17 {} -rdynamic -O3 -fopenmp -o {} {} {} -shared", - CFLAGS, plugin_path, file_path, LDFLAGS); + CFLAGS, plugin_path, source_files, LDFLAGS); #endif ExecuteCommand(cmd, _detail::MAX_COMPILE_TIME_MS, "Timeout while compiling plugin.", "Failed to compile plugin."); @@ -414,11 +421,20 @@ void lgraph::SingleLanguagePluginManager::LoadPlugin(const std::string& user, Kv } bool lgraph::SingleLanguagePluginManager::LoadPluginFromCode( - const std::string& user, const std::string& name_, const std::string& code, + const std::string& user, const std::string& name_, + const std::vector& code, + const std::vector& filename, plugin::CodeType code_type, const std::string& desc, bool read_only, const std::string& version) { // check input - if (code.empty()) THROW_CODE(InputError, "Code cannot be empty."); + bool empty_code = code.empty(); + for (auto& c : code) { + if (c.empty()) { + empty_code = true; + break; + } + } + if (empty_code) THROW_CODE(InputError, "Code cannot be empty."); if (!IsValidPluginName(name_)) throw InvalidPluginNameException(name_); if (version != plugin::PLUGIN_VERSION_1 && version != plugin::PLUGIN_VERSION_2) { throw InvalidPluginVersionException(version); @@ -451,17 +467,29 @@ bool lgraph::SingleLanguagePluginManager::LoadPluginFromCode( // load plugin from different type std::string exe; + std::string all_codes; + if (code_type == plugin::CodeType::CPP) { + all_codes = MergeCodeFiles(code, filename, name); + } switch (code_type) { case plugin::CodeType::SO: break; case plugin::CodeType::PY: - exe = CompilePluginFromCython(name, code); + if (code.size() != 1) { + THROW_CODE(InternalError, + FMA_FMT("code_type [{}] only supports uploading a single file.", code_type)); + } + exe = CompilePluginFromCython(name, code[0]); break; case plugin::CodeType::CPP: - exe = CompilePluginFromCpp(name, code); + exe = CompilePluginFromCpp(name, all_codes); break; case plugin::CodeType::ZIP: - exe = CompilePluginFromZip(name, code); + if (code.size() != 1) { + THROW_CODE(InternalError, + FMA_FMT("code_type [{}] only supports uploading a single file.", code_type)); + } + exe = CompilePluginFromZip(name, code[0]); break; default: THROW_CODE(InternalError, "Unhandled code_type [{}].", code_type); @@ -474,18 +502,19 @@ bool lgraph::SingleLanguagePluginManager::LoadPluginFromCode( switch (code_type) { case plugin::CodeType::PY: LoadPlugin(user, txn.GetTxn(), name, exe, desc, read_only, version); - UpdateCythonToKvStore(txn.GetTxn(), name, code); + UpdateCythonToKvStore(txn.GetTxn(), name, code[0]); break; case plugin::CodeType::SO: - LoadPlugin(user, txn.GetTxn(), name, code, desc, read_only, version); + LoadPlugin(user, txn.GetTxn(), name, code[0], desc, read_only, version); break; case plugin::CodeType::CPP: LoadPlugin(user, txn.GetTxn(), name, exe, desc, read_only, version); - UpdateCppToKvStore(txn.GetTxn(), name, code); + + UpdateCppToKvStore(txn.GetTxn(), name, all_codes); break; case plugin::CodeType::ZIP: LoadPlugin(user, txn.GetTxn(), name, exe, desc, read_only, version); - UpdateZipToKvStore(txn.GetTxn(), name, code); + UpdateZipToKvStore(txn.GetTxn(), name, code[0]); break; default: THROW_CODE(InternalError, "Unhandled code_type [{}].", code_type); @@ -562,6 +591,23 @@ bool lgraph::SingleLanguagePluginManager::Call(lgraph_api::Transaction* txn, return true; } +bool lgraph::SingleLanguagePluginManager::CallV2(lgraph_api::Transaction* txn, + const std::string& user, + AccessControlledDB* db_with_access_control, + const std::string& name_, + const std::string& request, + double timeout, bool in_process, + Result& output) { + std::string name = ToInternalName(name_); + AutoReadLock lock(lock_, GetMyThreadId()); + auto it = procedures_.find(name); + if (it == procedures_.end()) return false; + impl_->DoCallV2(txn, user, db_with_access_control, + name, it->second, request, timeout, in_process, + output); + return true; +} + bool lgraph::SingleLanguagePluginManager::isHashUpTodate(KvTransaction& txn, std::string name) { std::string hash_key = GetHashKey(name); auto hash_it = table_->GetIterator(txn, Value::ConstRef(hash_key)); @@ -698,11 +744,13 @@ bool lgraph::PluginManager::GetPluginCode(PluginType type, const std::string& us } bool lgraph::PluginManager::LoadPluginFromCode(PluginType type, const std::string& user, - const std::string& name, const std::string& code, + const std::string& name, + const std::vector& code, + const std::vector& filename, plugin::CodeType code_type, const std::string& desc, bool read_only, const std::string& version) { - return SelectManager(type)->LoadPluginFromCode(user, name, code, code_type, desc, read_only, - version); + return SelectManager(type)->LoadPluginFromCode(user, name, code, filename, code_type, desc, + read_only, version); } bool lgraph::PluginManager::DelPlugin(PluginType type, const std::string& user, @@ -723,3 +771,13 @@ bool lgraph::PluginManager::Call(lgraph_api::Transaction* txn, PluginType type, return SelectManager(type)->Call(txn, user, db_with_access_control, name_, request, timeout, in_process, output); } + +bool lgraph::PluginManager::CallV2(lgraph_api::Transaction* txn, PluginType type, + const std::string& user, + AccessControlledDB* db_with_access_control, + const std::string& name_, const std::string& request, + double timeout, bool in_process, Result& output) { + return SelectManager(type)->CallV2(txn, user, db_with_access_control, name_, request, timeout, + in_process, output); +} + diff --git a/src/plugin/plugin_manager.h b/src/plugin/plugin_manager.h index 63e635c8c3..fe45aedf80 100644 --- a/src/plugin/plugin_manager.h +++ b/src/plugin/plugin_manager.h @@ -84,7 +84,9 @@ class SingleLanguagePluginManager { // return true if success, false if plugin already exists // throws on error virtual bool LoadPluginFromCode(const std::string& user, const std::string& name, - const std::string& code, plugin::CodeType code_type, + const std::vector& code, + const std::vector& filename, + plugin::CodeType code_type, const std::string& desc, bool read_only, const std::string& version); @@ -104,6 +106,11 @@ class SingleLanguagePluginManager { const std::string& request, double timeout, bool in_process, std::string& output); + virtual bool CallV2(lgraph_api::Transaction* txn, const std::string& user, + AccessControlledDB* db_with_access_control, const std::string& name_, + const std::string& request, double timeout, bool in_process, + Result& output); + protected: void LoadAllPlugins(KvTransaction& txn); @@ -187,6 +194,38 @@ class SingleLanguagePluginManager { bool isHashUpTodate(KvTransaction& txn, std::string name); std::string SignatureToJsonString(const lgraph_api::SigSpec& spec); + + inline std::string MergeCodeFiles(const std::vector& code, + const std::vector& filename, const std::string& name) { + std::string all_codes; + if (filename.empty()) { + all_codes += FMA_FMT("//{}\n{}{}", name + ".cpp", code[0], + lgraph::plugin::PLUGIN_CODE_DELIMITER); + return all_codes; + } + for (size_t i = 0; i < code.size(); i++) { + all_codes += FMA_FMT("//{}\n{}{}", filename[i], code[i], + lgraph::plugin::PLUGIN_CODE_DELIMITER); + } + return all_codes; + } + + inline void SplitCode(std::vector& code, + std::vector& filename, const std::string& all_code) { + std::string::size_type startPos = 0; + std::string::size_type delimiterPos; + std::string::size_type filenamePos; + code.clear(); + filename.clear(); + while ((delimiterPos = all_code.find(lgraph::plugin::PLUGIN_CODE_DELIMITER, startPos)) != + std::string::npos) { + filenamePos = all_code.find('\n', startPos); + filename.emplace_back(all_code.substr(startPos + 2, filenamePos - startPos - 2)); + startPos = filenamePos + 1; + code.emplace_back(all_code.substr(startPos, delimiterPos - startPos)); + startPos = delimiterPos + std::string(lgraph::plugin::PLUGIN_CODE_DELIMITER).size(); + } + } }; class PluginManager { @@ -246,7 +285,8 @@ class PluginManager { * @return true if success, false if plugin already exists */ virtual bool LoadPluginFromCode(PluginType type, const std::string& user, - const std::string& name, const std::string& code, + const std::string& name, const std::vector& code, + const std::vector& filename, plugin::CodeType code_type, const std::string& desc, bool read_only, const std::string& version); @@ -288,6 +328,10 @@ class PluginManager { AccessControlledDB* db_with_access_control, const std::string& name_, const std::string& request, double timeout, bool in_process, std::string& output); + bool CallV2(lgraph_api::Transaction* txn, PluginType type, const std::string& user, + AccessControlledDB* db_with_access_control, const std::string& name_, + const std::string& request, double timeout, bool in_process, Result& output); + protected: inline std::unique_ptr& SelectManager(PluginType type) { return type == PluginType::CPP ? cpp_manager_ : python_manager_; diff --git a/src/plugin/plugin_manager_impl.h b/src/plugin/plugin_manager_impl.h index dce7f136a4..a7210a0f5e 100644 --- a/src/plugin/plugin_manager_impl.h +++ b/src/plugin/plugin_manager_impl.h @@ -179,5 +179,15 @@ class PluginManagerImplBase { double timeout, bool in_process, std::string& output) = 0; + + virtual void DoCallV2(lgraph_api::Transaction* txn, + const std::string& user, + AccessControlledDB* db_with_access_control, + const std::string name, + const PluginInfoBase* pinfo, + const std::string& request, + double timeout, + bool in_process, + Result& output) = 0; }; } // namespace lgraph diff --git a/src/plugin/python_plugin.cpp b/src/plugin/python_plugin.cpp index 5a6f4f8fc1..4a5cd9dd0e 100644 --- a/src/plugin/python_plugin.cpp +++ b/src/plugin/python_plugin.cpp @@ -139,6 +139,14 @@ void PythonPluginManagerImpl::DoCall(lgraph_api::Transaction* txn, const std::st } } +void PythonPluginManagerImpl::DoCallV2(lgraph_api::Transaction* txn, const std::string& user, + AccessControlledDB* db_with_access_control, + const std::string name, const PluginInfoBase* pinfo, + const std::string& request, double timeout, bool in_process, + Result& output) { + THROW_CODE(InputError, "Python does not yet support the V2 version procedure"); +} + // Run by the rest handling threads. Pushes the task to Python and wait for its finish. python_plugin::TaskOutput::ErrorCode PythonPluginManagerImpl::CallInternal( const std::string& user, const std::string& function, const std::string& input, double timeout, diff --git a/src/plugin/python_plugin.h b/src/plugin/python_plugin.h index 9280ce1402..04f6bcbfa5 100644 --- a/src/plugin/python_plugin.h +++ b/src/plugin/python_plugin.h @@ -301,6 +301,16 @@ class PythonPluginManagerImpl : public PluginManagerImplBase { bool in_process, std::string& output) override; + void DoCallV2(lgraph_api::Transaction* txn, + const std::string& user, + AccessControlledDB* db_with_access_control, + const std::string name, + const PluginInfoBase* pinfo, + const std::string& request, + double timeout, + bool in_process, + Result& output) override; + protected: python_plugin::TaskOutput::ErrorCode CallInternal(const std::string& user, const std::string& function, diff --git a/src/protobuf/ha.proto b/src/protobuf/ha.proto index 3adbfe3dc2..79fd8a94ff 100644 --- a/src/protobuf/ha.proto +++ b/src/protobuf/ha.proto @@ -582,9 +582,10 @@ message LoadPluginRequest { required string name = 1; required bool read_only = 2; - required bytes code = 3; + repeated bytes code = 3; optional string desc = 4; optional CodeType code_type = 5; + repeated string file_name = 6; }; message LoadPluginResponse {}; diff --git a/src/restful/server/json_convert.h b/src/restful/server/json_convert.h index def9dce385..73de2ec9c0 100644 --- a/src/restful/server/json_convert.h +++ b/src/restful/server/json_convert.h @@ -65,6 +65,7 @@ static const utility::string_t AUTH_METHOD = _TU("auth_method"); static const utility::string_t BEGIN_TIME = _TU("begin_time"); static const utility::string_t BRANCH = _TU("git_branch"); static const utility::string_t CODE = _TU("code_base64"); +static const utility::string_t FILENAMES = _TU("file_name"); static const utility::string_t VERSION = _TU("version"); static const utility::string_t CODE_TYPE = _TU("code_type"); static const utility::string_t COMMIT = _TU("git_commit"); diff --git a/src/restful/server/rest_server.cpp b/src/restful/server/rest_server.cpp index ac3dd840cc..9d004b0e7e 100644 --- a/src/restful/server/rest_server.cpp +++ b/src/restful/server/rest_server.cpp @@ -1989,10 +1989,13 @@ void RestServer::HandlePostPlugin(const std::string& user, const std::string& to LoadPluginRequest* req = preq->mutable_load_plugin_request(); bool read_only = false; - std::string code, version; + std::string version; + ::google::protobuf::RepeatedPtrField codes; if (!ExtractStringField(body, RestStrings::NAME, *req->mutable_name()) || !ExtractBoolField(body, RestStrings::READONLY, read_only) || - !ExtractStringField(body, RestStrings::CODE, code)) { + !ExtractObjectArray(body, RestStrings::CODE, &codes) || + (codes.size() > 1 && !ExtractObjectArray( + body, RestStrings::FILENAMES, req->mutable_file_name()))) { BEG_AUDIT_LOG(user, _TS(paths[1]), lgraph::LogApiType::Plugin, true, "POST " + _TS(relative_path)); return RespondBadJSON(request); @@ -2003,9 +2006,9 @@ void RestServer::HandlePostPlugin(const std::string& user, const std::string& to } preq->set_version(version); req->set_read_only(read_only); - { + for (auto &code : codes) { std::vector decoded = utility::conversions::from_base64(_TU(code)); - req->set_code(std::string(decoded.begin(), decoded.end())); + req->add_code(std::string(decoded.begin(), decoded.end())); } ExtractStringField(body, RestStrings::DESC, *req->mutable_desc()); LoadPluginRequest::CodeType code_type = (type == PluginManager::PluginType::CPP) diff --git a/src/server/state_machine.cpp b/src/server/state_machine.cpp index bdcacd6623..67563fe66c 100644 --- a/src/server/state_machine.cpp +++ b/src/server/state_machine.cpp @@ -1097,8 +1097,18 @@ bool lgraph::StateMachine::ApplyPluginRequest(const LGraphRequest* lgraph_req, plugin::CodeType code_type = GetPluginCodeType(preq.code_type()); BEG_AUDIT_LOG(user, req.graph(), lgraph::LogApiType::Plugin, true, FMA_FMT("Load plugin [{}]", preq.name())); - bool r = db->LoadPlugin(type, user, preq.name(), preq.code(), code_type, preq.desc(), - preq.read_only(), req.version()); + std::vector file_codes(preq.code().begin(), preq.code().end()); + std::vector file_names(preq.file_name().begin(), preq.file_name().end()); + if (file_codes.size() > 1 && code_type != plugin::CodeType::CPP) { + THROW_CODE(InputError, "Only cpp files support uploading multiple files"); + } + if (file_codes.size() > 1 && file_names.size() != file_codes.size()) { + THROW_CODE(InputError, + FMA_FMT("Get {} files but {} file_names.", + file_codes.size(), file_names.size())); + } + bool r = db->LoadPlugin(type, user, preq.name(), file_codes, file_names, code_type, + preq.desc(), preq.read_only(), req.version()); if (r) return RespondSuccess(resp); else diff --git a/test/integration/test_algo_v2.py b/test/integration/test_algo_v2.py new file mode 100644 index 0000000000..7034ea60c7 --- /dev/null +++ b/test/integration/test_algo_v2.py @@ -0,0 +1,85 @@ +import logging +import pytest +import json +import liblgraph_client_python +import math + +log = logging.getLogger(__name__) + +SERVEROPT = {"cmd":"./lgraph_server -c lgraph_standalone.json --directory ./testdb --port 7072 --rpc_port 9092", + "cleanup_dir":["./testdb"]} + +CLIENTOPT = {"host":"127.0.0.1:9092", "user":"admin", "password":"73@TuGraph"} + +IMPORTOPT = {"cmd":"./lgraph_import --config_file ./data/algo/fb.conf --dir ./testdb --user admin --password 73@TuGraph --graph default --overwrite 1", + "cleanup_dir":["./testdb", "./.import_tmp"]} + + +class TestAlgoV2: + + @pytest.mark.parametrize("importor", [IMPORTOPT], indirect=True) + @pytest.mark.parametrize("server", [SERVEROPT], indirect=True) + @pytest.mark.parametrize("client", [CLIENTOPT], indirect=True) + def test_algo_v2(self, importor, server, client): + algos = ["libbfs_v2.so", "liblcc_v2.so", "liblpa_v2.so", "libpagerank_v2.so", "libsssp_v2.so", "libwcc_v2.so"] + algo_dir = "./algo/" + for algo in algos: + ret = client.loadProcedure(algo_dir + algo, "CPP", algo.split('.')[0], "SO", "test plugin", True, "v2") + try: + assert ret[0] + except: + log.info(ret) + raise + + # Test BFS + ret = client.callCypher("MATCH (n:node{id: 0}) " + "CALL plugin.cpp.libbfs_v2(n) " + "YIELD node, parent " + "WITH node, parent " + "RETURN COUNT(parent)") + assert ret[0] + result = json.loads(ret[1])[0].get("COUNT(parent)") + assert result == 3829 + + # Test PageRank + ret = client.callCypher("CALL plugin.cpp.libpagerank_v2(10) " + "YIELD node, weight WITH node, weight " + "RETURN MAX(weight)") + assert ret[0] + result = json.loads(ret[1])[0].get("MAX(weight)") + assert math.isclose(result, 0.00939017583256023, rel_tol=1e-5) + + # Test WCC + ret = client.callCypher("CALL plugin.cpp.libwcc_v2() " + "YIELD node, label WITH node, label " + "RETURN COUNT(DISTINCT label)") + assert ret[0] + result = json.loads(ret[1])[0].get("COUNT(DISTINCT label)") + assert result == 1 + + # Test LPA + ret = client.callCypher("CALL plugin.cpp.liblpa_v2(10) " + "YIELD node, label WITH node, label " + "RETURN COUNT(DISTINCT label)") + assert ret[0] + result = json.loads(ret[1])[0].get("COUNT(DISTINCT label)") + assert result == 16 + + # Test LCC + ret = client.callCypher("CALL plugin.cpp.liblcc_v2() " + "YIELD node, score WITH node, score " + "WHERE node.id = 0 " + "RETURN score") + assert ret[0] + result = json.loads(ret[1])[0].get("score") + assert math.isclose(result, 0.041961653145874626, rel_tol=1e-5) + + # Test SSSP + ret = client.callCypher("MATCH (n:node{id:0}) " + "CALL plugin.cpp.libsssp_v2(n) " + "YIELD node, distance WITH node, distance " + "WHERE node.id = 10 " + "RETURN distance") + assert ret[0] + result = json.loads(ret[1])[0].get("distance") + assert result == 1 diff --git a/test/integration/test_procedure.py b/test/integration/test_procedure.py index 729e13573c..9c04a40102 100644 --- a/test/integration/test_procedure.py +++ b/test/integration/test_procedure.py @@ -20,6 +20,10 @@ "g++ -fno-gnu-unique -fPIC -g --std=c++17 -I ../../include -I ../../deps/install/include -rdynamic -O3 -fopenmp -DNDEBUG -o ./sortstr.so ../../test/test_procedures/sortstr.cpp ./liblgraph.so -shared"], "so_name":["./scan_graph.so", "./sortstr.so"]} +BUILDV2OPT = {"cmd": ["g++ -fno-gnu-unique -fPIC -g --std=c++17 -I ../../include -I ../../deps/install/include -rdynamic -O3 -fopenmp -DNDEBUG -o ./v2_pagerank.so ../../test/test_procedures/v2_pagerank.cpp ./liblgraph.so -shared", + "g++ -fno-gnu-unique -fPIC -g --std=c++17 -I ../../include -I ../../deps/install/include -rdynamic -O3 -fopenmp -DNDEBUG -o ./v2_test_path.so ../../test/test_procedures/v2_test_path.cpp ./liblgraph.so -shared"], + "so_name": ["v2_pagerank.so", "v2_test_path.so"]} + IMPORTCONTENT = { "schema" : '''{"schema" : [ { @@ -746,6 +750,32 @@ def test_plugin(self, build_so, importor, server, client): plugins = json.loads(ret[1]) assert len(plugins) == 1 + @pytest.mark.parametrize("build_so", [BUILDV2OPT], indirect=True) + @pytest.mark.parametrize("importor", [IMPORTOPT], indirect=True) + @pytest.mark.parametrize("server", [SERVEROPT], indirect=True) + @pytest.mark.parametrize("client", [CLIENTOPT], indirect=True) + def test_plugin_v2(self, build_so, importor, server, client): + pagerank_so = BUILDV2OPT.get("so_name")[0] + ret = client.loadProcedure(pagerank_so, "CPP", "v2_pagerank", "SO", "test plugin", True, "v2") + assert ret[0] + shortestpath_so = BUILDV2OPT.get("so_name")[1] + ret = client.loadProcedure(shortestpath_so, "CPP", "v2_test_path", "SO", "test plugin", True, "v2") + assert ret[0] + ret = client.callCypher("MATCH (a:Person {name:\"Christopher Nolan\"}), (b:Person {name: \"Corin Redgrave\"}) " + "CALL plugin.cpp.v2_test_path(a, b) YIELD length, nodeIds " + "RETURN length") + assert ret[0] + result = json.loads(ret[1])[0].get("length") + assert result == 5 + ret = client.callCypher("CALL plugin.cpp.v2_pagerank(10) " + "YIELD node, weight WITH node, weight " + "RETURN MAX(weight)") + assert ret[0] + result = json.loads(ret[1])[0].get("MAX(weight)") + import math + assert math.isclose(result, 0.07308246478538732, rel_tol=1e-5) + + @pytest.mark.parametrize("importor", [IMPORTOPT], indirect=True) @pytest.mark.parametrize("server", [SERVEROPT], indirect=True) @pytest.mark.parametrize("client", [CLIENTOPT], indirect=True) diff --git a/test/test_cpp_procedure.cpp b/test/test_cpp_procedure.cpp index a72ff1b2b5..caf2651095 100644 --- a/test/test_cpp_procedure.cpp +++ b/test/test_cpp_procedure.cpp @@ -25,6 +25,7 @@ #include "plugin/plugin_manager.h" #include "./test_tools.h" +#include "./graph_factory.h" class TestCppPlugin : public TuGraphTest {}; @@ -94,6 +95,10 @@ void build_so() { "../../test/test_procedures/bfs.cpp", LIBLGRAPH); rt = system(cmd.c_str()); UT_EXPECT_EQ(rt, 0); + cmd = UT_FMT(cmd_f.c_str(), INCLUDE_DIR, DEPS_INCLUDE_DIR, "./v2_pagerank.so", + "../../test/test_procedures/v2_pagerank.cpp", LIBLGRAPH); + rt = system(cmd.c_str()); + UT_EXPECT_EQ(rt, 0); } void read_code(const std::string& code_path, std::string& code) { @@ -121,6 +126,12 @@ TEST_F(TestCppPlugin, CppPlugin) { AutoCleanDir _1(db_dir); AutoCleanDir _2(plugin_dir); + GraphFactory::WriteYagoFiles(); + SubProcess import_client( + FMA_FMT("./lgraph_import -c yago.conf -d {}", db_dir)); + import_client.Wait(); + UT_EXPECT_EQ(import_client.GetExitCode(), 0); + UT_LOG() << "Test Begin..."; Galaxy galaxy(db_dir); AccessControlledDB db = galaxy.OpenGraph("admin", "default"); @@ -216,10 +227,20 @@ TEST_F(TestCppPlugin, CppPlugin) { std::string code_scan_graph = ""; std::string code_add_label = ""; std::string code_bfs = ""; + std::string code_v2_pagerank = ""; + std::string multi_procedure_name = "multi_files.cpp"; + std::string code_multi_procedure = ""; + std::string multi_core_name = "multi_files_core.cpp"; + std::string code_multi_core = ""; + std::string multi_header_name = "multi_files.h"; + std::string code_multi_header = ""; { // read file to string std::string code_so_path = "./sortstr.so"; std::string code_cpp_path = "../../test/test_procedures/sortstr.cpp"; + std::string multi_procedure_path = "../../test/test_procedures/multi_files.cpp"; + std::string multi_header_path = "../../test/test_procedures/multi_files.h"; + std::string multi_core_path = "../../test/test_procedures/multi_files_core.cpp"; #ifndef __clang__ std::string code_zip_path = "../../test/test_procedures/sortstr.zip"; #elif __APPLE__ @@ -230,6 +251,7 @@ TEST_F(TestCppPlugin, CppPlugin) { std::string code_scan_graph_path = "./scan_graph.so"; std::string code_add_label_path = "./add_label.so"; std::string code_bfs_path = "./bfs.so"; + std::string code_v2_pagerank_path = "./v2_pagerank.so"; build_so(); read_code(code_so_path, code_so); read_code(code_cpp_path, code_cpp); @@ -237,11 +259,19 @@ TEST_F(TestCppPlugin, CppPlugin) { read_code(code_scan_graph_path, code_scan_graph); read_code(code_add_label_path, code_add_label); read_code(code_bfs_path, code_bfs); + read_code(code_v2_pagerank_path, code_v2_pagerank); + read_code(multi_procedure_path, code_multi_procedure); + read_code(multi_core_path, code_multi_core); + read_code(multi_header_path, code_multi_header); UT_EXPECT_NE(code_so, ""); UT_EXPECT_NE(code_cpp, ""); UT_EXPECT_NE(code_zip, ""); UT_EXPECT_NE(code_scan_graph, ""); - UT_EXPECT_NE(code_add_label, ""); + UT_EXPECT_NE(code_bfs_path, ""); + UT_EXPECT_NE(code_v2_pagerank, ""); + UT_EXPECT_NE(code_multi_procedure, ""); + UT_EXPECT_NE(code_multi_core, ""); + UT_EXPECT_NE(code_multi_header, ""); } { // test PluginInfo @@ -263,31 +293,41 @@ TEST_F(TestCppPlugin, CppPlugin) { UT_EXPECT_TRUE(pm.procedures_.empty()); // test exception branch - UT_EXPECT_THROW_CODE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "#name", "", - plugin::CodeType::SO, "desc", true, "v1"), - InputError); + UT_EXPECT_THROW_CODE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "#name", + std::vector{}, + std::vector{"invalid.so"}, + plugin::CodeType::SO, "desc", true, "v1"), + InputError); UT_LOG() << "Test loading empty plugin code"; - UT_EXPECT_THROW_CODE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "name", "", - plugin::CodeType::SO, "desc", true, "v1"), - InputError); + UT_EXPECT_THROW_CODE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "name", + std::vector{""}, + std::vector{"empty.so"}, + plugin::CodeType::SO, "desc", true, "v1"), + InputError); UT_LOG() << "Test loading invalid plugin code (code_type: so)"; UT_EXPECT_THROW_CODE( - pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "name", "invalid_code", + pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "name", + std::vector{"invalid_code"}, + std::vector{"invalid.so"}, plugin::CodeType::SO, "desc", true, "v1"), InputError); UT_LOG() << "Test loading invalid plugin code (code_type: zip)"; UT_EXPECT_THROW_CODE( - pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "name", "invalid_code", + pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "name", + std::vector{"invalid_code"}, + std::vector{"invalid.zip"}, plugin::CodeType::ZIP, "desc", true, "v1"), InputError); UT_LOG() << "Test loading invalid plugin code (code_type: cpp)"; UT_EXPECT_THROW_CODE( - pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "name", "invalid_code", + pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "name", + std::vector{"invalid_code"}, + std::vector{"invalid.cpp"}, plugin::CodeType::CPP, "desc", true, "v1"), InputError); @@ -299,17 +339,38 @@ TEST_F(TestCppPlugin, CppPlugin) { UT_LOG() << "Test loading of plugins (code_type: so)"; UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "sortstr_so", - code_so, plugin::CodeType::SO, "sortstr so", true, + std::vector{code_so}, + std::vector{"sort.so"}, + plugin::CodeType::SO, "sortstr so", true, "v1")); UT_LOG() << "Test loading of plugins (code_type: cpp)"; UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "sortstr_cpp", - code_cpp, plugin::CodeType::CPP, "sortstr cpp", true, + std::vector{code_cpp}, + std::vector{"sort.cpp"}, + plugin::CodeType::CPP, "sortstr cpp", true, + "v1")); + + UT_LOG() << "Test loading multiple files for cpp"; + UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "multi_file", + std::vector{ + code_multi_core, + code_multi_procedure, + code_multi_header + }, + std::vector{ + multi_core_name, + multi_procedure_name, + multi_header_name + }, + plugin::CodeType::CPP, "multiple files", true, "v1")); UT_LOG() << "Test loading of plugins (code_type: zip)"; UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "sortstr_zip", - code_zip, plugin::CodeType::ZIP, "sortstr zip", true, + std::vector{code_zip}, + std::vector{"sort.zip"}, + plugin::CodeType::ZIP, "sortstr zip", true, "v1")); PluginCode pc; @@ -326,28 +387,46 @@ TEST_F(TestCppPlugin, CppPlugin) { UT_LOG() << "Test retrieving plugin (code_type: cpp)"; UT_EXPECT_TRUE(pm.GetPluginCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "sortstr_cpp", pc)); - UT_EXPECT_TRUE(pc.code.compare(code_cpp) == 0 && pc.code_type == "cpp"); + std::string code_cpp_merged; + UT_EXPECT_NO_THROW(code_cpp_merged = pm.MergeCodeFiles(std::vector{code_cpp}, + std::vector{"sort.cpp"}, + "sortstr_cpp")); + UT_EXPECT_TRUE(pc.code.compare(code_cpp_merged) == 0 && pc.code_type == "cpp"); UT_LOG() << "Load scan_graph plugin"; UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "scan_graph", - code_scan_graph, plugin::CodeType::SO, "scan graph v1", + std::vector{code_scan_graph}, + std::vector{"scan.so"}, + plugin::CodeType::SO, "scan graph v1", true, "v1")); UT_EXPECT_TRUE(!pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "scan_graph", - code_scan_graph, plugin::CodeType::SO, "scan graph", + std::vector{code_scan_graph}, + std::vector{"scan.so"}, + plugin::CodeType::SO, "scan graph", true, "v1")); UT_EXPECT_TRUE(pm.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "scan_graph")); UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "scan_graph", - code_scan_graph, plugin::CodeType::SO, "scan graph v2", + std::vector{code_scan_graph}, + std::vector{"scan.so"}, + plugin::CodeType::SO, "scan graph v2", true, "v1")); UT_LOG() << "Load add_label plugin"; UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "add_label", - code_add_label, plugin::CodeType::SO, "add label v1", + std::vector{code_add_label}, + std::vector{"add_label.so"}, + plugin::CodeType::SO, "add label v1", false, "v1")); + UT_LOG() << "Load v2_pagerank plugin"; + UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "v2_pagerank", + std::vector{code_v2_pagerank}, + std::vector{"v2_pagerank.so"}, + plugin::CodeType::SO, + "v2 pagerank", true, "v2")); UT_LOG() << "Test loaded plugins info"; auto& funcs = pm.procedures_; - UT_EXPECT_EQ(funcs.size(), 5); + UT_EXPECT_EQ(funcs.size(), 7); auto& zip = funcs["_fma_sortstr_zip"]; UT_EXPECT_EQ(zip->desc, "sortstr zip"); UT_EXPECT_EQ(zip->read_only, true); @@ -357,6 +436,9 @@ TEST_F(TestCppPlugin, CppPlugin) { auto& add_label = funcs["_fma_add_label"]; UT_EXPECT_EQ(add_label->desc, "add label v1"); UT_EXPECT_EQ(add_label->read_only, false); + auto& v2_pagerank = funcs["_fma_v2_pagerank"]; + UT_EXPECT_EQ(v2_pagerank->desc, "v2 pagerank"); + UT_EXPECT_EQ(v2_pagerank->read_only, true); // test call std::string output; @@ -367,7 +449,7 @@ TEST_F(TestCppPlugin, CppPlugin) { UT_EXPECT_TRUE(pm.Call(nullptr, lgraph::_detail::DEFAULT_ADMIN_NAME, &db, "scan_graph", "{\"scan_edges\":true, \"times\":2}", 0, true, output)); - UT_EXPECT_EQ(output, "{\"num_edges\":0,\"num_vertices\":0}"); + UT_EXPECT_EQ(output, "{\"num_edges\":56,\"num_vertices\":42}"); UT_EXPECT_TRUE(!pm.Call(nullptr, lgraph::_detail::DEFAULT_ADMIN_NAME, &db, "no_such_plugin", "{\"scan_edges\":true}", 2, true, output)); // bad argument causes Process to return false and output is used to return error @@ -385,12 +467,24 @@ TEST_F(TestCppPlugin, CppPlugin) { "{\"label\":\"vertex1\"}", 0, true, output), InputError); + { // test call v2 + Result output_v2; + UT_LOG() << "Test call v2 plugin"; + lgraph_api::GraphDB gdb(&db, true, false); + auto txn = gdb.CreateReadTxn(); + UT_EXPECT_TRUE(pm.CallV2(&txn, lgraph::_detail::DEFAULT_ADMIN_NAME, + nullptr, "v2_pagerank", + "{\"num_iteration\": 10}", + 0, true, output_v2)); + UT_EXPECT_EQ(output_v2.Size(), txn.GetNumVertices()); + } + { lgraph_api::GraphDB gdb(&db, true, false); auto txn = gdb.CreateReadTxn(); auto labels = txn.ListVertexLabels(); - UT_EXPECT_EQ(labels.size(), 1); - UT_EXPECT_EQ(labels.front(), "vertex1"); + UT_EXPECT_EQ(labels.size(), 4); + UT_EXPECT_EQ(labels.back(), "vertex1"); } { UT_EXPECT_EQ(pm.IsReadOnlyPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "scan_graph"), 1); @@ -399,7 +493,9 @@ TEST_F(TestCppPlugin, CppPlugin) { { UT_EXPECT_TRUE(pm.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "add_label")); UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "add_label", - code_add_label, plugin::CodeType::SO, + std::vector{code_add_label}, + std::vector{"add_label.so"}, + plugin::CodeType::SO, "add label v2", true, "v1")); // since add_label is now declared read-only, it should fail with an exception UT_EXPECT_THROW_CODE(pm.Call(nullptr, lgraph::_detail::DEFAULT_ADMIN_NAME, @@ -417,13 +513,17 @@ TEST_F(TestCppPlugin, CppPlugin) { UT_EXPECT_TRUE(pm.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "sortstr_so")); UT_EXPECT_TRUE(pm.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "sortstr_cpp")); UT_EXPECT_TRUE(pm.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "sortstr_zip")); + UT_EXPECT_TRUE(pm.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "multi_file")); + UT_EXPECT_TRUE( + pm.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "v2_pagerank")); // pm.UnloadAllPlugins(); // pm.DeleteAllPlugins("admin"); // UT_LOG() << "del succ"; UT_EXPECT_ANY_THROW( - pm.IsReadOnlyPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "scan_graph")); + pm.IsReadOnlyPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, + "scan_graph")); UT_EXPECT_TRUE(pm.procedures_.empty()); } { @@ -435,13 +535,18 @@ TEST_F(TestCppPlugin, CppPlugin) { { UT_LOG() << "Testing repeated load/delete"; UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "scan_graph", - code_scan_graph, plugin::CodeType::SO, + std::vector{code_scan_graph}, + std::vector{"scan.so"}, + plugin::CodeType::SO, "scan graph v1", true, "v1")); UT_EXPECT_TRUE(pm.Call(nullptr, lgraph::_detail::DEFAULT_ADMIN_NAME, &db, "scan_graph", "{\"scan_edges\":true, \"times\":2}", 0, true, output)); for (size_t i = 0; i < 300; i++) { UT_EXPECT_TRUE(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, - "sortstr_so", code_so, plugin::CodeType::SO, + "sortstr_so", + std::vector{code_so}, + std::vector{"sort.so"}, + plugin::CodeType::SO, "sortstr so", true, "v1")); std::string output; UT_EXPECT_TRUE(pm.Call(nullptr, lgraph::_detail::DEFAULT_ADMIN_NAME, &db, @@ -449,9 +554,12 @@ TEST_F(TestCppPlugin, CppPlugin) { UT_EXPECT_TRUE(pm.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "sortstr_so")); } pm.DeleteAllPlugins(lgraph::_detail::DEFAULT_ADMIN_NAME); - UT_EXPECT_ANY_THROW(pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "testa", - "#include ", (plugin::CodeType)6, - "test", true, "v1")); + UT_EXPECT_ANY_THROW( + pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "testa", + std::vector{"#include "}, + std::vector{"testa.cpp"}, + (plugin::CodeType)6, + "test", true, "v1")); } #ifndef __SANITIZE_ADDRESS__ { @@ -462,10 +570,11 @@ TEST_F(TestCppPlugin, CppPlugin) { UT_EXPECT_NO_THROW( r = pm.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "bfs_" + std::to_string(i), - code_bfs, plugin::CodeType::SO, + std::vector{code_bfs}, std::vector{"bfs.so"}, + plugin::CodeType::SO, "bfs v1", true, "v1")); UT_EXPECT_TRUE(r); - fma_common::SleepS(5); + fma_common::SleepS(1); } pm.DeleteAllPlugins(lgraph::_detail::DEFAULT_ADMIN_NAME); } diff --git a/test/test_cypher.cpp b/test/test_cypher.cpp index 2ac8d63d38..f3e0044326 100644 --- a/test/test_cypher.cpp +++ b/test/test_cypher.cpp @@ -1161,6 +1161,30 @@ int test_procedure(cypher::RTContext *ctx) { "CALL db.plugin.loadPlugin('CPP','" + i.first + "','" + encode + \ "','CPP','" + i.first + "', true, 'v1')"); } + std::vector> multi_file_info = { + {"multi_files_core.cpp", "../../test/test_procedures/multi_files_core.cpp"}, + {"multi_files.cpp", "../../test/test_procedures/multi_files.cpp"}, + {"multi_files.h", "../../test/test_procedures/multi_files.h"} + }; + std::map contents; + for (auto &i : multi_file_info) { + text.clear(); + f.open(i.second, std::ios::in); + std::string buf; + while (getline(f, buf)) { + text += buf; + text += "\n"; + } + f.close(); + encode = lgraph_api::encode_base64(text); + contents[i.first] = encode; + } + std::string cypher_q = FMA_FMT("CALL db.plugin.loadPlugin('CPP','multi', " + "\\{`{}`: \"{}\", `{}`: \"{}\", `{}`: \"{}\"\\}, 'CPP','multi', true, 'v1')", + "multi_files_core.cpp", contents["multi_files_core.cpp"], + "multi_files.cpp", contents["multi_files.cpp"], + "multi_files.h", contents["multi_files.h"]); + plugin_scripts.push_back(cypher_q); eval_scripts(ctx, plugin_scripts); static std::vector scripts = { @@ -1342,22 +1366,22 @@ int test_procedure(cypher::RTContext *ctx) { name, encoded, name)); }; - add_signatured_plugins("custom_shortestpath", - "../../test/test_procedures/custom_shortestpath.cpp"); + add_signatured_plugins("v2_test_path", + "../../test/test_procedures/v2_test_path.cpp"); call_signatured_plugins_scripts.emplace_back( "MATCH (a:Person {name: \"Christopher Nolan\"}), (b:Person {name: \"Corin Redgrave\"}) " - "CALL plugin.cpp.custom_shortestpath(a, b) YIELD length, nodeIds " + "CALL plugin.cpp.v2_test_path(a, b) YIELD length, nodeIds " "RETURN length, nodeIds AS path"); - add_signatured_plugins("custom_pagerank", "../../test/test_procedures/custom_pagerank.cpp"); + add_signatured_plugins("v2_pagerank", "../../test/test_procedures/v2_pagerank.cpp"); call_signatured_plugins_scripts.emplace_back( - "CALL plugin.cpp.custom_pagerank(10) " + "CALL plugin.cpp.v2_pagerank(10) " "YIELD node, weight WITH node, weight " "MATCH(node)-[r]->(n) RETURN node, r, n, weight"); call_signatured_plugins_scripts.emplace_back( "MATCH (a:Person {name: \"Christopher Nolan\"}), (b:Person {name: \"Corin Redgrave\"}) " - "CALL plugin.cpp.custom_shortestpath(a, b) YIELD length, nodeIds " + "CALL plugin.cpp.v2_test_path(a, b) YIELD length, nodeIds " "WITH length, nodeIds " "UNWIND nodeIds AS id " "RETURN id, length"); @@ -1369,18 +1393,18 @@ int test_procedure(cypher::RTContext *ctx) { "YIELD node, salt WITH node, salt " "MATCH(node)-[r]->(n) RETURN node, r, n, salt"); - add_signatured_plugins("custom_path_process", - "../../test/test_procedures/custom_path_process.cpp"); + add_signatured_plugins("v2_path_process", + "../../test/test_procedures/v2_path_process.cpp"); call_signatured_plugins_scripts.emplace_back( "MATCH p = (n {name:\"Rachel Kempson\"})-[*0..3]->() " - "CALL plugin.cpp.custom_path_process(nodes(p)) YIELD idSum " + "CALL plugin.cpp.v2_path_process(nodes(p)) YIELD idSum " "RETURN idSum"); - add_signatured_plugins("custom_algo", "../../test/test_procedures/custom_algo.cpp"); + add_signatured_plugins("v2_algo", "../../test/test_procedures/v2_algo.cpp"); call_signatured_plugins_scripts.emplace_back( - "CALL plugin.cpp.custom_algo() YIELD res RETURN res"); + "CALL plugin.cpp.v2_algo() YIELD res RETURN res"); call_signatured_plugins_scripts.emplace_back( - "CALL plugin.cpp.custom_algo()"); + "CALL plugin.cpp.v2_algo()"); eval_scripts(ctx, call_signatured_plugins_scripts); return 0; } diff --git a/test/test_ha_full_import.cpp b/test/test_ha_full_import.cpp index 0898278761..3382299700 100644 --- a/test/test_ha_full_import.cpp +++ b/test/test_ha_full_import.cpp @@ -210,7 +210,7 @@ TEST_F(TestHAFullImport, FullImportRemote) { lgraph::SubProcess online_import_client(import_cmd); online_import_client.Wait(); UT_EXPECT_EQ(online_import_client.GetExitCode(), 0); - fma_common::SleepS(20); + fma_common::SleepS(40); succeed = rpc_client->CallCypherToLeader(res, "match (n) return count(n)", "test"); UT_EXPECT_TRUE(succeed); v = web::json::value::parse(res); diff --git a/test/test_olap_on_db.cpp b/test/test_olap_on_db.cpp index 83aa0f1871..98c8cf44a5 100644 --- a/test/test_olap_on_db.cpp +++ b/test/test_olap_on_db.cpp @@ -191,247 +191,269 @@ TEST_F(TestOlapOnDB, OlapOnDB) { Galaxy g(db_path); g.SetCurrentUser("admin", "73@TuGraph"); GraphDB db = g.OpenGraph("default"); - auto txn = db.CreateReadTxn(); - - // test ConstructWithVid() - OlapOnDB test_db_one(db, txn, SNAPSHOT_PARALLEL | SNAPSHOT_UNDIRECTED, - nullptr, edge_convert_default); - UT_EXPECT_EQ(test_db_one.OutDegree(0), 5); - UT_EXPECT_EQ(test_db_one.InDegree(0), 5); - UT_EXPECT_EQ(test_db_one.OutDegree(6), 2); - UT_EXPECT_EQ(test_db_one.InDegree(6), 2); - UT_EXPECT_EQ(test_db_one.OutDegree(17), 4); - UT_EXPECT_EQ(test_db_one.InDegree(17), 4); - UT_EXPECT_EQ(test_db_one.OutDegree(20), 2); - UT_EXPECT_EQ(test_db_one.InDegree(20), 2); - auto active = test_db_one.AllocVertexSubset(); - active.Fill(); - UT_EXPECT_EQ(active.Size(), 21); - auto label = test_db_one.AllocVertexArray(); - UT_EXPECT_EQ(test_db_one.NumVertices(), 21); - UT_EXPECT_EQ(test_db_one.NumEdges(), 70); - test_db_one.ProcessVertexActive( - [&](size_t vi) { - label[vi] = -1; - return 1; - }, - active); - UT_EXPECT_EQ(label[0], -1); - for (auto& edge : test_db_one.OutEdges(0)) { - UT_EXPECT_EQ(edge.edge_data, 1.0); - } - auto vertex_lists = test_db_one.ExtractVertexData(vertex_extract); - UT_EXPECT_EQ(vertex_lists[0], 2); - - OlapOnDB unparallel_one(db, txn, SNAPSHOT_UNDIRECTED); - UT_EXPECT_EQ(unparallel_one.OutDegree(0), 5); - UT_EXPECT_EQ(unparallel_one.InDegree(0), 5); - UT_EXPECT_EQ(unparallel_one.OutDegree(6), 2); - UT_EXPECT_EQ(unparallel_one.InDegree(6), 2); - UT_EXPECT_EQ(unparallel_one.OutDegree(17), 4); - UT_EXPECT_EQ(unparallel_one.InDegree(17), 4); - UT_EXPECT_EQ(unparallel_one.OutDegree(20), 2); - UT_EXPECT_EQ(unparallel_one.InDegree(20), 2); - - OlapOnDB unparallel_db_two(db, txn, - SNAPSHOT_UNDIRECTED, nullptr, edge_convert_weight); - UT_EXPECT_EQ(unparallel_db_two.OutDegree(0), 5); - UT_EXPECT_EQ(unparallel_db_two.InDegree(0), 5); - UT_EXPECT_EQ(unparallel_db_two.OutDegree(6), 2); - UT_EXPECT_EQ(unparallel_db_two.InDegree(6), 2); - UT_EXPECT_EQ(unparallel_db_two.OutDegree(17), 4); - UT_EXPECT_EQ(unparallel_db_two.InDegree(17), 4); - UT_EXPECT_EQ(unparallel_db_two.OutDegree(20), 2); - UT_EXPECT_EQ(unparallel_db_two.InDegree(20), 2); - - for (auto& edge : unparallel_db_two.OutEdges(0)) { - UT_EXPECT_EQ(edge.edge_data, edge.neighbour); - } + auto test_construct = [](GraphDB* db, lgraph_api::Transaction& txn) { + // test ConstructWithVid() + OlapOnDB test_db_one(db, txn, SNAPSHOT_PARALLEL | SNAPSHOT_UNDIRECTED, nullptr, + edge_convert_default); + UT_EXPECT_EQ(test_db_one.OutDegree(0), 5); + UT_EXPECT_EQ(test_db_one.InDegree(0), 5); + UT_EXPECT_EQ(test_db_one.OutDegree(6), 2); + UT_EXPECT_EQ(test_db_one.InDegree(6), 2); + UT_EXPECT_EQ(test_db_one.OutDegree(17), 4); + UT_EXPECT_EQ(test_db_one.InDegree(17), 4); + UT_EXPECT_EQ(test_db_one.OutDegree(20), 2); + UT_EXPECT_EQ(test_db_one.InDegree(20), 2); + auto active = test_db_one.AllocVertexSubset(); + active.Fill(); + UT_EXPECT_EQ(active.Size(), 21); + auto label = test_db_one.AllocVertexArray(); + UT_EXPECT_EQ(test_db_one.NumVertices(), 21); + UT_EXPECT_EQ(test_db_one.NumEdges(), 70); + test_db_one.ProcessVertexActive( + [&](size_t vi) { + label[vi] = -1; + return 1; + }, + active); + UT_EXPECT_EQ(label[0], -1); + for (auto& edge : test_db_one.OutEdges(0)) { + UT_EXPECT_EQ(edge.edge_data, 1.0); + } - OlapOnDB parallel_one(db, txn, SNAPSHOT_PARALLEL, nullptr, edge_convert_weight); - UT_EXPECT_EQ(parallel_one.OutDegree(0), 5); - UT_EXPECT_EQ(parallel_one.InDegree(0), 0); - UT_EXPECT_EQ(parallel_one.OutDegree(6), 0); - UT_EXPECT_EQ(parallel_one.InDegree(6), 2); - UT_EXPECT_EQ(parallel_one.OutDegree(17), 1); - UT_EXPECT_EQ(parallel_one.InDegree(17), 3); - UT_EXPECT_EQ(parallel_one.OutDegree(20), 0); - UT_EXPECT_EQ(parallel_one.InDegree(20), 2); - - for (auto& edge : parallel_one.OutEdges(10)) { - UT_EXPECT_EQ(edge.edge_data, edge.neighbour); - } + auto vertex_lists = test_db_one.ExtractVertexData(vertex_extract); + UT_EXPECT_EQ(vertex_lists[0], 2); + + OlapOnDB unparallel_one(db, txn, SNAPSHOT_UNDIRECTED); + UT_EXPECT_EQ(unparallel_one.OutDegree(0), 5); + UT_EXPECT_EQ(unparallel_one.InDegree(0), 5); + UT_EXPECT_EQ(unparallel_one.OutDegree(6), 2); + UT_EXPECT_EQ(unparallel_one.InDegree(6), 2); + UT_EXPECT_EQ(unparallel_one.OutDegree(17), 4); + UT_EXPECT_EQ(unparallel_one.InDegree(17), 4); + UT_EXPECT_EQ(unparallel_one.OutDegree(20), 2); + UT_EXPECT_EQ(unparallel_one.InDegree(20), 2); + + OlapOnDB unparallel_db_two(db, txn, SNAPSHOT_UNDIRECTED, nullptr, + edge_convert_weight); + UT_EXPECT_EQ(unparallel_db_two.OutDegree(0), 5); + UT_EXPECT_EQ(unparallel_db_two.InDegree(0), 5); + UT_EXPECT_EQ(unparallel_db_two.OutDegree(6), 2); + UT_EXPECT_EQ(unparallel_db_two.InDegree(6), 2); + UT_EXPECT_EQ(unparallel_db_two.OutDegree(17), 4); + UT_EXPECT_EQ(unparallel_db_two.InDegree(17), 4); + UT_EXPECT_EQ(unparallel_db_two.OutDegree(20), 2); + UT_EXPECT_EQ(unparallel_db_two.InDegree(20), 2); + + for (auto& edge : unparallel_db_two.OutEdges(0)) { + UT_EXPECT_EQ(edge.edge_data, edge.neighbour); + } + + OlapOnDB parallel_one(db, txn, SNAPSHOT_PARALLEL, nullptr, + edge_convert_weight); + UT_EXPECT_EQ(parallel_one.OutDegree(0), 5); + UT_EXPECT_EQ(parallel_one.InDegree(0), 0); + UT_EXPECT_EQ(parallel_one.OutDegree(6), 0); + UT_EXPECT_EQ(parallel_one.InDegree(6), 2); + UT_EXPECT_EQ(parallel_one.OutDegree(17), 1); + UT_EXPECT_EQ(parallel_one.InDegree(17), 3); + UT_EXPECT_EQ(parallel_one.OutDegree(20), 0); + UT_EXPECT_EQ(parallel_one.InDegree(20), 2); + + for (auto& edge : parallel_one.OutEdges(10)) { + UT_EXPECT_EQ(edge.edge_data, edge.neighbour); + } + + OlapOnDB parallel_two(db, txn, SNAPSHOT_PARALLEL, nullptr, + edge_convert_default); + UT_EXPECT_EQ(parallel_two.OutDegree(0), 5); + UT_EXPECT_EQ(parallel_two.InDegree(0), 0); + UT_EXPECT_EQ(parallel_two.OutDegree(6), 0); + UT_EXPECT_EQ(parallel_two.InDegree(6), 2); + UT_EXPECT_EQ(parallel_two.OutDegree(17), 1); + UT_EXPECT_EQ(parallel_two.InDegree(17), 3); + UT_EXPECT_EQ(parallel_two.OutDegree(20), 0); + UT_EXPECT_EQ(parallel_two.InDegree(20), 2); + + for (auto& edge : parallel_two.OutEdges(10)) { + UT_EXPECT_EQ(edge.edge_data, 1.0); + } + + // test Construct() + OlapOnDB test_db_two(db, txn, SNAPSHOT_PARALLEL | SNAPSHOT_IDMAPPING); + UT_EXPECT_EQ(test_db_two.OutDegree(0), 5); + UT_EXPECT_EQ(test_db_two.InDegree(0), 0); + UT_EXPECT_EQ(test_db_two.OutDegree(6), 0); + UT_EXPECT_EQ(test_db_two.InDegree(6), 2); + UT_EXPECT_EQ(test_db_two.OutDegree(17), 1); + UT_EXPECT_EQ(test_db_two.InDegree(17), 3); + UT_EXPECT_EQ(test_db_two.OutDegree(20), 0); + UT_EXPECT_EQ(test_db_two.InDegree(20), 2); + UT_EXPECT_EQ(test_db_two.NumVertices(), 21); + UT_EXPECT_EQ(test_db_two.NumEdges(), 35); + UT_EXPECT_EQ(test_db_two.MappedVid(0), 0); + UT_EXPECT_EQ(test_db_two.OriginalVid(0), 0); + UT_EXPECT_EQ(test_db_two.MappedVid(test_db_two.OriginalVid(4)), 4); + + OlapOnDB unparallel_two(db, txn, SNAPSHOT_UNDIRECTED, vertex_filter, + edge_convert_weight); + UT_EXPECT_EQ(unparallel_two.OutDegree(0), 3); + UT_EXPECT_EQ(unparallel_two.InDegree(0), 3); + UT_EXPECT_EQ(unparallel_two.OutDegree(6), 2); + UT_EXPECT_EQ(unparallel_two.InDegree(6), 2); + UT_EXPECT_EQ(unparallel_two.OutDegree(13), 2); + UT_EXPECT_EQ(unparallel_two.InDegree(13), 2); + UT_EXPECT_EQ(unparallel_two.OutDegree(20), 0); + UT_EXPECT_EQ(unparallel_two.InDegree(20), 0); + UT_EXPECT_EQ(unparallel_two.NumVertices(), 20); + UT_EXPECT_EQ(unparallel_two.NumEdges(), 60); + UT_EXPECT_EQ(unparallel_two.MappedVid(unparallel_two.OriginalVid(17)), 17); + + // test ConstructWithDegree() + OlapOnDB test_db_three(db, txn, SNAPSHOT_PARALLEL | SNAPSHOT_UNDIRECTED); + UT_EXPECT_EQ(test_db_three.OutDegree(0), 5); + UT_EXPECT_EQ(test_db_three.InDegree(0), 5); + UT_EXPECT_EQ(test_db_three.OutDegree(6), 2); + UT_EXPECT_EQ(test_db_three.InDegree(6), 2); + UT_EXPECT_EQ(test_db_three.OutDegree(13), 5); + UT_EXPECT_EQ(test_db_three.InDegree(13), 5); + UT_EXPECT_EQ(test_db_three.OutDegree(20), 2); + UT_EXPECT_EQ(test_db_three.InDegree(20), 2); + UT_EXPECT_EQ(test_db_three.NumVertices(), 21); + UT_EXPECT_EQ(test_db_three.NumEdges(), 70); + UT_EXPECT_EQ(test_db_two.MappedVid(3), 3); + UT_EXPECT_EQ(test_db_two.OriginalVid(7), 7); + + OlapOnDB directed_three(db, txn, SNAPSHOT_PARALLEL); + UT_EXPECT_EQ(directed_three.OutDegree(0), 5); + UT_EXPECT_EQ(directed_three.InDegree(0), 0); + UT_EXPECT_EQ(directed_three.OutDegree(6), 0); + UT_EXPECT_EQ(directed_three.InDegree(6), 2); + UT_EXPECT_EQ(directed_three.OutDegree(13), 2); + UT_EXPECT_EQ(directed_three.InDegree(13), 3); + UT_EXPECT_EQ(directed_three.OutDegree(20), 0); + UT_EXPECT_EQ(directed_three.InDegree(20), 2); + UT_EXPECT_EQ(directed_three.NumVertices(), 21); + UT_EXPECT_EQ(directed_three.NumEdges(), 35); + UT_EXPECT_EQ(directed_three.MappedVid(1), 1); + UT_EXPECT_EQ(directed_three.OriginalVid(20), 20); + }; - OlapOnDB parallel_two(db, txn, SNAPSHOT_PARALLEL, - nullptr, edge_convert_default); - UT_EXPECT_EQ(parallel_two.OutDegree(0), 5); - UT_EXPECT_EQ(parallel_two.InDegree(0), 0); - UT_EXPECT_EQ(parallel_two.OutDegree(6), 0); - UT_EXPECT_EQ(parallel_two.InDegree(6), 2); - UT_EXPECT_EQ(parallel_two.OutDegree(17), 1); - UT_EXPECT_EQ(parallel_two.InDegree(17), 3); - UT_EXPECT_EQ(parallel_two.OutDegree(20), 0); - UT_EXPECT_EQ(parallel_two.InDegree(20), 2); - - for (auto& edge : parallel_two.OutEdges(10)) { - UT_EXPECT_EQ(edge.edge_data, 1.0); + { + // test ConstructWithVid, Construct and ConstructWithDegree + for (auto p_db : {&db, (lgraph_api::GraphDB *)nullptr}) { + auto txn = db.CreateReadTxn(); + test_construct(p_db, txn); + txn.Abort(); + txn = db.CreateWriteTxn(); + test_construct(p_db, txn); + txn.Abort(); + } } - // test Construct() - OlapOnDB test_db_two(db, txn, SNAPSHOT_PARALLEL | SNAPSHOT_IDMAPPING); - UT_EXPECT_EQ(test_db_two.OutDegree(0), 5); - UT_EXPECT_EQ(test_db_two.InDegree(0), 0); - UT_EXPECT_EQ(test_db_two.OutDegree(6), 0); - UT_EXPECT_EQ(test_db_two.InDegree(6), 2); - UT_EXPECT_EQ(test_db_two.OutDegree(17), 1); - UT_EXPECT_EQ(test_db_two.InDegree(17), 3); - UT_EXPECT_EQ(test_db_two.OutDegree(20), 0); - UT_EXPECT_EQ(test_db_two.InDegree(20), 2); - UT_EXPECT_EQ(test_db_two.NumVertices(), 21); - UT_EXPECT_EQ(test_db_two.NumEdges(), 35); - UT_EXPECT_EQ(test_db_two.MappedVid(0), 0); - UT_EXPECT_EQ(test_db_two.OriginalVid(0), 0); - UT_EXPECT_EQ(test_db_two.MappedVid(test_db_two.OriginalVid(4)), 4); - - OlapOnDB unparallel_two(db, txn, SNAPSHOT_UNDIRECTED, - vertex_filter, edge_convert_weight); - UT_EXPECT_EQ(unparallel_two.OutDegree(0), 3); - UT_EXPECT_EQ(unparallel_two.InDegree(0), 3); - UT_EXPECT_EQ(unparallel_two.OutDegree(6), 2); - UT_EXPECT_EQ(unparallel_two.InDegree(6), 2); - UT_EXPECT_EQ(unparallel_two.OutDegree(13), 2); - UT_EXPECT_EQ(unparallel_two.InDegree(13), 2); - UT_EXPECT_EQ(unparallel_two.OutDegree(20), 0); - UT_EXPECT_EQ(unparallel_two.InDegree(20), 0); - UT_EXPECT_EQ(unparallel_two.NumVertices(), 20); - UT_EXPECT_EQ(unparallel_two.NumEdges(), 60); - UT_EXPECT_EQ(unparallel_two.MappedVid(unparallel_two.OriginalVid(17)), 17); - - std::string vertex_label = "node"; - std::string edge_label = "edge"; - OlapOnDB filter_graph(db, txn, SNAPSHOT_PARALLEL, - [&vertex_label](VertexIterator& vit) { - return vit.GetLabel() == vertex_label; - }, [&edge_label](OutEdgeIterator& eit, Empty& edata) { + { + // test filter subgraphs based on a set of triples of point labels, + // edge labels, and point labels + auto txn = db.CreateReadTxn(); + std::string vertex_label = "node"; + std::string edge_label = "edge"; + OlapOnDB filter_graph( + db, txn, SNAPSHOT_PARALLEL, + [&vertex_label](VertexIterator& vit) { return vit.GetLabel() == vertex_label; }, + [&edge_label](OutEdgeIterator& eit, Empty& edata) { return eit.GetLabel() == edge_label; }); - UT_EXPECT_EQ(filter_graph.OutDegree(0), 5); - UT_EXPECT_EQ(filter_graph.InDegree(0), 0); - UT_EXPECT_EQ(filter_graph.OutDegree(6), 0); - UT_EXPECT_EQ(filter_graph.InDegree(6), 2); - UT_EXPECT_EQ(filter_graph.OutDegree(13), 2); - UT_EXPECT_EQ(filter_graph.InDegree(13), 3); - UT_EXPECT_EQ(filter_graph.OutDegree(20), 0); - UT_EXPECT_EQ(filter_graph.InDegree(20), 2); - UT_EXPECT_EQ(filter_graph.NumVertices(), 21); - UT_EXPECT_EQ(filter_graph.NumEdges(), 35); - - std::vector> label_list = {{"node", "edge", "node"}}; - OlapOnDB filter_graph_two(db, txn, - label_list, SNAPSHOT_PARALLEL); - UT_EXPECT_EQ(filter_graph_two.OutDegree(0), 5); - UT_EXPECT_EQ(filter_graph_two.InDegree(0), 0); - UT_EXPECT_EQ(filter_graph_two.OutDegree(6), 0); - UT_EXPECT_EQ(filter_graph_two.InDegree(6), 2); - UT_EXPECT_EQ(filter_graph_two.OutDegree(13), 2); - UT_EXPECT_EQ(filter_graph_two.InDegree(13), 3); - UT_EXPECT_EQ(filter_graph_two.OutDegree(20), 0); - UT_EXPECT_EQ(filter_graph_two.InDegree(20), 2); - UT_EXPECT_EQ(filter_graph_two.NumVertices(), 21); - UT_EXPECT_EQ(filter_graph_two.NumEdges(), 35); - - txn.Commit(); - - // WriteToGraphDB - ParallelVector parent = filter_graph.AllocVertexArray(); - for (int i = 0; i < parent.Size(); i++) { - parent[i] = i; - } - filter_graph.WriteToGraphDB(parent, "value"); - txn = db.CreateReadTxn(); - auto vit = txn.GetVertexIterator(); - vit.Goto(2); - UT_EXPECT_EQ(vit.GetField("value").ToString(), "2"); - vit.Goto(20); - UT_EXPECT_EQ(vit.GetField("value").ToString(), "20"); - - // WriteToFile - std::string file_path = "./test_write_to_db.csv"; - filter_graph.WriteToFile(parent, file_path); - - vertex_label = "id"; - edge_label = ""; - UT_EXPECT_THROW_MSG( - OlapOnDB(db, txn, SNAPSHOT_PARALLEL, - [&vertex_label](VertexIterator& vit) { - if (vertex_label == "") return true; - return vit.GetLabel() == vertex_label; - }, [&edge_label](OutEdgeIterator& eit, Empty& edata) { - if (edge_label == "") return true; - return eit.GetLabel() == edge_label; - }), "The graph vertex cannot be empty"); - - vertex_label = ""; - edge_label = "id"; - UT_EXPECT_THROW_MSG( - OlapOnDB(db, txn, SNAPSHOT_PARALLEL, - [&vertex_label](VertexIterator& vit) { - if (vertex_label == "") return true; - return vit.GetLabel() == vertex_label; - }, [&edge_label](OutEdgeIterator& eit, Empty& edata) { - if (edge_label == "") return true; - return eit.GetLabel() == edge_label; - }), "The graph edge cannot be empty"); - - // test ConstructWithDegree() - OlapOnDB test_db_three(db, txn, SNAPSHOT_PARALLEL | SNAPSHOT_UNDIRECTED); - UT_EXPECT_EQ(test_db_three.OutDegree(0), 5); - UT_EXPECT_EQ(test_db_three.InDegree(0), 5); - UT_EXPECT_EQ(test_db_three.OutDegree(6), 2); - UT_EXPECT_EQ(test_db_three.InDegree(6), 2); - UT_EXPECT_EQ(test_db_three.OutDegree(13), 5); - UT_EXPECT_EQ(test_db_three.InDegree(13), 5); - UT_EXPECT_EQ(test_db_three.OutDegree(20), 2); - UT_EXPECT_EQ(test_db_three.InDegree(20), 2); - UT_EXPECT_EQ(test_db_three.NumVertices(), 21); - UT_EXPECT_EQ(test_db_three.NumEdges(), 70); - UT_EXPECT_EQ(test_db_two.MappedVid(3), 3); - UT_EXPECT_EQ(test_db_two.OriginalVid(7), 7); - - OlapOnDB directed_three(db, txn, SNAPSHOT_PARALLEL); - UT_EXPECT_EQ(directed_three.OutDegree(0), 5); - UT_EXPECT_EQ(directed_three.InDegree(0), 0); - UT_EXPECT_EQ(directed_three.OutDegree(6), 0); - UT_EXPECT_EQ(directed_three.InDegree(6), 2); - UT_EXPECT_EQ(directed_three.OutDegree(13), 2); - UT_EXPECT_EQ(directed_three.InDegree(13), 3); - UT_EXPECT_EQ(directed_three.OutDegree(20), 0); - UT_EXPECT_EQ(directed_three.InDegree(20), 2); - UT_EXPECT_EQ(directed_three.NumVertices(), 21); - UT_EXPECT_EQ(directed_three.NumEdges(), 35); - UT_EXPECT_EQ(directed_three.MappedVid(1), 1); - UT_EXPECT_EQ(directed_three.OriginalVid(20), 20); - txn.Commit(); - - // test ExtractVertexData - auto write_txn = db.CreateWriteTxn(); - { - OlapOnDB test_db_four(db, write_txn, SNAPSHOT_PARALLEL); - UT_EXPECT_EQ(test_db_four.NumEdges(), 35); - auto vertex_list = test_db_four.ExtractVertexData(vertex_extract); - UT_EXPECT_EQ(vertex_list[0], 2); - UT_EXPECT_EQ(vertex_list.Size(), 21); - UT_EXPECT_EQ(test_db_four.InDegree(0), 0); + UT_EXPECT_EQ(filter_graph.OutDegree(0), 5); + UT_EXPECT_EQ(filter_graph.InDegree(0), 0); + UT_EXPECT_EQ(filter_graph.OutDegree(6), 0); + UT_EXPECT_EQ(filter_graph.InDegree(6), 2); + UT_EXPECT_EQ(filter_graph.OutDegree(13), 2); + UT_EXPECT_EQ(filter_graph.InDegree(13), 3); + UT_EXPECT_EQ(filter_graph.OutDegree(20), 0); + UT_EXPECT_EQ(filter_graph.InDegree(20), 2); + UT_EXPECT_EQ(filter_graph.NumVertices(), 21); + UT_EXPECT_EQ(filter_graph.NumEdges(), 35); + + std::vector> label_list = {{"node", "edge", "node"}}; + OlapOnDB filter_graph_two(db, txn, label_list, SNAPSHOT_PARALLEL); + UT_EXPECT_EQ(filter_graph_two.OutDegree(0), 5); + UT_EXPECT_EQ(filter_graph_two.InDegree(0), 0); + UT_EXPECT_EQ(filter_graph_two.OutDegree(6), 0); + UT_EXPECT_EQ(filter_graph_two.InDegree(6), 2); + UT_EXPECT_EQ(filter_graph_two.OutDegree(13), 2); + UT_EXPECT_EQ(filter_graph_two.InDegree(13), 3); + UT_EXPECT_EQ(filter_graph_two.OutDegree(20), 0); + UT_EXPECT_EQ(filter_graph_two.InDegree(20), 2); + UT_EXPECT_EQ(filter_graph_two.NumVertices(), 21); + UT_EXPECT_EQ(filter_graph_two.NumEdges(), 35); + + txn.Commit(); + + // WriteToGraphDB + ParallelVector parent = filter_graph.AllocVertexArray(); + for (int i = 0; i < parent.Size(); i++) { + parent[i] = i; + } + filter_graph.WriteToGraphDB(parent, "value"); + txn = db.CreateReadTxn(); + auto vit = txn.GetVertexIterator(); + vit.Goto(2); + UT_EXPECT_EQ(vit.GetField("value").ToString(), "2"); + vit.Goto(20); + UT_EXPECT_EQ(vit.GetField("value").ToString(), "20"); + + // WriteToFile + std::string file_path = "./test_write_to_db.csv"; + filter_graph.WriteToFile(parent, file_path); + + vertex_label = "id"; + edge_label = ""; + UT_EXPECT_THROW_MSG(OlapOnDB( + db, txn, SNAPSHOT_PARALLEL, + [&vertex_label](VertexIterator& vit) { + if (vertex_label == "") return true; + return vit.GetLabel() == vertex_label; + }, + [&edge_label](OutEdgeIterator& eit, Empty& edata) { + if (edge_label == "") return true; + return eit.GetLabel() == edge_label; + }), + "The graph vertex cannot be empty"); + + vertex_label = ""; + edge_label = "id"; + UT_EXPECT_THROW_MSG(OlapOnDB( + db, txn, SNAPSHOT_PARALLEL, + [&vertex_label](VertexIterator& vit) { + if (vertex_label == "") return true; + return vit.GetLabel() == vertex_label; + }, + [&edge_label](OutEdgeIterator& eit, Empty& edata) { + if (edge_label == "") return true; + return eit.GetLabel() == edge_label; + }), + "The graph edge cannot be empty"); } - { - OlapOnDB parallel_four(db, write_txn, SNAPSHOT_PARALLEL | SNAPSHOT_IDMAPPING); - auto extract_list = parallel_four.ExtractVertexData(vertex_extract); - UT_EXPECT_EQ(extract_list[0], 2); - UT_EXPECT_EQ(extract_list.Size(), 21); + + { // test ExtractVertexData + auto write_txn = db.CreateWriteTxn(); + { + OlapOnDB test_db_four(db, write_txn, SNAPSHOT_PARALLEL); + UT_EXPECT_EQ(test_db_four.NumEdges(), 35); + auto vertex_list = test_db_four.ExtractVertexData(vertex_extract); + UT_EXPECT_EQ(vertex_list[0], 2); + UT_EXPECT_EQ(vertex_list.Size(), 21); + UT_EXPECT_EQ(test_db_four.InDegree(0), 0); + } + { + OlapOnDB parallel_four(db, write_txn, SNAPSHOT_PARALLEL | SNAPSHOT_IDMAPPING); + auto extract_list = parallel_four.ExtractVertexData(vertex_extract); + UT_EXPECT_EQ(extract_list[0], 2); + UT_EXPECT_EQ(extract_list.Size(), 21); + } + write_txn.Commit(); } - write_txn.Commit(); - std::vector del_filename = {"test_olap_on_db.conf", - "test_vertices.csv", "test_weighted.csv", "test_write_to_db.csv"}; + std::vector del_filename = {"test_olap_on_db.conf", "test_vertices.csv", + "test_weighted.csv", "test_write_to_db.csv"}; ClearCsvFiles(del_filename); } diff --git a/test/test_procedures/custom_pagerank.cpp b/test/test_procedures/custom_pagerank.cpp deleted file mode 100644 index 83202b6915..0000000000 --- a/test/test_procedures/custom_pagerank.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/** -* Copyright 2024 AntGroup CO., Ltd. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -*/ - - -#include -#include "lgraph/lgraph.h" -#include "lgraph/lgraph_types.h" -#include "lgraph/lgraph_result.h" - -#include "tools/json.hpp" - -using json = nlohmann::json; - -using namespace lgraph_api; - -extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { - sig_spec.input_list = { - {.name = "num_iteration", .index = 0, .type = LGraphType::INTEGER}, - }; - sig_spec.result_list = { - {.name = "node", .index = 0, .type = LGraphType::NODE}, - {.name = "weight", .index = 1, .type = LGraphType::FLOAT} - }; - return true; -} - -extern "C" LGAPI bool ProcessInTxn(Transaction &txn, - const std::string &request, - std::string &response) { - int64_t num_iteration; - try { - json input = json::parse(request); - num_iteration = input["num_iteration"].get(); - } catch (std::exception &e) { - response = std::string("error parsing json: ") + e.what(); - return false; - } - // handle the page rank algo in dummy mode - // ... - - Result result({{"node", LGraphType::NODE}, - {"weight", LGraphType::FLOAT}, - }); - - - for (size_t i = 0; i < 2; i++) { - auto r = result.MutableRecord(); - auto vit = txn.GetVertexIterator(i); - r->Insert("node", vit); - r->Insert("weight", FieldData::Float(float(i) + 0.1*float(i))); - } - response = result.Dump(); - return true; -} - diff --git a/test/test_procedures/multi_files.cpp b/test/test_procedures/multi_files.cpp new file mode 100644 index 0000000000..b3c9f23e37 --- /dev/null +++ b/test/test_procedures/multi_files.cpp @@ -0,0 +1,115 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +*/ + +#include "./multi_files.h" + +extern "C" bool Process(GraphDB& db, const std::string& request, std::string& response) { + auto start_time = get_time(); + + // prepare + start_time = get_time(); + std::string root_value = "0"; + std::string root_label = ""; + std::string root_field = ""; + std::string vertex_label_filter = ""; + std::string edge_label_filter = ""; + std::string parent_id = ""; + std::string output_file = ""; + std::cout << "Input: " << request << std::endl; + try { + json input = json::parse(request); + parse_from_json(root_value, "root_value", input); + parse_from_json(root_label, "root_label", input); + parse_from_json(root_field, "root_field", input); + parse_from_json(vertex_label_filter, "vertex_label_filter", input); + parse_from_json(edge_label_filter, "edge_label_filter", input); + parse_from_json(parent_id, "parent_id", input); + parse_from_json(output_file, "output_file", input); + } catch (std::exception& e) { + response = "json parse error: " + std::string(e.what()); + std::cout << response << std::endl; + return false; + } + + auto txn = db.CreateReadTxn(); + + std::function vertex_filter = nullptr; + std::function edge_filter = nullptr; + + if (!vertex_label_filter.empty()) { + vertex_filter = [&vertex_label_filter](VertexIterator& vit) { + return vit.GetLabel() == vertex_label_filter; + }; + } + + if (!edge_label_filter.empty()) { + edge_filter = [&edge_label_filter](OutEdgeIterator& eit, Empty& edata) { + return eit.GetLabel() == edge_label_filter; + }; + } + + if (root_label.empty() || root_field.empty()) { + if (root_label.empty() && root_field.empty()) { + root_label = "node"; + root_field = "id"; + } else { + THROW_CODE(InputError, "root_label or root_field is empty"); + } + } + + OlapOnDB olapondb(db, txn, SNAPSHOT_PARALLEL, vertex_filter, edge_filter); + int64_t root_vid = + txn.GetVertexIndexIterator(root_label, root_field, root_value, root_value).GetVid(); + auto prepare_cost = get_time() - start_time; + + // core + start_time = get_time(); + ParallelVector parent = olapondb.AllocVertexArray(); + size_t count = BFSCore(olapondb, olapondb.MappedVid(root_vid), parent); + auto core_cost = get_time() - start_time; + + // output + start_time = get_time(); +#pragma omp parallel for + for (size_t i = 0; i < parent.Size(); i++) { + if (parent[i] != (size_t)-1) { + parent[i] = olapondb.OriginalVid(parent[i]); + } + } + if (output_file != "") { + olapondb.WriteToFile(parent, output_file); + } + txn.Commit(); + + if (parent_id != "") { + olapondb.WriteToGraphDB(parent, parent_id); + } + + printf("found_vertices = %ld\n", count); + auto output_cost = get_time() - start_time; + + // return + { + json output; + output["found_vertices"] = count; + output["num_vertices"] = olapondb.NumVertices(); + output["num_edges"] = olapondb.NumEdges(); + output["prepare_cost"] = prepare_cost; + output["core_cost"] = core_cost; + output["output_cost"] = output_cost; + output["total_cost"] = prepare_cost + core_cost + output_cost; + response = output.dump(); + } + return true; +} diff --git a/test/test_procedures/multi_files.h b/test/test_procedures/multi_files.h new file mode 100644 index 0000000000..1ccb8121bc --- /dev/null +++ b/test/test_procedures/multi_files.h @@ -0,0 +1,25 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#pragma once + +#include "lgraph/olap_base.h" +#include "lgraph/olap_on_db.h" +#include "tools/json.hpp" + +using namespace lgraph_api; +using namespace lgraph_api::olap; +using json = nlohmann::json; + +size_t BFSCore(OlapBase& graph, size_t root_vid, ParallelVector& parent); diff --git a/test/test_procedures/multi_files_core.cpp b/test/test_procedures/multi_files_core.cpp new file mode 100644 index 0000000000..3ecb1d3d9d --- /dev/null +++ b/test/test_procedures/multi_files_core.cpp @@ -0,0 +1,51 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + +#include "./multi_files.h" + +size_t BFSCore(OlapBase& graph, size_t root_vid, ParallelVector& parent) { + size_t root = root_vid; + auto active_in = graph.AllocVertexSubset(); + active_in.Add(root); + auto active_out = graph.AllocVertexSubset(); + parent.Fill((size_t)-1); + parent[root] = root; + + size_t num_activations = 1; + size_t discovered_vertices = 0; + for (int ii = 0; num_activations != 0; ii++) { + printf("activates(%d) <= %lu\n", ii, num_activations); + discovered_vertices += num_activations; + active_out.Clear(); + num_activations = graph.ProcessVertexActive( + [&](size_t vi) { + size_t num_activations = 0; + for (auto& edge : graph.OutEdges(vi)) { + size_t dst = edge.neighbour; + if (parent[dst] == (size_t)-1) { + auto lock = graph.GuardVertexLock(dst); + if (parent[dst] == (size_t)-1) { + parent[dst] = vi; + num_activations += 1; + active_out.Add(dst); + } + } + } + return num_activations; + }, + active_in); + active_in.Swap(active_out); + } + return discovered_vertices; +} diff --git a/test/test_procedures/peek_some_node_salt.cpp b/test/test_procedures/peek_some_node_salt.cpp index 8c976b3c12..d2d3ed1cf5 100644 --- a/test/test_procedures/peek_some_node_salt.cpp +++ b/test/test_procedures/peek_some_node_salt.cpp @@ -35,25 +35,29 @@ extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { extern "C" LGAPI bool ProcessInTxn(Transaction &txn, const std::string &request, - std::string &response) { + Result &response) { int64_t limit; try { json input = json::parse(request); limit = input["limit"].get(); } catch (std::exception &e) { - response = std::string("error parsing json: ") + e.what(); + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); return false; } - Result result({{"node", LGraphType::NODE}, + response.ResetHeader({{"node", LGraphType::NODE}, {"salt", LGraphType::FLOAT}, }); for (size_t i = 0; i < limit; i++) { - auto r = result.MutableRecord(); + auto r = response.MutableRecord(); auto vit = txn.GetVertexIterator(i); r->Insert("node", vit); r->Insert("salt", FieldData::Float(20.23*float(i))); } - response = result.Dump(); return true; } diff --git a/test/test_procedures/custom_algo.cpp b/test/test_procedures/v2_algo.cpp similarity index 85% rename from test/test_procedures/custom_algo.cpp rename to test/test_procedures/v2_algo.cpp index a9f68cfe53..f9336a0255 100644 --- a/test/test_procedures/custom_algo.cpp +++ b/test/test_procedures/v2_algo.cpp @@ -28,10 +28,9 @@ extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { extern "C" LGAPI bool ProcessInTxn(Transaction &txn, const std::string &request, - std::string &response) { - Result result({{"res", LGraphType::STRING}}); - auto r = result.MutableRecord(); + Result &response) { + response.ResetHeader({{"res", LGraphType::STRING}}); + auto r = response.MutableRecord(); r->Insert("res", FieldData::String("custom algo result")); - response = result.Dump(); return true; } diff --git a/test/test_procedures/v2_pagerank.cpp b/test/test_procedures/v2_pagerank.cpp new file mode 100644 index 0000000000..98da566043 --- /dev/null +++ b/test/test_procedures/v2_pagerank.cpp @@ -0,0 +1,156 @@ +/** +* Copyright 2024 AntGroup CO., Ltd. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ + + +#include +#include "lgraph/lgraph.h" +#include "lgraph/lgraph_types.h" +#include "lgraph/lgraph_result.h" + +#include "tools/json.hpp" + +using json = nlohmann::json; +using namespace lgraph_api; +using namespace lgraph_api::olap; + +#define __ADD_DANGLING__ + +void PageRankCore(OlapBase& graph, int num_iterations, ParallelVector& curr) { + auto all_vertices = graph.AllocVertexSubset(); + all_vertices.Fill(); + auto next = graph.AllocVertexArray(); + size_t num_vertices = graph.NumVertices(); + + double one_over_n = (double)1 / num_vertices; + double delta = 1; + double dangling = graph.ProcessVertexActive( + [&](size_t vi) { + curr[vi] = one_over_n; + if (graph.OutDegree(vi) > 0) { + curr[vi] /= graph.OutDegree(vi); + return 0.0; + } +#ifdef __ADD_DANGLING__ + return one_over_n; +#else + return 0.0; +#endif + }, + all_vertices); + dangling /= num_vertices; + + double d = (double)0.85; + for (int ii = 0; ii < num_iterations; ii++) { + printf("delta(%d)=%lf\n", ii, delta); + next.Fill((double)0); + delta = graph.ProcessVertexActive( + [&](size_t vi) { + double sum = 0; + for (auto& edge : graph.InEdges(vi)) { + size_t src = edge.neighbour; + sum += curr[src]; + } + next[vi] = sum; + next[vi] = (1 - d) * one_over_n + d * next[vi] + d * dangling; + if (ii == num_iterations - 1) { + return (double)0; + } else { + if (graph.OutDegree(vi) > 0) { + next[vi] /= graph.OutDegree(vi); + return fabs(next[vi] - curr[vi]) * graph.OutDegree(vi); + } else { + return fabs(next[vi] - curr[vi]); + } + } + }, + all_vertices); + curr.Swap(next); + +#ifdef __ADD_DANGLING__ + dangling = graph.ProcessVertexActive( + [&](size_t vi) { + if (graph.OutDegree(vi) == 0) + return curr[vi]; + return 0.0; + }, + all_vertices); + dangling /= num_vertices; +#endif + } +} + +extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { + sig_spec.input_list = { + {.name = "num_iteration", .index = 0, .type = LGraphType::INTEGER}, + }; + sig_spec.result_list = { + {.name = "node", .index = 0, .type = LGraphType::NODE}, + {.name = "weight", .index = 1, .type = LGraphType::DOUBLE} + }; + return true; +} + +extern "C" LGAPI bool ProcessInTxn(Transaction &txn, + const std::string &request, + Result &response) { + double start_time = get_time(); + int64_t num_iteration; + LOG_DEBUG() << "input: " << request; + try { + json input = json::parse(request); + num_iteration = input["num_iteration"].get(); + } catch (std::exception &e) { + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); + return false; + } + + OlapOnDB graph(txn, SNAPSHOT_PARALLEL); + auto prepare_cost = get_time() - start_time; + start_time = get_time(); + auto pr = graph.AllocVertexArray(); + PageRankCore(graph, num_iteration, pr); + size_t max_pr_vi = graph.ProcessVertexInRange( + [&](size_t vi) { return vi; }, 0, graph.NumVertices(), 0, + [&](size_t a, size_t b) { return pr[a] > pr[b] ? a : b; }); + LOG_INFO() << FMA_FMT("max_pr: pr[{}] = {}", max_pr_vi, pr[max_pr_vi]); + auto core_cost = get_time() - start_time; + + start_time = get_time(); + response.ResetHeader({ + {"node", LGraphType::NODE}, + {"weight", LGraphType::DOUBLE}, + }); + + response.Resize(pr.Size()); + graph.ProcessVertexInRange( + [&] (size_t vid) { + auto r = response.At(vid); + r->InsertVertexByID("node", graph.OriginalVid(vid)); + r->Insert("weight", FieldData::Double(pr[vid])); + return 0; + }, + 0, pr.Size()); + auto output_cost = get_time() - start_time; + + LOG_DEBUG() << "prepare_cost: " << prepare_cost << " (s)\n" + << "core_cost: " << core_cost << " (s)\n" + << "output_cost: " << output_cost << " (s)\n" + << "response.Size: " << response.Size(); + return true; +} diff --git a/test/test_procedures/custom_path_process.cpp b/test/test_procedures/v2_path_process.cpp similarity index 79% rename from test/test_procedures/custom_path_process.cpp rename to test/test_procedures/v2_path_process.cpp index d4c98740d0..02985a934b 100644 --- a/test/test_procedures/custom_path_process.cpp +++ b/test/test_procedures/v2_path_process.cpp @@ -37,13 +37,18 @@ extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { } extern "C" LGAPI bool ProcessInTxn(Transaction &txn, const std::string &request, - std::string &response) { + Result &response) { std::vector nodeIds; try { json input = json::parse(request); nodeIds = input["nodeIds"].get>(); } catch (std::exception &e) { - response = std::string("error parsing json: ") + e.what(); + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); return false; } @@ -52,10 +57,9 @@ extern "C" LGAPI bool ProcessInTxn(Transaction &txn, sum += id; } - Result result({{"idSum", LGraphType::INTEGER}}); + response.ResetHeader({{"idSum", LGraphType::INTEGER}}); - auto r = result.MutableRecord(); + auto r = response.MutableRecord(); r->Insert("idSum", FieldData::Int64(sum)); - response = result.Dump(); return true; } diff --git a/test/test_procedures/custom_shortestpath.cpp b/test/test_procedures/v2_test_path.cpp similarity index 85% rename from test/test_procedures/custom_shortestpath.cpp rename to test/test_procedures/v2_test_path.cpp index 333982c9c1..d1ecd4cf69 100644 --- a/test/test_procedures/custom_shortestpath.cpp +++ b/test/test_procedures/v2_test_path.cpp @@ -38,14 +38,19 @@ extern "C" LGAPI bool GetSignature(SigSpec &sig_spec) { } extern "C" LGAPI bool ProcessInTxn(Transaction &txn, const std::string &request, - std::string &response) { + Result &response) { std::string start_node, end_node; try { json input = json::parse(request); start_node = input["start"].get(); end_node = input["end"].get(); } catch (std::exception &e) { - response = std::string("error parsing json: ") + e.what(); + response.ResetHeader({ + {"errMsg", LGraphType::STRING} + }); + response.MutableRecord()->Insert( + "errMsg", + FieldData::String(std::string("error parsing json: ") + e.what())); return false; } // handle the shortest path algo in dummy mode @@ -55,18 +60,17 @@ extern "C" LGAPI bool ProcessInTxn(Transaction &txn, return strtoll(s.c_str() + id_begin + 1, NULL, 10); }; - Result result({{"length", LGraphType::INTEGER}, + response.ResetHeader({{"length", LGraphType::INTEGER}, {"nodeIds", LGraphType::LIST}, }); - auto r = result.MutableRecord(); + auto r = response.MutableRecord(); r->Insert("length", FieldData::Int32(5)); r->Insert("nodeIds", std::vector{ FieldData::Int64(parse_vertex_node(start_node)), FieldData::Int64(100), FieldData::Int64(200), FieldData::Int64(300), FieldData::Int64(parse_vertex_node(end_node)) }); - response = result.Dump(); return true; } diff --git a/test/test_python_plugin_manager.cpp b/test/test_python_plugin_manager.cpp index 3f16103709..a288674bfd 100644 --- a/test/test_python_plugin_manager.cpp +++ b/test/test_python_plugin_manager.cpp @@ -119,13 +119,16 @@ def Process(db, input): UT_LOG() << "Testing normal actions"; PluginTester manager(db.GetLightningGraph(), plugin_dir, "python_plugin", n_workers); UT_EXPECT_TRUE(manager.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, - "sleep", code_sleep, plugin::CodeType::PY, "sleep for n seconds", true, "v1")); + "sleep", std::vector{code_sleep}, std::vector{"sleep.py"}, + plugin::CodeType::PY, "sleep for n seconds", true, "v1")); UT_LOG() << "Testing normal actions1"; UT_EXPECT_TRUE(manager.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, - "scan_graph", code_read, plugin::CodeType::PY, + "scan_graph", std::vector{code_read}, + std::vector{"scan.py"}, plugin::CodeType::PY, "scan graph for at most n vertices", true, "v1")); UT_EXPECT_TRUE(manager.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, - "add", code_write, plugin::CodeType::PY, "write a vertex", true, "v1")); + "add", std::vector{code_write}, std::vector{"add.py"}, + plugin::CodeType::PY, "write a vertex", true, "v1")); UT_EXPECT_EQ(manager.procedures_.size(), 3); auto plugins = manager.ListPlugins(lgraph::_detail::DEFAULT_ADMIN_NAME); UT_EXPECT_EQ(plugins.size(), 3); @@ -151,10 +154,12 @@ def Process(db, input): UT_LOG() << "Updating plugin"; // already exists UT_EXPECT_TRUE(!manager.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, - "add", code_write, plugin::CodeType::PY, "write v2", false, "v1")); + "add", std::vector{code_write}, std::vector{"add.py"}, + plugin::CodeType::PY, "write v2", false, "v1")); UT_EXPECT_TRUE(manager.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "add")); UT_EXPECT_TRUE(manager.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, - "add", code_write, plugin::CodeType::PY, "write v2", false, "v1")); + "add", std::vector{code_write}, std::vector{"add.py"}, + plugin::CodeType::PY, "write v2", false, "v1")); plugins = manager.ListPlugins(lgraph::_detail::DEFAULT_ADMIN_NAME); UT_EXPECT_EQ(plugins.size(), 3); UT_EXPECT_EQ(plugins[0].name, "add"); @@ -186,7 +191,9 @@ def Process(db, input): UT_EXPECT_TRUE(output == ""); UT_EXPECT_TRUE(manager.DelPlugin(lgraph::_detail::DEFAULT_ADMIN_NAME, "sleep")); UT_EXPECT_TRUE(manager.LoadPluginFromCode(lgraph::_detail::DEFAULT_ADMIN_NAME, - "sleep", code_read, plugin::CodeType::PY, + "sleep", std::vector{code_read}, + std::vector{"sleep.py"}, + plugin::CodeType::PY, "code read but name sleep", true, "v1")); UT_EXPECT_TRUE(manager.GetPluginCode(lgraph::_detail::DEFAULT_ADMIN_NAME, "sleep", pc)); UT_EXPECT_TRUE(code_read.compare(pc.code) == 0 && pc.code_type == "py"); diff --git a/test/test_rest_client.cpp b/test/test_rest_client.cpp index 59d0de632b..c0d1eae125 100644 --- a/test/test_rest_client.cpp +++ b/test/test_rest_client.cpp @@ -218,6 +218,23 @@ TEST_P(TestRestClient, RestClient) { client->SetUserDesc(new_user, new_desc2); UT_EXPECT_EQ(client->GetUserInfo(new_user).desc, new_desc2); } + + DefineTest("LoadPlugin") { + Setup(); + auto ReadCode = [](const std::string& path) { + std::string ret; + fma_common::InputFmaStream ifs(path); + ret.resize(ifs.Size()); + ifs.Read(&ret[0], ret.size()); + return ret; + }; + std::string file_path = "../../test/test_procedures/sortstr.cpp"; + bool ret; + UT_EXPECT_NO_THROW(ret = client->LoadPlugin(db_name, lgraph_api::PluginCodeType::CPP, + lgraph::PluginDesc("sort", lgraph::plugin::PLUGIN_CODE_TYPE_CPP, "sort", + lgraph::plugin::PLUGIN_VERSION_1, true, ""), ReadCode(file_path))); + UT_EXPECT_EQ(ret, true); + } fma_common::file_system::RemoveDir(db_dir); } diff --git a/test/test_restful_base_operation.cpp b/test/test_restful_base_operation.cpp index cae2cdbc37..ad11d31cd0 100644 --- a/test/test_restful_base_operation.cpp +++ b/test/test_restful_base_operation.cpp @@ -723,6 +723,43 @@ TEST_P(TestRestfulBaseOperation, RestfulBaseOperation) { "V0KT" "oKICAgIHJldHVybiAoVHJ1ZSwgaW5wdXQpCg=="); + auto read_file = [] (std::string path) { + auto& fs = fma_common::FileSystem::GetFileSystem(path); + std::string res; + if (fs.FileExists(path)) { + fma_common::InputFmaStream ifs(path, 0); + size_t sz = ifs.Size(); + res.resize(sz); + size_t ssz = ifs.Read(&res[0], sz); + UT_EXPECT_TRUE(ssz == sz); + } + return res; + }; + std::string multi_core = read_file("../../test/test_procedures/multi_files_core.cpp"); + std::string multi_procedure = read_file("../../test/test_procedures/multi_files.cpp"); + std::string multi_header = read_file("../../test/test_procedures/multi_files.h"); + std::vector codes{multi_core, multi_procedure, multi_header}; + std::vector filenames{ + "multi_files_core.cpp", "multi_files.cpp", "multi_files.h"}; + UT_EXPECT_EQ(client.LoadPlugin(db_name, lgraph_api::PluginCodeType::CPP, + PluginDesc("cpp_test", lgraph::plugin::PLUGIN_CODE_TYPE_CPP, + "test cpp plugin", + lgraph::plugin::PLUGIN_VERSION_1, true, ""), + codes, filenames), + true); + + auto plugins = client.GetPlugin(db_name, true); + UT_EXPECT_EQ(plugins.size(), 1); + UT_EXPECT_EQ(plugins[0].name, "cpp_test"); + UT_EXPECT_EQ(plugins[0].desc, "test cpp plugin"); + UT_EXPECT_EQ(plugins[0].read_only, true); + plugins.clear(); + + client.DeletePlugin(db_name, true, "cpp_test"); + plugins = client.GetPlugin(db_name, true); + UT_EXPECT_EQ(plugins.size(), 0); + plugins.clear(); + #if LGRAPH_ENABLE_PYTHON_PLUGIN UT_EXPECT_EQ(client.LoadPlugin(db_name, lgraph_api::PluginCodeType::PY, PluginDesc("py_test", lgraph::plugin::PLUGIN_CODE_TYPE_PY, @@ -744,7 +781,7 @@ TEST_P(TestRestfulBaseOperation, RestfulBaseOperation) { lgraph::plugin::PLUGIN_VERSION_1, true, ""), code)); // cpp - auto plugins = client.GetPlugin(db_name, true); + plugins = client.GetPlugin(db_name, true); UT_EXPECT_EQ(plugins.size(), 0); plugins.clear(); // py diff --git a/test/test_rpc.cpp b/test/test_rpc.cpp index d25e705f0f..aff7397058 100644 --- a/test/test_rpc.cpp +++ b/test/test_rpc.cpp @@ -1628,6 +1628,16 @@ void test_cpp_procedure(lgraph::RpcClient& client) { ret = client.LoadProcedure(str, code_cpp_path, "CPP", "test_procedure5", "CPP", "this is a test procedure", true, "v1"); UT_EXPECT_TRUE(ret); + + std::string multi_procedure_path = "../../test/test_procedures/multi_files.cpp"; + std::string multi_header_path = "../../test/test_procedures/multi_files.h"; + std::string multi_core_path = "../../test/test_procedures/multi_files_core.cpp"; + ret = client.LoadProcedure(str, std::vector{ + multi_procedure_path, multi_header_path, multi_core_path}, + "CPP", "test_procedure6", "CPP", + "this is a test procedure", true, "v1"); + UT_EXPECT_TRUE(ret); + #ifndef __SANITIZE_ADDRESS__ ret = client.CallCypher(str, "CALL db.plugin.getPluginInfo('PY','countPersons')"); UT_EXPECT_FALSE(ret); @@ -1637,7 +1647,7 @@ void test_cpp_procedure(lgraph::RpcClient& client) { ret = client.ListProcedures(str, "CPP", "any"); UT_EXPECT_TRUE(ret); json_val = web::json::value::parse(str); - UT_EXPECT_EQ(json_val.as_array().size(), 5); + UT_EXPECT_EQ(json_val.as_array().size(), 6); UT_EXPECT_EQ( CheckObjectElementEqual(json_val, "plugin_description", "name", "test_procedure1", "STRING"), true); @@ -1653,6 +1663,9 @@ void test_cpp_procedure(lgraph::RpcClient& client) { UT_EXPECT_EQ( CheckObjectElementEqual(json_val, "plugin_description", "name", "test_procedure5", "STRING"), true); + UT_EXPECT_EQ( + CheckObjectElementEqual(json_val, "plugin_description", "name", "test_procedure6", + "STRING"), true); ret = client.CallProcedure(str, "CPP", "test_procedure1", "bcefg"); UT_EXPECT_TRUE(ret); json_val = web::json::value::parse(str);