Merge pull request #2803 from AK-ayush/feat-edit-mat

[REVIEW] Feature: pairwise edit distance for each string on a given nvstrings object
rapidsai · Sep 28, 2019 · 7f32395 · 7f32395
2 parents 989190f + ff9eced
commit 7f32395
Show file tree

Hide file tree

Showing 24 changed files with 349 additions and 42 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@
 - PR #2844 Add Series/DataFrame notnull
 - PR #2858 Add GTest type list utilities
 - PR #2655 CuPy-based Series and Dataframe .values property
+- PR #2803 Added `edit_distance_matrix()` function to calculate pairwise edit distance for each string on a given nvstrings object.
 
 ## Improvements
 

diff --git a/cpp/custrings/category/NVCategory.cu b/cpp/custrings/category/NVCategory.cu
@@ -26,6 +26,7 @@
 #include <thrust/copy.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVCategory.h"
 #include "nvstrings/NVStrings.h"

diff --git a/cpp/custrings/strings/NVStrings.cu b/cpp/custrings/strings/NVStrings.cu
@@ -28,6 +28,7 @@
 #include <thrust/unique.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 #include "nvstrings/ipc_transfer.h"

diff --git a/cpp/custrings/strings/NVStringsImpl.cu b/cpp/custrings/strings/NVStringsImpl.cu
@@ -24,6 +24,7 @@
 #include <thrust/sort.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/array.cu b/cpp/custrings/strings/array.cu
@@ -26,6 +26,7 @@
 #include <thrust/count.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/attrs.cu b/cpp/custrings/strings/attrs.cu
@@ -21,6 +21,7 @@
 #include <thrust/transform_scan.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/combine.cu b/cpp/custrings/strings/combine.cu
@@ -20,6 +20,7 @@
 #include <thrust/for_each.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/convert.cu b/cpp/custrings/strings/convert.cu
@@ -22,6 +22,7 @@
 #include <thrust/count.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/count.cu b/cpp/custrings/strings/count.cu
@@ -22,6 +22,7 @@
 #include <thrust/count.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/datetime.cu b/cpp/custrings/strings/datetime.cu
@@ -22,6 +22,7 @@
 #include <thrust/count.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/find.cu b/cpp/custrings/strings/find.cu
@@ -21,6 +21,7 @@
 #include <thrust/count.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/modify.cu b/cpp/custrings/strings/modify.cu
@@ -24,6 +24,7 @@
 #include <thrust/sort.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/split.cu b/cpp/custrings/strings/split.cu
@@ -21,6 +21,7 @@
 #include <thrust/extrema.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/strings/strip.cu b/cpp/custrings/strings/strip.cu
@@ -19,6 +19,7 @@
 #include <thrust/for_each.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 

diff --git a/cpp/custrings/tests/test_text.cu b/cpp/custrings/tests/test_text.cu
@@ -127,6 +127,27 @@ TEST(TestText, EditDistance)
     NVStrings::destroy(strs);
 }
 
+TEST(TestText, EditDistanceMatrix)
+{
+    std::vector<const char*> hstrs{ "dog", nullptr, "cat", "mouse",
+                                    "pup", "", "puppy" };
+    NVStrings* strs = NVStrings::create_from_array(hstrs.data(),hstrs.size());
+
+    thrust::device_vector<unsigned int> results(hstrs.size()*hstrs.size(),0);
+
+    NVText::edit_distance_matrix(NVText::levenshtein,*strs,results.data().get());
+    unsigned int expected[] = { 0,3,3,4,3,3,5,
+                                3,0,3,5,3,0,5,
+                                3,3,0,5,3,3,5,
+                                4,5,5,0,4,5,5,
+                                3,3,3,4,0,3,2,
+                                3,0,3,5,3,0,5,
+                                5,5,5,5,2,5,0};
+    for( int idx = 0; idx < (int) (hstrs.size()*hstrs.size()); ++idx )
+        EXPECT_EQ(results[idx],expected[idx]);
+    NVStrings::destroy(strs);
+}
+
 TEST(TestText, NGrams)
 {
     NVStrings* strs = NVStrings::create_from_array(tstrs.data(),tstrs.size());

diff --git a/cpp/custrings/text/NVText.cu b/cpp/custrings/text/NVText.cu
@@ -20,6 +20,7 @@
 #include <thrust/reduce.h>
 #include <rmm/rmm.h>
 #include <rmm/thrust_rmm_allocator.h>
+#include <utilities/error_utils.hpp>
 
 #include "nvstrings/NVStrings.h"
 #include "nvstrings/NVText.h"