WIP

DeanLight · Dec 23, 2024 · 623e8a0 · 623e8a0
1 parent e4c1491
commit 623e8a0
Show file tree

Hide file tree

Showing 10,038 changed files with 57 additions and 10,723 deletions.
diff --git a/nbs/benchmark/001_covid.ipynb b/nbs/benchmark/001_covid.ipynb
diff --git a/nbs/benchmark/covid_bench.py b/nbs/benchmark/covid_bench.py
@@ -1,20 +1,35 @@
 import time
+
+start_time = time.time()
+
 from glob import glob
 import pandas as pd
+import spacy
 from pandas import DataFrame
 from pathlib import Path
-from spannerlib import get_magic_session,Session,Span
-sess = get_magic_session()
+from spannerlib import get_magic_session,Session
+from spannerlib.ie_func.basic import rgx, rgx_split, rgx_is_match, span_contained, span_arity
+
+VERSION = "SPANNERFLOW"
+if VERSION in ["SPANNERFLOW", "SPANNERFLOW_PYTHON_IE"]:
+    from spannerflow.span import Span
+else:
+    from spannerlib import Span
+
+def is_adjacent(span1,span2):
+    yield span1.name==span2.name and span1.end +1 == span2.start
+
+
+
 
-import spacy
 nlp = spacy.load("en_core_web_sm")
 
 # configurations
 slog_file = Path('covid_bench_logic.pl')
 input_dir = Path('covid_data/sample_inputs')
 data_dir = Path('covid_data/rules_data')
 
-start_time = time.time()
+
 
 def split_sentence(text):
     """
@@ -138,10 +153,20 @@ def rewrite_docs(docs,span_label,new_version):
 def main(input_dir,data_dir,logic_file, start=0, end=10):
     global file_paths
     sess = Session()
+    sess.register('py_rgx', rgx, [str, Span], span_arity)
+    sess.register('py_rgx_split', rgx_split, [str, Span], [Span,Span])
+    sess.register('py_rgx_is_match', rgx_is_match, [str, Span], [bool])
+    sess.register('py_span_contained', span_contained, [Span, Span], [bool])
+    sess.register('is_adjacent',is_adjacent,[Span,Span],[bool])
     # define callback functions
-    sess.register('split_sentence',split_sentence,[(str,Span)],[Span])
-    sess.register('pos',pos_annotator,[(Span,str)],[Span,str])
-    sess.register('lemma',lemmatizer,[(Span,str)],[Span,str])
+    if VERSION in ["SPANNERFLOW", "SPANNERFLOW_PYTHON_IE"]:
+        sess.register('split_sentence',split_sentence,[Span],[Span])
+        sess.register('pos',pos_annotator,[Span],[Span,str])
+        sess.register('lemma',lemmatizer,[Span],[Span,str])
+    else:
+        sess.register('split_sentence',split_sentence,[str],[Span])
+        sess.register('pos',pos_annotator,[str],[Span,str])
+        sess.register('lemma',lemmatizer,[str],[Span,str])
     sess.register_agg('agg_mention',agg_mention,[str],[str])
     sess.register_agg('agg_doc_tags',AggDocumentTags,[str],[str])
 
@@ -169,7 +194,10 @@ def main(input_dir,data_dir,logic_file, start=0, end=10):
         [p.name,p.read_text(),'raw_text'] for p in file_paths
     ],columns=['Path','Doc','Version']
     )
-    sess.import_rel('Docs',raw_docs)
+    if VERSION in ["SPANNERFLOW", "SPANNERFLOW_PYTHON_IE"]:
+        sess.import_rel('Docs',raw_docs, scheme=[str, Span, str])
+    else:
+        sess.import_rel('Docs',raw_docs)
 
     # load logic, note that since we did not define the data relations in the logic file,
     # we need to load the logic after the data has been loaded
@@ -178,19 +206,32 @@ def main(input_dir,data_dir,logic_file, start=0, end=10):
     ## Rewritting the documents
     lemma_tags = sess.export('?Lemmas(P,D,W,L)')
     lemma_docs = rewrite_docs(raw_docs,lemma_tags,'lemma')
-    sess.import_rel('Docs',lemma_docs)
+    if VERSION in ["SPANNERFLOW", "SPANNERFLOW_PYTHON_IE"]:
+        sess.import_rel('Docs',lemma_docs, scheme=[str, Span, str])
+    else:
+        sess.import_rel('Docs',lemma_docs)
+
 
     lemma_concept_matches = sess.export('?LemmaConceptMatches(Path,Doc,Span,Label)')
     lemma_concepts = rewrite_docs(lemma_docs,lemma_concept_matches,'lemma_concept')
-    sess.import_rel('Docs',lemma_concepts)
+    if VERSION in ["SPANNERFLOW", "SPANNERFLOW_PYTHON_IE"]:
+        sess.import_rel('Docs',lemma_concepts, scheme=[str, Span, str])
+    else:
+        sess.import_rel('Docs',lemma_concepts)
 
     pos_concept_matches = sess.export('?PosConceptMatches(P,D,W,L)')
     pos_concept_docs = rewrite_docs(lemma_concepts,pos_concept_matches,'pos_concept')
-    sess.import_rel('Docs',pos_concept_docs)
+    if VERSION in ["SPANNERFLOW", "SPANNERFLOW_PYTHON_IE"]:
+        sess.import_rel('Docs',pos_concept_docs, scheme=[str, Span, str])
+    else:
+        sess.import_rel('Docs',pos_concept_docs)
 
     target_matches = sess.export('?TargetMatches(P,D,W,L)')
     target_rule_docs = rewrite_docs(pos_concept_docs,target_matches,'target_concept')
-    sess.import_rel('Docs',target_rule_docs)
+    if VERSION in ["SPANNERFLOW", "SPANNERFLOW_PYTHON_IE"]:
+        sess.import_rel('Docs',target_rule_docs, scheme=[str, Span, str])
+    else:
+        sess.import_rel('Docs',target_rule_docs)
 
     ## computing the tags based on the target concept documents
     doc_tags = sess.export('?DocumentTags(P,T)')
@@ -203,9 +244,9 @@ def main(input_dir,data_dir,logic_file, start=0, end=10):
 
     return classification
 
-
-for i in range(0, 1000, 20):
-    res = main(input_dir,data_dir,slog_file, start=i, end=i+20)
+k = 50
+for i in range(0, 100-k, k):
+    res = main(input_dir,data_dir,slog_file, start=i, end=i+k)
     print(res)
 
 end_time = time.time()

diff --git a/nbs/benchmark/covid_bench_logic.pl b/nbs/benchmark/covid_bench_logic.pl
@@ -28,7 +28,7 @@
 
 Sents(P,S)<-Docs(P,D,"target_concept"),split_sentence(D)->(S).
 
-SentPairs(P,S1,S2)<-Sents(P,S1),Sents(P,S2),expr_eval("{0}.end +1 == {1}.start",S1,S2)->(True).
+SentPairs(P,S1,S2)<-Sents(P,S1),Sents(P,S2),is_adjacent(S1,S2)->(True).
 
 # first we get the covid mentions and their surrounding sentences, using the span_contained ie function
 CovidMentions(Path, Span) <- Docs(Path,D,"target_concept"), rgx("COVID-19",D) -> (Span).

diff --git a/nbs/benchmark/covid_data/temp/sample100.txt b/nbs/benchmark/covid_data/temp/sample100.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1000.txt b/nbs/benchmark/covid_data/temp/sample1000.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10000.txt b/nbs/benchmark/covid_data/temp/sample10000.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10001.txt b/nbs/benchmark/covid_data/temp/sample10001.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10002.txt b/nbs/benchmark/covid_data/temp/sample10002.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10003.txt b/nbs/benchmark/covid_data/temp/sample10003.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10004.txt b/nbs/benchmark/covid_data/temp/sample10004.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10005.txt b/nbs/benchmark/covid_data/temp/sample10005.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10006.txt b/nbs/benchmark/covid_data/temp/sample10006.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10007.txt b/nbs/benchmark/covid_data/temp/sample10007.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10008.txt b/nbs/benchmark/covid_data/temp/sample10008.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10009.txt b/nbs/benchmark/covid_data/temp/sample10009.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1001.txt b/nbs/benchmark/covid_data/temp/sample1001.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10010.txt b/nbs/benchmark/covid_data/temp/sample10010.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10011.txt b/nbs/benchmark/covid_data/temp/sample10011.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10012.txt b/nbs/benchmark/covid_data/temp/sample10012.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10013.txt b/nbs/benchmark/covid_data/temp/sample10013.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10014.txt b/nbs/benchmark/covid_data/temp/sample10014.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10015.txt b/nbs/benchmark/covid_data/temp/sample10015.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10016.txt b/nbs/benchmark/covid_data/temp/sample10016.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10017.txt b/nbs/benchmark/covid_data/temp/sample10017.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10018.txt b/nbs/benchmark/covid_data/temp/sample10018.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10019.txt b/nbs/benchmark/covid_data/temp/sample10019.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1002.txt b/nbs/benchmark/covid_data/temp/sample1002.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10020.txt b/nbs/benchmark/covid_data/temp/sample10020.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10021.txt b/nbs/benchmark/covid_data/temp/sample10021.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10022.txt b/nbs/benchmark/covid_data/temp/sample10022.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10023.txt b/nbs/benchmark/covid_data/temp/sample10023.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10024.txt b/nbs/benchmark/covid_data/temp/sample10024.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10025.txt b/nbs/benchmark/covid_data/temp/sample10025.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10026.txt b/nbs/benchmark/covid_data/temp/sample10026.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10027.txt b/nbs/benchmark/covid_data/temp/sample10027.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10028.txt b/nbs/benchmark/covid_data/temp/sample10028.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10029.txt b/nbs/benchmark/covid_data/temp/sample10029.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1003.txt b/nbs/benchmark/covid_data/temp/sample1003.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10030.txt b/nbs/benchmark/covid_data/temp/sample10030.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10031.txt b/nbs/benchmark/covid_data/temp/sample10031.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10032.txt b/nbs/benchmark/covid_data/temp/sample10032.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10033.txt b/nbs/benchmark/covid_data/temp/sample10033.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10034.txt b/nbs/benchmark/covid_data/temp/sample10034.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10035.txt b/nbs/benchmark/covid_data/temp/sample10035.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10036.txt b/nbs/benchmark/covid_data/temp/sample10036.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10037.txt b/nbs/benchmark/covid_data/temp/sample10037.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10038.txt b/nbs/benchmark/covid_data/temp/sample10038.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10039.txt b/nbs/benchmark/covid_data/temp/sample10039.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1004.txt b/nbs/benchmark/covid_data/temp/sample1004.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10040.txt b/nbs/benchmark/covid_data/temp/sample10040.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10041.txt b/nbs/benchmark/covid_data/temp/sample10041.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10042.txt b/nbs/benchmark/covid_data/temp/sample10042.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10043.txt b/nbs/benchmark/covid_data/temp/sample10043.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10044.txt b/nbs/benchmark/covid_data/temp/sample10044.txt
diff --git a/nbs/benchmark/covid_data/temp/sample10045.txt b/nbs/benchmark/covid_data/temp/sample10045.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1005.txt b/nbs/benchmark/covid_data/temp/sample1005.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1006.txt b/nbs/benchmark/covid_data/temp/sample1006.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1007.txt b/nbs/benchmark/covid_data/temp/sample1007.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1008.txt b/nbs/benchmark/covid_data/temp/sample1008.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1009.txt b/nbs/benchmark/covid_data/temp/sample1009.txt
diff --git a/nbs/benchmark/covid_data/temp/sample101.txt b/nbs/benchmark/covid_data/temp/sample101.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1010.txt b/nbs/benchmark/covid_data/temp/sample1010.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1011.txt b/nbs/benchmark/covid_data/temp/sample1011.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1012.txt b/nbs/benchmark/covid_data/temp/sample1012.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1013.txt b/nbs/benchmark/covid_data/temp/sample1013.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1014.txt b/nbs/benchmark/covid_data/temp/sample1014.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1015.txt b/nbs/benchmark/covid_data/temp/sample1015.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1016.txt b/nbs/benchmark/covid_data/temp/sample1016.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1017.txt b/nbs/benchmark/covid_data/temp/sample1017.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1018.txt b/nbs/benchmark/covid_data/temp/sample1018.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1019.txt b/nbs/benchmark/covid_data/temp/sample1019.txt
diff --git a/nbs/benchmark/covid_data/temp/sample102.txt b/nbs/benchmark/covid_data/temp/sample102.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1020.txt b/nbs/benchmark/covid_data/temp/sample1020.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1021.txt b/nbs/benchmark/covid_data/temp/sample1021.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1022.txt b/nbs/benchmark/covid_data/temp/sample1022.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1023.txt b/nbs/benchmark/covid_data/temp/sample1023.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1024.txt b/nbs/benchmark/covid_data/temp/sample1024.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1025.txt b/nbs/benchmark/covid_data/temp/sample1025.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1026.txt b/nbs/benchmark/covid_data/temp/sample1026.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1027.txt b/nbs/benchmark/covid_data/temp/sample1027.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1028.txt b/nbs/benchmark/covid_data/temp/sample1028.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1029.txt b/nbs/benchmark/covid_data/temp/sample1029.txt
diff --git a/nbs/benchmark/covid_data/temp/sample103.txt b/nbs/benchmark/covid_data/temp/sample103.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1030.txt b/nbs/benchmark/covid_data/temp/sample1030.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1031.txt b/nbs/benchmark/covid_data/temp/sample1031.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1032.txt b/nbs/benchmark/covid_data/temp/sample1032.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1033.txt b/nbs/benchmark/covid_data/temp/sample1033.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1034.txt b/nbs/benchmark/covid_data/temp/sample1034.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1035.txt b/nbs/benchmark/covid_data/temp/sample1035.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1036.txt b/nbs/benchmark/covid_data/temp/sample1036.txt
diff --git a/nbs/benchmark/covid_data/temp/sample1037.txt b/nbs/benchmark/covid_data/temp/sample1037.txt