add element type frequency eval function. add copy script to all conn…

…ectors.

test_unstructured_ingest/evaluation-metrics.sh

@@ -26,6 +26,14 @@ trap cleanup EXIT
 
 EXPORT_DIR="$SCRIPT_DIR"/metrics
 PYTHONPATH=. ./unstructured/ingest/evaluate.py \
+    measure-text-edit-distance \
     --output_dir "$OUTPUT_DIR" \
     --source_dir "$CCT_DIR" \
     --export_dir "$EXPORT_DIR"
+
+ELEMENT_TYPE_DIR="$SCRIPT_DIR"/expected-structured-output
+PYTHONPATH=. ./unstructured/ingest/evaluate.py \
+    measure-element-type-accuracy \
+    --output_dir "$OUTPUT_DIR" \
+    --source_dir "$ELEMENT_TYPE_DIR" \
+    --export_dir "$EXPORT_DIR"

test_unstructured_ingest/metrics/aggregate-scores-cct.tsv

@@ -1,3 +1,3 @@
 strategy	average	sample_sd	population_sd	count
-cct-accuracy	0.774	0.124	0.087	2
-cct-%missing	0.065	0.035	0.025	2
+cct-accuracy	0.83	0.204	0.144	2
+cct-%missing	0.035	0.007	0.005	2

test_unstructured_ingest/metrics/all-docs-cct.tsv

@@ -1,3 +1,3 @@
 filename	connector	cct-accuracy	cct-%missing
+handbook-1p.docx	box	0.974	0.03
 example-10k.html	local	0.686	0.04
-science-exploration-1p.pptx	box	0.861	0.09

test_unstructured_ingest/test-ingest-against-api.sh

@@ -37,3 +37,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-num-files-output.sh 1 $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-airtable-diff.sh

@@ -48,3 +48,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --verbose
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-azure.sh

@@ -33,3 +33,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-biomed-api.sh

@@ -39,3 +39,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-biomed-path.sh

@@ -37,3 +37,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-confluence-diff.sh

@@ -45,3 +45,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-delta-table.sh

@@ -49,3 +49,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
 
 python "$SCRIPT_DIR"/python/test-ingest-delta-table-output.py --table-uri "$DESTINATION_TABLE"
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-discord.sh

@@ -41,3 +41,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-elasticsearch.sh

@@ -48,3 +48,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-embed.sh

@@ -34,3 +34,5 @@ set +e
 # once we have an alternative encoder that is deterministic, we test the diff here
 # until then just validating the file was created
 "$SCRIPT_DIR"/check-num-files-output.sh 1 "$OUTPUT_FOLDER_NAME"
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-gcs.sh

@@ -47,3 +47,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-github.sh

@@ -52,3 +52,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     $ACCESS_TOKEN_FLAGS
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-gitlab.sh

@@ -38,3 +38,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-num-files-output.sh 2 $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-google-drive.sh

@@ -48,3 +48,5 @@ PYTHONPATH=. unstructured/ingest/main.py \
 
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-jira.sh

@@ -68,3 +68,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-local-single-file-with-encoding.sh

@@ -31,3 +31,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 set +e
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-local-single-file-with-pdf-infer-table-structure.sh

@@ -33,3 +33,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 set +e
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-local-single-file.sh

@@ -32,3 +32,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 set +e
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-notion.sh

@@ -42,3 +42,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-onedrive.sh

@@ -46,3 +46,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-outlook.sh

@@ -46,3 +46,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-pdf-fast-reprocess.sh

@@ -42,3 +42,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-s3-compression.sh

@@ -35,3 +35,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --uncompress
 
 "$SCRIPT_DIR"/check-num-files-output.sh 12 $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-s3-minio.sh

@@ -47,3 +47,5 @@ AWS_SECRET_ACCESS_KEY=$secret_key AWS_ACCESS_KEY_ID=$access_key PYTHONPATH=. ./u
 
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-s3.sh

@@ -37,3 +37,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
 
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-salesforce.sh

@@ -58,3 +58,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-sharepoint.sh

@@ -54,3 +54,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-slack.sh

@@ -44,3 +44,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
    --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-diff-expected-output.sh $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

test_unstructured_ingest/test-ingest-wikipedia.sh

@@ -35,3 +35,5 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --work-dir "$WORK_DIR"
 
 "$SCRIPT_DIR"/check-num-files-output.sh 3 $OUTPUT_FOLDER_NAME
+
+"$SCRIPT_DIR"/evaluation-ingest-cp.sh "$OUTPUT_DIR" "$OUTPUT_FOLDER_NAME"

unstructured/ingest/evaluate.py

@@ -9,6 +9,7 @@
 import click
 
 from unstructured.metrics.text_extraction import calculate_accuracy, calculate_percent_missing_text
+from unstructured.metrics.element_type import calculate_element_type_percent_match, get_element_type_frequency
 from unstructured.staging.base import elements_from_json, elements_to_text
 
 logger = logging.getLogger("unstructured.ingest")
@@ -23,8 +24,12 @@
 
 logger.setLevel(logging.DEBUG)
 
+@click.group()
+def main():
+    pass
 
-@click.command()
+
+@main.command()
 @click.option("--output_dir", type=click.STRING, help="Directory to a structured output.")
 @click.option(
     "--output_list",
@@ -56,7 +61,7 @@
     help="A tuple of weights to the Levenshtein distance calculation. \
         See text_extraction.py/calculate_edit_distance for more details.",
 )
-def measure_edit_distance(
+def measure_text_edit_distance(
     output_dir: str,
     output_list: Optional[List[str]],
     source_dir: str,
@@ -123,6 +128,73 @@ def measure_edit_distance(
     _display(agg_rows, headers)
 
 
+@main.command()
+@click.option("--output_dir", type=click.STRING, help="Directory to a structured output.")
+@click.option(
+    "--output_list",
+    type=click.STRING,
+    multiple=True,
+    help="Optional: list of selected structured output file names under the \
+        directory to be evaluate. If none, all files under directory will be use.",
+)
+@click.option("--source_dir", type=click.STRING, help="Directory to a structured source.")
+@click.option(
+    "--source_list",
+    type=click.STRING,
+    multiple=True,
+    help="Optional: list of selected structured source file names under the directory \
+        to be evaluate. If none, all files under directory will be use.",
+)
+@click.option(
+    "--export_dir",
+    type=click.STRING,
+    default="metrics",
+    help="Directory to save the output evaluation metrics to. Default to \
+        [your_working_dir]/metrics/",
+)
+def measure_element_type_accuracy(
+    output_dir: str,
+    output_list: Optional[List[str]],
+    source_dir: str,
+    source_list: Optional[List[str]],
+    export_dir: str
+):
+    if not output_list:
+        output_list = _listdir_recursive(output_dir)
+    if not source_list:
+        source_list = _listdir_recursive(source_dir)
+
+    rows = []
+    accuracy_scores: List[float] = []
+
+    for doc in output_list:  # type: ignore
+        fn = (doc.split("/")[-1]).split(".json")[0]
+        connector = doc.split("/")[0]
+        if doc in source_list:  # type: ignore
+            output = get_element_type_frequency(_read_json(os.path.join(output_dir, doc)))
+            source = get_element_type_frequency(_read_json(os.path.join(source_dir, doc)))
+            accuracy = round(calculate_element_type_percent_match(output, source), 3)
+            rows.append([fn, connector, accuracy])
+            accuracy_scores.append(accuracy)
+
+    headers = ["filename", "connector", "element-type-accuracy"]
+    _write_to_file(export_dir, "all-docs-element-type-frequency.tsv", rows, headers)
+
+    headers = ["strategy", "average", "sample_sd", "population_sd", "count"]
+    agg_rows = []
+    agg_rows.append(
+        [
+            "element-type-accuracy",
+            _mean(accuracy_scores),
+            _stdev(accuracy_scores),
+            _pstdev(accuracy_scores),
+            len(accuracy_scores),
+        ],
+    )
+    _write_to_file(export_dir, "aggregate-scores-element-type.tsv", agg_rows, headers)
+    _display(agg_rows, headers)
+
+
 def _listdir_recursive(dir: str):
     listdir = []
     for dirpath, _, filenames in os.walk(dir):
@@ -137,6 +209,8 @@ def _listdir_recursive(dir: str):
 
 
 def _write_to_file(dir: str, filename: str, rows: List[Any], headers: List[Any]):
+    click.echo(dir)
+    click.echo(filename)
     if dir and not os.path.exists(dir):
         os.makedirs(dir)
     with open(os.path.join(os.path.join(dir, filename)), "w", newline="") as tsv:
@@ -164,6 +238,8 @@ def _display(rows, headers):
 
 
 def _mean(scores: List[float], rounding: Optional[int] = 3):
+    if len(scores) < 1:
+        return None
     if not rounding:
         return statistics.mean(scores)
     return round(statistics.mean(scores), rounding)
@@ -185,5 +261,11 @@ def _pstdev(scores: List[float], rounding: Optional[int] = 3):
     return round(statistics.pstdev(scores), rounding)
 
 
+def _read_json(path):
+    with open(path) as f:
+        jsontext = f.read()
+    return jsontext
+
+
 if __name__ == "__main__":
-    measure_edit_distance()
+    main()