🚧 Colour based tagging & non-ocr page_to_text

pyproject.toml

@@ -14,6 +14,9 @@ playwright = "^1.44.0"
 selenium = "^4.21.0"
 google-cloud-vision = "^3.7.2"
 azure-ai-vision-imageanalysis = "^1.0.0b2"
+pillow = "^10.4.0"
+numpy = "^2.0.1"
+tiktoken = "^0.7.0"
 
 
 [tool.poetry.group.dev.dependencies]

scripts/setup.sh

@@ -5,4 +5,8 @@ cd ..
 npm install
 npm run build
 
-poetry install
+poetry install
+
+cd ./tarsier-snapshots || exit 1
+poetry install
+poetry run bananalyze --download

tarsier-snapshots/snapshots/1JWoJWs3uZMt8Wa5ql6pr/non_ocr_2.txt

@@ -0,0 +1,48 @@
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+[0] We value your privacy
+     [1] We use cookies to enhance your browsing experience, serve personalized ads or content, and analyze our traffic. By clicking Accept
+     All", you consent to our use of cookies.
+                                                                                                                                                                                                            [$2] Customize                [$3] Reject All             [$4] Accept All
+                                                                                      [@7] Notice of Non-Discrimination             [@8] Careers [@9] Patient Portal/Pay my Bill              [@10] For Providers      [@11] Contact Us
+                                                                           [@12] About Us             [@26] For Patients               [@35] Our Services               [@65] Our Locations                 [@66] Find a Provider
+
+
+                **[69] Our Locations**
+                [$71] Search by location name                                                          [$73] Filter by category                                                               [$75] Search by ZIP code
+[$74] All categories Children's Hospital New Orleans East Jefferson General Hospital Lakeside Hospital Lakeview Hospital Lakeview Regional Physician Group New Orleans East Hospital Touro Tulane Medical Center University Medical Center New Orleans Urgent Care West Jefferson Medical Center
+
+
+
+
+
+
+
+                                                                                                                                                                                                                                                      [@77] Use my location
+
+                                                                                                                                                                                                                                                          [@78] Clear filters
+
+     [79] Amelia Health Center Cardiology and
+     Cardiovascular Surgery
+     [80] Touro
+     [81] 3715 Prytania St.
+     [82] Suite 400
+     [83] New Orleans, LA 70115
+     [@84] 504. 897. 8276
+     [@85] More information
+     [@86] Get directions
+
+     [87] Behavioral Health Center
+     [88] Children's Hospital New Orleans
+     [89] 210 State St.
+     [90] New Orleans, LA 10118
+     [@91] 504. 896. 7200
+     [@92] More information
+     [@93] Get directions
+
+     [94] Bienville Health Center Primary Care and
+     OB/GYN
+     [95] Touro
+
+     [98] New Orleans, LA 70119
+     [@99] 504. 252. 9488
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

tarsier-snapshots/snapshots/1qkOHewUy0Kqq9RVVSOoQ/non_ocr_2.txt

@@ -0,0 +1,265 @@
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+[9] Notice
+                                              [10] Road and parking lot construction in Madison, Wis. may result in travel delays and route changes to UW
+                                              Health clinic and hospital locations. Please plan accordingly. Read [@11] Read more more
+
+
+                                               [@20] Find a Doctor [@21] Conditions Services [@22] Locations Clinics                                                        [@23] Patients Families [@24] MyChart
+
+
+
+
+          [@29] UW Health Specialty Clinic - Sauk Prairie
+
+
+         **[31] Eye Care Clinic**
+               **Optometry**
+
+
+
+
+
+
+
+
+
+
+                  [@32] 250 26th Street, Suite 120 Prairie du Sac, WI 53578                     [@33] 608 643-6060             [$34] Closed now
+
+
+                 [@50] Parking and                   [@51] Hours of
+                 transportation                      operation                    [@52] Providers
+
+
+
+
+                **[53] Parking and**
+                **transportation**
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+                                                                                                                                                        [$58] Keyboard shortcuts [59] Map data 2023 [$60] 50 m [@61] Terms [@62] Report a map error
+Click to toggle between metric and imperial units
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+                                                                                                                                                            [64] UW Health Specialty Clinic - Sauk
+                                                                                                                                                            Prairie
+
+                                                                                                                                                            [66] 250 26th Street, Suite 120 Prairie
+                                                                                                                                                            du Sac, WI
+                                                                                                                                                             [68] 608 643-6060
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+                 [70] Hours of operation
+
+
+           [$71] Eye Care Clinic Optometry                  closed
+
+
+           [74] Monday-Friday                                     [75] 8am-5pm
+              [76] Upcoming special hours
+              [77] Nov 22                                       [78] Closed
+                                                                                [$79] UW Health Specialty Clinic - Sauk
+                                                                                Prairie
+                                                                                                                                  closed
+
+
+                                                                                   [82] Upcoming special hours
+                                                                                   [83] Nov 22                                       [84] Closed
+
+
+
+
+
+
+
+
+
+                **[85] Providers**
+
+
+
+
+
+
+
+
+
+
+
+
+
+           [@91] Telehealth [@92] Priority OrthoCare                      [@93] e-Visits [@94] Emergency Urgent care                                 [@95] UW School of Medicine and Public Health
+
+
+           **[@96] MyChart**
+
+           **[@97] Find a Doctor**
+
+           **[@98] Conditions Services**
+
+           **[@99] Locations Clinics**
+
+           **[@100] Patients Families**
+
+           **[@101] Refer a Patient**
+                                                                                                                 [@102] Pay a bill                                  [@103] Careers
+
+                                                                                                                 [@104] Refill a prescription                       [@105] News
+
+                                                                                                                 [@106] Obtain medical records                      [@107] Clinical Trials
+
+                                                                                                                 [@108] Order flowers and gifts                     [@109] Volunteering
+
+                                                                                                                 [@110] Send a greeting card                        [@111] About
+
+                                                                                                                 [@112] Make a donation                             [@113] Find a class or support group
+
+
+
+
+
+           [@114] About UW Health     [@115] Diversity, Equity and Inclusion   [@116] Media Center   [@117] Contact Us                                                           [@118] Make a donation
+
+
+           [@119] Notice of Privacy Practices HIPAA                                                                                  [@121] Language Access Notice of Nondiscrimination
+           [120] Donations to UW Health are managed by the University of
+           Wisconsin Foundation, a publicly supported charitable
+           organization under 501 c 3 of the Internal Revenue Code.
+                                                                                                                                     [@122] English [@123] Espa ol Spanish [@124] Hmoob Hmong
+                                                                                                                                                 [@125] Chinese [@126] Deutsch German           [@127] Arabic
+                                                                                                                                                  [@128] Russian       [@129] Korean [@130] Ti ng Vi t Vietnamese
+                                                                                                                                     [@131] Deitsch Pennsylvania Dutch                   [@132] Lao
+                                                                                                                                     [@133] Fran ais French   [@134] Polski Polish       [@135] Hindi
+                                                                                                                                     [@136] Shqip Albanian   [@137] Tagalog Tagalog Filipino
+
+           [138] Copyright 2023 University of Wisconsin Hospitals and Clinics Authority                         [@142] Terms and conditions [@143] Website privacy policy                    [@144] Employee home access
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------