From 8841abc7f2e016391c48dcb2723bb8371cd9227f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 18 Mar 2022 15:25:58 +0100 Subject: [PATCH] update to shapely 1.8 --- ocrd_cis/ocropy/common.py | 64 ++++++++++++++++- ocrd_cis/ocropy/resegment.py | 18 ++--- ocrd_cis/ocropy/segment.py | 131 +++++++++++++++++++++++++---------- setup.py | 2 +- 4 files changed, 168 insertions(+), 47 deletions(-) diff --git a/ocrd_cis/ocropy/common.py b/ocrd_cis/ocropy/common.py index d84e42b3..67e82002 100644 --- a/ocrd_cis/ocropy/common.py +++ b/ocrd_cis/ocropy/common.py @@ -7,6 +7,7 @@ from scipy.ndimage import measurements, filters, interpolation, morphology from scipy import stats, signal #from skimage.morphology import convex_hull_image +from skimage.measure import find_contours, approximate_polygon from PIL import Image from . import ocrolib @@ -996,7 +997,9 @@ def h_compatible(obj1, obj2, center1, center2): # (which must be split anyway) # - with tighter polygonal spread around foreground # - with spread of line labels against separator labels +# - with centerline extraction # - return bg line and sep labels intead of just fg line labels +# - return centerline coords, too @checks(ABINARY2) def compute_segmentation(binary, zoom=1.0, @@ -1046,6 +1049,7 @@ def compute_segmentation(binary, foreground may remain unlabelled for separators and other non-text like small noise, or large drop-capitals / images), + - list of Numpy arrays of centerline coordinates [x, y points in lr order] - Numpy array of horizontal foreground lines mask, - Numpy array of vertical foreground lines mask, - Numpy array of large/non-text foreground component mask, @@ -1141,10 +1145,66 @@ def compute_segmentation(binary, LOG.debug('sorting labels by reading order') llabels = morph.reading_order(llabels,rl,bt)[llabels] DSAVE('llabels_ordered', llabels) - + #segmentation = llabels*binary #return segmentation - return llabels, hlines, vlines, images, colseps, scale + clines = compute_centerlines(bottom, top, llabels, scale) + return llabels, clines, hlines, vlines, images, colseps, scale + +@checks(AFLOAT2,AFLOAT2,SEGMENTATION,NUMBER) +def compute_centerlines(bottom, top, lines, scale): + """Get the coordinates of center lines running between each bottom and top gradient peak.""" + # smooth bottom+top maps horizontally for centerline estimation + bottom = filters.gaussian_filter(bottom, (scale*0.25,scale), mode='constant') + top = filters.gaussian_filter(top, (scale*0.25,scale), mode='constant') + # idea: center is where bottom and top gradient meet in the middle + # (but between top and bottom, not between bottom and top) + # - calculation via numpy == or isclose is too fragile numerically: + #clines = np.isclose(top, bottom, rtol=0.1) & (np.diff(top - bottom, axis=0, append=0) < 0) + #DSAVE('clines', [clines, bottom, top], enabled=True) + # - calculation via skimage.measure contours is reliable, but produces polygon segments + gradients = bottom - top + #seeds = np.diff(bottom - top, axis=0, append=0) > 0 + seeds = lines > 0 + contours = find_contours(gradients, 0, mask=seeds) + img = np.zeros_like(bottom, np.int) + #img = gradients + from skimage import draw + clines = [] + for j, contour in enumerate(contours): + # map y,x to x,y points + contour = contour[:,::-1] + #contour = approximate_polygon(contour, 1.0) + if len(contour) <= 3: + # too short already + clines.append(contour[np.argsort(contour[:,0])]) + continue + img[draw.polygon_perimeter(contour[:,1], contour[:,0], img.shape)] = j + # ensure the segment runs from left-most to right-most point once, + # find the middle between both paths (back and forth) + left = contour[:,0].argmin() + contour = np.concatenate((contour[left:], contour[:left])) + right = contour[:,0].argmax() + if right >= len(contour)-2 or right <= 1: + # no plateau - no back path + clines.append(contour[np.argsort(contour[:,0])]) + continue + contour1 = contour[0:right] + contour2 = contour[right:] + interp1 = np.interp(contour2[:,0], contour1[:,0], contour1[:,1]) + interp2 = np.interp(contour1[:,0], contour2[:,0], contour2[:,1]) + order = np.argsort(contour[:,0]) + interpolated = [] + for i in order: + if i >= right: + interpolated.append([contour[i,0], 0.5 * (contour2[i-right,1] + interp1[i-right])]) + else: + interpolated.append([contour[i,0], 0.5 * (contour1[i,1] + interp2[i])]) + interpolated = np.array(interpolated) + img[draw.polygon_perimeter(interpolated[:,1], interpolated[:,0], img.shape)] = j+0.5 + clines.append(interpolated) + DSAVE("centerline contours", img, enabled=True) + return clines # from ocropus-gpageseg, but # - on both foreground and background, diff --git a/ocrd_cis/ocropy/resegment.py b/ocrd_cis/ocropy/resegment.py index 4bcc203a..81166432 100644 --- a/ocrd_cis/ocropy/resegment.py +++ b/ocrd_cis/ocropy/resegment.py @@ -4,7 +4,7 @@ from itertools import chain import numpy as np from skimage import draw -from shapely.geometry import Polygon, asPolygon, LineString +from shapely.geometry import Polygon, LineString from shapely.prepared import prep from shapely.ops import unary_union import alphashape @@ -209,7 +209,7 @@ def _process_segment(self, parent, parent_image, parent_coords, page_id, zoom, l segment_polygon = coordinates_of_segment(segment, parent_image, parent_coords) segment_polygon = make_valid(Polygon(segment_polygon)).buffer(margin) line_polygons.append(prep(segment_polygon)) - segment_polygon = np.array(segment_polygon.exterior, np.int)[:-1] + segment_polygon = np.array(segment_polygon.exterior.coords, np.int)[:-1] # draw.polygon: If any segment_polygon lies outside of parent # (causing negative/above-max indices), either fully or partially, # then this will silently ignore them. The caller does not need @@ -224,7 +224,7 @@ def _process_segment(self, parent, parent_image, parent_coords, page_id, zoom, l segment.id, page_id if fullpage else parent.id) segment_polygon = coordinates_of_segment(segment, parent_image, parent_coords) segment_polygon = make_valid(Polygon(segment_polygon)).buffer(margin) - segment_polygon = np.array(segment_polygon.exterior, np.int)[:-1] + segment_polygon = np.array(segment_polygon.exterior.coords, np.int)[:-1] ignore_bin[draw.polygon(segment_polygon[:, 1], segment_polygon[:, 0], parent_bin.shape)] = False @@ -271,7 +271,7 @@ def _process_segment(self, parent, parent_image, parent_coords, page_id, zoom, l # left-hand side if left-to-right, and vice versa scale * (-1) ** line_ltr, single_sided=True)], loc=line.id, scale=scale)) - line_polygon = np.array(line_polygon.exterior, np.int)[:-1] + line_polygon = np.array(line_polygon.exterior.coords, np.int)[:-1] line_y, line_x = draw.polygon(line_polygon[:, 1], line_polygon[:, 0], parent_bin.shape) @@ -280,12 +280,12 @@ def _process_segment(self, parent, parent_image, parent_coords, page_id, zoom, l scale=scale, loc=parent.id, threshold=threshold) return try: - new_line_labels, _, _, _, _, scale = compute_segmentation( + new_line_labels, _, _, _, _, _, scale = compute_segmentation( parent_bin, seps=ignore_bin, zoom=zoom, fullpage=fullpage, maxseps=0, maxcolseps=len(ignore), maximages=0) except Exception as err: - LOG.warning('Cannot line-segment %s "%s": %s', - tag, page_id if fullpage else parent.id, err) + LOG.error('Cannot line-segment %s "%s": %s', + tag, page_id if fullpage else parent.id, err) return LOG.info("Found %d new line labels for %d existing lines on %s '%s'", new_line_labels.max(), len(lines), tag, parent.id) @@ -476,7 +476,7 @@ def diff_polygons(poly1, poly2): if poly.type == 'MultiPolygon': poly = poly.convex_hull if poly.minimum_clearance < 1.0: - poly = asPolygon(np.round(poly.exterior.coords)) + poly = Polygon(np.round(poly.exterior.coords)) poly = make_valid(poly) return poly @@ -517,7 +517,7 @@ def join_polygons(polygons, loc='', scale=20): if jointp.minimum_clearance < 1.0: # follow-up calculations will necessarily be integer; # so anticipate rounding here and then ensure validity - jointp = asPolygon(np.round(jointp.exterior.coords)) + jointp = Polygon(np.round(jointp.exterior.coords)) jointp = make_valid(jointp) return jointp diff --git a/ocrd_cis/ocropy/segment.py b/ocrd_cis/ocropy/segment.py index b782fdde..f373bd12 100644 --- a/ocrd_cis/ocropy/segment.py +++ b/ocrd_cis/ocropy/segment.py @@ -5,7 +5,7 @@ from skimage import draw from skimage.morphology import convex_hull_image import cv2 -from shapely.geometry import Polygon, asPolygon +from shapely.geometry import Polygon, LineString from shapely.prepared import prep from shapely.ops import unary_union @@ -19,6 +19,7 @@ AlternativeImageType ) from ocrd_models.ocrd_page_generateds import ( + BaselineType, TableRegionType, ImageRegionType, RegionRefType, @@ -55,20 +56,21 @@ TOOL = 'ocrd-cis-ocropy-segment' -def masks2polygons(bg_labels, fg_bin, name, min_area=None, simplify=None): +def masks2polygons(bg_labels, baselines, fg_bin, name, min_area=None, simplify=None): """Convert label masks into polygon coordinates. Given a Numpy array of background labels ``bg_labels``, + (optionally) a Numpy array of a scalar field ``baselines``, and a Numpy array of the foreground ``fg_bin``, iterate through all labels (except zero and those labels which do not correspond to any foreground at all) to find - their outer contours. Each contour part which is not too - small and gives a (simplified) polygon of at least 4 points - becomes a polygon. (Thus, labels can be split into multiple - polygons.) + their outer contours and inner baselines. + Each contour part which is not too small and gives a + (simplified) polygon of at least 4 points becomes a polygon. + (Thus, labels can be split into multiple polygons.) Return a tuple: - - these polygons as a list of label, polygon tuples, and + - these polygons as a list of label, polygon, baseline tuples, and - a Numpy array of new background labels for that list. """ LOG = getLogger('processor.OcropySegment') @@ -78,7 +80,7 @@ def masks2polygons(bg_labels, fg_bin, name, min_area=None, simplify=None): if not label: # ignore if background continue - bg_mask = np.array(bg_labels == label, np.uint8) + bg_mask = np.array(bg_labels == label, np.bool) if not np.count_nonzero(bg_mask * fg_bin): # ignore if missing foreground LOG.debug('skipping label %d in %s due to empty fg', @@ -86,16 +88,19 @@ def masks2polygons(bg_labels, fg_bin, name, min_area=None, simplify=None): continue # simplify to convex hull if simplify is not None: - hull = convex_hull_image(bg_mask).astype(np.uint8) - conflicts = np.setdiff1d((hull>0) * simplify, - (bg_mask>0) * simplify) + hull = convex_hull_image(bg_mask.astype(np.uint8)).astype(np.bool) + conflicts = np.setdiff1d(hull * simplify, + bg_mask * simplify) if conflicts.any(): LOG.debug('Cannot simplify %d: convex hull would create additional intersections %s', label, str(conflicts)) else: bg_mask = hull + # find sharp baseline + if baselines is not None: + baselines = [LineString(base) for base in baselines] # find outer contour (parts): - contours, _ = cv2.findContours(bg_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours(bg_mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # determine areas of parts: areas = [cv2.contourArea(contour) for contour in contours] total_area = sum(areas) @@ -125,15 +130,66 @@ def masks2polygons(bg_labels, fg_bin, name, min_area=None, simplify=None): polygon = polygon.simplify(tolerance) if polygon.is_valid: break - polygon = polygon.exterior.coords[:-1] # keep open - if len(polygon) < 4: + poly = polygon.exterior.coords[:-1] # keep open + if len(poly) < 4: LOG.warning('Label %d contour %d has less than 4 points for %s', label, i, name) continue - results.append((label, polygon)) + # get baseline segments intersecting with this line mask + # and concatenate them from left to right + if baselines is not None: + base = [] + debug = True #name == 'region "phys00005_region0023"' + for baseline in baselines: + baseline = baseline.intersection(polygon) + # post-process + if (baseline.is_empty or + baseline.type in ['Point', 'MultiPoint']): + continue + base_x = [pt[0] for pt in base] + base_left = min(base_x, default=0) + base_right = max(base_x, default=0) + left = baseline.bounds[0] + right = baseline.bounds[2] + if debug: LOG.debug("baseline part at %s", baseline.bounds) + if (baseline.type == 'GeometryCollection' or + baseline.type.startswith('Multi')): + # heterogeneous result: filter point + for geom in baseline.geoms: + if geom.type == 'Point': + continue + left = geom.bounds[0] + right = geom.bounds[2] + if left > base_right: + if debug: LOG.debug("adding baseline part component to the right") + base.extend(geom.coords) + base_right = right + elif right < base_left: + if debug: LOG.debug("adding baseline part component to the left") + base = list(geom.coords) + base + base_left = left + else: + LOG.warning("baseline part component crosses existing x") + continue + elif left > base_right: + if debug: LOG.debug("adding baseline part to the right") + base.extend(baseline.coords) + elif right < base_left: + if debug: LOG.debug("adding baseline part to the left") + base = list(baseline.coords) + base + else: + LOG.warning("baseline part crosses existing x") + continue + assert all(p1[0] < p2[0] for p1, p2 in zip(base[:-1],base[1:])) + # LOG.debug("exporting baseline for %s label %d contour %d: %s", + # name, label, i, str(base)) + else: + base = None + results.append((label, poly, base)) result_labels[contour_labels == i+1] = len(results) return results, result_labels + class OcropySegment(Processor): def __init__(self, *args, **kwargs): @@ -471,7 +527,7 @@ def _process_element(self, element, ignore, image, coords, element_id, file_id, try: if report: raise Exception(report) - line_labels, hlines, vlines, images, colseps, scale = compute_segmentation( + line_labels, baselines, hlines, vlines, images, colseps, scale = compute_segmentation( # suppress separators and ignored regions for textline estimation # but keep them for h/v-line detection (in fullpage mode): element_bin, seps=(sep_bin+ignore_labels)>0, @@ -567,17 +623,17 @@ def _process_element(self, element, ignore, image, coords, element_id, file_id, seps=np.maximum(sepmask, colseps)) region_mask |= region_line_labels > 0 # find contours for region (can be non-contiguous) - regions, _ = masks2polygons(region_mask * region_label, element_bin, + regions, _ = masks2polygons(region_mask * region_label, None, element_bin, '%s "%s"' % (element_name, element_id), min_area=6000/zoom/zoom, simplify=ignore_labels * ~(sep_bin)) # find contours for lines (can be non-contiguous) - lines, _ = masks2polygons(region_line_labels, element_bin, + lines, _ = masks2polygons(region_line_labels, baselines, element_bin, 'region "%s"' % element_id, min_area=640/zoom/zoom) # create new lines in new regions (allocating by intersection) - line_polys = [Polygon(polygon) for _, polygon in lines] - for _, region_polygon in regions: + line_polys = [Polygon(polygon) for _, polygon, _ in lines] + for _, region_polygon, _ in regions: region_poly = prep(Polygon(region_polygon)) # convert back to absolute (page) coordinates: region_polygon = coordinates_for_segment(region_polygon, image, coords) @@ -597,7 +653,7 @@ def _process_element(self, element, ignore, image, coords, element_id, file_id, for i, line_poly in enumerate(line_polys): if not region_poly.intersects(line_poly): # .contains continue - line_label, line_polygon = lines[i] + line_label, line_polygon, line_baseline = lines[i] # convert back to absolute (page) coordinates: line_polygon = coordinates_for_segment(line_polygon, image, coords) line_polygon = polygon_for_parent(line_polygon, region) @@ -609,9 +665,11 @@ def _process_element(self, element, ignore, image, coords, element_id, file_id, line_no += 1 line_id = region_id + "_line%04d" % line_no LOG.debug('Line label %d becomes ID "%s"', line_label, line_id) - line = TextLineType( - id=line_id, Coords=CoordsType( - points=points_from_polygon(line_polygon))) + line = TextLineType(id=line_id, + Coords=CoordsType(points=points_from_polygon(line_polygon))) + if line_baseline: + line_baseline = coordinates_for_segment(line_baseline, image, coords) + line.set_Baseline(BaselineType(points=points_from_polygon(line_baseline))) region.add_TextLine(line) # if the region has received text lines, keep it if region.get_TextLine(): @@ -626,9 +684,9 @@ def _process_element(self, element, ignore, image, coords, element_id, file_id, LOG.info('Found %d large non-text/image regions for %s "%s"', num_images, element_name, element_id) # find contours around region labels (can be non-contiguous): - image_polygons, _ = masks2polygons(image_labels, element_bin, + image_polygons, _ = masks2polygons(image_labels, None, element_bin, '%s "%s"' % (element_name, element_id)) - for image_label, polygon in image_polygons: + for image_label, polygon, _ in image_polygons: # convert back to absolute (page) coordinates: region_polygon = coordinates_for_segment(polygon, image, coords) region_polygon = polygon_for_parent(region_polygon, element) @@ -647,11 +705,11 @@ def _process_element(self, element, ignore, image, coords, element_id, file_id, LOG.info('Found %d/%d h/v-lines for %s "%s"', num_hlines, num_vlines, element_name, element_id) # find contours around region labels (can be non-contiguous): - hline_polygons, _ = masks2polygons(hline_labels, element_bin, + hline_polygons, _ = masks2polygons(hline_labels, None, element_bin, '%s "%s"' % (element_name, element_id)) - vline_polygons, _ = masks2polygons(vline_labels, element_bin, + vline_polygons, _ = masks2polygons(vline_labels, None, element_bin, '%s "%s"' % (element_name, element_id)) - for _, polygon in hline_polygons + vline_polygons: + for _, polygon, _ in hline_polygons + vline_polygons: # convert back to absolute (page) coordinates: region_polygon = coordinates_for_segment(polygon, image, coords) region_polygon = polygon_for_parent(region_polygon, element) @@ -683,11 +741,11 @@ def _process_element(self, element, ignore, image, coords, element_id, file_id, # ensure the new line labels do not extrude from the region: line_labels = line_labels * region_mask # find contours around labels (can be non-contiguous): - line_polygons, _ = masks2polygons(line_labels, element_bin, + line_polygons, _ = masks2polygons(line_labels, baselines, element_bin, 'region "%s"' % element_id, min_area=640/zoom/zoom) line_no = 0 - for line_label, polygon in line_polygons: + for line_label, polygon, baseline in line_polygons: # convert back to absolute (page) coordinates: line_polygon = coordinates_for_segment(polygon, image, coords) line_polygon = polygon_for_parent(line_polygon, element) @@ -698,9 +756,12 @@ def _process_element(self, element, ignore, image, coords, element_id, file_id, # annotate result: line_no += 1 line_id = element_id + "_line%04d" % line_no - element.add_TextLine(TextLineType( - id=line_id, Coords=CoordsType( - points=points_from_polygon(line_polygon)))) + line = TextLineType(id=line_id, + Coords=CoordsType(points=points_from_polygon(line_polygon))) + if baseline: + line_baseline = coordinates_for_segment(baseline, image, coords) + line.set_Baseline(BaselineType(points=points_from_polygon(line_baseline))) + element.add_TextLine(line) if not sep_bin.any(): return # no derived image # annotate a text/image-separated image @@ -761,7 +822,7 @@ def make_intersection(poly1, poly2): if interp.minimum_clearance < 1.0: # follow-up calculations will necessarily be integer; # so anticipate rounding here and then ensure validity - interp = asPolygon(np.round(interp.exterior.coords)) + interp = Polygon(np.round(interp.exterior.coords)) interp = make_valid(interp) return interp diff --git a/setup.py b/setup.py index 72e11280..a0c371ed 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ 'scipy', 'numpy>=1.17.0', 'pillow>=7.1.2', - 'shapely>=1.7.1,<1.8', + 'shapely>=1.7.1', 'scikit-image', 'alphashape', 'opencv-python-headless',