From 6ffb3317bc638fce5d2fb2e9d33b4f383efdc0b0 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Wed, 22 Feb 2023 08:17:51 -0500 Subject: [PATCH 01/23] add validators for user creds --- ambuda/views/auth.py | 96 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 17 deletions(-) diff --git a/ambuda/views/auth.py b/ambuda/views/auth.py index 47d5c5dd..8b3c0707 100644 --- a/ambuda/views/auth.py +++ b/ambuda/views/auth.py @@ -5,9 +5,13 @@ https://www.uxmatters.com/mt/archives/2018/09/signon-signoff-and-registration.php Security reference: - - https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html - https://cheatsheetseries.owasp.org/cheatsheets/Forgot_Password_Cheat_Sheet.html + +Max lengths: +- https://cheatsheetseries.owasp.org/cheatsheets/Authentication_Cheat_Sheet.html suggests 64 characters for password +- https://www.rfc-editor.org/errata_search.php?rfc=3696 specifies 254 characters for email address + """ import secrets @@ -27,7 +31,15 @@ bp = Blueprint("auth", __name__) +# maximum lengths of authentication fields +MIN_EMAIL_ADDRESS_LEN = 4 +MAX_EMAIL_ADDRESS_LEN = 254 +MIN_PASSWORD_LEN = 8 +MAX_PASSWORD_LEN = 256 +MIN_USERNAME_LEN = 6 +MAX_USERNAME_LEN = 64 +# token lifetime MAX_TOKEN_LIFESPAN_IN_HOURS = 24 # FIXME: redirect to site.index once user accounts are more useful. POST_AUTH_ROUTE = "proofing.index" @@ -79,51 +91,99 @@ def _is_valid_reset_token(row: db.PasswordResetToken, raw_token: str, now=None): return True +# Copied from https://wtforms.readthedocs.io/en/2.3.x/validators/ +class FieldLength(object): + def __init__(self, min=-1, max=-1, message=None): + self.min = min + self.max = max + if not message: + message = "Field must be between %i and %i characters long." % (min, max) + self.message = message + + def __call__(self, form, field): + field_len = field.data and len(field.data) or 0 + if field_len < self.min or self.max != -1 and field_len > self.max: + raise val.ValidationError(self.message) + + +def get_field_validators(field_name: str, min_len: int, max_len: int): + return [ + val.DataRequired(), + FieldLength( + min=min_len, + max=max_len, + message=f"{field_name.capitalize()} must be between {min_len} and {max_len} characters long", + ), + ] + + +def get_username_validators(is_legacy: bool = False): + validators = [ + *get_field_validators("username", MIN_USERNAME_LEN, MAX_USERNAME_LEN), + ] + if not is_legacy: + validators.append( + val.Regexp("^[^\s]*$", message="Username must not contain spaces") + ) + return validators + +def get_password_validators(): + return [ + *get_field_validators("password", MIN_PASSWORD_LEN, MAX_PASSWORD_LEN), + ] + + +def get_email_validators(): + return [ + *get_field_validators("email", MIN_EMAIL_ADDRESS_LEN, MAX_EMAIL_ADDRESS_LEN), + val.Email(), + ] + + class SignupForm(FlaskForm): - username = StringField( - _l("Username"), [val.Length(min=6, max=25), val.DataRequired()] - ) - password = PasswordField(_l("Password"), [val.Length(min=8), val.DataRequired()]) - email = StringField(_l("Email address"), [val.DataRequired(), val.Email()]) + username = StringField(_l("Username"), [*get_username_validators()]) + password = PasswordField(_l("Password"), [*get_password_validators()]) + email = EmailField(_l("Email address"), [*get_email_validators()]) recaptcha = RecaptchaField() def validate_username(self, username): + # TODO: make username case insensitive user = q.user(username.data) if user: raise val.ValidationError("Please use a different username.") def validate_email(self, email): session = q.get_session() + # TODO: make email case insensitive user = session.query(db.User).filter_by(email=email.data).first() if user: raise val.ValidationError("Please use a different email address.") class SignInForm(FlaskForm): - username = StringField( - _l("Username"), [val.Length(min=6, max=25), val.DataRequired()] - ) - password = PasswordField(_l("Password"), [val.Length(min=8), val.DataRequired()]) + username = StringField(_l("Username"), [*get_username_validators(True)]) + password = PasswordField(_l("Password"), [*get_password_validators()]) class ResetPasswordForm(FlaskForm): - email = EmailField("Email", [val.DataRequired()]) + email = EmailField(_l("Email address"), [*get_email_validators()]) recaptcha = RecaptchaField() class ChangePasswordForm(FlaskForm): #: Old password. No validation requirements, in case we change our password #: criteria in the future. - old_password = PasswordField(_l("Old password"), [val.DataRequired()]) + old_password = PasswordField(_l("Old Password"), [*get_password_validators()]) + #: New password. - new_password = PasswordField( - _l("New password"), [val.Length(min=8), val.DataRequired()] - ) + new_password = PasswordField(_l("New password"), [*get_password_validators()]) class ResetPasswordFromTokenForm(FlaskForm): - password = PasswordField(_l("Password"), [val.DataRequired()]) - confirm_password = PasswordField(_l("Confirm password"), [val.DataRequired()]) + password = PasswordField(_l("Password"), [*get_password_validators()]) + confirm_password = PasswordField( + _l("Confirm password"), [*get_password_validators()] + ) @bp.route("/register", methods=["GET", "POST"]) @@ -133,6 +193,7 @@ def register(): return redirect(url_for("site.index")) form = SignupForm() + # save username and email in lowercase if form.validate_on_submit(): user = q.create_user( username=form.username.data, @@ -157,6 +218,7 @@ def sign_in(): return redirect(url_for("site.index")) form = SignInForm() + # TODO: make username case insensitive if form.validate_on_submit(): user = q.user(form.username.data) if user and user.check_password(form.password.data): From d15a8920dcf2481dc3e80e7b495bd5b5d33e40b9 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Thu, 23 Feb 2023 08:49:45 -0500 Subject: [PATCH 02/23] use get_x_validators() straight w/o unpacking --- ambuda/views/auth.py | 66 ++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/ambuda/views/auth.py b/ambuda/views/auth.py index 8b3c0707..d7d84228 100644 --- a/ambuda/views/auth.py +++ b/ambuda/views/auth.py @@ -91,59 +91,67 @@ def _is_valid_reset_token(row: db.PasswordResetToken, raw_token: str, now=None): return True +# The native val.Length() validator silently snips username and +# password to the maximum length. +# So, database stores the snipped username and password resulting +# in information loss. For instance, user may have copy pasted 240 +# chars our form will only store MAX_##_LEN bytes of it to the db. Creating +# our own validator will show a clear error that username cannot +# exceed MAX_##_LEN bytes. I'm not sure if it is a bug or a feature +# in the val.Length() # Copied from https://wtforms.readthedocs.io/en/2.3.x/validators/ class FieldLength(object): def __init__(self, min=-1, max=-1, message=None): self.min = min self.max = max if not message: - message = "Field must be between %i and %i characters long." % (min, max) + message = f"Field must be between {min} and {max} characters long." self.message = message def __call__(self, form, field): - field_len = field.data and len(field.data) or 0 + field_len = field.data and len(field.data or []) if field_len < self.min or self.max != -1 and field_len > self.max: raise val.ValidationError(self.message) def get_field_validators(field_name: str, min_len: int, max_len: int): + field_name_capitalized = field_name.capitalize() return [ val.DataRequired(), FieldLength( min=min_len, max=max_len, - message=f"{field_name.capitalize()} must be between {min_len} and {max_len} characters long", + message=f"{field_name_capitalized} must be between {min_len} and {max_len} characters long", ), ] -def get_username_validators(is_legacy: bool = False): - validators = [ - *get_field_validators("username", MIN_USERNAME_LEN, MAX_USERNAME_LEN), - ] - if not is_legacy: - validators.append( - val.Regexp("^[^\s]*$", message="Username must not contain spaces") - ) +def get_username_validators(): + validators = get_field_validators("username", MIN_USERNAME_LEN, MAX_USERNAME_LEN) + validators.append( + val.Regexp("^[^\s]*$", message="Username must not contain spaces") + ) return validators + +def get_legacy_username_validators(): + return get_field_validators("username", MIN_USERNAME_LEN, MAX_USERNAME_LEN) + + def get_password_validators(): - return [ - *get_field_validators("password", MIN_PASSWORD_LEN, MAX_PASSWORD_LEN), - ] + return get_field_validators("password", MIN_PASSWORD_LEN, MAX_PASSWORD_LEN) def get_email_validators(): - return [ - *get_field_validators("email", MIN_EMAIL_ADDRESS_LEN, MAX_EMAIL_ADDRESS_LEN), - val.Email(), - ] + validators = get_field_validators("email", MIN_EMAIL_ADDRESS_LEN, MAX_EMAIL_ADDRESS_LEN) + validators.append(val.Email()) + return validators class SignupForm(FlaskForm): - username = StringField(_l("Username"), [*get_username_validators()]) - password = PasswordField(_l("Password"), [*get_password_validators()]) - email = EmailField(_l("Email address"), [*get_email_validators()]) + username = StringField(_l("Username"), get_username_validators()) + password = PasswordField(_l("Password"), get_password_validators()) + email = EmailField(_l("Email address"), get_email_validators()) recaptcha = RecaptchaField() def validate_username(self, username): @@ -161,29 +169,27 @@ def validate_email(self, email): class SignInForm(FlaskForm): - username = StringField(_l("Username"), [*get_username_validators(True)]) - password = PasswordField(_l("Password"), [*get_password_validators()]) + username = StringField(_l("Username"), get_legacy_username_validators()) + password = PasswordField(_l("Password"), get_password_validators()) class ResetPasswordForm(FlaskForm): - email = EmailField(_l("Email address"), [*get_email_validators()]) + email = EmailField(_l("Email address"), get_email_validators()) recaptcha = RecaptchaField() class ChangePasswordForm(FlaskForm): #: Old password. No validation requirements, in case we change our password #: criteria in the future. - old_password = PasswordField(_l("Old Password"), [*get_password_validators()]) + old_password = PasswordField(_l("Old Password"), get_password_validators()) #: New password. - new_password = PasswordField(_l("New password"), [*get_password_validators()]) + new_password = PasswordField(_l("New password"), get_password_validators()) class ResetPasswordFromTokenForm(FlaskForm): - password = PasswordField(_l("Password"), [*get_password_validators()]) - confirm_password = PasswordField( - _l("Confirm password"), [*get_password_validators()] - ) + password = PasswordField(_l("Password"), get_password_validators()) + confirm_password = PasswordField(_l("Confirm password"), get_password_validators()) @bp.route("/register", methods=["GET", "POST"]) From 53fd4ce34d367ffbe4e29e5d17d1449bba839f75 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Fri, 24 Feb 2023 12:10:49 -0500 Subject: [PATCH 03/23] a better pythonic init and comparison of integers --- ambuda/views/auth.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ambuda/views/auth.py b/ambuda/views/auth.py index d7d84228..4713e0be 100644 --- a/ambuda/views/auth.py +++ b/ambuda/views/auth.py @@ -15,6 +15,7 @@ """ import secrets +import sys from datetime import datetime, timedelta from typing import Optional @@ -101,16 +102,16 @@ def _is_valid_reset_token(row: db.PasswordResetToken, raw_token: str, now=None): # in the val.Length() # Copied from https://wtforms.readthedocs.io/en/2.3.x/validators/ class FieldLength(object): - def __init__(self, min=-1, max=-1, message=None): - self.min = min - self.max = max + def __init__(self, min=None, max=None, message=None): + self.min = min or 0 + self.max = max or sys.maxsize if not message: message = f"Field must be between {min} and {max} characters long." self.message = message def __call__(self, form, field): field_len = field.data and len(field.data or []) - if field_len < self.min or self.max != -1 and field_len > self.max: + if not (self.min <= field_len <= self.max): raise val.ValidationError(self.message) From 3111d540492d4c46014f36013bacafa351480937 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Fri, 24 Feb 2023 13:14:35 -0500 Subject: [PATCH 04/23] ran py-lint --- ambuda/views/auth.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ambuda/views/auth.py b/ambuda/views/auth.py index 4713e0be..7de9a360 100644 --- a/ambuda/views/auth.py +++ b/ambuda/views/auth.py @@ -144,7 +144,9 @@ def get_password_validators(): def get_email_validators(): - validators = get_field_validators("email", MIN_EMAIL_ADDRESS_LEN, MAX_EMAIL_ADDRESS_LEN) + validators = get_field_validators( + "email", MIN_EMAIL_ADDRESS_LEN, MAX_EMAIL_ADDRESS_LEN + ) validators.append(val.Email()) return validators From 0fb78871a4507c07c3493d9396bb2c97b2a03150 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Wed, 1 Mar 2023 00:28:42 -0500 Subject: [PATCH 05/23] change `update` to `replace` in project.py --- ambuda/templates/proofing/projects/replace.html | 3 ++- ambuda/views/proofing/project.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index 3bbb8f98..f56a3786 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -35,7 +35,8 @@
{% for match in page.matches %} 
{{ match.query }}
- 
{{ match.update }}
+ 
{{ match.replace }}
+ {%- endfor %}
diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 9a375790..335d1a0a 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -303,9 +303,9 @@ def replace(slug): "proofing/projects/replace.html", project=project_, form=form ) - # search for "query" string and replace with "update" string + # search for "query" string and replace with "replace" string query = form.query.data - update = form.replace.data + replace = form.replace.data results = [] for page_ in project_.pages: @@ -322,8 +322,8 @@ def replace(slug): "query": escape(line).replace( query, Markup(f"{escape(query)}") ), - "update": escape(line).replace( - query, Markup(f"{escape(update)}") + "replace": escape(line).replace( + query, Markup(f"{escape(replace)}") ), } ) @@ -339,7 +339,7 @@ def replace(slug): project=project_, form=form, query=query, - update=update, + replace=replace, results=results, ) From e361b03cced00fa5b8d3b51a132928dc971abc3e Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Wed, 15 Mar 2023 18:55:59 -0400 Subject: [PATCH 06/23] submit edits and confirm them (wip) --- .../proofing/projects/confirm_replace.html | 47 ++++ .../templates/proofing/projects/replace.html | 99 +++++-- ambuda/views/proofing/project.py | 243 ++++++++++++++++-- 3 files changed, 333 insertions(+), 56 deletions(-) create mode 100644 ambuda/templates/proofing/projects/confirm_replace.html diff --git a/ambuda/templates/proofing/projects/confirm_replace.html b/ambuda/templates/proofing/projects/confirm_replace.html new file mode 100644 index 00000000..16e64921 --- /dev/null +++ b/ambuda/templates/proofing/projects/confirm_replace.html @@ -0,0 +1,47 @@ +{% extends 'proofing/base.html' %} +{% from "macros/forms.html" import field %} + +{% block title %}Confirm Replace | {{ project.title }}{% endblock %} + +{% block content %} +
+

Confirm Replace

+

Please confirm the changes you want to make:

+
+ +{% for result in results %} + {% set page = result.page %} + {% set matches = result.matches %} + {% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} + +
+

{{ project.title }}/{{ page.slug }}

+
    + {% for match in matches %} +
  • +

    Original Text:

    +

    {{ match.query }}

    +

    New Text:

    +

    {{ match.replace }}

    +
    • + {% endfor %} +
+
+{% endfor %} + +
+ {{ form.csrf_token }} + + + {% for result in results %} + {% set page = result.page %} + {% set matches = result.matches %} + {% for match in matches %} + + + {% endfor %} + {% endfor %} + + +
+{% endblock %} diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index f56a3786..06ad2194 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -10,39 +10,82 @@ {{ m.project_nav(project=project, active='edit') }}
+

Replace

Use this simple search and replace form to make edits across this project.

+
+ + {{ field(form.query) }} + {{ field(form.replace) }} + +
-
- {{ field(form.query) }} - {{ field(form.replace) }} - -
- -{% if query %} +{% if results %}
+

Matches

+ {% macro sp(s, p, n) %}{% if n == 1 %}{{ s }}{% else %}{{ p }}{% endif %}{% endmacro %} +
+ {{ submit_changes_form.csrf_token }} + + -{% macro sp(s, p, n) %}{% if n == 1 %}{{ s }}{% else %}{{ p }}{% endif %}{% endmacro %} - -{% set nr = results|length %} -

Found {{ nr }} {{ sp("page", "pages", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }}.

- -
    -{% for page in results %} -{% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} -
  • - {{ project.title }}/{{ page.slug }} -
    - {% for match in page.matches %} - 
    {{ match.query }}
    - 
    {{ match.replace }}
    - - {%- endfor %} + {% set nr = results|length %} +

    Found {{ nr }} {{ sp("page", "pages", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }}.

    +
      +
      + +
      - -{% endfor %} -
    - + {% for page in results %} + {% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} +
  • + {{ project.title }}/{{ page.slug }} +
    + {% for match in page.matches %} +
    + +

    Page {{ project.title }}/{{ page.slug }}: Line {{ loop.index }}

    + +
    + + +
    +
    + {% endfor %} +
    +
    • + {% endfor %} + {% if submit_changes_form %} + {{ submit_changes_form.submit(class="btn btn-submit") }} + {% else %} + + {% endif %} + +
+ {% endif %} + + +
-{% endif %} + {% endblock %} \ No newline at end of file diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 335d1a0a..39e056ad 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -1,3 +1,5 @@ +import logging + from celery.result import GroupResult from flask import ( Blueprint, @@ -9,13 +11,13 @@ url_for, ) from flask_babel import lazy_gettext as _l -from flask_login import login_required +from flask_login import current_user, login_required from flask_wtf import FlaskForm from markupsafe import Markup, escape from sqlalchemy import orm from werkzeug.exceptions import abort from werkzeug.utils import redirect -from wtforms import StringField +from wtforms import HiddenField, StringField, SubmitField from wtforms.validators import DataRequired, ValidationError from wtforms.widgets import TextArea @@ -24,10 +26,11 @@ from ambuda.tasks import app as celery_app from ambuda.tasks import ocr as ocr_tasks from ambuda.utils import project_utils, proofing_utils +from ambuda.utils.revisions import EditException, add_revision from ambuda.views.proofing.decorators import moderator_required, p2_required bp = Blueprint("project", __name__) - +LOG = logging.getLogger(__name__) def _is_valid_page_number_spec(_, field): try: @@ -102,6 +105,20 @@ class Meta: replace = StringField(_l("Replace"), validators=[DataRequired()]) +class SubmitChangesForm(ReplaceForm): + class Meta: + csrf = False + + submit = SubmitField("Submit Changes") + +class ConfirmReplaceForm(ReplaceForm): + class Meta: + csrf = False + + changes = HiddenField("Changes", validators=[DataRequired()]) + confirm = SubmitField("Confirm") + cancel = SubmitField("Cancel") + @bp.route("//") def summary(slug): @@ -285,38 +302,23 @@ def search(slug): results=results, ) - -@bp.route("//replace") -@login_required -def replace(slug): - """Search and replace a string across all of the project's pages. - - This is useful to replace a string across the project in one shot. +def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str): + """ + Gather all matches for the "query" string and pair them the "replace" string. """ - project_ = q.project(slug) - if project_ is None: - abort(404) - - form = ReplaceForm(request.args) - if not form.validate(): - return render_template( - "proofing/projects/replace.html", project=project_, form=form - ) - - # search for "query" string and replace with "replace" string - query = form.query.data - replace = form.replace.data results = [] + + LOG.info(f'Search/Replace text with {query} and {replace}') for page_ in project_.pages: if not page_.revisions: continue - matches = [] - latest = page_.revisions[-1] for line in latest.content.splitlines(): + LOG.info(f'Search/Replace > {page_.slug}: {line}') if query in line: + LOG.info(f'Search/Replace > appending search/replace {line}') matches.append( { "query": escape(line).replace( @@ -324,7 +326,8 @@ def replace(slug): ), "replace": escape(line).replace( query, Markup(f"{escape(replace)}") - ), + ), + "checked": False } ) if matches: @@ -337,11 +340,195 @@ def replace(slug): return render_template( "proofing/projects/replace.html", project=project_, - form=form, + form=replace_form, + submit_changes_form=SubmitChangesForm(), query=query, replace=replace, results=results, - ) + ) + + +@bp.route("//replace", methods=["GET", "POST"]) +@login_required +def replace(slug): + """Search and replace a string across all of the project's pages. + + This is useful to replace a string across the project in one shot. + """ + project_ = q.project(slug) + if project_ is None: + abort(404) + + form = ReplaceForm(request.form) + if not form.validate(): + invalid_keys = list(form.errors.keys()) + LOG.info(f'Invalid form - {request.method}, invalid keys: {invalid_keys}') + return render_template( + "proofing/projects/replace.html", project=project_, form=ReplaceForm() + ) + + # search for "query" string and replace with "update" string + query = form.query.data + replace = form.replace.data + render = _replace_text(project_, replace_form=form, query=query, replace=replace) + return render + + +def _select_changes(project_, submit_changes_form: SubmitChangesForm, query: str, replace: str): + """ + Mark "query" strings + """ + results = [] + LOG.info(f'{__name__}: Mark changes with {query} and {replace}') + for page_ in project_.pages: + if not page_.revisions: + continue + + latest = page_.revisions[-1] + matches = [] + for line_num, line in enumerate(latest.content.splitlines()): + form_key = f"match{page_.slug}-{line_num}" + if getattr(submit_changes_form, form_key, None) and getattr(submit_changes_form, form_key).data: + matches.append({ + 'query': line, + 'replace': line.replace(query, replace), + }) + + results.append({ + 'page': page_, + 'matches': matches + }) + + selected_count = sum(getattr(submit_changes_form, form_key).data == True for form_key in submit_changes_form._fields) + LOG.info(f'{__name__} > Number of selected changes = {selected_count}') + + return render_template("proofing/projects/confirm_replace.html", + project=project_, form=ConfirmReplaceForm(), query=query, replace=replace, results=results) + + +# def _mark_changes(project_, form: ReplaceForm, submit_changes_form: SubmitChangesForm, +# query: str, replace: str): +# """ +# Search for all "query" string +# """ +# results = [] +# LOG.info(f'Mark changes with {query} and {replace}') +# for page_ in project_.pages: +# if not page_.revisions: +# continue + +# latest = page_.revisions[-1] +# if submit_changes_form.get("check-all"): +# # check all matches +# LOG.info(f'Mark changes > {page_.slug}') +# for line_num, line in enumerate(latest.content.splitlines()): + +# form_key = f"match{page_.slug}-{line_num}" +# submit_changes_form[form_key].data = True +# LOG.info(f'Mark changes > Check-all > {form_key}') +# else: +# # handle individual match +# for i, line in enumerate(latest.content.splitlines()): +# form_key = f"match{page_.slug}-{line_num}" +# if submit_changes_form.get(form_key): +# submit_changes_form[form_key].data = True +# else: +# submit_changes_form[form_key].data = False +# selected_count = sum(submit_changes_form[form_key].data == True for form_key in submit_changes_form) +# LOG.info(f'{__name__} > Number of selected changes = {selected_count}') + + +# return render_template("proofing/projects/confirm_replace.html", +# project=project_, form=form, submit_changes_form=submit_changes_form, query=query, replace=replace, results=results) + + +@bp.route("//submit_changes", methods=["GET", "POST"]) +@login_required +def submit_changes(slug): + """Submit selected changes across all of the project's pages. + + This is useful to replace a string across the project in one shot. + """ + + project_ = q.project(slug) + if project_ is None: + abort(404) + + form = SubmitChangesForm(request.form) + if not form.validate(): + # elif request.form.get("form_submitted") is None: + invalid_keys = list(form.errors.keys()) + LOG.info(f'{__name__}: Invalid form values - {request.method}, invalid keys: {invalid_keys}') + return redirect(url_for("proofing.project.replace", slug=slug)) + + results = [] + render = None + # search for "query" string and replace with "update" string + query = form.query.data + replace = form.replace.data + + LOG.info(f'{__name__}: ({request.method})> Got to submit method with {query}->{replace} ') + LOG.info(f'{__name__}: {request.method} > {list(request.form.keys())}') + + render = _select_changes(project_, submit_changes_form=form, query=query, replace=replace) + + return render + + +@bp.route("//confirm_replace", methods=["GET", "POST"]) +@login_required +def confirm_replace(slug): + """Confirm changes to replace a string across all of the project's pages.""" + project_ = q.project(slug) + if project_ is None: + abort(404) + + form = ConfirmReplaceForm(request.form) + if not form.validate(): + flash("Invalid input.", "danger") + invalid_keys = list(form.errors.keys()) + LOG.info(f'{__name__}: Invalid form - {request.method}, invalid keys: {invalid_keys}') + return redirect(url_for("proofing.project.replace", slug=slug)) + + if form.confirm.data: + query = form.query.data + replace = form.replace.data + changes = [] + + # Get the changes from the form and store them in a list + for key, value in request.form.items(): + if key.startswith("match"): + parts = key.split("-") + page_slug = parts[1] + line_num = int(parts[2]) + if parts[3] == "replace": + changes.append((page_slug, line_num, value)) + + # Apply the changes to each page + for page in project_.pages: + lines = page.content.splitlines() + page_changed = False + for line_num, line in enumerate(lines): + for change in changes: + change_page_slug, change_line_num, change_replace_value = change + if page.slug == change_page_slug and line_num == change_line_num: + lines[line_num] = line.replace(query, change_replace_value) + page_changed = True + break + if page_changed: + break + + # Add a new revision if the page was changed + if page_changed: + new_content = "\n".join(lines) + new_summary = f'Replaced "{query}" with "{replace}" on page {page.slug}' + new_revision = add_revision(page, new_summary, new_content) + page.revisions.append(new_revision) + + flash("Changes applied.", "success") + return redirect(url_for("proofing.projects.replace", slug=slug)) + + return render_template("proofing/projects/confirm_replace.html", project=project_, form=form) @bp.route("//batch-ocr", methods=["GET", "POST"]) From 8bea0b2c03c5fb741a43baae4bb3e733016f20b0 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Thu, 16 Mar 2023 19:48:08 -0400 Subject: [PATCH 07/23] working all the way to add_revision() - wip --- .../proofing/projects/confirm_replace.html | 62 +++--- .../templates/proofing/projects/replace.html | 39 ++-- ambuda/views/proofing/project.py | 178 ++++++++++-------- 3 files changed, 151 insertions(+), 128 deletions(-) diff --git a/ambuda/templates/proofing/projects/confirm_replace.html b/ambuda/templates/proofing/projects/confirm_replace.html index 16e64921..2792fab6 100644 --- a/ambuda/templates/proofing/projects/confirm_replace.html +++ b/ambuda/templates/proofing/projects/confirm_replace.html @@ -1,47 +1,43 @@ {% extends 'proofing/base.html' %} {% from "macros/forms.html" import field %} +{% import "macros/proofing.html" as m %} {% block title %}Confirm Replace | {{ project.title }}{% endblock %} {% block content %} + +{{ m.project_header_nested('Confirm Replace', project) }} +{{ m.project_nav(project=project, active='edit') }} +

Confirm Replace

+ {% macro sp(s, p, n) %}{% if n == 1 %}{{ s }}{% else %}{{ p }}{% endif %}{% endmacro %}

Please confirm the changes you want to make:

-
- -{% for result in results %} - {% set page = result.page %} - {% set matches = result.matches %} - {% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} +
+ {{ form.csrf_token }} + + -
-

{{ project.title }}/{{ page.slug }}

-
    + {% set nr = results|length %} +

    Confirm edits on {{ nr }} {{ sp("page", "pages", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }} to be replaced by {{ replace }}.

    + + {% for result in results %} + {% set page = result.page %} + {% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} + {% set matches = result.matches %} {% for match in matches %} -
  • -

    Original Text:

    -

    {{ match.query }}

    -

    New Text:

    -

    {{ match.replace }}

    -
    • +
    +

    Page {{ project.title }}/{{ page.slug }}: Line {{ match.line_num }}

    + + +
    + + +
    {% endfor %} -
-
-{% endfor %} - - - {{ form.csrf_token }} - - - {% for result in results %} - {% set page = result.page %} - {% set matches = result.matches %} - {% for match in matches %} - - {% endfor %} - {% endfor %} - - -
+ + + + {% endblock %} diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index 06ad2194..8573c356 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -41,17 +41,17 @@

Matches

  • {{ project.title }}/{{ page.slug }}
    - {% for match in page.matches %} -
    - -

    Page {{ project.title }}/{{ page.slug }}: Line {{ loop.index }}

    - -
    - - -
    -
    - {% endfor %} + {% for match in page.matches %} +
    + +

    Page {{ project.title }}/{{ page.slug }}: Line {{ match.line_num }}

    + +
    + + +
    +
    + {% endfor %}
    • {% endfor %} @@ -62,12 +62,13 @@

    Matches

    {% endif %} - {% endif %} + {% endif %} + diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 39e056ad..8b23cccd 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -1,4 +1,5 @@ import logging +import re from celery.result import GroupResult from flask import ( @@ -17,7 +18,15 @@ from sqlalchemy import orm from werkzeug.exceptions import abort from werkzeug.utils import redirect -from wtforms import HiddenField, StringField, SubmitField +from wtforms import ( + BooleanField, + FieldList, + Form, + FormField, + HiddenField, + StringField, + SubmitField, +) from wtforms.validators import DataRequired, ValidationError from wtforms.widgets import TextArea @@ -87,6 +96,9 @@ class EditMetadataForm(FlaskForm): }, ) +class MatchForm(Form): + selected = BooleanField() + replace = HiddenField(validators=[DataRequired()]) class SearchForm(FlaskForm): class Meta: @@ -105,17 +117,24 @@ class Meta: replace = StringField(_l("Replace"), validators=[DataRequired()]) + +def validate_matches(form, field): + for match_form in field: + if match_form.errors: + raise ValidationError("Invalid match form values.") + + class SubmitChangesForm(ReplaceForm): class Meta: csrf = False - + + matches = FieldList(FormField(MatchForm), validators=[validate_matches]) submit = SubmitField("Submit Changes") class ConfirmReplaceForm(ReplaceForm): class Meta: csrf = False - changes = HiddenField("Changes", validators=[DataRequired()]) confirm = SubmitField("Confirm") cancel = SubmitField("Cancel") @@ -315,8 +334,9 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) continue matches = [] latest = page_.revisions[-1] - for line in latest.content.splitlines(): - LOG.info(f'Search/Replace > {page_.slug}: {line}') + LOG.info(f'{__name__}: {page_.slug}') + for line_num, line in enumerate(latest.content.splitlines()): + # LOG.info(f'Search/Replace > {page_.slug}: {line}') if query in line: LOG.info(f'Search/Replace > appending search/replace {line}') matches.append( @@ -327,7 +347,8 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) "replace": escape(line).replace( query, Markup(f"{escape(replace)}") ), - "checked": False + "checked": False, + "line_num": line_num } ) if matches: @@ -374,9 +395,9 @@ def replace(slug): return render -def _select_changes(project_, submit_changes_form: SubmitChangesForm, query: str, replace: str): +def _select_changes(project_, selected_keys, query: str, replace: str): """ - Mark "query" strings + Mark "query" strings """ results = [] LOG.info(f'{__name__}: Mark changes with {query} and {replace}') @@ -387,61 +408,33 @@ def _select_changes(project_, submit_changes_form: SubmitChangesForm, query: str latest = page_.revisions[-1] matches = [] for line_num, line in enumerate(latest.content.splitlines()): + form_key = f"match{page_.slug}-{line_num}" - if getattr(submit_changes_form, form_key, None) and getattr(submit_changes_form, form_key).data: + replace_form_key = f"match{page_.slug}-{line_num}-replace" + + if selected_keys.get(form_key) == "selected": + LOG.info(f'{__name__}: {form_key}: {selected_keys.get(form_key)}') + LOG.info(f'{__name__}: {replace_form_key}: {request.form.get(replace_form_key)}') + LOG.info(f'{__name__}: {form_key}: Appended') matches.append({ 'query': line, 'replace': line.replace(query, replace), + 'line_num': line_num, }) results.append({ 'page': page_, 'matches': matches }) + LOG.info(f'{__name__}: Total matches appended: {len(matches)}') - selected_count = sum(getattr(submit_changes_form, form_key).data == True for form_key in submit_changes_form._fields) + selected_count = sum(value == "selected" for value in selected_keys.values()) LOG.info(f'{__name__} > Number of selected changes = {selected_count}') return render_template("proofing/projects/confirm_replace.html", project=project_, form=ConfirmReplaceForm(), query=query, replace=replace, results=results) -# def _mark_changes(project_, form: ReplaceForm, submit_changes_form: SubmitChangesForm, -# query: str, replace: str): -# """ -# Search for all "query" string -# """ -# results = [] -# LOG.info(f'Mark changes with {query} and {replace}') -# for page_ in project_.pages: -# if not page_.revisions: -# continue - -# latest = page_.revisions[-1] -# if submit_changes_form.get("check-all"): -# # check all matches -# LOG.info(f'Mark changes > {page_.slug}') -# for line_num, line in enumerate(latest.content.splitlines()): - -# form_key = f"match{page_.slug}-{line_num}" -# submit_changes_form[form_key].data = True -# LOG.info(f'Mark changes > Check-all > {form_key}') -# else: -# # handle individual match -# for i, line in enumerate(latest.content.splitlines()): -# form_key = f"match{page_.slug}-{line_num}" -# if submit_changes_form.get(form_key): -# submit_changes_form[form_key].data = True -# else: -# submit_changes_form[form_key].data = False -# selected_count = sum(submit_changes_form[form_key].data == True for form_key in submit_changes_form) -# LOG.info(f'{__name__} > Number of selected changes = {selected_count}') - - -# return render_template("proofing/projects/confirm_replace.html", -# project=project_, form=form, submit_changes_form=submit_changes_form, query=query, replace=replace, results=results) - - @bp.route("//submit_changes", methods=["GET", "POST"]) @login_required def submit_changes(slug): @@ -454,12 +447,15 @@ def submit_changes(slug): if project_ is None: abort(404) + LOG.info(f'{__name__}: SUBMIT_CHANGES --- {request.method} > {list(request.form.keys())}') + + # FIXME: find a way to validate this form. Current `matches` are coming in the way of validators. form = SubmitChangesForm(request.form) - if not form.validate(): - # elif request.form.get("form_submitted") is None: - invalid_keys = list(form.errors.keys()) - LOG.info(f'{__name__}: Invalid form values - {request.method}, invalid keys: {invalid_keys}') - return redirect(url_for("proofing.project.replace", slug=slug)) + # if not form.validate(): + # # elif request.form.get("form_submitted") is None: + # invalid_keys = list(form.errors.keys()) + # LOG.info(f'{__name__}: Invalid form values - {request.method}, invalid keys: {invalid_keys}') + # return redirect(url_for("proofing.project.replace", slug=slug)) results = [] render = None @@ -469,8 +465,8 @@ def submit_changes(slug): LOG.info(f'{__name__}: ({request.method})> Got to submit method with {query}->{replace} ') LOG.info(f'{__name__}: {request.method} > {list(request.form.keys())}') - - render = _select_changes(project_, submit_changes_form=form, query=query, replace=replace) + selected_keys = {key: value for key, value in request.form.items() if key.startswith('match') and not key.endswith('replace')} + render = _select_changes(project_, selected_keys, query=query, replace=replace) return render @@ -482,7 +478,7 @@ def confirm_replace(slug): project_ = q.project(slug) if project_ is None: abort(404) - + LOG.info(f'{__name__}: CONFIRM_REPLACE {request.method} > Keys: {list(request.form.keys())}, Items: {list(request.form.items())}') form = ConfirmReplaceForm(request.form) if not form.validate(): flash("Invalid input.", "danger") @@ -490,45 +486,63 @@ def confirm_replace(slug): LOG.info(f'{__name__}: Invalid form - {request.method}, invalid keys: {invalid_keys}') return redirect(url_for("proofing.project.replace", slug=slug)) + if form.confirm.data: + LOG.info(f'{__name__}: {request.method} > Confirmed!') query = form.query.data replace = form.replace.data changes = [] # Get the changes from the form and store them in a list + pages = {} + + # Iterate over the dictionary `request.form` for key, value in request.form.items(): - if key.startswith("match"): - parts = key.split("-") - page_slug = parts[1] - line_num = int(parts[2]) - if parts[3] == "replace": - changes.append((page_slug, line_num, value)) - - # Apply the changes to each page - for page in project_.pages: - lines = page.content.splitlines() - page_changed = False - for line_num, line in enumerate(lines): - for change in changes: - change_page_slug, change_line_num, change_replace_value = change - if page.slug == change_page_slug and line_num == change_line_num: - lines[line_num] = line.replace(query, change_replace_value) - page_changed = True - break - if page_changed: - break - - # Add a new revision if the page was changed - if page_changed: - new_content = "\n".join(lines) + # Check if key matches the pattern + match = re.match(r"match(\d+)-(\d+)-replace", key) + if match: + # Extract page_slug and line_num from the key + page_slug = match.group(1) + line_num = int(match.group(2)) + if page_slug not in pages: + pages[page_slug] = {} + pages[page_slug][line_num] = value + + for page_slug, changed_lines in pages.items(): + # Get the corresponding `Page` object + LOG.info(f'{__name__}: Project - {project_.slug}, Page : {page_slug}') + + # Page query needs id for project and slug for page + page = q.page(project_.id, page_slug) + if not page: + LOG.error(f'{__name__}: Page not found for project - {project_.slug}, page : {page_slug}') + return render_template(url_for("proofing.projects.replace", slug=slug)) + + latest = page.revisions[-1] + current_lines = latest.content.splitlines() + # Iterate over the `lines` dictionary + for line_num, replace_value in changed_lines.items(): + # Check if the line_num exists in the dictionary for this page + if line_num in current_lines: + # Replace the line with the replacement value + LOG.info(f'{__name__}: Current - {current_lines[line_num]}, Revision : {replace_value}') + current_lines[line_num] = replace_value + # Join the lines into a single string + new_content = "\n".join(current_lines) + # Check if the page content has changed + if new_content != latest.content: + # Add a new revision to the page new_summary = f'Replaced "{query}" with "{replace}" on page {page.slug}' - new_revision = add_revision(page, new_summary, new_content) - page.revisions.append(new_revision) + new_revision = add_revision(page=page, summary=new_summary, content=new_content, status=page.status.name, version=page.version, author_id=current_user.id) + LOG.info(f'{__name__}: New reviion > {page_slug}: {new_revision}') flash("Changes applied.", "success") - return redirect(url_for("proofing.projects.replace", slug=slug)) + return redirect(url_for("proofing.projects.activity", slug=slug)) + elif form.cancel.data: + LOG.info(f'{__name__}: CONFIRM_REPLACE Cancelled') + return redirect(url_for("proofing.projects.edit", slug=slug)) - return render_template("proofing/projects/confirm_replace.html", project=project_, form=form) + return render_template(url_for("proofing.project.edit", slug=slug)) @bp.route("//batch-ocr", methods=["GET", "POST"]) From 4c266cf18e55c971b0ecd28e00125f6a786e9337 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Thu, 16 Mar 2023 20:09:35 -0400 Subject: [PATCH 08/23] search and replace is working now --- ambuda/views/proofing/project.py | 55 +++++++++++++++++--------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 8b23cccd..48c2fb65 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -328,17 +328,17 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) results = [] - LOG.info(f'Search/Replace text with {query} and {replace}') + LOG.debug(f'Search/Replace text with {query} and {replace}') for page_ in project_.pages: if not page_.revisions: continue matches = [] latest = page_.revisions[-1] - LOG.info(f'{__name__}: {page_.slug}') + LOG.debug(f'{__name__}: {page_.slug}') for line_num, line in enumerate(latest.content.splitlines()): - # LOG.info(f'Search/Replace > {page_.slug}: {line}') + # LOG.debug(f'Search/Replace > {page_.slug}: {line}') if query in line: - LOG.info(f'Search/Replace > appending search/replace {line}') + LOG.debug(f'Search/Replace > appending search/replace {line}') matches.append( { "query": escape(line).replace( @@ -383,7 +383,7 @@ def replace(slug): form = ReplaceForm(request.form) if not form.validate(): invalid_keys = list(form.errors.keys()) - LOG.info(f'Invalid form - {request.method}, invalid keys: {invalid_keys}') + LOG.debug(f'Invalid form - {request.method}, invalid keys: {invalid_keys}') return render_template( "proofing/projects/replace.html", project=project_, form=ReplaceForm() ) @@ -400,7 +400,7 @@ def _select_changes(project_, selected_keys, query: str, replace: str): Mark "query" strings """ results = [] - LOG.info(f'{__name__}: Mark changes with {query} and {replace}') + LOG.debug(f'{__name__}: Mark changes with {query} and {replace}') for page_ in project_.pages: if not page_.revisions: continue @@ -413,9 +413,9 @@ def _select_changes(project_, selected_keys, query: str, replace: str): replace_form_key = f"match{page_.slug}-{line_num}-replace" if selected_keys.get(form_key) == "selected": - LOG.info(f'{__name__}: {form_key}: {selected_keys.get(form_key)}') - LOG.info(f'{__name__}: {replace_form_key}: {request.form.get(replace_form_key)}') - LOG.info(f'{__name__}: {form_key}: Appended') + LOG.debug(f'{__name__}: {form_key}: {selected_keys.get(form_key)}') + LOG.debug(f'{__name__}: {replace_form_key}: {request.form.get(replace_form_key)}') + LOG.debug(f'{__name__}: {form_key}: Appended') matches.append({ 'query': line, 'replace': line.replace(query, replace), @@ -426,10 +426,10 @@ def _select_changes(project_, selected_keys, query: str, replace: str): 'page': page_, 'matches': matches }) - LOG.info(f'{__name__}: Total matches appended: {len(matches)}') + LOG.debug(f'{__name__}: Total matches appended: {len(matches)}') selected_count = sum(value == "selected" for value in selected_keys.values()) - LOG.info(f'{__name__} > Number of selected changes = {selected_count}') + LOG.debug(f'{__name__} > Number of selected changes = {selected_count}') return render_template("proofing/projects/confirm_replace.html", project=project_, form=ConfirmReplaceForm(), query=query, replace=replace, results=results) @@ -447,14 +447,14 @@ def submit_changes(slug): if project_ is None: abort(404) - LOG.info(f'{__name__}: SUBMIT_CHANGES --- {request.method} > {list(request.form.keys())}') + LOG.debug(f'{__name__}: SUBMIT_CHANGES --- {request.method} > {list(request.form.keys())}') # FIXME: find a way to validate this form. Current `matches` are coming in the way of validators. form = SubmitChangesForm(request.form) # if not form.validate(): # # elif request.form.get("form_submitted") is None: # invalid_keys = list(form.errors.keys()) - # LOG.info(f'{__name__}: Invalid form values - {request.method}, invalid keys: {invalid_keys}') + # LOG.debug(f'{__name__}: Invalid form values - {request.method}, invalid keys: {invalid_keys}') # return redirect(url_for("proofing.project.replace", slug=slug)) results = [] @@ -463,8 +463,8 @@ def submit_changes(slug): query = form.query.data replace = form.replace.data - LOG.info(f'{__name__}: ({request.method})> Got to submit method with {query}->{replace} ') - LOG.info(f'{__name__}: {request.method} > {list(request.form.keys())}') + LOG.debug(f'{__name__}: ({request.method})> Got to submit method with {query}->{replace} ') + LOG.debug(f'{__name__}: {request.method} > {list(request.form.keys())}') selected_keys = {key: value for key, value in request.form.items() if key.startswith('match') and not key.endswith('replace')} render = _select_changes(project_, selected_keys, query=query, replace=replace) @@ -478,17 +478,17 @@ def confirm_replace(slug): project_ = q.project(slug) if project_ is None: abort(404) - LOG.info(f'{__name__}: CONFIRM_REPLACE {request.method} > Keys: {list(request.form.keys())}, Items: {list(request.form.items())}') + LOG.debug(f'{__name__}: CONFIRM_REPLACE {request.method} > Keys: {list(request.form.keys())}, Items: {list(request.form.items())}') form = ConfirmReplaceForm(request.form) if not form.validate(): flash("Invalid input.", "danger") invalid_keys = list(form.errors.keys()) - LOG.info(f'{__name__}: Invalid form - {request.method}, invalid keys: {invalid_keys}') + LOG.error(f'{__name__}: Invalid form - {request.method}, invalid keys: {invalid_keys}') return redirect(url_for("proofing.project.replace", slug=slug)) if form.confirm.data: - LOG.info(f'{__name__}: {request.method} > Confirmed!') + LOG.debug(f'{__name__}: {request.method} > Confirmed!') query = form.query.data replace = form.replace.data changes = [] @@ -510,23 +510,26 @@ def confirm_replace(slug): for page_slug, changed_lines in pages.items(): # Get the corresponding `Page` object - LOG.info(f'{__name__}: Project - {project_.slug}, Page : {page_slug}') + LOG.debug(f'{__name__}: Project - {project_.slug}, Page : {page_slug}') # Page query needs id for project and slug for page page = q.page(project_.id, page_slug) if not page: LOG.error(f'{__name__}: Page not found for project - {project_.slug}, page : {page_slug}') - return render_template(url_for("proofing.projects.replace", slug=slug)) + return render_template(url_for("proofing.project.replace", slug=slug)) latest = page.revisions[-1] current_lines = latest.content.splitlines() # Iterate over the `lines` dictionary for line_num, replace_value in changed_lines.items(): # Check if the line_num exists in the dictionary for this page - if line_num in current_lines: + LOG.debug(f'{__name__}: Current - {current_lines[line_num]}, Length of lines = {len(current_lines)}') + if line_num < len(current_lines): # Replace the line with the replacement value - LOG.info(f'{__name__}: Current - {current_lines[line_num]}, Revision : {replace_value}') current_lines[line_num] = replace_value + else: + LOG.error(f'{__name__}: Invalid line number {line_num} in {page_slug} which has only {len(current_lines)}') + continue # Join the lines into a single string new_content = "\n".join(current_lines) # Check if the page content has changed @@ -534,13 +537,13 @@ def confirm_replace(slug): # Add a new revision to the page new_summary = f'Replaced "{query}" with "{replace}" on page {page.slug}' new_revision = add_revision(page=page, summary=new_summary, content=new_content, status=page.status.name, version=page.version, author_id=current_user.id) - LOG.info(f'{__name__}: New reviion > {page_slug}: {new_revision}') + LOG.debug(f'{__name__}: New reviion > {page_slug}: {new_revision}') flash("Changes applied.", "success") - return redirect(url_for("proofing.projects.activity", slug=slug)) + return redirect(url_for("proofing.project.activity", slug=slug)) elif form.cancel.data: - LOG.info(f'{__name__}: CONFIRM_REPLACE Cancelled') - return redirect(url_for("proofing.projects.edit", slug=slug)) + LOG.debug(f'{__name__}: CONFIRM_REPLACE Cancelled') + return redirect(url_for("proofing.project.edit", slug=slug)) return render_template(url_for("proofing.project.edit", slug=slug)) From 7406c3a4a8eacc68b8fe6a4d864848804c80545f Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Thu, 16 Mar 2023 20:23:47 -0400 Subject: [PATCH 09/23] fixed lint and unit tests --- ambuda/views/proofing/project.py | 130 +++++++++++++-------- test/ambuda/views/proofing/test_project.py | 10 ++ 2 files changed, 92 insertions(+), 48 deletions(-) diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 48c2fb65..678783b8 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -35,12 +35,13 @@ from ambuda.tasks import app as celery_app from ambuda.tasks import ocr as ocr_tasks from ambuda.utils import project_utils, proofing_utils -from ambuda.utils.revisions import EditException, add_revision +from ambuda.utils.revisions import add_revision from ambuda.views.proofing.decorators import moderator_required, p2_required bp = Blueprint("project", __name__) LOG = logging.getLogger(__name__) + def _is_valid_page_number_spec(_, field): try: _ = project_utils.parse_page_number_spec(field.data) @@ -96,10 +97,12 @@ class EditMetadataForm(FlaskForm): }, ) + class MatchForm(Form): selected = BooleanField() replace = HiddenField(validators=[DataRequired()]) + class SearchForm(FlaskForm): class Meta: csrf = False @@ -127,10 +130,11 @@ def validate_matches(form, field): class SubmitChangesForm(ReplaceForm): class Meta: csrf = False - + matches = FieldList(FormField(MatchForm), validators=[validate_matches]) submit = SubmitField("Submit Changes") + class ConfirmReplaceForm(ReplaceForm): class Meta: csrf = False @@ -321,6 +325,7 @@ def search(slug): results=results, ) + def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str): """ Gather all matches for the "query" string and pair them the "replace" string. @@ -328,17 +333,17 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) results = [] - LOG.debug(f'Search/Replace text with {query} and {replace}') + LOG.debug(f"Search/Replace text with {query} and {replace}") for page_ in project_.pages: if not page_.revisions: continue matches = [] latest = page_.revisions[-1] - LOG.debug(f'{__name__}: {page_.slug}') + LOG.debug(f"{__name__}: {page_.slug}") for line_num, line in enumerate(latest.content.splitlines()): # LOG.debug(f'Search/Replace > {page_.slug}: {line}') if query in line: - LOG.debug(f'Search/Replace > appending search/replace {line}') + LOG.debug(f"Search/Replace > appending search/replace {line}") matches.append( { "query": escape(line).replace( @@ -346,9 +351,9 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) ), "replace": escape(line).replace( query, Markup(f"{escape(replace)}") - ), + ), "checked": False, - "line_num": line_num + "line_num": line_num, } ) if matches: @@ -366,7 +371,7 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) query=query, replace=replace, results=results, - ) + ) @bp.route("//replace", methods=["GET", "POST"]) @@ -383,7 +388,7 @@ def replace(slug): form = ReplaceForm(request.form) if not form.validate(): invalid_keys = list(form.errors.keys()) - LOG.debug(f'Invalid form - {request.method}, invalid keys: {invalid_keys}') + LOG.debug(f"Invalid form - {request.method}, invalid keys: {invalid_keys}") return render_template( "proofing/projects/replace.html", project=project_, form=ReplaceForm() ) @@ -400,7 +405,7 @@ def _select_changes(project_, selected_keys, query: str, replace: str): Mark "query" strings """ results = [] - LOG.debug(f'{__name__}: Mark changes with {query} and {replace}') + LOG.debug(f"{__name__}: Mark changes with {query} and {replace}") for page_ in project_.pages: if not page_.revisions: continue @@ -411,28 +416,35 @@ def _select_changes(project_, selected_keys, query: str, replace: str): form_key = f"match{page_.slug}-{line_num}" replace_form_key = f"match{page_.slug}-{line_num}-replace" - + if selected_keys.get(form_key) == "selected": - LOG.debug(f'{__name__}: {form_key}: {selected_keys.get(form_key)}') - LOG.debug(f'{__name__}: {replace_form_key}: {request.form.get(replace_form_key)}') - LOG.debug(f'{__name__}: {form_key}: Appended') - matches.append({ - 'query': line, - 'replace': line.replace(query, replace), - 'line_num': line_num, - }) - - results.append({ - 'page': page_, - 'matches': matches - }) - LOG.debug(f'{__name__}: Total matches appended: {len(matches)}') + LOG.debug(f"{__name__}: {form_key}: {selected_keys.get(form_key)}") + LOG.debug( + f"{__name__}: {replace_form_key}: {request.form.get(replace_form_key)}" + ) + LOG.debug(f"{__name__}: {form_key}: Appended") + matches.append( + { + "query": line, + "replace": line.replace(query, replace), + "line_num": line_num, + } + ) + + results.append({"page": page_, "matches": matches}) + LOG.debug(f"{__name__}: Total matches appended: {len(matches)}") selected_count = sum(value == "selected" for value in selected_keys.values()) - LOG.debug(f'{__name__} > Number of selected changes = {selected_count}') + LOG.debug(f"{__name__} > Number of selected changes = {selected_count}") - return render_template("proofing/projects/confirm_replace.html", - project=project_, form=ConfirmReplaceForm(), query=query, replace=replace, results=results) + return render_template( + "proofing/projects/confirm_replace.html", + project=project_, + form=ConfirmReplaceForm(), + query=query, + replace=replace, + results=results, + ) @bp.route("//submit_changes", methods=["GET", "POST"]) @@ -447,7 +459,9 @@ def submit_changes(slug): if project_ is None: abort(404) - LOG.debug(f'{__name__}: SUBMIT_CHANGES --- {request.method} > {list(request.form.keys())}') + LOG.debug( + f"{__name__}: SUBMIT_CHANGES --- {request.method} > {list(request.form.keys())}" + ) # FIXME: find a way to validate this form. Current `matches` are coming in the way of validators. form = SubmitChangesForm(request.form) @@ -457,17 +471,22 @@ def submit_changes(slug): # LOG.debug(f'{__name__}: Invalid form values - {request.method}, invalid keys: {invalid_keys}') # return redirect(url_for("proofing.project.replace", slug=slug)) - results = [] render = None # search for "query" string and replace with "update" string query = form.query.data replace = form.replace.data - - LOG.debug(f'{__name__}: ({request.method})> Got to submit method with {query}->{replace} ') - LOG.debug(f'{__name__}: {request.method} > {list(request.form.keys())}') - selected_keys = {key: value for key, value in request.form.items() if key.startswith('match') and not key.endswith('replace')} + + LOG.debug( + f"{__name__}: ({request.method})> Got to submit method with {query}->{replace} " + ) + LOG.debug(f"{__name__}: {request.method} > {list(request.form.keys())}") + selected_keys = { + key: value + for key, value in request.form.items() + if key.startswith("match") and not key.endswith("replace") + } render = _select_changes(project_, selected_keys, query=query, replace=replace) - + return render @@ -478,20 +497,22 @@ def confirm_replace(slug): project_ = q.project(slug) if project_ is None: abort(404) - LOG.debug(f'{__name__}: CONFIRM_REPLACE {request.method} > Keys: {list(request.form.keys())}, Items: {list(request.form.items())}') + LOG.debug( + f"{__name__}: CONFIRM_REPLACE {request.method} > Keys: {list(request.form.keys())}, Items: {list(request.form.items())}" + ) form = ConfirmReplaceForm(request.form) if not form.validate(): flash("Invalid input.", "danger") invalid_keys = list(form.errors.keys()) - LOG.error(f'{__name__}: Invalid form - {request.method}, invalid keys: {invalid_keys}') + LOG.error( + f"{__name__}: Invalid form - {request.method}, invalid keys: {invalid_keys}" + ) return redirect(url_for("proofing.project.replace", slug=slug)) - if form.confirm.data: - LOG.debug(f'{__name__}: {request.method} > Confirmed!') + LOG.debug(f"{__name__}: {request.method} > Confirmed!") query = form.query.data replace = form.replace.data - changes = [] # Get the changes from the form and store them in a list pages = {} @@ -510,12 +531,14 @@ def confirm_replace(slug): for page_slug, changed_lines in pages.items(): # Get the corresponding `Page` object - LOG.debug(f'{__name__}: Project - {project_.slug}, Page : {page_slug}') + LOG.debug(f"{__name__}: Project - {project_.slug}, Page : {page_slug}") - # Page query needs id for project and slug for page + # Page query needs id for project and slug for page page = q.page(project_.id, page_slug) if not page: - LOG.error(f'{__name__}: Page not found for project - {project_.slug}, page : {page_slug}') + LOG.error( + f"{__name__}: Page not found for project - {project_.slug}, page : {page_slug}" + ) return render_template(url_for("proofing.project.replace", slug=slug)) latest = page.revisions[-1] @@ -523,12 +546,16 @@ def confirm_replace(slug): # Iterate over the `lines` dictionary for line_num, replace_value in changed_lines.items(): # Check if the line_num exists in the dictionary for this page - LOG.debug(f'{__name__}: Current - {current_lines[line_num]}, Length of lines = {len(current_lines)}') + LOG.debug( + f"{__name__}: Current - {current_lines[line_num]}, Length of lines = {len(current_lines)}" + ) if line_num < len(current_lines): # Replace the line with the replacement value current_lines[line_num] = replace_value else: - LOG.error(f'{__name__}: Invalid line number {line_num} in {page_slug} which has only {len(current_lines)}') + LOG.error( + f"{__name__}: Invalid line number {line_num} in {page_slug} which has only {len(current_lines)}" + ) continue # Join the lines into a single string new_content = "\n".join(current_lines) @@ -536,13 +563,20 @@ def confirm_replace(slug): if new_content != latest.content: # Add a new revision to the page new_summary = f'Replaced "{query}" with "{replace}" on page {page.slug}' - new_revision = add_revision(page=page, summary=new_summary, content=new_content, status=page.status.name, version=page.version, author_id=current_user.id) - LOG.debug(f'{__name__}: New reviion > {page_slug}: {new_revision}') + new_revision = add_revision( + page=page, + summary=new_summary, + content=new_content, + status=page.status.name, + version=page.version, + author_id=current_user.id, + ) + LOG.debug(f"{__name__}: New reviion > {page_slug}: {new_revision}") flash("Changes applied.", "success") return redirect(url_for("proofing.project.activity", slug=slug)) elif form.cancel.data: - LOG.debug(f'{__name__}: CONFIRM_REPLACE Cancelled') + LOG.debug(f"{__name__}: CONFIRM_REPLACE Cancelled") return redirect(url_for("proofing.project.edit", slug=slug)) return render_template(url_for("proofing.project.edit", slug=slug)) diff --git a/test/ambuda/views/proofing/test_project.py b/test/ambuda/views/proofing/test_project.py index 5dc3c0f9..61d59acf 100644 --- a/test/ambuda/views/proofing/test_project.py +++ b/test/ambuda/views/proofing/test_project.py @@ -105,6 +105,16 @@ def test_search__bad_project(rama_client): assert resp.status_code == 404 +def test_replace(client): + resp = client.get("/proofing/test-project/replace") + assert "Replace:" in resp.text + + +def test_replace__bad_project(rama_client): + resp = rama_client.get("/proofing/unknown/replace") + assert resp.status_code == 404 + + def test_admin__unauth(client): resp = client.get("/proofing/test-project/admin") assert resp.status_code == 302 From f91a755f65791e1a5fbbe0aca57f70831457c201 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Thu, 16 Mar 2023 20:53:33 -0400 Subject: [PATCH 10/23] added couple of simple tests --- test/ambuda/views/proofing/test_project.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/ambuda/views/proofing/test_project.py b/test/ambuda/views/proofing/test_project.py index 61d59acf..2393e581 100644 --- a/test/ambuda/views/proofing/test_project.py +++ b/test/ambuda/views/proofing/test_project.py @@ -105,16 +105,31 @@ def test_search__bad_project(rama_client): assert resp.status_code == 404 -def test_replace(client): - resp = client.get("/proofing/test-project/replace") +def test_replace(moderator_client): + resp = moderator_client.get("/proofing/test-project/replace") assert "Replace:" in resp.text +def test_replace__unauth(client): + resp = client.get("/proofing/test-project/replace") + assert resp.status_code == 302 + + def test_replace__bad_project(rama_client): resp = rama_client.get("/proofing/unknown/replace") assert resp.status_code == 404 +def test_submit_changes(moderator_client): + resp = moderator_client.get("/proofing/test-project/submit_changes") + assert "Changes:" in resp.text + + +def test_submit_changes(client): + resp = client.get("/proofing/test-project/submit_changes") + assert resp.status_code == 302 + + def test_admin__unauth(client): resp = client.get("/proofing/test-project/admin") assert resp.status_code == 302 From d2d4b0fa294d1cc87840752f2b342846632a7cb4 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Thu, 16 Mar 2023 22:33:05 -0400 Subject: [PATCH 11/23] minor edit to confirm confirm changes step. --- ...confirm_replace.html => confirm_changes.html} | 15 ++++++++------- ambuda/views/proofing/project.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 15 deletions(-) rename ambuda/templates/proofing/projects/{confirm_replace.html => confirm_changes.html} (72%) diff --git a/ambuda/templates/proofing/projects/confirm_replace.html b/ambuda/templates/proofing/projects/confirm_changes.html similarity index 72% rename from ambuda/templates/proofing/projects/confirm_replace.html rename to ambuda/templates/proofing/projects/confirm_changes.html index 2792fab6..19b52656 100644 --- a/ambuda/templates/proofing/projects/confirm_replace.html +++ b/ambuda/templates/proofing/projects/confirm_changes.html @@ -2,24 +2,25 @@ {% from "macros/forms.html" import field %} {% import "macros/proofing.html" as m %} -{% block title %}Confirm Replace | {{ project.title }}{% endblock %} +{% block title %} Search and Replace | {{ project.title }}{% endblock %} {% block content %} -{{ m.project_header_nested('Confirm Replace', project) }} +{{ m.project_header_nested('Review and Confirm Changes', project) }} {{ m.project_nav(project=project, active='edit') }}
    -

    Confirm Replace

    +

    Confirm Changes

    {% macro sp(s, p, n) %}{% if n == 1 %}{{ s }}{% else %}{{ p }}{% endif %}{% endmacro %} -

    Please confirm the changes you want to make:

    -
    +

    Please carefully review and confirm the changes you selected:

    + {{ form.csrf_token }} - {% set nr = results|length %} -

    Confirm edits on {{ nr }} {{ sp("page", "pages", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }} to be replaced by {{ replace }}.

    + {% set match_counts = results|map(attribute='matches')|map('length')|list %} + {% set nr = match_counts|sum %} +

    Confirm changes on {{ nr }} {{ sp("match", "matches", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }} to be replaced by {{ replace }}.

    {% for result in results %} {% set page = result.page %} diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 678783b8..1c482e36 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -135,7 +135,7 @@ class Meta: submit = SubmitField("Submit Changes") -class ConfirmReplaceForm(ReplaceForm): +class ConfirmChangesForm(ReplaceForm): class Meta: csrf = False @@ -438,9 +438,9 @@ def _select_changes(project_, selected_keys, query: str, replace: str): LOG.debug(f"{__name__} > Number of selected changes = {selected_count}") return render_template( - "proofing/projects/confirm_replace.html", + "proofing/projects/confirm_changes.html", project=project_, - form=ConfirmReplaceForm(), + form=ConfirmChangesForm(), query=query, replace=replace, results=results, @@ -490,17 +490,17 @@ def submit_changes(slug): return render -@bp.route("//confirm_replace", methods=["GET", "POST"]) +@bp.route("//confirm_changes", methods=["GET", "POST"]) @login_required -def confirm_replace(slug): +def confirm_changes(slug): """Confirm changes to replace a string across all of the project's pages.""" project_ = q.project(slug) if project_ is None: abort(404) LOG.debug( - f"{__name__}: CONFIRM_REPLACE {request.method} > Keys: {list(request.form.keys())}, Items: {list(request.form.items())}" + f"{__name__}: confirm_changes {request.method} > Keys: {list(request.form.keys())}, Items: {list(request.form.items())}" ) - form = ConfirmReplaceForm(request.form) + form = ConfirmChangesForm(request.form) if not form.validate(): flash("Invalid input.", "danger") invalid_keys = list(form.errors.keys()) @@ -576,7 +576,7 @@ def confirm_replace(slug): flash("Changes applied.", "success") return redirect(url_for("proofing.project.activity", slug=slug)) elif form.cancel.data: - LOG.debug(f"{__name__}: CONFIRM_REPLACE Cancelled") + LOG.debug(f"{__name__}: confirm_changes Cancelled") return redirect(url_for("proofing.project.edit", slug=slug)) return render_template(url_for("proofing.project.edit", slug=slug)) From 798017a767d3572fc55426e813015af8c91cb9b8 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Thu, 16 Mar 2023 22:39:39 -0400 Subject: [PATCH 12/23] fixed lint issue --- test/ambuda/views/proofing/test_project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ambuda/views/proofing/test_project.py b/test/ambuda/views/proofing/test_project.py index 2393e581..3d7185ff 100644 --- a/test/ambuda/views/proofing/test_project.py +++ b/test/ambuda/views/proofing/test_project.py @@ -125,7 +125,7 @@ def test_submit_changes(moderator_client): assert "Changes:" in resp.text -def test_submit_changes(client): +def test_submit_unauth(client): resp = client.get("/proofing/test-project/submit_changes") assert resp.status_code == 302 From a8f123c55b57fe419b90a3311f74855551274359 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Thu, 16 Mar 2023 23:03:33 -0400 Subject: [PATCH 13/23] added couple of tests to barely fix unittests --- test/ambuda/views/proofing/test_project.py | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test/ambuda/views/proofing/test_project.py b/test/ambuda/views/proofing/test_project.py index 3d7185ff..8a886722 100644 --- a/test/ambuda/views/proofing/test_project.py +++ b/test/ambuda/views/proofing/test_project.py @@ -110,6 +110,15 @@ def test_replace(moderator_client): assert "Replace:" in resp.text +def test_replace_post(moderator_client): + resp = moderator_client.post("/proofing/test-project/replace", + data={ + "query": "the", + "replace": "the",} + ) + assert resp.status_code == 200 + + def test_replace__unauth(client): resp = client.get("/proofing/test-project/replace") assert resp.status_code == 302 @@ -125,11 +134,34 @@ def test_submit_changes(moderator_client): assert "Changes:" in resp.text +def test_submit_changes_post(moderator_client): + resp = moderator_client.post("/proofing/test-project/submit_changes", + data={ + "query": "the", + "replace": "the", + "matches": [], + "submit": True, + } + ) + + assert resp.status_code == 200 + + def test_submit_unauth(client): resp = client.get("/proofing/test-project/submit_changes") assert resp.status_code == 302 +def test_confirm_changes(moderator_client): + resp = moderator_client.get("/proofing/test-project/confirm_changes") + assert "replace" in resp.text + + +def test_confirm_unauth(client): + resp = client.get("/proofing/test-project/confirm_changes") + assert resp.status_code == 302 + + def test_admin__unauth(client): resp = client.get("/proofing/test-project/admin") assert resp.status_code == 302 @@ -155,3 +187,13 @@ def test_admin__has_admin_role(admin_client): def test_admin__has_moderator_role__bad_project(admin_client): resp = admin_client.get("/proofing/unknown/admin") assert resp.status_code == 404 + + +def test_batch_ocr(moderator_client): + resp = moderator_client.get("/proofing/test-project/batch-ocr") + assert resp.status_code == 200 + + +def test_batch_ocr__unauth(client): + resp = client.get("/proofing/test-project/batch-ocr") + assert resp.status_code == 302 From dd98f24151e1971313e45ec498b22c7c0b5df35e Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Fri, 17 Mar 2023 13:52:47 -0400 Subject: [PATCH 14/23] add coverage report target in Makefile --- Makefile | 5 ++++ test/ambuda/views/proofing/test_project.py | 33 ++++++++++++---------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index ede40895..d2b7b839 100644 --- a/Makefile +++ b/Makefile @@ -209,6 +209,9 @@ test: py-venv-check coverage: pytest --cov=ambuda --cov-report=html test/ +coverage-report: coverage + coverage report --fail-under=80 + # Generate Ambuda's technical documentation. # After the command completes, open "docs/_build/index.html". docs: py-venv-check @@ -275,6 +278,8 @@ babel-update: py-venv-check babel-compile: py-venv-check pybabel compile -d ambuda/translations +# Clean up +# =============================================== clean: @rm -rf deploy/data/ @rm -rf ambuda/translations/* diff --git a/test/ambuda/views/proofing/test_project.py b/test/ambuda/views/proofing/test_project.py index 8a886722..e6192f50 100644 --- a/test/ambuda/views/proofing/test_project.py +++ b/test/ambuda/views/proofing/test_project.py @@ -111,12 +111,14 @@ def test_replace(moderator_client): def test_replace_post(moderator_client): - resp = moderator_client.post("/proofing/test-project/replace", - data={ - "query": "the", - "replace": "the",} - ) - assert resp.status_code == 200 + resp = moderator_client.post( + "/proofing/test-project/replace", + data={ + "query": "the", + "replace": "the", + }, + ) + assert resp.status_code == 200 def test_replace__unauth(client): @@ -135,15 +137,16 @@ def test_submit_changes(moderator_client): def test_submit_changes_post(moderator_client): - resp = moderator_client.post("/proofing/test-project/submit_changes", - data={ - "query": "the", - "replace": "the", - "matches": [], - "submit": True, - } - ) - + resp = moderator_client.post( + "/proofing/test-project/submit_changes", + data={ + "query": "the", + "replace": "the", + "matches": [], + "submit": True, + }, + ) + assert resp.status_code == 200 From c09ef66f61355d9ee1010b2aec61813fd8a233d3 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Fri, 17 Mar 2023 20:25:28 -0400 Subject: [PATCH 15/23] search and replace matching strings across all pages in a project --- Makefile | 5 + .../proofing/projects/confirm_changes.html | 44 +++ .../templates/proofing/projects/replace.html | 111 +++++-- ambuda/views/proofing/project.py | 288 ++++++++++++++++-- test/ambuda/views/proofing/test_project.py | 70 +++++ 5 files changed, 466 insertions(+), 52 deletions(-) create mode 100644 ambuda/templates/proofing/projects/confirm_changes.html diff --git a/Makefile b/Makefile index ede40895..d2b7b839 100644 --- a/Makefile +++ b/Makefile @@ -209,6 +209,9 @@ test: py-venv-check coverage: pytest --cov=ambuda --cov-report=html test/ +coverage-report: coverage + coverage report --fail-under=80 + # Generate Ambuda's technical documentation. # After the command completes, open "docs/_build/index.html". docs: py-venv-check @@ -275,6 +278,8 @@ babel-update: py-venv-check babel-compile: py-venv-check pybabel compile -d ambuda/translations +# Clean up +# =============================================== clean: @rm -rf deploy/data/ @rm -rf ambuda/translations/* diff --git a/ambuda/templates/proofing/projects/confirm_changes.html b/ambuda/templates/proofing/projects/confirm_changes.html new file mode 100644 index 00000000..19b52656 --- /dev/null +++ b/ambuda/templates/proofing/projects/confirm_changes.html @@ -0,0 +1,44 @@ +{% extends 'proofing/base.html' %} +{% from "macros/forms.html" import field %} +{% import "macros/proofing.html" as m %} + +{% block title %} Search and Replace | {{ project.title }}{% endblock %} + +{% block content %} + +{{ m.project_header_nested('Review and Confirm Changes', project) }} +{{ m.project_nav(project=project, active='edit') }} + +
    +

    Confirm Changes

    + {% macro sp(s, p, n) %}{% if n == 1 %}{{ s }}{% else %}{{ p }}{% endif %}{% endmacro %} +

    Please carefully review and confirm the changes you selected:

    + + {{ form.csrf_token }} + + + + {% set match_counts = results|map(attribute='matches')|map('length')|list %} + {% set nr = match_counts|sum %} +

    Confirm changes on {{ nr }} {{ sp("match", "matches", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }} to be replaced by {{ replace }}.

    + + {% for result in results %} + {% set page = result.page %} + {% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} + {% set matches = result.matches %} + {% for match in matches %} +
    +

    Page {{ project.title }}/{{ page.slug }}: Line {{ match.line_num }}

    + + +
    + + +
    + {% endfor %} + {% endfor %} + + + +
    +{% endblock %} diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index 3bbb8f98..8573c356 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -10,38 +10,95 @@ {{ m.project_nav(project=project, active='edit') }}
    +

    Replace

    Use this simple search and replace form to make edits across this project.

    +
    + + {{ field(form.query) }} + {{ field(form.replace) }} + +
    -
    - {{ field(form.query) }} - {{ field(form.replace) }} - -
    - -{% if query %} +{% if results %}
    +

    Matches

    + {% macro sp(s, p, n) %}{% if n == 1 %}{{ s }}{% else %}{{ p }}{% endif %}{% endmacro %} +
    + {{ submit_changes_form.csrf_token }} + + -{% macro sp(s, p, n) %}{% if n == 1 %}{{ s }}{% else %}{{ p }}{% endif %}{% endmacro %} - -{% set nr = results|length %} -

    Found {{ nr }} {{ sp("page", "pages", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }}.

    - -
      -{% for page in results %} -{% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} -
    • - {{ project.title }}/{{ page.slug }} -
      - {% for match in page.matches %} - 
      {{ match.query }}
      - 
      {{ match.update }}
      - {%- endfor %} + {% set nr = results|length %} +

      Found {{ nr }} {{ sp("page", "pages", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }}.

      +
        +
        + +
        - -{% endfor %} -
      - + {% for page in results %} + {% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} +
    • + {{ project.title }}/{{ page.slug }} +
      + {% for match in page.matches %} +
      + +

      Page {{ project.title }}/{{ page.slug }}: Line {{ match.line_num }}

      + +
      + + +
      +
      + {% endfor %} +
      +
      • + {% endfor %} + {% if submit_changes_form %} + {{ submit_changes_form.submit(class="btn btn-submit") }} + {% else %} + + {% endif %} + +
    + + {% endif %} + + +
    -{% endif %} + {% endblock %} \ No newline at end of file diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 9a375790..1c482e36 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -1,3 +1,6 @@ +import logging +import re + from celery.result import GroupResult from flask import ( Blueprint, @@ -9,13 +12,21 @@ url_for, ) from flask_babel import lazy_gettext as _l -from flask_login import login_required +from flask_login import current_user, login_required from flask_wtf import FlaskForm from markupsafe import Markup, escape from sqlalchemy import orm from werkzeug.exceptions import abort from werkzeug.utils import redirect -from wtforms import StringField +from wtforms import ( + BooleanField, + FieldList, + Form, + FormField, + HiddenField, + StringField, + SubmitField, +) from wtforms.validators import DataRequired, ValidationError from wtforms.widgets import TextArea @@ -24,9 +35,11 @@ from ambuda.tasks import app as celery_app from ambuda.tasks import ocr as ocr_tasks from ambuda.utils import project_utils, proofing_utils +from ambuda.utils.revisions import add_revision from ambuda.views.proofing.decorators import moderator_required, p2_required bp = Blueprint("project", __name__) +LOG = logging.getLogger(__name__) def _is_valid_page_number_spec(_, field): @@ -85,6 +98,11 @@ class EditMetadataForm(FlaskForm): ) +class MatchForm(Form): + selected = BooleanField() + replace = HiddenField(validators=[DataRequired()]) + + class SearchForm(FlaskForm): class Meta: csrf = False @@ -103,6 +121,28 @@ class Meta: replace = StringField(_l("Replace"), validators=[DataRequired()]) +def validate_matches(form, field): + for match_form in field: + if match_form.errors: + raise ValidationError("Invalid match form values.") + + +class SubmitChangesForm(ReplaceForm): + class Meta: + csrf = False + + matches = FieldList(FormField(MatchForm), validators=[validate_matches]) + submit = SubmitField("Submit Changes") + + +class ConfirmChangesForm(ReplaceForm): + class Meta: + csrf = False + + confirm = SubmitField("Confirm") + cancel = SubmitField("Cancel") + + @bp.route("//") def summary(slug): """Show basic information about the project.""" @@ -286,7 +326,55 @@ def search(slug): ) -@bp.route("//replace") +def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str): + """ + Gather all matches for the "query" string and pair them the "replace" string. + """ + + results = [] + + LOG.debug(f"Search/Replace text with {query} and {replace}") + for page_ in project_.pages: + if not page_.revisions: + continue + matches = [] + latest = page_.revisions[-1] + LOG.debug(f"{__name__}: {page_.slug}") + for line_num, line in enumerate(latest.content.splitlines()): + # LOG.debug(f'Search/Replace > {page_.slug}: {line}') + if query in line: + LOG.debug(f"Search/Replace > appending search/replace {line}") + matches.append( + { + "query": escape(line).replace( + query, Markup(f"{escape(query)}") + ), + "replace": escape(line).replace( + query, Markup(f"{escape(replace)}") + ), + "checked": False, + "line_num": line_num, + } + ) + if matches: + results.append( + { + "slug": page_.slug, + "matches": matches, + } + ) + return render_template( + "proofing/projects/replace.html", + project=project_, + form=replace_form, + submit_changes_form=SubmitChangesForm(), + query=query, + replace=replace, + results=results, + ) + + +@bp.route("//replace", methods=["GET", "POST"]) @login_required def replace(slug): """Search and replace a string across all of the project's pages. @@ -297,53 +385,203 @@ def replace(slug): if project_ is None: abort(404) - form = ReplaceForm(request.args) + form = ReplaceForm(request.form) if not form.validate(): + invalid_keys = list(form.errors.keys()) + LOG.debug(f"Invalid form - {request.method}, invalid keys: {invalid_keys}") return render_template( - "proofing/projects/replace.html", project=project_, form=form + "proofing/projects/replace.html", project=project_, form=ReplaceForm() ) # search for "query" string and replace with "update" string query = form.query.data - update = form.replace.data + replace = form.replace.data + render = _replace_text(project_, replace_form=form, query=query, replace=replace) + return render + +def _select_changes(project_, selected_keys, query: str, replace: str): + """ + Mark "query" strings + """ results = [] + LOG.debug(f"{__name__}: Mark changes with {query} and {replace}") for page_ in project_.pages: if not page_.revisions: continue + latest = page_.revisions[-1] matches = [] + for line_num, line in enumerate(latest.content.splitlines()): - latest = page_.revisions[-1] - for line in latest.content.splitlines(): - if query in line: + form_key = f"match{page_.slug}-{line_num}" + replace_form_key = f"match{page_.slug}-{line_num}-replace" + + if selected_keys.get(form_key) == "selected": + LOG.debug(f"{__name__}: {form_key}: {selected_keys.get(form_key)}") + LOG.debug( + f"{__name__}: {replace_form_key}: {request.form.get(replace_form_key)}" + ) + LOG.debug(f"{__name__}: {form_key}: Appended") matches.append( { - "query": escape(line).replace( - query, Markup(f"{escape(query)}") - ), - "update": escape(line).replace( - query, Markup(f"{escape(update)}") - ), + "query": line, + "replace": line.replace(query, replace), + "line_num": line_num, } ) - if matches: - results.append( - { - "slug": page_.slug, - "matches": matches, - } - ) + + results.append({"page": page_, "matches": matches}) + LOG.debug(f"{__name__}: Total matches appended: {len(matches)}") + + selected_count = sum(value == "selected" for value in selected_keys.values()) + LOG.debug(f"{__name__} > Number of selected changes = {selected_count}") + return render_template( - "proofing/projects/replace.html", + "proofing/projects/confirm_changes.html", project=project_, - form=form, + form=ConfirmChangesForm(), query=query, - update=update, + replace=replace, results=results, ) +@bp.route("//submit_changes", methods=["GET", "POST"]) +@login_required +def submit_changes(slug): + """Submit selected changes across all of the project's pages. + + This is useful to replace a string across the project in one shot. + """ + + project_ = q.project(slug) + if project_ is None: + abort(404) + + LOG.debug( + f"{__name__}: SUBMIT_CHANGES --- {request.method} > {list(request.form.keys())}" + ) + + # FIXME: find a way to validate this form. Current `matches` are coming in the way of validators. + form = SubmitChangesForm(request.form) + # if not form.validate(): + # # elif request.form.get("form_submitted") is None: + # invalid_keys = list(form.errors.keys()) + # LOG.debug(f'{__name__}: Invalid form values - {request.method}, invalid keys: {invalid_keys}') + # return redirect(url_for("proofing.project.replace", slug=slug)) + + render = None + # search for "query" string and replace with "update" string + query = form.query.data + replace = form.replace.data + + LOG.debug( + f"{__name__}: ({request.method})> Got to submit method with {query}->{replace} " + ) + LOG.debug(f"{__name__}: {request.method} > {list(request.form.keys())}") + selected_keys = { + key: value + for key, value in request.form.items() + if key.startswith("match") and not key.endswith("replace") + } + render = _select_changes(project_, selected_keys, query=query, replace=replace) + + return render + + +@bp.route("//confirm_changes", methods=["GET", "POST"]) +@login_required +def confirm_changes(slug): + """Confirm changes to replace a string across all of the project's pages.""" + project_ = q.project(slug) + if project_ is None: + abort(404) + LOG.debug( + f"{__name__}: confirm_changes {request.method} > Keys: {list(request.form.keys())}, Items: {list(request.form.items())}" + ) + form = ConfirmChangesForm(request.form) + if not form.validate(): + flash("Invalid input.", "danger") + invalid_keys = list(form.errors.keys()) + LOG.error( + f"{__name__}: Invalid form - {request.method}, invalid keys: {invalid_keys}" + ) + return redirect(url_for("proofing.project.replace", slug=slug)) + + if form.confirm.data: + LOG.debug(f"{__name__}: {request.method} > Confirmed!") + query = form.query.data + replace = form.replace.data + + # Get the changes from the form and store them in a list + pages = {} + + # Iterate over the dictionary `request.form` + for key, value in request.form.items(): + # Check if key matches the pattern + match = re.match(r"match(\d+)-(\d+)-replace", key) + if match: + # Extract page_slug and line_num from the key + page_slug = match.group(1) + line_num = int(match.group(2)) + if page_slug not in pages: + pages[page_slug] = {} + pages[page_slug][line_num] = value + + for page_slug, changed_lines in pages.items(): + # Get the corresponding `Page` object + LOG.debug(f"{__name__}: Project - {project_.slug}, Page : {page_slug}") + + # Page query needs id for project and slug for page + page = q.page(project_.id, page_slug) + if not page: + LOG.error( + f"{__name__}: Page not found for project - {project_.slug}, page : {page_slug}" + ) + return render_template(url_for("proofing.project.replace", slug=slug)) + + latest = page.revisions[-1] + current_lines = latest.content.splitlines() + # Iterate over the `lines` dictionary + for line_num, replace_value in changed_lines.items(): + # Check if the line_num exists in the dictionary for this page + LOG.debug( + f"{__name__}: Current - {current_lines[line_num]}, Length of lines = {len(current_lines)}" + ) + if line_num < len(current_lines): + # Replace the line with the replacement value + current_lines[line_num] = replace_value + else: + LOG.error( + f"{__name__}: Invalid line number {line_num} in {page_slug} which has only {len(current_lines)}" + ) + continue + # Join the lines into a single string + new_content = "\n".join(current_lines) + # Check if the page content has changed + if new_content != latest.content: + # Add a new revision to the page + new_summary = f'Replaced "{query}" with "{replace}" on page {page.slug}' + new_revision = add_revision( + page=page, + summary=new_summary, + content=new_content, + status=page.status.name, + version=page.version, + author_id=current_user.id, + ) + LOG.debug(f"{__name__}: New reviion > {page_slug}: {new_revision}") + + flash("Changes applied.", "success") + return redirect(url_for("proofing.project.activity", slug=slug)) + elif form.cancel.data: + LOG.debug(f"{__name__}: confirm_changes Cancelled") + return redirect(url_for("proofing.project.edit", slug=slug)) + + return render_template(url_for("proofing.project.edit", slug=slug)) + + @bp.route("//batch-ocr", methods=["GET", "POST"]) @p2_required def batch_ocr(slug): diff --git a/test/ambuda/views/proofing/test_project.py b/test/ambuda/views/proofing/test_project.py index 5dc3c0f9..e6192f50 100644 --- a/test/ambuda/views/proofing/test_project.py +++ b/test/ambuda/views/proofing/test_project.py @@ -105,6 +105,66 @@ def test_search__bad_project(rama_client): assert resp.status_code == 404 +def test_replace(moderator_client): + resp = moderator_client.get("/proofing/test-project/replace") + assert "Replace:" in resp.text + + +def test_replace_post(moderator_client): + resp = moderator_client.post( + "/proofing/test-project/replace", + data={ + "query": "the", + "replace": "the", + }, + ) + assert resp.status_code == 200 + + +def test_replace__unauth(client): + resp = client.get("/proofing/test-project/replace") + assert resp.status_code == 302 + + +def test_replace__bad_project(rama_client): + resp = rama_client.get("/proofing/unknown/replace") + assert resp.status_code == 404 + + +def test_submit_changes(moderator_client): + resp = moderator_client.get("/proofing/test-project/submit_changes") + assert "Changes:" in resp.text + + +def test_submit_changes_post(moderator_client): + resp = moderator_client.post( + "/proofing/test-project/submit_changes", + data={ + "query": "the", + "replace": "the", + "matches": [], + "submit": True, + }, + ) + + assert resp.status_code == 200 + + +def test_submit_unauth(client): + resp = client.get("/proofing/test-project/submit_changes") + assert resp.status_code == 302 + + +def test_confirm_changes(moderator_client): + resp = moderator_client.get("/proofing/test-project/confirm_changes") + assert "replace" in resp.text + + +def test_confirm_unauth(client): + resp = client.get("/proofing/test-project/confirm_changes") + assert resp.status_code == 302 + + def test_admin__unauth(client): resp = client.get("/proofing/test-project/admin") assert resp.status_code == 302 @@ -130,3 +190,13 @@ def test_admin__has_admin_role(admin_client): def test_admin__has_moderator_role__bad_project(admin_client): resp = admin_client.get("/proofing/unknown/admin") assert resp.status_code == 404 + + +def test_batch_ocr(moderator_client): + resp = moderator_client.get("/proofing/test-project/batch-ocr") + assert resp.status_code == 200 + + +def test_batch_ocr__unauth(client): + resp = client.get("/proofing/test-project/batch-ocr") + assert resp.status_code == 302 From ce583cf30c36a4bd2f9a7d78d012a21cb1e0e228 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu <44282098+kvchitrapu@users.noreply.github.com> Date: Sat, 18 Mar 2023 17:08:39 -0400 Subject: [PATCH 16/23] search and replace matching strings across all pages in a project (#79) From b330399a77d9fde58aa4b2a73919f735ce2a24a4 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Mon, 20 Mar 2023 23:45:29 -0400 Subject: [PATCH 17/23] added regex query support --- .../templates/proofing/projects/replace.html | 6 +-- ambuda/views/proofing/project.py | 38 +++++++++++-------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index 8573c356..e30c64ed 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -11,7 +11,7 @@

    Replace

    -

    Use this simple search and replace form to make edits across this project.

    +

    Use this simple search and replace form to make edits across this project. The search supports regex.s

    {{ field(form.query) }} @@ -45,9 +45,9 @@

    Matches

    Page {{ project.title }}/{{ page.slug }}: Line {{ match.line_num }}

    - +
    - +

    diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 1c482e36..f33e20a2 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -40,7 +40,7 @@ bp = Blueprint("project", __name__) LOG = logging.getLogger(__name__) - +LOG.setLevel(logging.DEBUG) def _is_valid_page_number_spec(_, field): try: @@ -333,6 +333,8 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) results = [] + query_pattern = re.compile(query, re.UNICODE) # Compile the regex pattern with Unicode support + LOG.debug(f"Search/Replace text with {query} and {replace}") for page_ in project_.pages: if not page_.revisions: @@ -341,21 +343,25 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) latest = page_.revisions[-1] LOG.debug(f"{__name__}: {page_.slug}") for line_num, line in enumerate(latest.content.splitlines()): - # LOG.debug(f'Search/Replace > {page_.slug}: {line}') - if query in line: - LOG.debug(f"Search/Replace > appending search/replace {line}") - matches.append( - { - "query": escape(line).replace( - query, Markup(f"{escape(query)}") - ), - "replace": escape(line).replace( - query, Markup(f"{escape(replace)}") - ), - "checked": False, - "line_num": line_num, - } - ) + if query_pattern.search(line): + try: + replaced_line = query_pattern.sub(replace, line) + marked_query = query_pattern.sub(lambda m: Markup(f'{escape(m.group(0))}'), line) + marked_replace = query_pattern.sub(Markup(f'{escape(replace)}'), line) + LOG.debug(f"Search/Replace > marked query: {marked_query}") + LOG.debug(f"Search/Replace > marked replace: {marked_replace}") + matches.append( + { + "query": marked_query, + "replace": marked_replace, + "checked": False, + "line_num": line_num, + } + ) + except TimeoutError: + # Handle the timeout for regex operation, e.g., log a warning or show an error message + LOG.warning(f"Regex operation timed out for line {line_num}: {line}") + if matches: results.append( { From 26ff48752026ecc75301d11b32bfc8a639dcf01d Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Mon, 20 Mar 2023 23:57:09 -0400 Subject: [PATCH 18/23] fixed lint issue --- ambuda/views/proofing/project.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index f33e20a2..ce8791d3 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -42,6 +42,7 @@ LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) + def _is_valid_page_number_spec(_, field): try: _ = project_utils.parse_page_number_spec(field.data) @@ -333,7 +334,9 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) results = [] - query_pattern = re.compile(query, re.UNICODE) # Compile the regex pattern with Unicode support + query_pattern = re.compile( + query, re.UNICODE + ) # Compile the regex pattern with Unicode support LOG.debug(f"Search/Replace text with {query} and {replace}") for page_ in project_.pages: @@ -345,9 +348,12 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) for line_num, line in enumerate(latest.content.splitlines()): if query_pattern.search(line): try: - replaced_line = query_pattern.sub(replace, line) - marked_query = query_pattern.sub(lambda m: Markup(f'{escape(m.group(0))}'), line) - marked_replace = query_pattern.sub(Markup(f'{escape(replace)}'), line) + marked_query = query_pattern.sub( + lambda m: Markup(f"{escape(m.group(0))}"), line + ) + marked_replace = query_pattern.sub( + Markup(f"{escape(replace)}"), line + ) LOG.debug(f"Search/Replace > marked query: {marked_query}") LOG.debug(f"Search/Replace > marked replace: {marked_replace}") matches.append( @@ -360,8 +366,10 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) ) except TimeoutError: # Handle the timeout for regex operation, e.g., log a warning or show an error message - LOG.warning(f"Regex operation timed out for line {line_num}: {line}") - + LOG.warning( + f"Regex operation timed out for line {line_num}: {line}" + ) + if matches: results.append( { From 3a6c4aebad5cb9e4d76a6e84cae73e386c55e3f1 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu <44282098+kvchitrapu@users.noreply.github.com> Date: Tue, 21 Mar 2023 00:11:07 -0400 Subject: [PATCH 19/23] Issue 470 search replace (#82) * search and replace matching strings across all pages in a project * added regex query support * fixed lint issue --- .../templates/proofing/projects/replace.html | 6 +-- ambuda/views/proofing/project.py | 43 ++++++++++++------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index 8573c356..e30c64ed 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -11,7 +11,7 @@

    Replace

    -

    Use this simple search and replace form to make edits across this project.

    +

    Use this simple search and replace form to make edits across this project. The search supports regex.s

    {{ field(form.query) }} @@ -45,9 +45,9 @@

    Matches

    Page {{ project.title }}/{{ page.slug }}: Line {{ match.line_num }}

    - +
    - +

    diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index 1c482e36..d5df1c70 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -333,6 +333,10 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) results = [] + query_pattern = re.compile( + query, re.UNICODE + ) # Compile the regex pattern with Unicode support + LOG.debug(f"Search/Replace text with {query} and {replace}") for page_ in project_.pages: if not page_.revisions: @@ -341,21 +345,30 @@ def _replace_text(project_, replace_form: ReplaceForm, query: str, replace: str) latest = page_.revisions[-1] LOG.debug(f"{__name__}: {page_.slug}") for line_num, line in enumerate(latest.content.splitlines()): - # LOG.debug(f'Search/Replace > {page_.slug}: {line}') - if query in line: - LOG.debug(f"Search/Replace > appending search/replace {line}") - matches.append( - { - "query": escape(line).replace( - query, Markup(f"{escape(query)}") - ), - "replace": escape(line).replace( - query, Markup(f"{escape(replace)}") - ), - "checked": False, - "line_num": line_num, - } - ) + if query_pattern.search(line): + try: + marked_query = query_pattern.sub( + lambda m: Markup(f"{escape(m.group(0))}"), line + ) + marked_replace = query_pattern.sub( + Markup(f"{escape(replace)}"), line + ) + LOG.debug(f"Search/Replace > marked query: {marked_query}") + LOG.debug(f"Search/Replace > marked replace: {marked_replace}") + matches.append( + { + "query": marked_query, + "replace": marked_replace, + "checked": False, + "line_num": line_num, + } + ) + except TimeoutError: + # Handle the timeout for regex operation, e.g., log a warning or show an error message + LOG.warning( + f"Regex operation timed out for line {line_num}: {line}" + ) + if matches: results.append( { From 4c7b0c4490dd4dd7370e1113fa89a20648741381 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Tue, 21 Mar 2023 07:59:27 -0400 Subject: [PATCH 20/23] updated Search & Replace comment --- ambuda/templates/proofing/projects/replace.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index e30c64ed..54344294 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -11,7 +11,7 @@

    Replace

    -

    Use this simple search and replace form to make edits across this project. The search supports regex.s

    +

    Use this simple search and replace form to make edits across this project. The search supports regular expressions./p> {{ field(form.query) }} From e34fba17ca9d6f85c22115b5bc8bdb41d5d7fb9e Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu <44282098+kvchitrapu@users.noreply.github.com> Date: Tue, 21 Mar 2023 08:30:48 -0400 Subject: [PATCH 21/23] search and replace with regular expression query patterns (#83) * search and replace matching strings across all pages in a project * added regex query support * fixed lint issue * updated Search & Replace comment --- ambuda/templates/proofing/projects/replace.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index e30c64ed..54344294 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -11,7 +11,7 @@

    Replace

    -

    Use this simple search and replace form to make edits across this project. The search supports regex.s

    +

    Use this simple search and replace form to make edits across this project. The search supports regular expressions./p> {{ field(form.query) }} From ea65bfcccd8050956e1024d13c2f77919f80a9a5 Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Tue, 21 Mar 2023 22:56:48 -0400 Subject: [PATCH 22/23] replace the revision --- .../templates/proofing/projects/replace.html | 74 +++++++++---------- ambuda/views/proofing/project.py | 6 +- test/ambuda/views/proofing/test_project.py | 8 +- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/ambuda/templates/proofing/projects/replace.html b/ambuda/templates/proofing/projects/replace.html index 54344294..ebb151c2 100644 --- a/ambuda/templates/proofing/projects/replace.html +++ b/ambuda/templates/proofing/projects/replace.html @@ -11,7 +11,7 @@

    Replace

    -

    Use this simple search and replace form to make edits across this project. The search supports regular expressions./p> +

    Use this simple search and replace form to make edits across this project. The search supports regular expressions.

    {{ field(form.query) }} @@ -33,8 +33,8 @@

    Matches

    Found {{ nr }} {{ sp("page", "pages", nr) }} that {{ sp("contains", "contain", nr) }} {{ query }}.

      - - + +
      {% for page in results %} {% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %} @@ -61,44 +61,44 @@

      Matches

      {% endif %} -
    - - {% endif %} - + - + + // Add change event listeners to each individual checkbox + checkboxes.forEach(function(checkbox) { + checkbox.addEventListener("change", function(event) { + updateReplaceFieldVisibility(event.target); + + // Update the state of the 'select-all' checkbox + checkAll.checked = Array.from(checkboxes).every(checkbox => checkbox.checked); + }); + }); + })(); +
    - +{% endif %} {% endblock %} \ No newline at end of file diff --git a/ambuda/views/proofing/project.py b/ambuda/views/proofing/project.py index d5df1c70..5df87c1e 100644 --- a/ambuda/views/proofing/project.py +++ b/ambuda/views/proofing/project.py @@ -419,6 +419,9 @@ def _select_changes(project_, selected_keys, query: str, replace: str): """ results = [] LOG.debug(f"{__name__}: Mark changes with {query} and {replace}") + query_pattern = re.compile( + query, re.UNICODE + ) # Compile the regex pattern with Unicode support for page_ in project_.pages: if not page_.revisions: continue @@ -436,10 +439,11 @@ def _select_changes(project_, selected_keys, query: str, replace: str): f"{__name__}: {replace_form_key}: {request.form.get(replace_form_key)}" ) LOG.debug(f"{__name__}: {form_key}: Appended") + replaced_line = query_pattern.sub(replace, line) matches.append( { "query": line, - "replace": line.replace(query, replace), + "replace": replaced_line, "line_num": line_num, } ) diff --git a/test/ambuda/views/proofing/test_project.py b/test/ambuda/views/proofing/test_project.py index e6192f50..e586cfb0 100644 --- a/test/ambuda/views/proofing/test_project.py +++ b/test/ambuda/views/proofing/test_project.py @@ -132,7 +132,13 @@ def test_replace__bad_project(rama_client): def test_submit_changes(moderator_client): - resp = moderator_client.get("/proofing/test-project/submit_changes") + query = "test_query" + replace = "test_replace" + form_data = { + "query": query, + "replace": replace + } + resp = moderator_client.post("/proofing/test-project/submit_changes", data=form_data) assert "Changes:" in resp.text From 2cffa01f2e4b3fb18130783841b6d3046016d9bf Mon Sep 17 00:00:00 2001 From: Kishore Chitrapu Date: Tue, 21 Mar 2023 22:57:23 -0400 Subject: [PATCH 23/23] add unittest for submit_changes --- test/ambuda/views/proofing/test_project.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/ambuda/views/proofing/test_project.py b/test/ambuda/views/proofing/test_project.py index e586cfb0..f4178d23 100644 --- a/test/ambuda/views/proofing/test_project.py +++ b/test/ambuda/views/proofing/test_project.py @@ -134,11 +134,10 @@ def test_replace__bad_project(rama_client): def test_submit_changes(moderator_client): query = "test_query" replace = "test_replace" - form_data = { - "query": query, - "replace": replace - } - resp = moderator_client.post("/proofing/test-project/submit_changes", data=form_data) + form_data = {"query": query, "replace": replace} + resp = moderator_client.post( + "/proofing/test-project/submit_changes", data=form_data + ) assert "Changes:" in resp.text