feat (WIP): bring png OCR scanning support (#1670)

* Add pytesseract

* Add simple ocr endpoint

replace extension argument

* feat/ocr-editor gui

* fix frontend linting issues

* Add service unit tests

* Add split text modes & single ingredient/instruction editing

* make split mode really reactive

* Remove default step and ingredient

* make the linter haappy

* Accept only image uploads

* Add automatic recipe title suggestion

* Correct regex

* fix incorrect array.map method usage

* make the linter happy again

* Swap route to use asset name

* Rearange buttons

* fix test data

* feat: Allow making image the recipe image

* Add translation

* Make the linter happy

* Restrict function setPropertyValueByPath generic

* Restrict template literal type

* Add a more friendly icon to creation page

* update poetry lock file

* Correct sloppy ocr classes

* Make MyPy happy

* Rewrite safer tests

* Add tesseract to backend test CI container dependencies

* Make canvas element a component global

* Remove unwanted spaces in selected text

* Add way to know if recipe was created with ocr

* Access to ocr-editor for ocr recipes

* Update Alembic revision

* Make the frontend build

* Fix scrolling offset bug

* Allow creation of recipes with custom settings

* Fix rebasing mistakes

* Add format_tsv_output test

* Exclude the tests data directory only

* Enforce camelCase for frontend functions

* Remove import of unused component

* Fix type and class initialization

* Add multi-language support

* Highlight words in mount

* Fix image ratio bug

* Better ocr creation page

* Revert awkward feature to scroll in Selection mode

* Rebasing alembic migrations sux

* Remove obsolete getShared function

* Add function docstring

* Move down ocr creation option

* Make toolbar icons more generic

* Show help at the bottom of the page

* move ocr types to own file

* Use template ref for the canvas

* Use i18n.tc to get strings directly

* Correct naming mistake

* Move Ocr editor to own directory

* Create Ocr Editor parts

* Safeguard recipe properties access

* Add loading frontend animation due to longer request time

* minor cleanup chores

Co-authored-by: Miroito <[email protected]>

.github/workflows/partial-backend.yml

@@ -62,7 +62,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install libsasl2-dev libldap2-dev libssl-dev
+          sudo apt-get install libsasl2-dev libldap2-dev libssl-dev tesseract-ocr-all
           poetry install
           poetry add "psycopg2-binary==2.8.6"
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'

.pre-commit-config.yaml

@@ -9,6 +9,7 @@ repos:
       - id: check-toml
       - id: end-of-file-fixer
       - id: trailing-whitespace
+        exclude: ^tests/data/
   - repo: https://github.com/sondrelg/pep585-upgrade
     rev: "v1.0.1" # Use the sha / tag you want to point at
     hooks:

alembic/versions/2022-08-05-17.07.07_089bfa50d0ed_add_is_ocr_recipe_column_to_recipes.py

@@ -0,0 +1,27 @@
+"""Add is_ocr_recipe column to recipes
+
+Revision ID: 089bfa50d0ed
+Revises: f30cf048c228
+Create Date: 2022-08-05 17:07:07.389271
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "089bfa50d0ed"
+down_revision = "188374910655"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("recipes", sa.Column("is_ocr_recipe", sa.Boolean(), default=False, nullable=True))
+    op.execute("UPDATE recipes SET is_ocr_recipe = FALSE")
+    #  SQLITE does not support ALTER COLUMN, so the column will stay nullable to prevent making this migration a mess
+    #  The Recipe pydantic model and the SQL server use False as default value anyway for this column so Null should be a very rare sight
+
+
+def downgrade():
+    op.drop_column("recipes", "is_ocr_recipe")

frontend/api/class-interfaces/ocr.ts

@@ -0,0 +1,18 @@
+import { BaseAPI } from "~/api/_base";
+
+const prefix = "/api";
+
+export class OcrAPI extends BaseAPI {
+
+  // Currently unused in favor for the endpoint using asset names
+  async fileToTsv(file: File) {
+    const formData = new FormData();
+    formData.append("file", file);
+    return await this.requests.post(`${prefix}/ocr/file-to-tsv`, formData);
+  }
+
+  async assetToTsv(recipeSlug: string, assetName: string) {
+    return await this.requests.post(`${prefix}/ocr/asset-to-tsv`, { recipeSlug, assetName });
+  }
+
+}

frontend/api/class-interfaces/recipes/recipe.ts

@@ -34,6 +34,7 @@ const routes = {
   recipesCategory: `${prefix}/recipes/category`,
   recipesParseIngredient: `${prefix}/parser/ingredient`,
   recipesParseIngredients: `${prefix}/parser/ingredients`,
+  recipesCreateFromOcr: `${prefix}/recipes/create-ocr`,
 
   recipesRecipeSlug: (recipe_slug: string) => `${prefix}/recipes/${recipe_slug}`,
   recipesRecipeSlugExport: (recipe_slug: string) => `${prefix}/recipes/${recipe_slug}/exports`,
@@ -116,4 +117,13 @@ export class RecipeAPI extends BaseCRUDAPI<CreateRecipe, Recipe, Recipe> {
   getZipRedirectUrl(recipeSlug: string, token: string) {
     return `${routes.recipesRecipeSlugExportZip(recipeSlug)}?token=${token}`;
   }
+
+  async createFromOcr(file: File, makeFileRecipeImage: boolean) {
+    const formData = new FormData();
+    formData.append("file", file);
+    formData.append("extension", file.name.split(".").pop() ?? "");
+    formData.append("makefilerecipeimage", String(makeFileRecipeImage));
+
+    return await this.requests.post(routes.recipesCreateFromOcr, formData);
+  }
 }

frontend/api/index.ts

@@ -24,6 +24,7 @@ import { MultiPurposeLabelsApi } from "./class-interfaces/group-multiple-purpose
 import { GroupEventNotifierApi } from "./class-interfaces/group-event-notifier";
 import { MealPlanRulesApi } from "./class-interfaces/group-mealplan-rules";
 import { GroupDataSeederApi } from "./class-interfaces/group-seeder";
+import {OcrAPI} from "./class-interfaces/ocr";
 import { ApiRequestInstance } from "~/types/api";
 
 class Api {
@@ -52,6 +53,7 @@ class Api {
   public groupEventNotifier: GroupEventNotifierApi;
   public upload: UploadFile;
   public seeders: GroupDataSeederApi;
+  public ocr: OcrAPI;
 
   constructor(requests: ApiRequestInstance) {
     // Recipes
@@ -90,6 +92,9 @@ class Api {
     this.bulk = new BulkActionsAPI(requests);
     this.groupEventNotifier = new GroupEventNotifierApi(requests);
 
+    // ocr
+    this.ocr = new OcrAPI(requests);
+
     Object.freeze(this);
   }
 }

frontend/components/Domain/Recipe/RecipeActionMenu.vue

@@ -90,6 +90,7 @@ const SAVE_EVENT = "save";
 const DELETE_EVENT = "delete";
 const CLOSE_EVENT = "close";
 const JSON_EVENT = "json";
+const OCR_EVENT = "ocr";
 
 export default defineComponent({
   components: { RecipeContextMenu, RecipeFavoriteBadge },
@@ -122,8 +123,12 @@ export default defineComponent({
       type: Boolean,
       default: false,
     },
+    showOcrButton: {
+      type: Boolean,
+      default: false,
+    },
   },
-  setup(_, context) {
+  setup(props, context) {
     const deleteDialog = ref(false);
 
     const { i18n, $globals } = useContext();
@@ -154,22 +159,26 @@ export default defineComponent({
       },
     ];
 
+    if (props.showOcrButton) {
+      editorButtons.splice(2, 0, {
+        text: i18n.t("ocr-editor.ocr-editor"),
+        icon: $globals.icons.eye,
+        event: OCR_EVENT,
+        color: "accent",
+      });
+    }
+
     function emitHandler(event: string) {
       switch (event) {
         case CLOSE_EVENT:
           context.emit(CLOSE_EVENT);
           context.emit("input", false);
           break;
-        case SAVE_EVENT:
-          context.emit(SAVE_EVENT);
-          break;
-        case JSON_EVENT:
-          context.emit(JSON_EVENT);
-          break;
         case DELETE_EVENT:
           deleteDialog.value = true;
           break;
         default:
+          context.emit(event);
           break;
       }
     }

frontend/components/Domain/Recipe/RecipeDialogBulkAdd.vue

@@ -2,7 +2,7 @@
   <div class="text-center">
     <v-dialog v-model="dialog" width="800">
       <template #activator="{ on, attrs }">
-        <BaseButton v-bind="attrs" v-on="on" @click="inputText = ''">
+        <BaseButton v-bind="attrs" v-on="on" @click="inputText = inputTextProp">
           {{ $t("new-recipe.bulk-add") }}
         </BaseButton>
       </template>
@@ -58,10 +58,17 @@
 <script lang="ts">
 import { reactive, toRefs, defineComponent, useContext } from "@nuxtjs/composition-api";
 export default defineComponent({
-  setup(_, context) {
+  props: {
+    inputTextProp: {
+      type: String,
+      required: false,
+      default: "",
+    },
+  },
+  setup(props, context) {
     const state = reactive({
       dialog: false,
-      inputText: "",
+      inputText: props.inputTextProp,
     });
 
     function splitText() {

frontend/components/Domain/Recipe/RecipeIngredientEditor.vue

@@ -8,6 +8,7 @@
       class="mx-1 mt-3 mb-4"
       :placeholder="$t('recipe.section-title')"
       style="max-width: 500px"
+      @click="$emit('clickIngredientField', 'title')"
     >
     </v-text-field>
     <v-row :no-gutters="$vuetify.breakpoint.mdAndUp" dense class="d-flex flex-wrap my-1">
@@ -81,7 +82,15 @@
       </v-col>
       <v-col sm="12" md="" cols="12">
         <div class="d-flex">
-          <v-text-field v-model="value.note" hide-details dense solo class="mx-1" :placeholder="$t('recipe.notes')">
+          <v-text-field
+            v-model="value.note"
+            hide-details
+            dense
+            solo
+            class="mx-1"
+            :placeholder="$t('recipe.notes')"
+            @click="$emit('clickIngredientField', 'note')"
+          >
             <v-icon v-if="disableAmount && $listeners && $listeners.delete" slot="prepend" class="mr-n1 handle">
               {{ $globals.icons.arrowUpDown }}
             </v-icon>
@@ -93,12 +102,12 @@
             :buttons="[
               {
                 icon: $globals.icons.delete,
-                text: $t('general.delete'),
+                text: $tc('general.delete'),
                 event: 'delete',
               },
               {
                 icon: $globals.icons.dotsVertical,
-                text: $t('general.menu'),
+                text: $tc('general.menu'),
                 event: 'open',
                 children: contextMenuOptions,
               },

frontend/components/Domain/Recipe/RecipeInstructions.vue

@@ -176,6 +176,7 @@
                   blur: imageUploadMode,
                 }"
                 @drop.stop.prevent="handleImageDrop(index, $event)"
+                @click="$emit('clickInstructionField', `${index}.text`)"
               >
                 <MarkdownEditor
                   v-model="value[index]['text']"