diff --git a/app/BUILD.gn b/app/BUILD.gn index 9b569e65c42a..936e92f32230 100644 --- a/app/BUILD.gn +++ b/app/BUILD.gn @@ -6,6 +6,7 @@ import("//brave/browser/shell_integrations/buildflags/buildflags.gni") import("//brave/components/brave_vpn/common/buildflags/buildflags.gni") import("//brave/components/speedreader/common/buildflags/buildflags.gni") +import("//brave/components/text_recognition/common/buildflags/buildflags.gni") import("//brave/resources/brave_grit.gni") import("//build/config/features.gni") import("//build/config/locales.gni") @@ -19,6 +20,7 @@ brave_grit("brave_generated_resources_grit") { "enable_speedreader=$enable_speedreader", "enable_brave_vpn=$enable_brave_vpn", "enable_pin_shortcut=$enable_pin_shortcut", + "enable_text_recognition=$enable_text_recognition", ] source = "brave_generated_resources.grd" output_dir = "$root_gen_dir/brave" diff --git a/app/brave_command_ids.h b/app/brave_command_ids.h index 82cb09673e50..78e947e7d5d5 100644 --- a/app/brave_command_ids.h +++ b/app/brave_command_ids.h @@ -51,6 +51,7 @@ #define IDC_COPY_CLEAN_LINK 56040 #define IDC_TOGGLE_TAB_MUTE 56041 #define IDC_SIDEBAR_TOGGLE_POSITION 56042 +#define IDC_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE 56043 #define IDC_CONTENT_CONTEXT_IMPORT_IPNS_KEYS_START 56100 #define IDC_CONTENT_CONTEXT_IMPORT_IPNS_KEYS_END 56199 diff --git a/app/brave_generated_resources.grd b/app/brave_generated_resources.grd index 8fc645b07260..2e7c1393fae6 100644 --- a/app/brave_generated_resources.grd +++ b/app/brave_generated_resources.grd @@ -896,6 +896,25 @@ Or change later at $2brave://settings/ext + + + + Copy Text From Image + + + Close + + + Copying text from image... + + + Text copied from image + + + Text copy failed + + + diff --git a/browser/ui/BUILD.gn b/browser/ui/BUILD.gn index 46e9454e533a..64d47edbf4c1 100644 --- a/browser/ui/BUILD.gn +++ b/browser/ui/BUILD.gn @@ -12,6 +12,7 @@ import("//brave/components/ipfs/buildflags/buildflags.gni") import("//brave/components/ntp_background_images/buildflags/buildflags.gni") import("//brave/components/playlist/common/buildflags/buildflags.gni") import("//brave/components/speedreader/common/buildflags/buildflags.gni") +import("//brave/components/text_recognition/common/buildflags/buildflags.gni") import("//brave/components/tor/buildflags/buildflags.gni") import("//build/config/features.gni") import("//chrome/common/features.gni") @@ -443,6 +444,15 @@ source_set("ui") { ] } + if (enable_text_recognition) { + sources += [ + "views/text_recognition_dialog_view.cc", + "views/text_recognition_dialog_view.h", + ] + + deps += [ "//brave/components/text_recognition/browser" ] + } + # This is no longer compiled into Chromium on Android, but we still # need it if (is_android) { diff --git a/browser/ui/browser_dialogs.h b/browser/ui/browser_dialogs.h index 2eb4b26549cb..a4e3d6e408ce 100644 --- a/browser/ui/browser_dialogs.h +++ b/browser/ui/browser_dialogs.h @@ -9,6 +9,11 @@ #include "base/callback_forward.h" class Browser; +class SkBitmap; + +namespace content { +class WebContents; +} // namespace content namespace brave { @@ -18,6 +23,10 @@ void ShowCrashReportPermissionAskDialog(Browser* browser); // Run |callback| when dialog closed. void ShowObsoleteSystemConfirmDialog(base::OnceCallback callback); +// Show web modal dialog for showing text that recognized from |image|. +void ShowTextRecognitionDialog(content::WebContents* web_contents, + const SkBitmap& image); + } // namespace brave #endif // BRAVE_BROWSER_UI_BROWSER_DIALOGS_H_ diff --git a/browser/ui/views/text_recognition_dialog_view.cc b/browser/ui/views/text_recognition_dialog_view.cc new file mode 100644 index 000000000000..c47374c514b1 --- /dev/null +++ b/browser/ui/views/text_recognition_dialog_view.cc @@ -0,0 +1,126 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "brave/browser/ui/views/text_recognition_dialog_view.h" + +#include + +#include "base/bind.h" +#include "base/strings/utf_string_conversions.h" +#include "base/task/task_traits.h" +#include "base/task/thread_pool.h" +#include "brave/components/l10n/common/localization_util.h" +#include "brave/components/text_recognition/browser/text_recognition.h" +#include "brave/grit/brave_generated_resources.h" +#include "components/constrained_window/constrained_window_views.h" +#include "ui/base/clipboard/scoped_clipboard_writer.h" +#include "ui/base/metadata/metadata_impl_macros.h" +#include "ui/gfx/geometry/insets.h" +#include "ui/views/controls/label.h" +#include "ui/views/controls/scroll_view.h" +#include "ui/views/layout/flex_layout.h" +#include "ui/views/widget/widget.h" +#include "ui/views/window/dialog_client_view.h" + +namespace brave { + +void ShowTextRecognitionDialog(content::WebContents* web_contents, + const SkBitmap& image) { + constrained_window::ShowWebModalDialogViews( + new TextRecognitionDialogView(image), web_contents) + ->Show(); +} + +} // namespace brave + +TextRecognitionDialogView::TextRecognitionDialogView(const SkBitmap& image) { + SetModalType(ui::MODAL_TYPE_CHILD); + SetButtons(ui::DIALOG_BUTTON_OK); + SetButtonLabel(ui::DIALOG_BUTTON_OK, + brave_l10n::GetLocalizedResourceUTF16String( + IDS_TEXT_RECOG_DIALOG_CLOSE_BUTTON)); + SetShowCloseButton(false); + + SetLayoutManager(std::make_unique()) + ->SetOrientation(views::LayoutOrientation::kVertical) + .SetMainAxisAlignment(views::LayoutAlignment::kStart) + .SetInteriorMargin(gfx::Insets::TLBR(24, 26, 0, 26)); + + header_label_ = AddChildView(std::make_unique()); + const int size_diff = 14 - views::Label::GetDefaultFontList().GetFontSize(); + header_label_->SetFontList( + views::Label::GetDefaultFontList() + .DeriveWithSizeDelta(size_diff) + .DeriveWithWeight(gfx::Font::Weight::SEMIBOLD)); + header_label_->SetHorizontalAlignment(gfx::ALIGN_LEFT); + header_label_->SetProperty(views::kMarginsKey, + gfx::Insets::TLBR(0, 0, 10, 0)); + + if (image.empty()) { + UpdateContents({}); + return; + } + + StartExtractingText(image); +} + +TextRecognitionDialogView::~TextRecognitionDialogView() = default; + +void TextRecognitionDialogView::StartExtractingText(const SkBitmap& image) { + header_label_->SetText(brave_l10n::GetLocalizedResourceUTF16String( + IDS_TEXT_RECOG_DIALOG_HEADER_IN_PROGRESS)); + + base::ThreadPool::PostTaskAndReplyWithResult( + FROM_HERE, + {base::MayBlock(), base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN}, + base::BindOnce(&GetTextFromImage, image), + base::BindOnce(&TextRecognitionDialogView::OnGetTextFromImage, + weak_factory_.GetWeakPtr())); +} + +void TextRecognitionDialogView::OnGetTextFromImage( + const std::vector& text) { + UpdateContents(text); + AdjustWidgetSize(); +} + +void TextRecognitionDialogView::UpdateContents( + const std::vector& text) { + if (text.empty()) { + header_label_->SetText(brave_l10n::GetLocalizedResourceUTF16String( + IDS_TEXT_RECOG_DIALOG_HEADER_FAILED)); + return; + } + + header_label_->SetText(brave_l10n::GetLocalizedResourceUTF16String( + IDS_TEXT_RECOG_DIALOG_HEADER_COMPLETE)); + + // Treat each string in |text| as a separated line string. + std::string unified_string; + for (const auto& t : text) { + unified_string += t; + unified_string += '\n'; + } + + ui::ScopedClipboardWriter(ui::ClipboardBuffer::kCopyPaste) + .WriteText(base::UTF8ToUTF16(unified_string)); + + auto* scroll_view = AddChildView(std::make_unique()); + scroll_view->SetProperty(views::kMarginsKey, gfx::Insets::VH(0, 10)); + scroll_view->ClipHeightTo(0, 350); + auto* label = scroll_view->SetContents( + std::make_unique(base::UTF8ToUTF16(unified_string))); + + label->SetHorizontalAlignment(gfx::ALIGN_LEFT); + label->SetSelectable(true); + label->SetMultiLine(true); +} + +void TextRecognitionDialogView::AdjustWidgetSize() { + GetWidget()->SetSize(GetDialogClientView()->GetPreferredSize()); +} + +BEGIN_METADATA(TextRecognitionDialogView, views::DialogDelegateView) +END_METADATA diff --git a/browser/ui/views/text_recognition_dialog_view.h b/browser/ui/views/text_recognition_dialog_view.h new file mode 100644 index 000000000000..12d7151fd271 --- /dev/null +++ b/browser/ui/views/text_recognition_dialog_view.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef BRAVE_BROWSER_UI_VIEWS_TEXT_RECOGNITION_DIALOG_VIEW_H_ +#define BRAVE_BROWSER_UI_VIEWS_TEXT_RECOGNITION_DIALOG_VIEW_H_ + +#include +#include + +#include "base/memory/raw_ptr.h" +#include "base/memory/weak_ptr.h" +#include "ui/base/metadata/metadata_header_macros.h" +#include "ui/views/window/dialog_delegate.h" + +class SkBitmap; + +namespace views { +class Label; +} // namespace views + +class TextRecognitionDialogView : public views::DialogDelegateView { + public: + METADATA_HEADER(TextRecognitionDialogView); + + explicit TextRecognitionDialogView(const SkBitmap& image); + TextRecognitionDialogView(const TextRecognitionDialogView&) = delete; + TextRecognitionDialogView& operator=(const TextRecognitionDialogView&) = + delete; + ~TextRecognitionDialogView() override; + + private: + void StartExtractingText(const SkBitmap& image); + void OnGetTextFromImage(const std::vector& text); + + // Show |text| in this dialog and copy it to clipboard. + void UpdateContents(const std::vector& text); + void AdjustWidgetSize(); + + raw_ptr header_label_ = nullptr; + base::WeakPtrFactory weak_factory_{this}; +}; + +#endif // BRAVE_BROWSER_UI_VIEWS_TEXT_RECOGNITION_DIALOG_VIEW_H_ diff --git a/chromium_src/chrome/browser/DEPS b/chromium_src/chrome/browser/DEPS index 50541737a6c4..3cd1a4f6d5a0 100644 --- a/chromium_src/chrome/browser/DEPS +++ b/chromium_src/chrome/browser/DEPS @@ -32,6 +32,7 @@ include_rules = [ "+brave/components/privacy_sandbox", "+brave/components/sidebar", "+brave/components/sync", + "+brave/components/text_recognition", "+brave/components/tor", "+brave/components/translate", "+brave/components/url_sanitizer", diff --git a/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc b/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc index 5e9ec53a00b6..0353940eac07 100644 --- a/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc +++ b/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc @@ -1,6 +1,7 @@ -// Copyright 2018 The Brave Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. +/* Copyright (c) 2018 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ #include "chrome/browser/renderer_context_menu/render_view_context_menu.h" @@ -9,6 +10,7 @@ #include "brave/browser/profiles/profile_util.h" #include "brave/browser/renderer_context_menu/brave_spelling_options_submenu_observer.h" #include "brave/browser/ui/browser_commands.h" +#include "brave/browser/ui/browser_dialogs.h" #include "brave/components/ipfs/buildflags/buildflags.h" #include "brave/components/tor/buildflags/buildflags.h" #include "brave/grit/brave_theme_resources.h" @@ -137,6 +139,16 @@ void OnTorProfileCreated(const GURL& link_url, #endif +#if BUILDFLAG(ENABLE_TEXT_RECOGNITION) +void OnGetImage(base::WeakPtr web_contents, + const SkBitmap& image) { + if (!web_contents) + return; + + brave::ShowTextRecognitionDialog(web_contents.get(), image); +} +#endif + } // namespace BraveRenderViewContextMenu::BraveRenderViewContextMenu( @@ -152,6 +164,10 @@ BraveRenderViewContextMenu::BraveRenderViewContextMenu( bool BraveRenderViewContextMenu::IsCommandIdEnabled(int id) const { switch (id) { +#if BUILDFLAG(ENABLE_TEXT_RECOGNITION) + case IDC_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE: + return params_.has_image_contents; +#endif case IDC_COPY_CLEAN_LINK: return params_.link_url.is_valid(); case IDC_CONTENT_CONTEXT_FORCE_PASTE: @@ -249,12 +265,27 @@ void BraveRenderViewContextMenu::ExecuteCommand(int id, int event_flags) { base::BindRepeating(OnTorProfileCreated, params_.link_url, HasAlreadyOpenedTorWindow(GetProfile()))); break; +#endif +#if BUILDFLAG(ENABLE_TEXT_RECOGNITION) + case IDC_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE: + CopyTextFromImage(); + break; #endif default: RenderViewContextMenu_Chromium::ExecuteCommand(id, event_flags); } } +#if BUILDFLAG(ENABLE_TEXT_RECOGNITION) +void BraveRenderViewContextMenu::CopyTextFromImage() { + RenderFrameHost* frame_host = GetRenderFrameHost(); + if (frame_host) + frame_host->GetImageAt( + params_.x, params_.y, + base::BindOnce(OnGetImage, source_web_contents_->GetWeakPtr())); +} +#endif + void BraveRenderViewContextMenu::AddSpellCheckServiceItem(bool is_checked) { // Call our implementation, not the one in the base class. // Assumption: @@ -382,6 +413,18 @@ void BraveRenderViewContextMenu::InitMenu() { IDC_CONTENT_CONTEXT_FORCE_PASTE, IDS_CONTENT_CONTEXT_FORCE_PASTE); } +#if BUILDFLAG(ENABLE_TEXT_RECOGNITION) + bool media_image = content_type_->SupportsGroup( + ContextMenuContentType::ITEM_GROUP_MEDIA_IMAGE); + if (media_image) { + index = + menu_model_.GetIndexOfCommandId(IDC_CONTENT_CONTEXT_COPYIMAGELOCATION); + DCHECK(index); + menu_model_.InsertItemWithStringIdAt( + index.value() + 1, IDC_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE, + IDS_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE); + } +#endif #if BUILDFLAG(ENABLE_TOR) // Add Open Link with Tor diff --git a/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.h b/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.h index aafab8fc6d05..7b5136edccb7 100644 --- a/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.h +++ b/chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.h @@ -1,11 +1,13 @@ -// Copyright 2018 The Brave Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. +/* Copyright (c) 2018 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ #ifndef BRAVE_CHROMIUM_SRC_CHROME_BROWSER_RENDERER_CONTEXT_MENU_RENDER_VIEW_CONTEXT_MENU_H_ #define BRAVE_CHROMIUM_SRC_CHROME_BROWSER_RENDERER_CONTEXT_MENU_RENDER_VIEW_CONTEXT_MENU_H_ #include "brave/components/ipfs/buildflags/buildflags.h" +#include "brave/components/text_recognition/common/buildflags/buildflags.h" #define BRAVE_RENDER_VIEW_CONTEXT_MENU_H_ \ private: \ @@ -53,6 +55,10 @@ class BraveRenderViewContextMenu : public RenderViewContextMenu_Chromium { ui::SimpleMenuModel ipfs_submenu_model_; #endif + +#if BUILDFLAG(ENABLE_TEXT_RECOGNITION) + void CopyTextFromImage(); +#endif }; // Use our own subclass as the real RenderViewContextMenu. diff --git a/chromium_src/chrome/browser/sources.gni b/chromium_src/chrome/browser/sources.gni index 021fc8e8cc15..2c68a4644304 100644 --- a/chromium_src/chrome/browser/sources.gni +++ b/chromium_src/chrome/browser/sources.gni @@ -4,12 +4,14 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import("//brave/components/brave_vpn/common/buildflags/buildflags.gni") +import("//brave/components/text_recognition/common/buildflags/buildflags.gni") import("//build/config/ui.gni") brave_chromium_src_chrome_browser_deps = [ "//base", "//brave/components/brave_vpn/common/buildflags", "//brave/components/playlist/common/buildflags", + "//brave/components/text_recognition/common/buildflags", "//chrome/common:channel_info", "//components/version_info", ] diff --git a/chromium_src/content/browser/renderer_host/render_frame_host_impl.cc b/chromium_src/content/browser/renderer_host/render_frame_host_impl.cc new file mode 100644 index 000000000000..e5ea308c39a5 --- /dev/null +++ b/chromium_src/content/browser/renderer_host/render_frame_host_impl.cc @@ -0,0 +1,20 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "src/content/browser/renderer_host/render_frame_host_impl.cc" + +namespace content { + +void RenderFrameHostImpl::GetImageAt( + int x, + int y, + base::OnceCallback callback) { + gfx::PointF point_in_view = + GetView()->TransformRootPointToViewCoordSpace(gfx::PointF(x, y)); + GetAssociatedLocalFrame()->GetImageAt( + gfx::Point(point_in_view.x(), point_in_view.y()), std::move(callback)); +} + +} // namespace content diff --git a/chromium_src/content/browser/renderer_host/render_frame_host_impl.h b/chromium_src/content/browser/renderer_host/render_frame_host_impl.h new file mode 100644 index 000000000000..e54337b8bbc8 --- /dev/null +++ b/chromium_src/content/browser/renderer_host/render_frame_host_impl.h @@ -0,0 +1,21 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef BRAVE_CHROMIUM_SRC_CONTENT_BROWSER_RENDERER_HOST_RENDER_FRAME_HOST_IMPL_H_ +#define BRAVE_CHROMIUM_SRC_CONTENT_BROWSER_RENDERER_HOST_RENDER_FRAME_HOST_IMPL_H_ + +#include "content/public/browser/render_frame_host.h" +#include "third_party/blink/public/mojom/frame/frame.mojom.h" + +#define CopyImageAt \ + GetImageAt(int x, int y, base::OnceCallback callback) \ + override; \ + void CopyImageAt + +#include "src/content/browser/renderer_host/render_frame_host_impl.h" + +#undef CopyImageAt + +#endif // BRAVE_CHROMIUM_SRC_CONTENT_BROWSER_RENDERER_HOST_RENDER_FRAME_HOST_IMPL_H_ diff --git a/chromium_src/content/public/browser/render_frame_host.h b/chromium_src/content/public/browser/render_frame_host.h new file mode 100644 index 000000000000..61b78521c2e5 --- /dev/null +++ b/chromium_src/content/public/browser/render_frame_host.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef BRAVE_CHROMIUM_SRC_CONTENT_PUBLIC_BROWSER_RENDER_FRAME_HOST_H_ +#define BRAVE_CHROMIUM_SRC_CONTENT_PUBLIC_BROWSER_RENDER_FRAME_HOST_H_ + +class SkBitmap; + +#define CopyImageAt \ + GetImageAt(int x, int y, \ + base::OnceCallback callback) = 0; \ + virtual void CopyImageAt + +#include "src/content/public/browser/render_frame_host.h" + +#undef CopyImageAt + +#endif // BRAVE_CHROMIUM_SRC_CONTENT_PUBLIC_BROWSER_RENDER_FRAME_HOST_H_ diff --git a/chromium_src/third_party/blink/public/mojom/frame/frame.mojom b/chromium_src/third_party/blink/public/mojom/frame/frame.mojom new file mode 100644 index 000000000000..62348b2032f7 --- /dev/null +++ b/chromium_src/third_party/blink/public/mojom/frame/frame.mojom @@ -0,0 +1,15 @@ +// Copyright (c) 2023 The Brave Authors. All rights reserved. +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +module blink.mojom; + +import "skia/public/mojom/bitmap.mojom"; +import "ui/gfx/geometry/mojom/geometry.mojom"; + +[BraveExtend] +interface LocalFrame { + GetImageAt(gfx.mojom.Point window_point) + => (skia.mojom.BitmapN32? bitmap_image); +}; diff --git a/chromium_src/third_party/blink/renderer/core/frame/local_frame.cc b/chromium_src/third_party/blink/renderer/core/frame/local_frame.cc index b506dc4d9e5b..af3f5f263677 100644 --- a/chromium_src/third_party/blink/renderer/core/frame/local_frame.cc +++ b/chromium_src/third_party/blink/renderer/core/frame/local_frame.cc @@ -6,7 +6,11 @@ #include "third_party/blink/renderer/core/frame/local_frame.h" #include "brave/components/brave_page_graph/common/buildflags.h" +#include "skia/ext/skia_utils_base.h" +#include "third_party/blink/renderer/core/html/canvas/html_canvas_element.h" +#include "third_party/blink/renderer/core/layout/layout_image.h" #include "third_party/blink/renderer/core/probe/core_probes.h" +#include "third_party/blink/renderer/platform/graphics/graphics_types_3d.h" #if BUILDFLAG(ENABLE_BRAVE_PAGE_GRAPH) #include "brave/third_party/blink/renderer/core/brave_page_graph/page_graph.h" @@ -23,4 +27,79 @@ #include "src/third_party/blink/renderer/core/frame/local_frame.cc" +namespace blink { + +namespace { + +static scoped_refptr ImageFromNode(const Node& node) { + DCHECK(!node.GetDocument().NeedsLayoutTreeUpdate()); + DocumentLifecycle::DisallowTransitionScope disallow_transition( + node.GetDocument().Lifecycle()); + + const LayoutObject* const layout_object = node.GetLayoutObject(); + if (!layout_object) + return nullptr; + + if (layout_object->IsCanvas()) { + return To(const_cast(node)) + .Snapshot(kFrontBuffer); + } + + if (!layout_object->IsImage()) + return nullptr; + + const auto& layout_image = To(*layout_object); + const ImageResourceContent* const cached_image = layout_image.CachedImage(); + if (!cached_image || cached_image->ErrorOccurred()) + return nullptr; + return cached_image->GetImage(); +} + +} // namespace + +SkBitmap LocalFrame::GetImageAtViewportPoint(const gfx::Point& viewport_point) { + HitTestResult result = HitTestResultForVisualViewportPos(viewport_point); + if (!IsA(result.InnerNodeOrImageMapImage()) && + result.AbsoluteImageURL().IsEmpty()) { + // There isn't actually an image at these coordinates. Might be because + // the window scrolled while the context menu was open or because the page + // changed itself between when we thought there was an image here and when + // we actually tried to retrieve the image. + // + // FIXME: implement a cache of the most recent HitTestResult to avoid having + // to do two hit tests. + return {}; + } + + const scoped_refptr image = + ImageFromNode(*result.InnerNodeOrImageMapImage()); + if (!image.get()) + return {}; + + PaintImage paint_image = image->PaintImageForCurrentFrame(); + // Orient the data. + if (!image->HasDefaultOrientation()) { + paint_image = Image::ResizeAndOrientImage( + paint_image, image->CurrentFrameOrientation(), gfx::Vector2dF(1, 1), 1, + kInterpolationNone); + } + SkBitmap bitmap; + if (sk_sp sk_image = paint_image.GetSwSkImage()) + sk_image->asLegacyBitmap(&bitmap); + + // The bitmap backing a canvas can be in non-native skia pixel order (aka + // RGBA when kN32_SkColorType is BGRA-ordered, or higher bit-depth color-types + // like F16. The IPC to the browser requires the bitmap to be in N32 format + // so we convert it here if needed. + SkBitmap n32_bitmap; + if (skia::SkBitmapToN32OpaqueOrPremul(bitmap, &n32_bitmap) && + !n32_bitmap.isNull()) { + return n32_bitmap; + } + + return {}; +} + +} // namespace blink + #undef FrameAttachedToParent diff --git a/chromium_src/third_party/blink/renderer/core/frame/local_frame.h b/chromium_src/third_party/blink/renderer/core/frame/local_frame.h new file mode 100644 index 000000000000..f2090d5d8591 --- /dev/null +++ b/chromium_src/third_party/blink/renderer/core/frame/local_frame.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_H_ +#define BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_H_ + +class SkBitmap; + +#define CopyImageAtViewportPoint \ + CopyImageAtViewportPoint_UnUsed() {} \ + SkBitmap GetImageAtViewportPoint(const gfx::Point& viewport_point); \ + void CopyImageAtViewportPoint + +#include "src/third_party/blink/renderer/core/frame/local_frame.h" + +#undef CopyImageAtViewportPoint + +#endif // BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_H_ diff --git a/chromium_src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.cc b/chromium_src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.cc new file mode 100644 index 000000000000..78d2736888ac --- /dev/null +++ b/chromium_src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.cc @@ -0,0 +1,17 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.cc" + +namespace blink { + +void LocalFrameMojoHandler::GetImageAt(const gfx::Point& window_point, + GetImageAtCallback callback) { + gfx::Point viewport_position = + frame_->GetWidgetForLocalRoot()->DIPsToRoundedBlinkSpace(window_point); + std::move(callback).Run(frame_->GetImageAtViewportPoint(viewport_position)); +} + +} // namespace blink diff --git a/chromium_src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.h b/chromium_src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.h new file mode 100644 index 000000000000..bd3314d45fd7 --- /dev/null +++ b/chromium_src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_MOJO_HANDLER_H_ +#define BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_MOJO_HANDLER_H_ + +#include "third_party/blink/public/mojom/frame/frame.mojom-blink.h" + +#define CopyImageAt \ + GetImageAt(const gfx::Point& window_point, GetImageAtCallback callback) \ + final; \ + void CopyImageAt + +#include "src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.h" + +#undef CopyImageAt + +#endif // BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_MOJO_HANDLER_H_ diff --git a/components/text_recognition/DEPS b/components/text_recognition/DEPS new file mode 100644 index 000000000000..833ba2c614eb --- /dev/null +++ b/components/text_recognition/DEPS @@ -0,0 +1,5 @@ +include_rules = [ + "+base", + "+skia", + "+third_party/skia", +] diff --git a/components/text_recognition/browser/BUILD.gn b/components/text_recognition/browser/BUILD.gn new file mode 100644 index 000000000000..8c7228dadc6a --- /dev/null +++ b/components/text_recognition/browser/BUILD.gn @@ -0,0 +1,28 @@ +# Copyright (c) 2023 The Brave Authors. All rights reserved. +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. + +import("//brave/components/text_recognition/common/buildflags/buildflags.gni") + +assert(enable_text_recognition) + +source_set("browser") { + sources = [ "text_recognition.h" ] + + if (is_mac) { + sources += [ "text_recognition.mm" ] + + frameworks = [ + "Foundation.framework", + "Vision.framework", + ] + } + + deps = [ + "//base", + "//skia", + ] + + configs += [ "//build/config/compiler:enable_arc" ] +} diff --git a/components/text_recognition/browser/text_recognition.h b/components/text_recognition/browser/text_recognition.h new file mode 100644 index 000000000000..0b9f20b7244a --- /dev/null +++ b/components/text_recognition/browser/text_recognition.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef BRAVE_COMPONENTS_TEXT_RECOGNITION_BROWSER_TEXT_RECOGNITION_H_ +#define BRAVE_COMPONENTS_TEXT_RECOGNITION_BROWSER_TEXT_RECOGNITION_H_ + +#include +#include + +class SkBitmap; + +// Returns recognized texts from |image|. +std::vector GetTextFromImage(const SkBitmap& image); + +#endif // BRAVE_COMPONENTS_TEXT_RECOGNITION_BROWSER_TEXT_RECOGNITION_H_ diff --git a/components/text_recognition/browser/text_recognition.mm b/components/text_recognition/browser/text_recognition.mm new file mode 100644 index 000000000000..5cb87b65a22f --- /dev/null +++ b/components/text_recognition/browser/text_recognition.mm @@ -0,0 +1,75 @@ +/* Copyright (c) 2023 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "brave/components/text_recognition/browser/text_recognition.h" + +#import +#import + +#include "base/logging.h" +#include "base/mac/foundation_util.h" +#include "base/mac/mac_util.h" +#include "base/mac/scoped_cftyperef.h" +#include "base/strings/sys_string_conversions.h" +#include "skia/ext/skia_utils_base.h" +#include "skia/ext/skia_utils_mac.h" +#include "third_party/skia/include/core/SkBitmap.h" +#include "third_party/skia/include/utils/mac/SkCGUtils.h" + +std::vector GetTextFromImage(const SkBitmap& image) { + std::vector result; + if (@available(macOS 10.15, *)) { + // The bitmap type is sanitized to be N32 before we get here. The conversion + // to an NSImage would not explode if we got this wrong, so this is not a + // security CHECK. + DCHECK_EQ(image.colorType(), kN32_SkColorType); + + auto* p_result = &result; + VNRecognizeTextRequest* textRecognitionRequest = + [[VNRecognizeTextRequest alloc] initWithCompletionHandler:^( + VNRequest* _Nonnull request, + NSError* _Nullable error) { + NSArray* observations = request.results; + + [observations enumerateObjectsUsingBlock:^( + VNRecognizedTextObservation* _Nonnull obj, + NSUInteger idx, BOOL* _Nonnull stop) { + // Ask first top candidate for a recognized text string. + VNRecognizedText* recognizedText = + [obj topCandidates:1].firstObject; + + p_result->push_back( + base::SysNSStringToUTF8([recognizedText string])); + }]; + }]; + + textRecognitionRequest.recognitionLevel = + VNRequestTextRecognitionLevelAccurate; + textRecognitionRequest.usesLanguageCorrection = true; + // Copied FF's supported language list. + // See https://support.mozilla.org/en-US/kb/text-recognition + // English, Chinese, Portuguese, French, Italian, German, and Spanish + textRecognitionRequest.recognitionLanguages = @[ + @"en-US", @"zh-Hans", @"zh-Hant", @"pt-BR", @"fr-FR", @"it-IT", @"de-DE", + @"es-ES" + ]; + + if (@available(macOS 13.0, *)) { + textRecognitionRequest.automaticallyDetectsLanguage = true; + } + + NSError* error = nil; + base::ScopedCFTypeRef cg_image(SkCreateCGImageRef(image)); + VNImageRequestHandler* requestHandler = + [[VNImageRequestHandler alloc] initWithCGImage:cg_image.get() + options:@{}]; + [requestHandler performRequests:@[ textRecognitionRequest ] error:&error]; + if (error) { + LOG(ERROR) << base::SysNSStringToUTF8([error localizedDescription]); + } + } + + return result; +} diff --git a/components/text_recognition/common/buildflags/BUILD.gn b/components/text_recognition/common/buildflags/BUILD.gn new file mode 100644 index 000000000000..f2c61f4b526b --- /dev/null +++ b/components/text_recognition/common/buildflags/BUILD.gn @@ -0,0 +1,12 @@ +# Copyright (c) 2023 The Brave Authors. All rights reserved. +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. + +import("//brave/components/text_recognition/common/buildflags/buildflags.gni") +import("//build/buildflag_header.gni") + +buildflag_header("buildflags") { + header = "buildflags.h" + flags = [ "ENABLE_TEXT_RECOGNITION=$enable_text_recognition" ] +} diff --git a/components/text_recognition/common/buildflags/buildflags.gni b/components/text_recognition/common/buildflags/buildflags.gni new file mode 100644 index 000000000000..3ea4cd80e398 --- /dev/null +++ b/components/text_recognition/common/buildflags/buildflags.gni @@ -0,0 +1,8 @@ +# Copyright (c) 2023 The Brave Authors. All rights reserved. +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. + +declare_args() { + enable_text_recognition = is_mac +}