Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GSoC'24] Enhancement of Media Drop Feature in NoteEditor #16749

Merged
merged 2 commits into from
Aug 15, 2024
Merged

[GSoC'24] Enhancement of Media Drop Feature in NoteEditor #16749

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
935086e
rename 'image' to 'media'
SanjaySargam Jul 24, 2024
33498e8
note-editor: drop files
SanjaySargam Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AnkiDroid/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ dependencies {
implementation libs.androidx.appcompat
implementation libs.androidx.browser
implementation libs.androidx.core.ktx
implementation libs.androidx.draganddrop
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Regarding the commit message of the second commit, I'd suggest using > on the start of the lines, so that markdown displays it as a quote

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you need a space after > for it to work in markdown

implementation libs.androidx.exifinterface
implementation libs.androidx.fragment.ktx
implementation libs.androidx.media
Expand Down
6 changes: 6 additions & 0 deletions AnkiDroid/src/main/AndroidManifest.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"
android:installLocation="auto">

<!-- At the time of writing, ankidroid minSdk is 23. This library requires API 24.-->
<!-- In order to use it, we use <uses-sdk tools:overrideLibrary instead of-->
<!-- <uses-feature android:name. We must ensure this is never called when running on API 23.-->
<uses-sdk tools:overrideLibrary="androidx.draganddrop"/>

<uses-feature android:name="android.hardware.camera" android:required="false" />
<uses-feature android:name="android.hardware.camera.any" android:required="false" />
<uses-feature android:name="android.hardware.audio.output" android:required="false" />
Expand Down
66 changes: 9 additions & 57 deletions AnkiDroid/src/main/java/com/ichi2/anki/FieldEditText.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.ichi2.anki

import android.content.ClipDescription
import android.content.ClipboardManager
import android.content.Context
import android.graphics.drawable.Drawable
Expand All @@ -25,25 +26,18 @@ import android.os.LocaleList
import android.os.Parcelable
import android.text.InputType
import android.util.AttributeSet
import android.view.View
import android.view.inputmethod.EditorInfo
import android.view.inputmethod.InputConnection
import android.widget.EditText
import androidx.annotation.RequiresApi
import androidx.annotation.VisibleForTesting
import androidx.core.view.ContentInfoCompat
import androidx.core.view.OnReceiveContentListener
import androidx.core.view.ViewCompat
import androidx.core.view.inputmethod.EditorInfoCompat
import androidx.core.view.inputmethod.InputConnectionCompat
import com.google.android.material.color.MaterialColors
import com.ichi2.anki.preferences.sharedPrefs
import com.ichi2.anki.servicelayer.NoteService
import com.ichi2.ui.FixedEditText
import com.ichi2.utils.ClipboardUtil.IMAGE_MIME_TYPES
import com.ichi2.utils.ClipboardUtil.getDescription
import com.ichi2.utils.ClipboardUtil.getPlainText
import com.ichi2.utils.ClipboardUtil.getUri
import com.ichi2.utils.ClipboardUtil.hasImage
import com.ichi2.utils.ClipboardUtil.hasMedia
import com.ichi2.utils.KotlinCleanup
import kotlinx.parcelize.Parcelize
import timber.log.Timber
Expand Down Expand Up @@ -100,49 +94,6 @@ class FieldEditText : FixedEditText, NoteService.NoteField {
this.pasteListener = pasteListener
}

@KotlinCleanup("add extension method to iterate clip items")
override fun onCreateInputConnection(editorInfo: EditorInfo): InputConnection? {
david-allison marked this conversation as resolved.
Show resolved Hide resolved
val inputConnection = super.onCreateInputConnection(editorInfo) ?: return null
EditorInfoCompat.setContentMimeTypes(editorInfo, IMAGE_MIME_TYPES)
ViewCompat.setOnReceiveContentListener(
this,
IMAGE_MIME_TYPES,
object : OnReceiveContentListener {
override fun onReceiveContent(view: View, payload: ContentInfoCompat): ContentInfoCompat? {
val pair = payload.partition { item -> item.uri != null }
val uriContent = pair.first
val remaining = pair.second

if (pasteListener == null || uriContent == null) {
return remaining
}

val clip = uriContent.clip
val description = clip.description

if (!hasImage(description)) {
return remaining
}

for (i in 0 until clip.itemCount) {
val uri = clip.getItemAt(i).uri
try {
onPaste(uri)
} catch (e: Exception) {
Timber.w(e)
CrashReportService.sendExceptionReport(e, "NoteEditor::onImage")
return remaining
}
}

return remaining
}
}
)

return InputConnectionCompat.createWrapper(this, inputConnection, editorInfo)
}
david-allison marked this conversation as resolved.
Show resolved Hide resolved

override fun onSelectionChanged(selStart: Int, selEnd: Int) {
if (selectionChangeListener != null) {
try {
Expand Down Expand Up @@ -192,9 +143,10 @@ class FieldEditText : FixedEditText, NoteService.NoteField {

override fun onTextContextMenuItem(id: Int): Boolean {
// The current function is called both by Ctrl+V and pasting from the context menu
// It does not deal with drag and drop
if (id == android.R.id.paste) {
if (hasImage(clipboard)) {
return onPaste(getUri(clipboard))
if (hasMedia(clipboard)) {
return onPaste(getUri(clipboard), getDescription(clipboard))
}
return pastePlainText()
}
Expand All @@ -215,11 +167,11 @@ class FieldEditText : FixedEditText, NoteService.NoteField {
return false
}

private fun onPaste(mediaUri: Uri?): Boolean {
private fun onPaste(mediaUri: Uri?, description: ClipDescription?): Boolean {
return if (mediaUri == null) {
false
} else {
pasteListener!!.onPaste(this, mediaUri)
pasteListener!!.onPaste(this, mediaUri, description)
}
}

Expand Down Expand Up @@ -252,7 +204,7 @@ class FieldEditText : FixedEditText, NoteService.NoteField {
}

fun interface PasteListener {
fun onPaste(editText: EditText, uri: Uri?): Boolean
fun onPaste(editText: EditText, uri: Uri?, description: ClipDescription?): Boolean
}

companion object {
Expand Down
37 changes: 30 additions & 7 deletions AnkiDroid/src/main/java/com/ichi2/anki/MediaRegistration.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,17 @@
*/
package com.ichi2.anki

import android.content.ClipDescription
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.net.Uri
import androidx.annotation.CheckResult
import com.ichi2.anki.multimediacard.fields.ImageField
import com.ichi2.anki.multimediacard.fields.MediaClipField
import com.ichi2.compat.CompatHelper
import com.ichi2.libanki.exception.EmptyMediaException
import com.ichi2.utils.ClipboardUtil
import com.ichi2.utils.ContentResolverUtil.getFileName
import com.ichi2.utils.FileUtil.getFileNameAndExtension
import timber.log.Timber
Expand All @@ -47,7 +50,7 @@ class MediaRegistration(private val context: Context) {
* @return HTML referring to the loaded image
*/
@Throws(IOException::class)
fun loadMediaIntoCollection(uri: Uri): String? {
fun loadMediaIntoCollection(uri: Uri, description: ClipDescription): String? {
val fileName: String
val filename = getFileName(context.contentResolver, uri)
val fd = openInputStreamWithURI(uri)
Expand All @@ -59,9 +62,12 @@ class MediaRegistration(private val context: Context) {
}
var clipCopy: File
var bytesWritten: Long
val isImage = ClipboardUtil.hasImage(description)
val isVideo = ClipboardUtil.hasVideo(description)

openInputStreamWithURI(uri).use { copyFd ->
// no conversion to jpg in cases of gif and jpg and if png image with alpha channel
if (shouldConvertToJPG(fileNameAndExtension.value, copyFd)) {
if (shouldConvertToJPG(fileNameAndExtension.value, copyFd, isImage)) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd have considered to send the description directly to the function. So that you could use the mime type to check for SVG.

clipCopy = File.createTempFile(fileName, ".jpg")
bytesWritten = CompatHelper.compat.copyFile(fd, clipCopy.absolutePath)
// return null if jpg conversion false.
Expand All @@ -81,11 +87,22 @@ class MediaRegistration(private val context: Context) {
Timber.d("File was %d bytes", bytesWritten)
if (bytesWritten > MEDIA_MAX_SIZE) {
Timber.w("File was too large: %d bytes", bytesWritten)
showThemedToast(context, context.getString(R.string.note_editor_paste_too_large), false)
val message = if (isImage) {
context.getString(R.string.note_editor_image_too_large)
} else if (isVideo) {
context.getString(R.string.note_editor_video_too_large)
} else {
context.getString(R.string.note_editor_audio_too_large)
}
showThemedToast(context, message, false)
File(tempFilePath).delete()
return null
}
val field = ImageField()
val field = if (isImage) {
ImageField()
} else {
MediaClipField()
}
david-allison marked this conversation as resolved.
Show resolved Hide resolved
field.hasTemporaryMedia = true
field.mediaPath = tempFilePath
return field.formattedValue
Expand All @@ -112,7 +129,13 @@ class MediaRegistration(private val context: Context) {
return true // successful conversion to jpg.
}

private fun shouldConvertToJPG(fileNameExtension: String, fileStream: InputStream): Boolean {
private fun shouldConvertToJPG(fileNameExtension: String, fileStream: InputStream, isImage: Boolean): Boolean {
if (!isImage) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While you are touching this file, would you mind:

  • setting fileNameExtension to lowercase
  • add jpeg, which is also a valid for jpg
  • maybe simplify to:
        return when (fileNameExtension.lowercase()) {
            ".svg", ".jpg", ".jpeg", ".gif" -> false
            ".png" -> !doesInputStreamContainTransparency(fileStream)
            else -> true
        }

return false
}
if (".svg" == fileNameExtension) {
return false
}
if (".jpg" == fileNameExtension) {
return false // we are already a jpg, no conversion
}
Expand All @@ -129,11 +152,11 @@ class MediaRegistration(private val context: Context) {
return fileNameAndExtension.key.length <= 3
}

fun onPaste(uri: Uri): String? {
fun onPaste(uri: Uri, description: ClipDescription): String? {
return try {
// check if cache already holds registered file or not
if (!pastedMediaCache.containsKey(uri.toString())) {
pastedMediaCache[uri.toString()] = loadMediaIntoCollection(uri)
pastedMediaCache[uri.toString()] = loadMediaIntoCollection(uri, description)
}
pastedMediaCache[uri.toString()]
} catch (ex: NullPointerException) {
Expand Down
59 changes: 54 additions & 5 deletions AnkiDroid/src/main/java/com/ichi2/anki/NoteEditor.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import android.app.Activity
import android.app.Activity.RESULT_CANCELED
import android.content.BroadcastReceiver
import android.content.ClipData
import android.content.ClipDescription
import android.content.ClipboardManager
import android.content.Context
import android.content.Intent
Expand Down Expand Up @@ -67,7 +68,11 @@ import androidx.core.content.edit
import androidx.core.content.res.ResourcesCompat
import androidx.core.os.BundleCompat
import androidx.core.text.HtmlCompat
import androidx.core.util.component1
import androidx.core.util.component2
import androidx.core.view.OnReceiveContentListener
import androidx.core.view.isVisible
import androidx.draganddrop.DropHelper
import androidx.fragment.app.activityViewModels
import androidx.lifecycle.lifecycleScope
import anki.config.ConfigKey
Expand Down Expand Up @@ -126,6 +131,7 @@ import com.ichi2.anki.utils.ext.isImageOcclusion
import com.ichi2.anki.utils.ext.sharedPrefs
import com.ichi2.anki.widgets.DeckDropDownAdapter.SubtitleListener
import com.ichi2.annotations.NeedsTest
import com.ichi2.compat.CompatHelper
import com.ichi2.compat.CompatHelper.Companion.getSerializableCompat
import com.ichi2.compat.CompatHelper.Companion.registerReceiverCompat
import com.ichi2.libanki.Card
Expand All @@ -144,6 +150,8 @@ import com.ichi2.libanki.load
import com.ichi2.libanki.note
import com.ichi2.libanki.undoableOp
import com.ichi2.utils.ClipboardUtil
import com.ichi2.utils.ClipboardUtil.hasMedia
import com.ichi2.utils.ClipboardUtil.items
import com.ichi2.utils.HashUtil
import com.ichi2.utils.ImageUtils
import com.ichi2.utils.ImportUtils
Expand All @@ -169,7 +177,6 @@ import java.io.File
import java.util.LinkedList
import java.util.Locale
import java.util.function.Consumer
import kotlin.collections.ArrayList
import kotlin.math.max
import kotlin.math.min
import kotlin.math.roundToInt
Expand Down Expand Up @@ -333,6 +340,37 @@ class NoteEditor : AnkiFragment(R.layout.note_editor), DeckSelectionListener, Su
}
)

/**
* Listener for handling content received via drag and drop or copy and paste.
* This listener processes URIs contained in the payload and attempts to paste the content into the target EditText view.
*/
private val onReceiveContentListener = OnReceiveContentListener { view, payload ->
val (uriContent, remaining) = payload.partition { item -> item.uri != null }

if (uriContent == null) {
return@OnReceiveContentListener remaining
}

val clip = uriContent.clip
val description = clip.description

if (!hasMedia(description)) {
return@OnReceiveContentListener remaining
}

for (uri in clip.items().map { it.uri }) {
try {
onPaste(view as EditText, uri, description)
} catch (e: Exception) {
Timber.w(e)
CrashReportService.sendExceptionReport(e, "NoteEditor::onReceiveContent")
return@OnReceiveContentListener remaining
}
}

return@OnReceiveContentListener remaining
}

private inner class NoteEditorActivityResultCallback(private val callback: (result: ActivityResult) -> Unit) : ActivityResultCallback<ActivityResult> {
override fun onActivityResult(result: ActivityResult) {
Timber.d("onActivityResult() with result: %s", result.resultCode)
Expand Down Expand Up @@ -1592,12 +1630,23 @@ class NoteEditor : AnkiFragment(R.layout.note_editor), DeckSelectionListener, Su
val editLineView = editLines[i]
customViewIds.add(editLineView.id)
val newEditText = editLineView.editText
newEditText.setPasteListener { editText: EditText?, uri: Uri? ->
newEditText.setPasteListener { editText: EditText?, uri: Uri?, description: ClipDescription? ->
onPaste(
editText!!,
uri!!
uri!!,
description!!
)
}
CompatHelper.compat.configureView(
requireActivity(),
editLineView,
DropHelper.Options.Builder()
.setHighlightColor(R.color.material_lime_green_A700)
.setHighlightCornerRadiusPx(0)
.addInnerEditTexts(newEditText)
.build(),
onReceiveContentListener
)
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
if (i == 0) {
findViewById<View>(R.id.note_deck_spinner).nextFocusForwardId = newEditText.id
Expand Down Expand Up @@ -1835,8 +1884,8 @@ class NoteEditor : AnkiFragment(R.layout.note_editor), DeckSelectionListener, Su
}
}

private fun onPaste(editText: EditText, uri: Uri): Boolean {
val mediaTag = mediaRegistration!!.onPaste(uri) ?: return false
private fun onPaste(editText: EditText, uri: Uri, description: ClipDescription): Boolean {
val mediaTag = mediaRegistration!!.onPaste(uri, description) ?: return false
insertStringInField(editText, mediaTag)
return true
}
Expand Down
16 changes: 16 additions & 0 deletions AnkiDroid/src/main/java/com/ichi2/compat/Compat.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package com.ichi2.compat

import android.app.Activity
import android.content.Context
import android.content.Intent
import android.content.pm.PackageInfo
Expand All @@ -30,6 +31,8 @@ import android.net.Uri
import android.os.Bundle
import android.view.View
import androidx.annotation.CheckResult
import androidx.core.view.OnReceiveContentListener
import androidx.draganddrop.DropHelper
import java.io.File
import java.io.FileNotFoundException
import java.io.IOException
Expand Down Expand Up @@ -201,6 +204,19 @@ interface Compat {
@Throws(IOException::class)
fun contentOfDirectory(directory: File): FileStream

/**
* If possible, configures a [View] for drag and drop operations, including highlighting that
* indicates the view is a drop target. Sets a listener that enables the view to handle dropped data.
*
* @see DropHelper.configureView
*/
fun configureView(
activity: Activity,
view: View,
options: DropHelper.Options,
onReceiveContentListener: OnReceiveContentListener
)

/**
* Converts a locale to a 'two letter' code (ISO-639-1 + ISO 3166-1 alpha-2)
* Locale("spa", "MEX", "001") => Locale("es", "MX", "001")
Expand Down
Loading