Release overlay resources at end of frame processing

When implementing pause and resume, overlay appears as black frame on resume, suggesting the texture isn't being remade properly due to lack of proper deletion. PiperOrigin-RevId: 554832580
SimonHung · Aug 10, 2023 · 7be15b7 · 7be15b7
1 parent 5c78290
commit 7be15b7
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 2 deletions.
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java
@@ -22,6 +22,7 @@
 import android.net.Uri;
 import android.opengl.GLES20;
 import android.opengl.GLUtils;
+import androidx.media3.common.C;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.util.BitmapLoader;
 import androidx.media3.common.util.GlUtil;
@@ -31,6 +32,7 @@
 import com.google.common.util.concurrent.ListenableFuture;
 import java.util.concurrent.ExecutionException;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
 
 /**
  * Creates {@link TextureOverlay}s from {@link Bitmap}s.
@@ -40,7 +42,11 @@
 @UnstableApi
 public abstract class BitmapOverlay extends TextureOverlay {
   private int lastTextureId;
-  private @MonotonicNonNull Bitmap lastBitmap;
+  private @Nullable Bitmap lastBitmap;
+
+  BitmapOverlay() {
+    lastTextureId = C.INDEX_UNSET;
+  }
 
   /**
    * Returns the overlay bitmap displayed at the specified timestamp.
@@ -68,6 +74,9 @@ public int getTextureId(long presentationTimeUs) throws VideoFrameProcessingExce
     if (bitmap != lastBitmap) {
       try {
         lastBitmap = bitmap;
+        if (lastTextureId != -1) {
+          GlUtil.deleteTexture(lastTextureId);
+        }
         lastTextureId =
             GlUtil.createTexture(
                 bitmap.getWidth(),
@@ -77,7 +86,7 @@ public int getTextureId(long presentationTimeUs) throws VideoFrameProcessingExce
         GLUtils.texImage2D(
             GLES20.GL_TEXTURE_2D,
             /* level= */ 0,
-            BitmapUtil.flipBitmapVertically(lastBitmap),
+            BitmapUtil.flipBitmapVertically(checkNotNull(lastBitmap)),
             /* border= */ 0);
         GlUtil.checkGlError();
       } catch (GlUtil.GlException e) {
@@ -162,4 +171,18 @@ public OverlaySettings getOverlaySettings(long presentationTimeUs) {
       }
     };
   }
+
+  @Override
+  public void release() throws VideoFrameProcessingException {
+    super.release();
+    lastBitmap = null;
+    if (lastTextureId != -1) {
+      try {
+        GlUtil.deleteTexture(lastTextureId);
+      } catch (GlUtil.GlException e) {
+        throw new VideoFrameProcessingException(e);
+      }
+    }
+    lastTextureId = C.INDEX_UNSET;
+  }
 }
diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java
@@ -299,6 +299,9 @@ public void release() throws VideoFrameProcessingException {
     } catch (GlUtil.GlException e) {
       throw new VideoFrameProcessingException(e);
     }
+    for (int i = 0; i < overlays.size(); i++) {
+      overlays.get(i).release();
+    }
   }
 
   private static String createVertexShader(int numOverlays) {

diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureOverlay.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureOverlay.java
@@ -60,4 +60,11 @@ public void configure(Size videoSize) {}
   public OverlaySettings getOverlaySettings(long presentationTimeUs) {
     return new OverlaySettings.Builder().build();
   }
+
+  /**
+   * Releases all resources.
+   *
+   * @throws VideoFrameProcessingException If an error occurs while releasing resources.
+   */
+  public void release() throws VideoFrameProcessingException {}
 }