Update effect to take in and use a GlObjectsProvider

PiperOrigin-RevId: 514744747

demos/transformer/src/withMediaPipe/java/com/google/android/exoplayer2/transformerdemo/MediaPipeShaderProgram.java

@@ -25,6 +25,7 @@
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.effect.GlShaderProgram;
+import com.google.android.exoplayer2.util.GlObjectsProvider;
 import com.google.android.exoplayer2.util.GlTextureInfo;
 import com.google.android.exoplayer2.util.LibraryLoader;
 import com.google.android.exoplayer2.util.Util;
@@ -159,6 +160,9 @@ public void setErrorListener(Executor executor, ErrorListener errorListener) {
                 () -> errorListener.onError(new VideoFrameProcessingException(error))));
   }
 
+  @Override
+  public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {}
+
   @Override
   public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
     AppTextureFrame appTextureFrame =

library/common/src/main/java/com/google/android/exoplayer2/util/VideoFrameProcessor.java

@@ -41,6 +41,15 @@ public interface VideoFrameProcessor {
 
   /** A factory for {@link VideoFrameProcessor} instances. */
   interface Factory {
+
+    /**
+     * Sets the {@link GlObjectsProvider}.
+     *
+     * <p>Must be called before {@link #create}.
+     */
+    Factory setGlObjectsProvider(GlObjectsProvider glObjectsProvider);
+
+    // TODO(271433904): Turn parameters with default values into setters.
     /**
      * Creates a new {@link VideoFrameProcessor} instance.
      *

library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java

@@ -28,6 +28,7 @@
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.util.DebugViewProvider;
 import com.google.android.exoplayer2.util.FrameInfo;
+import com.google.android.exoplayer2.util.GlObjectsProvider;
 import com.google.android.exoplayer2.util.GlTextureInfo;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.SurfaceInfo;
@@ -410,6 +411,9 @@ public void setOutputListener(OutputListener outputListener) {
     @Override
     public void setErrorListener(Executor executor, ErrorListener errorListener) {}
 
+    @Override
+    public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {}
+
     @Override
     public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
       // No input is queued in these tests. The BlankFrameProducer is used to produce frames.

library/effect/src/main/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessor.java

@@ -35,6 +35,7 @@
 import com.google.android.exoplayer2.util.DebugViewProvider;
 import com.google.android.exoplayer2.util.Effect;
 import com.google.android.exoplayer2.util.FrameInfo;
+import com.google.android.exoplayer2.util.GlObjectsProvider;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.Log;
 import com.google.android.exoplayer2.util.SurfaceInfo;
@@ -60,6 +61,20 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
 
   /** A factory for {@link DefaultVideoFrameProcessor} instances. */
   public static final class Factory implements VideoFrameProcessor.Factory {
+    private GlObjectsProvider glObjectsProvider = GlObjectsProvider.DEFAULT;
+
+    /**
+     * {@inheritDoc}
+     *
+     * <p>The default value is {@link GlObjectsProvider#DEFAULT}.
+     */
+    @Override
+    public DefaultVideoFrameProcessor.Factory setGlObjectsProvider(
+        GlObjectsProvider glObjectsProvider) {
+      this.glObjectsProvider = glObjectsProvider;
+      return this;
+    }
+
     /**
      * {@inheritDoc}
      *
@@ -137,7 +152,8 @@ public DefaultVideoFrameProcessor create(
                       releaseFramesAutomatically,
                       singleThreadExecutorService,
                       listenerExecutor,
-                      listener));
+                      listener,
+                      glObjectsProvider));
 
       try {
         return defaultVideoFrameProcessorFuture.get();
@@ -365,7 +381,8 @@ private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor(
       boolean releaseFramesAutomatically,
       ExecutorService singleThreadExecutorService,
       Executor executor,
-      Listener listener)
+      Listener listener,
+      GlObjectsProvider glObjectsProvider)
       throws GlUtil.GlException, VideoFrameProcessingException {
     checkState(Thread.currentThread().getName().equals(THREAD_NAME));
 
@@ -378,7 +395,8 @@ private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor(
             : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888;
     int openGlVersion =
         ColorInfo.isTransferHdr(inputColorInfo) || ColorInfo.isTransferHdr(outputColorInfo) ? 3 : 2;
-    EGLContext eglContext = GlUtil.createEglContext(eglDisplay, openGlVersion, configAttributes);
+    EGLContext eglContext =
+        glObjectsProvider.createEglContext(eglDisplay, openGlVersion, configAttributes);
     GlUtil.createFocusedPlaceholderEglSurface(eglContext, eglDisplay, configAttributes);
 
     // Not releaseFramesAutomatically means outputting to a display surface. HDR display surfaces
@@ -407,6 +425,7 @@ private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor(
             releaseFramesAutomatically,
             executor,
             listener);
+    setGlObjectProviderOnShaderPrograms(shaderPrograms, glObjectsProvider);
     VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor =
         new VideoFrameProcessingTaskExecutor(singleThreadExecutorService, listener);
     chainShaderProgramsWithListeners(
@@ -520,6 +539,15 @@ private static ImmutableList<GlShaderProgram> getGlShaderProgramsForGlEffects(
     return shaderProgramListBuilder.build();
   }
 
+  /** Sets the {@link GlObjectsProvider} on all of the {@linkplain GlShaderProgram}s provided. */
+  private static void setGlObjectProviderOnShaderPrograms(
+      ImmutableList<GlShaderProgram> shaderPrograms, GlObjectsProvider glObjectsProvider) {
+    for (int i = 0; i < shaderPrograms.size() - 1; i++) {
+      GlShaderProgram shaderProgram = shaderPrograms.get(i);
+      shaderProgram.setGlObjectsProvider(glObjectsProvider);
+    }
+  }
+
   /**
    * Chains the given {@link GlShaderProgram} instances using {@link
    * ChainingGlShaderProgramListener} instances.

library/effect/src/main/java/com/google/android/exoplayer2/effect/FinalShaderProgramWrapper.java

@@ -34,6 +34,7 @@
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.util.DebugViewProvider;
+import com.google.android.exoplayer2.util.GlObjectsProvider;
 import com.google.android.exoplayer2.util.GlTextureInfo;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.Log;
@@ -86,9 +87,11 @@
   private int inputHeight;
   @Nullable private DefaultShaderProgram defaultShaderProgram;
   @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
+  private GlObjectsProvider glObjectsProvider;
   private InputListener inputListener;
   private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
   @Nullable private SurfaceView debugSurfaceView;
+  private boolean frameProcessingStarted;
 
   private volatile boolean outputSizeOrRotationChanged;
 
@@ -130,10 +133,19 @@ public FinalShaderProgramWrapper(
 
     textureTransformMatrix = GlUtil.create4x4IdentityMatrix();
     streamOffsetUsQueue = new ConcurrentLinkedQueue<>();
+    glObjectsProvider = GlObjectsProvider.DEFAULT;
     inputListener = new InputListener() {};
     availableFrames = new ConcurrentLinkedQueue<>();
   }
 
+  @Override
+  public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
+    checkState(
+        !frameProcessingStarted,
+        "The GlObjectsProvider cannot be set after frame processing has started.");
+    this.glObjectsProvider = glObjectsProvider;
+  }
+
   @Override
   public void setInputListener(InputListener inputListener) {
     this.inputListener = inputListener;
@@ -154,6 +166,7 @@ public void setErrorListener(Executor executor, ErrorListener errorListener) {
 
   @Override
   public void signalEndOfCurrentInputStream() {
+    frameProcessingStarted = true;
     checkState(!streamOffsetUsQueue.isEmpty(), "No input stream to end.");
     streamOffsetUsQueue.remove();
     if (streamOffsetUsQueue.isEmpty()) {
@@ -178,6 +191,7 @@ public void appendStream(long streamOffsetUs) {
 
   @Override
   public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
+    frameProcessingStarted = true;
     long streamOffsetUs =
         checkStateNotNull(streamOffsetUsQueue.peek(), "No input stream specified.");
     long offsetPresentationTimeUs = presentationTimeUs + streamOffsetUs;
@@ -199,6 +213,7 @@ public void releaseOutputFrame(GlTextureInfo outputTexture) {
   }
 
   public void releaseOutputFrame(long releaseTimeNs) {
+    frameProcessingStarted = true;
     checkState(!releaseFramesAutomatically);
     Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
     renderFrameToSurfaces(
@@ -209,6 +224,7 @@ public void releaseOutputFrame(long releaseTimeNs) {
 
   @Override
   public void flush() {
+    frameProcessingStarted = true;
     // Drops all frames that aren't released yet.
     availableFrames.clear();
     if (defaultShaderProgram != null) {
@@ -304,7 +320,7 @@ private synchronized void maybeRenderFrameToOutputSurface(
         outputEglSurface,
         outputSurfaceInfo.width,
         outputSurfaceInfo.height);
-    GlUtil.clearOutputFrame();
+    glObjectsProvider.clearOutputFrame();
     defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
 
     EGLExt.eglPresentationTimeANDROID(
@@ -446,7 +462,7 @@ private void maybeRenderFrameToDebugSurface(GlTextureInfo inputTexture, long pre
     try {
       debugSurfaceViewWrapper.maybeRenderToSurfaceView(
           () -> {
-            GlUtil.clearOutputFrame();
+            glObjectsProvider.clearOutputFrame();
             @C.ColorTransfer
             int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
             defaultShaderProgram.setOutputColorTransfer(

library/effect/src/main/java/com/google/android/exoplayer2/effect/FrameCacheGlShaderProgram.java

@@ -19,7 +19,7 @@
 
 import android.content.Context;
 import android.opengl.GLES20;
-import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.util.GlObjectsProvider;
 import com.google.android.exoplayer2.util.GlProgram;
 import com.google.android.exoplayer2.util.GlTextureInfo;
 import com.google.android.exoplayer2.util.GlUtil;
@@ -49,10 +49,12 @@
   private final int capacity;
   private final boolean useHdr;
 
+  private GlObjectsProvider glObjectsProvider;
   private InputListener inputListener;
   private OutputListener outputListener;
   private ErrorListener errorListener;
   private Executor errorListenerExecutor;
+  private boolean frameProcessingStarted;
 
   /** Creates a new instance. */
   public FrameCacheGlShaderProgram(Context context, int capacity, boolean useHdr)
@@ -80,12 +82,21 @@ public FrameCacheGlShaderProgram(Context context, int capacity, boolean useHdr)
         GlUtil.getNormalizedCoordinateBounds(),
         GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
 
+    glObjectsProvider = GlObjectsProvider.DEFAULT;
     inputListener = new InputListener() {};
     outputListener = new OutputListener() {};
     errorListener = videoFrameProcessingException -> {};
     errorListenerExecutor = MoreExecutors.directExecutor();
   }
 
+  @Override
+  public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
+    checkState(
+        !frameProcessingStarted,
+        "The GlObjectsProvider cannot be set after frame processing has started.");
+    this.glObjectsProvider = glObjectsProvider;
+  }
+
   @Override
   public void setInputListener(InputListener inputListener) {
     this.inputListener = inputListener;
@@ -115,6 +126,7 @@ public void setErrorListener(Executor errorListenerExecutor, ErrorListener error
 
   @Override
   public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
+    frameProcessingStarted = true;
     try {
       configureAllOutputTextures(inputTexture.width, inputTexture.height);
 
@@ -125,7 +137,7 @@ public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs)
       // Copy frame to fbo.
       GlUtil.focusFramebufferUsingCurrentContext(
           outputTexture.fboId, outputTexture.width, outputTexture.height);
-      GlUtil.clearOutputFrame();
+      glObjectsProvider.clearOutputFrame();
       drawFrame(inputTexture.texId);
       inputListener.onInputFrameProcessed(inputTexture);
       outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
@@ -147,6 +159,7 @@ private void drawFrame(int inputTexId) throws GlUtil.GlException {
 
   @Override
   public void releaseOutputFrame(GlTextureInfo outputTexture) {
+    frameProcessingStarted = true;
     checkState(inUseOutputTextures.contains(outputTexture));
     inUseOutputTextures.remove(outputTexture);
     freeOutputTextures.add(outputTexture);
@@ -155,11 +168,13 @@ public void releaseOutputFrame(GlTextureInfo outputTexture) {
 
   @Override
   public void signalEndOfCurrentInputStream() {
+    frameProcessingStarted = true;
     outputListener.onCurrentOutputStreamEnded();
   }
 
   @Override
   public void flush() {
+    frameProcessingStarted = true;
     freeOutputTextures.addAll(inUseOutputTextures);
     inUseOutputTextures.clear();
     inputListener.onFlush();
@@ -170,6 +185,7 @@ public void flush() {
 
   @Override
   public void release() throws VideoFrameProcessingException {
+    frameProcessingStarted = true;
     try {
       deleteAllOutputTextures();
     } catch (GlUtil.GlException e) {
@@ -196,9 +212,8 @@ private void createAllOutputTextures(int width, int height) throws GlUtil.GlExce
     checkState(inUseOutputTextures.isEmpty());
     for (int i = 0; i < capacity; i++) {
       int outputTexId = GlUtil.createTexture(width, height, useHdr);
-      int outputFboId = GlUtil.createFboForTexture(outputTexId);
       GlTextureInfo outputTexture =
-          new GlTextureInfo(outputTexId, outputFboId, /* rboId= */ C.INDEX_UNSET, width, height);
+          glObjectsProvider.createBuffersForTexture(outputTexId, width, height);
       freeOutputTextures.add(outputTexture);
     }
   }

library/effect/src/main/java/com/google/android/exoplayer2/effect/GlShaderProgram.java

@@ -15,6 +15,7 @@
  */
 package com.google.android.exoplayer2.effect;
 
+import com.google.android.exoplayer2.util.GlObjectsProvider;
 import com.google.android.exoplayer2.util.GlTextureInfo;
 import com.google.android.exoplayer2.util.VideoFrameProcessingException;
 import java.util.concurrent.Executor;
@@ -146,6 +147,13 @@ interface ErrorListener {
    */
   void setErrorListener(Executor executor, ErrorListener errorListener);
 
+  /**
+   * Sets the {@link GlObjectsProvider}.
+   *
+   * <p>This method should not be called after any of the frame processing methods.
+   */
+  void setGlObjectsProvider(GlObjectsProvider glObjectsProvider);
+
   /**
    * Processes an input frame if possible.
    *