From 023f4369b80d53a0cd8893ae24a3363e51c45ff0 Mon Sep 17 00:00:00 2001
From: x270880x <x270880x@gmail.com>
Date: Tue, 6 Oct 2026 00:21:38 +0300
Subject: [PATCH] Release the filter FBO when a filter is removed

initFBOLink() gives every added filter a framebuffer, a depth renderbuffer
and an RGBA texture the size of the stream. Nothing ever deletes them:
release() of each filter only deletes its program (object filters also
their content texture), and there is no glDeleteFramebuffers or
glDeleteRenderbuffers in the library. So every removeFilter and
clearFilters leaks one frame-size FBO until the EGL context is destroyed. Apps that change overlays during a stream run the
GPU out of memory.

MainRender now frees the FBO, the renderbuffer and the texture whenever it
drops a filter, after the filter's own release(), on the GL thread that
created them. The ids are reset to 0, so a second release does nothing and
a filter added again gets new ones.

setFilter keeps reusing the old filter's FBO for the new filter. The old
filter gets a new RenderHandler before release(), so it no longer shares
the ids with the filter that took its place.

removeFilter(filter) now releases the filter only if it was in the list,
so removing a filter twice can't delete ids that belong to something else.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../input/gl/render/BaseRenderOffScreen.java     |  9 +++++++++
 .../pedro/encoder/input/gl/render/MainRender.kt  | 16 +++++++++++-----
 .../gl/render/filters/BaseFilterRender.java      |  4 ++++
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/encoder/src/main/java/com/pedro/encoder/input/gl/render/BaseRenderOffScreen.java b/encoder/src/main/java/com/pedro/encoder/input/gl/render/BaseRenderOffScreen.java
index be84bc906d..5d789023bc 100644
--- a/encoder/src/main/java/com/pedro/encoder/input/gl/render/BaseRenderOffScreen.java
+++ b/encoder/src/main/java/com/pedro/encoder/input/gl/render/BaseRenderOffScreen.java
@@ -86,4 +86,13 @@ protected void initFBO(int width, int height, int[] fboId, int[] rboId, int[] te
     }
     GlUtil.checkGlError("initFBO_E");
   }
+
+  protected void releaseFBO(int[] fboId, int[] rboId, int[] texId) {
+    GLES20.glDeleteFramebuffers(1, fboId, 0);
+    GLES20.glDeleteRenderbuffers(1, rboId, 0);
+    GLES20.glDeleteTextures(1, texId, 0);
+    fboId[0] = 0;
+    rboId[0] = 0;
+    texId[0] = 0;
+  }
 }
diff --git a/encoder/src/main/java/com/pedro/encoder/input/gl/render/MainRender.kt b/encoder/src/main/java/com/pedro/encoder/input/gl/render/MainRender.kt
index 0d2ef1ec41..07e9ea8779 100644
--- a/encoder/src/main/java/com/pedro/encoder/input/gl/render/MainRender.kt
+++ b/encoder/src/main/java/com/pedro/encoder/input/gl/render/MainRender.kt
@@ -96,14 +96,21 @@ class MainRender {
   fun release() {
     running.set(false)
     cameraRender.release()
-    for (baseFilterRender in filterRenders) baseFilterRender.release()
+    for (baseFilterRender in filterRenders) releaseFilter(baseFilterRender)
     filterRenders.clear()
     screenRender.release()
   }
 
+  private fun releaseFilter(baseFilterRender: BaseFilterRender) {
+    baseFilterRender.release()
+    baseFilterRender.releaseFBOLink()
+  }
+
   private fun setFilter(position: Int, baseFilterRender: BaseFilterRender) {
     val id = filterRenders[position].previousTexId
     val renderHandler = filterRenders[position].renderHandler
+    //the fbo is reused by the new filter, so the old one must not release it
+    filterRenders[position].renderHandler = RenderHandler()
     filterRenders[position].release()
     filterRenders[position] = baseFilterRender
     filterRenders[position].previousTexId = id
@@ -125,18 +132,17 @@ class MainRender {
 
   private fun clearFilters() {
     for (baseFilterRender in filterRenders) {
-      baseFilterRender.release()
+      releaseFilter(baseFilterRender)
     }
     filterRenders.clear()
   }
 
   private fun removeFilter(position: Int) {
-    filterRenders.removeAt(position).release()
+    releaseFilter(filterRenders.removeAt(position))
   }
 
   private fun removeFilter(baseFilterRender: BaseFilterRender) {
-    baseFilterRender.release()
-    filterRenders.remove(baseFilterRender)
+    if (filterRenders.remove(baseFilterRender)) releaseFilter(baseFilterRender)
   }
 
   private fun reOrderFilters(filters: List<BaseFilterRender>) {
diff --git a/encoder/src/main/java/com/pedro/encoder/input/gl/render/filters/BaseFilterRender.java b/encoder/src/main/java/com/pedro/encoder/input/gl/render/filters/BaseFilterRender.java
index a3dc56c52e..666c35cafe 100644
--- a/encoder/src/main/java/com/pedro/encoder/input/gl/render/filters/BaseFilterRender.java
+++ b/encoder/src/main/java/com/pedro/encoder/input/gl/render/filters/BaseFilterRender.java
@@ -63,6 +63,10 @@ public void initFBOLink() {
         renderHandler.getTexId());
   }
 
+  public void releaseFBOLink() {
+    releaseFBO(renderHandler.getFboId(), renderHandler.getRboId(), renderHandler.getTexId());
+  }
+
   protected abstract void initGlFilter(Context context);
 
   public void draw() {