diff --git a/.github/workflows/build-apk.yml b/.github/workflows/build-apk.yml new file mode 100644 index 0000000..ce98774 --- /dev/null +++ b/.github/workflows/build-apk.yml @@ -0,0 +1,40 @@ +name: Build APK + +on: + push: + branches: [master] + pull_request: + +jobs: + build-apk: + name: "Build release APK" + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Run setup-libraries composite action + uses: ./lib_ass/src/main/cpp/libass-cmake/.github/actions/setup-libraries + with: + libass-cmake-path: ${{ github.workspace }}/lib_ass/src/main/cpp/libass-cmake + + # Gradle 8.14 can't run on the runner's default JDK (25). + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "21" + + - name: Build release APK + run: ./gradlew :app:assembleRelease + + - name: Upload APK + uses: actions/upload-artifact@v7 + with: + name: demo-libass-android + path: app/build/outputs/apk/release/*.apk + archive: false + if-no-files-found: error diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e999897..037fc4d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,6 +18,13 @@ jobs: with: libass-cmake-path: ${{ github.workspace }}/lib_ass/src/main/cpp/libass-cmake + # Gradle 8.14 can't run on the runner's default JDK (25). + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "21" + - name: Publish to Maven Central Repository run: ./gradlew publishAndReleaseToMavenCentral --no-configuration-cache env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f012c3..c0322eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,35 @@ on: pull_request: jobs: + # The EFFECTS_ATLAS overlay's unit tests. + overlay-tests: + name: "Overlay unit tests" + # For Mesa 26.0+: before it, llvmpipe rounds some 8-bit blends down instead of to nearest, which the + # exact blending tests catch. + runs-on: ubuntu-26.04 + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake ninja-build pkg-config libass-dev \ + libegl-dev libgles-dev libegl-mesa0 libgl1-mesa-dri + + - name: Build + run: | + cmake -S lib_ass_kt/src/main/cpp/overlay -B overlay-build -G Ninja + cmake --build overlay-build + + - name: Run tests + env: + LIBGL_ALWAYS_SOFTWARE: "1" + # Without a GLES context, the GL tests fail rather than skip. + ASS_OVERLAY_REQUIRE_GL: "1" + run: ctest --test-dir overlay-build --output-on-failure + run-tests: name: "Test (${{ matrix.arch }})" runs-on: ubuntu-latest @@ -25,6 +54,13 @@ jobs: with: libass-cmake-path: ${{ github.workspace }}/lib_ass/src/main/cpp/libass-cmake + # Gradle 8.14 can't run on the runner's default JDK (25). + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "21" + - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules @@ -34,6 +70,6 @@ jobs: - name: Run tests uses: reactivecircus/android-emulator-runner@v2 with: - api-level: 21 + api-level: 23 arch: ${{ matrix.arch }} script: ./gradlew test connectedAndroidTest diff --git a/.gitignore b/.gitignore index 347e252..2acb069 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ google-services.json # Android Profiling *.hprof + +# cmake build dir +lib_ass_kt/src/main/cpp/overlay/builddir/ \ No newline at end of file diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 6702b33..0d0ed24 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -5,11 +5,11 @@ plugins { android { namespace = "io.github.peerless2012.ass.demo" - compileSdk = 35 + compileSdk = 36 defaultConfig { applicationId = "io.github.peerless2012.ass" - minSdk = 21 + minSdk = 23 targetSdk = 35 versionCode = 1 versionName = "1.0" diff --git a/app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt b/app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt index 1c4dfe3..08e038b 100644 --- a/app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt +++ b/app/src/main/java/io/github/peerless2012/ass/demo/MainActivity.kt @@ -5,7 +5,9 @@ import android.os.Bundle import android.view.Menu import android.view.MenuItem import android.widget.EditText +import android.widget.TextView import androidx.activity.enableEdgeToEdge +import androidx.activity.result.contract.ActivityResultContracts import androidx.annotation.OptIn import androidx.appcompat.app.AlertDialog import androidx.appcompat.app.AppCompatActivity @@ -30,10 +32,30 @@ class MainActivity : AppCompatActivity() { private var url = "http://192.168.3.6:8080/files/c.mkv" + private var renderType = AssRenderType.EFFECTS_ATLAS + private lateinit var player: ExoPlayer private lateinit var playerView: PlayerView + private lateinit var droppedFramesView: TextView + + private val droppedFramesUpdater = object : Runnable { + @OptIn(UnstableApi::class) + override fun run() { + val counters = player.videoDecoderCounters + counters?.ensureUpdated() + droppedFramesView.text = "Dropped frames: ${counters?.droppedBufferCount ?: 0}" + droppedFramesView.postDelayed(this, 500) + } + } + + private val openFileLauncher = registerForActivityResult(ActivityResultContracts.OpenDocument()) { uri -> + if (uri != null) { + playUrl(uri.toString()) + } + } + @UnstableApi override fun onCreate(savedInstanceState: Bundle?) { enableEdgeToEdge() @@ -48,10 +70,17 @@ class MainActivity : AppCompatActivity() { setSupportActionBar(toolbar) playerView = findViewById(R.id.main_player) + var startPosition = 0*1000L + if (savedInstanceState != null) { + url = savedInstanceState.getString(KEY_URL, url) + renderType = AssRenderType.valueOf(savedInstanceState.getString(KEY_RENDER_TYPE, renderType.name)) + startPosition = savedInstanceState.getLong(KEY_POSITION) + } + player = ExoPlayer.Builder(this) .buildWithAssSupport( this, - AssRenderType.OVERLAY_OPEN_GL, + renderType, AssHandlerConfig(maxRenderPixels = 720*480), playerView.subtitleView ) @@ -81,8 +110,11 @@ class MainActivity : AppCompatActivity() { val mediaItem = MediaItem.Builder() .setUri(url) // .setSubtitleConfigurations(ImmutableList.of(enConfig, jpConfig, zhConfig)) - player.setMediaItem(mediaItem.build(), 30*1000) + player.setMediaItem(mediaItem.build(), startPosition) player.prepare() + + droppedFramesView = findViewById(R.id.main_dropped_frames) + droppedFramesView.post(droppedFramesUpdater) } override fun onCreateOptionsMenu(menu: Menu): Boolean { @@ -95,6 +127,8 @@ class MainActivity : AppCompatActivity() { when(item.itemId) { R.id.menu_gc-> System.gc() R.id.menu_url-> switchUrl() + R.id.menu_open_file -> openFileLauncher.launch(arrayOf("video/*")) + R.id.menu_render_type -> selectRenderType() R.id.menu_audio -> selectTrack(C.TRACK_TYPE_AUDIO) R.id.menu_sub -> selectTrack(C.TRACK_TYPE_TEXT) R.id.menu_sub_add -> { @@ -133,10 +167,7 @@ class MainActivity : AppCompatActivity() { .setTitle("Set url") .setPositiveButton("Confirm" ) { dialog, which -> - url = urlInput.text.toString() - player.stop() - player.setMediaItem(MediaItem.fromUri(url)) - player.prepare() + playUrl(urlInput.text.toString()) } .setNegativeButton("Cancel", null) .setView(urlInput) @@ -144,6 +175,30 @@ class MainActivity : AppCompatActivity() { .show() } + private fun selectRenderType() { + val renderTypes = AssRenderType.entries + AlertDialog.Builder(this) + .setTitle("Render type") + .setSingleChoiceItems(renderTypes.map { it.name }.toTypedArray(), renderTypes.indexOf(renderType)) { dialog, which -> + dialog.dismiss() + if (renderTypes[which] != renderType) { + // The render type is fixed when the player is built, so recreate the activity to rebuild it. + renderType = renderTypes[which] + recreate() + } + } + .setNegativeButton("Cancel", null) + .create() + .show() + } + + private fun playUrl(newUrl: String) { + url = newUrl + player.stop() + player.setMediaItem(MediaItem.fromUri(url)) + player.prepare() + } + @OptIn(UnstableApi::class) private fun selectTrack(trackType: Int) { TrackSelectionDialogBuilder(this, "Select track", player, trackType) @@ -152,9 +207,23 @@ class MainActivity : AppCompatActivity() { .show() } + override fun onSaveInstanceState(outState: Bundle) { + super.onSaveInstanceState(outState) + outState.putString(KEY_URL, url) + outState.putString(KEY_RENDER_TYPE, renderType.name) + outState.putLong(KEY_POSITION, player.currentPosition) + } + override fun onDestroy() { + droppedFramesView.removeCallbacks(droppedFramesUpdater) player.release() super.onDestroy() } + companion object { + private const val KEY_URL = "url" + private const val KEY_RENDER_TYPE = "render_type" + private const val KEY_POSITION = "position" + } + } \ No newline at end of file diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index d94ebd0..06cbafb 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -17,4 +17,15 @@ android:layout_width="match_parent" android:layout_height="wrap_content"/> + + \ No newline at end of file diff --git a/app/src/main/res/menu/main_menu.xml b/app/src/main/res/menu/main_menu.xml index 642c1f7..c34f74f 100644 --- a/app/src/main/res/menu/main_menu.xml +++ b/app/src/main/res/menu/main_menu.xml @@ -5,6 +5,10 @@ + + + + diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1a1982c..c6357a2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,7 +11,7 @@ appcompat = "1.7.1" material = "1.12.0" activity = "1.10.1" constraintlayout = "2.2.1" -media3 = "1.8.0" +media3 = "1.11.1" publish = "0.34.0" [libraries] diff --git a/lib_ass/build.gradle.kts b/lib_ass/build.gradle.kts index 6b01c88..0a8b5be 100644 --- a/lib_ass/build.gradle.kts +++ b/lib_ass/build.gradle.kts @@ -12,7 +12,7 @@ android { ndkVersion = "28.1.13356709" defaultConfig { - minSdk = 21 + minSdk = 23 testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" consumerProguardFiles("consumer-rules.pro") externalNativeBuild { diff --git a/lib_ass_kt/build.gradle.kts b/lib_ass_kt/build.gradle.kts index 3bda173..953d2b7 100644 --- a/lib_ass_kt/build.gradle.kts +++ b/lib_ass_kt/build.gradle.kts @@ -12,7 +12,7 @@ android { ndkVersion = "28.1.13356709" defaultConfig { - minSdk = 21 + minSdk = 23 testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" consumerProguardFiles("consumer-rules.pro") externalNativeBuild { diff --git a/lib_ass_kt/src/main/cpp/AssKt.c b/lib_ass_kt/src/main/cpp/AssKt.c index c691327..eddb294 100644 --- a/lib_ass_kt/src/main/cpp/AssKt.c +++ b/lib_ass_kt/src/main/cpp/AssKt.c @@ -6,11 +6,11 @@ #include #include #include -#include #include "ass/ass.h" #include "fontconfig/fontconfig.h" #include "GLES2/gl2.h" #include "GLES2/gl2ext.h" +#include "AssOverlay.h" #define LOG_TAG "SubtitleRenderer" @@ -361,6 +361,16 @@ void nativeAssRenderDeinit(JNIEnv* env, jclass clazz, jlong render) { } } +jlong nativeAssOverlayDraw(JNIEnv* env, jclass clazz, jlong overlay, jlong render, jlong track, + jint fbo, jint frameWidth, jint frameHeight, jlong timeMs) { + return (jlong) assOverlayDraw((AssOverlay *) overlay, (ASS_Renderer *) render, (ASS_Track *) track, + (unsigned int) fbo, frameWidth, frameHeight, timeMs); +} + +void nativeAssOverlayRelease(JNIEnv* env, jclass clazz, jlong overlay) { + assOverlayRelease((AssOverlay *) overlay); +} + static JNINativeMethod renderMethodTable[] = { {"nativeAssRenderInit", "(J)J", (void*)nativeAssRenderInit}, {"nativeAssRenderSetFontScale", "(JF)V", (void*)nativeAssRenderSetFontScale}, @@ -369,6 +379,8 @@ static JNINativeMethod renderMethodTable[] = { {"nativeAssRenderSetFrameSize", "(JII)V", (void*)nativeAssRenderSetFrameSize}, {"nativeAssRenderFrame", "(JJJI)Lio/github/peerless2012/ass/AssFrame;", (void*) nativeAssRenderFrame}, {"nativeAssRenderDeinit", "(J)V", (void*)nativeAssRenderDeinit}, + {"nativeAssOverlayDraw", "(JJJIIIJ)J", (void*) nativeAssOverlayDraw}, + {"nativeAssOverlayRelease", "(J)V", (void*) nativeAssOverlayRelease}, }; JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { JNIEnv *env = NULL; diff --git a/lib_ass_kt/src/main/cpp/CMakeLists.txt b/lib_ass_kt/src/main/cpp/CMakeLists.txt index da87fac..012a124 100644 --- a/lib_ass_kt/src/main/cpp/CMakeLists.txt +++ b/lib_ass_kt/src/main/cpp/CMakeLists.txt @@ -27,11 +27,13 @@ include_directories(${CMAKE_SOURCE_DIR}/include) add_library(${CMAKE_PROJECT_NAME} SHARED AssKt.c) # Add these two lines. find_package(lib_ass REQUIRED CONFIG) +add_subdirectory(overlay) set_target_properties(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME "asskt") target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE lib_ass::ass + ass_overlay android jnigraphics GLESv2 z - log) \ No newline at end of file + log) diff --git a/lib_ass_kt/src/main/cpp/overlay/AssAtlas.c b/lib_ass_kt/src/main/cpp/overlay/AssAtlas.c new file mode 100644 index 0000000..d02fa56 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssAtlas.c @@ -0,0 +1,371 @@ +#include "AssAtlas.h" + +#include +#include +#include + +#include "AssOverlayLog.h" + +// Piece capacity to start from; it doubles whenever a frame has more pieces. +#define MIN_PIECE_CAP 64 +// Page widths double from this until they fit a page's pieces. +#define MIN_PAGE_WIDTH 256 + +/** + * \brief Whether an image has nothing to draw: libass emits zero-sized ones. + * \param img libass image + * \return whether img is zero-sized + */ +static bool isEmpty(const ASS_Image* img) { + return img->w <= 0 || img->h <= 0; +} + +/** + * \brief Makes room for pieces and their packing scratch. + * + * Capacity at least doubles, so a piece count creeping up frame by frame rarely reallocates. + * \param atlas atlas to grow + * \param count how many pieces must fit + * \return false on OOM + */ +static bool ensurePieces(AssAtlas* atlas, int count) { + if (count <= atlas->pieceCap) { + return true; + } + + int newCap = atlas->pieceCap > 0 ? atlas->pieceCap * 2 : MIN_PIECE_CAP; + if (newCap < count) { + newCap = count; + } + + AssAtlasPiece* pieces = (AssAtlasPiece*) realloc(atlas->pieces, (size_t) newCap * sizeof(AssAtlasPiece)); + if (pieces == NULL) { + return false; + } + + atlas->pieces = pieces; + AssAtlasPackEntry* packOrder = (AssAtlasPackEntry*) realloc(atlas->packOrder, (size_t) newCap * sizeof(AssAtlasPackEntry)); + if (packOrder == NULL) { + return false; + } + + atlas->packOrder = packOrder; + if (!assSkylineReserve(&atlas->skyline, newCap)) { + return false; + } + + int slotCap = atlas->bitmapSlotCap > 0 ? atlas->bitmapSlotCap : 2 * MIN_PIECE_CAP; + while (slotCap < 2 * newCap) { + slotCap *= 2; + } + + int* slots = (int*) realloc(atlas->bitmapSlots, (size_t) slotCap * sizeof(int)); + if (slots == NULL) { + return false; + } + + atlas->bitmapSlots = slots; + atlas->bitmapSlotCap = slotCap; + atlas->pieceCap = newCap; + return true; +} + +/** + * \brief Makes room for pages. Capacity at least doubles. + * \param atlas atlas to grow + * \param count how many pages must fit + * \return false on OOM + */ +static bool ensurePages(AssAtlas* atlas, int count) { + if (count <= atlas->pageCap) { + return true; + } + int newCap = atlas->pageCap > 0 ? atlas->pageCap * 2 : 2; + if (newCap < count) { + newCap = count; + } + AssAtlasPage* pages = (AssAtlasPage*) realloc(atlas->pages, (size_t) newCap * sizeof(AssAtlasPage)); + if (pages == NULL) { + return false; + } + atlas->pages = pages; + atlas->pageCap = newCap; + return true; +} + +/** + * \brief Links every piece to the closest earlier piece showing the same bitmap. + * + * The same bitmap means the same pixels, size and stride. The link is stored in each piece's sameAs. + * \param atlas atlas whose pieces to link + */ +static void linkRepeats(AssAtlas* atlas) { + const uint32_t mask = (uint32_t) atlas->bitmapSlotCap - 1; + + // Set all slot to -1 + memset(atlas->bitmapSlots, 0xFF, (size_t) atlas->bitmapSlotCap * sizeof(int)); + + for (int i = 0; i < atlas->pieceCount; i++) { + AssAtlasPiece* p = &atlas->pieces[i]; + p->sameAs = -1; + + // Linear probing from a Fibonacci hash of the pointer. At most half the slots are used, so + // the probe always ends on a match or an empty slot. + uint32_t slot = (uint32_t) (((uint64_t) (uintptr_t) p->bitmap * 0x9E3779B97F4A7C15ULL) >> 32) & mask; + + for (; atlas->bitmapSlots[slot] >= 0; slot = (slot + 1) & mask) { + const AssAtlasPiece* q = &atlas->pieces[atlas->bitmapSlots[slot]]; + if (q->bitmap == p->bitmap && q->w == p->w && q->h == p->h && q->stride == p->stride) { + p->sameAs = atlas->bitmapSlots[slot]; + break; + } + } + + atlas->bitmapSlots[slot] = i; + } +} + +/** + * \brief The texels a piece adds to a page. + * \param atlas atlas holding the pieces + * \param first first piece of the page's run + * \param i piece to add + * \return the piece's pixel count, or 0 if the run already has its bitmap + */ +static long long runTexels(const AssAtlas* atlas, int first, int i) { + const AssAtlasPiece* p = &atlas->pieces[i]; + return p->sameAs >= first ? 0 : (long long) p->w * p->h; +} + +/** + * \brief qsort comparator ordering pack entries tallest first, then widest. + * + * The skyline wastes little that way. Ties keep list order, so the layout is deterministic. + * \param a first AssAtlasPackEntry + * \param b second AssAtlasPackEntry + * \return negative if a goes first, positive if b does + */ +static int compareTallestFirst(const void* a, const void* b) { + const AssAtlasPackEntry* p = (const AssAtlasPackEntry*) a; + const AssAtlasPackEntry* q = (const AssAtlasPackEntry*) b; + if (p->h != q->h) { + return q->h - p->h; + } + + if (p->w != q->w) { + return q->w - p->w; + } + + return p->piece - q->piece; +} + +/** + * \brief Picks the width of a roughly square page. + * + * One as wide as the max texture size (often 16384) would turn a row of large signs into a + * ~16384 x 1080 upload. Powers of two keep page sizes, and so textures, stable from frame to frame. + * \param areaPixelCount texels the page must hold + * \param widest width of the widest piece + * \param maxSize the largest texture size + * \return a power of two from MIN_PAGE_WIDTH on, capped at maxSize + */ +static int pageWidth(long long areaPixelCount, int widest, int maxSize) { + int width = MIN_PAGE_WIDTH; + while ((long long) width * width < areaPixelCount && width < maxSize) { + width *= 2; + } + + while (width < widest && width < maxSize) { + width *= 2; + } + + return width < maxSize ? width : maxSize; +} + +/** + * \brief Packs the run pieces[first, end[ into one page, tallest first, storing each distinct + * bitmap once. + * + * Placement order is free: only the pieces' list order decides the paint order. Pieces sit edge to + * edge: they are drawn 1:1 with GL_NEAREST, so no fragment samples outside its own piece. + * \param atlas atlas holding the pieces + * \param first first piece of the run + * \param end one past the run's last piece + * \param usedW receives the packed width + * \param usedH receives the packed height + * \return false if the run doesn't fit in one page + */ +static bool packPage(AssAtlas* atlas, int first, int end, int* usedW, int* usedH) { + long long areaPixelCount = 0; + int widest = 0; + for (int i = first; i < end; i++) { + areaPixelCount += runTexels(atlas, first, i); + if (atlas->pieces[i].w > widest) { + widest = atlas->pieces[i].w; + } + } + + // The page is as tall as the max texture size: what it actually uses is read back afterward. + assSkylineReset(&atlas->skyline, pageWidth(areaPixelCount, widest, atlas->maxPageSize), atlas->maxPageSize); + *usedW = 0; + *usedH = 0; + for (int k = 0; k < atlas->pieceCount; k++) { + const int i = atlas->packOrder[k].piece; + AssAtlasPiece* p = &atlas->pieces[i]; + if (i < first || i >= end || p->sameAs >= first) { + continue; + } + if (!assSkylineAdd(&atlas->skyline, p->w, p->h, &p->atlasX, &p->atlasY)) { + return false; + } + if (p->atlasX + p->w > *usedW) { + *usedW = p->atlasX + p->w; + } + if (p->atlasY + p->h > *usedH) { + *usedH = p->atlasY + p->h; + } + } + // In list order, so a repeat's copy, possibly a repeat itself, is already placed. + for (int i = first; i < end; i++) { + AssAtlasPiece* p = &atlas->pieces[i]; + if (p->sameAs < first) { + continue; + } + p->atlasX = atlas->pieces[p->sameAs].atlasX; + p->atlasY = atlas->pieces[p->sameAs].atlasY; + } + return true; +} + +/** + * \brief Adds a page holding the maximum pieces from first on that fits in it. + * \param atlas atlas to add the page to + * \param first first piece of the run + * \return where the run ends, or -1 on OOM + */ +static int addPage(AssAtlas* atlas, int first) { + // Start from the longest run whose distinct bitmaps don't add up to more than a whole page. + const long long pagePixelCount = (long long) atlas->maxPageSize * atlas->maxPageSize; + + long long runArea = runTexels(atlas, first, first); + int end = first + 1; + while (end < atlas->pieceCount) { + const long long piecePixelCount = runTexels(atlas, first, end); + if (runArea + piecePixelCount > pagePixelCount) { + break; + } + runArea += piecePixelCount; + end++; + } + + int usedW, usedH; + while (!packPage(atlas, first, end, &usedW, &usedH)) { + // Packing never fills a page completely: retry without the run's last 1/8 of texels, and at + // least its last piece. A lone piece always fits, as larger ones were dropped. + const long long target = runArea - runArea / 8; + runArea = runTexels(atlas, first, first); + int shorter = first + 1; + while (shorter < end - 1) { + const long long piecePixelCount = runTexels(atlas, first, shorter); + if (runArea + piecePixelCount > target) { + break; + } + runArea += piecePixelCount; + shorter++; + } + end = shorter; + } + if (!ensurePages(atlas, atlas->pageCount + 1)) { + return -1; + } + atlas->pages[atlas->pageCount++] = (AssAtlasPage) {first, end - first, usedW, usedH}; + return end; +} + +void assAtlasPack(AssAtlas* atlas, const ASS_Image* images) { + atlas->pieceCount = 0; + atlas->pageCount = 0; + + int count = 0; + for (const ASS_Image* img = images; img != NULL; img = img->next) { + if (!isEmpty(img)) { + count++; + } + } + + if (count == 0 || !ensurePieces(atlas, count)) { + return; + } + + for (const ASS_Image* img = images; img != NULL; img = img->next) { + if (isEmpty(img)) { + continue; + } + if (img->w > atlas->maxPageSize || img->h > atlas->maxPageSize) { + // This should never happen. + LOGE("dropping %dx%d image: larger than max atlas size %d", img->w, img->h, atlas->maxPageSize); + continue; + } + const int i = atlas->pieceCount++; + atlas->pieces[i] = (AssAtlasPiece) { + .dstX = img->dst_x, + .dstY = img->dst_y, + .w = img->w, + .h = img->h, + .color = img->color, + .bitmap = img->bitmap, + .stride = img->stride, + }; + atlas->packOrder[i] = (AssAtlasPackEntry) {img->h, img->w, i}; + } + + if (atlas->pieceCount == 0) { + return; + } + + linkRepeats(atlas); + qsort(atlas->packOrder, (size_t) atlas->pieceCount, sizeof(AssAtlasPackEntry), compareTallestFirst); + + // A frame that fits in one page is the case where the first page's run is the whole list. + for (int first = 0; first < atlas->pieceCount;) { + const int end = addPage(atlas, first); + if (end < 0) { // OOM: keep the pages added so far + atlas->pieceCount = first; + return; + } + first = end; + } +} + +void assAtlasFillPage(const AssAtlas* atlas, const AssAtlasPage* page, unsigned char* buf) { + const size_t rowBytes = (size_t) page->usedW; + for (int i = page->firstPiece; i < page->firstPiece + page->pieceCount; i++) { + const AssAtlasPiece* p = &atlas->pieces[i]; + if (p->sameAs >= page->firstPiece) { + continue; // its copy on this page holds the texels + } + unsigned char* dst = buf + (size_t) p->atlasY * rowBytes + p->atlasX; + const unsigned char* src = p->bitmap; + for (int row = 0; row < p->h; row++) { + memcpy(dst, src, (size_t) p->w); + dst += rowBytes; + src += p->stride; + } + } +} + +void assAtlasTruncate(AssAtlas* atlas, int pageCount) { + if (pageCount >= atlas->pageCount) { + return; + } + atlas->pieceCount = atlas->pages[pageCount].firstPiece; + atlas->pageCount = pageCount; +} + +void assAtlasFree(AssAtlas* atlas) { + free(atlas->pieces); + free(atlas->pages); + free(atlas->packOrder); + assSkylineFree(&atlas->skyline); + free(atlas->bitmapSlots); +} diff --git a/lib_ass_kt/src/main/cpp/overlay/AssAtlas.h b/lib_ass_kt/src/main/cpp/overlay/AssAtlas.h new file mode 100644 index 0000000..b9888f5 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssAtlas.h @@ -0,0 +1,106 @@ +#ifndef LIBASS_ANDROID_ASS_ATLAS_H +#define LIBASS_ANDROID_ASS_ATLAS_H + +#include +#include + +#include "AssSkyline.h" +#include "ass/ass.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Where a frame's subtitle bitmaps go in the atlas pages. CPU side only: AssOverlayGl.c uploads +// each page into a texture. + +// One libass image, and where its bitmap sits in its page. +typedef struct { + int atlasX; + int atlasY; + int dstX; + int dstY; + int w; + int h; + uint32_t color; // libass RGBA, A being transparency + // Owned by libass: only valid until the next ass_render_frame call. + const unsigned char* bitmap; + int stride; + // The closest earlier piece showing the very same bitmap, or -1. When that one is on the same + // page, this piece shares its texels instead of being stored again. + int sameAs; +} AssAtlasPiece; + +// A page is one atlas texture: a single 8-bit texture (GL_R8, or GL_ALPHA on GLES2) that many pieces +// are packed into side by side. +// +// Why more than one page: a texture can't be larger than maxPageSize (GL_MAX_TEXTURE_SIZE). +// When a frame's bitmaps don't all fit in one maxPageSize x maxPageSize texture, +// the rest go in a second page, then a third... +typedef struct { + int firstPiece, pieceCount; + // The packed area, from (0, 0): the part of the texture that is uploaded. The texture itself may + // be larger, as AssOverlayGl.c reuses it from frame to frame. + int usedW, usedH; +} AssAtlasPage; + +typedef struct { + int h, w, piece; +} AssAtlasPackEntry; + +typedef struct { + int maxPageSize; // the largest texture the GPU takes (GL_MAX_TEXTURE_SIZE) + + AssAtlasPiece* pieces; // a.k.a libass bitmap + int pieceCount; // the pieces of the current frame, valid in pieces[0, pieceCount[ + int pieceCap; // how many pieces are allocated; never shrinks + + AssAtlasPage* pages; // a.k.a. texture: one per page + int pageCount; // the pages of the current frame, valid in pages[0, pageCount[ + int pageCap; // how many pages are allocated; never shrinks + + // Packing scratch, sized along with the pieces. + AssAtlasPackEntry* packOrder; + AssSkyline skyline; + int* bitmapSlots; // hash table of piece indices, -1 when empty + int bitmapSlotCap; // a power of two, at least twice pieceCap +} AssAtlas; + +/** + * \brief Lays out images from scratch. + * + * Images larger than a page are dropped, which takes a frame larger than the GPU's max texture + * size. On OOM, fewer images are laid out, possibly none. + * \param atlas atlas to lay out into + * \param images libass image list, as returned by ass_render_frame + */ +void assAtlasPack(AssAtlas* atlas, const ASS_Image* images); + +/** + * \brief Writes a page's bitmaps into a buffer. + * + * Bytes between the pieces are left as they are: they are never sampled. + * \param atlas atlas holding the page's pieces + * \param page page to fill + * \param buf a usedW x usedH area with rows usedW bytes apart + */ +void assAtlasFillPage(const AssAtlas* atlas, const AssAtlasPage* page, unsigned char* buf); + +/** + * \brief Drops the pages from pageCount on, with their pieces. + * \param atlas atlas to truncate + * \param pageCount how many pages to keep + */ +void assAtlasTruncate(AssAtlas* atlas, int pageCount); + +/** + * \brief Frees the atlas's arrays. The AssAtlas struct itself belongs to the caller. + * \param atlas atlas to free + */ +void assAtlasFree(AssAtlas* atlas); + +#ifdef __cplusplus +} +#endif + +#endif // LIBASS_ANDROID_ASS_ATLAS_H diff --git a/lib_ass_kt/src/main/cpp/overlay/AssOverlay.c b/lib_ass_kt/src/main/cpp/overlay/AssOverlay.c new file mode 100644 index 0000000..9aac4d1 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssOverlay.c @@ -0,0 +1,133 @@ +#include "AssOverlay.h" + +#include +#include + +#include "AssAtlas.h" +#include "AssOverlayGl.h" +#include "AssOverlayLog.h" + +#define ASS_OVERLAY_SLOW_LOG_MS 10 + +/** + * \brief Reads the monotonic clock. + * \return the time in milliseconds, from an arbitrary start + */ +static long long nowMs(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (long long) ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL; +} + +/** + * \brief Frees the overlay, its GL objects and its atlas. + * \param overlay overlay to free, or NULL + */ +static void destroyOverlay(AssOverlay* overlay) { + if (overlay == NULL) { + return; + } + assOverlayGlRelease(&overlay->gl); + assAtlasFree(&overlay->atlas); + free(overlay); +} + +/** + * \brief Allocates an overlay and sets up its GL state. + * \return the new overlay, or NULL on OOM or if the shader program can't be built + */ +static AssOverlay* createOverlay(void) { + AssOverlay* overlay = (AssOverlay*) calloc(1, sizeof(AssOverlay)); + if (overlay == NULL) { + LOGE("Failed to allocate the overlay"); + return NULL; + } + if (!assOverlayGlInit(&overlay->gl)) { + destroyOverlay(overlay); + return NULL; + } + overlay->atlas.maxPageSize = overlay->gl.maxTextureSize; + return overlay; +} + +/** + * \brief Brings the atlas and its quads in line with libass's latest output. + * \param overlay overlay whose atlas to update + * \param images libass image list (returned by ass_render_frame) + * \param changed ass_render_frame's detect_change: 0 if nothing changed since the previous call, + * 1 if only positions did, 2 if the content did + */ +static void updateAtlas(AssOverlay* overlay, const ASS_Image* images, int changed) { + if (changed == 0) { + return; + } + + AssAtlas* atlas = &overlay->atlas; + + assAtlasPack(atlas, images); + if (atlas->pieceCount == 0) { + return; + } + + const int uploadedPages = assOverlayGlUpload(&overlay->gl, atlas); + assAtlasTruncate(atlas, uploadedPages); // fewer only on OOM + if (atlas->pieceCount == 0) { + return; + } + if (!assOverlayGlSetQuads(&overlay->gl, atlas, overlay->frameW, overlay->frameH)) { + assAtlasTruncate(atlas, 0); + } +} + +AssOverlay* assOverlayDraw(AssOverlay* overlay, ASS_Renderer* render, ASS_Track* track, + unsigned int fbo, int frameWidth, int frameHeight, long long timeMs) { + + if (overlay == NULL) { + overlay = createOverlay(); + if (overlay == NULL) { + return NULL; + } + } + + if (render == NULL || track == NULL) { + LOGE("Cannot draw the overlay without a renderer and a track (render=%p, track=%p)", + (void*) render, (void*) track); + return overlay; + } + + + if (frameWidth != overlay->frameW || frameHeight != overlay->frameH) { + overlay->frameW = frameWidth; + overlay->frameH = frameHeight; + ass_set_frame_size(render, frameWidth, frameHeight); + LOGI("ass_set_frame_size(%d, %d)", frameWidth, frameHeight); + } + + int changed = 0; + const long long start = nowMs(); + const ASS_Image* images = ass_render_frame(render, track, timeMs, &changed); + const long long rendered = nowMs(); + // The atlas starts empty, whatever libass reports. + if (!overlay->hasRendered) { + overlay->hasRendered = true; + changed = 2; + } + updateAtlas(overlay, images, changed); + const long long updated = nowMs(); + + if (updated - start >= ASS_OVERLAY_SLOW_LOG_MS) { + LOGW("timeMs=%lld (changed=%d): ass_render_frame %lldms, atlas update %lldms (pieces=%d, pages=%d)", + timeMs, changed, rendered - start, updated - rendered, overlay->atlas.pieceCount, overlay->atlas.pageCount); + } + + // The atlas outlives changed == 0 frames, but each call's FBO wraps a new video frame that doesn't + // have the subtitles on it yet. + if (overlay->atlas.pieceCount > 0) { + assOverlayGlDraw(&overlay->gl, &overlay->atlas, (GLuint) fbo, overlay->frameW, overlay->frameH); + } + return overlay; +} + +void assOverlayRelease(AssOverlay* overlay) { + destroyOverlay(overlay); +} diff --git a/lib_ass_kt/src/main/cpp/overlay/AssOverlay.h b/lib_ass_kt/src/main/cpp/overlay/AssOverlay.h new file mode 100644 index 0000000..823507c --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssOverlay.h @@ -0,0 +1,48 @@ +#ifndef LIBASS_ANDROID_ASS_OVERLAY_H +#define LIBASS_ANDROID_ASS_OVERLAY_H + +#include "ass/ass.h" +#include "AssOverlayGl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + AssOverlayGl gl; + AssAtlas atlas; + // The video frame's pixel size: libass's frame size and the GL viewport. + int frameW, frameH; + // Whether libass has rendered a frame for this overlay. Until then, its detect_change compares with + // whatever the renderer rendered last, possibly for an overlay released since. + bool hasRendered; +} AssOverlay; + +/** + * \brief Renders `track` at `timeMs` and draws it onto a video frame. + * + * The first call creates the overlay, and later calls reuse it. Nothing is drawn if render or + * track is NULL. + * \param overlay NULL on the first call, then whatever the previous call returned + * \param render libass renderer + * \param track track to render + * \param fbo GL framebuffer wrapping the video frame's texture + * \param frameWidth video frame width, in pixels + * \param frameHeight video frame height, in pixels + * \param timeMs time to render the track at, in milliseconds + * \return the overlay to pass to the next call, or NULL if it can't be created + */ +AssOverlay* assOverlayDraw(AssOverlay* overlay, ASS_Renderer* render, ASS_Track* track, + unsigned int fbo, int frameWidth, int frameHeight, long long timeMs); + +/** + * \brief Frees the overlay and its GL objects. + * \param overlay overlay to free, or NULL + */ +void assOverlayRelease(AssOverlay* overlay); + +#ifdef __cplusplus +} +#endif + +#endif // LIBASS_ANDROID_ASS_OVERLAY_H diff --git a/lib_ass_kt/src/main/cpp/overlay/AssOverlayGl.c b/lib_ass_kt/src/main/cpp/overlay/AssOverlayGl.c new file mode 100644 index 0000000..7d90f8a --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssOverlayGl.c @@ -0,0 +1,528 @@ +#include "AssOverlayGl.h" + +#include +#include +#include + +#include "AssOverlayLog.h" + +// Page texture capacities are rounded up to a multiple of this. +#define TEXTURE_GROW_STEP 256 +// Each piece is a quad made of two triangles. +#define VERTICES_PER_PIECE 6 +// glGetError returns one error flag per call, and an implementation can have several. +#define MAX_GL_ERRORS 8 + +// One draw call covers many pieces, so their colors travel as a vertex attribute. The output is +// premultiplied by opacity and coverage, for a premultiplied "over" blend onto the video frame. +static const char* kVertexShader = + "attribute vec2 a_Position;\n" + "attribute vec2 a_TexCoord;\n" + "attribute vec4 a_Color;\n" + "varying vec2 v_TexCoord;\n" + "varying vec4 v_Color;\n" + "void main() {\n" + " gl_Position = vec4(a_Position, 0.0, 1.0);\n" + " v_TexCoord = a_TexCoord;\n" + " v_Color = a_Color;\n" + "}\n"; + +// The output is in whole 8-bit steps, alpha and color each rounded once, so the GPU's conversion to +// the frame's 8 bits is exact. The blend's only rounding is then dst * (255 - alpha) / 255, which is +// never halfway between two steps (255 is odd): every GPU that rounds to nearest writes the same byte. +static const char* kFragmentShader = + // highp: mediump is fp16 on most mobile GPUs. It can't hold the 16-bit products below, nor address + // single texels of a large page (UVs in [0.5, 1) of a 4096 page step by 2 texels): pieces would + // sample their neighbors. Samplers default to lowp, which may not tell two 8-bit values apart. + "#ifdef GL_FRAGMENT_PRECISION_HIGH\n" + "precision highp float;\n" + "precision highp sampler2D;\n" + "#else\n" + "precision mediump float;\n" + "precision mediump sampler2D;\n" + "#endif\n" + "uniform sampler2D u_Texture;\n" + "varying vec2 v_TexCoord;\n" + "varying vec4 v_Color;\n" + // Back to the 8-bit integer that x was made from. + "vec4 bytes(vec4 x) {\n" + " return floor(x * 255.0 + 0.5);\n" + "}\n" + "void main() {\n" + " vec4 color = bytes(v_Color);\n" + " float coverage = bytes(texture2D(u_Texture, v_TexCoord)).a;\n" + " float a = floor(color.a * coverage / 255.0 + 0.5);\n" + " gl_FragColor = vec4(floor(color.rgb * a / 255.0 + 0.5), a) / 255.0;\n" + "}\n"; + +/** + * \brief Reads and clears the GL errors raised since the last call, logging each. + * + * The overlay handles its own errors: left pending, they'd make media3's next GlUtil.checkGlError() + * throw and fail playback. + * \param what what the errors come from, for the log + * \return true if there was none + */ +static bool clearGlErrors(const char* what) { + bool none = true; + for (int i = 0; i < MAX_GL_ERRORS; i++) { + const GLenum error = glGetError(); + if (error == GL_NO_ERROR) { + break; + } + LOGE("%s: GL error 0x%x", what, error); + none = false; + } + return none; +} + +/** + * \brief Compiles a shader, logging the compile error if any. + * \param type GL_VERTEX_SHADER or GL_FRAGMENT_SHADER + * \param source GLSL source + * \return the shader, or 0 on failure + */ +static GLuint compileShader(GLenum type, const char* source) { + GLuint shader = glCreateShader(type); + if (shader == 0) { + LOGE("glCreateShader(0x%x) failed: GL error 0x%x", type, glGetError()); + return 0; + } + + glShaderSource(shader, 1, &source, NULL); + glCompileShader(shader); + GLint status = GL_FALSE; + glGetShaderiv(shader, GL_COMPILE_STATUS, &status); + if (status != GL_TRUE) { + char log[512]; + glGetShaderInfoLog(shader, sizeof(log), NULL, log); + LOGE("shader compile error: %s", log); + glDeleteShader(shader); + return 0; + } + return shader; +} + +/** + * \brief Compiles both shaders and links them into a program, logging the link error if any. + * \param vertexSource vertex shader GLSL source + * \param fragmentSource fragment shader GLSL source + * \return the program, or 0 on failure + */ +static GLuint linkProgram(const char* vertexSource, const char* fragmentSource) { + GLuint vertexShader = compileShader(GL_VERTEX_SHADER, vertexSource); + GLuint fragmentShader = compileShader(GL_FRAGMENT_SHADER, fragmentSource); + if (vertexShader == 0 || fragmentShader == 0) { + if (vertexShader != 0) { + glDeleteShader(vertexShader); + } + if (fragmentShader != 0) { + glDeleteShader(fragmentShader); + } + return 0; + } + + GLuint program = glCreateProgram(); + glAttachShader(program, vertexShader); + glAttachShader(program, fragmentShader); + glLinkProgram(program); + GLint status = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &status); + glDeleteShader(vertexShader); + glDeleteShader(fragmentShader); + if (status != GL_TRUE) { + char log[512]; + glGetProgramInfoLog(program, sizeof(log), NULL, log); + LOGE("program link error: %s", log); + glDeleteProgram(program); + return 0; + } + return program; +} + +bool assOverlayGlInit(AssOverlayGl* gl) { + glGetIntegerv(GL_MAX_TEXTURE_SIZE, &gl->maxTextureSize); + // GLES guarantees GL_VERSION starts with "OpenGL ES .". + const char* version = (const char*) glGetString(GL_VERSION); + int major = 0; + if (version != NULL) { + sscanf(version, "OpenGL ES %d", &major); + } + gl->isGles3 = major >= 3; + LOGI("GL_VERSION=%s, max texture size %d, %s path", + version != NULL ? version : "?", gl->maxTextureSize, gl->isGles3 ? "GLES3 PBO/R8" : "GLES2"); + + gl->program = linkProgram(kVertexShader, kFragmentShader); + if (gl->program == 0) { + LOGE("linkProgram failed"); + return false; + } + + gl->aPosition = glGetAttribLocation(gl->program, "a_Position"); + gl->aTexCoord = glGetAttribLocation(gl->program, "a_TexCoord"); + gl->aColor = glGetAttribLocation(gl->program, "a_Color"); + // Page textures are always bound to unit 0. + glUseProgram(gl->program); + glUniform1i(glGetUniformLocation(gl->program, "u_Texture"), 0); + glUseProgram(0); + glGenBuffers(1, &gl->vbo); + return clearGlErrors("init"); +} + +void assOverlayGlRelease(AssOverlayGl* gl) { + for (int i = 0; i < gl->textureCap; i++) { + if (gl->textures[i].id != 0) { + glDeleteTextures(1, &gl->textures[i].id); + } + } + if (gl->program != 0) { + glDeleteProgram(gl->program); + } + if (gl->vbo != 0) { + glDeleteBuffers(1, &gl->vbo); + } + if (gl->pbo != 0) { + glDeleteBuffers(1, &gl->pbo); + } + free(gl->textures); + free(gl->staging); + free(gl->vertices); +} + +// --- Page textures ------------------------------------------------------------------------------- + +/** + * \brief Makes room for page textures. Capacity at least doubles. + * + * New entries have no GL texture yet. + * \param gl GL state to grow + * \param count how many textures must fit + * \return false on OOM + */ +static bool ensureTextures(AssOverlayGl* gl, int count) { + if (count <= gl->textureCap) { + return true; + } + int newCap = gl->textureCap > 0 ? gl->textureCap * 2 : 2; + if (newCap < count) { + newCap = count; + } + AssOverlayTexture* textures = (AssOverlayTexture*) realloc(gl->textures, (size_t) newCap * sizeof(AssOverlayTexture)); + if (textures == NULL) { + return false; + } + memset(textures + gl->textureCap, 0, (size_t) (newCap - gl->textureCap) * sizeof(AssOverlayTexture)); + gl->textures = textures; + gl->textureCap = newCap; + return true; +} + +/** + * \brief Rounds value up to a multiple of step. + * \param value value to round, non-negative + * \param step multiple to round to, positive + * \return the smallest multiple of step that is at least value + */ +static int roundUp(int value, int step) { + return (value + step - 1) / step * step; +} + +/** + * \brief Binds the texture for `page`, (re)allocating it first if it's missing or too small. + * \param gl GL state, for the max texture size and the texture format + * \param texture texture to bind, its id and capacity updated if it is (re)allocated + * \param page page the texture must hold + * \return false if the texture can't be allocated, which leaves it deleted + */ +static bool bindPageTexture(const AssOverlayGl* gl, AssOverlayTexture* texture, const AssAtlasPage* page) { + if (texture->id != 0 && page->usedW <= texture->capW && page->usedH <= texture->capH) { + glBindTexture(GL_TEXTURE_2D, texture->id); + return true; + } + // A dimension that overflows gets 1/8 headroom, so a page creeping up by a few rows doesn't + // reallocate a multi-MB texture every time. + int capW = texture->capW >= page->usedW ? texture->capW : page->usedW + page->usedW / 8; + int capH = texture->capH >= page->usedH ? texture->capH : page->usedH + page->usedH / 8; + capW = roundUp(capW, TEXTURE_GROW_STEP); + capH = roundUp(capH, TEXTURE_GROW_STEP); + if (capW > gl->maxTextureSize) { + capW = gl->maxTextureSize; + } + if (capH > gl->maxTextureSize) { + capH = gl->maxTextureSize; + } + if (texture->id != 0) { + glDeleteTextures(1, &texture->id); + } + glGenTextures(1, &texture->id); + glBindTexture(GL_TEXTURE_2D, texture->id); + // GLES2 needs CLAMP_TO_EDGE for non-power-of-two textures, which otherwise sample as black. + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + // Quads are drawn 1:1, so every fragment samples a texel center: GL_NEAREST returns exactly that + // texel, never one of a neighboring piece. + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + if (gl->isGles3) { + // R8 is native everywhere, while several drivers emulate ALPHA by converting it on the CPU at + // every upload. The swizzle keeps the shader's `.a` read working. + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_A, GL_RED); + glTexStorage2D(GL_TEXTURE_2D, 1, GL_R8, capW, capH); + } else { + glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, capW, capH, 0, GL_ALPHA, GL_UNSIGNED_BYTE, NULL); + } + if (!clearGlErrors("page texture allocation")) { + glDeleteTextures(1, &texture->id); + *texture = (AssOverlayTexture) {0}; + return false; + } + texture->capW = capW; + texture->capH = capH; + return true; +} + +// --- Uploads ------------------------------------------------------------------------------------- + +/** + * \brief The pixel format of page uploads, matching the textures' internal format. + * \param gl GL state + * \return GL_RED on GLES3, GL_ALPHA on GLES2 + */ +static GLenum textureFormat(const AssOverlayGl* gl) { + return gl->isGles3 ? GL_RED : GL_ALPHA; +} + +/** + * \brief Makes the staging buffer at least `bytes` long, with 1/8 headroom. + * \param gl GL state holding the staging buffer + * \param bytes how many bytes must fit + * \return false on OOM + */ +static bool ensureStaging(AssOverlayGl* gl, size_t bytes) { + if (bytes <= gl->stagingCap) { + return true; + } + size_t newCap = bytes + bytes / 8; + unsigned char* staging = (unsigned char*) realloc(gl->staging, newCap); + if (staging == NULL) { + return false; + } + gl->staging = staging; + gl->stagingCap = newCap; + return true; +} + +/** + * \brief Uploads a page into the bound texture through the PBO. GLES3 only. + * + * Writes the page's bitmaps straight into the mapped PBO - the one unavoidable CPU copy - and lets + * the driver move them into the texture GPU-side, asynchronously. + * \param gl GL state holding the PBO + * \param atlas atlas holding the page's pieces + * \param page page to upload, its texture bound + * \return false if the PBO can't be allocated or mapped, or the upload fails + */ +static bool uploadThroughPbo(AssOverlayGl* gl, const AssAtlas* atlas, const AssAtlasPage* page) { + const size_t bytes = (size_t) page->usedW * page->usedH; + if (gl->pbo == 0) { + glGenBuffers(1, &gl->pbo); + } + glBindBuffer(GL_PIXEL_UNPACK_BUFFER, gl->pbo); + // Sized to the packed area rather than the texture's capacity, with 1/8 headroom: the first use of + // new storage is the expensive part. + if (gl->pboSize < bytes) { + gl->pboSize = bytes + bytes / 8; + glBufferData(GL_PIXEL_UNPACK_BUFFER, (GLsizeiptr) gl->pboSize, NULL, GL_STREAM_DRAW); + if (!clearGlErrors("PBO allocation")) { + gl->pboSize = 0; + glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); + return false; + } + } + // Synchronized: if the previous upload from the buffer hasn't run yet, the driver waits for it. + unsigned char* mapped = (unsigned char*) glMapBufferRange(GL_PIXEL_UNPACK_BUFFER, 0, (GLsizeiptr) bytes, + GL_MAP_WRITE_BIT); + bool uploaded = false; + if (mapped != NULL) { + assAtlasFillPage(atlas, page, mapped); + if (glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER)) { + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, page->usedW, page->usedH, textureFormat(gl), GL_UNSIGNED_BYTE, (const void*) 0); + uploaded = true; + } + } + // Unbound before any client-memory upload, ours or media3's. + glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); + // A failed map raises an error too. + const bool noError = clearGlErrors("PBO upload"); + return uploaded && noError; +} + +/** + * \brief Copies a page's bitmaps into the staging buffer and uploads them from there into the bound + * texture. + * \param gl GL state holding the staging buffer + * \param atlas atlas holding the page's pieces + * \param page page to upload, its texture bound + * \return false on OOM, or if the upload fails + */ +static bool uploadFromStaging(AssOverlayGl* gl, const AssAtlas* atlas, const AssAtlasPage* page) { + if (!ensureStaging(gl, (size_t) page->usedW * page->usedH)) { + return false; + } + assAtlasFillPage(atlas, page, gl->staging); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, page->usedW, page->usedH, textureFormat(gl), GL_UNSIGNED_BYTE, gl->staging); + return clearGlErrors("page upload"); +} + +int assOverlayGlUpload(AssOverlayGl* gl, const AssAtlas* atlas) { + if (!ensureTextures(gl, atlas->pageCount)) { + return 0; + } + // Page rows are usedW bytes, unpadded. The previous alignment is restored for media3's own uploads. + GLint previousAlignment; + glGetIntegerv(GL_UNPACK_ALIGNMENT, &previousAlignment); + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + int pageIdx = 0; + for (; pageIdx < atlas->pageCount; pageIdx++) { + const AssAtlasPage* page = &atlas->pages[pageIdx]; + const bool uploaded = bindPageTexture(gl, &gl->textures[pageIdx], page) + && ((gl->isGles3 && uploadThroughPbo(gl, atlas, page)) || uploadFromStaging(gl, atlas, page)); + if (!uploaded) { + break; // out of CPU or GPU memory + } + } + glPixelStorei(GL_UNPACK_ALIGNMENT, previousAlignment); + glBindTexture(GL_TEXTURE_2D, 0); + return pageIdx; +} + +// --- Quads --------------------------------------------------------------------------------------- + +/** + * \brief Splits a libass color into its channels, turning transparency into alpha. + * + * libass packs a color as 0xRRGGBBTT, TT being transparency (0 = opaque) rather than alpha. + * \param color libass color + * \param r receives the red channel + * \param g receives the green channel + * \param b receives the blue channel + * \param a receives the alpha channel (0xFF = opaque) + */ +static void unpackColor(uint32_t color, uint8_t* r, uint8_t* g, uint8_t* b, uint8_t* a) { + *r = (uint8_t) (color >> 24); + *g = (uint8_t) (color >> 16); + *b = (uint8_t) (color >> 8); + *a = (uint8_t) (0xFF - (color & 0xFF)); +} + +/** + * \brief Makes room for vertices. Capacity at least doubles. + * \param gl GL state to grow + * \param count how many vertices must fit + * \return false on OOM + */ +static bool ensureVertices(AssOverlayGl* gl, int count) { + if (count <= gl->vertexCap) { + return true; + } + int newCap = gl->vertexCap > 0 ? gl->vertexCap * 2 : 64 * VERTICES_PER_PIECE; + if (newCap < count) { + newCap = count; + } + AssOverlayVertex* vertices = (AssOverlayVertex*) realloc(gl->vertices, (size_t) newCap * sizeof(AssOverlayVertex)); + if (vertices == NULL) { + return false; + } + gl->vertices = vertices; + gl->vertexCap = newCap; + return true; +} + +bool assOverlayGlSetQuads(AssOverlayGl* gl, const AssAtlas* atlas, int frameW, int frameH) { + const int count = atlas->pieceCount * VERTICES_PER_PIECE; + if (!ensureVertices(gl, count)) { + return false; + } + const float sx = 2.0f / (float) frameW; + const float sy = 2.0f / (float) frameH; + AssOverlayVertex* out = gl->vertices; + for (int pageIdx = 0; pageIdx < atlas->pageCount; pageIdx++) { + const AssAtlasPage* page = &atlas->pages[pageIdx]; + const float su = 1.0f / (float) gl->textures[pageIdx].capW; + const float sv = 1.0f / (float) gl->textures[pageIdx].capH; + for (int i = page->firstPiece; i < page->firstPiece + page->pieceCount; i++) { + const AssAtlasPiece* p = &atlas->pieces[i]; + // libass's y goes down, clip space's goes up. + const float x0 = (float) p->dstX * sx - 1.0f; + const float x1 = (float) (p->dstX + p->w) * sx - 1.0f; + const float y0 = 1.0f - (float) p->dstY * sy; + const float y1 = 1.0f - (float) (p->dstY + p->h) * sy; + // w x h pixels sample w x h texels: each fragment hits a texel center. + const float u0 = (float) p->atlasX * su; + const float u1 = (float) (p->atlasX + p->w) * su; + const float v0 = (float) p->atlasY * sv; + const float v1 = (float) (p->atlasY + p->h) * sv; + uint8_t r, g, b, a; + unpackColor(p->color, &r, &g, &b, &a); + out[0] = (AssOverlayVertex) {x0, y0, u0, v0, r, g, b, a}; + out[1] = (AssOverlayVertex) {x1, y0, u1, v0, r, g, b, a}; + out[2] = (AssOverlayVertex) {x0, y1, u0, v1, r, g, b, a}; + out[3] = out[1]; + out[4] = (AssOverlayVertex) {x1, y1, u1, v1, r, g, b, a}; + out[5] = out[2]; + out += VERTICES_PER_PIECE; + } + } + glBindBuffer(GL_ARRAY_BUFFER, gl->vbo); + // Storage is only respecified to grow, and otherwise updated in place rather than orphaned (on + // ANGLE, orphaning is a new allocation every time). A draw still reading it is safe: the driver + // keeps glBufferSubData from affecting commands already issued. + if (gl->vboCap < gl->vertexCap) { + glBufferData(GL_ARRAY_BUFFER, (GLsizeiptr) ((size_t) gl->vertexCap * sizeof(AssOverlayVertex)), NULL, GL_DYNAMIC_DRAW); + gl->vboCap = gl->vertexCap; + } + glBufferSubData(GL_ARRAY_BUFFER, 0, (GLsizeiptr) ((size_t) count * sizeof(AssOverlayVertex)), gl->vertices); + glBindBuffer(GL_ARRAY_BUFFER, 0); + if (!clearGlErrors("quad upload")) { + gl->vboCap = 0; // its storage is respecified next time + return false; + } + return true; +} + +// --- Draw ---------------------------------------------------------------------------------------- + +void assOverlayGlDraw(const AssOverlayGl* gl, const AssAtlas* atlas, GLuint fbo, int frameW, int frameH) { + GLint previousFbo = 0; + glGetIntegerv(GL_FRAMEBUFFER_BINDING, &previousFbo); + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + glViewport(0, 0, frameW, frameH); + glEnable(GL_BLEND); + glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA); // premultiplied "over" + glUseProgram(gl->program); + glBindBuffer(GL_ARRAY_BUFFER, gl->vbo); + const GLsizei stride = sizeof(AssOverlayVertex); + glEnableVertexAttribArray((GLuint) gl->aPosition); + glVertexAttribPointer((GLuint) gl->aPosition, 2, GL_FLOAT, GL_FALSE, stride, (void*) offsetof(AssOverlayVertex, x)); + glEnableVertexAttribArray((GLuint) gl->aTexCoord); + glVertexAttribPointer((GLuint) gl->aTexCoord, 2, GL_FLOAT, GL_FALSE, stride, (void*) offsetof(AssOverlayVertex, u)); + glEnableVertexAttribArray((GLuint) gl->aColor); + glVertexAttribPointer((GLuint) gl->aColor, 4, GL_UNSIGNED_BYTE, GL_TRUE, stride, (void*) offsetof(AssOverlayVertex, r)); + glActiveTexture(GL_TEXTURE0); + for (int i = 0; i < atlas->pageCount; i++) { + const AssAtlasPage* page = &atlas->pages[i]; + glBindTexture(GL_TEXTURE_2D, gl->textures[i].id); + glDrawArrays(GL_TRIANGLES, page->firstPiece * VERTICES_PER_PIECE, page->pieceCount * VERTICES_PER_PIECE); + } + + // Leave the GL state the way media3's own shader programs expect it. + glDisable(GL_BLEND); + glDisableVertexAttribArray((GLuint) gl->aPosition); + glDisableVertexAttribArray((GLuint) gl->aTexCoord); + glDisableVertexAttribArray((GLuint) gl->aColor); + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindTexture(GL_TEXTURE_2D, 0); + glUseProgram(0); + glBindFramebuffer(GL_FRAMEBUFFER, (GLuint) previousFbo); + clearGlErrors("draw"); +} diff --git a/lib_ass_kt/src/main/cpp/overlay/AssOverlayGl.h b/lib_ass_kt/src/main/cpp/overlay/AssOverlayGl.h new file mode 100644 index 0000000..2ce9216 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssOverlayGl.h @@ -0,0 +1,109 @@ +#ifndef LIBASS_ANDROID_ASS_OVERLAY_GL_H +#define LIBASS_ANDROID_ASS_OVERLAY_GL_H + +#include +#include +#include + +#include "AssAtlas.h" +#include "GLES3/gl3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// The texture holding one atlas page. Its capacity only grows, so it is rarely reallocated. +typedef struct { + GLuint id; + int capW, capH; +} AssOverlayTexture; + +typedef struct { + float x, y; // clip space + float u, v; // texture coordinates + uint8_t r, g, b, a; // normalized to [0, 1] by the attribute pointer +} AssOverlayVertex; + +// The GL side of the overlay: shader program, page textures, uploads and quads. GL thread only, with +// the context current. +typedef struct { + int maxTextureSize; + bool isGles3; // R8 textures and PBO uploads; GLES2 gets ALPHA textures and client-memory uploads + + GLuint program; + GLint aPosition, aTexCoord, aColor; + + AssOverlayTexture* textures; // textures[i] holds atlas page i; spare ones are kept for reuse + int textureCap; + + // The GLES3 upload buffer. Grow-only and never orphaned: on ANGLE, orphaning a multi-MB buffer + // allocates and page-faults new storage on every upload (10-18 ms). Instead it's mapped + // synchronized, which only waits if its previous upload hasn't run yet, and that upload has + // normally long run by the next changed frame: one buffer is enough. + GLuint pbo; + size_t pboSize; + // For when there is no PBO (GLES2) or it can't be mapped. + unsigned char* staging; + size_t stagingCap; + + GLuint vbo; + int vboCap; // in vertices + AssOverlayVertex* vertices; // a quad (6 vertices) per atlas piece + int vertexCap; +} AssOverlayGl; + +/** + * \brief Builds the shader program and reads the context's limits. + * + * Whether it succeeds or not, assOverlayGlRelease frees what it created. + * \param gl GL state to set up, zero-initialized + * \return false if the shader program can't be built or a GL call fails + */ +bool assOverlayGlInit(AssOverlayGl* gl); + +/** + * \brief Frees the GL objects and arrays. The AssOverlayGl struct itself belongs to the caller. + * \param gl GL state to free, passed to assOverlayGlInit before + */ +void assOverlayGlRelease(AssOverlayGl* gl); + +/** + * \brief Uploads every atlas page into its texture. + * + * Page i goes into textures[i], which is (re)allocated first if it's missing or too small. + * \param gl GL state holding the textures + * \param atlas atlas whose pages to upload + * \return how many pages were uploaded, from the first on: fewer than the atlas has only when CPU or + * GPU memory runs out + */ +int assOverlayGlUpload(AssOverlayGl* gl, const AssAtlas* atlas); + +/** + * \brief Rebuilds the quads that draw the atlas pieces onto a frameW x frameH frame. + * + * Texture coordinates are relative to the page textures' sizes, so the pages must be uploaded first. + * \param gl GL state holding the vertex buffer + * \param atlas atlas whose pieces to draw + * \param frameW video frame width, in pixels + * \param frameH video frame height, in pixels + * \return false on OOM, or if the vertex buffer can't be filled: the quads can't be drawn then + */ +bool assOverlayGlSetQuads(AssOverlayGl* gl, const AssAtlas* atlas, int frameW, int frameH); + +/** + * \brief Draws the quads onto `fbo`, one call per page, then restores the GL state media3 expects. + * + * Errors the draw raises are logged and cleared, so they don't fail playback. + * \param gl GL state holding the program, textures and quads + * \param atlas atlas the quads were built from + * \param fbo GL framebuffer wrapping the video frame's texture + * \param frameW video frame width, in pixels + * \param frameH video frame height, in pixels + */ +void assOverlayGlDraw(const AssOverlayGl* gl, const AssAtlas* atlas, GLuint fbo, int frameW, int frameH); + +#ifdef __cplusplus +} +#endif + +#endif // LIBASS_ANDROID_ASS_OVERLAY_GL_H diff --git a/lib_ass_kt/src/main/cpp/overlay/AssOverlayLog.h b/lib_ass_kt/src/main/cpp/overlay/AssOverlayLog.h new file mode 100644 index 0000000..7a5b50e --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssOverlayLog.h @@ -0,0 +1,23 @@ +#ifndef LIBASS_ANDROID_ASS_OVERLAY_LOG_H +#define LIBASS_ANDROID_ASS_OVERLAY_LOG_H + +#define ASS_OVERLAY_LOG_TAG "AssOverlay" + +#ifdef __ANDROID__ +#include + +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, ASS_OVERLAY_LOG_TAG, __VA_ARGS__) +#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, ASS_OVERLAY_LOG_TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, ASS_OVERLAY_LOG_TAG, __VA_ARGS__) +#else +// Off Android (unit tests, desktop builds), logs go to stderr in logcat's "level/tag: message" form. +#include + +#define ASS_OVERLAY_LOG(level, ...) \ + (fprintf(stderr, level "/" ASS_OVERLAY_LOG_TAG ": "), fprintf(stderr, __VA_ARGS__), fputc('\n', stderr)) +#define LOGI(...) ASS_OVERLAY_LOG("I", __VA_ARGS__) +#define LOGW(...) ASS_OVERLAY_LOG("W", __VA_ARGS__) +#define LOGE(...) ASS_OVERLAY_LOG("E", __VA_ARGS__) +#endif + +#endif // LIBASS_ANDROID_ASS_OVERLAY_LOG_H diff --git a/lib_ass_kt/src/main/cpp/overlay/AssSkyline.c b/lib_ass_kt/src/main/cpp/overlay/AssSkyline.c new file mode 100644 index 0000000..431b3e3 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssSkyline.c @@ -0,0 +1,141 @@ +#include "AssSkyline.h" + +#include +#include +#include + +// One step of the skyline: over the columns [x, next node's x), the rows from y on are free. +// +// skyline->nodes[0, count[ are the steps from left to right, level neighbors merged so neighbors +// always differ in y, and nodes[count] marks the right edge: (width, 0). In the example from +// AssSkyline.h, they end up as (0, 7), (2, 5), (4, 6) and (9, 0), then (10, 0). +// +// The spots tried are the left ends of the steps: moving a rectangle left to the start of the step +// it starts on never puts it lower, so no other column fits higher. +struct AssSkylineNode { + int x, y; +}; + +bool assSkylineReserve(AssSkyline* skyline, int count) { + // An empty skyline is one node plus the right-edge marker, and each rectangle adds at most one + // node: count rectangles need at most count + 2. + const int nodeCap = count + 2; + if (nodeCap <= skyline->nodeCap) { + return true; + } + + AssSkylineNode* nodes = (AssSkylineNode*) realloc(skyline->nodes, (size_t) nodeCap * sizeof(AssSkylineNode)); + if (nodes == NULL) { + return false; + } + + skyline->nodes = nodes; + skyline->nodeCap = nodeCap; + return true; +} + +void assSkylineReset(AssSkyline* skyline, int width, int maxHeight) { + skyline->nodes[0] = (AssSkylineNode) {0, 0}; + skyline->nodes[1] = (AssSkylineNode) {width, 0}; + skyline->count = 1; + skyline->width = width; + skyline->maxHeight = maxHeight; +} + +/** + * \brief Finds the node a w x h rectangle fits highest under, then leftmost. + * The rectangle's left edge goes on the node's x. + * \param skyline skyline to search + * \param w rectangle width + * \param h rectangle height + * \param fitY receives the rectangle's row; meaningless if -1 is returned + * \return the node or -1 if the rectangle fits nowhere. + */ +static int findHighestNode(const AssSkyline* skyline, int w, int h, int* fitY) { + const AssSkylineNode* nodes = skyline->nodes; + int bestNode = -1; + int bestY = skyline->maxHeight - h + 1; + + for (int i = 0; i < skyline->count && nodes[i].x + w <= skyline->width; i++) { + const int right = nodes[i].x + w; + int y = 0; + for (int k = i; nodes[k].x < right && y < bestY; k++) { + if (nodes[k].y > y) { + y = nodes[k].y; + } + } + if (y < bestY) { + bestY = y; + bestNode = i; + } + } + + *fitY = bestY; + return bestNode; +} + +/** + * \brief Removes node i, moving the nodes after it, right-edge marker included, down by one. + * \param skyline skyline to update + * \param i node to remove + */ +static void removeNode(AssSkyline* skyline, int i) { + memmove(&skyline->nodes[i], &skyline->nodes[i + 1], (size_t) (skyline->count - i) * sizeof(AssSkylineNode)); + skyline->count--; +} + +/** + * \brief Lowers the skyline over the columns of a rectangle just placed under node i. + * The last node the rectangle only partly covers keeps its uncovered part, and level neighbors + * merge, so neighbors always differ in y. + * \param skyline skyline to update + * \param i node the rectangle was placed under, its left edge on the node's x + * \param w rectangle width + * \param bottom first row below the rectangle: its columns are free from there on + */ +static void lowerSkyline(AssSkyline* skyline, int i, int w, int bottom) { + AssSkylineNode* nodes = skyline->nodes; + const int right = nodes[i].x + w; + // the first node from the rectangle's right edge on + int end = i + 1; + while (nodes[end].x < right) { + end++; + } + + const bool partial = nodes[end].x > right; + const int uncoveredY = nodes[end - 1].y; + + // Nodes [i, end[ become node i, plus the uncovered part. + const int kept = partial ? 2 : 1; + memmove(&nodes[i + kept], &nodes[end], (size_t) (skyline->count + 1 - end) * sizeof(AssSkylineNode)); + skyline->count += kept - (end - i); + nodes[i].y = bottom; + + if (partial) { + nodes[i + 1] = (AssSkylineNode) {right, uncoveredY}; // above `bottom`, so it can't merge + } else if (i + 1 < skyline->count && nodes[i + 1].y == bottom) { + removeNode(skyline, i + 1); + } + + if (i > 0 && nodes[i - 1].y == bottom) { + removeNode(skyline, i); + } +} + +bool assSkylineAdd(AssSkyline* skyline, int w, int h, int* x, int* y) { + int fitY; + const int node = findHighestNode(skyline, w, h, &fitY); + if (node < 0) { + return false; + } + + *x = skyline->nodes[node].x; + *y = fitY; + + lowerSkyline(skyline, node, w, fitY + h); + return true; +} + +void assSkylineFree(AssSkyline* skyline) { + free(skyline->nodes); +} diff --git a/lib_ass_kt/src/main/cpp/overlay/AssSkyline.h b/lib_ass_kt/src/main/cpp/overlay/AssSkyline.h new file mode 100644 index 0000000..dc3ad27 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/AssSkyline.h @@ -0,0 +1,74 @@ +#ifndef LIBASS_ANDROID_ASS_SKYLINE_H +#define LIBASS_ANDROID_ASS_SKYLINE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct AssSkylineNode AssSkylineNode; + +// Packs rectangles into a width x maxHeight area, each at the highest spot it fits - the closest +// to row 0 - then the leftmost. Only the bottom edge of what is packed is tracked - the skyline, +// hanging from row 0 - so the space above a rectangle that sticks out past the one it sits under +// is lost; fed tallest first, little is. +// +// For example, A 4x5, B 3x4, C 5x2 and D 2x2 packed in that order into a 10 wide area, drawn as +// the page is laid out in memory, row 0 first: +// +// 0123456789 +// 0 AAAABBB##. A at (0, 0): the area is empty. +// 1 AAAABBB##. B at (4, 0): beside A is higher than under it. +// 2 AAAABBB##. C at (4, 4): under B is higher than under A, and from x = 7 it would stick out +// 3 AAAABBB##. past the right edge. +// 4 AAAACCCCC. D at (0, 5): under A is higher than under C, and from x = 9 it would stick out. +// 5 DD..CCCCC. +// 6 DD........ ## is lost: C closes it off from below, and only C's bottom is tracked. +// Column 9 is still free from row 0. +typedef struct { + AssSkylineNode* nodes; + int count, nodeCap; + int width, maxHeight; +} AssSkyline; + +/** + * \brief Makes room for packing count rectangles between two resets. Never shrinks. + * \param skyline skyline to grow + * \param count how many rectangles a packing may hold + * \return false on OOM, leaving the room as it was + */ +bool assSkylineReserve(AssSkyline* skyline, int count); + +/** + * \brief Starts an empty width x maxHeight area, keeping the room reserved. + * \param skyline skyline to reset + * \param width area width + * \param maxHeight area height: a rectangle that would reach past it doesn't fit + */ +void assSkylineReset(AssSkyline* skyline, int width, int maxHeight); + +/** + * \brief Places a w x h rectangle at the highest spot it fits, then the leftmost. + * + * At most as many rectangles as reserved may be placed between two resets. + * \param skyline skyline to place the rectangle in + * \param w rectangle width + * \param h rectangle height + * \param x receives the rectangle's first column + * \param y receives the rectangle's first row + * \return false if the rectangle fits nowhere, leaving the skyline, x and y unchanged + */ +bool assSkylineAdd(AssSkyline* skyline, int w, int h, int* x, int* y); + +/** + * \brief Frees the room reserved. + * \param skyline skyline to free + */ +void assSkylineFree(AssSkyline* skyline); + +#ifdef __cplusplus +} +#endif + +#endif // LIBASS_ANDROID_ASS_SKYLINE_H diff --git a/lib_ass_kt/src/main/cpp/overlay/CMakeLists.txt b/lib_ass_kt/src/main/cpp/overlay/CMakeLists.txt new file mode 100644 index 0000000..f3f8667 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/CMakeLists.txt @@ -0,0 +1,36 @@ +# The EFFECTS_ATLAS overlay, +# It can be build on Unix to run the units tests. + +cmake_minimum_required(VERSION 3.22.1) +project(ass_overlay C) + +if (ANDROID) + if (NOT TARGET lib_ass::ass) + find_package(lib_ass REQUIRED CONFIG) + endif () + set(ASS_OVERLAY_LIBASS lib_ass::ass) + set(ASS_OVERLAY_GL GLESv3) +else () + find_package(PkgConfig REQUIRED) + pkg_check_modules(LIBASS REQUIRED IMPORTED_TARGET libass) + set(ASS_OVERLAY_LIBASS PkgConfig::LIBASS) + pkg_check_modules(GLESV2 REQUIRED IMPORTED_TARGET glesv2) + set(ASS_OVERLAY_GL PkgConfig::GLESV2) +endif () + +if (PROJECT_IS_TOP_LEVEL AND CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + add_compile_options(-fsanitize=address,undefined -fno-sanitize-recover=all) + add_link_options(-fsanitize=address,undefined) +endif () + +add_library(ass_overlay OBJECT AssSkyline.c AssAtlas.c AssOverlay.c AssOverlayGl.c) +target_include_directories(ass_overlay PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(ass_overlay PUBLIC ${ASS_OVERLAY_LIBASS} PRIVATE ${ASS_OVERLAY_GL}) +if (ANDROID) + target_link_libraries(ass_overlay PRIVATE log) +endif () + +if (PROJECT_IS_TOP_LEVEL) + enable_testing() + add_subdirectory(tests) +endif () diff --git a/lib_ass_kt/src/main/cpp/overlay/README.md b/lib_ass_kt/src/main/cpp/overlay/README.md new file mode 100644 index 0000000..474bef9 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/README.md @@ -0,0 +1,41 @@ +# EFFECTS_ATLAS overlay + +The native code behind `AssRenderType.EFFECTS_ATLAS`. It draws ASS subtitles with OpenGL ES straight onto each decoded video frame, inside media3's video effects pipeline. + +It also builds on a desktop, where it has unit tests. + +## The idea + +libass doesn't output a picture. `ass_render_frame` returns a list of `ASS_Image`s: small bitmaps, each with a position and a single color. Blended one over the other in list order, they make the subtitles. + +Rather than compositing them into a full-frame RGBA picture on the CPU and uploading that on every frame, the overlay hands the images to the GPU as they are: + +1. **Pack.** The bitmaps are packed side by side into an 8-bit texture, the atlas page. A bitmap + that shows up several times in a frame is stored once per page. If the bitmaps don't all fit in a + texture of `GL_MAX_TEXTURE_SIZE`, the rest go in a second page, then a third... +2. **Upload.** Only the packed area of each page is uploaded: one byte per texel, and only as large + as the subtitles, not the frame. +3. **Draw.** Each image becomes a quad carrying its color as a vertex attribute, so a whole page is + one draw call. The fragment shader scales the color by the texel's coverage and blends it, + premultiplied, "over" the frame. Quads keep libass's list order, so images stack as libass meant. + +On GLES 3, pages are `GL_R8` textures uploaded through a pixel buffer object. +On GLES 2, they are `GL_ALPHA` textures uploaded from client memory. + +### Limitations + +- It doesn't support HDR or Dolby Vision video. + +### Build and run the tests + +From this directory: + +```sh +cmake -S . -B builddir +cmake --build builddir +ctest --test-dir builddir --output-on-failure +``` + +The GL tests expect the GPU to blend exactly as the 8-bit integer math does. On Mesa's software +renderer, llvmpipe (as with `LIBGL_ALWAYS_SOFTWARE=1`), they need Mesa 26.0 or later: before it, +llvmpipe rounds some blends down instead of to nearest, and the tests fail by one step. diff --git a/lib_ass_kt/src/main/cpp/overlay/tests/AssAtlasTest.cpp b/lib_ass_kt/src/main/cpp/overlay/tests/AssAtlasTest.cpp new file mode 100644 index 0000000..44c7d19 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/tests/AssAtlasTest.cpp @@ -0,0 +1,541 @@ +#include + +#include +#include +#include +#include + +#include "AssAtlas.h" +#include "AssTestImages.h" + +namespace { + +struct Page { + int firstPiece, pieceCount, usedW, usedH; + + bool operator==(const Page& other) const { + return firstPiece == other.firstPiece && pieceCount == other.pieceCount && usedW == other.usedW && + usedH == other.usedH; + } +}; + +std::ostream& operator<<(std::ostream& os, const Page& page) { + return os << "{first " << page.firstPiece << ", count " << page.pieceCount << ", " << page.usedW << "x" + << page.usedH << "}"; +} + +std::vector pages(const AssAtlas& atlas) { + std::vector result; + for (int k = 0; k < atlas.pageCount; k++) { + const AssAtlasPage& p = atlas.pages[k]; + result.push_back({p.firstPiece, p.pieceCount, p.usedW, p.usedH}); + } + return result; +} + +std::vector sameAs(const AssAtlas& atlas) { + std::vector result; + for (int i = 0; i < atlas.pieceCount; i++) { + result.push_back(atlas.pieces[i].sameAs); + } + return result; +} + +bool overlaps(const AssAtlasPiece& a, const AssAtlasPiece& b) { + return a.atlasX < b.atlasX + b.w && b.atlasX < a.atlasX + a.w && a.atlasY < b.atlasY + b.h && + b.atlasY < a.atlasY + a.h; +} + +// Checks what holds for any layout: the pages split the pieces into runs in list order, each page +// fits in a max-size texture, the bitmaps stored on a page lie in its used area without +// overlapping, and a repeat on its copy's page sits on that copy. +void expectValidLayout(const AssAtlas& atlas) { + int nextPiece = 0; + for (int k = 0; k < atlas.pageCount; k++) { + const AssAtlasPage& page = atlas.pages[k]; + SCOPED_TRACE(testing::Message() << "page " << k); + EXPECT_EQ(page.firstPiece, nextPiece); + EXPECT_GT(page.pieceCount, 0); + nextPiece = page.firstPiece + page.pieceCount; + EXPECT_GT(page.usedW, 0); + EXPECT_GT(page.usedH, 0); + EXPECT_LE(page.usedW, atlas.maxPageSize); + EXPECT_LE(page.usedH, atlas.maxPageSize); + + for (int i = page.firstPiece; i < nextPiece; i++) { + const AssAtlasPiece& p = atlas.pieces[i]; + SCOPED_TRACE(testing::Message() << "piece " << i); + EXPECT_LT(p.sameAs, i); + if (p.sameAs >= page.firstPiece) { + EXPECT_EQ(p.atlasX, atlas.pieces[p.sameAs].atlasX); + EXPECT_EQ(p.atlasY, atlas.pieces[p.sameAs].atlasY); + continue; + } + EXPECT_GE(p.atlasX, 0); + EXPECT_GE(p.atlasY, 0); + EXPECT_LE(p.atlasX + p.w, page.usedW); + EXPECT_LE(p.atlasY + p.h, page.usedH); + for (int j = page.firstPiece; j < i; j++) { + if (atlas.pieces[j].sameAs >= page.firstPiece) { + continue; + } + EXPECT_FALSE(overlaps(p, atlas.pieces[j])) << "overlaps piece " << j; + } + } + } + EXPECT_EQ(nextPiece, atlas.pieceCount); +} + +// Fills each page and checks that every piece, repeats included, reads back its own bitmap. +void expectPagesHoldBitmaps(const AssAtlas& atlas) { + for (int k = 0; k < atlas.pageCount; k++) { + const AssAtlasPage& page = atlas.pages[k]; + std::vector buf((size_t) page.usedW * page.usedH); + assAtlasFillPage(&atlas, &page, buf.data()); + + for (int i = page.firstPiece; i < page.firstPiece + page.pieceCount; i++) { + const AssAtlasPiece& p = atlas.pieces[i]; + for (int y = 0; y < p.h; y++) { + for (int x = 0; x < p.w; x++) { + const unsigned char actual = buf[(size_t) (p.atlasY + y) * page.usedW + p.atlasX + x]; + const unsigned char expected = p.bitmap[(size_t) y * p.stride + x]; + if (actual != expected) { + ADD_FAILURE() << "page " << k << ", piece " << i << ": texel (" << x << ", " << y + << ") is " << (int) actual << " instead of " << (int) expected; + return; + } + } + } + } + } +} + +// Checks the pieces against the images: every image with something to draw that fits in a page is +// kept, in list order, as it is, and linked to the closest earlier piece showing the very same bitmap. +void expectPieces(const AssAtlas& atlas, const ImageList& images) { + std::vector kept; + for (const ASS_Image& img : images.images()) { + if (img.w > 0 && img.h > 0 && img.w <= atlas.maxPageSize && img.h <= atlas.maxPageSize) { + kept.push_back(&img); + } + } + ASSERT_EQ(atlas.pieceCount, (int) kept.size()); + + for (int i = 0; i < atlas.pieceCount; i++) { + const AssAtlasPiece& p = atlas.pieces[i]; + const ASS_Image& img = *kept[(size_t) i]; + SCOPED_TRACE(testing::Message() << "piece " << i); + EXPECT_EQ(p.dstX, img.dst_x); + EXPECT_EQ(p.dstY, img.dst_y); + EXPECT_EQ(p.w, img.w); + EXPECT_EQ(p.h, img.h); + EXPECT_EQ(p.color, img.color); + EXPECT_EQ((const void*) p.bitmap, (const void*) img.bitmap); + EXPECT_EQ(p.stride, img.stride); + + int sameAs = -1; + for (int j = i - 1; j >= 0 && sameAs < 0; j--) { + const ASS_Image& other = *kept[(size_t) j]; + if (other.bitmap == img.bitmap && other.w == img.w && other.h == img.h && other.stride == img.stride) { + sameAs = j; + } + } + EXPECT_EQ(p.sameAs, sameAs); + } +} + +class AssAtlasTest : public testing::Test { +protected: + void TearDown() override { + assAtlasFree(&atlas); + } + + void pack(ImageList& images, int maxPageSize = 256) { + atlas.maxPageSize = maxPageSize; + assAtlasPack(&atlas, images.head()); + } + + AssAtlas atlas{}; +}; + +} // namespace + +TEST_F(AssAtlasTest, EmptyList) { + ImageList images; + pack(images); + + EXPECT_EQ(atlas.pieceCount, 0); + EXPECT_EQ(atlas.pageCount, 0); +} + +TEST_F(AssAtlasTest, SkipsZeroSizedImages) { + const Bitmap a = makeBitmap(4, 4, 1); + ImageList images; + images.addRaw(0, 4, 0, nullptr); + images.add(a); + images.addRaw(4, 0, 4, nullptr); + pack(images); + + ASSERT_EQ(atlas.pieceCount, 1); + EXPECT_EQ(atlas.pieces[0].bitmap, a.data.data()); + EXPECT_EQ(pages(atlas), (std::vector{{0, 1, 4, 4}})); +} + +TEST_F(AssAtlasTest, OnlyZeroSizedImages) { + ImageList images; + images.addRaw(0, 0, 0, nullptr); + pack(images); + + EXPECT_EQ(atlas.pieceCount, 0); + EXPECT_EQ(atlas.pageCount, 0); +} + +TEST_F(AssAtlasTest, KeepsImagesInListOrder) { + const Bitmap a = makeBitmap(3, 5, 1, 8); + const Bitmap b = makeBitmap(6, 2, 2); + ImageList images; + images.add(a, 10, 20, 0x11223344); + images.add(b, -3, 7, 0xAABBCCDD); + pack(images); + + ASSERT_EQ(atlas.pieceCount, 2); + const AssAtlasPiece& pa = atlas.pieces[0]; + EXPECT_EQ(pa.dstX, 10); + EXPECT_EQ(pa.dstY, 20); + EXPECT_EQ(pa.w, 3); + EXPECT_EQ(pa.h, 5); + EXPECT_EQ(pa.color, 0x11223344u); + EXPECT_EQ(pa.bitmap, a.data.data()); + EXPECT_EQ(pa.stride, 8); + const AssAtlasPiece& pb = atlas.pieces[1]; + EXPECT_EQ(pb.dstX, -3); + EXPECT_EQ(pb.dstY, 7); + EXPECT_EQ(pb.w, 6); + EXPECT_EQ(pb.h, 2); + EXPECT_EQ(pb.color, 0xAABBCCDDu); + EXPECT_EQ(pb.bitmap, b.data.data()); + EXPECT_EQ(pb.stride, 6); + expectValidLayout(atlas); +} + +TEST_F(AssAtlasTest, DropsImagesLargerThanAPage) { + const Bitmap tooWide = makeBitmap(17, 1, 1); + const Bitmap fits = makeBitmap(16, 16, 2); + const Bitmap tooTall = makeBitmap(1, 17, 3); + ImageList images; + images.add(tooWide); + images.add(fits); + images.add(tooTall); + pack(images, 16); + + ASSERT_EQ(atlas.pieceCount, 1); + EXPECT_EQ(atlas.pieces[0].bitmap, fits.data.data()); + EXPECT_EQ(pages(atlas), (std::vector{{0, 1, 16, 16}})); +} + +// A piece wider than the page its area calls for widens the page, up to a max size that need not be +// a power of two. +TEST_F(AssAtlasTest, WidensPagesForWidePieces) { + const Bitmap a = makeBitmap(600, 2, 1); + ImageList images; + images.add(a); + pack(images, 1000); + + EXPECT_EQ(pages(atlas), (std::vector{{0, 1, 600, 2}})); + expectPagesHoldBitmaps(atlas); +} + +// Placement goes tallest first, then widest, while the pieces keep their list order. +TEST_F(AssAtlasTest, PlacesTallestFirst) { + const Bitmap small = makeBitmap(4, 2, 1); + const Bitmap tall = makeBitmap(4, 8, 2); + const Bitmap wide = makeBitmap(8, 2, 3); + ImageList images; + images.add(small); + images.add(tall); + images.add(wide); + pack(images); + + ASSERT_EQ(atlas.pieceCount, 3); + EXPECT_EQ(atlas.pieces[1].atlasX, 0); // tall + EXPECT_EQ(atlas.pieces[1].atlasY, 0); + EXPECT_EQ(atlas.pieces[2].atlasX, 4); // wide, beside tall + EXPECT_EQ(atlas.pieces[2].atlasY, 0); + EXPECT_EQ(atlas.pieces[0].atlasX, 12); // small, beside wide + EXPECT_EQ(atlas.pieces[0].atlasY, 0); + EXPECT_EQ(pages(atlas), (std::vector{{0, 3, 16, 8}})); + expectPagesHoldBitmaps(atlas); +} + +TEST_F(AssAtlasTest, ReadsBitmapsByStride) { + const Bitmap a = makeBitmap(3, 2, 1, 8); + ImageList images; + images.add(a); + pack(images); + + ASSERT_EQ(pages(atlas), (std::vector{{0, 1, 3, 2}})); + std::vector buf(3 * 2); + assAtlasFillPage(&atlas, &atlas.pages[0], buf.data()); + const std::vector expected{ + a.data[0], a.data[1], a.data[2], + a.data[8], a.data[9], a.data[10], + }; + EXPECT_EQ(buf, expected); +} + +TEST_F(AssAtlasTest, StoresRepeatsOnce) { + const Bitmap a = makeBitmap(8, 8, 1); + const Bitmap b = makeBitmap(8, 8, 2); + ImageList images; + images.add(a, 0, 0, 0x11111111); + images.add(b, 10, 0, 0x22222222); + images.add(a, 20, 0, 0x33333333); + pack(images); + + EXPECT_EQ(sameAs(atlas), (std::vector{-1, -1, 0})); + // A repeat only shares its copy's texels: it is still drawn where and how it is itself. + EXPECT_EQ(atlas.pieces[2].dstX, 20); + EXPECT_EQ(atlas.pieces[2].color, 0x33333333u); + // Only a and b take room. + EXPECT_EQ(pages(atlas), (std::vector{{0, 3, 16, 8}})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +// Repeats take no room, so as many as there are fit on the page of their copy. +TEST_F(AssAtlasTest, RepeatsAddNoTexelsToAPage) { + const Bitmap a = makeBitmap(16, 16, 1); + ImageList images; + for (int i = 0; i < 4; i++) { + images.add(a, i * 20, 0); + } + pack(images, 16); + + EXPECT_EQ(sameAs(atlas), (std::vector{-1, 0, 1, 2})); + EXPECT_EQ(pages(atlas), (std::vector{{0, 4, 16, 16}})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +TEST_F(AssAtlasTest, RepeatsLinkToTheClosestEarlierPiece) { + const Bitmap a = makeBitmap(4, 4, 1); + const Bitmap b = makeBitmap(4, 4, 2); + ImageList images; + images.add(a); + images.add(b); + images.add(a); + images.add(b); + images.add(a); + pack(images); + + EXPECT_EQ(sameAs(atlas), (std::vector{-1, -1, 0, 1, 2})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +// The same pixels read with another size or stride are another bitmap. +TEST_F(AssAtlasTest, SamePointerWithAnotherShapeIsNotARepeat) { + const Bitmap a = makeBitmap(8, 8, 1); + const unsigned char* pixels = a.data.data(); + ImageList images; + images.addRaw(8, 8, 8, pixels); + images.addRaw(4, 8, 8, pixels); + images.addRaw(8, 4, 8, pixels); + images.addRaw(8, 4, 16, pixels); + pack(images); + + EXPECT_EQ(sameAs(atlas), (std::vector{-1, -1, -1, -1})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +// Pieces sharing a pointer start their lookup at the same slot: a repeat must look past the other +// shapes there to find its copy. +TEST_F(AssAtlasTest, RepeatsAreFoundPastAnotherShape) { + const Bitmap a = makeBitmap(8, 8, 1); + const unsigned char* pixels = a.data.data(); + ImageList images; + images.addRaw(8, 8, 8, pixels); + images.addRaw(4, 8, 8, pixels); + images.addRaw(8, 8, 8, pixels); + images.addRaw(4, 8, 8, pixels); + pack(images); + + EXPECT_EQ(sameAs(atlas), (std::vector{-1, -1, 0, 1})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +TEST_F(AssAtlasTest, SplitsIntoPagesInListOrder) { + std::vector bitmaps; + for (int i = 0; i < 9; i++) { + bitmaps.push_back(makeBitmap(8, 8, i)); + } + ImageList images; + for (const Bitmap& bitmap : bitmaps) { + images.add(bitmap); + } + pack(images, 16); + + // Four 8x8 pieces fill a 16x16 page. + EXPECT_EQ(pages(atlas), (std::vector{{0, 4, 16, 16}, {4, 4, 16, 16}, {8, 1, 8, 8}})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +// Three 9x9 pieces add up to less than a 16x16 page, but no two fit side by side or one above the +// other: the run shrinks until it packs. +TEST_F(AssAtlasTest, ShortensRunsThatDontPack) { + std::vector bitmaps; + for (int i = 0; i < 3; i++) { + bitmaps.push_back(makeBitmap(9, 9, i)); + } + ImageList images; + for (const Bitmap& bitmap : bitmaps) { + images.add(bitmap); + } + pack(images, 16); + + EXPECT_EQ(pages(atlas), (std::vector{{0, 1, 9, 9}, {1, 1, 9, 9}, {2, 1, 9, 9}})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +// A page only shares texels within itself, so a repeat whose copy is on an earlier page is stored +// again, and a repeat of that one shares its texels. +TEST_F(AssAtlasTest, RepeatOfAnEarlierPageIsStoredAgain) { + const Bitmap a = makeBitmap(16, 16, 1); + const Bitmap b = makeBitmap(16, 16, 2); + ImageList images; + images.add(a); + images.add(b); + images.add(a); + images.add(a); + pack(images, 16); + + EXPECT_EQ(sameAs(atlas), (std::vector{-1, -1, 0, 2})); + EXPECT_EQ(pages(atlas), (std::vector{{0, 1, 16, 16}, {1, 1, 16, 16}, {2, 2, 16, 16}})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +TEST_F(AssAtlasTest, Truncate) { + std::vector bitmaps; + for (int i = 0; i < 9; i++) { + bitmaps.push_back(makeBitmap(8, 8, i)); + } + ImageList images; + for (const Bitmap& bitmap : bitmaps) { + images.add(bitmap); + } + pack(images, 16); + ASSERT_EQ(atlas.pageCount, 3); + + assAtlasTruncate(&atlas, 3); // keeps everything + EXPECT_EQ(atlas.pageCount, 3); + EXPECT_EQ(atlas.pieceCount, 9); + + assAtlasTruncate(&atlas, 1); + EXPECT_EQ(atlas.pageCount, 1); + EXPECT_EQ(atlas.pieceCount, 4); + expectValidLayout(atlas); + + assAtlasTruncate(&atlas, 0); + EXPECT_EQ(atlas.pageCount, 0); + EXPECT_EQ(atlas.pieceCount, 0); +} + +// Each pack starts from scratch, and the arrays grow when a frame has more pieces than any before. +TEST_F(AssAtlasTest, RepacksEachFrame) { + std::vector bitmaps; + for (int i = 0; i < 100; i++) { + bitmaps.push_back(makeBitmap(1 + i % 13, 1 + i % 7, i)); + } + + ImageList small; + small.add(bitmaps[0]); + small.add(bitmaps[1]); + pack(small); + EXPECT_EQ(atlas.pieceCount, 2); + EXPECT_EQ(atlas.pageCount, 1); + + // 300 pieces: each bitmap three times. + ImageList large; + for (int round = 0; round < 3; round++) { + for (const Bitmap& bitmap : bitmaps) { + large.add(bitmap); + } + } + pack(large); + ASSERT_EQ(atlas.pieceCount, 300); + for (int i = 0; i < 300; i++) { + EXPECT_EQ(atlas.pieces[i].sameAs, i < 100 ? -1 : i - 100) << "piece " << i; + } + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); + + pack(small); + EXPECT_EQ(atlas.pieceCount, 2); + EXPECT_EQ(sameAs(atlas), (std::vector{-1, -1})); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); +} + +// Random frames, packed one after the other into the same atlas: mostly small images, at times one +// as large as a page or larger, zero-sized ones, repeats of an earlier bitmap, and earlier bitmaps read +// with another size, into pages whose size need not be a power of two. +TEST_F(AssAtlasTest, RandomFrames) { + std::mt19937 rng(20261014); + // rng() % n rather than std::uniform_int_distribution, whose algorithm varies between standard + // libraries: a seed makes the same data everywhere. + const auto random = [&rng](int min, int max) { return min + (int) (rng() % (uint32_t) (max - min + 1)); }; + for (int frame = 0; frame < 300; frame++) { + SCOPED_TRACE(testing::Message() << "frame " << frame); + const int maxPageSize = random(4, 80); + std::deque bitmaps; // a deque, so that adding a bitmap never moves the earlier ones + ImageList images; + const int count = random(0, random(1, 120)); + for (int i = 0; i < count; i++) { + const int dstX = random(-8, 200); + const int dstY = random(-8, 200); + const auto color = (uint32_t) rng(); + const int kind = random(0, 19); + const ASS_Image* earlier = + images.images().empty() ? nullptr : &images.images()[(size_t) random(0, (int) images.images().size() - 1)]; + // Each random() call is a statement of its own, as the order of function arguments is + // unspecified. + if (kind == 0) { + const int side = random(1, 4); + if (random(0, 1) == 0) { + images.addRaw(0, side, 0, nullptr, dstX, dstY, color); + } else { + images.addRaw(side, 0, side, nullptr, dstX, dstY, color); + } + } else if (kind <= 5 && earlier != nullptr && earlier->bitmap != nullptr) { + // The same bitmap, or the same pixels read with another size. + int w = earlier->w; + int h = earlier->h; + if (kind > 3) { + w = random(1, w); + h = random(1, h); + } + images.addRaw(w, h, earlier->stride, earlier->bitmap, dstX, dstY, color); + } else { + const int maxSide = kind == 6 ? maxPageSize + 4 : random(1, maxPageSize); + const int w = random(1, maxSide); + const int h = random(1, maxSide); + const int stride = w + random(0, 3); + bitmaps.push_back(makeBitmap(w, h, frame * 1000 + i, stride)); + images.add(bitmaps.back(), dstX, dstY, color); + } + } + + pack(images, maxPageSize); + expectPieces(atlas, images); + expectValidLayout(atlas); + expectPagesHoldBitmaps(atlas); + if (HasFailure()) { + return; + } + } +} diff --git a/lib_ass_kt/src/main/cpp/overlay/tests/AssGlTest.cpp b/lib_ass_kt/src/main/cpp/overlay/tests/AssGlTest.cpp new file mode 100644 index 0000000..c975471 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/tests/AssGlTest.cpp @@ -0,0 +1,178 @@ +#include "AssGlTest.h" + +#include +#include + +#include +#include +#include + +// GPU drivers and the libraries they pull in, such as libdbus, leave allocations behind at exit, and so +// does fontconfig 2.15 when libass has it parse its config files. LeakSanitizer reads this when the +// build has ASan. +extern "C" const char* __lsan_default_suppressions() { + return "leak:libdbus-1.so\n" + "leak:libEGL_mesa.so\n" + "leak:libEGL_nvidia.so\n" + "leak:libfontconfig.so\n" + "leak:libgallium\n" + "leak:libLLVM\n" + "leak:libnvidia-\n"; +} + +std::ostream& operator<<(std::ostream& os, const Rgba& c) { + return os << "(" << (int) c.r << ", " << (int) c.g << ", " << (int) c.b << ", " << (int) c.a << ")"; +} + +std::vector expectedFrame(const ASS_Image* images, int frameW, int frameH) { + // Round to nearest + const auto divide255 = [](int x) { + return (x + 127) / 255; + }; + std::vector frame((size_t) frameW * frameH, kBackground); + for (const ASS_Image* img = images; img != nullptr; img = img->next) { + const int r = (int) (img->color >> 24 & 0xFF); + const int g = (int) (img->color >> 16 & 0xFF); + const int b = (int) (img->color >> 8 & 0xFF); + const int opacity = 255 - (int) (img->color & 0xFF); + for (int y = 0; y < img->h; y++) { + for (int x = 0; x < img->w; x++) { + const int fx = img->dst_x + x; + const int fy = img->dst_y + y; + if (fx < 0 || fx >= frameW || fy < 0 || fy >= frameH) { + continue; + } + const int a = divide255(opacity * img->bitmap[(size_t) y * img->stride + x]); + Rgba& px = frame[(size_t) fy * frameW + fx]; + const auto over = [&](int src, uint8_t dst) { + return (uint8_t) (divide255(src * a) + divide255(dst * (255 - a))); + }; + px = {over(r, px.r), over(g, px.g), over(b, px.b), over(255, px.a)}; + } + } + } + return frame; +} + +void expectFrame(const std::vector& actual, const std::vector& expected, int frameW) { + ASSERT_EQ(actual.size(), expected.size()); + int mismatches = 0; + for (size_t i = 0; i < actual.size(); i++) { + const Rgba& p = actual[i]; + const Rgba& q = expected[i]; + if (p.r == q.r && p.g == q.g && p.b == q.b && p.a == q.a) { + continue; + } + if (++mismatches <= 5) { + ADD_FAILURE() << "pixel (" << i % frameW << ", " << i / frameW << ") is " << p << " instead of " << q; + } + } + EXPECT_EQ(mismatches, 0) << "mismatching pixels"; +} + +VideoFrame::VideoFrame(int width, int height) : width_(width), height_(height) { + glGenTextures(1, &texture_); + glBindTexture(GL_TEXTURE_2D, texture_); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + glBindTexture(GL_TEXTURE_2D, 0); + glGenFramebuffers(1, &fbo_); + glBindFramebuffer(GL_FRAMEBUFFER, fbo_); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, texture_, 0); + EXPECT_EQ(glCheckFramebufferStatus(GL_FRAMEBUFFER), (GLenum) GL_FRAMEBUFFER_COMPLETE); + glClearColor(kBackground.r / 255.0f, kBackground.g / 255.0f, kBackground.b / 255.0f, kBackground.a / 255.0f); + glClear(GL_COLOR_BUFFER_BIT); + glBindFramebuffer(GL_FRAMEBUFFER, 0); +} + +VideoFrame::~VideoFrame() { + glDeleteFramebuffers(1, &fbo_); + glDeleteTextures(1, &texture_); +} + +// GL's rows go up, so the frame's top row is the last one glReadPixels returns. +std::vector VideoFrame::read() const { + std::vector rows((size_t) width_ * height_); + glBindFramebuffer(GL_FRAMEBUFFER, fbo_); + glReadPixels(0, 0, width_, height_, GL_RGBA, GL_UNSIGNED_BYTE, rows.data()); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + std::vector frame((size_t) width_ * height_); + for (int y = 0; y < height_; y++) { + memcpy(&frame[(size_t) y * width_], &rows[(size_t) (height_ - 1 - y) * width_], (size_t) width_ * sizeof(Rgba)); + } + return frame; +} + +namespace { + +EGLDisplay display = EGL_NO_DISPLAY; +EGLContext context = EGL_NO_CONTEXT; +std::string noContextReason; + +void createContext() { + const char* clientExtensions = eglQueryString(EGL_NO_DISPLAY, EGL_EXTENSIONS); + if (clientExtensions != nullptr && strstr(clientExtensions, "EGL_MESA_platform_surfaceless") != nullptr) { + display = eglGetPlatformDisplay(EGL_PLATFORM_SURFACELESS_MESA, EGL_DEFAULT_DISPLAY, nullptr); + } + if (display == EGL_NO_DISPLAY) { + display = eglGetDisplay(EGL_DEFAULT_DISPLAY); + } + if (display == EGL_NO_DISPLAY || !eglInitialize(display, nullptr, nullptr)) { + noContextReason = "no EGL display"; + return; + } + // The tests draw into their own FBO: the context needs no surface. + if (strstr(eglQueryString(display, EGL_EXTENSIONS), "EGL_KHR_surfaceless_context") == nullptr) { + noContextReason = "no EGL_KHR_surfaceless_context"; + return; + } + + eglBindAPI(EGL_OPENGL_ES_API); + const EGLint configAttribs[] = {EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, EGL_SURFACE_TYPE, 0, EGL_NONE}; + EGLConfig config; + EGLint configCount = 0; + if (!eglChooseConfig(display, configAttribs, &config, 1, &configCount) || configCount == 0) { + noContextReason = "no GLES EGL config"; + return; + } + // GLES3 if there is one: it runs both upload paths. + for (const EGLint version : {3, 2}) { + const EGLint contextAttribs[] = {EGL_CONTEXT_CLIENT_VERSION, version, EGL_NONE}; + context = eglCreateContext(display, config, EGL_NO_CONTEXT, contextAttribs); + if (context != EGL_NO_CONTEXT) { + break; + } + } + if (context == EGL_NO_CONTEXT || !eglMakeCurrent(display, EGL_NO_SURFACE, EGL_NO_SURFACE, context)) { + noContextReason = "no GLES context"; + context = EGL_NO_CONTEXT; + } +} + +} // namespace + +void AssGlTest::SetUpTestSuite() { + createContext(); +} + +void AssGlTest::TearDownTestSuite() { + if (display == EGL_NO_DISPLAY) { + return; + } + eglMakeCurrent(display, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT); + if (context != EGL_NO_CONTEXT) { + eglDestroyContext(display, context); + } + eglTerminate(display); + context = EGL_NO_CONTEXT; + display = EGL_NO_DISPLAY; +} + +void AssGlTest::SetUp() { + if (context != EGL_NO_CONTEXT) { + return; + } + if (getenv("ASS_OVERLAY_REQUIRE_GL") != nullptr) { + FAIL() << noContextReason << ", and ASS_OVERLAY_REQUIRE_GL is set"; + } + GTEST_SKIP() << noContextReason; +} diff --git a/lib_ass_kt/src/main/cpp/overlay/tests/AssGlTest.h b/lib_ass_kt/src/main/cpp/overlay/tests/AssGlTest.h new file mode 100644 index 0000000..b580864 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/tests/AssGlTest.h @@ -0,0 +1,68 @@ +#ifndef LIBASS_ANDROID_ASS_GL_TEST_H +#define LIBASS_ANDROID_ASS_GL_TEST_H + +#include + +#include +#include +#include + +#include "GLES3/gl3.h" +#include "ass/ass.h" + +// What the tests that draw share: a headless GLES context, the video frame they draw onto, and what +// the overlay must draw, composited on the CPU. + +struct Rgba { + uint8_t r, g, b, a; +}; + +std::ostream& operator<<(std::ostream& os, const Rgba& c); + +// The video frame before the subtitles go on it. +constexpr Rgba kBackground{40, 80, 120, 255}; + +// What the overlay must draw: each image in list order, its coverage times its opacity blended "over" +// what is under it, in 8-bit integers as the shader computes them. Top row first, as libass's rows. +std::vector expectedFrame(const ASS_Image* images, int frameW, int frameH); + +// Expects every pixel to be exactly the expected one. +void expectFrame(const std::vector& actual, const std::vector& expected, int frameW); + +// The video frame: an RGBA texture of kBackground wrapped in an FBO, as media3 hands it over. +class VideoFrame { +public: + VideoFrame(int width, int height); + ~VideoFrame(); + VideoFrame(const VideoFrame&) = delete; + VideoFrame& operator=(const VideoFrame&) = delete; + + GLuint fbo() const { + return fbo_; + } + + // The frame as it is now, top row first. + std::vector read() const; + +private: + int width_, height_; + GLuint texture_ = 0; + GLuint fbo_ = 0; +}; + +// A test that draws, on a headless GLES context shared by its test suite: on +// EGL_MESA_platform_surfaceless when the EGL has it, as it needs no display server, else on the +// default display. Each test then runs on whatever GPU that picks; LIBGL_ALWAYS_SOFTWARE=1 picks +// Mesa's llvmpipe. +// +// Without a context, the tests are skipped, or fail when ASS_OVERLAY_REQUIRE_GL is set: in CI, a +// missing context is a broken setup, not a machine without a GPU. +class AssGlTest : public testing::Test { +protected: + static void SetUpTestSuite(); + static void TearDownTestSuite(); + // Derived fixtures call it first, and return if it skipped or failed the test. + void SetUp() override; +}; + +#endif // LIBASS_ANDROID_ASS_GL_TEST_H diff --git a/lib_ass_kt/src/main/cpp/overlay/tests/AssOverlayGlTest.cpp b/lib_ass_kt/src/main/cpp/overlay/tests/AssOverlayGlTest.cpp new file mode 100644 index 0000000..467aab2 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/tests/AssOverlayGlTest.cpp @@ -0,0 +1,259 @@ +#include + +#include +#include +#include + +#include "AssGlTest.h" +#include "AssOverlayGl.h" +#include "AssTestImages.h" + +namespace { + +// Runs once per upload path: true for GLES3's (R8 textures, PBO uploads), false for GLES2's (ALPHA +// textures, client-memory uploads). A GLES3 context runs both. +class AssOverlayGlTest : public AssGlTest, public testing::WithParamInterface { +protected: + void SetUp() override { + AssGlTest::SetUp(); + if (IsSkipped() || HasFatalFailure()) { + return; + } + ASSERT_TRUE(assOverlayGlInit(&gl)); + if (GetParam() && !gl.isGles3) { + GTEST_SKIP() << "GLES2 context"; + } + gl.isGles3 = GetParam(); + atlas.maxPageSize = gl.maxTextureSize; + } + + void TearDown() override { + frame.reset(); + assOverlayGlRelease(&gl); + assAtlasFree(&atlas); + } + + // What AssOverlay.c does with a frame's images: packs them, uploads the pages, then draws them onto + // a new frameW x frameH video frame. Returns the frame, top row first. + std::vector draw(ImageList& images, int frameW, int frameH) { + frame = std::make_unique(frameW, frameH); + assAtlasPack(&atlas, images.head()); + EXPECT_EQ(assOverlayGlUpload(&gl, &atlas), atlas.pageCount); + EXPECT_TRUE(assOverlayGlSetQuads(&gl, &atlas, frameW, frameH)); + assOverlayGlDraw(&gl, &atlas, frame->fbo(), frameW, frameH); + EXPECT_EQ(glGetError(), (GLenum) GL_NO_ERROR); + return frame->read(); + } + + AssOverlayGl gl{}; + AssAtlas atlas{}; + // The last video frame drawn. + std::unique_ptr frame; +}; + +} // namespace + +// Each image lands at its libass position, upright, with its own color and coverage, clipped to the +// frame. +TEST_P(AssOverlayGlTest, DrawsImagesWhereLibassPlacesThem) { + const Bitmap a = makeBitmap(10, 6, 1); + const Bitmap b = makeBitmap(7, 9, 2, 12); + const Bitmap c = makeBitmap(8, 12, 3); + ImageList images; + images.add(a, 5, 3, 0xFF000000); // opaque red + images.add(b, 30, 20, 0x00FF0080); // half transparent green + images.add(c, -3, 40, 0x2040E010); // past the left and bottom edges + + expectFrame(draw(images, 64, 48), expectedFrame(images.head(), 64, 48), 64); +} + +// Placement goes tallest first, but drawing goes in list order: a later image blends over an earlier +// one. +TEST_P(AssOverlayGlTest, BlendsImagesInListOrder) { + const Bitmap low = makeBitmap(20, 4, 1); + const Bitmap tall = makeBitmap(6, 20, 2); + ImageList images; + images.add(low, 10, 10, 0xFF000060); + images.add(tall, 15, 5, 0x0000FF60); + + expectFrame(draw(images, 40, 30), expectedFrame(images.head(), 40, 30), 40); +} + +// Every coverage at every opacity, first over the background, then over what the first pass drew: the +// GPU must draw exactly what the 8-bit integer math does. +TEST_P(AssOverlayGlTest, BlendsEveryCoverageAtEveryOpacityExactly) { + const Bitmap ramp = makeBitmap(256, 1, 0); // coverage x at column x + const Bitmap shiftedRamp = makeBitmap(256, 1, 4); // coverage x + 124 + ImageList images; + for (uint32_t y = 0; y < 256; y++) { + images.add(ramp, 0, (int) y, y << 24 | (255 - y) << 16 | (y * 37 & 0xFF) << 8 | y); + } + for (uint32_t y = 0; y < 256; y++) { + // 91 and 53 are odd: each pass still has every transparency once. + images.add(shiftedRamp, 0, (int) y, (y * 53 & 0xFF) << 24 | (y ^ 0xA5) << 16 | (255 - y) << 8 | (y * 91 & 0xFF)); + } + + expectFrame(draw(images, 256, 256), expectedFrame(images.head(), 256, 256), 256); +} + +// One draw per page, each with its own texture, in list order: the images overlap across pages. +TEST_P(AssOverlayGlTest, DrawsEveryPage) { + std::vector bitmaps; + for (int i = 0; i < 9; i++) { + bitmaps.push_back(makeBitmap(8, 8, i)); + } + ImageList images; + for (int i = 0; i < 9; i++) { + images.add(bitmaps[i], i * 3, i * 2, 0x10E03040u + (uint32_t) i * 0x18000000u); + } + atlas.maxPageSize = 16; // four 8x8 pieces per page + + const std::vector drawn = draw(images, 40, 30); + EXPECT_EQ(atlas.pageCount, 3); + expectFrame(drawn, expectedFrame(images.head(), 40, 30), 40); +} + +// A repeat samples its copy's texels, with its own position and color. +TEST_P(AssOverlayGlTest, DrawsRepeatsFromTheirCopy) { + const Bitmap a = makeBitmap(8, 8, 1); + const Bitmap b = makeBitmap(8, 8, 2); + ImageList images; + images.add(a, 2, 2, 0xFF000000); + images.add(b, 12, 2, 0x00FF0000); + images.add(a, 22, 2, 0x0000FF40); + + const std::vector drawn = draw(images, 32, 12); + ASSERT_EQ(atlas.pieceCount, 3); + EXPECT_EQ(atlas.pieces[2].sameAs, 0); + expectFrame(drawn, expectedFrame(images.head(), 32, 12), 32); +} + +// A page that fits its texture reuses it, and one that doesn't gets a larger one. Either way the +// quads sample the texture as it is, whatever an earlier frame left in it. +TEST_P(AssOverlayGlTest, ReusesAndGrowsTextures) { + const Bitmap first = makeBitmap(10, 10, 1); + ImageList firstImages; + firstImages.add(first, 4, 4, 0xFFFFFF00); + expectFrame(draw(firstImages, 320, 20), expectedFrame(firstImages.head(), 320, 20), 320); + const GLuint texture = gl.textures[0].id; + EXPECT_EQ(gl.textures[0].capW, 256); + EXPECT_EQ(gl.textures[0].capH, 256); + + const Bitmap smaller = makeBitmap(20, 8, 2); + ImageList smallerImages; + smallerImages.add(smaller, 100, 6, 0x80FF4020); + expectFrame(draw(smallerImages, 320, 20), expectedFrame(smallerImages.head(), 320, 20), 320); + EXPECT_EQ(gl.textures[0].id, texture); + EXPECT_EQ(gl.textures[0].capW, 256); + EXPECT_EQ(gl.textures[0].capH, 256); + + const Bitmap wider = makeBitmap(300, 4, 3); + ImageList widerImages; + widerImages.add(wider, 10, 8, 0x4080FF00); + expectFrame(draw(widerImages, 320, 20), expectedFrame(widerImages.head(), 320, 20), 320); + EXPECT_EQ(gl.textures[0].capW, 512); // 300 plus 1/8, rounded up to 256 + EXPECT_EQ(gl.textures[0].capH, 256); +} + +// A frame with fewer pages keeps the spare textures, and a later frame with more pages reuses them, +// whatever an earlier frame left in them. +TEST_P(AssOverlayGlTest, ReusesSpareTextures) { + std::vector bitmaps; + for (int i = 0; i < 19; i++) { + bitmaps.push_back(makeBitmap(8, 8, i)); + } + ImageList threePages; + for (int i = 0; i < 9; i++) { + threePages.add(bitmaps[i], i * 3, i * 2, 0x10E03040u + (uint32_t) i * 0x18000000u); + } + ImageList onePage; + onePage.add(bitmaps[9], 20, 10, 0xFF804000); + ImageList otherThreePages; + for (int i = 0; i < 9; i++) { + otherThreePages.add(bitmaps[10 + i], 30 - i * 3, i * 2, 0x40E0A020u + (uint32_t) i * 0x00180000u); + } + atlas.maxPageSize = 16; // four 8x8 pieces per page + + expectFrame(draw(threePages, 40, 30), expectedFrame(threePages.head(), 40, 30), 40); + ASSERT_EQ(atlas.pageCount, 3); + const std::vector textures{gl.textures[0].id, gl.textures[1].id, gl.textures[2].id}; + + expectFrame(draw(onePage, 40, 30), expectedFrame(onePage.head(), 40, 30), 40); + EXPECT_EQ(atlas.pageCount, 1); + + expectFrame(draw(otherThreePages, 40, 30), expectedFrame(otherThreePages.head(), 40, 30), 40); + EXPECT_EQ(atlas.pageCount, 3); + EXPECT_EQ((std::vector{gl.textures[0].id, gl.textures[1].id, gl.textures[2].id}), textures); +} + +// A page texture that can't be allocated isn't counted as uploaded, and its GL error doesn't reach +// media3, whose next GlUtil.checkGlError() would fail playback. Going past GL_MAX_TEXTURE_SIZE fails +// like running out of GPU memory does. The next frame allocates the texture again. +TEST_P(AssOverlayGlTest, HandlesAPageTextureThatCantBeAllocated) { + const int maxTextureSize = gl.maxTextureSize; + gl.maxTextureSize = maxTextureSize * 2; + atlas.maxPageSize = gl.maxTextureSize; + const Bitmap tooWide = makeBitmap(maxTextureSize + 1, 1, 1); + ImageList tooWideImages; + tooWideImages.add(tooWide); + assAtlasPack(&atlas, tooWideImages.head()); + ASSERT_EQ(atlas.pageCount, 1); + + EXPECT_EQ(assOverlayGlUpload(&gl, &atlas), 0); + EXPECT_EQ(glGetError(), (GLenum) GL_NO_ERROR); + EXPECT_EQ(gl.textures[0].id, 0u); + + gl.maxTextureSize = maxTextureSize; + atlas.maxPageSize = maxTextureSize; + const Bitmap a = makeBitmap(8, 8, 2); + ImageList images; + images.add(a, 2, 2, 0xFF000000); + expectFrame(draw(images, 16, 16), expectedFrame(images.head(), 16, 16), 16); +} + +// media3's own shader programs run right after the overlay, in the same context. +TEST_P(AssOverlayGlTest, LeavesTheGlStateMedia3Expects) { + const Bitmap a = makeBitmap(8, 8, 1); + ImageList images; + images.add(a, 2, 2, 0xFF000000); + GLuint previousFbo; + glGenFramebuffers(1, &previousFbo); + GLint suiteAlignment; + glGetIntegerv(GL_UNPACK_ALIGNMENT, &suiteAlignment); + glPixelStorei(GL_UNPACK_ALIGNMENT, 8); + + draw(images, 16, 16); + glBindFramebuffer(GL_FRAMEBUFFER, previousFbo); + assOverlayGlDraw(&gl, &atlas, frame->fbo(), 16, 16); + + GLint value = -1; + glGetIntegerv(GL_FRAMEBUFFER_BINDING, &value); + EXPECT_EQ(value, (GLint) previousFbo); + EXPECT_FALSE(glIsEnabled(GL_BLEND)); + glGetIntegerv(GL_CURRENT_PROGRAM, &value); + EXPECT_EQ(value, 0); + glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &value); + EXPECT_EQ(value, 0); + glGetIntegerv(GL_TEXTURE_BINDING_2D, &value); + EXPECT_EQ(value, 0); + for (const GLint attrib : {gl.aPosition, gl.aTexCoord, gl.aColor}) { + glGetVertexAttribiv((GLuint) attrib, GL_VERTEX_ATTRIB_ARRAY_ENABLED, &value); + EXPECT_EQ(value, GL_FALSE) << "attribute " << attrib; + } + if (GetParam()) { + // Left bound, it would turn media3's client-memory uploads into PBO offsets. + glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &value); + EXPECT_EQ(value, 0); + } + glGetIntegerv(GL_UNPACK_ALIGNMENT, &value); + EXPECT_EQ(value, 8); + EXPECT_EQ(glGetError(), (GLenum) GL_NO_ERROR); + + glPixelStorei(GL_UNPACK_ALIGNMENT, suiteAlignment); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glDeleteFramebuffers(1, &previousFbo); +} + +INSTANTIATE_TEST_SUITE_P(, AssOverlayGlTest, testing::Bool(), [](const testing::TestParamInfo& info) { + return info.param ? "Gles3" : "Gles2"; +}); diff --git a/lib_ass_kt/src/main/cpp/overlay/tests/AssOverlayTest.cpp b/lib_ass_kt/src/main/cpp/overlay/tests/AssOverlayTest.cpp new file mode 100644 index 0000000..22433df --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/tests/AssOverlayTest.cpp @@ -0,0 +1,274 @@ +#include + +#include +#include +#include + +#include "AssGlTest.h" +#include "AssOverlay.h" + +namespace { + +// Two drawings, in script coordinates that are the frame's pixels at 64x48, each drawn as an outline +// and a fill image: a triangle shown from 1s to 2s, and a half transparent square shown from 1s to 3s +// that moves 32 pixels right over its first second. +constexpr const char* kScript = R"([Script Info] +ScriptType: v4.00+ +PlayResX: 64 +PlayResY: 48 +ScaledBorderAndShadow: yes + +[V4+ Styles] +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding +Style: Default,sans-serif,20,&H000000FF,&H000000FF,&H00FF0000,&H00000000,0,0,0,0,100,100,0,0,1,1,0,7,0,0,0,1 + +[Events] +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text +Dialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,{\pos(10,8)\p1}m 0 0 l 20 0 0 12{\p0} +Dialogue: 0,0:00:01.00,0:00:03.00,Default,,0,0,0,,{\move(4,30,36,30,0,1000)\1a&H80&\p1}m 0 0 l 8 0 8 8 0 8{\p0} +)"; + +// A scene of what tracks are made of, in script coordinates that are the frame's pixels at 320x180: +// lines that fade or come and go, a sign shown twice at once (libass hands over the same bitmap for +// both), a drawing that turns and grows to the frame's width, past the first page texture's size, a +// blurred and clipped one that moves, and from 3s on a line that stays still. +constexpr const char* kScene = R"([Script Info] +ScriptType: v4.00+ +PlayResX: 320 +PlayResY: 180 +ScaledBorderAndShadow: yes +WrapStyle: 2 + +[V4+ Styles] +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding +Style: Default,sans-serif,24,&H00FFFFFF,&H000000FF,&H00000000,&H80000000,0,0,0,0,100,100,0,0,1,2,1,2,8,8,8,1 +Style: Sign,sans-serif,16,&H0000FFFF,&H000000FF,&H00400000,&H00000000,1,0,0,0,100,100,0,0,1,1,0,7,0,0,0,1 + +[Events] +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text +Dialogue: 0,0:00:00.00,0:00:01.50,Default,,0,0,0,,{\fad(300,300)}Hello there +Dialogue: 0,0:00:01.20,0:00:03.00,Default,,0,0,0,,Two lines\Nat once +Dialogue: 0,0:00:00.50,0:00:02.50,Sign,,0,0,0,,{\pos(20,20)}Sign +Dialogue: 0,0:00:00.50,0:00:02.50,Sign,,0,0,0,,{\pos(200,20)}Sign +Dialogue: 1,0:00:00.00,0:00:03.00,Sign,,0,0,0,,{\pos(160,90)\an5\t(0,3000,\fscx2500\fscy2500\frz30\1c&HFF00FF&)\p1}m 0 0 l 12 0 12 8 0 8{\p0} +Dialogue: 2,0:00:00.00,0:00:03.00,Sign,,0,0,0,,{\move(0,120,280,120)\blur2\clip(40,100,240,160)\p1}m 0 0 l 40 0 20 32{\p0} +Dialogue: 0,0:00:02.50,0:00:04.00,Default,,0,0,0,,The end +)"; + +// libass's warnings and errors, as its default callback would print them; not its chatter. +void onLibassMessage(int level, const char* format, va_list args, void*) { + if (level > 2) { + return; + } + fputs("libass: ", stderr); + vfprintf(stderr, format, args); + fputc('\n', stderr); +} + +// Where the subtitles are expected, but libass rendered none, it most likely found no font. +constexpr const char* kNoImages = "libass rendered no images: no font?"; + +// What a draw showed. +struct Drawn { + // ass_render_frame's detect_change: 0 if nothing changed since the previous draw, 1 if only + // positions did, 2 if the content did + int changed; + // how many images libass rendered + int images; +}; + +// Draws a track with assOverlayDraw, as AssRender.kt does for each video frame. +class AssOverlayTest : public AssGlTest { +protected: + void SetUp() override { + AssGlTest::SetUp(); + if (IsSkipped() || HasFatalFailure()) { + return; + } + library = ass_library_init(); + ASSERT_NE(library, nullptr); + ass_set_message_cb(library, onLibassMessage, nullptr); + renderer = newRenderer(); + mirror = newRenderer(); + readTrack(kScript); + } + + void TearDown() override { + assOverlayRelease(overlay); // its GL objects, while the context is current + if (track != nullptr) { + ass_free_track(track); + } + if (mirror != nullptr) { + ass_renderer_done(mirror); + } + if (renderer != nullptr) { + ass_renderer_done(renderer); + } + if (library != nullptr) { + ass_library_done(library); + } + } + + // Replaces the track, before the first draw. + void readTrack(const char* script) { + if (track != nullptr) { + ass_free_track(track); + } + std::string text = script; + track = ass_read_memory(library, text.data(), text.size(), nullptr); + ASSERT_NE(track, nullptr); + } + + // Draws the track at timeMs onto a new frameW x frameH video frame, and expects the frame to show + // what libass renders for it. + Drawn draw(long long timeMs, int frameW = 64, int frameH = 48) { + const VideoFrame frame(frameW, frameH); + overlay = assOverlayDraw(overlay, renderer, track, frame.fbo(), frameW, frameH, timeMs); + EXPECT_NE(overlay, nullptr); + EXPECT_EQ(glGetError(), (GLenum) GL_NO_ERROR); + + // The mirror gets the very calls the overlay makes to its renderer, so it renders the same + // images, and reports the same change. + if (frameW != mirrorW || frameH != mirrorH) { + ass_set_frame_size(mirror, frameW, frameH); + mirrorW = frameW; + mirrorH = frameH; + } + Drawn drawn{-1, 0}; + const ASS_Image* images = ass_render_frame(mirror, track, timeMs, &drawn.changed); + for (const ASS_Image* img = images; img != nullptr; img = img->next) { + drawn.images++; + } + expectFrame(frame.read(), expectedFrame(images, frameW, frameH), frameW); + return drawn; + } + + ASS_Library* library = nullptr; + // The overlay's renderer, and one that renders the same frames for the tests to compare with. + ASS_Renderer* renderer = nullptr; + ASS_Renderer* mirror = nullptr; + int mirrorW = 0, mirrorH = 0; + ASS_Track* track = nullptr; + AssOverlay* overlay = nullptr; + +private: + ASS_Renderer* newRenderer() { + ASS_Renderer* r = ass_renderer_init(library); + EXPECT_NE(r, nullptr); + if (r != nullptr) { + ass_set_fonts(r, nullptr, "sans-serif", ASS_FONTPROVIDER_AUTODETECT, nullptr, 0); + } + return r; + } +}; + +} // namespace + +// Each frame shows the subtitles at its time, if any. +TEST_F(AssOverlayTest, DrawsTheTrackAtTheGivenTime) { + EXPECT_EQ(draw(500).images, 0); + EXPECT_GT(draw(1000).images, 0) << kNoImages; + EXPECT_GT(draw(2500).images, 0) << kNoImages; +} + +// A new overlay on a renderer that already rendered the frame, as when the subtitles are turned off +// and on again: libass reports no change, but the new overlay has drawn nothing yet. +TEST_F(AssOverlayTest, NewOverlayDrawsUnchangedSubtitles) { + ASSERT_GT(draw(1000).images, 0) << kNoImages; + assOverlayRelease(overlay); + overlay = nullptr; + EXPECT_EQ(draw(1000).changed, 0); +} + +// The first draw creates the overlay, and later ones reuse it. +TEST_F(AssOverlayTest, KeepsItsOverlay) { + draw(1000); + const AssOverlay* first = overlay; + ASSERT_NE(first, nullptr); + draw(1500); + EXPECT_EQ(overlay, first); +} + +// When libass reports no change, the overlay draws the atlas it already has: each draw is onto a new +// video frame, without the subtitles on it. +TEST_F(AssOverlayTest, RedrawsUnchangedSubtitlesOnEveryFrame) { + ASSERT_GT(draw(1000).images, 0) << kNoImages; + for (int i = 0; i < 3; i++) { + EXPECT_EQ(draw(1000).changed, 0); + } +} + +// When libass reports that only positions changed, the subtitles move. +TEST_F(AssOverlayTest, FollowsMovingSubtitles) { + ASSERT_GT(draw(1000).images, 0) << kNoImages; + EXPECT_EQ(draw(1500).changed, 1); +} + +// When the subtitles change, the new ones replace them. +TEST_F(AssOverlayTest, FollowsChangingSubtitles) { + ASSERT_GT(draw(1000).images, 0) << kNoImages; + const Drawn drawn = draw(2500); // the triangle is gone + EXPECT_EQ(drawn.changed, 2); + EXPECT_GT(drawn.images, 0); +} + +// Once the events end, libass reports a change to no images: nothing is drawn, then or on the next +// unchanged frame. +TEST_F(AssOverlayTest, ClearsSubtitlesWhenEventsEnd) { + ASSERT_GT(draw(2500).images, 0) << kNoImages; + const Drawn ended = draw(3500); + EXPECT_EQ(ended.changed, 2); + EXPECT_EQ(ended.images, 0); + EXPECT_EQ(draw(3600).changed, 0); +} + +// A new frame size goes to libass, which renders the subtitles for it, and the quads are laid out +// for it, growing or shrinking. +TEST_F(AssOverlayTest, FollowsFrameSizeChanges) { + ASSERT_GT(draw(1000, 64, 48).images, 0) << kNoImages; + EXPECT_EQ(draw(1000, 128, 96).changed, 2); + EXPECT_EQ(draw(1000, 40, 30).changed, 2); +} + +// Without a renderer or a track, the overlay is still created, for the next call, but draws nothing. +TEST_F(AssOverlayTest, DrawsNothingWithoutARendererOrATrack) { + const VideoFrame frame(64, 48); + overlay = assOverlayDraw(nullptr, nullptr, track, frame.fbo(), 64, 48, 1000); + ASSERT_NE(overlay, nullptr); + EXPECT_EQ(assOverlayDraw(overlay, renderer, nullptr, frame.fbo(), 64, 48, 1000), overlay); + EXPECT_EQ(glGetError(), (GLenum) GL_NO_ERROR); + expectFrame(frame.read(), expectedFrame(nullptr, 64, 48), 64); + + // It draws once it has both. + EXPECT_GT(draw(1000).images, 0) << kNoImages; +} + +// The scene drawn frame by frame at 23.976 fps, as a video plays. +TEST_F(AssOverlayTest, PlaysAScene) { + readTrack(kScene); + int shown = 0; + int shownUnchanged = 0; + for (long long frame = 0;; frame++) { + const long long timeMs = frame * 1001 / 24; + if (timeMs > 4200) { + break; + } + SCOPED_TRACE(testing::Message() << "frame " << frame << ", at " << timeMs << " ms"); + const Drawn drawn = draw(timeMs, 320, 180); + if (drawn.images > 0) { + shown++; + if (drawn.changed == 0) { + shownUnchanged++; + } + } + if (HasFailure()) { + return; + } + } + EXPECT_GT(shown, 0) << kNoImages; + EXPECT_GT(shownUnchanged, 0) << "the line from 3s on doesn't stay still"; +} + +TEST(AssOverlay, ReleaseTakesNull) { + assOverlayRelease(nullptr); +} diff --git a/lib_ass_kt/src/main/cpp/overlay/tests/AssSkylineTest.cpp b/lib_ass_kt/src/main/cpp/overlay/tests/AssSkylineTest.cpp new file mode 100644 index 0000000..70f48fa --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/tests/AssSkylineTest.cpp @@ -0,0 +1,159 @@ +#include + +#include +#include +#include +#include +#include + +#include "AssSkyline.h" + +namespace { + +using Point = std::pair; + +class AssSkylineTest : public testing::Test { +protected: + void TearDown() override { + assSkylineFree(&skyline); + } + + void reset(int count, int width, int maxHeight) { + assSkylineFree(&skyline); + skyline = {}; + ASSERT_TRUE(assSkylineReserve(&skyline, count)); + assSkylineReset(&skyline, width, maxHeight); + } + + std::optional add(int w, int h) { + int x, y; + if (!assSkylineAdd(&skyline, w, h, &x, &y)) { + return std::nullopt; + } + return Point(x, y); + } + + // A rectangle that fits nowhere leaves x and y as they were. RandomRectangles checks it leaves + // the skyline as it was too, through the rectangles placed after it. + void expectRejected(int w, int h) { + int x = -1, y = -1; + EXPECT_FALSE(assSkylineAdd(&skyline, w, h, &x, &y)) << w << "x" << h; + EXPECT_EQ(Point(x, y), Point(-1, -1)); + } + + AssSkyline skyline{}; +}; + +} // namespace + +// The example from AssSkyline.h. +TEST_F(AssSkylineTest, HeaderExample) { + reset(4, 10, 100); + + EXPECT_EQ(add(4, 5), Point(0, 0)); // A + EXPECT_EQ(add(3, 4), Point(4, 0)); // B + EXPECT_EQ(add(5, 2), Point(4, 4)); // C + EXPECT_EQ(add(2, 2), Point(0, 5)); // D +} + +TEST_F(AssSkylineTest, RejectsTooWide) { + reset(1, 10, 100); + + expectRejected(11, 1); +} + +TEST_F(AssSkylineTest, RejectsTooTall) { + reset(2, 10, 100); + + ASSERT_EQ(add(10, 60), Point(0, 0)); + expectRejected(1, 41); +} + +TEST_F(AssSkylineTest, FillsRemainingHeight) { + reset(2, 10, 100); + + ASSERT_EQ(add(10, 60), Point(0, 0)); + EXPECT_EQ(add(1, 40), Point(0, 60)); +} + +// Of two spots as high, the leftmost wins. +// 0123456789 +// 0 AAABBBCCCC +// 1 AAABBBCCCC +// 2 DDDBBB.... +// 3 ...BBB.... +// 4 ...BBB.... +TEST_F(AssSkylineTest, TiesGoLeftmost) { + reset(4, 10, 5); + + ASSERT_EQ(add(3, 2), Point(0, 0)); + ASSERT_EQ(add(3, 5), Point(3, 0)); + ASSERT_EQ(add(4, 2), Point(6, 0)); + EXPECT_EQ(add(3, 1), Point(0, 2)); +} + +// A reset forgets what was packed, area size included, and keeps the room reserved. +TEST_F(AssSkylineTest, ResetStartsOver) { + reset(2, 10, 100); + ASSERT_EQ(add(10, 60), Point(0, 0)); + + assSkylineReset(&skyline, 5, 10); + expectRejected(6, 1); + EXPECT_EQ(add(5, 10), Point(0, 0)); + expectRejected(1, 1); +} + +// Rectangles each one column wide and one row shorter than the one before, with a column left free +// after them, leave a staircase: the most steps as many rectangles can. Room for exactly that many +// must still hold them; taking more only shows under ASan. +TEST_F(AssSkylineTest, ReservedRoomSuffices) { + const int count = 64; + reset(count, count + 1, count); + + for (int i = 0; i < count; i++) { + EXPECT_EQ(add(1, count - i), Point(i, 0)); + } +} + +// Random rectangles into random areas, checked against a model of the area: the bottom of what is +// packed over each column. Each rectangle must go at the highest spot it fits, then the leftmost - +// trying every column - or be rejected. +TEST_F(AssSkylineTest, RandomRectangles) { + std::mt19937 rng(20261014); + // rng() % n rather than std::uniform_int_distribution, whose algorithm varies between standard + // libraries: a seed makes the same data everywhere. + const auto random = [&rng](int min, int max) { return min + (int) (rng() % (uint32_t) (max - min + 1)); }; + for (int round = 0; round < 2000; round++) { + SCOPED_TRACE(testing::Message() << "round " << round); + const int width = random(1, 48); + const int maxHeight = random(1, 48); + const int count = random(1, 40); + reset(count, width, maxHeight); + std::vector bottom((size_t) width, 0); + + for (int k = 0; k < count; k++) { + // Mostly small, at times too wide. + const int w = random(1, random(1, width + 1)); + const int h = random(1, random(1, maxHeight / 2 + 1)); + int bestX = -1; + int bestY = maxHeight - h + 1; + for (int x = 0; x + w <= width; x++) { + const int y = *std::max_element(bottom.begin() + x, bottom.begin() + x + w); + if (y < bestY) { + bestX = x; + bestY = y; + } + } + + if (bestX < 0) { + expectRejected(w, h); + } else { + EXPECT_EQ(add(w, h), Point(bestX, bestY)) << w << "x" << h; + std::fill(bottom.begin() + bestX, bottom.begin() + bestX + w, bestY + h); + } + if (HasFailure()) { + return; + } + } + } +} diff --git a/lib_ass_kt/src/main/cpp/overlay/tests/AssTestImages.h b/lib_ass_kt/src/main/cpp/overlay/tests/AssTestImages.h new file mode 100644 index 0000000..21865ed --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/tests/AssTestImages.h @@ -0,0 +1,67 @@ +#ifndef LIBASS_ANDROID_ASS_TEST_IMAGES_H +#define LIBASS_ANDROID_ASS_TEST_IMAGES_H + +#include +#include +#include +#include + +#include "ass/ass.h" + +// Stand-ins for libass's output, shared by the tests. + +// A w x h bitmap with rows stride bytes apart. Each texel's value depends on seed and its position, +// so a piece copied from the wrong place or with the wrong stride doesn't match. +struct Bitmap { + int w, h, stride; + std::vector data; +}; + +inline Bitmap makeBitmap(int w, int h, int seed, int stride = 0) { + Bitmap bitmap{w, h, stride > 0 ? stride : w, {}}; + // Padding past w is never sampled, so it must never reach a page. + bitmap.data.assign((size_t) bitmap.stride * h, 0xEE); + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + bitmap.data[(size_t) y * bitmap.stride + x] = (unsigned char) (seed * 31 + y * w + x); + } + } + return bitmap; +} + +// A libass image list, as ass_render_frame returns it. +class ImageList { +public: + void add(const Bitmap& bitmap, int dstX = 0, int dstY = 0, uint32_t color = 0) { + addRaw(bitmap.w, bitmap.h, bitmap.stride, bitmap.data.data(), dstX, dstY, color); + } + + void addRaw(int w, int h, int stride, const unsigned char* bitmap, int dstX = 0, int dstY = 0, uint32_t color = 0) { + ASS_Image img{}; + img.w = w; + img.h = h; + img.stride = stride; + img.bitmap = const_cast(bitmap); + img.color = color; + img.dst_x = dstX; + img.dst_y = dstY; + images_.push_back(img); + } + + const ASS_Image* head() { + for (size_t i = 0; i < images_.size(); i++) { + images_[i].next = i + 1 < images_.size() ? &images_[i + 1] : nullptr; + } + return images_.empty() ? nullptr : &images_[0]; + } + + const std::deque& images() const { + return images_; + } + +private: + // A deque, so that adding an image never moves the earlier ones. + std::deque images_; +}; + +#endif // LIBASS_ANDROID_ASS_TEST_IMAGES_H diff --git a/lib_ass_kt/src/main/cpp/overlay/tests/CMakeLists.txt b/lib_ass_kt/src/main/cpp/overlay/tests/CMakeLists.txt new file mode 100644 index 0000000..840a9d1 --- /dev/null +++ b/lib_ass_kt/src/main/cpp/overlay/tests/CMakeLists.txt @@ -0,0 +1,28 @@ +enable_language(CXX) + +# Give the extracted files the extraction time, so a GoogleTest upgrade rebuilds everything it touches. +if (POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif () + +include(FetchContent) +FetchContent_Declare(googletest + URL https://github.com/google/googletest/releases/download/v1.18.0/googletest-1.18.0.tar.gz + URL_HASH SHA256=6e3191c1455468b3fc35a417fb565c1c5071aee1b7e7f85e30cf48a98d37d8b5 +) +set(BUILD_GMOCK OFF) +set(INSTALL_GTEST OFF) +FetchContent_MakeAvailable(googletest) +include(GoogleTest) + +foreach (test AssAtlasTest AssOverlayGlTest AssOverlayTest AssSkylineTest) + add_executable(${test} ${test}.cpp) + target_link_libraries(${test} PRIVATE ass_overlay GTest::gtest_main) + gtest_discover_tests(${test} PROPERTIES TIMEOUT 30) +endforeach () + +pkg_check_modules(EGL REQUIRED IMPORTED_TARGET egl) +foreach (test AssOverlayGlTest AssOverlayTest) + target_sources(${test} PRIVATE AssGlTest.cpp) + target_link_libraries(${test} PRIVATE ${ASS_OVERLAY_GL} PkgConfig::EGL) +endforeach () diff --git a/lib_ass_kt/src/main/java/io/github/peerless2012/ass/AssRender.kt b/lib_ass_kt/src/main/java/io/github/peerless2012/ass/AssRender.kt index 877cbdf..3ce7523 100644 --- a/lib_ass_kt/src/main/java/io/github/peerless2012/ass/AssRender.kt +++ b/lib_ass_kt/src/main/java/io/github/peerless2012/ass/AssRender.kt @@ -34,6 +34,12 @@ class AssRender(nativeAss: Long, private val lock: ReentrantLock) { @JvmStatic external fun nativeAssRenderDeinit(render: Long) + + @JvmStatic + external fun nativeAssOverlayDraw(overlay: Long, render: Long, track: Long, fbo: Int, frameWidth: Int, frameHeight: Int, timeMs: Long): Long + + @JvmStatic + external fun nativeAssOverlayRelease(overlay: Long) } private var nativeRender: Long = nativeAssRenderInit(nativeAss) @@ -87,6 +93,25 @@ class AssRender(nativeAss: Long, private val lock: ReentrantLock) { } } + // Only for EFFECTS_ATLAS. Draws onto `fbo` with `overlay`, the handle the previous call returned (0 + // the first time), and returns the handle to pass next. An overlay's GL objects belong to the GL + // context it was created in, so the caller keeps one per context. + fun drawOverlayFrame(overlay: Long, fbo: Int, frameWidth: Int, frameHeight: Int, timeMs: Long): Long { + lock.withLock { + if (released || nativeRender == 0L) return overlay + val t = track ?: return overlay + if (t.released || t.nativeAssTrack == 0L) return overlay + return nativeAssOverlayDraw(overlay, nativeRender, t.nativeAssTrack, fbo, frameWidth, frameHeight, timeMs) + } + } + + // Only for EFFECTS_ATLAS. Frees an overlay from drawOverlayFrame, on its GL context's thread. + fun releaseOverlay(overlay: Long) { + if (overlay != 0L) { + nativeAssOverlayRelease(overlay) + } + } + fun release() { lock.withLock { if (released) return diff --git a/lib_ass_media/README.md b/lib_ass_media/README.md index 5387d94..9bdd1d1 100644 --- a/lib_ass_media/README.md +++ b/lib_ass_media/README.md @@ -4,13 +4,14 @@ A media3 extension library for libass. Apps that use media3 can use this module to add libass into their player. ## Feature -There are three ways to render ass subtitles which are defined in `AssRenderType`. +There are multiple ways to render ass subtitles which are defined in `AssRenderType`. | Type | Feature | Anim | HDR/DV | Block render/UI | | :----: | :----: | :----: | :----: | :----: | | CUES | SubtitleView & Cue | ❌ | ✅ | ❌ | | EFFECTS_CANVAS | Effect | ✅ | ❓ | ✅ | | EFFECTS_OPEN_GL | Effect | ✅ | ❓ | ✅ | +| EFFECTS_ATLAS | Effect | ✅ | ❓ | ✅ | | OVERLAY_CANVAS | Overlay | ✅ | ✅ | ✅ | | OVERLAY_OPEN_GL | Overlay | ✅ | ✅ | ❌ | @@ -36,14 +37,17 @@ Just like `EFFECTS_CANVAS`, but uses OpenGL to render. and the offscreen text is According to tests, the `EFFECTS_OPEN_GL` will take a 1/3 of the time to render. -### 4. OVERLAY_CANVAS +### 4. EFFECTS_ATLAS +The fastest render type. It use OpenGL to render, but it use a atlas to push the libass bitmap to the GPU which makes it much more performant than any other mode. It have similar performance to mpv. + +### 5. OVERLAY_CANVAS The ass/ssa subtitles will be calculated at runtime, and an `Overlay` widget is added in `SubtitleView` to render the subtitles. The `libass` render result will copy into a bitmap, and draw in the `Canvas`. This will block UI thread when rendering. -### 4. OVERLAY_OPEN_GL +### 6. OVERLAY_OPEN_GL Just like `OVERLAY_CANVAS`, but the `libass` render result will passed to `OpenGL` as a texture to avoid creating temporary bitmap. It uses about half as much memory as `OVERLAY_CANVAS`. @@ -123,7 +127,7 @@ player = ExoPlayer.Builder(this) | :----: | :----: | :---- | | `glyphSize` | 10000 | Maximum number of glyph cache entries in libass | | `cacheSize` | 128 | Maximum bitmap cache size in MB for libass | -| `maxRenderPixels` | 0 | Maximum pixel count for subtitle rendering (0 = no limit) | +| `maxRenderPixels` | 0 | Maximum pixel count for subtitle rendering (0 = no limit). Ignored by `EFFECTS_ATLAS` | ### Render Downscaling (`maxRenderPixels`) @@ -157,5 +161,6 @@ Actual render size: 1920 x 1080 - **OVERLAY_OPEN_GL**: Renders at reduced size, scales `glViewport` coordinates to surface size - **OVERLAY_CANVAS**: Renders at reduced size, uses `drawBitmap` with scaled `RectF` - **EFFECTS_OPEN_GL**: Creates smaller FBO, uses `getVertexTransformation` to scale up in the OverlayEffect pipeline +- **EFFECTS_ATLAS**: It always renders at video size - **EFFECTS_CANVAS**: Renders at reduced size, scales canvas draw coordinates - **CUES**: Not affected (pre-renders all subtitles at parse time) diff --git a/lib_ass_media/build.gradle.kts b/lib_ass_media/build.gradle.kts index 82c1cee..2f92bc4 100644 --- a/lib_ass_media/build.gradle.kts +++ b/lib_ass_media/build.gradle.kts @@ -11,7 +11,7 @@ android { compileSdk = 36 defaultConfig { - minSdk = 21 + minSdk = 23 testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" consumerProguardFiles("consumer-rules.pro") } diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt index 159034d..2d49e9d 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandler.kt @@ -17,6 +17,7 @@ import io.github.peerless2012.ass.AssRender import io.github.peerless2012.ass.AssTrack import io.github.peerless2012.ass.Ass import io.github.peerless2012.ass.media.parser.AssHeaderParser +import io.github.peerless2012.ass.media.render.AssFrameTimeMapper import io.github.peerless2012.ass.media.render.AssOverlayManager import io.github.peerless2012.ass.media.type.AssRenderType @@ -87,6 +88,13 @@ class AssHandler( var videoTimeCallback: ((Long) -> Unit)? = null + /** Maps video effect timestamps to media time. Fed by the video renderer for [AssRenderType.EFFECTS_ATLAS]. */ + internal val frameTimeMapper = AssFrameTimeMapper() + + /** [AssRenderType.EFFECTS_ATLAS]'s overlay is the only one to set the frame size: it uses each video + * frame's size, which can differ from the video track's size (pixel aspect ratio, rotation). */ + private val frameSizeSetByOverlay = renderType == AssRenderType.EFFECTS_ATLAS + /** The overlay manager for toggling the effects renderer. */ private var overlayManager: AssOverlayManager? = null @@ -103,8 +111,8 @@ class AssHandler( fun init(player: ExoPlayer) { player.addListener(this) handler = Handler(player.applicationLooper) - if (renderType == AssRenderType.EFFECTS_CANVAS || renderType == AssRenderType.EFFECTS_OPEN_GL) { - overlayManager = AssOverlayManager(this, player, renderType == AssRenderType.EFFECTS_OPEN_GL) + if (renderType == AssRenderType.EFFECTS_CANVAS || renderType == AssRenderType.EFFECTS_OPEN_GL || renderType == AssRenderType.EFFECTS_ATLAS) { + overlayManager = AssOverlayManager(this, player, renderType) } } @@ -171,7 +179,7 @@ class AssHandler( if (renderType == AssRenderType.OVERLAY_CANVAS || renderType == AssRenderType.OVERLAY_OPEN_GL) { val renderSize = computeRenderSize(surfaceSize.width, surfaceSize.height) render.setFrameSize(renderSize.width, renderSize.height) - } else { + } else if (!frameSizeSetByOverlay) { val renderSize = computeRenderSize(videoSize.width, videoSize.height) render.setFrameSize(renderSize.width, renderSize.height) } @@ -282,7 +290,7 @@ class AssHandler( if (videoSize.isValid) { render.setStorageSize(videoSize.width, videoSize.height) } - if (videoSize.isValid) { + if (videoSize.isValid && !frameSizeSetByOverlay) { val renderSize = computeRenderSize(videoSize.width, videoSize.height) render.setFrameSize(renderSize.width, renderSize.height) } @@ -291,7 +299,7 @@ class AssHandler( val renderSize = computeRenderSize(surfaceSize.width, surfaceSize.height) render.setFrameSize(renderSize.width, renderSize.height) } - } else { + } else if (!frameSizeSetByOverlay) { if (videoSize.isValid) { val renderSize = computeRenderSize(videoSize.width, videoSize.height) render.setFrameSize(renderSize.width, renderSize.height) diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandlerConfig.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandlerConfig.kt index 71ca5c0..e3db425 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandlerConfig.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/AssHandlerConfig.kt @@ -18,7 +18,8 @@ data class AssHandlerConfig( * - 2560 * 1440 = 3_686_400 (limit to 1440p) * - 0 = no limit, render at full frame size (default) * - * Only applies to OVERLAY and EFFECTS render types. CUES mode is not affected. + * Only applies to OVERLAY and EFFECTS render types, except EFFECTS_ATLAS, which + * always renders at video size. CUES mode is not affected. */ val maxRenderPixels: Int = 0 ) diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/factory/AssRenderersFactory.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/factory/AssRenderersFactory.kt index c25ba1d..1d85cc0 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/factory/AssRenderersFactory.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/factory/AssRenderersFactory.kt @@ -1,15 +1,19 @@ package io.github.peerless2012.ass.media.factory import android.os.Handler +import android.util.Log import androidx.media3.common.util.UnstableApi import androidx.media3.exoplayer.Renderer import androidx.media3.exoplayer.RenderersFactory import androidx.media3.exoplayer.audio.AudioRendererEventListener import androidx.media3.exoplayer.metadata.MetadataOutput import androidx.media3.exoplayer.text.TextOutput +import androidx.media3.exoplayer.video.MediaCodecVideoRenderer import androidx.media3.exoplayer.video.VideoRendererEventListener import io.github.peerless2012.ass.media.AssHandler import io.github.peerless2012.ass.media.render.AssRenderer +import io.github.peerless2012.ass.media.render.AssVideoRendererWrapper +import io.github.peerless2012.ass.media.type.AssRenderType /** * @Author peerless2012 @@ -27,8 +31,17 @@ class AssRenderersFactory(private val assHandler: AssHandler, private val render textRendererOutput: TextOutput, metadataRendererOutput: MetadataOutput ): Array { - val renderers = renderersFactory.createRenderers(eventHandler, videoRendererEventListener, audioRendererEventListener, textRendererOutput, metadataRendererOutput) - return (renderers.toMutableList() + AssRenderer(assHandler)).toTypedArray() + val renderers = renderersFactory.createRenderers(eventHandler, videoRendererEventListener, audioRendererEventListener, textRendererOutput, metadataRendererOutput).toMutableList() + if (assHandler.renderType == AssRenderType.EFFECTS_ATLAS) { + // Only MediaCodecVideoRenderer supports video effects. + val index = renderers.indexOfFirst { it is MediaCodecVideoRenderer } + if (index >= 0) { + renderers[index] = AssVideoRendererWrapper(renderers[index], assHandler.frameTimeMapper) + } else { + Log.e("AssRenderersFactory", "No MediaCodecVideoRenderer found, EFFECTS_ATLAS subtitles will be drawn at the wrong time or not at all") + } + } + return (renderers + AssRenderer(assHandler)).toTypedArray() } override fun createSecondaryRenderer( @@ -39,6 +52,8 @@ class AssRenderersFactory(private val assHandler: AssHandler, private val render textRendererOutput: TextOutput, metadataRendererOutput: MetadataOutput ): Renderer? { + // A pre-warming video renderer would send frames whose timestamps the frame time mapper doesn't track. + if (renderer is AssVideoRendererWrapper) return null return renderersFactory.createSecondaryRenderer( renderer, eventHandler, diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssFrameTimeMapper.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssFrameTimeMapper.kt new file mode 100644 index 0000000..b0236b2 --- /dev/null +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssFrameTimeMapper.kt @@ -0,0 +1,72 @@ +package io.github.peerless2012.ass.media.render + +import androidx.media3.common.C + +/** + * Maps the presentation time a video effect receives back to the media time of the frame, which is + * the time base of the ASS events. + * + * With video effects, `MediaCodecVideoRenderer` shifts each buffer timestamp (renderer time, which + * is media time + stream offset) by `-startPositionUs` before the frame reaches the effects. + * `startPositionUs` is the start position of the first stream the renderer is enabled with after + * it is created or reset, and it keeps that value across seeks, disable/enable cycles and playlist + * transitions. This class mirrors that from the calls [AssVideoRendererWrapper] sees, and keeps + * every stream's offset so that a frame is mapped with the offset of the stream it belongs to. + * + * The renderer calls come from the playback thread and [toMediaTimeUs] from the GL thread. + */ +internal class AssFrameTimeMapper { + + private class Stream(val startPositionUs: Long, val offsetUs: Long) + + /** The renderer time that media3 subtracts from every frame timestamp. */ + private var baseUs = C.TIME_UNSET + + /** In renderer-time order. Each stream ends where the next one starts. */ + private val streams = ArrayDeque() + + @Synchronized + fun onEnabled(startPositionUs: Long, offsetUs: Long) { + if (baseUs == C.TIME_UNSET) { + baseUs = startPositionUs + } + streams.clear() + streams.addLast(Stream(startPositionUs, offsetUs)) + } + + @Synchronized + fun onStreamReplaced(startPositionUs: Long, offsetUs: Long) { + streams.addLast(Stream(startPositionUs, offsetUs)) + } + + /** Like `MediaCodecRenderer`, only the last stream is output after a position reset. */ + @Synchronized + fun onPositionReset() { + while (streams.size > 1) { + streams.removeFirst() + } + } + + @Synchronized + fun onReset() { + baseUs = C.TIME_UNSET + streams.clear() + } + + /** + * Returns the media time of the frame with the given effect presentation time, or + * [C.TIME_UNSET] if the video renderer hasn't been enabled. + */ + @Synchronized + fun toMediaTimeUs(framePresentationTimeUs: Long): Long { + if (baseUs == C.TIME_UNSET || streams.isEmpty()) { + return C.TIME_UNSET + } + val rendererTimeUs = framePresentationTimeUs + baseUs + // Frames arrive in order, so the streams that ended before this frame aren't needed anymore. + while (streams.size > 1 && streams[1].startPositionUs <= rendererTimeUs) { + streams.removeFirst() + } + return rendererTimeUs - streams.first().offsetUs + } +} diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssGlEffect.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssGlEffect.kt new file mode 100644 index 0000000..6c49d67 --- /dev/null +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssGlEffect.kt @@ -0,0 +1,28 @@ +package io.github.peerless2012.ass.media.render + +import android.content.Context +import android.util.Log +import androidx.annotation.OptIn +import androidx.media3.common.util.UnstableApi +import androidx.media3.effect.GlEffect +import androidx.media3.effect.GlShaderProgram +import io.github.peerless2012.ass.AssRender +import io.github.peerless2012.ass.media.AssHandler + +@OptIn(UnstableApi::class) +class AssGlEffect( + private val handler: AssHandler, + private val render: AssRender +) : GlEffect { + + override fun toGlShaderProgram(context: Context, useHdr: Boolean): GlShaderProgram { + if (useHdr) { + Log.w(TAG, "HDR is not supported: subtitles are blended as 8-bit sRGB and colors will be wrong") + } + return AssGlShaderProgram(handler, render) + } + + companion object { + private const val TAG = "AssGlEffect" + } +} diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssGlShaderProgram.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssGlShaderProgram.kt new file mode 100644 index 0000000..f1bbecf --- /dev/null +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssGlShaderProgram.kt @@ -0,0 +1,133 @@ +package io.github.peerless2012.ass.media.render + +import android.util.Log +import androidx.annotation.OptIn +import androidx.media3.common.C +import androidx.media3.common.GlObjectsProvider +import androidx.media3.common.GlTextureInfo +import androidx.media3.common.VideoFrameProcessingException +import androidx.media3.common.util.GlUtil +import androidx.media3.common.util.UnstableApi +import androidx.media3.effect.GlShaderProgram +import io.github.peerless2012.ass.AssRender +import io.github.peerless2012.ass.media.AssHandler +import java.util.concurrent.Executor + +/** + * Draws ASS subtitles in place onto each video frame's own texture and forwards that same texture + * downstream. + * + * Implements [GlShaderProgram] directly instead of extending `BaseGlShaderProgram`, which always + * renders into a new, cleared output texture: every frame would first need a full-frame copy of the + * video before the subtitles could be drawn on top. Drawing in place avoids that copy, so a frame + * without visible subtitles costs no GPU work at all, and it saves a full-size output texture. + * + * Subtitles are rendered synchronously on the GL thread by [AssRender.drawOverlayFrame], so every + * frame is accurate, but a pathologically slow subtitle frame stalls video delivery. + */ +@OptIn(UnstableApi::class) +class AssGlShaderProgram( + private val handler: AssHandler, + private val render: AssRender +) : GlShaderProgram { + + private var inputListener: GlShaderProgram.InputListener = object : GlShaderProgram.InputListener {} + private var outputListener: GlShaderProgram.OutputListener = object : GlShaderProgram.OutputListener {} + private var errorListener = GlShaderProgram.ErrorListener {} + private var errorExecutor = Executor { it.run() } + + /** True while downstream holds the frame we forwarded. We accept one frame at a time, so this + * mirrors `BaseGlShaderProgram.setInputListener`, which only signals readiness for free output + * textures: a new input listener mustn't be told we're ready while a frame is still held. */ + private var frameInFlight = false + + /** Only populated when upstream doesn't already hand us an FBO for a given texture (fboId + * unset) — keyed by texId so a texture media3's pool reuses doesn't leak a new FBO every time. */ + private val fallbackFbos = HashMap() + + /** The native overlay, created by the first draw. Its GL objects belong to this program's GL + * context, so it's this program's alone, never shared through [render]. */ + private var overlay = 0L + + override fun setInputListener(listener: GlShaderProgram.InputListener) { + inputListener = listener + if (!frameInFlight) { + listener.onReadyToAcceptInputFrame() + } + } + + override fun setOutputListener(listener: GlShaderProgram.OutputListener) { + outputListener = listener + } + + override fun setErrorListener(executor: Executor, listener: GlShaderProgram.ErrorListener) { + errorExecutor = executor + errorListener = listener + } + + override fun queueInputFrame( + glObjectsProvider: GlObjectsProvider, + inputTexture: GlTextureInfo, + presentationTimeUs: Long + ) { + try { + val fbo = if (inputTexture.fboId != C.INDEX_UNSET) { + inputTexture.fboId + } else { + fallbackFbos.getOrPut(inputTexture.texId) { + GlUtil.createFboForTexture(inputTexture.texId) + } + } + // presentationTimeUs isn't media time, see AssFrameTimeMapper. Without a mapping (the player + // wasn't built with AssRenderersFactory), fall back to it like the other effect overlays do. + val mediaTimeUs = handler.frameTimeMapper.toMediaTimeUs(presentationTimeUs) + .takeIf { it != C.TIME_UNSET } ?: presentationTimeUs + val startNs = System.nanoTime() + overlay = render.drawOverlayFrame( + overlay, + fbo, + inputTexture.width, + inputTexture.height, + mediaTimeUs / 1000 + ) + val elapsedMs = (System.nanoTime() - startNs) / 1_000_000 + if (elapsedMs >= SLOW_FRAME_LOG_THRESHOLD_MS) { + Log.w(TAG, "drawOverlayFrame took ${elapsedMs}ms for presentationTimeUs=$presentationTimeUs") + } + } catch (e: Exception) { + errorExecutor.execute { + errorListener.onError(VideoFrameProcessingException.from(e, presentationTimeUs)) + } + } + frameInFlight = true + outputListener.onOutputFrameAvailable(inputTexture, presentationTimeUs) + } + + override fun releaseOutputFrame(outputTexture: GlTextureInfo) { + frameInFlight = false + inputListener.onInputFrameProcessed(outputTexture) + inputListener.onReadyToAcceptInputFrame() + } + + override fun signalEndOfCurrentInputStream() { + outputListener.onCurrentOutputStreamEnded() + } + + override fun flush() { + frameInFlight = false + inputListener.onFlush() + inputListener.onReadyToAcceptInputFrame() + } + + override fun release() { + fallbackFbos.values.forEach { GlUtil.deleteFbo(it) } + fallbackFbos.clear() + render.releaseOverlay(overlay) + overlay = 0L + } + + private companion object { + private const val TAG = "AssGlShaderProgram" + private const val SLOW_FRAME_LOG_THRESHOLD_MS = 8 + } +} diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssOverlayManager.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssOverlayManager.kt index 287b0ff..c3068c5 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssOverlayManager.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssOverlayManager.kt @@ -1,17 +1,19 @@ package io.github.peerless2012.ass.media.render import androidx.annotation.OptIn +import androidx.media3.common.Effect import androidx.media3.common.util.UnstableApi import androidx.media3.effect.OverlayEffect import androidx.media3.exoplayer.ExoPlayer import io.github.peerless2012.ass.AssRender import io.github.peerless2012.ass.media.AssHandler +import io.github.peerless2012.ass.media.type.AssRenderType @OptIn(UnstableApi::class) class AssOverlayManager( private val handler: AssHandler, private val player: ExoPlayer, - private val tex: Boolean + private val renderType: AssRenderType ) { private var currentRenderer : AssRender? = null @@ -23,12 +25,11 @@ class AssOverlayManager( fun enable(renderer: AssRender) { if (renderer == currentRenderer) return this.currentRenderer = renderer - val overlay = if (tex) { - AssTexOverlay(handler, renderer) - } else { - AssCanvasOverlay(handler, renderer) + val effect: Effect = when (renderType) { + AssRenderType.EFFECTS_ATLAS -> AssGlEffect(handler, renderer) + AssRenderType.EFFECTS_OPEN_GL -> OverlayEffect(listOf(AssTexOverlay(handler, renderer))) + else -> OverlayEffect(listOf(AssCanvasOverlay(handler, renderer))) } - val effect = OverlayEffect(listOf(overlay)) player.setVideoEffects(listOf(effect)) } diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssVideoRendererWrapper.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssVideoRendererWrapper.kt new file mode 100644 index 0000000..807df9a --- /dev/null +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/render/AssVideoRendererWrapper.kt @@ -0,0 +1,66 @@ +package io.github.peerless2012.ass.media.render + +import androidx.media3.common.Format +import androidx.media3.common.util.UnstableApi +import androidx.media3.exoplayer.ForwardingRenderer +import androidx.media3.exoplayer.Renderer +import androidx.media3.exoplayer.RendererConfiguration +import androidx.media3.exoplayer.source.MediaSource +import androidx.media3.exoplayer.source.SampleStream + +/** + * Wraps the player's `MediaCodecVideoRenderer` to report the stream changes that decide the + * timestamps its video effects receive to [mapper]. + */ +@UnstableApi +internal class AssVideoRendererWrapper( + renderer: Renderer, + private val mapper: AssFrameTimeMapper +) : ForwardingRenderer(renderer) { + + override fun enable( + configuration: RendererConfiguration, + formats: Array, + stream: SampleStream, + positionUs: Long, + joining: Boolean, + mayRenderStartOfStream: Boolean, + startPositionUs: Long, + offsetUs: Long, + mediaPeriodId: MediaSource.MediaPeriodId + ) { + mapper.onEnabled(startPositionUs, offsetUs) + super.enable( + configuration, + formats, + stream, + positionUs, + joining, + mayRenderStartOfStream, + startPositionUs, + offsetUs, + mediaPeriodId + ) + } + + override fun replaceStream( + formats: Array, + stream: SampleStream, + startPositionUs: Long, + offsetUs: Long, + mediaPeriodId: MediaSource.MediaPeriodId + ) { + mapper.onStreamReplaced(startPositionUs, offsetUs) + super.replaceStream(formats, stream, startPositionUs, offsetUs, mediaPeriodId) + } + + override fun resetPosition(positionUs: Long, sampleStreamIsResetToKeyFrame: Boolean) { + mapper.onPositionReset() + super.resetPosition(positionUs, sampleStreamIsResetToKeyFrame) + } + + override fun reset() { + mapper.onReset() + super.reset() + } +} diff --git a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/type/AssRenderType.kt b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/type/AssRenderType.kt index ef050ca..581b0ac 100644 --- a/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/type/AssRenderType.kt +++ b/lib_ass_media/src/main/java/io/github/peerless2012/ass/media/type/AssRenderType.kt @@ -22,6 +22,14 @@ enum class AssRenderType { @Deprecated("Use OVERLAY instead.") EFFECTS_OPEN_GL, + /** + * Use Effect(Powered by a native GLES `GlEffect`/`GlShaderProgram` with a GPU texture atlas). + * + * This is the most performant and frame accurate mode. + * It doesn't support HDR and Dolby Vision. + */ + EFFECTS_ATLAS, + /** * Use Widget overlay(Powered by Canvas). */ diff --git a/lib_ass_media/src/test/java/io/github/peerless2012/ass/media/render/AssFrameTimeMapperTest.kt b/lib_ass_media/src/test/java/io/github/peerless2012/ass/media/render/AssFrameTimeMapperTest.kt new file mode 100644 index 0000000..a03f16a --- /dev/null +++ b/lib_ass_media/src/test/java/io/github/peerless2012/ass/media/render/AssFrameTimeMapperTest.kt @@ -0,0 +1,100 @@ +package io.github.peerless2012.ass.media.render + +import androidx.media3.common.C +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * Each test feeds the mapper the renderer calls ExoPlayer makes, builds the timestamp media3 gives + * the effect for a frame (buffer time - startPositionUs of the first stream since reset), and + * checks that the frame's media time comes back. + */ +class AssFrameTimeMapperTest { + + private val mapper = AssFrameTimeMapper() + + @Test + fun unsetBeforeEnabled() { + assertEquals(C.TIME_UNSET, mapper.toMediaTimeUs(0)) + } + + @Test + fun playbackStartedAtZero() { + mapper.onEnabled(startPositionUs = OFFSET, offsetUs = OFFSET) + + assertEquals(5 * S, mapper.toMediaTimeUs(effectTimeUs(5 * S, OFFSET, base = OFFSET))) + } + + @Test + fun playbackStartedLater() { + val base = OFFSET + 600 * S + mapper.onEnabled(startPositionUs = base, offsetUs = OFFSET) + + assertEquals(605 * S, mapper.toMediaTimeUs(effectTimeUs(605 * S, OFFSET, base))) + } + + @Test + fun seekInsideTheItemKeepsTheMapping() { + val base = OFFSET + 600 * S + mapper.onEnabled(startPositionUs = base, offsetUs = OFFSET) + mapper.onPositionReset() + + assertEquals(5 * S, mapper.toMediaTimeUs(effectTimeUs(5 * S, OFFSET, base))) + } + + @Test + fun reEnableWithoutResetKeepsTheFirstStartPosition() { + // Seek to another item: the renderer is disabled and enabled again, but not reset. + val base = OFFSET + 600 * S + mapper.onEnabled(startPositionUs = base, offsetUs = OFFSET) + mapper.onEnabled(startPositionUs = OFFSET, offsetUs = OFFSET) + + assertEquals(3 * S, mapper.toMediaTimeUs(effectTimeUs(3 * S, OFFSET, base))) + } + + @Test + fun playlistTransition() { + val firstDuration = 1200 * S + val secondOffset = OFFSET + firstDuration + mapper.onEnabled(startPositionUs = OFFSET, offsetUs = OFFSET) + // The next item's stream is handed over before the current item's last frames are processed. + mapper.onStreamReplaced(startPositionUs = secondOffset, offsetUs = secondOffset) + + assertEquals(1199 * S, mapper.toMediaTimeUs(effectTimeUs(1199 * S, OFFSET, base = OFFSET))) + assertEquals(5 * S, mapper.toMediaTimeUs(effectTimeUs(5 * S, secondOffset, base = OFFSET))) + } + + @Test + fun seekAfterPlaylistTransitionUsesTheLastStream() { + val firstDuration = 1200 * S + val secondOffset = OFFSET + firstDuration + mapper.onEnabled(startPositionUs = OFFSET, offsetUs = OFFSET) + mapper.onStreamReplaced(startPositionUs = secondOffset, offsetUs = secondOffset) + mapper.onPositionReset() + + assertEquals(0L, mapper.toMediaTimeUs(effectTimeUs(0, secondOffset, base = OFFSET))) + } + + @Test + fun resetTakesTheNextStartPosition() { + mapper.onEnabled(startPositionUs = OFFSET + 600 * S, offsetUs = OFFSET) + mapper.onReset() + assertEquals(C.TIME_UNSET, mapper.toMediaTimeUs(0)) + + val base = OFFSET + 30 * S + mapper.onEnabled(startPositionUs = base, offsetUs = OFFSET) + + assertEquals(40 * S, mapper.toMediaTimeUs(effectTimeUs(40 * S, OFFSET, base))) + } + + /** What media3 1.11 hands to video effects for a frame at [mediaTimeUs]. */ + private fun effectTimeUs(mediaTimeUs: Long, offsetUs: Long, base: Long): Long { + return mediaTimeUs + offsetUs - base + } + + private companion object { + /** MediaPeriodQueue.INITIAL_RENDERER_POSITION_OFFSET_US */ + const val OFFSET = 1_000_000_000_000L + const val S = 1_000_000L + } +}