diff --git a/dev/demux.html b/dev/demux.html index 140d3cad1..b06e64073 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -14,48 +14,57 @@ const file = fileInput.files[0]; const input = new Mediabunny.Input({ formats: Mediabunny.ALL_FORMATS, - source: new Mediabunny.BlobSource(file), + source: new Mediabunny.BufferSource(await file.arrayBuffer()), }); - + const track = await input.getPrimaryVideoTrack(); - if (!track) { - throw new Error('The synthetic file has no video track.'); - } + const cache = new Mediabunny.PacketCache({ maxCacheSize: Infinity }); + const reader = new Mediabunny.PacketReader(track, { cache }); + + for (let i = 0; i < 50; i++) { + console.time() + let packet = await reader.getFirst(); + let count = 0; + while (packet) { + packet = reader.getNext(packet); + if (packet instanceof Promise) packet = await packet; + + count++; + + /* + if (packet.type === 'key') { + i++; + } + + if (i === 2) { + break; + } + */ + } + console.timeEnd() - let encodedPackets = 0; - for await (const packet of new Mediabunny.EncodedPacketSink(track).packets()) { - console.log(packet.timestamp) - encodedPackets++; + console.log(count) } - let decodedFrames = 0; - let lastTimestamp = null; - const sink = new Mediabunny.VideoSampleSink(track, { - hardwareAcceleration: 'prefer-software', - }); - for await (const sample of sink.samples()) { - console.log(sample.timestamp) - decodedFrames++; - lastTimestamp = sample.timestamp; - sample.close(); - } + console.log("Done") - input.dispose(); - const reproduced = encodedPackets === 48 && decodedFrames === 47; - console.log({ - decodedFrames, - encodedPackets, - lastTimestamp, - reproduced, - }); + //console.log(cache); +// + //reader.getAt(0.5) + //reader.getAt(0.5) + //console.log(await reader.getAt(0.5)); + //console.log(await reader.getAt(0.99)); /* - const track = await input.getPrimaryVideoTrack(); - const sink = new Mediabunny.EncodedPacketSink(track); + const packetSink = new Mediabunny.EncodedPacketSink(track); - const packet = await sink.getFirstPacket(); - console.log(packet); + const first = await packetSink.getFirstPacket({ verifyKeyPackets: true }); + const second = await packetSink.getNextPacket(first, { verifyKeyPackets: true }); + const third = await packetSink.getNextPacket(second, { verifyKeyPackets: true }); + const fourth = await packetSink.getNextPacket(third, { verifyKeyPackets: true }); + console.log(first, second, third, fourth); */ + /* for await (const packet of packetSink.packets()) { @@ -292,14 +301,104 @@ } */ + //alert(total / iters); /* - const videoTrack = await input.getPrimaryVideoTrack(); - const sink = new Mediabunny.EncodedPacketSink(videoTrack); + console.log(await cursor.seekTo(2.00)); + console.log(await cursor.next()); + console.log(await cursor.next()); + */ - for await (const packet of sink.packets()) { - console.log(packet); + //console.log(await cursor.seekTo(4.9)) + //console.log(await cursor.seekTo(5.1)) + //return; + + /* + const mh = [ + cursor.seekTo(2.05), + //cursor.seekTo(4.9), + //cursor.seekTo(4.9), + //cursor.seekTo(4.9), + //cursor.seekTo(4.9), + //cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.next(), + cursor.close(), + //cursor.next(), + //cursor.next(), + //cursor.next(), + //cursor.next(), + //cursor.close(), + //cursor.seekTo(2.10), + //cursor.seekTo(5), + //cursor.close(), + //cursor.seekTo(2.00), + ]; + */ + + /* + for (const yo of mh) { + const samp = await yo; + console.log(samp) + samp?.close(); + } + + console.log("done") + */ + + //console.log(await Promise.all(mh)); + + //console.log(await cursor.seekTo(2.05)); + //console.log(await cursor.seekTo(2.10)); + //console.log(await cursor.seekTo(10.2)); + //console.log(await cursor.seekTo(5.2)); + + /* + setTimeout(async () => { + console.log("this") + + }, 500); + */ + + + /* + const iters = 5; + let total = 0; + + for (let i = 0; i < iters; i++) { + const file = fileInput.files[0]; + const input = new Mediabunny.Input({ + formats: Mediabunny.ALL_FORMATS, + source: new Mediabunny.BufferSource(await file.arrayBuffer()), + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + const cursor = new Mediabunny.PacketCursor(audioTrack); + + await cursor.seekToStart(); + + const start = performance.now(); + await cursor.iterate(() => {}); + //alert(performance.now() - start) + + total += performance.now() - start; } + + alert(total / iters); */ /* @@ -381,6 +480,8 @@ total += end - start; //console.log(total / file.size, end - start); } + alert(elapsed / iters); + const videoTrack = await input.getPrimaryVideoTrack(); const sink = new Mediabunny.EncodedPacketSink(videoTrack); diff --git a/eslint.config.mjs b/eslint.config.mjs index 3c240225a..6d58d8096 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -30,6 +30,19 @@ export default tseslint.config( '@typescript-eslint/require-await': 'off', '@stylistic/yield-star-spacing': ['error', { before: false, after: true }], '@typescript-eslint/no-unsafe-enum-comparison': 'off', + '@typescript-eslint/no-unused-vars': [ + 'error', + { + // From https://typescript-eslint.io/rules/no-unused-vars/ + "args": "after-used", + "argsIgnorePattern": "^_", + "caughtErrors": "all", + "caughtErrorsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "varsIgnorePattern": "^_", + "ignoreRestSiblings": true, + }, + ], '@typescript-eslint/no-unsafe-unary-minus': 'off', '@typescript-eslint/no-deprecated': 'error', '@typescript-eslint/consistent-type-exports': 'error', diff --git a/examples/media-player/media-player.ts b/examples/media-player/media-player.ts index 7882ae2be..b9bc361b4 100644 --- a/examples/media-player/media-player.ts +++ b/examples/media-player/media-player.ts @@ -1,12 +1,10 @@ import { ALL_FORMATS, - AudioBufferSink, + AudioSampleCursor, BlobSource, - CanvasSink, Input, UrlSource, - WrappedAudioBuffer, - WrappedCanvas, + VideoSampleCursor, } from 'mediabunny'; import { registerAc3Decoder } from '@mediabunny/ac3'; import { registerDtsDecoder } from '@mediabunny/dts'; @@ -51,8 +49,8 @@ let audioContext: AudioContext | null = null; let gainNode: GainNode | null = null; let fileLoaded = false; -let videoSink: CanvasSink | null = null; -let audioSink: AudioBufferSink | null = null; +let videoCursor: VideoSampleCursor | null = null; +let audioCursor: AudioSampleCursor | null = null; let firstTimestamp = 0; let endTimestamp = 0; @@ -63,17 +61,8 @@ let playing = false; /** The timestamp within the media file when the playback was started. */ let playbackTimeAtStart = 0; -let videoFrameIterator: AsyncGenerator | null = null; -let audioBufferIterator: AsyncGenerator | null = null; -let nextFrame: WrappedCanvas | null = null; const queuedAudioNodes: Set = new Set(); -/** - * Used to prevent async race conditions. When seekId is incremented, already-running async functions will be prevented - * from having an effect. - */ -let asyncId = 0; - let liveRefreshIntervalId = -1; let draggingProgressBar = false; @@ -91,10 +80,8 @@ const initMediaPlayer = async (resource: File | string) => { pause(); } - void videoFrameIterator?.return(); - void audioBufferIterator?.return(); - - asyncId++; + void videoCursor?.close(); + void audioCursor?.close(); fileLoaded = false; fileNameElement.textContent = resource instanceof File ? resource.name : resource; @@ -193,15 +180,14 @@ const initMediaPlayer = async (resource: File | string) => { playerContainer.style.background = videoCanBeTransparent ? 'transparent' : ''; - // For video, let's use a CanvasSink as it handles rotation and closing video samples for us. - // Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases. - videoSink = videoTrack && new CanvasSink(videoTrack, { - poolSize: 2, - fit: 'contain', // In case the video changes dimensions over time - alpha: videoCanBeTransparent, - }); - // For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API - audioSink = audioTrack && new AudioBufferSink(audioTrack); + if (videoTrack) { + videoCursor = new VideoSampleCursor(videoTrack); + await videoCursor.seekToFirst(); + } + if (audioTrack) { + audioCursor = new AudioSampleCursor(audioTrack); + await audioCursor.seekToFirst(); + } // Show the canvas if there's a video track, otherwise hide it if (videoTrack) { @@ -223,8 +209,6 @@ const initMediaPlayer = async (resource: File | string) => { fileLoaded = true; - await startVideoIterator(); - if (audioContext.state === 'running') { // Start playback automatically if the audio context permits // await play(); @@ -233,7 +217,7 @@ const initMediaPlayer = async (resource: File | string) => { loadingElement.style.display = 'none'; playerContainer.style.display = ''; - if (!videoSink) { + if (!videoCursor) { // If there's only an audio track, always show the controls controlsElement.style.opacity = '1'; controlsElement.style.pointerEvents = ''; @@ -282,33 +266,7 @@ const initMediaPlayer = async (resource: File | string) => { /** === VIDEO RENDERING LOGIC === */ -/** Creates a new video frame iterator and renders the first video frame. */ -const startVideoIterator = async () => { - if (!videoSink) { - return; - } - - asyncId++; - - await videoFrameIterator?.return(); // Dispose of the current iterator - - // Create a new iterator - videoFrameIterator = videoSink.canvases(getPlaybackTime()); - - // Get the first two frames - const firstFrame = (await videoFrameIterator.next()).value ?? null; - const secondFrame = (await videoFrameIterator.next()).value ?? null; - - nextFrame = secondFrame; - - if (firstFrame) { - // Draw the first frame - context.clearRect(0, 0, canvas.width, canvas.height); - context.drawImage(firstFrame.canvas, 0, 0); - } -}; - -/** Runs every frame; updates the canvas if necessary. */ +/** Runs every frame and updates the canvas if possible. */ const render = (requestFrame = true) => { if (fileLoaded) { const playbackTime = getPlaybackTime(); @@ -318,14 +276,17 @@ const render = (requestFrame = true) => { playbackTimeAtStart = endTimestamp; } - // Check if the current playback time has caught up to the next frame - if (nextFrame && nextFrame.timestamp <= playbackTime) { - context.clearRect(0, 0, canvas.width, canvas.height); - context.drawImage(nextFrame.canvas, 0, 0); - nextFrame = null; + if (videoCursor) { + if (videoCursor.isIdle()) { + // The seek is instant if the frame has already been decoded under the hood; if it is not, we'll just + // render the old frame until the new one is ready. + void videoCursor.seekTo(playbackTime); + } - // Request the next frame - void updateNextFrame(); + context.clearRect(0, 0, canvas.width, canvas.height); + if (videoCursor.current) { + videoCursor.current.drawWithFit(context, { fit: 'contain' }); + } } if (!draggingProgressBar) { @@ -342,50 +303,30 @@ render(); // Also call the render function on an interval to make sure the video keeps updating even if the tab isn't visible setInterval(() => render(false), 500); -/** Iterates over the video frame iterator until it finds a video frame in the future. */ -const updateNextFrame = async () => { - const currentAsyncId = asyncId; - - // We have a loop here because we may need to iterate over multiple frames until we reach a frame in the future - while (true) { - const newNextFrame = (await videoFrameIterator!.next()).value ?? null; - if (!newNextFrame) { - break; - } - - if (currentAsyncId !== asyncId) { - break; - } - - const playbackTime = getPlaybackTime(); - if (newNextFrame.timestamp <= playbackTime) { - // Draw it immediately - context.clearRect(0, 0, canvas.width, canvas.height); - context.drawImage(newNextFrame.canvas, 0, 0); - } else { - // Save it for later - nextFrame = newNextFrame; - break; - } - } -}; - /** === AUDIO PLAYBACK LOGIC === */ +let currentAudioIteratorId = 0; + /** Loops over the audio buffer iterator, scheduling the audio to be played in the audio context. */ const runAudioIterator = async () => { - if (!audioSink) { + if (!audioCursor) { return; } + const id = ++currentAudioIteratorId; + // To play back audio, we loop over all audio chunks (typically very short) of the file and play them at the correct // timestamp. The result is a continuous, uninterrupted audio signal. - for await (const { buffer, timestamp } of audioBufferIterator!) { + for await (const sample of audioCursor) { + if (id !== currentAudioIteratorId) { + break; + } + const node = audioContext!.createBufferSource(); - node.buffer = buffer; + node.buffer = sample.toAudioBuffer(); node.connect(gainNode!); - let startTimestamp = audioContextStartTime! + timestamp - playbackTimeAtStart; + let startTimestamp = audioContextStartTime! + sample.timestamp - playbackTimeAtStart; // Round timestamp to the context's sample boundaries to prevent subsample audio glitches startTimestamp = Math.round(audioContext!.sampleRate * startTimestamp) / audioContext!.sampleRate; @@ -405,15 +346,20 @@ const runAudioIterator = async () => { // If we're more than a second ahead of the current playback time, let's slow down the loop until time has // passed. - if (timestamp - getPlaybackTime() >= 1) { + if (sample.timestamp - getPlaybackTime() >= 1) { await new Promise((resolve) => { - const id = setInterval(() => { - if (timestamp - getPlaybackTime() < 1) { - clearInterval(id); + const timeoutId = setInterval(() => { + if (sample.timestamp - getPlaybackTime() < 1 || id !== currentAudioIteratorId) { + clearInterval(timeoutId); resolve(); } }, 100); }); + + // This check is required to prevent advancing the cursor when it is no longer in our control + if (id !== currentAudioIteratorId) { + break; + } } } }; @@ -439,16 +385,15 @@ const play = async () => { if (getPlaybackTime() === endTimestamp) { // If we're at the end, let's snap back to the start playbackTimeAtStart = firstTimestamp; - await startVideoIterator(); + await videoCursor?.seekTo(firstTimestamp); } audioContextStartTime = audioContext!.currentTime; playing = true; - if (audioSink) { + if (audioCursor) { // Start the audio iterator - void audioBufferIterator?.return(); - audioBufferIterator = audioSink?.buffers(getPlaybackTime()); + void audioCursor.seekTo(getPlaybackTime()); void runAudioIterator(); } @@ -459,8 +404,8 @@ const play = async () => { const pause = () => { playbackTimeAtStart = getPlaybackTime(); playing = false; - void audioBufferIterator?.return(); // This stops any for-loops that are iterating the iterator - audioBufferIterator = null; + + currentAudioIteratorId++; // This stops any ongoing cursor iteration // Stop all audio nodes that were already queued to play for (const node of queuedAudioNodes) { @@ -491,7 +436,7 @@ const seekToTime = async (seconds: number) => { playbackTimeAtStart = seconds; - await startVideoIterator(); + await videoCursor?.seekTo(seconds); if (wasPlaying && playbackTimeAtStart < endTimestamp) { void play(); @@ -590,7 +535,7 @@ volumeBarContainer.addEventListener('pointermove', (event) => { /** === CONTROL UI LOGIC === */ const showControlsTemporarily = () => { - if (!videoSink) { + if (!videoCursor) { // Shouldn't run if there's only an audio track return; } @@ -623,7 +568,7 @@ playerContainer.addEventListener('pointermove', (event) => { } }); playerContainer.addEventListener('pointerleave', (event) => { - if (!videoSink) { + if (!videoCursor) { // Shouldn't run if there's only an audio track return; } diff --git a/examples/thumbnail-generation/thumbnail-generation.ts b/examples/thumbnail-generation/thumbnail-generation.ts index 7add9bc8d..717020519 100644 --- a/examples/thumbnail-generation/thumbnail-generation.ts +++ b/examples/thumbnail-generation/thumbnail-generation.ts @@ -1,9 +1,10 @@ -import { Input, ALL_FORMATS, BlobSource, UrlSource, CanvasSink } from 'mediabunny'; +import { Input, ALL_FORMATS, BlobSource, UrlSource, VideoSampleCursor } from 'mediabunny'; import { registerAc3Decoder } from '@mediabunny/ac3'; import { registerDtsDecoder } from '@mediabunny/dts'; import { registerProresDecoder } from '@mediabunny/prores'; import SampleFileUrl from '../../docs/assets/big-buck-bunny-trimmed.mp4'; +import { canvasTransformer } from '../../src/cursors.js'; // Enable codecs that aren't natively supported by WebCodecs. registerAc3Decoder(); @@ -80,16 +81,23 @@ const generateThumbnails = async (resource: File | string) => { (_, i) => firstTimestamp + i * (lastTimestamp - firstTimestamp) / THUMBNAIL_COUNT, ); - // Create a CanvasSink for extracting resized frames from the video track - const sink = new CanvasSink(videoTrack, { - width: Math.floor(width * window.devicePixelRatio), - height: Math.floor(height * window.devicePixelRatio), - fit: 'fill', + // Create a cursor to extract frames from the video + const cursor = new VideoSampleCursor(videoTrack, { + transform: canvasTransformer({ + width: Math.floor(width * window.devicePixelRatio), + height: Math.floor(height * window.devicePixelRatio), + fit: 'fill', + }), }); + // Queue all cursor commands at once, including closing the cursor; this allows it to optimize access patterns + const promises = timestamps.map(x => cursor.seekTo(x)); + void cursor.close(); + // Iterate over all thumbnail canvases let i = 0; - for await (const wrappedCanvas of sink.canvasesAtTimestamps(timestamps)) { + for (const promise of promises) { + const wrappedCanvas = await promise; const container = thumbnailElements[i]!; if (wrappedCanvas) { diff --git a/package-lock.json b/package-lock.json index badd60773..8d6776a5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -353,9 +353,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", - "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, "license": "MIT", "engines": { @@ -363,13 +363,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.0.tgz", - "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", + "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.0" + "@babel/types": "^7.28.5" }, "bin": { "parser": "bin/babel-parser.js" @@ -389,14 +389,14 @@ } }, "node_modules/@babel/types": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.0.tgz", - "integrity": "sha512-jYnje+JyZG5YThjHiF28oT4SIZLnYOcSBb6+SDaFIyzDVSkXQmQQYclJ2R+YxcdmK0AX6x1E5OQNtuh3jHDrUg==", + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", + "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1" + "@babel/helper-validator-identifier": "^7.28.5" }, "engines": { "node": ">=6.9.0" diff --git a/package.json b/package.json index dc6ce8ce4..292e78c7d 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "sideEffects": false, "scripts": { "build": "./scripts/build.sh", + "build-loose": "./scripts/build.sh --loose", "watch": "tsx scripts/bundle.ts --watch", "lint": "eslint .", "test": "npm run pre-test && npx vitest --run", @@ -87,6 +88,7 @@ "@tailwindcss/vite": "^4.1.7", "@types/markdown-it-footnote": "^3.0.4", "@types/node": "^22.13.10", + "@vitest/browser": "3.2.4", "esbuild": "^0.25.1", "esbuild-plugin-external-global": "^1.0.1", "eslint": "^9.22.0", @@ -103,7 +105,6 @@ "vitepress-plugin-llms": "^1.5.1", "vitepress-plugin-mermaid": "^2.0.17", "vitest": "3.2.4", - "@vitest/browser": "3.2.4", "webdriverio": "9.19.2" }, "keywords": [ diff --git a/scripts/build.sh b/scripts/build.sh index dec13899a..0a60adb6e 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -3,6 +3,17 @@ set -e # This script must be executed via `npm run build` +LOOSE=false +if [ "$1" = "--loose" ]; then + LOOSE=true +fi + +if [ "$LOOSE" = true ]; then + API_EXTRACTOR_FLAGS="--local" +else + API_EXTRACTOR_FLAGS="" +fi + # Clear the stuff from last build rm -rf dist rm -rf packages/mp3-encoder/dist @@ -38,27 +49,29 @@ npm run fix-build-import-paths tsx scripts/bundle.ts # Declaration file rollup and checks -api-extractor run -api-extractor run -c packages/mp3-encoder/api-extractor.json -api-extractor run -c packages/ac3/api-extractor.json -api-extractor run -c packages/dts/api-extractor.json -api-extractor run -c packages/aac-encoder/api-extractor.json -api-extractor run -c packages/flac-encoder/api-extractor.json -api-extractor run -c packages/prores/api-extractor.json -api-extractor run -c packages/server/api-extractor.json +api-extractor run $API_EXTRACTOR_FLAGS +api-extractor run $API_EXTRACTOR_FLAGS -c packages/mp3-encoder/api-extractor.json +api-extractor run $API_EXTRACTOR_FLAGS -c packages/ac3/api-extractor.json +api-extractor run $API_EXTRACTOR_FLAGS -c packages/dts/api-extractor.json +api-extractor run $API_EXTRACTOR_FLAGS -c packages/aac-encoder/api-extractor.json +api-extractor run $API_EXTRACTOR_FLAGS -c packages/flac-encoder/api-extractor.json +api-extractor run $API_EXTRACTOR_FLAGS -c packages/prores/api-extractor.json +api-extractor run $API_EXTRACTOR_FLAGS -c packages/server/api-extractor.json -# Checks that all symbols are documented -tsx scripts/check-docblocks.ts dist/mediabunny.d.ts -tsx scripts/check-docblocks.ts packages/mp3-encoder/dist/mediabunny-mp3-encoder.d.ts -tsx scripts/check-docblocks.ts packages/ac3/dist/mediabunny-ac3.d.ts -tsx scripts/check-docblocks.ts packages/dts/dist/mediabunny-dts.d.ts -tsx scripts/check-docblocks.ts packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts -tsx scripts/check-docblocks.ts packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts -tsx scripts/check-docblocks.ts packages/prores/dist/mediabunny-prores.d.ts -tsx scripts/check-docblocks.ts packages/server/dist/mediabunny-server.d.ts +if [ "$LOOSE" = false ]; then + # Checks that all symbols are documented + tsx scripts/check-docblocks.ts dist/mediabunny.d.ts + tsx scripts/check-docblocks.ts packages/mp3-encoder/dist/mediabunny-mp3-encoder.d.ts + tsx scripts/check-docblocks.ts packages/ac3/dist/mediabunny-ac3.d.ts + tsx scripts/check-docblocks.ts packages/dts/dist/mediabunny-dts.d.ts + tsx scripts/check-docblocks.ts packages/aac-encoder/dist/mediabunny-aac-encoder.d.ts + tsx scripts/check-docblocks.ts packages/flac-encoder/dist/mediabunny-flac-encoder.d.ts + tsx scripts/check-docblocks.ts packages/prores/dist/mediabunny-prores.d.ts + tsx scripts/check-docblocks.ts packages/server/dist/mediabunny-server.d.ts -# Checks that API docs are generatable -npm run docs:generate -- --dry + # Checks that API docs are generatable + npm run docs:generate -- --dry +fi # Appends stuff to the declaration files to register the global variables these libraries expose echo 'export as namespace Mediabunny;' >> dist/mediabunny.d.ts diff --git a/scripts/generate-api-docs.ts b/scripts/generate-api-docs.ts index 319daf5bb..8cdf2ee9d 100644 --- a/scripts/generate-api-docs.ts +++ b/scripts/generate-api-docs.ts @@ -3,7 +3,6 @@ // good, which is the point of a custom script for this: full, precise control. /* eslint-disable @typescript-eslint/restrict-template-expressions */ -/* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable @stylistic/max-len */ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ diff --git a/src/adts/adts-demuxer.ts b/src/adts/adts-demuxer.ts index 96423e265..08ab8d960 100644 --- a/src/adts/adts-demuxer.ts +++ b/src/adts/adts-demuxer.ts @@ -16,17 +16,17 @@ import { } from '../id3'; import { Input } from '../input'; import { InputAudioTrackBacking } from '../input-track'; -import { PacketRetrievalOptions } from '../media-sink'; import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; import { assert, AsyncMutex, - binarySearchExact, binarySearchLessOrEqual, isThenable, + MaybeRelevantPromise, + ResultValue, UNDETERMINED_LANGUAGE, } from '../misc'; -import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; +import { EncodedPacket, PacketRetrievalOptions, PacketRetrievalResult, PLACEHOLDER_DATA } from '../packet'; import { readBytes, Reader } from '../reader'; import { AdtsFrameHeader, @@ -69,18 +69,21 @@ export class AdtsDemuxer extends Demuxer { return this.metadataPromise ??= (async () => { // Keep loading until we find the first frame header while (!this.firstFrameHeader && !this.lastSampleLoaded) { - await this.advanceReader(); + const result = new ResultValue(); + const promise = this.advanceReader(result); + if (result.pending) await promise; } - // There has to be a frame if this demuxer got selected - assert(this.firstFrameHeader); + if (!this.firstFrameHeader) { + throw new Error('No valid ADTS frame found.'); + } // Create the single audio track this.trackBackings = [new AdtsAudioTrackBacking(this)]; })(); } - async advanceReader() { + async advanceReader(res: ResultValue): MaybeRelevantPromise { if (this.lastLoadedPos === 0) { // Skip all ID3v2 tags at the start of the file while (true) { @@ -89,7 +92,7 @@ export class AdtsDemuxer extends Demuxer { if (!slice) { this.lastSampleLoaded = true; - return; + return res.set(); } const id3V2Header = readId3V2Header(slice); @@ -109,19 +112,19 @@ export class AdtsDemuxer extends Demuxer { if (isThenable(slice)) slice = await slice; if (!slice) { this.lastSampleLoaded = true; - return; + return res.set(); } const header = readAdtsFrameHeader(slice); if (!header) { this.lastSampleLoaded = true; - return; + return res.set(); } if (this.reader.fileSize !== null && header.startPos + header.frameLength > this.reader.fileSize) { // Frame doesn't fit in the rest of the file this.lastSampleLoaded = true; - return; + return res.set(); } if (!this.firstFrameHeader) { @@ -142,6 +145,8 @@ export class AdtsDemuxer extends Demuxer { this.loadedSamples.push(sample); this.nextTimestampInSamples += SAMPLES_PER_AAC_FRAME; this.lastLoadedPos = header.startPos + header.frameLength; + + return res.set(); } async getMimeType() { @@ -154,41 +159,38 @@ export class AdtsDemuxer extends Demuxer { } async getMetadataTags() { - const release = await this.readingMutex.acquire(); + using lock = this.readingMutex.lock(); + if (lock.pending) await lock.ready; - try { - await this.readMetadata(); + await this.readMetadata(); - if (this.metadataTags) { - return this.metadataTags; - } + if (this.metadataTags) { + return this.metadataTags; + } - this.metadataTags = {}; - let currentPos = 0; + this.metadataTags = {}; + let currentPos = 0; - while (true) { - let headerSlice = this.reader.requestSlice(currentPos, ID3_V2_HEADER_SIZE); - if (isThenable(headerSlice)) headerSlice = await headerSlice; - if (!headerSlice) break; + while (true) { + let headerSlice = this.reader.requestSlice(currentPos, ID3_V2_HEADER_SIZE); + if (isThenable(headerSlice)) headerSlice = await headerSlice; + if (!headerSlice) break; - const id3V2Header = readId3V2Header(headerSlice); - if (!id3V2Header) { - break; - } - - let contentSlice = this.reader.requestSlice(headerSlice.filePos, id3V2Header.size); - if (isThenable(contentSlice)) contentSlice = await contentSlice; - if (!contentSlice) break; + const id3V2Header = readId3V2Header(headerSlice); + if (!id3V2Header) { + break; + } - parseId3V2Tag(contentSlice, id3V2Header, this.metadataTags); + let contentSlice = this.reader.requestSlice(headerSlice.filePos, id3V2Header.size); + if (isThenable(contentSlice)) contentSlice = await contentSlice; + if (!contentSlice) break; - currentPos = headerSlice.filePos + id3V2Header.size; - } + parseId3V2Tag(contentSlice, id3V2Header, this.metadataTags); - return this.metadataTags; - } finally { - release(); + currentPos = headerSlice.filePos + id3V2Header.size; } + + return this.metadataTags; } } @@ -292,14 +294,18 @@ class AdtsAudioTrackBacking implements InputAudioTrackBacking { }; } - async getPacketAtIndex(sampleIndex: number, options: PacketRetrievalOptions) { + async getPacketAtIndex( + res: ResultValue, + sampleIndex: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { if (sampleIndex === -1) { - return null; + return res.set(new PacketRetrievalResult(null)); } const rawSample = this.demuxer.loadedSamples[sampleIndex]; if (!rawSample) { - return null; + return res.set(new PacketRetrievalResult(null)); } let data: Uint8Array; @@ -310,92 +316,102 @@ class AdtsAudioTrackBacking implements InputAudioTrackBacking { if (isThenable(slice)) slice = await slice; if (!slice) { - return null; // Data didn't fit into the rest of the file + return res.set(new PacketRetrievalResult(null)); // Data didn't fit into the rest of the file } data = readBytes(slice, rawSample.dataSize); } - return new EncodedPacket( + return res.set(new PacketRetrievalResult(new EncodedPacket( data, 'key', rawSample.timestamp, rawSample.duration, sampleIndex, rawSample.dataSize, - ); + ))); } - getFirstPacket(options: PacketRetrievalOptions) { - return this.getPacketAtIndex(0, options); + getFirstPacket(res: ResultValue, options: PacketRetrievalOptions) { + return this.getPacketAtIndex(res, 0, options); } - async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - const release = await this.demuxer.readingMutex.acquire(); + async getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + using lock = this.demuxer.readingMutex.lock(); + if (lock.pending) await lock.ready; + + const sampleIndex = packet.sequenceNumber; + assert(sampleIndex >= 0); + + const nextIndex = sampleIndex + 1; + + // Ensure the next sample exists + while ( + nextIndex >= this.demuxer.loadedSamples.length + && !this.demuxer.lastSampleLoaded + ) { + const result = new ResultValue(); + const promise = this.demuxer.advanceReader(result); + if (result.pending) await promise; + } - try { - const sampleIndex = binarySearchExact( + return this.getPacketAtIndex(res, nextIndex, options); + } + + async getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + using lock = this.demuxer.readingMutex.lock(); + if (lock.pending) await lock.ready; + + while (true) { + const index = binarySearchLessOrEqual( this.demuxer.loadedSamples, - packet.timestamp, + timestamp, x => x.timestamp, ); - if (sampleIndex === -1) { - throw new Error('Packet was not created from this track.'); - } - const nextIndex = sampleIndex + 1; - // Ensure the next sample exists - while ( - nextIndex >= this.demuxer.loadedSamples.length - && !this.demuxer.lastSampleLoaded - ) { - await this.demuxer.advanceReader(); + if (index === -1 && this.demuxer.loadedSamples.length > 0) { + // We're before the first sample + return res.set(new PacketRetrievalResult(null)); } - return this.getPacketAtIndex(nextIndex, options); - } finally { - release(); - } - } - - async getPacket(timestamp: number, options: PacketRetrievalOptions) { - const release = await this.demuxer.readingMutex.acquire(); - - try { - while (true) { - const index = binarySearchLessOrEqual( - this.demuxer.loadedSamples, - timestamp, - x => x.timestamp, - ); - if (index === -1 && this.demuxer.loadedSamples.length > 0) { - // We're before the first sample - return null; - } - - if (this.demuxer.lastSampleLoaded) { - // All data is loaded, return what we found - return this.getPacketAtIndex(index, options); - } - - if (index >= 0 && index + 1 < this.demuxer.loadedSamples.length) { - // The next packet also exists, we're done - return this.getPacketAtIndex(index, options); - } + if (this.demuxer.lastSampleLoaded) { + // All data is loaded, return what we found + return this.getPacketAtIndex(res, index, options); + } - // Otherwise, keep loading data - await this.demuxer.advanceReader(); + if (index >= 0 && index + 1 < this.demuxer.loadedSamples.length) { + // The next packet also exists, we're done + return this.getPacketAtIndex(res, index, options); } - } finally { - release(); + + // Otherwise, keep loading data + const result = new ResultValue(); + const promise = this.demuxer.advanceReader(result); + if (result.pending) await promise; } } - getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { - return this.getPacket(timestamp, options); + getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this.getPacket(res, timestamp, options); } - getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - return this.getNextPacket(packet, options); + getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this.getNextPacket(res, packet, options); } } diff --git a/src/adts/adts-muxer.ts b/src/adts/adts-muxer.ts index 1e33ee798..c8643107b 100644 --- a/src/adts/adts-muxer.ts +++ b/src/adts/adts-muxer.ts @@ -32,7 +32,8 @@ export class AdtsMuxer extends Muxer { } async start() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.writer = await this.output._getRootWriter(true); @@ -40,8 +41,6 @@ export class AdtsMuxer extends Muxer { const id3Writer = new Id3V2Writer(this.writer); id3Writer.writeId3V2Tag(this.output._metadataTags); } - - release(); } async getMimeType() { @@ -57,60 +56,57 @@ export class AdtsMuxer extends Muxer { packet: EncodedPacket, meta?: EncodedAudioChunkMetadata, ) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); + this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); - // First packet - determine input format from metadata - if (this.inputIsAdts === null) { - validateAudioChunkMetadata(meta, track.source._codec); + // First packet - determine input format from metadata + if (this.inputIsAdts === null) { + validateAudioChunkMetadata(meta, track.source._codec); - const description = meta?.decoderConfig?.description; + const description = meta?.decoderConfig?.description; - // Follows from the Mediabunny Codec Registry: - this.inputIsAdts = !description; + // Follows from the Mediabunny Codec Registry: + this.inputIsAdts = !description; - if (!this.inputIsAdts) { - const config = parseAacAudioSpecificConfig(toUint8Array(description!)); - const template = buildAdtsHeaderTemplate(config); - this.header = template.header; - this.headerBitstream = template.bitstream; - } + if (!this.inputIsAdts) { + const config = parseAacAudioSpecificConfig(toUint8Array(description!)); + const template = buildAdtsHeaderTemplate(config); + this.header = template.header; + this.headerBitstream = template.bitstream; } + } - if (this.inputIsAdts) { - // Packets are already ADTS frames, write them directly - const startPos = this.writer.getPos(); - this.writer.write(packet.data); + if (this.inputIsAdts) { + // Packets are already ADTS frames, write them directly + const startPos = this.writer.getPos(); + this.writer.write(packet.data); - if (this.format._options.onFrame) { - this.format._options.onFrame(packet.data, startPos); - } - } else { - assert(this.header); + if (this.format._options.onFrame) { + this.format._options.onFrame(packet.data, startPos); + } + } else { + assert(this.header); - // Packets are raw AAC, we gotta turn it into ADTS - const frameLength = packet.data.byteLength + this.header.byteLength; - writeAdtsFrameLength(this.headerBitstream!, frameLength); + // Packets are raw AAC, we gotta turn it into ADTS + const frameLength = packet.data.byteLength + this.header.byteLength; + writeAdtsFrameLength(this.headerBitstream!, frameLength); - const startPos = this.writer.getPos(); - this.writer.write(this.header); - this.writer.write(packet.data); + const startPos = this.writer.getPos(); + this.writer.write(this.header); + this.writer.write(packet.data); - if (this.format._options.onFrame) { - const frameBytes = new Uint8Array(frameLength); - frameBytes.set(this.header, 0); - frameBytes.set(packet.data, this.header.byteLength); + if (this.format._options.onFrame) { + const frameBytes = new Uint8Array(frameLength); + frameBytes.set(this.header, 0); + frameBytes.set(packet.data, this.header.byteLength); - this.format._options.onFrame(frameBytes, startPos); - } + this.format._options.onFrame(frameBytes, startPos); } - - await this.writer.flush(); - } finally { - release(); } + + await this.writer.flush(); } async addSubtitleCue() { @@ -118,12 +114,11 @@ export class AdtsMuxer extends Muxer { } async finalize() { - const release = await this.mutex.acquire(); // Required so that finalize() can't resolve before other calls + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; // Required so that finalize() can't resolve before other calls if (this.inputIsAdts === null) { throw new Error('Cannot finalize an empty ADTS file: not a single packet was added.'); } - - release(); } } diff --git a/src/conversion.ts b/src/conversion.ts index 83c85cd48..999a91301 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -25,11 +25,7 @@ import { import { Input } from './input'; import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; import { Logging } from './logging'; -import { - AudioSampleSink, - EncodedPacketSink, - VideoSampleSink, -} from './media-sink'; +import { AudioSampleCursor, PacketCursor, VideoSampleCursor } from './cursors'; import { AudioSource, EncodedVideoPacketSource, @@ -52,7 +48,7 @@ import { } from './misc'; import { Output, OutputTrackGroup } from './output'; import { Mp4OutputFormat } from './output-format'; -import { EncodedPacket } from './packet'; +import { EncodedPacket, PacketReader } from './packet'; import { AudioSample, clampCropRectangle, @@ -745,6 +741,8 @@ export class Conversion { _computeProgress = false; /** @internal */ _lastProgress = 0; + /** @internal */ + _lastProcessedTime = 0; /** * Whether this conversion, as it has been configured, is valid and can be executed. If this field is `false`, check @@ -1408,8 +1406,9 @@ export class Conversion { } if (this._computeProgress) { - const minTimestamp = Math.min(...this._maxTimestamps.values()); - this.onProgress?.(1, minTimestamp); + // By this point, all tracks have closed and removed themselves from _maxTimestamps, so we use the last + // processed time we saw while they were still open + this.onProgress?.(1, this._lastProcessedTime); } } } @@ -1531,9 +1530,9 @@ export class Conversion { if (!needsTranscode) { // Check if we can copy it - const sink = new EncodedPacketSink(track); - let startPacket = await sink.getKeyPacket(this._startTimestamp, { verifyKeyPackets: true }) - ?? await sink.getFirstKeyPacket({ verifyKeyPackets: true }); + const reader = new PacketReader(track); + let startPacket = await reader.getKeyAt(this._startTimestamp, { verifyKeyPackets: true }) + ?? await reader.getFirstKey({ verifyKeyPackets: true }); if ( startPacket @@ -1541,7 +1540,7 @@ export class Conversion { && startPacket.timestamp + startPacket.duration <= this._startTimestamp && this._copyBoundaryPolicy === 'shrink' ) { - startPacket = await sink.getNextKeyPacket(startPacket, { verifyKeyPackets: true }); + startPacket = await reader.getNextKey(startPacket, { verifyKeyPackets: true }); } copyStartPacket = startPacket; @@ -1627,17 +1626,18 @@ export class Conversion { videoSource = source; this._registerTrackPump(async (pump) => { - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track, { options: { verifyKeyPackets: true } }); + const reader = new PacketReader(track); const decoderConfig = await track.getDecoderConfig(); const meta: EncodedVideoChunkMetadata = { decoderConfig: decoderConfig ?? undefined }; let maxTimestamp: number | null = null; + if (copyStartPacket) { + await cursor.seekToKey(copyStartPacket.timestamp); + } + // eslint-disable-next-line curly - if (copyStartPacket) for await (const packet of sink.packets( - copyStartPacket, - undefined, - { verifyKeyPackets: true }, - )) { + if (copyStartPacket) for await (const packet of cursor) { if (this._state === 'canceled') { break; } @@ -1654,7 +1654,7 @@ export class Conversion { const lookahead = 6; // Heuristic, but should be enough for most streams for (let i = 0; i < lookahead; i++) { - const next = await sink.getNextPacket(current, { metadataOnly: true }); + const next = await reader.getNext(current, { metadataOnly: true }); if (!next) { break; } @@ -1703,7 +1703,7 @@ export class Conversion { type: packetType, }); - this._reportProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration); + this._updateProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration); await source.add(modifiedPacket, meta); if (this._synchronizer.shouldWait(outputTrackId, modifiedPacket.timestamp)) { @@ -1714,7 +1714,7 @@ export class Conversion { } source.close(); - this._synchronizer.closeTrack(outputTrackId); + this._closeTrack(outputTrackId); }); } else { // We need to decode & reencode the video @@ -1795,13 +1795,13 @@ export class Conversion { await tempOutput.start(); // Let's just use the first sample to test - const sink = new VideoSampleSink(track); - using firstSample = await sink.getSample(await track.getFirstTimestamp()); + const cursor = new VideoSampleCursor(track, { closeSamples: false }); + using firstSample = await cursor.seekToFirst(); + await cursor.close(); if (firstSample) { try { await tempSource.add(firstSample); - firstSample.close(); await tempOutput.finalize(); } catch (error) { Logging._warn( @@ -1851,13 +1851,18 @@ export class Conversion { videoSource = source; this._registerTrackPump(async (pump) => { - const sink = new VideoSampleSink(track); + await using cursor = new VideoSampleCursor(track); + await cursor.seekTo(this._startTimestamp); - for await (using sample of sink.samples(this._startTimestamp, this._endTimestamp)) { + for await (const sample of cursor) { if (this._state === 'canceled') { break; } + if (sample.timestamp >= this._endTimestamp) { + break; + } + const clampedStartTimestamp = Math.max(this._startTimestamp, sample.timestamp); const clampedEndTimestamp = Math.min(this._endTimestamp, sample.timestamp + sample.duration); @@ -1866,12 +1871,14 @@ export class Conversion { continue; } - sample.setTimestamp(clampedStartTimestamp + this._timestampOffset); - sample.setDuration(clampedEndTimestamp - clampedStartTimestamp); + using adjustedSample = sample.clone({ + timestamp: clampedStartTimestamp + this._timestampOffset, + duration: clampedEndTimestamp - clampedStartTimestamp, + }); - this._reportProgress(outputTrackId, sample.timestamp + sample.duration); - await source.add(sample); - sample.close(); + this._updateProgress(outputTrackId, adjustedSample.timestamp + adjustedSample.duration); + await source.add(adjustedSample); + adjustedSample.close(); if (lastSampleTimestamp !== null) { if (this._synchronizer.shouldWait(outputTrackId, lastSampleTimestamp)) { @@ -1883,7 +1890,7 @@ export class Conversion { } source.close(); - this._synchronizer.closeTrack(outputTrackId); + this._closeTrack(outputTrackId); }); } @@ -1963,9 +1970,9 @@ export class Conversion { if (!needsTranscode) { // Check if we can copy it - const sink = new EncodedPacketSink(track); - let startPacket = await sink.getKeyPacket(this._startTimestamp) - ?? await sink.getFirstKeyPacket(); + const reader = new PacketReader(track); + let startPacket = await reader.getKeyAt(this._startTimestamp) + ?? await reader.getFirstKey(); if ( startPacket @@ -1981,14 +1988,14 @@ export class Conversion { ) ) ) { - startPacket = await sink.getNextKeyPacket(startPacket); + startPacket = await reader.getNextKey(startPacket); } const hasDecoderWarmup = (NON_PCM_AUDIO_CODECS as readonly AudioCodec[]).includes(sourceCodec) && sourceCodec !== 'flac'; if (startPacket && this._copyBoundaryPolicy === 'expand' && hasDecoderWarmup) { // Go one packet back - const previousPacket = await sink.getKeyPacket( + const previousPacket = await reader.getKeyAt( startPacket.timestamp - 1 / (await track.getTimeResolution()), ); if (previousPacket) { @@ -2072,13 +2079,17 @@ export class Conversion { audioSource = source; this._registerTrackPump(async (pump) => { - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); const decoderConfig = await track.getDecoderConfig(); const meta: EncodedAudioChunkMetadata = { decoderConfig: decoderConfig ?? undefined }; let maxTimestamp: number | null = null; + if (copyStartPacket) { + await cursor.seekToKey(copyStartPacket.timestamp); + } + // eslint-disable-next-line curly - if (copyStartPacket) for await (const packet of sink.packets(copyStartPacket)) { + if (copyStartPacket) for await (const packet of cursor) { if (this._state === 'canceled') { break; } @@ -2106,7 +2117,7 @@ export class Conversion { duration: packet.duration, }); - this._reportProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration); + this._updateProgress(outputTrackId, modifiedPacket.timestamp + modifiedPacket.duration); await source.add(modifiedPacket, meta); if (this._synchronizer.shouldWait(outputTrackId, modifiedPacket.timestamp)) { @@ -2117,7 +2128,7 @@ export class Conversion { } source.close(); - this._synchronizer.closeTrack(outputTrackId); + this._closeTrack(outputTrackId); }); } else { // We need to decode & reencode the audio @@ -2216,12 +2227,18 @@ export class Conversion { this._registerTrackPump(async (pump) => { let needsPadding: boolean | null = null; - const sink = new AudioSampleSink(track); - for await (using sample of sink.samples(this._startTimestamp, this._endTimestamp)) { + await using cursor = new AudioSampleCursor(track); + await cursor.seekTo(this._startTimestamp); + + for await (const sample of cursor) { if (this._state === 'canceled') { break; } + if (sample.timestamp >= this._endTimestamp) { + break; + } + let startFrame = 0; let endFrame = sample.numberOfFrames; @@ -2234,30 +2251,24 @@ export class Conversion { if (startFrame >= endFrame) { // Sample lies wholly out of trim region - sample.close(); continue; // No break since we may be before the start } - // Can't assign to "using" identifiers so we gotta do this - let finalSampleLet: AudioSample; - if (startFrame > 0 || endFrame < sample.numberOfFrames) { - // Trim the sample if it sticks out of the trim region on either end - const trimmedSample = sample.trim(startFrame, endFrame); - sample.close(); - finalSampleLet = trimmedSample; - - if (trimmedSample.numberOfFrames === 0) { - trimmedSample.close(); - continue; - } - } else { - finalSampleLet = sample; - } + // Trim the sample if it sticks out of the trim region on either end + using trimmedSample = startFrame > 0 || endFrame < sample.numberOfFrames + ? sample.trim(startFrame, endFrame) + : null; - using finalSample = finalSampleLet; + if (trimmedSample?.numberOfFrames === 0) { + continue; + } - // Offset the timestamp as needed - finalSample.setTimestamp(finalSample.timestamp + this._timestampOffset); + // Offset the timestamp as needed. The clone also gives us a sample that we own (the one yielded + // by the cursor belongs to the cursor). + const sourceSample = trimmedSample ?? sample; + using finalSample = sourceSample.clone({ + timestamp: sourceSample.timestamp + this._timestampOffset, + }); if (needsPadding === null) { needsPadding = finalSample.timestamp > 0 && !this.output.format.supportsTimestampedMediaData; @@ -2296,7 +2307,7 @@ export class Conversion { } source.close(); - this._synchronizer.closeTrack(outputTrackId); + this._closeTrack(outputTrackId); }); } @@ -2340,7 +2351,7 @@ export class Conversion { outputTrackId: number, getLastSampleTimestamp: () => number | null, ) { - this._reportProgress(outputTrackId, sample.timestamp + sample.duration); + this._updateProgress(outputTrackId, sample.timestamp + sample.duration); await source.add(sample); sample.close(); @@ -2387,20 +2398,38 @@ export class Conversion { } /** @internal */ - _reportProgress(trackId: number, endTimestamp: number) { + _closeTrack(id: number) { + this._synchronizer.closeTrack(id); + + this._maxTimestamps.delete(id); + this._reportProgress(); + } + + /** @internal */ + _updateProgress(trackId: number, endTimestamp: number) { if (!this._computeProgress) { return; } - assert(this._totalDuration !== null); this._maxTimestamps.set( trackId, Math.max(endTimestamp, this._maxTimestamps.get(trackId)!), ); + this._reportProgress(); + } + + /** @internal */ + _reportProgress() { + if (!this._computeProgress || this._maxTimestamps.size === 0) { + return; + } + assert(this._totalDuration !== null); const minTimestamp = Math.min(...this._maxTimestamps.values()); const newProgress = clamp(minTimestamp / this._totalDuration, 0, 1); + this._lastProcessedTime = minTimestamp; + if (newProgress !== this._lastProgress) { this._lastProgress = newProgress; this.onProgress?.(newProgress, minTimestamp); diff --git a/src/cursors.ts b/src/cursors.ts new file mode 100644 index 000000000..2e55e1be0 --- /dev/null +++ b/src/cursors.ts @@ -0,0 +1,1610 @@ +/*! + * Copyright (c) 2026-present, Vanilagy and contributors + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +import { PCM_AUDIO_CODECS } from './codec'; +import { AudioDecoderWrapper, DecoderWrapper, PcmAudioDecoderWrapper, VideoDecoderWrapper } from './decode'; +import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; +import { + assert, + AsyncMutex, + AsyncMutexLock, + ForgivingCallSerializer, + defer, + isFirefox, + isThenable, + last, + MaybePromise, + missingWebCodecsClassMessage, + polyfillSymbolDispose, + promiseWithResolvers, + ResultValue, + Rotation, + AsyncGate, + MaybeRelevantPromise, +} from './misc'; +import { + EncodedPacket, + PacketCache, + PacketReader, + PacketRetrievalOptions, + validatePacketRetrievalOptions, + validateTimestamp, +} from './packet'; +import { AudioSample, clampCropRectangle, CropRectangle, validateCropRectangle, VideoSample } from './sample'; + +polyfillSymbolDispose(); + +export type PacketCursorConfig = { + options?: PacketRetrievalOptions; + cache?: PacketCache; +}; + +export class PacketCursor { + track: T; + current: EncodedPacket | null = null; + + private _reader: PacketReader; + private _options: PacketRetrievalOptions; + private _nextIsFirst = true; + private _callSerializer = new ForgivingCallSerializer(); + + constructor(track: T, config: PacketCursorConfig = {}) { + if (!(track instanceof InputTrack)) { + throw new TypeError('track must be an InputTrack.'); + } + if (typeof config !== 'object' || config === null) { + throw new TypeError('config must be an object.'); + } + if (config.options !== undefined) { + validatePacketRetrievalOptions(config.options, 'config.options'); + } + if (config.cache !== undefined && !(config.cache instanceof PacketCache)) { + throw new TypeError('config.cache, when provided, must be a PacketCache.'); + } + + this.track = track; + this._reader = new PacketReader(track, { cache: config.cache }); + this._options = config.options ?? {}; + } + + private _seekToFirstDirect(): MaybePromise { + const result = this._reader.getFirst(this._options); + + const onPacket = (packet: EncodedPacket | null) => { + this._nextIsFirst = false; + return this.current = packet; + }; + + if (isThenable(result)) { + return result.then(onPacket); + } else { + return onPacket(result); + } + } + + seekToFirst(): MaybePromise { + return this._callSerializer.call(() => this._seekToFirstDirect()); + } + + seekToFirstKey(): MaybePromise { + return this._callSerializer.call(() => { + const result = this._reader.getFirstKey(this._options); + + const onPacket = (packet: EncodedPacket | null) => { + this._nextIsFirst = false; + return this.current = packet; + }; + + if (isThenable(result)) { + return result.then(onPacket); + } else { + return onPacket(result); + } + }); + } + + seekTo(timestamp: number): MaybePromise { + validateTimestamp(timestamp); + + return this._callSerializer.call(() => { + const result = this._reader.getAt(timestamp, this._options); + + const onPacket = (packet: EncodedPacket | null) => { + this._nextIsFirst = !packet; + return this.current = packet; + }; + + if (isThenable(result)) { + return result.then(onPacket); + } else { + return onPacket(result); + } + }); + } + + seekToKey(timestamp: number): MaybePromise { + validateTimestamp(timestamp); + + return this._callSerializer.call(() => { + const result = this._reader.getKeyAt(timestamp, this._options); + + const onPacket = (packet: EncodedPacket | null) => { + this._nextIsFirst = !packet; + return this.current = packet; + }; + + if (isThenable(result)) { + return result.then(onPacket); + } else { + return onPacket(result); + } + }); + } + + next(): MaybePromise { + return this._callSerializer.call(() => { + if (this._nextIsFirst) { + return this._seekToFirstDirect(); + } + + if (!this.current) { + return null; + } + + const result = this._reader.getNext(this.current, this._options); + + const onPacket = (packet: EncodedPacket | null) => { + return this.current = packet; + }; + + if (isThenable(result)) { + return result.then(onPacket); + } else { + return onPacket(result); + } + }); + } + + nextKey(): MaybePromise { + return this._callSerializer.call(() => { + if (this._nextIsFirst) { + return this._seekToFirstDirect(); + } + + if (!this.current) { + return null; + } + + const result = this._reader.getNextKey(this.current, this._options); + + const onPacket = (packet: EncodedPacket | null) => { + return this.current = packet; + }; + + if (isThenable(result)) { + return result.then(onPacket); + } else { + return onPacket(result); + } + }); + } + + async iterate( + callback: (packet: EncodedPacket) => MaybePromise, + ) { + if (typeof callback !== 'function') { + throw new TypeError('callback must be a function.'); + } + + const donePromise = this._callSerializer.waitUntilIdle(); + if (donePromise) await donePromise; + + while (true) { + if (this.current) { + let result = callback(this.current); + if (isThenable(result)) result = await result; + + if (result === false) { + break; + } + } + + const result = this.next(); + if (isThenable(result)) await result; + + if (!this.current) { + break; + } + } + } + + // eslint-disable-next-line @stylistic/generator-star-spacing + async *[Symbol.asyncIterator]() { + const donePromise = this._callSerializer.waitUntilIdle(); + if (donePromise) await donePromise; + + while (true) { + if (this.current) { + yield this.current; + } + + const result = this.next(); + if (isThenable(result)) await result; + + if (!this.current) { + break; + } + } + } + + waitUntilIdle() { + return this._callSerializer.waitUntilIdle(); + } + + isIdle() { + return this._callSerializer.isIdle(); + } +} + +type PendingRequest = { + timestamp: number; + promise: Promise; + resolve: (sample: T | null) => void; + reject: (error: unknown) => void; + successor: PendingRequest | null; +}; + +export type SampleTransformer = (sample: Sample) => TransformedSample; + +export type SampleCursorOptions = { + closeSamples?: boolean; + transform?: SampleTransformer; + skipLiveWait?: boolean; + cache?: PacketCache; +}; + +const validateSampleCursorOptions = ( + options: SampleCursorOptions, +) => { + if (!options || typeof options !== 'object') { + throw new TypeError('options must an object.'); + } + if (options.closeSamples !== undefined && typeof options.closeSamples !== 'boolean') { + throw new TypeError('options.closeSamples, when provided, must be a boolean.'); + } + if (options.transform !== undefined && typeof options.transform !== 'function') { + throw new TypeError('options.transform, when provided, must be a function.'); + } + if (options.skipLiveWait !== undefined && typeof options.skipLiveWait !== 'boolean') { + throw new TypeError('options.skipLiveWait, when provided, must be a boolean.'); + } + if (options.cache !== undefined && !(options.cache instanceof PacketCache)) { + throw new TypeError('options.cache, when provided, must be a PacketCache.'); + } +}; + +export abstract class SampleCursor< + Sample extends VideoSample | AudioSample, + TransformedSample = Sample, +> implements AsyncDisposable { + track: InputTrack; + current: TransformedSample | null = null; + + private _transform: SampleTransformer; + private _closeSamples: boolean; + private _retrievalOptions: PacketRetrievalOptions; + + /** @internal */ + _mutex = new AsyncMutex(); + + private _packetReader: PacketReader; + private _packetCursor: PacketCursor; + /** @internal */ + _decoder: DecoderWrapper | null = null; + private _currentSample: Sample | null = null; + /** Updated when _currentSample is updated, but reset when a new pump is started. */ + private _currentSampleTimestamp: number | null = null; + /** The queue of samples that have been decoded and are now waiting. */ + private _sampleQueue: Sample[] = []; + private _pendingRequests: PendingRequest[] = []; + private _lastPendingRequest: PendingRequest | null = null; + /** Whether the next sample is the first sample. */ + private _nextIsFirst = true; + private _queuedResets = 0; + + /** @internal */ + _pumpRunning = false; + /** Used to pause and resume the pump. */ + private _pumpGate = new AsyncGate(); + private _pumpStopQueued = false; + private _pumpStopped = new AsyncGate(); + /** The minimum target packet until which the pump should decode. */ + private _pumpTarget: EncodedPacket | null = null; + private _lastTarget: EncodedPacket | null = null; + private _decoderFlushPromise: Promise | null = null; + /** + * When this value is above 0, the pump is instructed to be lazy: that is, only decode packets until the target and + * not further to increase decoder efficiency. + */ + private _lazyPump = 0; + + private _closed = false; + private _closePromise: Promise | null = null; + private _error: unknown = null; + private _errorSet = false; + + /** + * A bunch of utilities for simulating certain behaviors for testing. + * @internal + */ + _debug = { + enabled: false, + pumpsStarted: 0, + seekPackets: [] as (EncodedPacket | null)[], + decodedPackets: [] as EncodedPacket[], + throwInDecoderInit: false, + throwInPump: false, + throwDecoderError: false, + unthrottledPump: false, + pumpEnded: new AsyncGate(), + }; + + get closed(): boolean { + return this._closed; + } + + get errored(): boolean { + return this._errorSet; + } + + protected constructor( + track: InputTrack, + options: SampleCursorOptions, + ) { + this.track = track; + this._retrievalOptions = { skipLiveWait: options.skipLiveWait }; + this._packetReader = new PacketReader(track, { cache: options.cache }); + this._packetCursor = new PacketCursor(track, { options: this._retrievalOptions, cache: options.cache }); + this._closeSamples = options.closeSamples ?? true; + this._transform = options.transform ?? (sample => sample as unknown as TransformedSample); + + track.input._openSampleCursors.add(this); + + const lock = this._mutex.lock(); + assert(!lock.pending); + + void this._initDecoder() + .then(decoder => this._decoder = decoder) + .catch(error => this._closeWithError(error, false)) + .finally(() => lock.release()); + } + + private _getSample( + callback: (result: ResultValue) => MaybeRelevantPromise, + ): MaybePromise { + this._ensureWillBeOpen(); + + try { + const result = new ResultValue(); + const promise = callback(result); + + if (result.pending) { + return promise + .then(() => result.value) + .catch(this._closeWithErrorAndThrow.bind(this)); + } else { + return result.value; + } + } catch (error) { + this._closeWithErrorAndThrow(error); + } + } + + seekToFirst(): MaybePromise { + // The first packet, as far as sample cursors are concerned, is the first *key* packet - it's the first packet + // with which decoding can begin. + return this._getSample(result => this._seekToPacket( + result, + this._packetReader.getFirstKey({ ...this._retrievalOptions, verifyKeyPackets: true }), + )); + } + + seekTo(timestamp: number): MaybePromise { + validateTimestamp(timestamp); + return this._getSample(result => this._seekToPacket( + result, + this._packetReader.getAt(timestamp, this._retrievalOptions), + )); + } + + seekToKey(timestamp: number): MaybePromise { + validateTimestamp(timestamp); + return this._getSample(result => this._seekToPacket( + result, + this._packetReader.getKeyAt(timestamp, { ...this._retrievalOptions, verifyKeyPackets: true }), + )); + } + + next(): MaybePromise { + return this._getSample(result => this._nextInternal(result)); + } + + nextKey(): MaybePromise { + return this._getSample(result => this._nextKeyInternal(result)); + } + + hasNext(): MaybePromise { + this._ensureWillBeOpen(); + + try { + const result = new ResultValue(); + const promise = this._hasNextInternal(result); + + if (result.pending) { + return promise + .then(() => result.value) + .catch(this._closeWithErrorAndThrow.bind(this)); + } else { + return result.value; + } + } catch (error) { + this._closeWithErrorAndThrow(error); + } + } + + async iterate( + callback: (sample: TransformedSample) => MaybePromise, + ) { + if (typeof callback !== 'function') { + throw new TypeError('callback must be a function.'); + } + + this._ensureWillBeOpen(); + + const waitPromise = this.waitUntilIdle(); + if (waitPromise) await waitPromise; + + this._ensureNotClosed(); + + while (true) { + if (this.current) { + let result = callback(this.current); + if (isThenable(result)) result = await result; + + if (result === false) { + break; + } + } + + const result = this.next(); + if (isThenable(result)) await result; + + if (!this.current) { + break; + } + } + } + + // eslint-disable-next-line @stylistic/generator-star-spacing + async *[Symbol.asyncIterator]() { + this._ensureWillBeOpen(); + + const waitPromise = this.waitUntilIdle(); + if (waitPromise) await waitPromise; + + this._ensureNotClosed(); + + while (true) { + if (this.current) { + yield this.current; + } + + const result = this.next(); + if (isThenable(result)) await result; + + if (!this.current) { + break; + } + } + } + + close() { + return this._closePromise ??= this._closed + ? Promise.resolve() + : this._closeInternal(); + } + + [Symbol.asyncDispose]() { + return this.close(); + } + + async reset() { + this._lazyPump++; + + this._queuedResets++; + using _ = defer(() => this._queuedResets--); + + using lock = this._mutex.lock(); + if (lock.pending) await lock.ready; + + if (!this._closed) { + await this._closeInternal(false); + } + + // All of this should automatically be true after a close + assert(!this._pumpRunning); + assert(!this._pumpStopQueued); + assert(!this._currentSample); + assert(!this.current); + assert(this._sampleQueue.length === 0); + assert(this._pendingRequests.length === 0); + assert(this._lastPendingRequest === null); + assert(this._pumpTarget === null); + assert(!this._decoder || this._decoder.closed); + + this._closed = false; + this._closePromise = null; + this._error = null; + this._errorSet = false; + this._nextIsFirst = true; + this._lazyPump = 0; + + this.track.input._openSampleCursors.add(this); + + try { + const newDecoder = await this._initDecoder(); + this._decoder = newDecoder; + } catch (error) { + this._closeWithErrorAndThrow(error, false); + } + } + + /** + * Returns a Promise that resolves when currently pending operations (at the time of calling this method) + * are settled, or `null` if there are none. + */ + waitUntilIdle(): Promise | null { + const lock = this._mutex.lock(); + if (!lock.pending && this._pendingRequests.length === 0) { + lock.release(); + return null; + } + + const getLastPendingPromise = () => { + lock.release(); + + if (this._pendingRequests.length === 0) { + return; + } + + let lastRequest = last(this._pendingRequests)!; + while (lastRequest.successor) { + lastRequest = lastRequest.successor; + } + + return lastRequest.promise + .catch(() => {}) + .then(() => {}); + }; + + if (lock.pending) { + assert(lock.ready); + return lock.ready.then(getLastPendingPromise); + } else { + return getLastPendingPromise() ?? null; + } + } + + isIdle(): boolean { + return this._pendingRequests.length === 0 && !this._mutex.locked; + } + + /** @internal */ + abstract _initDecoder(): Promise>; + + protected _onDecoderSample(sample: Sample): void { + try { + if (this._debug.enabled && this._debug.throwDecoderError) { + sample.close(); + + if (!this._closed) { + // Let's fake a decoder error this way + return this._onDecoderError(new Error('Fake decoder error!')); + } else { + return; + } + } + + if (this._pendingRequests.length === 0) { + if (this._pumpStopQueued || !this._pumpRunning) { + // Don't care about it anymore + sample.close(); + } else { + // Let's save it for later + this._sampleQueue.push(sample); + } + } else { + // If requests are pending, the sample queue must be empty; samples are queued only when there is + // nobody to hand them to + assert(this._sampleQueue.length === 0); + + let given = false; + let nextInsertionIndex = 0; + + // Let's hand the sample to all matching requests + for (let i = 0; i < this._pendingRequests.length; i++) { + const request = this._pendingRequests[i]!; + if (request.timestamp > sample.timestamp) { + break; + } + + this._setCurrentRaw(sample); + request.resolve(this._transformSample()); + + this._pendingRequests.splice(i--, 1); + given = true; + + if (request.successor) { + // If the request has a successor request, "unlock" that successor and add it to the + // start of the queue + this._pendingRequests.splice(nextInsertionIndex, 0, request.successor); + i++; + nextInsertionIndex++; + } + } + + if (!given) { + sample.close(); + } + } + + this._pumpGate.open(); + } catch (error) { + void this._closeWithError(error); + } + } + + protected _onDecoderError(error: unknown): void { + void this._closeWithError(error); + } + + protected _onDecoderDequeue() { + this._pumpGate.open(); + } + + private _setCurrentRaw(newCurrentRaw: Sample | null) { + if (this._currentSample === newCurrentRaw) { + return; + } + + this._currentSample?.close(); + this._currentSample = newCurrentRaw; + this._currentSampleTimestamp = newCurrentRaw?.timestamp ?? null; + this.current = null; + } + + private _transformSample() { + assert(this._currentSample && !this._currentSample.closed); + + if (this._closeSamples) { + // Here, the transformation is memoized: repeated calls will not transform the same sample twice. + return this.current ??= this._transform(this._currentSample); + } else { + // Here, the transformation happens every time since the sample is also cloned every time + const clone = this._currentSample.clone(); + const transformed = this._transform(clone as Sample); + + return this.current = transformed; + } + } + + private async _seekToPacket( + res: ResultValue, + targetPacketPromise: MaybePromise, + lock?: AsyncMutexLock, + ): MaybeRelevantPromise { + this._lazyPump++; + + if (!lock) { + lock = this._mutex.lock(); + if (lock.pending) await lock.ready; + } + + using deferred = defer(() => { + lock?.release(); + this._lazyPump--; + }); + + this._ensureNotClosed(); + + // First, let's wait for the packet to be retrieved + const targetPacket = isThenable(targetPacketPromise) + ? await targetPacketPromise + : targetPacketPromise; + + if (this._debug.enabled) { + this._debug.seekPackets.push(targetPacket); + } + + // A null packet means we're before the first packet + this._nextIsFirst = !targetPacket; + + if (!targetPacket) { + this._lastTarget = null; + this._setCurrentRaw(null); + return res.set(null); + } + + if (this._currentSample?.timestamp === targetPacket.timestamp && !this._currentSample.closed) { + // We can reuse the current sample, the timestamp is the same + return res.set(this._transformSample()); + } + + let needsNewPump: boolean; + + let lastRequest = last(this._pendingRequests) ?? null; + while (lastRequest?.successor) { + lastRequest = lastRequest.successor; + } + + if (lastRequest?.timestamp === -Infinity) { + // next() requests are queued, so in order to know if we need to start a new pump or not, we'll need to wait + await lastRequest.promise + .then(() => {}) + .catch(() => {}); + + this._ensureNotClosed(); + } + + const lastTimestamp = Math.max( + this._lastTarget?.timestamp ?? -Infinity, + this._currentSampleTimestamp ?? -Infinity, // This one's revelant if we had next() requests + ); + + if (lastTimestamp !== -Infinity && lastTimestamp <= targetPacket.timestamp) { + const findQueuedSatisfyingSample = () => { + while (this._sampleQueue.length > 0) { + const nextSample = this._sampleQueue.shift()!; + this._pumpGate.open(); + + if (targetPacket.timestamp <= nextSample.timestamp) { + return nextSample; + } else { + nextSample.close(); + } + } + }; + + // First, let's see if an already-decoded sample can satisfy the request + let satisfyingSample = findQueuedSatisfyingSample(); + if (satisfyingSample) { + this._setCurrentRaw(satisfyingSample); + return res.set(this._transformSample()); + } + + if (this.track.type === 'audio' && targetPacket.timestamp - lastTimestamp < 0.1) { + // The difference is too small for it to be worth to set up a new pump, relevant for audio tracks + needsNewPump = false; + } else { + if (this._packetCursor.current) { + // We need to see if the target packet is ahead of the decoder, GOP-wise + let nextKey = this._packetReader.getNextKey( + this._packetCursor.current, + { ...this._retrievalOptions, verifyKeyPackets: true }, + ); + if (isThenable(nextKey)) nextKey = await nextKey; + + needsNewPump = !!nextKey && targetPacket.sequenceNumber >= nextKey.sequenceNumber; + } else { + needsNewPump = true; + } + } + + // We need to check the queue again because the pump may have advanced ever since the last await + satisfyingSample = findQueuedSatisfyingSample(); + if (satisfyingSample) { + this._setCurrentRaw(satisfyingSample); + return res.set(this._transformSample()); + } + + if (!this._pumpRunning) { + if (targetPacket.timestamp > lastTimestamp) { + // The pump decoded everything to the end and nothing satisfied the request; nothing ever will + this._setCurrentRaw(null); + return res.set(null); + } + + // The sample at lastTimestamp has already been consumed, so it must be decoded again + needsNewPump = true; + } + } else { + // This is the first packet or we went backwards, create a new pump + needsNewPump = true; + } + + if (needsNewPump && this._pumpRunning) { + await this._stopPump(); + } + + if (!this._pumpTarget || targetPacket.sequenceNumber > this._pumpTarget.sequenceNumber) { + this._pumpTarget = targetPacket; + } + this._lastTarget = targetPacket; + + if (needsNewPump) { + // Set the cursor to the right spot + const result = this._packetCursor.seekToKey(targetPacket.timestamp); + if (isThenable(result)) await result; + + // Start the new pump + void this._runPump(); + } + + this._ensureNotClosed(); + + // A request may only ever be registered when the sample queue is empty, otherwise the queued samples would + // never be reconsidered and could clog up the pump + assert(this._sampleQueue.length === 0); + // There must be a pump running if we reach this point. If there isn't, then the request would never be + // fulfilled, which would cause a hang. + assert(this._pumpRunning); + + // Add the request to the queue + const request = promiseWithResolvers(); + const pendingRequest: PendingRequest = { + timestamp: targetPacket.timestamp, + promise: request.promise, + resolve: request.resolve, + reject: request.reject, + successor: null, + }; + this._pendingRequests.push(pendingRequest); + this._lastPendingRequest = pendingRequest; + + this._pumpGate.open(); + deferred.execute(); // Waiting for the return would be too long + + return res.set(await request.promise); + } + + private async _nextInternal(res: ResultValue): MaybeRelevantPromise { + using lock = this._mutex.lock(); + if (lock.pending) await lock.ready; + + this._ensureNotClosed(); + + if (this._nextIsFirst) { + // Easy, just seek to the first sample + // await is important so that the lock doesn't release too early + return await this._seekToPacket( + res, + this._packetReader.getFirstKey({ ...this._retrievalOptions, verifyKeyPackets: true }), + lock, + ); + } + + // See if the request can be satisfied using already-decoded samples + if (this._sampleQueue.length > 0) { + const nextSample = this._sampleQueue.shift()!; + this._pumpGate.open(); + + this._setCurrentRaw(nextSample); + return res.set(this._transformSample()); + } + + if (!this._pumpRunning) { + // No pump is running (but the cursor isn't closed), the pump must've reached the end + this._setCurrentRaw(null); + return res.set(null); + } + + assert(this._lastPendingRequest); + + this._ensureNotClosed(); + + // A request may only ever be registered when the sample queue is empty + assert(this._sampleQueue.length === 0); + + // Instead of figuring out what the next presentation timestamp is (no easy way to do that), we simply add a + // new request that can be fulfilled by *any* timestamp. This way, whatever the decoder produces next (and the + // decoder is required to output samples in presentation order) is what we'll return. + const request = promiseWithResolvers(); + const pendingRequest: PendingRequest = { + timestamp: -Infinity, + promise: request.promise, + resolve: request.resolve, + reject: request.reject, + successor: null, + }; + + if (this._pendingRequests.length === 0) { + this._pendingRequests.push(pendingRequest); + } else { + // The request will only get "unlocked" when the previous request is fulfilled + this._lastPendingRequest.successor = pendingRequest; + } + this._lastPendingRequest = pendingRequest; + + this._pumpGate.open(); + lock.release(); // Waiting for the return would be too long + + return res.set(await request.promise); + } + + private async _nextKeyInternal(res: ResultValue): MaybeRelevantPromise { + using lock = this._mutex.lock(); + if (lock.pending) await lock.ready; + + this._ensureNotClosed(); + + if (this._nextIsFirst) { + // await is important so that the lock doesn't release too early + return await this._seekToPacket( + res, + this._packetReader.getFirstKey({ ...this._retrievalOptions, verifyKeyPackets: true }), + lock, + ); + } + + let timestampToCheck: number; + + const lastPendingRequest = last(this._pendingRequests); + if (lastPendingRequest && !lastPendingRequest.successor) { + timestampToCheck = lastPendingRequest.timestamp; + } else { + if (lastPendingRequest?.successor) { + let last = lastPendingRequest.successor; + while (last.successor) { + last = last.successor; + } + + await last.promise + .then(() => {}) + .catch(() => {}); + + this._ensureNotClosed(); + } + + if (this._currentSampleTimestamp !== null) { + timestampToCheck = this._currentSampleTimestamp; + } else { + // We're at the end + return res.set(null); + } + } + + // The reason we don't just call readNextKey directly is as follows: readNextKey retrieves the next key in + // *decode* order, however we want the next key in *presentation* order. We know that at least the key frames + // are ascending in timestamp, so we first get the current key (based on a presentation-order search), then + // get the next key after that, which will be the answer we're looking for. + + let key = this._packetReader.getKeyAt(timestampToCheck, { ...this._retrievalOptions, verifyKeyPackets: true }); + if (isThenable(key)) key = await key; + assert(key); // Must be + + let nextKey = this._packetReader.getNextKey(key, { ...this._retrievalOptions, verifyKeyPackets: true }); + if (isThenable(nextKey)) nextKey = await nextKey; + + if (!nextKey) { + this._setCurrentRaw(null); + return res.set(null); + } + + return await this._seekToPacket(res, nextKey, lock); + } + + private async _hasNextInternal(res: ResultValue): MaybeRelevantPromise { + using lock = this._mutex.lock(); + if (lock.pending) await lock.ready; + + this._ensureNotClosed(); + + if (this._nextIsFirst) { + let first = this._packetReader.getFirstKey({ ...this._retrievalOptions, verifyKeyPackets: true }); + if (isThenable(first)) first = await first; + + return res.set(!!first); + } + + if (!this._pumpRunning) { + return res.set(false); + } + + if (this._decoderFlushPromise) { + await this._decoderFlushPromise; + } + + return res.set(this._sampleQueue.length > 0 || this._pumpRunning); + } + + /** + * Starts the "pump process", which handles pushing packets into the decoder. It throttles itself if it is far + * enough ahead and must be woken up again by the outside. It also stops itself when the outside tells it to. + */ + private async _runPump() { + assert(this._packetCursor.current); + assert(this._pumpTarget); + assert(this._decoder); + + // Close whatever's left from the previous pump run (only relevant if the previous pump ended naturally, i.e. + // it wasn't stopped) + for (const sample of this._sampleQueue) { + sample.close(); + } + this._sampleQueue.length = 0; + + try { + this._pumpRunning = true; + this._currentSampleTimestamp = null; + + if (this._debug.enabled) { + this._debug.pumpsStarted++; + } + + // Yield once so that the caller can register its request before the first packet is decoded (some decoders + // are synchronous) + await Promise.resolve(); + + // Main loop + while (this._packetCursor.current) { + if (this._debug.enabled && this._debug.throwInPump) { + throw new Error('Fake pump error!'); + } + + const isAheadOfTarget = this._packetCursor.current.sequenceNumber > this._pumpTarget.sequenceNumber; + const nextRequestExists = this._pendingRequests.some(x => x.successor || x.timestamp === -Infinity); + if (isAheadOfTarget && !nextRequestExists) { + if (this._pumpStopQueued) { + break; + } + + if (this._lazyPump > 0) { + await this._pumpGate.wait(); + continue; + } else { + // We're eager! That means even if we're past the target, we'll keep decoding samples to + // prefill the sample queue to have samples ready. This is the common case when not batching + // commands. + } + } + + const decodeQueueSize = this._decoder.getDecodeQueueSize(); + if ( + this._sampleQueue.length + decodeQueueSize >= 4 + && !this._pumpStopQueued + && !(this._debug.enabled && this._debug.unthrottledPump) + ) { + await this._pumpGate.wait(); + continue; + } + + // Send the packet to the decoder + this._decoder.decode(this._packetCursor.current); + + if (this._debug.enabled) { + this._debug.decodedPackets.push(this._packetCursor.current); + } + + // Advance the cursor + const maybePromise = this._packetCursor.next(); + if (isThenable(maybePromise)) await maybePromise; + } + + if (!this._closed || this._pendingRequests.length > 0) { + const { promise, resolve } = promiseWithResolvers(); + this._decoderFlushPromise = promise; + + try { + await this._decoder.flush(); + } finally { + resolve(); + this._decoderFlushPromise = null; + } + } else { + // We're closed with no pending requests, don't bother flushing what's left + } + + // Resolve whatever requests remain with null. The reason this is correct: assume there are still pending + // requests. Then the above flush() call ensured that all possible samples that we can get from the decoder + // we have received. If some of the pending requests are still unsatisfied after seeing all samples we + // decoded, then there is no other way for them to be solved but with null. + const resolveWithNull = (request: PendingRequest) => { + this._setCurrentRaw(null); // Note that this only runs if there exists at least one pending request + + request.resolve(null); + if (request.successor) { + resolveWithNull(request.successor); + } + }; + this._pendingRequests.forEach(resolveWithNull); + } catch (error) { + if (!this._decoder.closed && this._pendingRequests.length > 0) { + // The pump errored but the decoder is still fine, let's first flush the decoder before continuing + try { + await this._decoder.flush(); + } catch { + // The original pump error is the interesting one; _closeWithError below settles the requests + } + } + + this._pumpRunning = false; // So that close() doesn't attempt to stop the pump + void this._closeWithError(error); + } finally { + this._pendingRequests.length = 0; + this._lastPendingRequest = null; + this._pumpRunning = false; + this._pumpTarget = null; + this._pumpStopQueued = false; + this._lastTarget = null; + + this._pumpStopped.open(); + + if (this._debug.enabled) { + this._debug.pumpEnded.open(); + } + } + } + + private async _stopPump() { + assert(this._pumpRunning); + + this._pumpStopQueued = true; + this._pumpGate.open(); + + for (const sample of this._sampleQueue) { + sample.close(); + } + this._sampleQueue.length = 0; + + await this._pumpStopped.wait(); + } + + private async _closeInternal(doLock = true) { + this._lazyPump++; + this.track.input._openSampleCursors.delete(this); + + let lock: AsyncMutexLock | null = null; + if (doLock) { + lock = this._mutex.lock(); + if (lock.pending) await lock.ready; + } + using _ = defer(() => lock?.release()); + + this._closed = true; + + if (this._pumpRunning) { + await this._stopPump(); + } + + for (const sample of this._sampleQueue) { + sample.close(); + } + this._sampleQueue.length = 0; + + this._setCurrentRaw(null); + this._decoder?.close(); + this._decoder = null; + } + + private _closeWithError(error: unknown, doLock?: boolean) { + if (this._closed) { + return; + } + + this._closed = true; + this._error = error; + this._errorSet = true; + + const rejectWithError = (request: PendingRequest) => { + request.reject(error); + if (request.successor) { + rejectWithError(request.successor); + } + }; + this._pendingRequests.forEach(rejectWithError); + + return this._closeInternal(doLock); + } + + private _closeWithErrorAndThrow(error: unknown, doLock?: boolean): never { + void this._closeWithError(error, doLock); + throw error; + } + + /** Ensures that the cursor is not currently closed. */ + private _ensureNotClosed() { + if (this.closed) { + if (this._errorSet) { + throw this._error; + } else { + throw new Error('This cursor has been closed and can no longer be used.'); + } + } + } + + /** Ensures that the cursor is either open or will be open again at some point, even if it currently closed. */ + private _ensureWillBeOpen() { + if (this._queuedResets > 0) { + return; + } + + this._ensureNotClosed(); + } +} + +export type VideoSampleCursorOptions = + SampleCursorOptions & { + /** + * A hint that configures the hardware acceleration method of the decoder. This is best left on + * `'no-preference'`, the default. + */ + hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software'; + /** + * Hint that the selected decoder should be configured to minimize the number of packets that have to be decoded + * before video frames are output. + */ + optimizeForLatency?: boolean; + }; + +const validateVideoSampleCursorOptions = ( + options: VideoSampleCursorOptions, +) => { + validateSampleCursorOptions(options); + + if ( + options.hardwareAcceleration !== undefined + && !['no-preference', 'prefer-hardware', 'prefer-software'].includes(options.hardwareAcceleration) + ) { + throw new TypeError( + 'options.hardwareAcceleration, when provided, must be \'no-preference\', \'prefer-hardware\' or' + + ' \'prefer-software\'.', + ); + } + if (options.optimizeForLatency !== undefined && typeof options.optimizeForLatency !== 'boolean') { + throw new TypeError('options.optimizeForLatency, when provided, must be a boolean.'); + } +}; + +export class VideoSampleCursor extends SampleCursor { + override track!: InputVideoTrack; + + private _options: VideoSampleCursorOptions; + + constructor( + track: InputVideoTrack, + options: VideoSampleCursorOptions = {}, + ) { + if (!(track instanceof InputVideoTrack)) { + throw new TypeError('track must be an InputVideoTrack.'); + } + validateVideoSampleCursorOptions(options); + + super(track, options); + + this._options = options; + } + + /** @internal */ + override async _initDecoder(): Promise> { + if (!(await this.track.canDecode())) { + if (typeof VideoDecoder === 'undefined') { + throw new Error(missingWebCodecsClassMessage('VideoDecoder')); + } + + throw new Error( + 'This video track cannot be decoded in this environment. Make sure to check decodability before using' + + ' a track.', + ); + } + + if (this._debug.enabled && this._debug.throwInDecoderInit) { + throw new Error('Fake decoder init error!'); + } + + const codec = await this.track.getCodec(); + const rotation = await this.track.getRotation(); + const flip = await this.track.getFlip(); + let decoderConfig = await this.track.getDecoderConfig(); + const timeResolution = await this.track.getTimeResolution(); + assert(codec && decoderConfig); + + decoderConfig = { + ...decoderConfig, + hardwareAcceleration: this._options.hardwareAcceleration, + optimizeForLatency: this._options.optimizeForLatency, + }; + + const decoder = new VideoDecoderWrapper( + sample => this._onDecoderSample(sample), + error => this._onDecoderError(error), + codec, + decoderConfig, + rotation, + flip, + timeResolution, + ); + + decoder.onDequeue = () => this._onDecoderDequeue(); + + return decoder; + } +} + +export type AudioSampleCursorOptions = + SampleCursorOptions; + +const validateAudioSampleCursorOptions = ( + options: AudioSampleCursorOptions, +) => { + validateSampleCursorOptions(options); +}; + +export class AudioSampleCursor extends SampleCursor { + override track!: InputAudioTrack; + + constructor( + track: InputAudioTrack, + options: AudioSampleCursorOptions = {}, + ) { + if (!(track instanceof InputAudioTrack)) { + throw new TypeError('track must be an InputAudioTrack.'); + } + validateAudioSampleCursorOptions(options); + + super(track, options); + } + + /** @internal */ + override async _initDecoder(): Promise> { + if (!(await this.track.canDecode())) { + if (typeof AudioDecoder === 'undefined') { + throw new Error(missingWebCodecsClassMessage('AudioDecoder')); + } + + throw new Error( + 'This audio track cannot be decoded in this environment. Make sure to check decodability before using' + + ' a track.', + ); + } + + if (this._debug.enabled && this._debug.throwInDecoderInit) { + throw new Error('Fake decoder init error!'); + } + + const codec = await this.track.getCodec(); + const decoderConfig = await this.track.getDecoderConfig(); + assert(codec && decoderConfig); + + let decoder: AudioDecoderWrapper | PcmAudioDecoderWrapper; + if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) { + decoder = new PcmAudioDecoderWrapper( + sample => this._onDecoderSample(sample), + error => this._onDecoderError(error), + decoderConfig, + ); + } else { + decoder = new AudioDecoderWrapper( + sample => this._onDecoderSample(sample), + error => this._onDecoderError(error), + codec, + decoderConfig, + ); + } + + decoder.onDequeue = () => this._onDecoderDequeue(); + + return decoder; + } +} + +/** + * A canvas with additional timing information (timestamp & duration). + * @public + */ +export class WrappedCanvas { + /** A canvas element or offscreen canvas. */ + canvas: HTMLCanvasElement | OffscreenCanvas; + /** The timestamp of the corresponding video sample, in seconds. */ + timestamp: number; + /** The duration of the corresponding video sample, in seconds. */ + duration: number; + + constructor(canvas: HTMLCanvasElement | OffscreenCanvas, timestamp: number, duration: number) { + this.canvas = canvas; + this.timestamp = timestamp; + this.duration = duration; + } +}; + +/** + * Options for constructing a canvas transformer to be used with {@link VideoSampleCursor}. + * @public + */ +export type CanvasTransformerOptions = { + /** + * Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set + * this to `true` when reading transparent videos. + */ + alpha?: boolean; + /** + * The width of the output canvas in pixels, defaulting to the display width of the video track. If height is not + * set, it will be deduced automatically based on aspect ratio. + */ + width?: number; + /** + * The height of the output canvas in pixels, defaulting to the display height of the video track. If width is not + * set, it will be deduced automatically based on aspect ratio. + */ + height?: number; + /** + * The fitting algorithm in case both width and height are set. + * + * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio. + * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to + * letterboxing. + * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. + */ + fit?: 'fill' | 'contain' | 'cover'; + /** + * The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata. + * Rotation is applied before flipping. + */ + rotation?: Rotation; + /** + * Whether to flip the raw video frame horizontally (about the vertical axis). Defaults to the flip set in the file + * metadata. The flip is applied after rotation but before cropping and resizing. + */ + flip?: boolean; + /** + * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to + * the dimensions of the input video track. Cropping is performed after rotation and flip but before resizing. The + * crop region is in the _display pixel space_ of the underlying video data. + */ + crop?: CropRectangle; + /** + * When set, specifies the number of canvases in the pool. These canvases will be reused in a ring buffer / + * round-robin type fashion. This keeps the amount of allocated VRAM constant and relieves the browser from + * constantly allocating/deallocating canvases. A pool size of 0 or `undefined` disables the pool and means a new + * canvas is created each time. + */ + poolSize?: number; +}; + +export const canvasTransformer = ( + options: CanvasTransformerOptions = {}, +): SampleTransformer => { + if (options && typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if (options.alpha !== undefined && typeof options.alpha !== 'boolean') { + throw new TypeError('options.alpha, when provided, must be a boolean.'); + } + if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) { + throw new TypeError('options.width, when defined, must be a positive integer.'); + } + if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) { + throw new TypeError('options.height, when defined, must be a positive integer.'); + } + if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) { + throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".'); + } + if ( + options.width !== undefined + && options.height !== undefined + && options.fit === undefined + ) { + throw new TypeError( + 'When both options.width and options.height are provided, options.fit must also be provided.', + ); + } + if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) { + throw new TypeError('options.rotation, when provided, must be 0, 90, 180 or 270.'); + } + if (options.flip !== undefined && typeof options.flip !== 'boolean') { + throw new TypeError('options.flip, when provided, must be a boolean.'); + } + if (options.crop !== undefined) { + validateCropRectangle(options.crop, 'options.'); + } + if ( + options.poolSize !== undefined + && (typeof options.poolSize !== 'number' || !Number.isInteger(options.poolSize) || options.poolSize < 0) + ) { + throw new TypeError('poolSize must be a non-negative integer.'); + } + + let needsSetup = true; + let alpha: boolean; + let width: number; + let height: number; + let fit: 'fill' | 'contain' | 'cover'; + let rotation: Rotation; + let flip: boolean; + let crop: { left: number; top: number; width: number; height: number } | undefined; + let canvasPool: (HTMLCanvasElement | OffscreenCanvas | null)[]; + let nextCanvasIndex = 0; + + return (sample) => { + if (needsSetup) { + rotation = options.rotation ?? sample.rotation; + flip = options.flip ?? sample.flip; + + const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 + ? [sample.squarePixelWidth, sample.squarePixelHeight] + : [sample.squarePixelHeight, sample.squarePixelWidth]; + + crop = options.crop; + if (crop) { + crop = clampCropRectangle(crop, rotatedWidth, rotatedHeight); + } + + [width, height] = crop + ? [crop.width, crop.height] + : [rotatedWidth, rotatedHeight]; + const originalAspectRatio = width / height; + + // If width and height aren't defined together, deduce the missing value using the aspect ratio + if (options.width !== undefined && options.height === undefined) { + width = options.width; + height = Math.round(width / originalAspectRatio); + } else if (options.width === undefined && options.height !== undefined) { + height = options.height; + width = Math.round(height * originalAspectRatio); + } else if (options.width !== undefined && options.height !== undefined) { + width = options.width; + height = options.height; + } + + alpha = options.alpha ?? false; + fit = options.fit ?? 'fill'; + canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null); + needsSetup = false; + } + + let canvas = canvasPool[nextCanvasIndex]; + let canvasIsNew = false; + + if (!canvas) { + if (typeof document !== 'undefined') { + // Prefer an HTMLCanvasElement + canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + } else { + canvas = new OffscreenCanvas(width, height); + } + + if (canvasPool.length > 0) { + canvasPool[nextCanvasIndex] = canvas; + } + + canvasIsNew = true; + } + + if (canvasPool.length > 0) { + nextCanvasIndex = (nextCanvasIndex + 1) % canvasPool.length; + } + + const context = canvas.getContext('2d', { + alpha: alpha || isFirefox(), // Firefox has VideoFrame glitches with opaque canvases + }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; + assert(context); + + sample._drawWithFitAndMipmapping(canvas, context, { + fit, + rotation, + flip, + crop, + targetIsFresh: canvasIsNew, + fillBlack: !alpha && isFirefox(), + }); + sample.close(); + + return new WrappedCanvas(canvas, sample.timestamp, sample.duration); + }; +}; diff --git a/src/decode.ts b/src/decode.ts index 62f5c5e72..ea60b07e3 100644 --- a/src/decode.ts +++ b/src/decode.ts @@ -15,12 +15,1333 @@ import { guessDescriptionForVideo, inferCodecFromCodecString, MediaCodec, + parsePcmCodec, PCM_AUDIO_CODECS, + PcmAudioCodec, VIDEO_CODECS, VideoCodec, } from './codec'; -import { customAudioDecoders, customVideoDecoders } from './custom-coder'; -import { isAllowSharedBufferSource, SetOptional } from './misc'; +import { + addAvcBitstreamRestriction, + AvcNalUnitType, + concatAvcNalUnits, + deserializeAvcDecoderConfigurationRecord, + determineVideoPacketType, + extractNalUnitTypeForAvc, + extractNalUnitTypeForHevc, + HevcNalUnitType, + iterateAvcNalUnits, + iterateHevcNalUnits, + parseAvcSps, + sanitizeHevcPacketForChromium, + serializeAvcDecoderConfigurationRecord, +} from './codec-data'; +import { CustomAudioDecoder, customAudioDecoders, CustomVideoDecoder, customVideoDecoders } from './custom-coder'; +import { + assert, + assertNever, + clamp, + colorSpaceIsComplete, + getInt24, + getUint24, + insertSorted, + isAllowSharedBufferSource, + isChromium, + isWebKit, + last, + NaiveCallSerializer, + promiseWithResolvers, + removeItem, + Rotation, + SetOptional, + toDataView, + toUint8Array, +} from './misc'; +import { EncodedPacket } from './packet'; +import { fromAlaw, fromUlaw } from './pcm'; +import { AudioSample, VideoSample, VideoSamplePixelFormat } from './sample'; + +export abstract class DecoderWrapper< + MediaSample extends VideoSample | AudioSample, +> { + constructor( + public onSample: (sample: MediaSample) => unknown, + public onError: (error: unknown) => unknown, + ) {} + + abstract getDecodeQueueSize(): number; + abstract decode(packet: EncodedPacket): void; + abstract flush(): Promise; + abstract close(): void; + + abstract get closed(): boolean; +} + +export class VideoDecoderWrapper extends DecoderWrapper { + decoder: VideoDecoder | null = null; + + customDecoder: CustomVideoDecoder | null = null; + customDecoderCallSerializer = new NaiveCallSerializer(); + customDecoderQueueSize = 0; + customDecoderClosed = false; + + inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted. + frameQueue: VideoFrame[] = []; // Safari-specific thing, check usage. + currentPacketIndex = 0; + raslSkipped = false; // For HEVC stuff + + // Alpha stuff + alphaDecoder: VideoDecoder | null = null; + alphaHadKeyframe = false; + colorQueue: VideoFrame[] = []; + alphaQueue: (VideoFrame | null)[] = []; + merger: ColorAlphaMerger | null = null; + decodedAlphaChunkCount = 0; + alphaDecoderQueueSize = 0; + /** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */ + nullAlphaFrameQueue: number[] = []; + currentAlphaPacketIndex = 0; + alphaRaslSkipped = false; // For HEVC stuff + finalFrames: { frame: VideoFrame | null }[] = []; + mergeAlphaPromises: Promise[] = []; + + onDequeue: (() => unknown) | null = null; + + constructor( + onSample: (sample: VideoSample) => unknown, + onError: (error: unknown) => unknown, + public codec: VideoCodec, + public decoderConfig: VideoDecoderConfig, + public rotation: Rotation, + public flip: boolean, + public timeResolution: number, + ) { + super(onSample, onError); + + const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig)); + if (MatchingCustomDecoder) { + // @ts-expect-error "Can't create instance of abstract class 🤓" + this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder; + // @ts-expect-error It's technically readonly + this.customDecoder.codec = codec; + // @ts-expect-error It's technically readonly + this.customDecoder.config = decoderConfig; + // @ts-expect-error It's technically readonly + this.customDecoder.onSample = (sample) => { + if (!(sample instanceof VideoSample)) { + throw new TypeError('The argument passed to onSample must be a VideoSample.'); + } + + // @ts-expect-error Readonly + sample.rotation = this.rotation; + // @ts-expect-error Readonly + sample.flip = this.flip; + + this.onSample(sample); + }; + // @ts-expect-error It's technically readonly + this.customDecoder.onError = (error) => { + onError(error); + }; + + void this.customDecoderCallSerializer + .call(() => this.customDecoder!.init()) + .catch(error => onError(error)); + } else { + const colorHandler = (frame: VideoFrame) => { + if (this.alphaQueue.length > 0) { + // Even when no alpha data is present (most of the time), there will be nulls in this queue + const alphaFrame = this.alphaQueue.shift(); + assert(alphaFrame !== undefined); + + void this.mergeAlpha(frame, alphaFrame); + } else { + this.colorQueue.push(frame); + } + }; + + if (isChromium()) { + if (codec === 'avc' && this.decoderConfig.description) { + const record = deserializeAvcDecoderConfigurationRecord( + toUint8Array(this.decoderConfig.description), + ); + if (record && record.sequenceParameterSets.length > 0) { + const sps = parseAvcSps(record.sequenceParameterSets[0]!); + if (sps) { + if (sps.frameMbsOnlyFlag === 0) { + // Chromium has/had a bug with playing interlaced AVC + // (https://issues.chromium.org/issues/456919096) which can be worked around by + // requesting that software decoding be used. So, here we peek into the AVC description, + // if present, and switch to software decoding if we find interlaced content. + this.decoderConfig = { + ...this.decoderConfig, + hardwareAcceleration: 'prefer-software', + }; + } + + if (sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) { + // Modify the SPS to fix potential loss of B frames + record.sequenceParameterSets[0] = addAvcBitstreamRestriction(sps); + this.decoderConfig = { + ...this.decoderConfig, + description: serializeAvcDecoderConfigurationRecord(record), + }; + } + } + } + } + + if (!colorSpaceIsComplete(this.decoderConfig.colorSpace)) { + // Found via https://github.com/remotion-dev/remotion/issues/10841. + // If the color space is incomplete (which is often that it's just partially filled), Chromium has + // some nasty logic where it doesn't pass that information along to the GPU at all. The result is + // that information is genuinely lost, like the color matrix for example. Chromium has other code + // paths where it just fills the missing values with a hardcoded default, so we do the exact same + // thing here, with the same hardcoded defaults: + this.decoderConfig = { + ...this.decoderConfig, + colorSpace: { + primaries: this.decoderConfig.colorSpace?.primaries ?? 'bt709', + matrix: this.decoderConfig.colorSpace?.matrix ?? 'bt709', + transfer: this.decoderConfig.colorSpace?.transfer ?? 'bt709', + fullRange: this.decoderConfig.colorSpace?.fullRange ?? false, + }, + }; + } + } + + const stack = new Error('Decoding error').stack; + + this.decoder = new VideoDecoder({ + output: (frame) => { + try { + colorHandler(frame); + } catch (error) { + this.onError(error); + } + }, + error: (error) => { + error.stack = stack; // Provide a more useful stack trace, the default one sucks + this.onError(error); + }, + }); + this.decoder.configure(this.decoderConfig); + + this.decoder.addEventListener('dequeue', () => { + this.onDequeue?.(); + }); + } + } + + getDecodeQueueSize() { + if (this.customDecoder) { + return this.customDecoderQueueSize; + } else { + assert(this.decoder); + + let result = Math.max( + this.decoder.decodeQueueSize, + this.alphaDecoder?.decodeQueueSize ?? 0, + ); + + // Frames in an in-flight alpha merge must also count towards the queue size + result += this.finalFrames.length; + + return result; + } + } + + decode(packet: EncodedPacket) { + assert(!packet.isMetadataOnly); + + if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) { + if (this.hasHevcRaslPicture(packet.data)) { + return; // Drop + } + + this.raslSkipped = true; + } + + if (this.customDecoder) { + this.customDecoderQueueSize++; + void this.customDecoderCallSerializer + .call(() => this.customDecoder!.decode(packet)) + .catch(error => this.onError(error)) + .finally(() => { + this.customDecoderQueueSize--; + this.onDequeue?.(); + }); + } else { + assert(this.decoder); + + if (!isWebKit()) { + insertSorted(this.inputTimestamps, packet.timestamp, x => x); + } + + if (isChromium() && this.currentPacketIndex === 0) { + if (this.codec === 'avc') { + // Workaround for https://issues.chromium.org/issues/470109459 + const filteredNalUnits: Uint8Array[] = []; + let hasFrameData = false; + + for (const loc of iterateAvcNalUnits(packet.data, this.decoderConfig)) { + const type = extractNalUnitTypeForAvc(packet.data[loc.offset]!); + hasFrameData ||= type >= 1 && type <= 5; + + if (type === AvcNalUnitType.AUD) { + if (hasFrameData) { + // Already has actual frame data, so treat an AUD as simply the end of the packet + break; + } else { + // If packets contain an AUD and have NALUs before it, this trips up Chromium's key + // frame detector. Clear the NALUs if an AUD is encountered. + // https://github.com/Vanilagy/mediabunny/issues/396 + filteredNalUnits.length = 0; + } + } + + // These trip up Chromium's key frame detection, so let's strip them + if (!(type >= 20 && type <= 31)) { + filteredNalUnits.push(packet.data.subarray(loc.offset, loc.offset + loc.length)); + } + } + + if (!this.decoderConfig.description) { + // Do SPS fixups if necessary + for (let i = 0; i < filteredNalUnits.length; i++) { + const nalUnit = filteredNalUnits[i]!; + if (extractNalUnitTypeForAvc(nalUnit[0]!) !== AvcNalUnitType.SPS) { + continue; + } + + const sps = parseAvcSps(nalUnit); + if (sps && sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) { + filteredNalUnits[i] = addAvcBitstreamRestriction(sps); + } + + break; + } + } + + const newData = concatAvcNalUnits(filteredNalUnits, this.decoderConfig); + packet = new EncodedPacket(newData, packet.type, packet.timestamp, packet.duration); + } else if (this.codec === 'hevc') { + // Workaround for https://issues.chromium.org/issues/507611247 + const sanitizedData = sanitizeHevcPacketForChromium(packet.data, this.decoderConfig); + if (sanitizedData) { + packet = new EncodedPacket(sanitizedData, packet.type, packet.timestamp, packet.duration); + } + } + } + + this.decoder.decode(packet.toEncodedVideoChunk()); + this.decodeAlphaData(packet); + } + + this.currentPacketIndex++; + } + + decodeAlphaData(packet: EncodedPacket) { + if (!packet.sideData.alpha) { + // No alpha side data in the packet, most common case + this.pushNullAlphaFrame(); + return; + } + + if (!this.merger) { + this.merger = new ColorAlphaMerger(); + } + + // Check if we need to set up the alpha decoder + if (!this.alphaDecoder) { + const alphaHandler = (frame: VideoFrame) => { + if (this.colorQueue.length > 0) { + const colorFrame = this.colorQueue.shift(); + assert(colorFrame !== undefined); + + void this.mergeAlpha(colorFrame, frame); + } else { + this.alphaQueue.push(frame); + } + + // Check if any null frames have been queued for this point + this.decodedAlphaChunkCount++; + while ( + this.nullAlphaFrameQueue.length > 0 + && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount + ) { + this.nullAlphaFrameQueue.shift(); + + if (this.colorQueue.length > 0) { + const colorFrame = this.colorQueue.shift(); + assert(colorFrame !== undefined); + + void this.mergeAlpha(colorFrame, null); + } else { + this.alphaQueue.push(null); + } + } + + this.alphaDecoderQueueSize--; + }; + + const stack = new Error('Decoding error').stack; + + this.alphaDecoder = new VideoDecoder({ + output: (frame) => { + try { + alphaHandler(frame); + } catch (error) { + this.onError(error); + } + }, + error: (error) => { + error.stack = stack; // Provide a more useful stack trace, the default one sucks + this.onError(error); + }, + }); + this.alphaDecoder.configure({ + ...this.decoderConfig, + // Alpha is always full range, regardless of what the color track says. The decoder only honors the + // override if all fields are set tho. + colorSpace: { + fullRange: true, + // These fields are irrelevant: + matrix: 'bt709', + primaries: 'bt709', + transfer: 'bt709', + }, + }); + + this.alphaDecoder.addEventListener('dequeue', () => { + this.onDequeue?.(); + }); + } + + const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha); + + // Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start + // decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque. + if (!this.alphaHadKeyframe) { + this.alphaHadKeyframe = type === 'key'; + } + + if (this.alphaHadKeyframe) { + // Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but + // here for symmetry. + if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) { + if (this.hasHevcRaslPicture(packet.sideData.alpha)) { + this.pushNullAlphaFrame(); + return; + } + + this.alphaRaslSkipped = true; + } + + this.currentAlphaPacketIndex++; + this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type)); + this.alphaDecoderQueueSize++; + } else { + this.pushNullAlphaFrame(); + } + } + + pushNullAlphaFrame() { + if (this.alphaDecoderQueueSize === 0) { + // Easy + this.alphaQueue.push(null); + } else { + // There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order + // data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload + // has finished. + this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize); + } + } + + /** + * If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as + * CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave + * differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good + * and causes bugs upstream. So, let's take the dropping into our own hands. + */ + hasHevcRaslPicture(packetData: Uint8Array) { + for (const loc of iterateHevcNalUnits(packetData, this.decoderConfig)) { + const type = extractNalUnitTypeForHevc(packetData[loc.offset]!); + if (type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R) { + return true; + } + } + + return false; + } + + /** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */ + frameHandler(frame: VideoFrame) { + if (isWebKit()) { + // For correct B-frame handling, we don't just hand over the frames directly but instead add them to + // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue + // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we + // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are + // emitted in presentation order, but Safari doesn't always follow this rule. + if (this.frameQueue.length > 0 && (frame.timestamp >= last(this.frameQueue)!.timestamp)) { + for (const frame of this.frameQueue) { + this.finalizeAndEmitSample(frame); + } + + this.frameQueue.length = 0; + } + + insertSorted(this.frameQueue, frame, x => x.timestamp); + } else { + // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are + // required to emit decoded frames in presentation order *while* retaining the timestamp of their + // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a + // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and + // ensure they are sorted by doing this. + const timestamp = this.inputTimestamps.shift(); + + // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the + // correspondence should be 1:1. + assert(timestamp !== undefined); + this.finalizeAndEmitSample(frame, timestamp); + } + } + + finalizeAndEmitSample(frame: VideoFrame, timestampOverride?: number) { + const sample = new VideoSample(frame, { + // Round the timestamps to the time resolution + timestamp: Math.round( + (timestampOverride ?? (frame.timestamp / 1e6)) * this.timeResolution, + ) / this.timeResolution, + duration: Math.round( + (frame.duration ?? 0) / 1e6 * this.timeResolution, + ) / this.timeResolution, + rotation: this.rotation, + flip: this.flip, + }); + + this.onSample(sample); + } + + async mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) { + const resolver = promiseWithResolvers(); + this.mergeAlphaPromises.push(resolver.promise); + + // Alpha merging is concurrent but the frames must still be emitted in the same order in which the merging + // began. Therefore, serialize the results in an array. + const result: { frame: VideoFrame | null } = { frame: null }; + this.finalFrames.push(result); + + let finalFrameRemoved = false; + + try { + if (!alpha) { + // Nothing needs to be merged + result.frame = color; + } else { + assert(this.merger); + + // The merger takes ownership of the frames, so no need to close them ourselves + result.frame = await this.merger.merge(color, alpha); + } + + // Emit any leading frames that are ready, preserving input order + while (this.finalFrames.length > 0 && this.finalFrames[0]!.frame !== null) { + const next = this.finalFrames.shift()!; + finalFrameRemoved = true; + this.frameHandler(next.frame!); + } + } catch (error) { + removeItem(this.finalFrames, result); + finalFrameRemoved = true; + this.onError(error); + } finally { + removeItem(this.mergeAlphaPromises, resolver.promise); + resolver.resolve(); + + if (finalFrameRemoved) { + // Since final frames affect the decode queue size + this.onDequeue?.(); + } + } + } + + async flush() { + if (this.customDecoder) { + await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush()); + } else { + assert(this.decoder); + await Promise.all([ + this.decoder.flush(), + this.alphaDecoder?.flush(), + ]); + await Promise.all(this.mergeAlphaPromises); + + this.colorQueue.forEach(x => x.close()); + this.colorQueue.length = 0; + this.alphaQueue.forEach(x => x?.close()); + this.alphaQueue.length = 0; + + this.alphaHadKeyframe = false; + this.decodedAlphaChunkCount = 0; + this.alphaDecoderQueueSize = 0; + this.nullAlphaFrameQueue.length = 0; + this.currentAlphaPacketIndex = 0; + this.alphaRaslSkipped = false; + } + + if (isWebKit()) { + for (const sample of this.frameQueue) { + this.finalizeAndEmitSample(sample); + } + + this.frameQueue.length = 0; + } + + this.currentPacketIndex = 0; + this.raslSkipped = false; + } + + close() { + if (this.customDecoder) { + if (!this.customDecoderClosed) { + this.customDecoderClosed = true; + void this.customDecoderCallSerializer.call(() => this.customDecoder!.close()); + } + } else { + assert(this.decoder); + + if (this.decoder.state !== 'closed') { + this.decoder.close(); + } + if (this.alphaDecoder && this.alphaDecoder.state !== 'closed') { + this.alphaDecoder.close(); + } + + this.colorQueue.forEach(x => x.close()); + this.colorQueue.length = 0; + this.alphaQueue.forEach(x => x?.close()); + this.alphaQueue.length = 0; + + this.merger?.close(); + } + + for (const sample of this.frameQueue) { + sample.close(); + } + this.frameQueue.length = 0; + } + + get closed() { + if (this.customDecoder) { + if (this.customDecoderClosed) { + return true; + } + + return !this.customDecoderCallSerializer.errored; + } else { + assert(this.decoder); + return this.decoder.state === 'closed'; + } + } +} + +let mergerWorkerUrl: string | null = null; + +/** Utility class that merges together color and alpha information on the CPU in a pool of workers. */ +class ColorAlphaMerger { + private workers: Worker[] = []; + private nextWorkerIndex = 0; + private pendingRequests = new Map>>(); + private nextRequestId = 0; + + merge(color: VideoFrame, alpha: VideoFrame): Promise { + if (this.workers.length === 0) { + if (!mergerWorkerUrl) { + const blob = new Blob( + [`(${colorAlphaMergerWorkerCode.toString()})()`], + { type: 'application/javascript' }, + ); + mergerWorkerUrl = URL.createObjectURL(blob); + } + + const poolSize = clamp(navigator.hardwareConcurrency, 1, 4); + for (let i = 0; i < poolSize; i++) { + const worker = new Worker(mergerWorkerUrl); + + worker.addEventListener('message', (event: MessageEvent) => { + const data = event.data; + const pending = this.pendingRequests.get(data.id); + if (!pending) { + return; + } + this.pendingRequests.delete(data.id); + + if ('error' in data) { + pending.reject(new Error(data.error)); + } else { + pending.resolve(data.frame); + } + }); + + worker.addEventListener('error', (event) => { + const error = new Error(event.message || 'Color/alpha merge worker error.'); + for (const pending of this.pendingRequests.values()) { + pending.reject(error); + } + this.pendingRequests.clear(); + }); + + this.workers.push(worker); + } + } + + const id = this.nextRequestId++; + const pending = promiseWithResolvers(); + this.pendingRequests.set(id, pending); + + // Hand the job to the next worker in round-robin fashion + const worker = this.workers[this.nextWorkerIndex]!; + this.nextWorkerIndex = (this.nextWorkerIndex + 1) % this.workers.length; + worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] }); + + return pending.promise; + } + + close() { + for (const worker of this.workers) { + worker.terminate(); + } + this.workers.length = 0; + + const error = new Error('Color/alpha merger closed.'); + for (const pending of this.pendingRequests.values()) { + pending.reject(error); + } + this.pendingRequests.clear(); + } +} + +type ColorAlphaMergerWorkerRequest = { + id: number; + color: VideoFrame; + alpha: VideoFrame; +}; + +type ColorAlphaMergerWorkerResponse = + | { id: number; frame: VideoFrame } + | { id: number; error: string }; + +const colorAlphaMergerWorkerCode = () => { + // These buffers are reused across frames as long as the size matches, since consecutive frames usually share + // dimensions + let cpuAlphaBuffer: Uint8Array | null = null; + let cpuColorBuffer: Uint8Array | null = null; + + // Serialize execution internally so concurrent requests don't race on the shared cpu*Buffer state. + let chain: Promise = Promise.resolve(); + self.addEventListener('message', (event: MessageEvent) => { + const { id, color, alpha } = event.data; + chain = chain.then(async () => { + try { + const frame = await merge(color, alpha); + self.postMessage({ id, frame }, { transfer: [frame] }); + } catch (error) { + self.postMessage({ id, error: (error as Error).message }); + } finally { + // We took ownership of the inputs via transfer; close them now that the merge (or its error) is done. + color.close(); + alpha.close(); + } + }); + }); + + const merge = async (color: VideoFrame, alpha: VideoFrame): Promise => { + const format = color.format as VideoSamplePixelFormat | null; + const alphaFormat = alpha.format as VideoSamplePixelFormat | null; + if (!format || !alphaFormat) { + throw new Error('CPU color/alpha merging requires a known VideoFrame format.'); + } + + // The alpha frame must have the same bit depth as the color frame + const colorIs10 = format.includes('P10'); + const colorIs12 = format.includes('P12'); + const alphaIs10 = alphaFormat.includes('P10'); + const alphaIs12 = alphaFormat.includes('P12'); + if (alphaIs10 !== colorIs10 || alphaIs12 !== colorIs12) { + throw new Error( + `CPU color/alpha merging requires the alpha frame to have the same bit depth as the color frame` + + ` (color: '${format}', alpha: '${alphaFormat}').`, + ); + } + + if (format === 'RGBX' || format === 'RGBA' || format === 'BGRX' || format === 'BGRA') { + return await mergeInterleavedRgba(color, alpha, format); + } else if ( + format === 'I420' || format === 'I420P10' || format === 'I420P12' + || format === 'I422' || format === 'I422P10' || format === 'I422P12' + || format === 'I444' || format === 'I444P10' || format === 'I444P12' + ) { + return await mergePlanarYuv(color, alpha, format); + } else if (format === 'NV12') { + return await mergeNv12(color, alpha); + } + + throw new Error(`CPU color/alpha merging does not support format '${format}'.`); + }; + + const mergeInterleavedRgba = async ( + color: VideoFrame, + alpha: VideoFrame, + format: 'RGBX' | 'RGBA' | 'BGRX' | 'BGRA', + ): Promise => { + const width = color.visibleRect?.width ?? color.codedWidth; + const height = color.visibleRect?.height ?? color.codedHeight; + + const pixelCount = width * height; + const output = new Uint8Array(pixelCount * 4); + + // Color goes straight into the output buffer via copyTo, no intermediate copy needed + await color.copyTo(output); + + // And now add the alpha data + const alphaY = await readAlpha(alpha, width, height, 1); + for (let i = 0, j = 3; i < pixelCount; i++, j += 4) { + output[j] = alphaY[i]!; + } + + const outputFormat = (format === 'RGBX' || format === 'RGBA') ? 'RGBA' : 'BGRA'; + const init = { + format: outputFormat, + codedWidth: width, + codedHeight: height, + timestamp: color.timestamp, + duration: color.duration ?? undefined, + transfer: [output.buffer], + } as const; + + return new VideoFrame(output, init); + }; + + const mergePlanarYuv = async ( + color: VideoFrame, + alpha: VideoFrame, + format: + | 'I420' | 'I420P10' | 'I420P12' + | 'I422' | 'I422P10' | 'I422P12' + | 'I444' | 'I444P10' | 'I444P12', + ): Promise => { + const width = color.visibleRect?.width ?? color.codedWidth; + const height = color.visibleRect?.height ?? color.codedHeight; + + const is10 = format.includes('P10'); + const is12 = format.includes('P12'); + const bytesPerSample = (is10 || is12) ? 2 : 1; + + let chromaW: number; + let chromaH: number; + if (format.startsWith('I420')) { + chromaW = Math.ceil(width / 2); + chromaH = Math.ceil(height / 2); + } else if (format.startsWith('I422')) { + chromaW = Math.ceil(width / 2); + chromaH = height; + } else { + chromaW = width; + chromaH = height; + } + + const ySamples = width * height; + const uvSamples = chromaW * chromaH; + const yBytes = ySamples * bytesPerSample; + const uvBytes = uvSamples * bytesPerSample; + const aBytes = ySamples * bytesPerSample; + + const outputBytes = yBytes + 2 * uvBytes + aBytes; + const output = new Uint8Array(outputBytes); + + // Write color planes directly into the output buffer via copyTo, no intermediate copy + await color.copyTo(output); + + const alphaY = await readAlpha(alpha, width, height, bytesPerSample); + const aOffset = yBytes + 2 * uvBytes; + output.set(alphaY, aOffset); + + const outputFormat = (format.slice(0, 4) + 'A' + format.slice(4)) as VideoPixelFormat; + + const init = { + format: outputFormat, + codedWidth: width, + codedHeight: height, + timestamp: color.timestamp, + duration: color.duration ?? undefined, + transfer: [output.buffer], + }; + + return new VideoFrame(output, init); + }; + + const mergeNv12 = async ( + color: VideoFrame, + alpha: VideoFrame, + ): Promise => { + const width = color.visibleRect?.width ?? color.codedWidth; + const height = color.visibleRect?.height ?? color.codedHeight; + + const ySize = width * height; + const chromaW = Math.ceil(width / 2); + const chromaH = Math.ceil(height / 2); + const uvSize = chromaW * chromaH; + + const sourceSize = color.allocationSize(); + if (!cpuColorBuffer || cpuColorBuffer.byteLength !== sourceSize) { + cpuColorBuffer = new Uint8Array(sourceSize); + } + await color.copyTo(cpuColorBuffer); + + const output = new Uint8Array(ySize + 2 * uvSize + ySize); + + // Y plane copies straight over + output.set(cpuColorBuffer.subarray(0, ySize), 0); + + // Deinterleave the UV plane into separate U and V planes + const uOffset = ySize; + const vOffset = ySize + uvSize; + const uvStart = ySize; + for (let i = 0; i < uvSize; i++) { + output[uOffset + i] = cpuColorBuffer[uvStart + i * 2]!; + output[vOffset + i] = cpuColorBuffer[uvStart + i * 2 + 1]!; + } + + const alphaY = await readAlpha(alpha, width, height, 1); + output.set(alphaY, ySize + 2 * uvSize); + + const init = { + format: 'I420A', + codedWidth: width, + codedHeight: height, + timestamp: color.timestamp, + duration: color.duration ?? undefined, + transfer: [output.buffer], + } as const; + + return new VideoFrame(output, init); + }; + + const readAlpha = async (alpha: VideoFrame, width: number, height: number, bytesPerSample: number) => { + const size = alpha.allocationSize(); + if (!cpuAlphaBuffer || cpuAlphaBuffer.byteLength !== size) { + cpuAlphaBuffer = new Uint8Array(size); + } + await alpha.copyTo(cpuAlphaBuffer); + + const format = alpha.format; + if (format === 'RGBA' || format === 'BGRA' || format === 'RGBX' || format === 'BGRX') { + // Pack alpha data tightly. Assume alpha is stored in RGB, so sample just from R for simplicity. + const rOffset = (format === 'RGBA' || format === 'RGBX') ? 0 : 2; + const pixelCount = width * height; + for (let i = 0; i < pixelCount; i++) { + cpuAlphaBuffer[i] = cpuAlphaBuffer[i * 4 + rOffset]!; + } + return cpuAlphaBuffer.subarray(0, pixelCount); + } else { + // For Y-plane-first formats (I*** and NV12), the leading width*height samples are the Y plane + const yPlane = cpuAlphaBuffer.subarray(0, width * height * bytesPerSample); + + if (alpha.colorSpace.fullRange === false) { + // Some decoders hand us limited-range alpha, so stretch it back out to full range + const bitDepth = format!.includes('P12') ? 12 : format!.includes('P10') ? 10 : 8; + const low = 16 << (bitDepth - 8); + const high = 235 << (bitDepth - 8); + const max = (1 << bitDepth) - 1; + const samples = bytesPerSample === 2 + ? new Uint16Array(yPlane.buffer, 0, width * height) + : yPlane; + + for (let i = 0; i < samples.length; i++) { + const value = Math.round((samples[i]! - low) * max / (high - low)); + samples[i] = Math.min(Math.max(value, 0), max); + } + } + + return yPlane; + } + }; +}; + +export class AudioDecoderWrapper extends DecoderWrapper { + decoder: AudioDecoder | null = null; + + customDecoder: CustomAudioDecoder | null = null; + customDecoderCallSerializer = new NaiveCallSerializer(); + customDecoderQueueSize = 0; + customDecoderClosed = false; + + // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially + // inaccurate) packet timestamps. + currentTimestamp: number | null = null; + // Chromium does not respect negative packet timestamps, so we must do the fixin' ourselves + expectedFirstTimestamp: number | null = null; + timestampOffset = 0; + + onDequeue: (() => unknown) | null = null; + + constructor( + onSample: (sample: AudioSample) => unknown, + onError: (error: unknown) => unknown, + codec: AudioCodec, + decoderConfig: AudioDecoderConfig, + ) { + super(onSample, onError); + + const sampleHandler = (sample: AudioSample) => { + let sampleTimestamp = sample.timestamp; + + if (this.expectedFirstTimestamp !== null && this.currentTimestamp === null) { + this.timestampOffset = this.expectedFirstTimestamp - sampleTimestamp; + } + + sampleTimestamp += this.timestampOffset; + + if ( + this.currentTimestamp === null + || Math.abs(sampleTimestamp - this.currentTimestamp) >= sample.duration + ) { + // We need to sync with the sample timestamp again + this.currentTimestamp = sampleTimestamp; + } + + const preciseTimestamp = this.currentTimestamp; + this.currentTimestamp += sample.duration; + + if (sample.numberOfFrames === 0) { + // We skip zero-data (empty) AudioSamples. These are sometimes emitted, for example, by Firefox when it + // decodes Vorbis (at the start). + sample.close(); + return; + } + + // Round the timestamp to the sample rate + const sampleRate = decoderConfig.sampleRate; + // @ts-expect-error Readonly + sample.timestamp = Math.round(preciseTimestamp * sampleRate) / sampleRate; + + onSample(sample); + }; + + const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig)); + if (MatchingCustomDecoder) { + // @ts-expect-error "Can't create instance of abstract class 🤓" + this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder; + // @ts-expect-error It's technically readonly + this.customDecoder.codec = codec; + // @ts-expect-error It's technically readonly + this.customDecoder.config = decoderConfig; + // @ts-expect-error It's technically readonly + this.customDecoder.onSample = (sample) => { + if (!(sample instanceof AudioSample)) { + throw new TypeError('The argument passed to onSample must be an AudioSample.'); + } + + sampleHandler(sample); + }; + // @ts-expect-error It's technically readonly + this.customDecoder.onError = (error) => { + onError(error); + }; + + void this.customDecoderCallSerializer + .call(() => this.customDecoder!.init()) + .catch(error => onError(error)); + } else { + const stack = new Error('Decoding error').stack; + + this.decoder = new AudioDecoder({ + output: (data) => { + try { + sampleHandler(new AudioSample(data)); + } catch (error) { + this.onError(error); + } + }, + error: (error) => { + error.stack = stack; // Provide a more useful stack trace, the default one sucks + this.onError(error); + }, + }); + this.decoder.configure(decoderConfig); + + this.decoder.addEventListener('dequeue', () => { + this.onDequeue?.(); + }); + } + } + + getDecodeQueueSize() { + if (this.customDecoder) { + return this.customDecoderQueueSize; + } else { + assert(this.decoder); + return this.decoder.decodeQueueSize; + } + } + + decode(packet: EncodedPacket) { + assert(!packet.isMetadataOnly); + + if (this.customDecoder) { + this.customDecoderQueueSize++; + void this.customDecoderCallSerializer + .call(() => this.customDecoder!.decode(packet)) + .catch(error => this.onError(error)) + .finally(() => { + this.customDecoderQueueSize--; + this.onDequeue?.(); + }); + } else { + assert(this.decoder); + + this.expectedFirstTimestamp ??= packet.timestamp; + this.decoder.decode(packet.toEncodedAudioChunk()); + } + } + + async flush() { + if (this.customDecoder) { + await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush()); + } else { + assert(this.decoder); + await this.decoder.flush(); + } + + this.currentTimestamp = null; + this.expectedFirstTimestamp = null; + this.timestampOffset = 0; + } + + close() { + if (this.customDecoder) { + if (!this.customDecoderClosed) { + this.customDecoderClosed = true; + void this.customDecoderCallSerializer.call(() => this.customDecoder!.close()); + } + } else { + assert(this.decoder); + + if (this.decoder.state !== 'closed') { + this.decoder.close(); + } + } + } + + get closed() { + if (this.customDecoder) { + if (this.customDecoderClosed) { + return true; + } + + return !this.customDecoderCallSerializer.errored; + } else { + assert(this.decoder); + return this.decoder.state === 'closed'; + } + } +} + +// There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple +// decoder that maps any input PCM format into a PCM format supported by the browser. +export class PcmAudioDecoderWrapper extends DecoderWrapper { + codec: PcmAudioCodec; + + inputSampleSize: 1 | 2 | 3 | 4 | 8; + readInputValue: (view: DataView, byteOffset: number) => number; + + outputSampleSize: 1 | 2 | 4; + outputFormat: 'u8' | 's16' | 's32' | 'f32'; + writeOutputValue: (view: DataView, byteOffset: number, value: number) => void; + + // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially + // inaccurate) packet timestamps. + currentTimestamp: number | null = null; + + isClosed = false; + + onDequeue: (() => unknown) | null = null; + + constructor( + onSample: (sample: AudioSample) => unknown, + onError: (error: unknown) => unknown, + public decoderConfig: AudioDecoderConfig, + ) { + super(onSample, onError); + + assert((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)); + this.codec = decoderConfig.codec as PcmAudioCodec; + + const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec); + this.inputSampleSize = sampleSize; + + switch (sampleSize) { + case 1: { + if (dataType === 'unsigned') { + this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7; + } else if (dataType === 'signed') { + this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset); + } else if (dataType === 'ulaw') { + this.readInputValue = (view, byteOffset) => fromUlaw(view.getUint8(byteOffset)); + } else if (dataType === 'alaw') { + this.readInputValue = (view, byteOffset) => fromAlaw(view.getUint8(byteOffset)); + } else { + assert(false); + } + }; break; + case 2: { + if (dataType === 'unsigned') { + this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15; + } else if (dataType === 'signed') { + this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian); + } else { + assert(false); + } + }; break; + case 3: { + if (dataType === 'unsigned') { + this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23; + } else if (dataType === 'signed') { + this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian); + } else { + assert(false); + } + }; break; + case 4: { + if (dataType === 'unsigned') { + this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31; + } else if (dataType === 'signed') { + this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian); + } else if (dataType === 'float') { + this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian); + } else { + assert(false); + } + }; break; + case 8: { + if (dataType === 'float') { + this.readInputValue = (view, byteOffset) => view.getFloat64(byteOffset, littleEndian); + } else { + assert(false); + } + }; break; + default: { + assertNever(sampleSize); + assert(false); + }; + } + + switch (sampleSize) { + case 1: { + if (dataType === 'ulaw' || dataType === 'alaw') { + this.outputSampleSize = 2; + this.outputFormat = 's16'; + this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true); + } else { + this.outputSampleSize = 1; + this.outputFormat = 'u8'; + this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7); + } + }; break; + case 2: { + this.outputSampleSize = 2; + this.outputFormat = 's16'; + this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true); + }; break; + case 3: { + this.outputSampleSize = 4; + this.outputFormat = 's32'; + // From https://www.w3.org/TR/webcodecs: + // AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are + // stored in s32, each sample MUST be left-shifted by 8 bits. + this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true); + }; break; + case 4: { + this.outputSampleSize = 4; + + if (dataType === 'float') { + this.outputFormat = 'f32'; + this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true); + } else { + this.outputFormat = 's32'; + this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true); + } + }; break; + case 8: { + this.outputSampleSize = 4; + + this.outputFormat = 'f32'; + this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true); + }; break; + default: { + assertNever(sampleSize); + assert(false); + }; + }; + } + + getDecodeQueueSize() { + return 0; + } + + decode(packet: EncodedPacket) { + this.onDequeue?.(); + + const inputView = toDataView(packet.data); + + const numberOfFrames = packet.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize; + + const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize; + const outputBuffer = new ArrayBuffer(outputBufferSize); + const outputView = new DataView(outputBuffer); + + for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) { + const inputIndex = i * this.inputSampleSize; + const outputIndex = i * this.outputSampleSize; + + const value = this.readInputValue(inputView, inputIndex); + this.writeOutputValue(outputView, outputIndex, value); + } + + const preciseDuration = numberOfFrames / this.decoderConfig.sampleRate; + if (this.currentTimestamp === null || Math.abs(packet.timestamp - this.currentTimestamp) >= preciseDuration) { + // We need to sync with the packet timestamp again + this.currentTimestamp = packet.timestamp; + } + + const preciseTimestamp = this.currentTimestamp; + this.currentTimestamp += preciseDuration; + + const audioSample = new AudioSample({ + format: this.outputFormat, + data: outputBuffer, + numberOfChannels: this.decoderConfig.numberOfChannels, + sampleRate: this.decoderConfig.sampleRate, + numberOfFrames, + timestamp: preciseTimestamp, + }); + + this.onSample(audioSample); + } + + async flush() { + // Do nothing + } + + close() { + this.isClosed = true; + } + + get closed() { + return this.isClosed; + } +} export const canDecodeVideoMemo = new Map>(); export const canDecodeAudioMemo = new Map>(); diff --git a/src/flac/flac-demuxer.ts b/src/flac/flac-demuxer.ts index 7a581d00e..3d2852cc7 100644 --- a/src/flac/flac-demuxer.ts +++ b/src/flac/flac-demuxer.ts @@ -10,16 +10,17 @@ import { FlacBlockType, readVorbisComments } from '../codec-data'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrackBacking } from '../input-track'; -import { PacketRetrievalOptions } from '../media-sink'; import { assert, AsyncMutex, binarySearchLessOrEqual, isThenable, + MaybeRelevantPromise, + ResultValue, textDecoder, UNDETERMINED_LANGUAGE, } from '../misc'; -import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; +import { EncodedPacket, PacketRetrievalOptions, PacketRetrievalResult, PLACEHOLDER_DATA } from '../packet'; import { FileSlice, readBytes, @@ -286,13 +287,16 @@ export class FlacDemuxer extends Demuxer { })()); } - async readNextFlacFrame({ - startPos, - isFirstPacket, - }: { - startPos: number; - isFirstPacket: boolean; - }): Promise { + async readNextFlacFrame( + res: ResultValue, + { + startPos, + isFirstPacket, + }: { + startPos: number; + isFirstPacket: boolean; + }, + ): MaybeRelevantPromise { assert(this.audioInfo); // we expect that there are at least `minimumFrameSize` bytes left in the file @@ -337,14 +341,15 @@ export class FlacDemuxer extends Demuxer { const maximumSliceLength = effectiveMaxFrameSize + maximumHeaderLength; - const slice = await this.reader.requestSliceRange( + let slice = this.reader.requestSliceRange( startPos, maximumHeaderLength, maximumSliceLength, ); + if (isThenable(slice)) slice = await slice; if (!slice) { - return null; + return res.set(null); } const frameHeader = this.readFlacFrameHeader({ @@ -353,7 +358,7 @@ export class FlacDemuxer extends Demuxer { }); if (!frameHeader) { - return null; + return res.set(null); } // We don't know exactly how long the packet is, we only know the `minimumFrameSize` and `maximumFrameSize` @@ -367,13 +372,13 @@ export class FlacDemuxer extends Demuxer { while (true) { // Reached end of the file, packet is over if (slice.filePos > slice.end - minimumHeaderLength) { - return { + return res.set({ num: frameHeader.num, blockSize: frameHeader.blockSize, sampleRate: frameHeader.sampleRate, size: slice.end - startPos, isLastFrame: true, - }; + }); } const nextByte = readU8(slice); @@ -419,13 +424,13 @@ export class FlacDemuxer extends Demuxer { } } - return { + return res.set({ num: frameHeader.num, blockSize: frameHeader.blockSize, sampleRate: frameHeader.sampleRate, size: lengthIfNextFlacFrameHeaderIsLegit, isLastFrame: false, - }; + }); } } } @@ -533,21 +538,24 @@ export class FlacDemuxer extends Demuxer { return { num, blockSize, sampleRate }; } - async advanceReader() { - await this.readMetadata(); + async advanceReader(res: ResultValue): MaybeRelevantPromise { assert(this.lastLoadedPos !== null); assert(this.audioInfo); const startPos = this.lastLoadedPos; - const frame = await this.readNextFlacFrame({ + + const frameResult = new ResultValue(); + const promise = this.readNextFlacFrame(frameResult, { startPos, isFirstPacket: this.loadedSamples.length === 0, }); + if (frameResult.pending) await promise; + const frame = frameResult.value; if (!frame) { // Unexpected case, failed to read next FLAC frame // handling gracefully this.lastSampleLoaded = true; - return; + return res.set(); } const lastSample = this.loadedSamples[this.loadedSamples.length - 1]; @@ -567,8 +575,9 @@ export class FlacDemuxer extends Demuxer { if (frame.isLastFrame) { this.lastSampleLoaded = true; - return; } + + return res.set(); } } @@ -670,102 +679,118 @@ class FlacAudioTrackBacking implements InputAudioTrackBacking { } async getPacket( + res: ResultValue, timestamp: number, options: PacketRetrievalOptions, - ): Promise { + ): MaybeRelevantPromise { assert(this.demuxer.audioInfo); if (timestamp < 0) { - return null; + return res.set(new PacketRetrievalResult(null)); } - const release = await this.demuxer.readingMutex.acquire(); + using lock = this.demuxer.readingMutex.lock(); + if (lock.pending) await lock.ready; - try { - while (true) { - const packetIndex = binarySearchLessOrEqual( - this.demuxer.loadedSamples, - timestamp, - x => x.blockOffset / this.demuxer.audioInfo!.sampleRate, - ); - if (packetIndex === -1) { - await this.demuxer.advanceReader(); - continue; - } + const advanceResult = new ResultValue(); + + while (true) { + const packetIndex = binarySearchLessOrEqual( + this.demuxer.loadedSamples, + timestamp, + x => x.blockOffset / this.demuxer.audioInfo!.sampleRate, + ); + if (packetIndex === -1) { + advanceResult.reset(); + const promise = this.demuxer.advanceReader(advanceResult); + if (advanceResult.pending) await promise; - const packet = this.demuxer.loadedSamples[packetIndex]!; - const sampleTimestamp + continue; + } + + const packet = this.demuxer.loadedSamples[packetIndex]!; + const sampleTimestamp = packet.blockOffset / this.demuxer.audioInfo.sampleRate; - const sampleDuration + const sampleDuration = packet.blockSize / this.demuxer.audioInfo.sampleRate; - if (sampleTimestamp + sampleDuration <= timestamp) { - if (this.demuxer.lastSampleLoaded) { - return this.getPacketAtIndex( - this.demuxer.loadedSamples.length - 1, - options, - ); - } - - await this.demuxer.advanceReader(); - continue; + if (sampleTimestamp + sampleDuration <= timestamp) { + if (this.demuxer.lastSampleLoaded) { + return this.getPacketAtIndex( + res, + this.demuxer.loadedSamples.length - 1, + options, + ); } - return this.getPacketAtIndex(packetIndex, options); + advanceResult.reset(); + const promise = this.demuxer.advanceReader(advanceResult); + if (advanceResult.pending) await promise; + + continue; } - } finally { - release(); + + return this.getPacketAtIndex(res, packetIndex, options); } } async getNextPacket( + res: ResultValue, packet: EncodedPacket, options: PacketRetrievalOptions, - ): Promise { - const release = await this.demuxer.readingMutex.acquire(); - try { - const nextIndex = packet.sequenceNumber + 1; - if ( - this.demuxer.lastSampleLoaded - && nextIndex >= this.demuxer.loadedSamples.length - ) { - return null; - } + ): MaybeRelevantPromise { + using lock = this.demuxer.readingMutex.lock(); + if (lock.pending) await lock.ready; - // Ensure the next sample exists - while ( - nextIndex >= this.demuxer.loadedSamples.length - && !this.demuxer.lastSampleLoaded - ) { - await this.demuxer.advanceReader(); - } - return this.getPacketAtIndex(nextIndex, options); - } finally { - release(); + assert(packet.sequenceNumber >= 0); + + const nextIndex = packet.sequenceNumber + 1; + if ( + this.demuxer.lastSampleLoaded + && nextIndex >= this.demuxer.loadedSamples.length + ) { + return res.set(new PacketRetrievalResult(null)); } + + const advanceResult = new ResultValue(); + + // Ensure the next sample exists + while ( + nextIndex >= this.demuxer.loadedSamples.length + && !this.demuxer.lastSampleLoaded + ) { + advanceResult.reset(); + const promise = this.demuxer.advanceReader(advanceResult); + if (advanceResult.pending) await promise; + } + + return this.getPacketAtIndex(res, nextIndex, options); } getKeyPacket( + res: ResultValue, timestamp: number, options: PacketRetrievalOptions, - ): Promise { - return this.getPacket(timestamp, options); + ): MaybeRelevantPromise { + return this.getPacket(res, timestamp, options); } getNextKeyPacket( + res: ResultValue, packet: EncodedPacket, options: PacketRetrievalOptions, - ): Promise { - return this.getNextPacket(packet, options); + ): MaybeRelevantPromise { + return this.getNextPacket(res, packet, options); } async getPacketAtIndex( + res: ResultValue, sampleIndex: number, options: PacketRetrievalOptions, - ): Promise { + ): MaybeRelevantPromise { const rawSample = this.demuxer.loadedSamples[sampleIndex]; if (!rawSample) { - return null; + return res.set(new PacketRetrievalResult(null)); } let data: Uint8Array; @@ -779,7 +804,7 @@ class FlacAudioTrackBacking implements InputAudioTrackBacking { if (isThenable(slice)) slice = await slice; if (!slice) { - return null; // Data didn't fit into the rest of the file + return res.set(new PacketRetrievalResult(null)); // Data didn't fit into the rest of the file } data = readBytes(slice, rawSample.byteSize); @@ -788,27 +813,33 @@ class FlacAudioTrackBacking implements InputAudioTrackBacking { assert(this.demuxer.audioInfo); const timestamp = rawSample.blockOffset / this.demuxer.audioInfo.sampleRate; const duration = rawSample.blockSize / this.demuxer.audioInfo.sampleRate; - return new EncodedPacket( + + return res.set(new PacketRetrievalResult(new EncodedPacket( data, 'key', timestamp, duration, sampleIndex, rawSample.byteSize, - ); + ))); } async getFirstPacket( + res: ResultValue, options: PacketRetrievalOptions, - ): Promise { + ): MaybeRelevantPromise { + const advanceResult = new ResultValue(); + // Ensure the next sample exists while ( this.demuxer.loadedSamples.length === 0 && !this.demuxer.lastSampleLoaded ) { - await this.demuxer.advanceReader(); + advanceResult.reset(); + const promise = this.demuxer.advanceReader(advanceResult); + if (advanceResult.pending) await promise; } - return this.getPacketAtIndex(0, options); + return this.getPacketAtIndex(res, 0, options); } } diff --git a/src/flac/flac-muxer.ts b/src/flac/flac-muxer.ts index 411240c6b..5b6792b01 100644 --- a/src/flac/flac-muxer.ts +++ b/src/flac/flac-muxer.ts @@ -52,7 +52,8 @@ export class FlacMuxer extends Muxer { } async start() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.writer = await this.output._getRootWriter(!!this.format._options.appendOnly); this.writer.write(FLAC_HEADER); @@ -65,8 +66,6 @@ export class FlacMuxer extends Muxer { validateAudioChunkMetadata({ decoderConfig: track.metadata.decoderConfig }, track.source._codec); this.applyDecoderConfig(track.metadata.decoderConfig); } - - release(); } applyDecoderConfig(decoderConfig: AudioDecoderConfig) { @@ -259,57 +258,54 @@ export class FlacMuxer extends Muxer { packet: EncodedPacket, meta?: EncodedAudioChunkMetadata, ): Promise { - const release = await this.mutex.acquire(); - - try { - this.validateTimestamp( - track, - packet.timestamp, - packet.type === 'key', - ); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - if (this.sampleRate === null) { - // It's the first packet - validateAudioChunkMetadata(meta, track.source._codec); + this.validateTimestamp( + track, + packet.timestamp, + packet.type === 'key', + ); - assert(meta); - assert(meta.decoderConfig); + if (this.sampleRate === null) { + // It's the first packet + validateAudioChunkMetadata(meta, track.source._codec); - this.applyDecoderConfig(meta.decoderConfig); - } + assert(meta); + assert(meta.decoderConfig); - if (!this.metadataWritten) { - this.writeVorbisCommentAndPictureBlock(); - } + this.applyDecoderConfig(meta.decoderConfig); + } - const slice = FileSlice.tempFromBytes(packet.data); - slice.skip(2); - const bytes = readBytes(slice, 2); - const bitstream = new Bitstream(bytes); - const blockSizeOrUncommon = getBlockSizeOrUncommon(bitstream.readBits(4)); - if (blockSizeOrUncommon === null) { - throw new Error('Invalid FLAC frame: Invalid block size.'); - } + if (!this.metadataWritten) { + this.writeVorbisCommentAndPictureBlock(); + } - readCodedNumber(slice); // num - const blockSize = readBlockSize(slice, blockSizeOrUncommon); + const slice = FileSlice.tempFromBytes(packet.data); + slice.skip(2); + const bytes = readBytes(slice, 2); + const bitstream = new Bitstream(bytes); + const blockSizeOrUncommon = getBlockSizeOrUncommon(bitstream.readBits(4)); + if (blockSizeOrUncommon === null) { + throw new Error('Invalid FLAC frame: Invalid block size.'); + } - if (!this.format._options.appendOnly) { - this.blockSizes.push(blockSize); - this.frameSizes.push(packet.data.length); - } + readCodedNumber(slice); // num + const blockSize = readBlockSize(slice, blockSizeOrUncommon); - const startPos = this.writer.getPos(); - this.writer.write(packet.data); + if (!this.format._options.appendOnly) { + this.blockSizes.push(blockSize); + this.frameSizes.push(packet.data.length); + } - if (this.format._options.onFrame) { - this.format._options.onFrame(packet.data, startPos); - } + const startPos = this.writer.getPos(); + this.writer.write(packet.data); - await this.writer.flush(); - } finally { - release(); + if (this.format._options.onFrame) { + this.format._options.onFrame(packet.data, startPos); } + + await this.writer.flush(); } override addSubtitleCue(): Promise { @@ -317,7 +313,8 @@ export class FlacMuxer extends Muxer { } async finalize(): Promise { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; if (this.sampleRate === null) { throw new Error( @@ -377,7 +374,5 @@ export class FlacMuxer extends Muxer { totalSamples, }); } - - release(); } } diff --git a/src/hls/hls-demuxer.ts b/src/hls/hls-demuxer.ts index 4b203ed3d..6f56bc338 100644 --- a/src/hls/hls-demuxer.ts +++ b/src/hls/hls-demuxer.ts @@ -14,11 +14,18 @@ import { InputTrackBacking, InputVideoTrackBacking, } from '../input-track'; -import { PacketRetrievalOptions } from '../media-sink'; import { DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata'; import { TrackType } from '../output'; -import { assert, joinPaths, MaybePromise, TransformationMatrix, UNDETERMINED_LANGUAGE } from '../misc'; -import { EncodedPacket } from '../packet'; +import { + assert, + joinPaths, + MaybePromise, + MaybeRelevantPromise, + ResultValue, + TransformationMatrix, + UNDETERMINED_LANGUAGE, +} from '../misc'; +import { EncodedPacket, PacketRetrievalOptions, PacketRetrievalResult } from '../packet'; import { readAllLines } from '../reader'; import { AttributeList, @@ -754,29 +761,58 @@ abstract class HlsInputTrackBacking implements InputTrackBacking { return this.internalTrack.hasOnlyKeyPackets || null; } - async getFirstPacket(options: PacketRetrievalOptions): Promise { - await this.hydrate(); - return this.internalTrack.backingTrack!.getFirstPacket(options); + async getFirstPacket( + res: ResultValue, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + if (!this.internalTrack.backingTrack) { + await this.hydrate(); + } + return this.internalTrack.backingTrack!.getFirstPacket(res, options); } - async getPacket(timestamp: number, options: PacketRetrievalOptions): Promise { - await this.hydrate(); - return this.internalTrack.backingTrack!.getPacket(timestamp, options); + async getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + if (!this.internalTrack.backingTrack) { + await this.hydrate(); + } + return this.internalTrack.backingTrack!.getPacket(res, timestamp, options); } - async getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise { - await this.hydrate(); - return this.internalTrack.backingTrack!.getKeyPacket(timestamp, options); + async getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + if (!this.internalTrack.backingTrack) { + await this.hydrate(); + } + return this.internalTrack.backingTrack!.getKeyPacket(res, timestamp, options); } - async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { - await this.hydrate(); - return this.internalTrack.backingTrack!.getNextPacket(packet, options); + async getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + if (!this.internalTrack.backingTrack) { + await this.hydrate(); + } + return this.internalTrack.backingTrack!.getNextPacket(res, packet, options); } - async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { - await this.hydrate(); - return this.internalTrack.backingTrack!.getNextKeyPacket(packet, options); + async getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + if (!this.internalTrack.backingTrack) { + await this.hydrate(); + } + return this.internalTrack.backingTrack!.getNextKeyPacket(res, packet, options); } } diff --git a/src/hls/hls-muxer.ts b/src/hls/hls-muxer.ts index d06b9ff52..8c6952925 100644 --- a/src/hls/hls-muxer.ts +++ b/src/hls/hls-muxer.ts @@ -172,7 +172,8 @@ export class HlsMuxer extends Muxer { } async start(): Promise { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; const someRelative = this.output.tracks.some(t => t.metadata.isRelativeToUnixEpoch); const someNotRelative = this.output.tracks.some(t => !t.metadata.isRelativeToUnixEpoch); @@ -553,8 +554,6 @@ export class HlsMuxer extends Muxer { ); } } - - release(); } async getMimeType(): Promise { @@ -581,13 +580,10 @@ export class HlsMuxer extends Muxer { const playlist = this.playlists.find(x => x.tracks.includes(track)); assert(playlist); // If there isn't one then the assignment algo failed innit - const release = await playlist.mutex.acquire(); + using lock = playlist.mutex.lock(); + if (lock.pending) await lock.ready; - try { - await this.advancePlaylist(playlist); - } finally { - release(); - } + await this.advancePlaylist(playlist); } getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket | null, meta?: EncodedVideoChunkMetadata) { @@ -658,25 +654,22 @@ export class HlsMuxer extends Muxer { const trackData = this.getVideoTrackData(track, packet, meta); const playlist = trackData.playlist; - const release = await playlist.mutex.acquire(); - - try { - this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); - trackData.packets.push(packet); + using lock = playlist.mutex.lock(); + if (lock.pending) await lock.ready; - if (playlist.currentSegmentStartTimestamp === null) { - playlist.currentSegmentStartTimestamp = packet.timestamp; - } else if (!playlist.currentSegmentStartTimestampIsFixed) { - playlist.currentSegmentStartTimestamp = Math.min( - playlist.currentSegmentStartTimestamp, - packet.timestamp, - ); - } + this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); + trackData.packets.push(packet); - await this.advancePlaylist(playlist); - } finally { - release(); + if (playlist.currentSegmentStartTimestamp === null) { + playlist.currentSegmentStartTimestamp = packet.timestamp; + } else if (!playlist.currentSegmentStartTimestampIsFixed) { + playlist.currentSegmentStartTimestamp = Math.min( + playlist.currentSegmentStartTimestamp, + packet.timestamp, + ); } + + await this.advancePlaylist(playlist); } async addEncodedAudioPacket( @@ -687,25 +680,22 @@ export class HlsMuxer extends Muxer { const trackData = this.getAudioTrackData(track, packet, meta); const playlist = trackData.playlist; - const release = await playlist.mutex.acquire(); - - try { - this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); - trackData.packets.push(packet); + using lock = playlist.mutex.lock(); + if (lock.pending) await lock.ready; - if (playlist.currentSegmentStartTimestamp === null) { - playlist.currentSegmentStartTimestamp = packet.timestamp; - } else if (!playlist.currentSegmentStartTimestampIsFixed) { - playlist.currentSegmentStartTimestamp = Math.min( - playlist.currentSegmentStartTimestamp, - packet.timestamp, - ); - } + this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); + trackData.packets.push(packet); - await this.advancePlaylist(playlist); - } finally { - release(); + if (playlist.currentSegmentStartTimestamp === null) { + playlist.currentSegmentStartTimestamp = packet.timestamp; + } else if (!playlist.currentSegmentStartTimestampIsFixed) { + playlist.currentSegmentStartTimestamp = Math.min( + playlist.currentSegmentStartTimestamp, + packet.timestamp, + ); } + + await this.advancePlaylist(playlist); } async addSubtitleCue( @@ -1623,35 +1613,32 @@ export class HlsMuxer extends Muxer { this.format._options.onMaster?.(masterPlaylistText); - const release = await this.mutex.acquire(); - - try { - let writer: Writer; - if (this.numWrittenMasterPlaylists === 0) { - // For the first master playlist write, we use the normal root writer getter, so that the target - // returned by Output.target emits valid write events. - writer = await this.output._getRootWriter(true); - } else { - // For subsequent master playlist writes, we *must* obtain a different target in order to overwrite - // the file. - const target = await this.output._getTarget({ - path: pathedTarget.rootPath, - isRoot: true, - mimeType: HLS_MIME_TYPE, - }); - writer = new Writer(target, true); - writer.start(); - } + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; + + let writer: Writer; + if (this.numWrittenMasterPlaylists === 0) { + // For the first master playlist write, we use the normal root writer getter, so that the target + // returned by Output.target emits valid write events. + writer = await this.output._getRootWriter(true); + } else { + // For subsequent master playlist writes, we *must* obtain a different target in order to overwrite + // the file. + const target = await this.output._getTarget({ + path: pathedTarget.rootPath, + isRoot: true, + mimeType: HLS_MIME_TYPE, + }); + writer = new Writer(target, true); + writer.start(); + } - writer.write(textEncoder.encode(masterPlaylistText)); + writer.write(textEncoder.encode(masterPlaylistText)); - await writer.flush(); - await writer.finalize(); + await writer.flush(); + await writer.finalize(); - this.numWrittenMasterPlaylists++; - } finally { - release(); - } + this.numWrittenMasterPlaylists++; } private async tryWriteMasterPlaylist() { @@ -1668,8 +1655,11 @@ export class HlsMuxer extends Muxer { } async finalize() { - const releases = await Promise.all(this.playlists.map(p => p.mutex.acquire())); - releases.forEach(release => release()); + // Wait for all playlist mutexes + await Promise.all(this.playlists.map(async (playlist) => { + using lock = playlist.mutex.lock(); + if (lock.pending) await lock.ready; + })); for (const trackData of this.trackDatas) { trackData.closed = true; diff --git a/src/hls/hls-segmented-input.ts b/src/hls/hls-segmented-input.ts index 30b8da014..a13c80d62 100644 --- a/src/hls/hls-segmented-input.ts +++ b/src/hls/hls-segmented-input.ts @@ -8,7 +8,13 @@ import { AES_128_BLOCK_SIZE, createAes128CbcDecryptStream } from '../aes'; import { ENCRYPTION_KEY_CACHE_GROUP, Input } from '../input'; -import { Segment, SegmentedInput, SegmentedInputTrackDeclaration, SegmentRetrievalOptions } from '../segmented-input'; +import { + Segment, + SegmentedInput, + SegmentedInputTrackDeclaration, + SegmentRetrievalOptions, + SegmentRetrievalResult, +} from '../segmented-input'; import { toDataView, joinPaths, @@ -18,6 +24,8 @@ import { arrayArgmin, wait, base64ToBytes, + MaybeRelevantPromise, + ResultValue, } from '../misc'; import { readAllLines, readBytes, Reader } from '../reader'; import { CustomPathedSource, PathedSource, ReadableStreamSource, SourceRef, SourceRequest } from '../source'; @@ -540,15 +548,19 @@ export class HlsSegmentedInput extends SegmentedInput { } } - async getFirstSegment() { + async getFirstSegment(res: ResultValue): MaybeRelevantPromise { if (this.segments.length === 0) { await this.runUpdateSegments(); } - return this.segments[0] ?? null; + return res.set(new SegmentRetrievalResult(this.segments[0] ?? null)); } - async getSegmentAt(timestamp: number, options: SegmentRetrievalOptions) { + async getSegmentAt( + res: ResultValue, + timestamp: number, + options: SegmentRetrievalOptions, + ): MaybeRelevantPromise { if (this.segments.length === 0) { await this.runUpdateSegments(); } @@ -559,16 +571,24 @@ export class HlsSegmentedInput extends SegmentedInput { while (true) { const index = binarySearchLessOrEqual(this.segments, timestamp, x => x.timestamp); if (index === -1) { - return null; + return res.set(new SegmentRetrievalResult(null)); } - if (index < this.segments.length - 1 || this.streamHasEnded || isLazy) { - return this.segments[index]!; + if (index < this.segments.length - 1 || this.streamHasEnded) { + return res.set(new SegmentRetrievalResult(this.segments[index]!)); } const segment = this.segments[index]!; if (timestamp < segment.timestamp + segment.duration) { - return segment; + return res.set(new SegmentRetrievalResult(segment)); + } + + if (isLazy) { + // The timestamp lies past the live edge, so a future segment may still end up containing it + const result = new SegmentRetrievalResult(segment); + result.provisional = true; + + return res.set(result); } await this.runUpdateSegments(); @@ -579,7 +599,11 @@ export class HlsSegmentedInput extends SegmentedInput { } } - async getNextSegment(segment: Segment, options: SegmentRetrievalOptions) { + async getNextSegment( + res: ResultValue, + segment: Segment, + options: SegmentRetrievalOptions, + ): MaybeRelevantPromise { const index = this.segments.indexOf(segment as HlsSegment); assert(index !== -1); @@ -590,11 +614,19 @@ export class HlsSegmentedInput extends SegmentedInput { while (true) { if (nextIndex < this.segments.length) { - return this.segments[nextIndex]!; + return res.set(new SegmentRetrievalResult(this.segments[nextIndex]!)); } - if (this.streamHasEnded || isLazy) { - return null; + if (this.streamHasEnded) { + return res.set(new SegmentRetrievalResult(null)); + } + + if (isLazy) { + // There's no next segment yet, but one may still come + const result = new SegmentRetrievalResult(null); + result.provisional = true; + + return res.set(result); } await this.runUpdateSegments(); @@ -605,11 +637,11 @@ export class HlsSegmentedInput extends SegmentedInput { } } - async getPreviousSegment(segment: Segment) { + async getPreviousSegment(res: ResultValue, segment: Segment): MaybeRelevantPromise { const index = this.segments.indexOf(segment as HlsSegment); assert(index !== -1); - return this.segments[index - 1] ?? null; + return res.set(new SegmentRetrievalResult(this.segments[index - 1] ?? null)); } getInputForSegment(segment: Segment): Input { diff --git a/src/index.ts b/src/index.ts index 798c19291..d187ce770 100644 --- a/src/index.ts +++ b/src/index.ts @@ -97,6 +97,7 @@ export { SUBTITLE_CODECS, } from './codec'; export { + VideoDecoderWrapper, canDecode, canDecodeVideo, canDecodeAudio, @@ -264,6 +265,10 @@ export { EncodedPacket, type EncodedPacketSideData, type PacketType, + PacketCache, + type PacketCacheOptions, + PacketReader, + type PacketReaderOptions, } from './packet'; export { AudioSample, @@ -283,18 +288,16 @@ export { registerVideoSampleTransformer, } from './sample'; export { - AudioBufferSink, - AudioSampleSink, - BaseMediaSampleSink, - CanvasSink, - type CanvasSinkOptions, - EncodedPacketSink, - type PacketRetrievalOptions, - VideoSampleSink, - type VideoSinkDecoderOptions, - type WrappedAudioBuffer, - type WrappedCanvas, -} from './media-sink'; + PacketCursor, + type PacketCursorConfig, + SampleCursor, + VideoSampleCursor, + AudioSampleCursor, + type VideoSampleCursorOptions, + type AudioSampleCursorOptions, + canvasTransformer, + type CanvasTransformerOptions, +} from './cursors'; export { Conversion, type ConversionOptions, diff --git a/src/input-format.ts b/src/input-format.ts index 7a420a358..6942fc355 100644 --- a/src/input-format.ts +++ b/src/input-format.ts @@ -22,7 +22,7 @@ import { } from './matroska/ebml'; import { MatroskaDemuxer } from './matroska/matroska-demuxer'; import { Mp3Demuxer } from './mp3/mp3-demuxer'; -import { MP3_FRAME_HEADER_SIZE, getXingOffset, INFO, XING } from '../shared/mp3-misc'; +import { MP3_FRAME_HEADER_SIZE, getXingOffset, INFO, XING, Mp3FrameHeader } from '../shared/mp3-misc'; import { ID3_V2_HEADER_SIZE, readId3V2Header } from './id3'; import { readNextMp3FrameHeader } from './mp3/mp3-reader'; import { OggDemuxer } from './ogg/ogg-demuxer'; @@ -36,7 +36,7 @@ import { TS_PACKET_SIZE } from './mpeg-ts/mpeg-ts-misc'; import { HlsDemuxer } from './hls/hls-demuxer'; import { HLS_MIME_TYPE } from './hls/hls-misc'; import { PathedSource } from './source'; -import { isThenable, MaybePromise } from './misc'; +import { isThenable, MaybePromise, ResultValue } from './misc'; /** * Base class representing an input media file format. @@ -335,7 +335,15 @@ export class Mp3InputFormat extends InputFormat { currentPos = slice.filePos + id3V2Header.size; } - const firstResult = await readNextMp3FrameHeader(input._reader, currentPos, currentPos + 4096); + const result = new ResultValue<{ + header: Mp3FrameHeader; + startPos: number; + } | null>(); + + let promise = readNextMp3FrameHeader(result, input._reader, currentPos, currentPos + 4096); + if (result.pending) await promise; + + const firstResult = result.value; if (!firstResult) { return false; } @@ -359,11 +367,11 @@ export class Mp3InputFormat extends InputFormat { // Fine, we found one frame header, but we're still not entirely sure this is MP3. Let's check if we can find // another header right after it: - const secondResult = await readNextMp3FrameHeader( - input._reader, - currentPos, - currentPos + MP3_FRAME_HEADER_SIZE, - ); + result.reset(); + promise = readNextMp3FrameHeader(result, input._reader, currentPos, currentPos + MP3_FRAME_HEADER_SIZE); + if (result.pending) await promise; + + const secondResult = result.value; if (!secondResult) { return false; } diff --git a/src/input-track.ts b/src/input-track.ts index 8020ad811..c6d0b3e13 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -11,7 +11,6 @@ import { determineVideoPacketType } from './codec-data'; import { customAudioDecoders, customVideoDecoders } from './custom-coder'; import { Input } from './input'; import { Logging } from './logging'; -import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink'; import { assert, extractRotationFromMatrix, @@ -19,14 +18,17 @@ import { isThenable, matrixIsFlipped, MaybePromise, + MaybeRelevantPromise, Rational, + ResultValue, roundToDivisor, simplifyRational, TransformationMatrix, } from './misc'; import { TrackType } from './output'; -import { EncodedPacket, PacketType } from './packet'; +import { EncodedPacket, PacketRetrievalOptions, PacketRetrievalResult, PacketType } from './packet'; import { TrackDisposition } from './metadata'; +import { PacketCursor } from './cursors'; import { DurationMetadataRequestOptions } from './demuxer'; /** @@ -134,13 +136,34 @@ export interface InputTrackBacking { getDecoderConfig(): Promise; getMetadataCodecParameterString?(): MaybePromise; - getFirstPacket(options: PacketRetrievalOptions): Promise; - getPacket(timestamp: number, options: PacketRetrievalOptions): Promise; - getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise; - getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise; - getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise; + getFirstPacket( + res: ResultValue, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise; + getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise; + getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise; + getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise; + getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise; } +let nextInputTrackId = 0; + /** * Represents a media track in an input file. * @group Input files & tracks @@ -151,6 +174,9 @@ export abstract class InputTrack { readonly input: Input; /** @internal */ _backing: InputTrackBacking; + /** @internal */ + _uniqueId = nextInputTrackId++; + /** @internal */ constructor(input: Input, backing: InputTrackBacking) { this.input = input; @@ -342,7 +368,9 @@ export abstract class InputTrack { * with a negative timestamp should not be presented. */ async getFirstTimestamp() { - const firstPacket = await this._backing.getFirstPacket({ metadataOnly: true }); + const resultValue = new ResultValue(); + await this._backing.getFirstPacket(resultValue, { metadataOnly: true }); + const firstPacket = resultValue.value.packet; return firstPacket?.timestamp ?? 0; } @@ -354,7 +382,9 @@ export abstract class InputTrack { * in the options. */ async computeDuration(options?: PacketRetrievalOptions) { - const lastPacket = await this._backing.getPacket(Infinity, { metadataOnly: true, ...options }); + const resultValue = new ResultValue(); + await this._backing.getPacket(resultValue, Infinity, { metadataOnly: true, ...options }); + const lastPacket = resultValue.value.packet; const result = (lastPacket?.timestamp ?? 0) + (lastPacket?.duration ?? 0); return roundToDivisor(result, await this.getTimeResolution()); @@ -387,20 +417,20 @@ export abstract class InputTrack { * {@link PacketRetrievalOptions.skipLiveWait} to `true` in the options. */ async computePacketStats(targetPacketCount = Infinity, options?: PacketRetrievalOptions): Promise { - const sink = new EncodedPacketSink(this); + const cursor = new PacketCursor(this, { options: { ...options, metadataOnly: true } }); let startTimestamp = Infinity; let endTimestamp = -Infinity; let packetCount = 0; let totalPacketBytes = 0; - for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true, ...options })) { + await cursor.iterate((packet) => { if ( packetCount >= targetPacketCount // This additional condition is needed to produce correct results with out-of-presentation-order packets && packet.timestamp >= endTimestamp ) { - break; + return false; } startTimestamp = Math.min(startTimestamp, packet.timestamp); @@ -408,7 +438,7 @@ export abstract class InputTrack { packetCount++; totalPacketBytes += packet.byteLength; - } + }); return { packetCount, @@ -914,25 +944,25 @@ export class InputVideoTrack extends InputTrack { const timeResolution = await this.getTimeResolution(); const targetPacketCount = options.targetPacketCount ?? 256; - const sink = new EncodedPacketSink(this); + const cursor = new PacketCursor(this, { options: { metadataOnly: true } }); const timestamps: number[] = []; let maxTimestamp = -Infinity; let probedPacketCount = 0; - for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true })) { + await cursor.iterate((packet) => { if ( timestamps.length >= targetPacketCount // Needed for out-of-presentation-order packets. && packet.timestamp >= maxTimestamp ) { - break; + return false; } timestamps.push(packet.timestamp); maxTimestamp = Math.max(maxTimestamp, packet.timestamp); probedPacketCount++; - } + }); const ticks = new Float64Array(timestamps.length); diff --git a/src/input.ts b/src/input.ts index a58fd76e6..05937658f 100644 --- a/src/input.ts +++ b/src/input.ts @@ -8,6 +8,7 @@ import { Demuxer, DurationMetadataRequestOptions } from './demuxer'; import { InputFormat, InputFormatOptions, validateInputFormatOptions } from './input-format'; +import { SampleCursor } from './cursors'; import { InputAudioTrack, InputAudioTrackBacking, @@ -22,7 +23,6 @@ import { prefer, desc, } from './input-track'; -import { PacketRetrievalOptions } from './media-sink'; import { arrayArgmin, arrayCount, @@ -30,6 +30,7 @@ import { EventEmitter, polyfillSymbolDispose, removeItem, + ResultValue, } from './misc'; import { Reader } from './reader'; import { @@ -39,6 +40,7 @@ import { SourceRequest, sourceRequestsAreEqual, } from './source'; +import { PacketRetrievalOptions, PacketRetrievalResult } from './packet'; polyfillSymbolDispose(); @@ -120,7 +122,6 @@ export class Input extends EventEmitter _backingToTrack = new Map(); /** @internal */ _disposed = false; - /** @internal */ _nextSourceCacheAge = 0; /** @internal */ _sourceRefs: SourceRef[] = []; @@ -133,6 +134,10 @@ export class Input extends EventEmitter promise: Promise; }[] = []; + /** @internal */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + _openSampleCursors = new Set>(); + /** @internal */ _formatOptions: InputFormatOptions; /** @internal */ @@ -337,7 +342,13 @@ export class Input extends EventEmitter } // Only count the timestamps of tracks that have at least one packet - const firstPackets = await Promise.all(filtered.map(x => x._backing.getFirstPacket({ metadataOnly: true }))); + const firstPackets = await Promise.all(filtered.map(async (x) => { + const result = new ResultValue(); + const promise = x._backing.getFirstPacket(result, { metadataOnly: true }); + if (result.pending) await promise; + + return result.value.packet; + })); const result = Math.min(...firstPackets.map(x => x?.timestamp ?? Infinity)); return result === Infinity ? 0 : result; @@ -536,6 +547,10 @@ export class Input extends EventEmitter .then(demuxer => demuxer.dispose()) .catch(() => {}); } + + for (const cursor of [...this._openSampleCursors]) { + void cursor.close(); + } } /** diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 17d648dc2..e3a5a6926 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -54,7 +54,6 @@ import { InputTrackBacking, InputVideoTrackBacking, } from '../input-track'; -import { PacketRetrievalOptions } from '../media-sink'; import { assert, binarySearchExact, @@ -74,12 +73,14 @@ import { UNDETERMINED_LANGUAGE, toDataView, roundIfAlmostInteger, + ResultValue, + MaybeRelevantPromise, hexStringToBytes, HEX_STRING_REGEX, EMPTY_COLOR_SPACE, colorSpaceIsComplete, } from '../misc'; -import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; +import { EncodedPacket, PacketRetrievalOptions, PacketRetrievalResult, PLACEHOLDER_DATA } from '../packet'; import { buildIsobmffMimeType, parsePsshBoxContents, psshBoxesAreEqual, PsshBox } from './isobmff-misc'; import { MAX_BOX_HEADER_SIZE, @@ -687,9 +688,9 @@ export class IsobmffDemuxer extends Demuxer { return sampleTable; } - async readFragment(startPos: number): Promise { + async readFragment(res: ResultValue, startPos: number): MaybeRelevantPromise { if (this.lastReadFragment?.moofOffset === startPos) { - return this.lastReadFragment; + return res.set(this.lastReadFragment); } let headerSlice = this.reader.requestSliceRange(startPos, MIN_BOX_HEADER_SIZE, MAX_BOX_HEADER_SIZE); @@ -775,7 +776,7 @@ export class IsobmffDemuxer extends Demuxer { } } - return fragment; + return res.set(fragment); } readContiguousBoxes(slice: FileSlice) { @@ -2857,12 +2858,6 @@ export class IsobmffDemuxer extends Demuxer { } abstract class IsobmffTrackBacking implements InputTrackBacking { - packetToSampleIndex = new WeakMap(); - packetToFragmentLocation = new WeakMap(); - constructor(public internalTrack: InternalTrack) {} abstract getType(): TrackType; @@ -2944,7 +2939,9 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { assert(track.trackBacking); - const firstPacket = await track.trackBacking.getFirstPacket({ metadataOnly: true }); + const result = new ResultValue(); + await track.trackBacking.getFirstPacket(result, { metadataOnly: true }); + const firstPacket = result.value.packet; return (firstPacket?.timestamp ?? 0) + track.durationInMediaTimescale / track.timescale; } @@ -2952,14 +2949,21 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { return null; } - async getFirstPacket(options: PacketRetrievalOptions) { - const regularPacket = await this.fetchPacketForSampleIndex(0, options); - if (regularPacket || !this.internalTrack.demuxer.isFragmented) { + async getFirstPacket( + res: ResultValue, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + const result = new ResultValue(); + const promise = this.fetchPacketForSampleIndex(result, 0, options); + if (result.pending) await promise; + + if (result.value.packet || !this.internalTrack.demuxer.isFragmented) { // If there's a non-fragmented packet, always prefer that - return regularPacket; + return res.set(result.value); } return this.performFragmentedLookup( + res, null, (fragment) => { const trackData = fragment.trackData.get(this.internalTrack.id); @@ -2988,19 +2992,22 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { return roundIfAlmostInteger(timestamp * this.internalTrack.timescale) + this.internalTrack.editListOffset; } - async getPacket(timestamp: number, options: PacketRetrievalOptions) { + async getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp); const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack); - const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale); - const regularPacket = await this.fetchPacketForSampleIndex(sampleIndex, options); - if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) { // Prefer the non-fragmented packet - return regularPacket; + const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale); + return this.fetchPacketForSampleIndex(res, sampleIndex, options); } return this.performFragmentedLookup( + res, null, (fragment) => { const trackData = fragment.trackData.get(this.internalTrack.id); @@ -3025,28 +3032,34 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { ); } - async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - const regularSampleIndex = this.packetToSampleIndex.get(packet); + async getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + const isFromFragment = packet._internal !== undefined; + if (!isFromFragment) { + const sampleIndex = packet.sequenceNumber; // sequenceNumber = sampleIndex in this case + assert(sampleIndex >= 0); - if (regularSampleIndex !== undefined) { - // Prefer the non-fragmented packet - return this.fetchPacketForSampleIndex(regularSampleIndex + 1, options); + return this.fetchPacketForSampleIndex(res, sampleIndex + 1, options); } - const locationInFragment = this.packetToFragmentLocation.get(packet); - if (locationInFragment === undefined) { - throw new Error('Packet was not created from this track.'); - } + const moofOffset = packet._internal; + assert(typeof moofOffset === 'number'); + + const sampleIndex = packet.sequenceNumber - moofOffset; // since sequenceNumber = moofOffset + sampleIndex return this.performFragmentedLookup( - locationInFragment.fragment, + res, + moofOffset, (fragment) => { - if (fragment === locationInFragment.fragment) { + if (fragment.moofOffset === moofOffset) { const trackData = fragment.trackData.get(this.internalTrack.id)!; - if (locationInFragment.sampleIndex + 1 < trackData.samples.length) { + if (sampleIndex + 1 < trackData.samples.length) { // We can simply take the next sample in the fragment return { - sampleIndex: locationInFragment.sampleIndex + 1, + sampleIndex: sampleIndex + 1, correctSampleFound: true, }; } @@ -3071,19 +3084,22 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { ); } - async getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { + async getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp); const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack); - const sampleIndex = getKeyframeSampleIndexForTimestamp(sampleTable, timestampInTimescale); - const regularPacket = await this.fetchPacketForSampleIndex(sampleIndex, options); - if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) { // Prefer the non-fragmented packet - return regularPacket; + const sampleIndex = getKeyframeSampleIndexForTimestamp(sampleTable, timestampInTimescale); + return this.fetchPacketForSampleIndex(res, sampleIndex, options); } return this.performFragmentedLookup( + res, null, (fragment) => { const trackData = fragment.trackData.get(this.internalTrack.id); @@ -3107,27 +3123,34 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { ); } - async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - const regularSampleIndex = this.packetToSampleIndex.get(packet); - if (regularSampleIndex !== undefined) { - // Prefer the non-fragmented packet + async getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + const isFromFragment = packet._internal !== undefined; + if (!isFromFragment) { + const sampleIndex = packet.sequenceNumber; // sequenceNumber = sampleIndex in this case + assert(sampleIndex >= 0); + const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack); - const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, regularSampleIndex); - return this.fetchPacketForSampleIndex(nextKeyFrameSampleIndex, options); + const nextKeyFrameSampleIndex = getNextKeyframeIndexForSample(sampleTable, sampleIndex); + return this.fetchPacketForSampleIndex(res, nextKeyFrameSampleIndex, options); } - const locationInFragment = this.packetToFragmentLocation.get(packet); - if (locationInFragment === undefined) { - throw new Error('Packet was not created from this track.'); - } + const moofOffset = packet._internal; + assert(typeof moofOffset === 'number'); + + const sampleIndex = packet.sequenceNumber - moofOffset; // since sequenceNumber = moofOffset + sampleIndex return this.performFragmentedLookup( - locationInFragment.fragment, + res, + moofOffset, (fragment) => { - if (fragment === locationInFragment.fragment) { + if (fragment.moofOffset === moofOffset) { const trackData = fragment.trackData.get(this.internalTrack.id)!; const nextKeyFrameIndex = trackData.samples.findIndex( - (x, i) => x.isKeyFrame && i > locationInFragment.sampleIndex, + (x, i) => x.isKeyFrame && i > sampleIndex, ); if (nextKeyFrameIndex !== -1) { @@ -3161,15 +3184,19 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { ); } - private async fetchPacketForSampleIndex(sampleIndex: number, options: PacketRetrievalOptions) { + private async fetchPacketForSampleIndex( + res: ResultValue, + sampleIndex: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { if (sampleIndex === -1) { - return null; + return res.set(new PacketRetrievalResult(null)); } const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack); const sampleInfo = getSampleInfo(sampleTable, sampleIndex); if (!sampleInfo) { - return null; + return res.set(new PacketRetrievalResult(null)); } let data: Uint8Array; @@ -3182,7 +3209,7 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { ); if (isThenable(slice)) slice = await slice; if (!slice) { - return null; // Data is outside + return res.set(new PacketRetrievalResult(null)); // Data is outside } data = readBytes(slice, sampleInfo.sampleSize); @@ -3221,14 +3248,17 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { sampleInfo.sampleSize, ); - this.packetToSampleIndex.set(packet, sampleIndex); - - return packet; + return res.set(new PacketRetrievalResult(packet)); } - private async fetchPacketInFragment(fragment: Fragment, sampleIndex: number, options: PacketRetrievalOptions) { + private async fetchPacketInFragment( + res: ResultValue, + fragment: Fragment, + sampleIndex: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { if (sampleIndex === -1) { - return null; + return res.set(new PacketRetrievalResult(null)); } const trackData = fragment.trackData.get(this.internalTrack.id)!; @@ -3245,7 +3275,7 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { ); if (isThenable(slice)) slice = await slice; if (!slice) { - return null; // Data is outside + return res.set(new PacketRetrievalResult(null)); // Data is outside } data = readBytes(slice, fragmentSample.byteSize); @@ -3271,15 +3301,16 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { fragmentSample.byteSize, ); - this.packetToFragmentLocation.set(packet, { fragment, sampleIndex }); + packet._internal = fragment.moofOffset; - return packet; + return res.set(new PacketRetrievalResult(packet)); } /** Looks for a packet in the fragments while trying to load as few fragments as possible to retrieve it. */ private async performFragmentedLookup( - // The fragment where we start looking - startFragment: Fragment | null, + res: ResultValue, + // The position where we'll start the lookup + startOffset: number | null, // This function returns the best-matching sample in a given fragment getMatchInFragment: (fragment: Fragment) => { sampleIndex: number; correctSampleFound: boolean }, // The timestamp with which we can search the lookup table @@ -3287,20 +3318,25 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { // The timestamp for which we know the correct sample will not come after it latestTimestamp: number, options: PacketRetrievalOptions, - ): Promise { + ): MaybeRelevantPromise { const demuxer = this.internalTrack.demuxer; + let currentPos = startOffset ?? 0; let currentFragment: Fragment | null = null; let bestFragment: Fragment | null = null; let bestSampleIndex = -1; - if (startFragment) { + if (startOffset !== null && demuxer.lastReadFragment?.moofOffset === startOffset) { + const startFragment = demuxer.lastReadFragment; const { sampleIndex, correctSampleFound } = getMatchInFragment(startFragment); if (correctSampleFound) { - return this.fetchPacketInFragment(startFragment, sampleIndex, options); + return this.fetchPacketInFragment(res, startFragment, sampleIndex, options); } + currentPos = startFragment.moofOffset + startFragment.moofSize; // Start reading from the next fragment + currentFragment = startFragment; + if (sampleIndex !== -1) { bestFragment = startFragment; bestSampleIndex = sampleIndex; @@ -3332,18 +3368,10 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { positionCacheEntry?.moofOffset ?? 0, ) || null; - let currentPos: number; - - if (!startFragment) { - currentPos = lookupEntryPosition ?? 0; - } else { - if (lookupEntryPosition === null || startFragment.moofOffset >= lookupEntryPosition) { - currentPos = startFragment.moofOffset + startFragment.moofSize; - currentFragment = startFragment; - } else { - // Use the lookup entry - currentPos = lookupEntryPosition; - } + if (lookupEntryPosition !== null && lookupEntryPosition > currentPos) { + // Use the lookup entry + currentPos = lookupEntryPosition; + currentFragment = null; } while (true) { @@ -3367,10 +3395,15 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { } if (boxInfo.name === 'moof') { - currentFragment = await demuxer.readFragment(boxStartPos); + const result = new ResultValue(); + const promise = demuxer.readFragment(result, boxStartPos); + if (result.pending) await promise; + + currentFragment = result.value; + const { sampleIndex, correctSampleFound } = getMatchInFragment(currentFragment); if (correctSampleFound) { - return this.fetchPacketInFragment(currentFragment, sampleIndex, options); + return this.fetchPacketInFragment(res, currentFragment, sampleIndex, options); } if (sampleIndex !== -1) { bestFragment = currentFragment; @@ -3390,6 +3423,7 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { const newSearchTimestamp = previousLookupEntry?.timestamp ?? -Infinity; return this.performFragmentedLookup( + res, null, getMatchInFragment, newSearchTimestamp, @@ -3400,10 +3434,10 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { if (bestFragment) { // If we finished looping but didn't find a perfect match, still return the best match we found - return this.fetchPacketInFragment(bestFragment, bestSampleIndex, options); + return this.fetchPacketInFragment(res, bestFragment, bestSampleIndex, options); } - return null; + return res.set(new PacketRetrievalResult(null)); } } @@ -3470,12 +3504,20 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo } return this.decoderConfigPromise ??= (async (): Promise => { + const getFirstPacket = async () => { + const result = new ResultValue(); + const promise = this.getFirstPacket(result, {}); + if (result.pending) await promise; + + return result.value.packet; + }; + if (this.internalTrack.info.codec === 'avc' && !this.internalTrack.info.codecDescription) { - const firstPacket = await this.getFirstPacket({}); + const firstPacket = await getFirstPacket(); this.internalTrack.info.avcCodecInfo = firstPacket && extractAvcDecoderConfigurationRecord(firstPacket.data); } else if (this.internalTrack.info.codec === 'hevc' && !this.internalTrack.info.codecDescription) { - const firstPacket = await this.getFirstPacket({}); + const firstPacket = await getFirstPacket(); this.internalTrack.info.hevcCodecInfo = firstPacket && extractHevcDecoderConfigurationRecord(firstPacket.data); } else if ( @@ -3486,7 +3528,7 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo || !vp9CodecInfoHasColorInfo(this.internalTrack.info.vp9CodecInfo) ) ) { - const firstPacket = await this.getFirstPacket({}); + const firstPacket = await getFirstPacket(); const packetInfo = firstPacket && extractVp9CodecInfoFromPacket(firstPacket.data); if (packetInfo) { @@ -3508,14 +3550,14 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo ) ) ) { - const firstPacket = await this.getFirstPacket({}); + const firstPacket = await getFirstPacket(); const packetInfo = firstPacket && extractAv1CodecInfoFromPacket(firstPacket.data); if (packetInfo) { this.internalTrack.info.av1CodecInfo = packetInfo; } } else if (this.internalTrack.info.codec === 'prores' && !this.internalTrack.info.proresCodecInfo) { - const firstPacket = await this.getFirstPacket({}); + const firstPacket = await getFirstPacket(); this.internalTrack.info.proresCodecInfo = firstPacket && extractProresCodecInfoFromPacket(firstPacket.data); } @@ -3583,7 +3625,11 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking implements InputAudio return this.decoderConfigPromise ??= (async (): Promise => { if (this.internalTrack.info.codec === 'dts' && !this.internalTrack.info.dtsFormat) { // Gotta check the packet to determine the DTS variant - const firstPacket = await this.getFirstPacket({}); + const result = new ResultValue(); + const promise = this.getFirstPacket(result, {}); + if (result.pending) await promise; + + const firstPacket = result.value.packet; this.internalTrack.info.dtsFormat = firstPacket && extractDtsFourCcFromPacket(firstPacket.data); } @@ -3668,8 +3714,6 @@ type SampleInfo = { duration: number; sampleOffset: number; sampleSize: number; - chunkOffset: number; - chunkSize: number; isKeyFrame: boolean; }; @@ -3704,20 +3748,13 @@ const getSampleInfo = (sampleTable: SampleTable, sampleIndex: number): SampleInf const startSampleIndexOfChunk = chunkEntry.startSampleIndex + (chunkIndex - chunkEntry.startChunkIndex) * chunkEntry.samplesPerChunk; - let chunkSize = 0; let sampleOffset = chunkOffset; if (sampleTable.sampleSizes.length === 1) { sampleOffset += sampleSize * (sampleIndex - startSampleIndexOfChunk); - chunkSize += sampleSize * chunkEntry.samplesPerChunk; } else { - for (let i = startSampleIndexOfChunk; i < startSampleIndexOfChunk + chunkEntry.samplesPerChunk; i++) { - const sampleSize = sampleTable.sampleSizes[i]!; - - if (i < sampleIndex) { - sampleOffset += sampleSize; - } - chunkSize += sampleSize; + for (let i = startSampleIndexOfChunk; i < sampleIndex; i++) { + sampleOffset += sampleTable.sampleSizes[i]!; } } @@ -3740,8 +3777,6 @@ const getSampleInfo = (sampleTable: SampleTable, sampleIndex: number): SampleInf duration, sampleOffset, sampleSize, - chunkOffset, - chunkSize, isKeyFrame: sampleTable.keySampleIndices ? binarySearchExact(sampleTable.keySampleIndices, sampleIndex, x => x) !== -1 : true, diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 6af8668c8..ae40842be 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -248,7 +248,8 @@ export class IsobmffMuxer extends Muxer { } async start() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; if (!this.isCmaf) { this.writer = await this.output._getRootWriter(target => ( @@ -360,8 +361,6 @@ export class IsobmffMuxer extends Muxer { ); } } - - release(); } private allTracksAreKnown() { @@ -666,116 +665,110 @@ export class IsobmffMuxer extends Muxer { } async addEncodedVideoPacket(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - const trackData = this.getVideoTrackData(track, packet, meta); + const trackData = this.getVideoTrackData(track, packet, meta); - let packetData = packet.data; - if (trackData.info.requiresAnnexBTransformation) { - const nalUnits = [...iterateNalUnitsInAnnexB(packetData)] - .map(loc => packetData.subarray(loc.offset, loc.offset + loc.length)); - if (nalUnits.length === 0) { - // It's not valid Annex B data - throw new Error( - 'Failed to transform packet data. Make sure all packets are provided in Annex B format, as' - + ' specified in ITU-T-REC-H.264 and ITU-T-REC-H.265.', - ); - } - - // We don't strip things like SPS or PPS NALUs here, mainly because they can also appear in the middle - // of a stream and potentially modify the parameters of it. So, let's just leave them in to be sure. - packetData = concatNalUnitsInLengthPrefixed(nalUnits, 4); + let packetData = packet.data; + if (trackData.info.requiresAnnexBTransformation) { + const nalUnits = [...iterateNalUnitsInAnnexB(packetData)] + .map(loc => packetData.subarray(loc.offset, loc.offset + loc.length)); + if (nalUnits.length === 0) { + // It's not valid Annex B data + throw new Error( + 'Failed to transform packet data. Make sure all packets are provided in Annex B format, as' + + ' specified in ITU-T-REC-H.264 and ITU-T-REC-H.265.', + ); } - this.validateTimestamp( - trackData.track, - packet.timestamp, - packet.type === 'key', - ); - const internalSample = this.createSampleForTrack( - trackData, - packetData, - packet.timestamp, - packet.duration, - packet.type, - ); - - await this.registerSample(trackData, internalSample); - } finally { - release(); + // We don't strip things like SPS or PPS NALUs here, mainly because they can also appear in the middle + // of a stream and potentially modify the parameters of it. So, let's just leave them in to be sure. + packetData = concatNalUnitsInLengthPrefixed(nalUnits, 4); } + + this.validateTimestamp( + trackData.track, + packet.timestamp, + packet.type === 'key', + ); + const internalSample = this.createSampleForTrack( + trackData, + packetData, + packet.timestamp, + packet.duration, + packet.type, + ); + + await this.registerSample(trackData, internalSample); } async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - const trackData = this.getAudioTrackData(track, packet, meta); + const trackData = this.getAudioTrackData(track, packet, meta); - let packetData = packet.data; - if (trackData.info.requiresAdtsStripping) { - const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packetData)); - if (!adtsFrame) { - throw new Error('Expected ADTS frame, didn\'t get one.'); - } - - const headerLength = adtsFrame.crcCheck === null - ? MIN_ADTS_FRAME_HEADER_SIZE - : MAX_ADTS_FRAME_HEADER_SIZE; - packetData = packetData.subarray(headerLength); + let packetData = packet.data; + if (trackData.info.requiresAdtsStripping) { + const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packetData)); + if (!adtsFrame) { + throw new Error('Expected ADTS frame, didn\'t get one.'); } - this.validateTimestamp( - trackData.track, - packet.timestamp, - packet.type === 'key', - ); + const headerLength = adtsFrame.crcCheck === null + ? MIN_ADTS_FRAME_HEADER_SIZE + : MAX_ADTS_FRAME_HEADER_SIZE; + packetData = packetData.subarray(headerLength); + } - let timestamp = packet.timestamp; - let duration = packet.duration; + this.validateTimestamp( + trackData.track, + packet.timestamp, + packet.type === 'key', + ); - if (trackData.info.requiresPcmTransformation) { - // Packets may have only approximate timestamp/duration information, but for our PCM logic, we need it - // to be precise. So here, we refine the values. + let timestamp = packet.timestamp; + let duration = packet.duration; - const pcmInfo = parsePcmCodec( - trackData.info.decoderConfig.codec as PcmAudioCodec, - ); - const frameSize = pcmInfo.sampleSize * trackData.info.numberOfChannels; + if (trackData.info.requiresPcmTransformation) { + // Packets may have only approximate timestamp/duration information, but for our PCM logic, we need it + // to be precise. So here, we refine the values. - // Compute the precise duration - duration = packetData.byteLength / frameSize / trackData.info.sampleRate; + const pcmInfo = parsePcmCodec( + trackData.info.decoderConfig.codec as PcmAudioCodec, + ); + const frameSize = pcmInfo.sampleSize * trackData.info.numberOfChannels; - if (trackData.info.expectedNextPcmPacketTimestamp !== null) { - const diff = timestamp - trackData.info.expectedNextPcmPacketTimestamp; - if (diff < 0.01) { - timestamp = trackData.info.expectedNextPcmPacketTimestamp; - } else { - const paddedDuration = await this.padWithSilence( - trackData, - trackData.info.expectedNextPcmPacketTimestamp, - diff, - ); - timestamp = trackData.info.expectedNextPcmPacketTimestamp + paddedDuration; - } - } + // Compute the precise duration + duration = packetData.byteLength / frameSize / trackData.info.sampleRate; - trackData.info.expectedNextPcmPacketTimestamp = timestamp + duration; + if (trackData.info.expectedNextPcmPacketTimestamp !== null) { + const diff = timestamp - trackData.info.expectedNextPcmPacketTimestamp; + if (diff < 0.01) { + timestamp = trackData.info.expectedNextPcmPacketTimestamp; + } else { + const paddedDuration = await this.padWithSilence( + trackData, + trackData.info.expectedNextPcmPacketTimestamp, + diff, + ); + timestamp = trackData.info.expectedNextPcmPacketTimestamp + paddedDuration; + } } - const internalSample = this.createSampleForTrack( - trackData, - packetData, - timestamp, - duration, - packet.type, - ); - - await this.registerSample(trackData, internalSample); - } finally { - release(); + trackData.info.expectedNextPcmPacketTimestamp = timestamp + duration; } + + const internalSample = this.createSampleForTrack( + trackData, + packetData, + timestamp, + duration, + packet.type, + ); + + await this.registerSample(trackData, internalSample); } private async padWithSilence(trackData: IsobmffAudioTrackData, timestamp: number, duration: number) { @@ -803,21 +796,18 @@ export class IsobmffMuxer extends Muxer { } async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - const trackData = this.getSubtitleTrackData(track, meta); + const trackData = this.getSubtitleTrackData(track, meta); - this.validateTimestamp(trackData.track, cue.timestamp, true); + this.validateTimestamp(trackData.track, cue.timestamp, true); - if (track.source._codec === 'webvtt') { - trackData.cueQueue.push(cue); - await this.processWebVTTCues(trackData, cue.timestamp); - } else { - // TODO - } - } finally { - release(); + if (track.source._codec === 'webvtt') { + trackData.cueQueue.push(cue); + await this.processWebVTTCues(trackData, cue.timestamp); + } else { + // TODO } } @@ -1486,7 +1476,8 @@ export class IsobmffMuxer extends Muxer { // eslint-disable-next-line @typescript-eslint/no-misused-promises override async onTrackClose(track: OutputTrack) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; const trackData = this.trackDatas.find(x => x.track === track); if (trackData) { @@ -1507,8 +1498,6 @@ export class IsobmffMuxer extends Muxer { // Since a track is now closed, we may be able to write out chunks that were previously waiting await this.interleaveSamples(); } - - release(); } ensureOneEnabledTrack() { @@ -1537,27 +1526,25 @@ export class IsobmffMuxer extends Muxer { async forceFragmentFinalization() { assert(this.isFragmented); - const release = await this.mutex.acquire(); - - try { - for (const trackData of this.trackDatas) { - if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') { - await this.processWebVTTCues(trackData, Infinity); - } + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - this.processTimestamps(trackData); + for (const trackData of this.trackDatas) { + if (trackData.type === 'subtitle' && trackData.track.source._codec === 'webvtt') { + await this.processWebVTTCues(trackData, Infinity); } - await this.interleaveSamples(true); - await this.finalizeFragment(); - } finally { - release(); + this.processTimestamps(trackData); } + + await this.interleaveSamples(true); + await this.finalizeFragment(); } /** Finalizes the file, making it ready for use. Must be called after all video and audio chunks have been added. */ async finalize() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.allTracksKnown.resolve(); this.ensureOneEnabledTrack(); @@ -1755,7 +1742,5 @@ export class IsobmffMuxer extends Muxer { this.formatOptions.onMoov(data, start); } } - - release(); } } diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index f0524c32f..1c43b99ad 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -37,7 +37,6 @@ import { InputVideoTrackBacking, } from '../input-track'; import { AttachedFile, DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata'; -import { PacketRetrievalOptions } from '../media-sink'; import { assert, binarySearchLessOrEqual, @@ -49,10 +48,12 @@ import { isThenable, last, MATRIX_COEFFICIENTS_MAP_INVERSE, + MaybeRelevantPromise, centeredTransformationMatrix, DEG_TO_RAD, multiplyMatrices, normalizeRotation, + ResultValue, Rotation, rotationMatrix, roundIfAlmostInteger, @@ -62,7 +63,13 @@ import { TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, } from '../misc'; -import { EncodedPacket, EncodedPacketSideData, PLACEHOLDER_DATA } from '../packet'; +import { + EncodedPacket, + EncodedPacketSideData, + PacketRetrievalOptions, + PacketRetrievalResult, + PLACEHOLDER_DATA, +} from '../packet'; import { assertDefinedSize, CODEC_STRING_MAP, @@ -654,9 +661,9 @@ export class MatroskaDemuxer extends Demuxer { this.currentSegment = null; } - async readCluster(startPos: number, segment: Segment) { + async readCluster(res: ResultValue, startPos: number, segment: Segment): MaybeRelevantPromise { if (segment.lastReadCluster?.elementStartPos === startPos) { - return segment.lastReadCluster; + return res.set(segment.lastReadCluster); } let headerSlice = this.reader.requestSliceRange(startPos, MIN_HEADER_SIZE, MAX_HEADER_SIZE); @@ -786,7 +793,7 @@ export class MatroskaDemuxer extends Demuxer { } segment.lastReadCluster = cluster; - return cluster; + return res.set(cluster); } getTrackDataInCluster(cluster: Cluster, trackNumber: number) { @@ -2031,11 +2038,6 @@ export class MatroskaDemuxer extends Demuxer { } abstract class MatroskaTrackBacking implements InputTrackBacking { - packetToClusterLocation = new WeakMap(); - constructor(public internalTrack: InternalTrack) {} abstract getType(): TrackType; @@ -2117,8 +2119,12 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { return null; } - async getFirstPacket(options: PacketRetrievalOptions) { + async getFirstPacket( + res: ResultValue, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { return this.performClusterLookup( + res, null, (cluster) => { const trackData = cluster.trackData.get(this.internalTrack.id); @@ -2147,10 +2153,15 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { return roundIfAlmostInteger(timestamp * this.internalTrack.segment.timestampFactor); } - async getPacket(timestamp: number, options: PacketRetrievalOptions) { + async getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { const timestampInTimescale = this.intoTimescale(timestamp); return this.performClusterLookup( + res, null, (cluster) => { const trackData = cluster.trackData.get(this.internalTrack.id); @@ -2175,21 +2186,27 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { ); } - async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - const locationInCluster = this.packetToClusterLocation.get(packet); - if (locationInCluster === undefined) { - throw new Error('Packet was not created from this track.'); - } + async getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + const clusterStartPos = packet._internal; + assert(typeof clusterStartPos === 'number'); + + // sequenceNumber = clusterStartPos + blockIndex, so: + const blockIndex = packet.sequenceNumber - clusterStartPos; return this.performClusterLookup( - locationInCluster.cluster, + res, + clusterStartPos, (cluster) => { - if (cluster === locationInCluster.cluster) { + if (cluster.elementStartPos === clusterStartPos) { const trackData = cluster.trackData.get(this.internalTrack.id)!; - if (locationInCluster.blockIndex + 1 < trackData.blocks.length) { + if (blockIndex + 1 < trackData.blocks.length) { // We can simply take the next block in the cluster return { - blockIndex: locationInCluster.blockIndex + 1, + blockIndex: blockIndex + 1, correctBlockFound: true, }; } @@ -2214,10 +2231,15 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { ); } - async getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { + async getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { const timestampInTimescale = this.intoTimescale(timestamp); return this.performClusterLookup( + res, null, (cluster) => { const trackData = cluster.trackData.get(this.internalTrack.id); @@ -2241,19 +2263,25 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { ); } - async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - const locationInCluster = this.packetToClusterLocation.get(packet); - if (locationInCluster === undefined) { - throw new Error('Packet was not created from this track.'); - } + async getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + const clusterStartPos = packet._internal; + assert(typeof clusterStartPos === 'number'); + + // sequenceNumber = clusterStartPos + blockIndex, so: + const blockIndex = packet.sequenceNumber - clusterStartPos; return this.performClusterLookup( - locationInCluster.cluster, + res, + clusterStartPos, (cluster) => { - if (cluster === locationInCluster.cluster) { + if (cluster.elementStartPos === clusterStartPos) { const trackData = cluster.trackData.get(this.internalTrack.id)!; const nextKeyFrameIndex = trackData.blocks.findIndex( - (x, i) => x.isKeyFrame && i > locationInCluster.blockIndex, + (x, i) => x.isKeyFrame && i > blockIndex, ); if (nextKeyFrameIndex !== -1) { @@ -2287,7 +2315,11 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { ); } - private async fetchPacketInCluster(cluster: Cluster, blockIndex: number, options: PacketRetrievalOptions) { + private fetchPacketInCluster( + cluster: Cluster, + blockIndex: number, + options: PacketRetrievalOptions, + ) { if (blockIndex === -1) { return null; } @@ -2346,20 +2378,21 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { block.isKeyFrame ? 'key' : 'delta', timestamp, duration, - cluster.dataStartPos + blockIndex, + cluster.elementStartPos + blockIndex, block.data.byteLength, sideData, ); - this.packetToClusterLocation.set(packet, { cluster, blockIndex }); + packet._internal = cluster.elementStartPos; return packet; } /** Looks for a packet in the clusters while trying to load as few clusters as possible to retrieve it. */ private async performClusterLookup( - // The cluster where we start looking - startCluster: Cluster | null, + res: ResultValue, + // The position where we'll start the lookup + startOffset: number | null, // This function returns the best-matching block in a given cluster getMatchInCluster: (cluster: Cluster) => { blockIndex: number; correctBlockFound: boolean }, // The timestamp with which we can search the lookup table @@ -2367,20 +2400,25 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { // The timestamp for which we know the correct block will not come after it latestTimestamp: number, options: PacketRetrievalOptions, - ): Promise { + ): MaybeRelevantPromise { const { demuxer, segment } = this.internalTrack; + let currentPos = startOffset ?? 0; let currentCluster: Cluster | null = null; let bestCluster: Cluster | null = null; let bestBlockIndex = -1; - if (startCluster) { + if (startOffset !== null && segment.lastReadCluster?.elementStartPos === startOffset) { + const startCluster = segment.lastReadCluster; const { blockIndex, correctBlockFound } = getMatchInCluster(startCluster); if (correctBlockFound) { - return this.fetchPacketInCluster(startCluster, blockIndex, options); + return res.set(new PacketRetrievalResult(this.fetchPacketInCluster(startCluster, blockIndex, options))); } + currentPos = startCluster.elementEndPos; // Start reading from the next cluster + currentCluster = startCluster; + if (blockIndex !== -1) { bestCluster = startCluster; bestBlockIndex = blockIndex; @@ -2413,18 +2451,14 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { positionCacheEntry?.elementStartPos ?? 0, ) || null; - let currentPos: number; + if (lookupEntryPosition !== null && lookupEntryPosition > currentPos) { + // Use the lookup entry + currentPos = lookupEntryPosition; + currentCluster = null; + } - if (!startCluster) { + if (startOffset === null) { currentPos = lookupEntryPosition ?? segment.clusterSeekStartPos; - } else { - if (lookupEntryPosition === null || startCluster.elementStartPos >= lookupEntryPosition) { - currentPos = startCluster.elementEndPos; - currentCluster = startCluster; - } else { - // Use the lookup entry - currentPos = lookupEntryPosition; - } } while (segment.elementEndPos === null || currentPos <= segment.elementEndPos - MIN_HEADER_SIZE) { @@ -2470,13 +2504,20 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { const dataStartPos = slice.filePos; if (id === EBMLId.Cluster) { - currentCluster = await demuxer.readCluster(elementStartPos, segment); + const result = new ResultValue(); + const promise = demuxer.readCluster(result, elementStartPos, segment); + if (result.pending) await promise; + + currentCluster = result.value; + // readCluster computes the proper size even if it's undefined in the header, so let's use that instead size = currentCluster.elementEndPos - dataStartPos; const { blockIndex, correctBlockFound } = getMatchInCluster(currentCluster); if (correctBlockFound) { - return this.fetchPacketInCluster(currentCluster, blockIndex, options); + return res.set( + new PacketRetrievalResult(this.fetchPacketInCluster(currentCluster, blockIndex, options)), + ); } if (blockIndex !== -1) { @@ -2530,15 +2571,17 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { assert(!previousCuePoint || previousCuePoint.time < cuePoint.time); const newSearchTimestamp = previousCuePoint?.time ?? -Infinity; - return this.performClusterLookup(null, getMatchInCluster, newSearchTimestamp, latestTimestamp, options); + return this.performClusterLookup( + res, null, getMatchInCluster, newSearchTimestamp, latestTimestamp, options, + ); } if (bestCluster) { // If we finished looping but didn't find a perfect match, still return the best match we found - return this.fetchPacketInCluster(bestCluster, bestBlockIndex, options); + return res.set(new PacketRetrievalResult(this.fetchPacketInCluster(bestCluster, bestBlockIndex, options))); } - return null; + return res.set(new PacketRetrievalResult(null)); } } @@ -2626,7 +2669,11 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid || (this.internalTrack.info.codec === 'hevc' && !this.internalTrack.info.codecDescription); if (needsPacketForAdditionalInfo) { - firstPacket = await this.getFirstPacket({}); + const result = new ResultValue(); + const promise = this.getFirstPacket(result, {}); + if (result.pending) await promise; + + firstPacket = result.value.packet; } const codecInfo = { @@ -2717,7 +2764,11 @@ class MatroskaAudioTrackBacking extends MatroskaTrackBacking implements InputAud return this.decoderConfigPromise ??= (async (): Promise => { if (this.internalTrack.info.codec === 'dts' && !this.internalTrack.info.dtsFormat) { // Gotta check the packet to determine the DTS variant - const firstPacket = await this.getFirstPacket({}); + const result = new ResultValue(); + const promise = this.getFirstPacket(result, {}); + if (result.pending) await promise; + + const firstPacket = result.value.packet; this.internalTrack.info.dtsFormat = firstPacket && extractDtsFourCcFromPacket(firstPacket.data); } diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index a0f5fcc57..d969d716b 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -175,7 +175,8 @@ export class MatroskaMuxer extends Muxer { } async start() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.writer = await this.output._getRootWriter(!!this.format._options.appendOnly); this.ebmlWriter = new EBMLWriter(this.writer); @@ -202,8 +203,6 @@ export class MatroskaMuxer extends Muxer { ); } } - - release(); } private writeEBMLHeader() { @@ -983,115 +982,106 @@ export class MatroskaMuxer extends Muxer { } async addEncodedVideoPacket(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) { - const release = await this.mutex.acquire(); - - try { - const trackData = this.getVideoTrackData(track, packet, meta); - trackData.info.alphaMode ??= !!packet.sideData.alpha; + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - let packetData = packet.data; - if (track.source._codec === 'prores') { - if (packetData.byteLength < 8) { - throw new Error('ProRes packet too small, expected at least 8 bytes.'); - } + const trackData = this.getVideoTrackData(track, packet, meta); + trackData.info.alphaMode ??= !!packet.sideData.alpha; - // Trim off the frame container atom header. FFmpeg does this too and cites the "Matroska spec" as the - // reason, despite the spec not saying anything about this. - packetData = packetData.subarray(8); + let packetData = packet.data; + if (track.source._codec === 'prores') { + if (packetData.byteLength < 8) { + throw new Error('ProRes packet too small, expected at least 8 bytes.'); } - const isKeyFrame = packet.type === 'key'; - this.validateTimestamp(trackData.track, packet.timestamp, isKeyFrame); + // Trim off the frame container atom header. FFmpeg does this too and cites the "Matroska spec" as the + // reason, despite the spec not saying anything about this. + packetData = packetData.subarray(8); + } - let timestamp = packet.timestamp; - let duration = packet.duration; + const isKeyFrame = packet.type === 'key'; + this.validateTimestamp(trackData.track, packet.timestamp, isKeyFrame); - if (track.metadata.frameRate !== undefined) { - // Constrain the time values to the frame rate - timestamp = roundToDivisor(timestamp, track.metadata.frameRate); - duration = roundToDivisor(duration, track.metadata.frameRate); - } + let timestamp = packet.timestamp; + let duration = packet.duration; - const additions = trackData.info.alphaMode - ? packet.sideData.alpha ?? null - : null; + if (track.metadata.frameRate !== undefined) { + // Constrain the time values to the frame rate + timestamp = roundToDivisor(timestamp, track.metadata.frameRate); + duration = roundToDivisor(duration, track.metadata.frameRate); + } - const videoChunk = this.createInternalChunk(packetData, timestamp, duration, packet.type, additions); - if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk); + const additions = trackData.info.alphaMode + ? packet.sideData.alpha ?? null + : null; - trackData.chunkQueue.push(videoChunk); - await this.interleaveChunks(); - } finally { - release(); - } + const videoChunk = this.createInternalChunk(packetData, timestamp, duration, packet.type, additions); + if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk); + + trackData.chunkQueue.push(videoChunk); + await this.interleaveChunks(); } async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - const trackData = this.getAudioTrackData(track, packet, meta); + const trackData = this.getAudioTrackData(track, packet, meta); - let packetData = packet.data; - if (trackData.info.requiresAdtsStripping) { - const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packetData)); - if (!adtsFrame) { - throw new Error('Expected ADTS frame, didn\'t get one.'); - } - - const headerLength = adtsFrame.crcCheck === null - ? MIN_ADTS_FRAME_HEADER_SIZE - : MAX_ADTS_FRAME_HEADER_SIZE; - packetData = packetData.subarray(headerLength); + let packetData = packet.data; + if (trackData.info.requiresAdtsStripping) { + const adtsFrame = readAdtsFrameHeader(FileSlice.tempFromBytes(packetData)); + if (!adtsFrame) { + throw new Error('Expected ADTS frame, didn\'t get one.'); } - const isKeyFrame = packet.type === 'key'; - this.validateTimestamp(trackData.track, packet.timestamp, isKeyFrame); - - const audioChunk = this.createInternalChunk(packetData, packet.timestamp, packet.duration, packet.type); - trackData.chunkQueue.push(audioChunk); - await this.interleaveChunks(); - } finally { - release(); + const headerLength = adtsFrame.crcCheck === null + ? MIN_ADTS_FRAME_HEADER_SIZE + : MAX_ADTS_FRAME_HEADER_SIZE; + packetData = packetData.subarray(headerLength); } + + const isKeyFrame = packet.type === 'key'; + this.validateTimestamp(trackData.track, packet.timestamp, isKeyFrame); + + const audioChunk = this.createInternalChunk(packetData, packet.timestamp, packet.duration, packet.type); + trackData.chunkQueue.push(audioChunk); + await this.interleaveChunks(); } async addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - const trackData = this.getSubtitleTrackData(track, meta); + const trackData = this.getSubtitleTrackData(track, meta); - this.validateTimestamp(trackData.track, cue.timestamp, true); + this.validateTimestamp(trackData.track, cue.timestamp, true); - let bodyText = cue.text; - const timestampMs = Math.round(cue.timestamp * 1000); + let bodyText = cue.text; + const timestampMs = Math.round(cue.timestamp * 1000); - // Replace in-body timestamps so that they're relative to the cue start time - inlineTimestampRegex.lastIndex = 0; - bodyText = bodyText.replace(inlineTimestampRegex, (match) => { - const time = parseSubtitleTimestamp(match.slice(1, -1)); - const offsetTime = time - timestampMs; + // Replace in-body timestamps so that they're relative to the cue start time + inlineTimestampRegex.lastIndex = 0; + bodyText = bodyText.replace(inlineTimestampRegex, (match) => { + const time = parseSubtitleTimestamp(match.slice(1, -1)); + const offsetTime = time - timestampMs; - return `<${formatSubtitleTimestamp(offsetTime)}>`; - }); + return `<${formatSubtitleTimestamp(offsetTime)}>`; + }); - const body = textEncoder.encode(bodyText); - const additions = `${cue.settings ?? ''}\n${cue.identifier ?? ''}\n${cue.notes ?? ''}`; + const body = textEncoder.encode(bodyText); + const additions = `${cue.settings ?? ''}\n${cue.identifier ?? ''}\n${cue.notes ?? ''}`; - const subtitleChunk = this.createInternalChunk( - body, - cue.timestamp, - cue.duration, - 'key', - additions.trim() ? textEncoder.encode(additions) : null, - ); + const subtitleChunk = this.createInternalChunk( + body, + cue.timestamp, + cue.duration, + 'key', + additions.trim() ? textEncoder.encode(additions) : null, + ); - trackData.chunkQueue.push(subtitleChunk); - await this.interleaveChunks(); - } finally { - release(); - } + trackData.chunkQueue.push(subtitleChunk); + await this.interleaveChunks(); } private async interleaveChunks(isFinalCall = false) { @@ -1396,7 +1386,8 @@ export class MatroskaMuxer extends Muxer { // eslint-disable-next-line @typescript-eslint/no-misused-promises override async onTrackClose(track: OutputTrack) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; const trackData = this.trackDatas.find(x => x.track === track); if (trackData) { @@ -1409,15 +1400,12 @@ export class MatroskaMuxer extends Muxer { // Since a track is now closed, we may be able to write out chunks that were previously waiting await this.interleaveChunks(); - - release(); } /** Finalizes the file, making it ready for use. Must be called after all media chunks have been added. */ async finalize() { - const release = await this.mutex.acquire(); - - this.allTracksKnown.resolve(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; for (const trackData of this.trackDatas) { trackData.closed = true; @@ -1455,7 +1443,5 @@ export class MatroskaMuxer extends Muxer { this.maybeCreateSeekHead(true); this.ebmlWriter.writeEBML(this.seekHead); } - - release(); } } diff --git a/src/media-sink.ts b/src/media-sink.ts deleted file mode 100644 index 45e4dd88e..000000000 --- a/src/media-sink.ts +++ /dev/null @@ -1,2708 +0,0 @@ -/*! - * Copyright (c) 2026-present, Vanilagy and contributors - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. - */ - -import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec'; -import { - addAvcBitstreamRestriction, - AvcNalUnitType, - concatAvcNalUnits, - deserializeAvcDecoderConfigurationRecord, - determineVideoPacketType, - extractNalUnitTypeForAvc, - extractNalUnitTypeForHevc, - HevcNalUnitType, - iterateAvcNalUnits, - iterateHevcNalUnits, - parseAvcSps, - sanitizeHevcPacketForChromium, - serializeAvcDecoderConfigurationRecord, -} from './codec-data'; -import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; -import { InputDisposedError } from './input'; -import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; -import { - AnyIterable, - assert, - assertNever, - CallSerializer, - clamp, - colorSpaceIsComplete, - getInt24, - getUint24, - insertSorted, - isChromium, - isFirefox, - isNumber, - isWebKit, - last, - mapAsyncGenerator, - missingWebCodecsClassMessage, - promiseWithResolvers, - removeItem, - Rotation, - toAsyncIterator, - toDataView, - toUint8Array, - validateAnyIterable, -} from './misc'; -import { EncodedPacket } from './packet'; -import { fromAlaw, fromUlaw } from './pcm'; -import { - AudioSample, - clampCropRectangle, - CropRectangle, - validateCropRectangle, - VideoSample, - VideoSamplePixelFormat, -} from './sample'; - -/** - * Additional options for controlling packet retrieval. - * @group Media sinks - * @public - */ -export type PacketRetrievalOptions = { - /** - * When set to `true`, only packet metadata (like timestamp) will be retrieved - the actual packet data will not - * be loaded. - */ - metadataOnly?: boolean; - - /** - * When set to `true`, key packets will be verified upon retrieval by looking into the packet's bitstream. - * If not enabled, the packet types will be determined solely by what's stored in the containing file and may be - * incorrect, potentially leading to decoder errors. Since determining a packet's actual type requires looking into - * its data, this option cannot be enabled together with `metadataOnly`. - */ - verifyKeyPackets?: boolean; - - /** - * When querying packets in live media that are in the future relative to the current live edge, Mediabunny will, - * by default, wait for the stream to advance until the query can be satisfied. In a sense, Mediabunny simply treats - * live streams as media files that are still being written, and any read that depends on future information will - * wait until it can be fulfilled. - * - * If you want to query packets based only on the currently known information, set this field to `true` - this way, - * Mediabunny will never wait for the live stream to catch up. - * - * For non-live media, this field has no effect. - */ - skipLiveWait?: boolean; -}; - -const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => { - if (!options || typeof options !== 'object') { - throw new TypeError('options must be an object.'); - } - if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') { - throw new TypeError('options.metadataOnly, when defined, must be a boolean.'); - } - if (options.verifyKeyPackets !== undefined && typeof options.verifyKeyPackets !== 'boolean') { - throw new TypeError('options.verifyKeyPackets, when defined, must be a boolean.'); - } - if (options.verifyKeyPackets && options.metadataOnly) { - throw new TypeError('options.verifyKeyPackets and options.metadataOnly cannot be enabled together.'); - } - if (options.skipLiveWait !== undefined && typeof options.skipLiveWait !== 'boolean') { - throw new TypeError('options.skipLiveWait, when defined, must be a boolean.'); - } -}; - -const validateTimestamp = (timestamp: number) => { - if (!isNumber(timestamp)) { - throw new TypeError('timestamp must be a number.'); // It can be non-finite, that's fine - } -}; - -const maybeFixPacketType = ( - track: InputTrack, - promise: Promise, - options: PacketRetrievalOptions, -) => { - if (options.verifyKeyPackets) { - return promise.then(async (packet) => { - if (!packet || packet.type === 'delta') { - return packet; - } - - const determinedType = await track.determinePacketType(packet); - if (determinedType) { - // @ts-expect-error Technically readonly - packet.type = determinedType; - } - - return packet; - }); - } else { - return promise; - } -}; - -/** - * Sink for retrieving encoded packets from an input track. - * @group Media sinks - * @public - */ -export class EncodedPacketSink { - /** @internal */ - _track: InputTrack; - - /** Creates a new {@link EncodedPacketSink} for the given {@link InputTrack}. */ - constructor(track: InputTrack) { - if (!(track instanceof InputTrack)) { - throw new TypeError('track must be an InputTrack.'); - } - - this._track = track; - } - - /** - * Retrieves the track's first packet (in decode order), or null if it has no packets. The first packet is very - * likely to be a key packet, but it doesn't have to be. - */ - async getFirstPacket(options: PacketRetrievalOptions = {}) { - validatePacketRetrievalOptions(options); - - if (this._track.input._disposed) { - throw new InputDisposedError(); - } - - return maybeFixPacketType(this._track, this._track._backing.getFirstPacket(options), options); - } - - /** Retrieves the track's first key packet (in decode order), or null if it has no key packets. */ - async getFirstKeyPacket(options: PacketRetrievalOptions = {}) { - validatePacketRetrievalOptions(options); - - const firstPacket = await this.getFirstPacket(options); - if (!firstPacket) { - return null; - } - - if (firstPacket.type === 'key') { - // Great - return firstPacket; - } - - return this.getNextKeyPacket(firstPacket, options); - } - - /** - * Retrieves the packet corresponding to the given timestamp, in seconds. More specifically, returns the last packet - * (in presentation order) with a start timestamp less than or equal to the given timestamp. This method can be - * used to retrieve a track's last packet using `getPacket(Infinity)`. The method returns null if the timestamp - * is before the first packet in the track. - * - * @param timestamp - The timestamp used for retrieval, in seconds. - */ - async getPacket(timestamp: number, options: PacketRetrievalOptions = {}) { - validateTimestamp(timestamp); - validatePacketRetrievalOptions(options); - - if (this._track.input._disposed) { - throw new InputDisposedError(); - } - - return maybeFixPacketType(this._track, this._track._backing.getPacket(timestamp, options), options); - } - - /** - * Retrieves the packet following the given packet (in decode order), or null if the given packet is the - * last packet. - */ - async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}) { - if (!(packet instanceof EncodedPacket)) { - throw new TypeError('packet must be an EncodedPacket.'); - } - validatePacketRetrievalOptions(options); - - if (this._track.input._disposed) { - throw new InputDisposedError(); - } - - return maybeFixPacketType(this._track, this._track._backing.getNextPacket(packet, options), options); - } - - /** - * Retrieves the key packet corresponding to the given timestamp, in seconds. More specifically, returns the last - * key packet (in presentation order) with a start timestamp less than or equal to the given timestamp. A key packet - * is a packet that doesn't require previous packets to be decoded. This method can be used to retrieve a track's - * last key packet using `getKeyPacket(Infinity)`. The method returns null if the timestamp is before the first - * key packet in the track. - * - * To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`. - * - * @param timestamp - The timestamp used for retrieval, in seconds. - */ - async getKeyPacket(timestamp: number, options: PacketRetrievalOptions = {}): Promise { - validateTimestamp(timestamp); - validatePacketRetrievalOptions(options); - - if (this._track.input._disposed) { - throw new InputDisposedError(); - } - - if (!options.verifyKeyPackets) { - return this._track._backing.getKeyPacket(timestamp, options); - } - - const packet = await this._track._backing.getKeyPacket(timestamp, options); - if (!packet) { - return packet; - } - assert(packet.type === 'key'); - - const determinedType = await this._track.determinePacketType(packet); - if (determinedType === 'delta') { - // Try returning the previous key packet (in hopes that it's actually a key packet) - return this.getKeyPacket(packet.timestamp - 1 / await this._track.getTimeResolution(), options); - } - - return packet; - } - - /** - * Retrieves the key packet following the given packet (in decode order), or null if the given packet is the last - * key packet. - * - * To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`. - */ - async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}): Promise { - if (!(packet instanceof EncodedPacket)) { - throw new TypeError('packet must be an EncodedPacket.'); - } - validatePacketRetrievalOptions(options); - - if (this._track.input._disposed) { - throw new InputDisposedError(); - } - - if (!options.verifyKeyPackets) { - return this._track._backing.getNextKeyPacket(packet, options); - } - - const nextPacket = await this._track._backing.getNextKeyPacket(packet, options); - if (!nextPacket) { - return nextPacket; - } - assert(nextPacket.type === 'key'); - - const determinedType = await this._track.determinePacketType(nextPacket); - if (determinedType === 'delta') { - // Try returning the next key packet (in hopes that it's actually a key packet) - return this.getNextKeyPacket(nextPacket, options); - } - - return nextPacket; - } - - /** - * Creates an async iterator that yields the packets in this track in decode order. To enable fast iteration, this - * method will intelligently preload packets based on the speed of the consumer. - * - * @param startPacket - (optional) The packet from which iteration should begin. This packet will also be yielded. - * @param endPacket - (optional) The packet at which iteration should end. This packet will _not_ be yielded. - */ - packets( - startPacket?: EncodedPacket, - endPacket?: EncodedPacket, - options: PacketRetrievalOptions = {}, - ): AsyncGenerator { - if (startPacket !== undefined && !(startPacket instanceof EncodedPacket)) { - throw new TypeError('startPacket must be an EncodedPacket.'); - } - if (startPacket !== undefined && startPacket.isMetadataOnly && !options?.metadataOnly) { - throw new TypeError('startPacket can only be metadata-only if options.metadataOnly is enabled.'); - } - if (endPacket !== undefined && !(endPacket instanceof EncodedPacket)) { - throw new TypeError('endPacket must be an EncodedPacket.'); - } - validatePacketRetrievalOptions(options); - - if (this._track.input._disposed) { - throw new InputDisposedError(); - } - - const packetQueue: EncodedPacket[] = []; - - let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers(); - let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers(); - let ended = false; - let terminated = false; - - // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this - // method but instead in a different context. This error should not go unnoticed and must be bubbled up to - // the consumer. - let outOfBandError = null as unknown; - let hasOutOfBandError = false; - - const timestamps: number[] = []; - // The queue should always be big enough to hold 1 second worth of packets - const maxQueueSize = () => Math.max(2, timestamps.length); - - // The following is the "pump" process that keeps pumping packets into the queue - (async () => { - let packet = startPacket ?? await this.getFirstPacket(options); - - while (packet && !terminated && !this._track.input._disposed) { - if (endPacket && packet.sequenceNumber >= endPacket?.sequenceNumber) { - break; - } - - if (packetQueue.length > maxQueueSize()) { - ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers()); - await queueDequeue; - continue; - } - - packetQueue.push(packet); - - onQueueNotEmpty(); - ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers()); - - packet = await this.getNextPacket(packet, options); - } - - ended = true; - onQueueNotEmpty(); - })().catch((error) => { - if (!hasOutOfBandError) { - outOfBandError = error; - hasOutOfBandError = true; - onQueueNotEmpty(); - } - }); - - const track = this._track; - - return { - async next() { - while (true) { - if (track.input._disposed) { - throw new InputDisposedError(); - } else if (terminated) { - return { value: undefined, done: true }; - } else if (hasOutOfBandError) { - throw outOfBandError; - } else if (packetQueue.length > 0) { - const value = packetQueue.shift()!; - const now = performance.now(); - timestamps.push(now); - - while (timestamps.length > 0 && now - timestamps[0]! >= 1000) { - timestamps.shift(); - } - - onQueueDequeue(); - - return { value, done: false }; - } else if (ended) { - return { value: undefined, done: true }; - } else { - await queueNotEmpty; - } - } - }, - async return() { - terminated = true; - onQueueDequeue(); - onQueueNotEmpty(); - - return { value: undefined, done: true }; - }, - async throw(error) { - throw error; - }, - [Symbol.asyncIterator]() { - return this; - }, - }; - } -} - -abstract class DecoderWrapper< - MediaSample extends VideoSample | AudioSample, -> { - constructor( - public onSample: (sample: MediaSample) => unknown, - public onError: (error: unknown) => unknown, - ) {} - - abstract getDecodeQueueSize(): number; - abstract decode(packet: EncodedPacket): void; - abstract flush(): Promise; - abstract close(): void; -} - -/** - * Base class for decoded media sample sinks. - * @group Media sinks - * @public - */ -export abstract class BaseMediaSampleSink< - MediaSample extends VideoSample | AudioSample, -> { - /** @internal */ - abstract _track: InputTrack; - - /** @internal */ - abstract _createDecoder( - onSample: (sample: MediaSample) => unknown, - onError: (error: unknown) => unknown - ): Promise>; - /** @internal */ - abstract _createPacketSink(): EncodedPacketSink; - - /** @internal */ - protected mediaSamplesInRange( - startTimestamp = -Infinity, - endTimestamp = Infinity, - options: PacketRetrievalOptions, - ): AsyncGenerator { - validateTimestamp(startTimestamp); - validateTimestamp(endTimestamp); - - const sampleQueue: MediaSample[] = []; - let firstSampleQueued = false; - let lastSample: MediaSample | null = null; - let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers(); - let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers(); - let decoderIsFlushed = false; - let ended = false; - let terminated = false; - let decoder: DecoderWrapper | null = null; - - // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this - // method but instead in a different context. This error should not go unnoticed and must be bubbled up to - // the consumer. - let outOfBandError = null as unknown; - let hasOutOfBandError = false; - - const packetRetrievalOptions: PacketRetrievalOptions = { - ...options, - verifyKeyPackets: true, - metadataOnly: false, - }; - - // The following is the "pump" process that keeps pumping packets into the decoder - (async () => { - decoder = await this._createDecoder((sample) => { - onQueueDequeue(); - if (sample.timestamp >= endTimestamp) { - ended = true; - } - - if (ended) { - sample.close(); - return; - } - - if (lastSample) { - if (sample.timestamp > startTimestamp) { - // We don't know ahead of time what the first first is. This is because the first first is the - // last first whose timestamp is less than or equal to the start timestamp. Therefore we need to - // wait for the first first after the start timestamp, and then we'll know that the previous - // first was the first first. - sampleQueue.push(lastSample); - firstSampleQueued = true; - } else { - lastSample.close(); - } - } - - if (sample.timestamp >= startTimestamp) { - sampleQueue.push(sample); - firstSampleQueued = true; - } - - lastSample = firstSampleQueued ? null : sample; - - if (sampleQueue.length > 0) { - onQueueNotEmpty(); - ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers()); - } - }, (error) => { - if (!hasOutOfBandError) { - outOfBandError = error; - hasOutOfBandError = true; - onQueueNotEmpty(); - } - }); - - const packetSink = this._createPacketSink(); - const keyPacket = await packetSink.getKeyPacket(startTimestamp, packetRetrievalOptions) - ?? await packetSink.getFirstKeyPacket(packetRetrievalOptions); - - let currentPacket: EncodedPacket | null = keyPacket; - - // B-frames make it exceedingly difficult to properly define an upper bound for packet iteration if an end - // timestamp is set, so we just don't do it. The case that makes it especially tricky is when the frames - // following a key frame have a lower timestamp than the keyframe; something that quite frequently happens - // in HEVC streams. The price to pay for not upper-bounding the packet iterator is a slight increase in - // decoder work at the end of the range, but the added correctness and reliability makes this tradeoff worth - // it. - const endPacket = undefined; - - const packets = packetSink.packets(keyPacket ?? undefined, endPacket, packetRetrievalOptions); - await packets.next(); // Skip the start packet as we already have it - - while (currentPacket && !ended && !this._track.input._disposed) { - const maxQueueSize = computeMaxQueueSize(sampleQueue.length); - if (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize) { - ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers()); - await queueDequeue; - continue; - } - - decoder.decode(currentPacket); - - const packetResult = await packets.next(); - if (packetResult.done) { - break; - } - - currentPacket = packetResult.value; - } - - await packets.return(); - - if (!terminated && !this._track.input._disposed) { - await decoder.flush(); - } - - if (!firstSampleQueued && lastSample) { - sampleQueue.push(lastSample); - } - - decoderIsFlushed = true; - onQueueNotEmpty(); // To unstuck (unstick?) the generator - })().catch((error) => { - if (!hasOutOfBandError) { - outOfBandError = error; - hasOutOfBandError = true; - onQueueNotEmpty(); - } - }).finally(() => { - decoder?.close(); - }); - - const track = this._track; - const closeSamples = () => { - lastSample?.close(); - for (const sample of sampleQueue) { - sample.close(); - } - }; - - return { - async next() { - while (true) { - if (track.input._disposed) { - // Once next() throws, the consumer will never call return(), so terminate the - // iteration here - otherwise, the pump keeps queueing decoded samples that - // nothing can ever close. - terminated = true; - ended = true; - closeSamples(); - throw new InputDisposedError(); - } else if (terminated) { - return { value: undefined, done: true }; - } else if (hasOutOfBandError) { - terminated = true; - ended = true; - closeSamples(); - throw outOfBandError; - } else if (sampleQueue.length > 0) { - const value = sampleQueue.shift()!; - onQueueDequeue(); - return { value, done: false }; - } else if (!decoderIsFlushed) { - await queueNotEmpty; - } else { - return { value: undefined, done: true }; - } - } - }, - async return() { - terminated = true; - ended = true; - onQueueDequeue(); - onQueueNotEmpty(); - closeSamples(); - - return { value: undefined, done: true }; - }, - async throw(error) { - throw error; - }, - [Symbol.asyncIterator]() { - return this; - }, - }; - } - - /** @internal */ - protected mediaSamplesAtTimestamps( - timestamps: AnyIterable, - options: PacketRetrievalOptions, - ): AsyncGenerator { - validateAnyIterable(timestamps); - const timestampIterator = toAsyncIterator(timestamps); - const timestampsOfInterest: number[] = []; - - const sampleQueue: (MediaSample | null)[] = []; - let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers(); - let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers(); - let decoderIsFlushed = false; - let terminated = false; - let decoder: DecoderWrapper | null = null; - - // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this - // method but instead in a different context. This error should not go unnoticed and must be bubbled up to - // the consumer. - let outOfBandError = null as unknown; - let hasOutOfBandError = false; - - const pushToQueue = (sample: MediaSample | null) => { - sampleQueue.push(sample); - onQueueNotEmpty(); - ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers()); - }; - - const retrievalOptions: PacketRetrievalOptions = { - ...options, - verifyKeyPackets: true, - metadataOnly: false, - }; - - // The following is the "pump" process that keeps pumping packets into the decoder - (async () => { - decoder = await this._createDecoder((sample) => { - onQueueDequeue(); - - if (terminated) { - sample.close(); - return; - } - - let sampleUses = 0; - while ( - timestampsOfInterest.length > 0 - && sample.timestamp - timestampsOfInterest[0]! > -1e-10 // Give it a little epsilon - ) { - sampleUses++; - timestampsOfInterest.shift(); - } - - if (sampleUses > 0) { - for (let i = 0; i < sampleUses; i++) { - // Clone the sample if we need to emit it multiple times - pushToQueue((i < sampleUses - 1 ? sample.clone() : sample) as MediaSample); - } - } else { - sample.close(); - } - }, (error) => { - if (!hasOutOfBandError) { - outOfBandError = error; - hasOutOfBandError = true; - onQueueNotEmpty(); - } - }); - - const packetSink = this._createPacketSink(); - let lastPacket: EncodedPacket | null = null; - let lastKeyPacket: EncodedPacket | null = null; - - // The end sequence number (inclusive) in the next batch of packets that will be decoded. The batch starts - // at the last key frame and goes until this sequence number. - let maxSequenceNumber = -1; - - const decodePackets = async () => { - assert(lastKeyPacket); - assert(decoder); - - // Start at the current key packet - let currentPacket = lastKeyPacket; - decoder.decode(currentPacket); - - while (currentPacket.sequenceNumber < maxSequenceNumber) { - const maxQueueSize = computeMaxQueueSize(sampleQueue.length); - while (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize && !terminated) { - ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers()); - await queueDequeue; - } - - if (terminated) { - break; - } - - const nextPacket = await packetSink.getNextPacket(currentPacket, retrievalOptions); - assert(nextPacket); - - decoder.decode(nextPacket); - currentPacket = nextPacket; - } - - maxSequenceNumber = -1; - }; - - const flushDecoder = async () => { - assert(decoder); - await decoder.flush(); - - // We don't expect this list to have any elements in it anymore, but in case it does, let's emit - // nulls for every remaining element, then clear it. - for (let i = 0; i < timestampsOfInterest.length; i++) { - pushToQueue(null); - } - timestampsOfInterest.length = 0; - }; - - for await (const timestamp of timestampIterator) { - validateTimestamp(timestamp); - - if (terminated || this._track.input._disposed) { - break; - } - - const targetPacket = await packetSink.getPacket(timestamp, retrievalOptions); - const keyPacket = targetPacket && await packetSink.getKeyPacket(timestamp, retrievalOptions); - - if (!keyPacket) { - if (maxSequenceNumber !== -1) { - await decodePackets(); - await flushDecoder(); - } - - pushToQueue(null); - lastPacket = null; - continue; - } - - // Check if the key packet has changed or if we're going back in time - if ( - lastPacket - && ( - keyPacket.sequenceNumber !== lastKeyPacket!.sequenceNumber - || targetPacket.timestamp < lastPacket.timestamp - ) - ) { - await decodePackets(); - await flushDecoder(); // Always flush here, improves decoder compatibility - } - - timestampsOfInterest.push(targetPacket.timestamp); - maxSequenceNumber = Math.max(targetPacket.sequenceNumber, maxSequenceNumber); - - lastPacket = targetPacket; - lastKeyPacket = keyPacket; - } - - if (!terminated && !this._track.input._disposed) { - if (maxSequenceNumber !== -1) { - // We still need to decode packets - await decodePackets(); - } - - await flushDecoder(); - } - - decoderIsFlushed = true; - onQueueNotEmpty(); // To unstuck the generator - })().catch((error) => { - if (!hasOutOfBandError) { - outOfBandError = error; - hasOutOfBandError = true; - onQueueNotEmpty(); - } - }).finally(() => { - decoder?.close(); - }); - - const track = this._track; - const closeSamples = () => { - for (const sample of sampleQueue) { - sample?.close(); - } - }; - - return { - async next() { - while (true) { - if (track.input._disposed) { - // Once next() throws, the consumer will never call return(), so terminate the - // iteration here - otherwise, the pump keeps queueing decoded samples that - // nothing can ever close. - terminated = true; - closeSamples(); - throw new InputDisposedError(); - } else if (terminated) { - return { value: undefined, done: true }; - } else if (hasOutOfBandError) { - terminated = true; - closeSamples(); - throw outOfBandError; - } else if (sampleQueue.length > 0) { - const value = sampleQueue.shift(); - assert(value !== undefined); - onQueueDequeue(); - return { value, done: false }; - } else if (!decoderIsFlushed) { - await queueNotEmpty; - } else { - return { value: undefined, done: true }; - } - } - }, - async return() { - terminated = true; - onQueueDequeue(); - onQueueNotEmpty(); - closeSamples(); - - return { value: undefined, done: true }; - }, - async throw(error) { - throw error; - }, - [Symbol.asyncIterator]() { - return this; - }, - }; - } -} - -const computeMaxQueueSize = (decodedSampleQueueSize: number) => { - // If we have decoded samples lying around, limit the total queue size to a small value (decoded samples can use up - // a lot of memory). If not, we're fine with a much bigger queue of encoded packets waiting to be decoded. In fact, - // some decoders only start flushing out decoded chunks when the packet queue is large enough. - return decodedSampleQueueSize === 0 ? 40 : 8; -}; - -class VideoDecoderWrapper extends DecoderWrapper { - decoder: VideoDecoder | null = null; - - customDecoder: CustomVideoDecoder | null = null; - customDecoderCallSerializer = new CallSerializer(); - customDecoderQueueSize = 0; - - inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted. - sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage. - currentPacketIndex = 0; - raslSkipped = false; // For HEVC stuff - - // Alpha stuff - alphaDecoder: VideoDecoder | null = null; - alphaHadKeyframe = false; - colorQueue: VideoFrame[] = []; - alphaQueue: (VideoFrame | null)[] = []; - merger: ColorAlphaMerger | null = null; - decodedAlphaChunkCount = 0; - alphaDecoderQueueSize = 0; - /** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */ - nullAlphaFrameQueue: number[] = []; - currentAlphaPacketIndex = 0; - alphaRaslSkipped = false; // For HEVC stuff - finalSamples: { sample: VideoSample | null }[] = []; - mergeAlphaPromises: Promise[] = []; - - constructor( - onSample: (sample: VideoSample) => unknown, - onError: (error: unknown) => unknown, - public codec: VideoCodec, - public decoderConfig: VideoDecoderConfig, - public rotation: Rotation, - public flip: boolean, - public timeResolution: number, - ) { - super(onSample, onError); - - const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig)); - if (MatchingCustomDecoder) { - // @ts-expect-error "Can't create instance of abstract class 🤓" - this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder; - // @ts-expect-error It's technically readonly - this.customDecoder.codec = codec; - // @ts-expect-error It's technically readonly - this.customDecoder.config = decoderConfig; - // @ts-expect-error It's technically readonly - this.customDecoder.onSample = (sample) => { - if (!(sample instanceof VideoSample)) { - throw new TypeError('The argument passed to onSample must be a VideoSample.'); - } - - this.finalizeAndEmitSample(sample); - }; - // @ts-expect-error It's technically readonly - this.customDecoder.onError = (error) => { - onError(error); - }; - - void this.customDecoderCallSerializer - .call(() => this.customDecoder!.init()) - .catch(error => onError(error)); - } else { - const colorHandler = (frame: VideoFrame) => { - if (this.alphaQueue.length > 0) { - // Even when no alpha data is present (most of the time), there will be nulls in this queue - const alphaFrame = this.alphaQueue.shift(); - assert(alphaFrame !== undefined); - - void this.mergeAlpha(frame, alphaFrame); - } else { - this.colorQueue.push(frame); - } - }; - - if (isChromium()) { - if (codec === 'avc' && this.decoderConfig.description) { - const record = deserializeAvcDecoderConfigurationRecord( - toUint8Array(this.decoderConfig.description), - ); - if (record && record.sequenceParameterSets.length > 0) { - const sps = parseAvcSps(record.sequenceParameterSets[0]!); - if (sps) { - if (sps.frameMbsOnlyFlag === 0) { - // Chromium has/had a bug with playing interlaced AVC - // (https://issues.chromium.org/issues/456919096) which can be worked around by - // requesting that software decoding be used. So, here we peek into the AVC description, - // if present, and switch to software decoding if we find interlaced content. - this.decoderConfig = { - ...this.decoderConfig, - hardwareAcceleration: 'prefer-software', - }; - } - - if (sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) { - // Modify the SPS to fix potential loss of B frames - record.sequenceParameterSets[0] = addAvcBitstreamRestriction(sps); - this.decoderConfig = { - ...this.decoderConfig, - description: serializeAvcDecoderConfigurationRecord(record), - }; - } - } - } - } - - if (!colorSpaceIsComplete(this.decoderConfig.colorSpace)) { - // Found via https://github.com/remotion-dev/remotion/issues/10841. - // If the color space is incomplete (which is often that it's just partially filled), Chromium has - // some nasty logic where it doesn't pass that information along to the GPU at all. The result is - // that information is genuinely lost, like the color matrix for example. Chromium has other code - // paths where it just fills the missing values with a hardcoded default, so we do the exact same - // thing here, with the same hardcoded defaults: - this.decoderConfig = { - ...this.decoderConfig, - colorSpace: { - primaries: this.decoderConfig.colorSpace?.primaries ?? 'bt709', - matrix: this.decoderConfig.colorSpace?.matrix ?? 'bt709', - transfer: this.decoderConfig.colorSpace?.transfer ?? 'bt709', - fullRange: this.decoderConfig.colorSpace?.fullRange ?? false, - }, - }; - } - } - - const stack = new Error('Decoding error').stack; - - this.decoder = new VideoDecoder({ - output: (frame) => { - try { - colorHandler(frame); - } catch (error) { - this.onError(error); - } - }, - error: (error) => { - error.stack = stack; // Provide a more useful stack trace, the default one sucks - this.onError(error); - }, - }); - this.decoder.configure(this.decoderConfig); - } - } - - getDecodeQueueSize() { - if (this.customDecoder) { - return this.customDecoderQueueSize; - } else { - assert(this.decoder); - - return Math.max( - this.decoder.decodeQueueSize, - this.alphaDecoder?.decodeQueueSize ?? 0, - ); - } - } - - decode(packet: EncodedPacket) { - if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) { - if (this.hasHevcRaslPicture(packet.data)) { - return; // Drop - } - - this.raslSkipped = true; - } - - if (this.customDecoder) { - this.customDecoderQueueSize++; - void this.customDecoderCallSerializer - .call(() => this.customDecoder!.decode(packet)) - .catch(error => this.onError(error)) - .finally(() => this.customDecoderQueueSize--); - } else { - assert(this.decoder); - - if (!isWebKit()) { - insertSorted(this.inputTimestamps, packet.timestamp, x => x); - } - - if (isChromium() && this.currentPacketIndex === 0) { - if (this.codec === 'avc') { - // Workaround for https://issues.chromium.org/issues/470109459 - const filteredNalUnits: Uint8Array[] = []; - let hasFrameData = false; - - for (const loc of iterateAvcNalUnits(packet.data, this.decoderConfig)) { - const type = extractNalUnitTypeForAvc(packet.data[loc.offset]!); - hasFrameData ||= type >= 1 && type <= 5; - - if (type === AvcNalUnitType.AUD) { - if (hasFrameData) { - // Already has actual frame data, so treat an AUD as simply the end of the packet - break; - } else { - // If packets contain an AUD and have NALUs before it, this trips up Chromium's key - // frame detector. Clear the NALUs if an AUD is encountered. - // https://github.com/Vanilagy/mediabunny/issues/396 - filteredNalUnits.length = 0; - } - } - - // These trip up Chromium's key frame detection, so let's strip them - if (!(type >= 20 && type <= 31)) { - filteredNalUnits.push(packet.data.subarray(loc.offset, loc.offset + loc.length)); - } - } - - if (!this.decoderConfig.description) { - // Do SPS fixups if necessary - for (let i = 0; i < filteredNalUnits.length; i++) { - const nalUnit = filteredNalUnits[i]!; - if (extractNalUnitTypeForAvc(nalUnit[0]!) !== AvcNalUnitType.SPS) { - continue; - } - - const sps = parseAvcSps(nalUnit); - if (sps && sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) { - filteredNalUnits[i] = addAvcBitstreamRestriction(sps); - } - - break; - } - } - - const newData = concatAvcNalUnits(filteredNalUnits, this.decoderConfig); - packet = new EncodedPacket(newData, packet.type, packet.timestamp, packet.duration); - } else if (this.codec === 'hevc') { - // Workaround for https://issues.chromium.org/issues/507611247 - const sanitizedData = sanitizeHevcPacketForChromium(packet.data, this.decoderConfig); - if (sanitizedData) { - packet = new EncodedPacket(sanitizedData, packet.type, packet.timestamp, packet.duration); - } - } - } - - this.decoder.decode(packet.toEncodedVideoChunk()); - this.decodeAlphaData(packet); - } - - this.currentPacketIndex++; - } - - decodeAlphaData(packet: EncodedPacket) { - if (!packet.sideData.alpha) { - // No alpha side data in the packet, most common case - this.pushNullAlphaFrame(); - return; - } - - if (!this.merger) { - this.merger = new ColorAlphaMerger(); - } - - // Check if we need to set up the alpha decoder - if (!this.alphaDecoder) { - const alphaHandler = (frame: VideoFrame) => { - if (this.colorQueue.length > 0) { - const colorFrame = this.colorQueue.shift(); - assert(colorFrame !== undefined); - - void this.mergeAlpha(colorFrame, frame); - } else { - this.alphaQueue.push(frame); - } - - // Check if any null frames have been queued for this point - this.decodedAlphaChunkCount++; - while ( - this.nullAlphaFrameQueue.length > 0 - && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount - ) { - this.nullAlphaFrameQueue.shift(); - - if (this.colorQueue.length > 0) { - const colorFrame = this.colorQueue.shift(); - assert(colorFrame !== undefined); - - void this.mergeAlpha(colorFrame, null); - } else { - this.alphaQueue.push(null); - } - } - - this.alphaDecoderQueueSize--; - }; - - const stack = new Error('Decoding error').stack; - - this.alphaDecoder = new VideoDecoder({ - output: (frame) => { - try { - alphaHandler(frame); - } catch (error) { - this.onError(error); - } - }, - error: (error) => { - error.stack = stack; // Provide a more useful stack trace, the default one sucks - this.onError(error); - }, - }); - this.alphaDecoder.configure({ - ...this.decoderConfig, - // Alpha is always full range, regardless of what the color track says. The decoder only honors the - // override if all fields are set tho. - colorSpace: { - fullRange: true, - // These fields are irrelevant: - matrix: 'bt709', - primaries: 'bt709', - transfer: 'bt709', - }, - }); - } - - const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha); - - // Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start - // decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque. - if (!this.alphaHadKeyframe) { - this.alphaHadKeyframe = type === 'key'; - } - - if (this.alphaHadKeyframe) { - // Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but - // here for symmetry. - if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) { - if (this.hasHevcRaslPicture(packet.sideData.alpha)) { - this.pushNullAlphaFrame(); - return; - } - - this.alphaRaslSkipped = true; - } - - this.currentAlphaPacketIndex++; - this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type)); - this.alphaDecoderQueueSize++; - } else { - this.pushNullAlphaFrame(); - } - } - - pushNullAlphaFrame() { - if (this.alphaDecoderQueueSize === 0) { - // Easy - this.alphaQueue.push(null); - } else { - // There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order - // data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload - // has finished. - this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize); - } - } - - /** - * If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as - * CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave - * differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good - * and causes bugs upstream. So, let's take the dropping into our own hands. - */ - hasHevcRaslPicture(packetData: Uint8Array) { - for (const loc of iterateHevcNalUnits(packetData, this.decoderConfig)) { - const type = extractNalUnitTypeForHevc(packetData[loc.offset]!); - if (type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R) { - return true; - } - } - - return false; - } - - /** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */ - sampleHandler(sample: VideoSample) { - if (isWebKit()) { - // For correct B-frame handling, we don't just hand over the frames directly but instead add them to - // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue - // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we - // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are - // emitted in presentation order, but Safari doesn't always follow this rule. - if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) { - for (const sample of this.sampleQueue) { - this.finalizeAndEmitSample(sample); - } - - this.sampleQueue.length = 0; - } - - insertSorted(this.sampleQueue, sample, x => x.timestamp); - } else { - // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are - // required to emit decoded frames in presentation order *while* retaining the timestamp of their - // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a - // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and - // ensure they are sorted by doing this. - const timestamp = this.inputTimestamps.shift(); - - // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the - // correspondence should be 1:1. - assert(timestamp !== undefined); - - sample.setTimestamp(timestamp); - this.finalizeAndEmitSample(sample); - } - } - - finalizeAndEmitSample(sample: VideoSample) { - // Round the timestamps to the time resolution - sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution); - sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution); - sample.setRotation(this.rotation); - sample.setFlip(this.flip); - - this.onSample(sample); - } - - async mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) { - const resolver = promiseWithResolvers(); - this.mergeAlphaPromises.push(resolver.promise); - - // Alpha merging is concurrent but the samples must still be emitted in the same order in which the merging - // began. Therefore, serialize the results in an array. - const result: { sample: VideoSample | null } = { sample: null }; - this.finalSamples.push(result); - - try { - if (!alpha) { - // Nothing needs to be merged - result.sample = new VideoSample(color); - } else { - assert(this.merger); - - // The merger takes ownership of the frames, so no need to close them ourselves - const finalFrame = await this.merger.merge(color, alpha); - result.sample = new VideoSample(finalFrame); - } - - // Emit any leading samples that are ready, preserving input order - while (this.finalSamples.length > 0 && this.finalSamples[0]!.sample !== null) { - const next = this.finalSamples.shift()!; - this.sampleHandler(next.sample!); - } - } catch (error) { - removeItem(this.finalSamples, result); - this.onError(error); - } finally { - removeItem(this.mergeAlphaPromises, resolver.promise); - resolver.resolve(); - } - } - - async flush() { - if (this.customDecoder) { - await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush()); - } else { - assert(this.decoder); - await Promise.all([ - this.decoder.flush(), - this.alphaDecoder?.flush(), - ]); - await Promise.all(this.mergeAlphaPromises); - - this.colorQueue.forEach(x => x.close()); - this.colorQueue.length = 0; - this.alphaQueue.forEach(x => x?.close()); - this.alphaQueue.length = 0; - - this.alphaHadKeyframe = false; - this.decodedAlphaChunkCount = 0; - this.alphaDecoderQueueSize = 0; - this.nullAlphaFrameQueue.length = 0; - this.currentAlphaPacketIndex = 0; - this.alphaRaslSkipped = false; - } - - if (isWebKit()) { - for (const sample of this.sampleQueue) { - this.finalizeAndEmitSample(sample); - } - - this.sampleQueue.length = 0; - } - - this.currentPacketIndex = 0; - this.raslSkipped = false; - } - - close() { - if (this.customDecoder) { - void this.customDecoderCallSerializer.call(() => this.customDecoder!.close()); - } else { - assert(this.decoder); - - if (this.decoder.state !== 'closed') { - this.decoder.close(); - } - if (this.alphaDecoder && this.alphaDecoder.state !== 'closed') { - this.alphaDecoder.close(); - } - - this.colorQueue.forEach(x => x.close()); - this.colorQueue.length = 0; - this.alphaQueue.forEach(x => x?.close()); - this.alphaQueue.length = 0; - - this.merger?.close(); - } - - for (const sample of this.sampleQueue) { - sample.close(); - } - this.sampleQueue.length = 0; - } -} - -let mergerWorkerUrl: string | null = null; - -/** Utility class that merges together color and alpha information on the CPU in a pool of workers. */ -export class ColorAlphaMerger { - private workers: Worker[] = []; - private nextWorkerIndex = 0; - private pendingRequests = new Map>>(); - private nextRequestId = 0; - - merge(color: VideoFrame, alpha: VideoFrame): Promise { - if (this.workers.length === 0) { - if (!mergerWorkerUrl) { - const blob = new Blob( - [`(${colorAlphaMergerWorkerCode.toString()})()`], - { type: 'application/javascript' }, - ); - mergerWorkerUrl = URL.createObjectURL(blob); - } - - const poolSize = clamp(navigator.hardwareConcurrency, 1, 4); - for (let i = 0; i < poolSize; i++) { - const worker = new Worker(mergerWorkerUrl); - - worker.addEventListener('message', (event: MessageEvent) => { - const data = event.data; - const pending = this.pendingRequests.get(data.id); - if (!pending) { - return; - } - this.pendingRequests.delete(data.id); - - if ('error' in data) { - pending.reject(new Error(data.error)); - } else { - pending.resolve(data.frame); - } - }); - - worker.addEventListener('error', (event) => { - const error = new Error(event.message || 'Color/alpha merge worker error.'); - for (const pending of this.pendingRequests.values()) { - pending.reject(error); - } - this.pendingRequests.clear(); - }); - - this.workers.push(worker); - } - } - - const id = this.nextRequestId++; - const pending = promiseWithResolvers(); - this.pendingRequests.set(id, pending); - - // Hand the job to the next worker in round-robin fashion - const worker = this.workers[this.nextWorkerIndex]!; - this.nextWorkerIndex = (this.nextWorkerIndex + 1) % this.workers.length; - worker.postMessage({ id, color, alpha }, { transfer: [color, alpha] }); - - return pending.promise; - } - - close() { - for (const worker of this.workers) { - worker.terminate(); - } - this.workers.length = 0; - - const error = new Error('Color/alpha merger closed.'); - for (const pending of this.pendingRequests.values()) { - pending.reject(error); - } - this.pendingRequests.clear(); - } -} - -type ColorAlphaMergerWorkerRequest = { - id: number; - color: VideoFrame; - alpha: VideoFrame; -}; - -type ColorAlphaMergerWorkerResponse = - | { id: number; frame: VideoFrame } - | { id: number; error: string }; - -const colorAlphaMergerWorkerCode = () => { - // These buffers are reused across frames as long as the size matches, since consecutive frames usually share - // dimensions - let cpuAlphaBuffer: Uint8Array | null = null; - let cpuColorBuffer: Uint8Array | null = null; - - // Serialize execution internally so concurrent requests don't race on the shared cpu*Buffer state. - let chain: Promise = Promise.resolve(); - self.addEventListener('message', (event: MessageEvent) => { - const { id, color, alpha } = event.data; - chain = chain.then(async () => { - try { - const frame = await merge(color, alpha); - self.postMessage({ id, frame }, { transfer: [frame] }); - } catch (error) { - self.postMessage({ id, error: (error as Error).message }); - } finally { - // We took ownership of the inputs via transfer; close them now that the merge (or its error) is done. - color.close(); - alpha.close(); - } - }); - }); - - const merge = async (color: VideoFrame, alpha: VideoFrame): Promise => { - const format = color.format as VideoSamplePixelFormat | null; - const alphaFormat = alpha.format as VideoSamplePixelFormat | null; - if (!format || !alphaFormat) { - throw new Error('CPU color/alpha merging requires a known VideoFrame format.'); - } - - // The alpha frame must have the same bit depth as the color frame - const colorIs10 = format.includes('P10'); - const colorIs12 = format.includes('P12'); - const alphaIs10 = alphaFormat.includes('P10'); - const alphaIs12 = alphaFormat.includes('P12'); - if (alphaIs10 !== colorIs10 || alphaIs12 !== colorIs12) { - throw new Error( - `CPU color/alpha merging requires the alpha frame to have the same bit depth as the color frame` - + ` (color: '${format}', alpha: '${alphaFormat}').`, - ); - } - - if (format === 'RGBX' || format === 'RGBA' || format === 'BGRX' || format === 'BGRA') { - return await mergeInterleavedRgba(color, alpha, format); - } else if ( - format === 'I420' || format === 'I420P10' || format === 'I420P12' - || format === 'I422' || format === 'I422P10' || format === 'I422P12' - || format === 'I444' || format === 'I444P10' || format === 'I444P12' - ) { - return await mergePlanarYuv(color, alpha, format); - } else if (format === 'NV12') { - return await mergeNv12(color, alpha); - } - - throw new Error(`CPU color/alpha merging does not support format '${format}'.`); - }; - - const mergeInterleavedRgba = async ( - color: VideoFrame, - alpha: VideoFrame, - format: 'RGBX' | 'RGBA' | 'BGRX' | 'BGRA', - ): Promise => { - const width = color.visibleRect?.width ?? color.codedWidth; - const height = color.visibleRect?.height ?? color.codedHeight; - - const pixelCount = width * height; - const output = new Uint8Array(pixelCount * 4); - - // Color goes straight into the output buffer via copyTo, no intermediate copy needed - await color.copyTo(output); - - // And now add the alpha data - const alphaY = await readAlpha(alpha, width, height, 1); - for (let i = 0, j = 3; i < pixelCount; i++, j += 4) { - output[j] = alphaY[i]!; - } - - const outputFormat = (format === 'RGBX' || format === 'RGBA') ? 'RGBA' : 'BGRA'; - const init = { - format: outputFormat, - codedWidth: width, - codedHeight: height, - timestamp: color.timestamp, - duration: color.duration ?? undefined, - transfer: [output.buffer], - } as const; - - return new VideoFrame(output, init); - }; - - const mergePlanarYuv = async ( - color: VideoFrame, - alpha: VideoFrame, - format: - | 'I420' | 'I420P10' | 'I420P12' - | 'I422' | 'I422P10' | 'I422P12' - | 'I444' | 'I444P10' | 'I444P12', - ): Promise => { - const width = color.visibleRect?.width ?? color.codedWidth; - const height = color.visibleRect?.height ?? color.codedHeight; - - const is10 = format.includes('P10'); - const is12 = format.includes('P12'); - const bytesPerSample = (is10 || is12) ? 2 : 1; - - let chromaW: number; - let chromaH: number; - if (format.startsWith('I420')) { - chromaW = Math.ceil(width / 2); - chromaH = Math.ceil(height / 2); - } else if (format.startsWith('I422')) { - chromaW = Math.ceil(width / 2); - chromaH = height; - } else { - chromaW = width; - chromaH = height; - } - - const ySamples = width * height; - const uvSamples = chromaW * chromaH; - const yBytes = ySamples * bytesPerSample; - const uvBytes = uvSamples * bytesPerSample; - const aBytes = ySamples * bytesPerSample; - - const outputBytes = yBytes + 2 * uvBytes + aBytes; - const output = new Uint8Array(outputBytes); - - // Write color planes directly into the output buffer via copyTo, no intermediate copy - await color.copyTo(output); - - const alphaY = await readAlpha(alpha, width, height, bytesPerSample); - const aOffset = yBytes + 2 * uvBytes; - output.set(alphaY, aOffset); - - const outputFormat = (format.slice(0, 4) + 'A' + format.slice(4)) as VideoPixelFormat; - - const init = { - format: outputFormat, - codedWidth: width, - codedHeight: height, - timestamp: color.timestamp, - duration: color.duration ?? undefined, - transfer: [output.buffer], - }; - - return new VideoFrame(output, init); - }; - - const mergeNv12 = async ( - color: VideoFrame, - alpha: VideoFrame, - ): Promise => { - const width = color.visibleRect?.width ?? color.codedWidth; - const height = color.visibleRect?.height ?? color.codedHeight; - - const ySize = width * height; - const chromaW = Math.ceil(width / 2); - const chromaH = Math.ceil(height / 2); - const uvSize = chromaW * chromaH; - - const sourceSize = color.allocationSize(); - if (!cpuColorBuffer || cpuColorBuffer.byteLength !== sourceSize) { - cpuColorBuffer = new Uint8Array(sourceSize); - } - await color.copyTo(cpuColorBuffer); - - const output = new Uint8Array(ySize + 2 * uvSize + ySize); - - // Y plane copies straight over - output.set(cpuColorBuffer.subarray(0, ySize), 0); - - // Deinterleave the UV plane into separate U and V planes - const uOffset = ySize; - const vOffset = ySize + uvSize; - const uvStart = ySize; - for (let i = 0; i < uvSize; i++) { - output[uOffset + i] = cpuColorBuffer[uvStart + i * 2]!; - output[vOffset + i] = cpuColorBuffer[uvStart + i * 2 + 1]!; - } - - const alphaY = await readAlpha(alpha, width, height, 1); - output.set(alphaY, ySize + 2 * uvSize); - - const init = { - format: 'I420A', - codedWidth: width, - codedHeight: height, - timestamp: color.timestamp, - duration: color.duration ?? undefined, - transfer: [output.buffer], - } as const; - - return new VideoFrame(output, init); - }; - - const readAlpha = async (alpha: VideoFrame, width: number, height: number, bytesPerSample: number) => { - const size = alpha.allocationSize(); - if (!cpuAlphaBuffer || cpuAlphaBuffer.byteLength !== size) { - cpuAlphaBuffer = new Uint8Array(size); - } - await alpha.copyTo(cpuAlphaBuffer); - - const format = alpha.format; - if (format === 'RGBA' || format === 'BGRA' || format === 'RGBX' || format === 'BGRX') { - // Pack alpha data tightly. Assume alpha is stored in RGB, so sample just from R for simplicity. - const rOffset = (format === 'RGBA' || format === 'RGBX') ? 0 : 2; - const pixelCount = width * height; - for (let i = 0; i < pixelCount; i++) { - cpuAlphaBuffer[i] = cpuAlphaBuffer[i * 4 + rOffset]!; - } - return cpuAlphaBuffer.subarray(0, pixelCount); - } else { - // For Y-plane-first formats (I*** and NV12), the leading width*height samples are the Y plane - const yPlane = cpuAlphaBuffer.subarray(0, width * height * bytesPerSample); - - if (alpha.colorSpace.fullRange === false) { - // Some decoders hand us limited-range alpha, so stretch it back out to full range - const bitDepth = format!.includes('P12') ? 12 : format!.includes('P10') ? 10 : 8; - const low = 16 << (bitDepth - 8); - const high = 235 << (bitDepth - 8); - const max = (1 << bitDepth) - 1; - const samples = bytesPerSample === 2 - ? new Uint16Array(yPlane.buffer, 0, width * height) - : yPlane; - - for (let i = 0; i < samples.length; i++) { - const value = Math.round((samples[i]! - low) * max / (high - low)); - samples[i] = Math.min(Math.max(value, 0), max); - } - } - - return yPlane; - } - }; -}; - -/** - * Describes additional decoder preferences for video sinks. - * @group Media sinks - * @public - */ -export type VideoSinkDecoderOptions = { - /** - * A hint that configures the hardware acceleration method of the decoder. This is best left on `'no-preference'`, - * the default. - */ - hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software'; - /** - * Hint that the selected decoder should be configured to minimize the number of packets that have to be decoded - * before video frames are output. - */ - optimizeForLatency?: boolean; -}; - -const validateVideoSinkDecoderOptions = (decoderOptions: VideoSinkDecoderOptions) => { - if (!decoderOptions || typeof decoderOptions !== 'object') { - throw new TypeError('decoderOptions must be an object.'); - } - if ( - decoderOptions.hardwareAcceleration !== undefined - && !['no-preference', 'prefer-hardware', 'prefer-software'].includes(decoderOptions.hardwareAcceleration) - ) { - throw new TypeError( - 'decoderOptions.hardwareAcceleration, when provided, must be \'no-preference\', \'prefer-hardware\' or' - + ' \'prefer-software\'.', - ); - } - if (decoderOptions.optimizeForLatency !== undefined && typeof decoderOptions.optimizeForLatency !== 'boolean') { - throw new TypeError('decoderOptions.optimizeForLatency, when provided, must be a boolean.'); - } -}; - -/** - * A sink that retrieves decoded video samples (video frames) from a video track. - * @group Media sinks - * @public - */ -export class VideoSampleSink extends BaseMediaSampleSink { - /** @internal */ - _track: InputVideoTrack; - /** @internal */ - _decoderOptions: VideoSinkDecoderOptions; - - /** Creates a new {@link VideoSampleSink} for the given {@link InputVideoTrack}. */ - constructor(videoTrack: InputVideoTrack, decoderOptions: VideoSinkDecoderOptions = {}) { - if (!(videoTrack instanceof InputVideoTrack)) { - throw new TypeError('videoTrack must be an InputVideoTrack.'); - } - validateVideoSinkDecoderOptions(decoderOptions); - - super(); - - this._track = videoTrack; - this._decoderOptions = decoderOptions; - } - - /** @internal */ - async _createDecoder( - onSample: (sample: VideoSample) => unknown, - onError: (error: unknown) => unknown, - ) { - if (!(await this._track.canDecode())) { - if (typeof VideoDecoder === 'undefined') { - throw new Error(missingWebCodecsClassMessage('VideoDecoder')); - } - - throw new Error( - 'This video track cannot be decoded in this environment. Make sure to check decodability before using' - + ' a track.', - ); - } - - const codec = await this._track.getCodec(); - const rotation = await this._track.getRotation(); - const flip = await this._track.getFlip(); - let decoderConfig = await this._track.getDecoderConfig(); - const timeResolution = await this._track.getTimeResolution(); - assert(codec && decoderConfig); - - decoderConfig = { - ...decoderConfig, - hardwareAcceleration: this._decoderOptions.hardwareAcceleration, - optimizeForLatency: this._decoderOptions.optimizeForLatency, - }; - - return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, flip, timeResolution); - } - - /** @internal */ - _createPacketSink() { - return new EncodedPacketSink(this._track); - } - - /** - * Retrieves the video sample (frame) corresponding to the given timestamp, in seconds. More specifically, returns - * the last video sample (in presentation order) with a start timestamp less than or equal to the given timestamp. - * Returns null if the timestamp is before the track's first timestamp. - * - * @param timestamp - The timestamp used for retrieval, in seconds. - * @param options - Options used for the underlying packet retrieval. - */ - async getSample(timestamp: number, options: PacketRetrievalOptions = {}) { - validateTimestamp(timestamp); - - for await (const sample of this.mediaSamplesAtTimestamps([timestamp], options)) { - return sample; - } - throw new Error('Internal error: Iterator returned nothing.'); - } - - /** - * Creates an async iterator that yields the video samples (frames) of this track in presentation order. This method - * will intelligently pre-decode a few frames ahead to enable fast iteration. - * - * @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive). - * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive). - * @param options - Options used for the underlying packet retrieval. - */ - samples(startTimestamp?: number, endTimestamp?: number, options: PacketRetrievalOptions = {}) { - return this.mediaSamplesInRange(startTimestamp, endTimestamp, options); - } - - /** - * Creates an async iterator that yields a video sample (frame) for each timestamp in the argument. This method - * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most - * once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may - * yield null if no frame is available for a given timestamp. - * - * This method is good for sparse access of media data. If you want primarily sequential media access, prefer - * {@link VideoSampleSink.samples} instead. - * - * @param timestamps - An iterable or async iterable of timestamps in seconds. - * @param options - Options used for the underlying packet retrieval. - */ - samplesAtTimestamps(timestamps: AnyIterable, options: PacketRetrievalOptions = {}) { - return this.mediaSamplesAtTimestamps(timestamps, options); - } -} - -/** - * A canvas with additional timing information (timestamp & duration). - * @group Media sinks - * @public - */ -export type WrappedCanvas = { - /** A canvas element or offscreen canvas. */ - canvas: HTMLCanvasElement | OffscreenCanvas; - /** The timestamp of the corresponding video sample, in seconds. */ - timestamp: number; - /** The duration of the corresponding video sample, in seconds. */ - duration: number; -}; - -/** - * Options for constructing a CanvasSink. - * @group Media sinks - * @public - */ -export type CanvasSinkOptions = { - /** - * Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set - * this to `true` when using this sink to read transparent videos. - */ - alpha?: boolean; - /** - * The width of the output canvas in pixels, defaulting to the display width of the video track. If height is not - * set, it will be deduced automatically based on aspect ratio. - */ - width?: number; - /** - * The height of the output canvas in pixels, defaulting to the display height of the video track. If width is not - * set, it will be deduced automatically based on aspect ratio. - */ - height?: number; - /** - * The fitting algorithm in case both width and height are set. - * - * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio. - * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to - * letterboxing. - * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. - */ - fit?: 'fill' | 'contain' | 'cover'; - /** - * The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata. - * Rotation is applied before flipping. - */ - rotation?: Rotation; - /** - * Whether to flip the raw video frame horizontally (about the vertical axis). Defaults to the flip set in the file - * metadata. The flip is applied after rotation but before cropping and resizing. - */ - flip?: boolean; - /** - * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to - * the dimensions of the input video track. Cropping is performed after rotation and flip but before resizing. The - * crop region is in the _display pixel space_ of the underlying video data. - */ - crop?: CropRectangle; - /** - * When set, specifies the number of canvases in the pool. These canvases will be reused in a ring buffer / - * round-robin type fashion. This keeps the amount of allocated VRAM constant and relieves the browser from - * constantly allocating/deallocating canvases. A pool size of 0 or `undefined` disables the pool and means a new - * canvas is created each time. - */ - poolSize?: number; - /** Additional preferences for the underlying video decoder. */ - decoderOptions?: VideoSinkDecoderOptions; -}; - -/** - * A sink that renders video samples (frames) of the given video track to canvases. This is often more useful than - * directly retrieving frames, as it comes with common preprocessing steps such as resizing or applying rotation and - * flip metadata. - * - * This sink will yield `HTMLCanvasElement`s when in a DOM context, and `OffscreenCanvas`es otherwise. - * - * @group Media sinks - * @public - */ -export class CanvasSink { - /** @internal */ - _videoTrack: InputVideoTrack; - /** @internal */ - _alpha: boolean; - /** @internal */ - _width!: number; - /** @internal */ - _height!: number; - /** @internal */ - _options: CanvasSinkOptions; - /** @internal */ - _fit: 'fill' | 'contain' | 'cover'; - /** @internal */ - _rotation: Rotation = 0; - /** @internal */ - _flip = false; - /** @internal */ - _crop?: { left: number; top: number; width: number; height: number }; - /** @internal */ - _initPromise: Promise | null = null; - /** @internal */ - _videoSampleSink: VideoSampleSink; - /** @internal */ - _canvasPool: (HTMLCanvasElement | OffscreenCanvas | null)[]; - /** @internal */ - _nextCanvasIndex = 0; - - /** Creates a new {@link CanvasSink} for the given {@link InputVideoTrack}. */ - constructor(videoTrack: InputVideoTrack, options: CanvasSinkOptions = {}) { - if (!(videoTrack instanceof InputVideoTrack)) { - throw new TypeError('videoTrack must be an InputVideoTrack.'); - } - if (options && typeof options !== 'object') { - throw new TypeError('options must be an object.'); - } - if (options.alpha !== undefined && typeof options.alpha !== 'boolean') { - throw new TypeError('options.alpha, when provided, must be a boolean.'); - } - if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) { - throw new TypeError('options.width, when defined, must be a positive integer.'); - } - if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) { - throw new TypeError('options.height, when defined, must be a positive integer.'); - } - if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) { - throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".'); - } - if ( - options.width !== undefined - && options.height !== undefined - && options.fit === undefined - ) { - throw new TypeError( - 'When both options.width and options.height are provided, options.fit must also be provided.', - ); - } - if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) { - throw new TypeError('options.rotation, when provided, must be 0, 90, 180 or 270.'); - } - if (options.flip !== undefined && typeof options.flip !== 'boolean') { - throw new TypeError('options.flip, when provided, must be a boolean.'); - } - if (options.crop !== undefined) { - validateCropRectangle(options.crop, 'options.'); - } - if ( - options.poolSize !== undefined - && (typeof options.poolSize !== 'number' || !Number.isInteger(options.poolSize) || options.poolSize < 0) - ) { - throw new TypeError('poolSize must be a non-negative integer.'); - } - if (options.decoderOptions !== undefined) { - validateVideoSinkDecoderOptions(options.decoderOptions); - } - - this._videoTrack = videoTrack; - this._alpha = options.alpha ?? false; - this._options = options; - this._fit = options.fit ?? 'fill'; - this._videoSampleSink = new VideoSampleSink(videoTrack, options.decoderOptions); - this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null); - } - - /** @internal */ - _ensureInit() { - return this._initPromise ??= (async () => { - const options = this._options; - const videoTrack = this._videoTrack; - - const rotation = options.rotation ?? await videoTrack.getRotation(); - const flip = options.flip ?? await videoTrack.getFlip(); - const squarePixelWidth = await videoTrack.getSquarePixelWidth(); - const squarePixelHeight = await videoTrack.getSquarePixelHeight(); - - const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 - ? [squarePixelWidth, squarePixelHeight] - : [squarePixelHeight, squarePixelWidth]; - - let crop = options.crop; - if (crop) { - crop = clampCropRectangle(crop, rotatedWidth, rotatedHeight); - } - - let [width, height] = crop - ? [crop.width, crop.height] - : [rotatedWidth, rotatedHeight]; - const originalAspectRatio = width / height; - - // If width and height aren't defined together, deduce the missing value using the aspect ratio - if (options.width !== undefined && options.height === undefined) { - width = options.width; - height = Math.round(width / originalAspectRatio); - } else if (options.width === undefined && options.height !== undefined) { - height = options.height; - width = Math.round(height * originalAspectRatio); - } else if (options.width !== undefined && options.height !== undefined) { - width = options.width; - height = options.height; - } - - this._width = width; - this._height = height; - this._rotation = rotation; - this._flip = flip; - this._crop = crop; - })(); - } - - /** @internal */ - _videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas { - const width = this._width; - const height = this._height; - let canvas = this._canvasPool[this._nextCanvasIndex]; - let canvasIsNew = false; - - if (!canvas) { - if (typeof document !== 'undefined') { - // Prefer an HTMLCanvasElement - canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - } else { - canvas = new OffscreenCanvas(width, height); - } - - if (this._canvasPool.length > 0) { - this._canvasPool[this._nextCanvasIndex] = canvas; - } - - canvasIsNew = true; - } - - if (this._canvasPool.length > 0) { - this._nextCanvasIndex = (this._nextCanvasIndex + 1) % this._canvasPool.length; - } - - const context = canvas.getContext('2d', { - alpha: this._alpha || isFirefox(), // Firefox has VideoFrame glitches with opaque canvases - }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; - assert(context); - - sample._drawWithFitAndMipmapping(canvas, context, { - fit: this._fit, - rotation: this._rotation, - flip: this._flip, - crop: this._crop, - targetIsFresh: canvasIsNew, - fillBlack: !this._alpha && isFirefox(), - }); - - const result = { - canvas, - timestamp: sample.timestamp, - duration: sample.duration, - }; - - sample.close(); - return result; - } - - /** - * Retrieves a canvas with the video frame corresponding to the given timestamp, in seconds. More specifically, - * returns the last video frame (in presentation order) with a start timestamp less than or equal to the given - * timestamp. Returns null if the timestamp is before the track's first timestamp. - * - * @param timestamp - The timestamp used for retrieval, in seconds. - * @param options - Options used for the underlying packet retrieval. - */ - async getCanvas(timestamp: number, options?: PacketRetrievalOptions) { - validateTimestamp(timestamp); - await this._ensureInit(); - - const sample = await this._videoSampleSink.getSample(timestamp, options); - return sample && this._videoSampleToWrappedCanvas(sample); - } - - /** - * Creates an async iterator that yields canvases with the video frames of this track in presentation order. This - * method will intelligently pre-decode a few frames ahead to enable fast iteration. - * - * @param startTimestamp - The timestamp in seconds at which to start yielding canvases (inclusive). - * @param endTimestamp - The timestamp in seconds at which to stop yielding canvases (exclusive). - * @param options - Options used for the underlying packet retrieval. - */ - async* canvases(startTimestamp?: number, endTimestamp?: number, options?: PacketRetrievalOptions) { - await this._ensureInit(); - yield* mapAsyncGenerator( - this._videoSampleSink.samples(startTimestamp, endTimestamp, options), - sample => this._videoSampleToWrappedCanvas(sample), - ); - } - - /** - * Creates an async iterator that yields a canvas for each timestamp in the argument. This method uses an optimized - * decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most once, and is - * therefore more efficient than manually getting the canvas for every timestamp. The iterator may yield null if - * no frame is available for a given timestamp. - * - * This method is good for sparse access of media data. If you want primarily sequential media access, prefer - * {@link CanvasSink.canvases} instead. - * - * @param timestamps - An iterable or async iterable of timestamps in seconds. - * @param options - Options used for the underlying packet retrieval. - */ - async* canvasesAtTimestamps(timestamps: AnyIterable, options?: PacketRetrievalOptions) { - await this._ensureInit(); - yield* mapAsyncGenerator( - this._videoSampleSink.samplesAtTimestamps(timestamps, options), - sample => sample && this._videoSampleToWrappedCanvas(sample), - ); - } -} - -class AudioDecoderWrapper extends DecoderWrapper { - decoder: AudioDecoder | null = null; - - customDecoder: CustomAudioDecoder | null = null; - customDecoderCallSerializer = new CallSerializer(); - customDecoderQueueSize = 0; - - // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially - // inaccurate) packet timestamps. - currentTimestamp: number | null = null; - // Chromium does not respect negative packet timestamps, so we must do the fixin' ourselves - expectedFirstTimestamp: number | null = null; - timestampOffset = 0; - - constructor( - onSample: (sample: AudioSample) => unknown, - onError: (error: unknown) => unknown, - codec: AudioCodec, - decoderConfig: AudioDecoderConfig, - ) { - super(onSample, onError); - - const sampleHandler = (sample: AudioSample) => { - let sampleTimestamp = sample.timestamp; - - if (this.expectedFirstTimestamp !== null && this.currentTimestamp === null) { - this.timestampOffset = this.expectedFirstTimestamp - sampleTimestamp; ; - } - - sampleTimestamp += this.timestampOffset; - - if ( - this.currentTimestamp === null - || Math.abs(sampleTimestamp - this.currentTimestamp) >= sample.duration - ) { - // We need to sync with the sample timestamp again - this.currentTimestamp = sampleTimestamp; - } - - const preciseTimestamp = this.currentTimestamp; - this.currentTimestamp += sample.duration; - - if (sample.numberOfFrames === 0) { - // We skip zero-data (empty) AudioSamples. These are sometimes emitted, for example, by Firefox when it - // decodes Vorbis (at the start). - sample.close(); - return; - } - - // Round the timestamp to the sample rate - const sampleRate = decoderConfig.sampleRate; - sample.setTimestamp(Math.round(preciseTimestamp * sampleRate) / sampleRate); - - onSample(sample); - }; - - const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig)); - if (MatchingCustomDecoder) { - // @ts-expect-error "Can't create instance of abstract class 🤓" - this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder; - // @ts-expect-error It's technically readonly - this.customDecoder.codec = codec; - // @ts-expect-error It's technically readonly - this.customDecoder.config = decoderConfig; - // @ts-expect-error It's technically readonly - this.customDecoder.onSample = (sample) => { - if (!(sample instanceof AudioSample)) { - throw new TypeError('The argument passed to onSample must be an AudioSample.'); - } - - sampleHandler(sample); - }; - // @ts-expect-error It's technically readonly - this.customDecoder.onError = (error) => { - onError(error); - }; - - void this.customDecoderCallSerializer - .call(() => this.customDecoder!.init()) - .catch(error => onError(error)); - } else { - const stack = new Error('Decoding error').stack; - - this.decoder = new AudioDecoder({ - output: (data) => { - try { - sampleHandler(new AudioSample(data)); - } catch (error) { - this.onError(error); - } - }, - error: (error) => { - error.stack = stack; // Provide a more useful stack trace, the default one sucks - this.onError(error); - }, - }); - this.decoder.configure(decoderConfig); - } - } - - getDecodeQueueSize() { - if (this.customDecoder) { - return this.customDecoderQueueSize; - } else { - assert(this.decoder); - return this.decoder.decodeQueueSize; - } - } - - decode(packet: EncodedPacket) { - if (this.customDecoder) { - this.customDecoderQueueSize++; - void this.customDecoderCallSerializer - .call(() => this.customDecoder!.decode(packet)) - .catch(error => this.onError(error)) - .finally(() => this.customDecoderQueueSize--); - } else { - assert(this.decoder); - - this.expectedFirstTimestamp ??= packet.timestamp; - this.decoder.decode(packet.toEncodedAudioChunk()); - } - } - - async flush() { - if (this.customDecoder) { - await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush()); - } else { - assert(this.decoder); - await this.decoder.flush(); - } - - this.currentTimestamp = null; - this.expectedFirstTimestamp = null; - this.timestampOffset = 0; - } - - close() { - if (this.customDecoder) { - void this.customDecoderCallSerializer.call(() => this.customDecoder!.close()); - } else { - assert(this.decoder); - - if (this.decoder.state !== 'closed') { - this.decoder.close(); - } - } - } -} - -// There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple -// decoder that maps any input PCM format into a PCM format supported by the browser. -class PcmAudioDecoderWrapper extends DecoderWrapper { - codec: PcmAudioCodec; - - inputSampleSize: 1 | 2 | 3 | 4 | 8; - readInputValue: (view: DataView, byteOffset: number) => number; - - outputSampleSize: 1 | 2 | 4; - outputFormat: 'u8' | 's16' | 's32' | 'f32'; - writeOutputValue: (view: DataView, byteOffset: number, value: number) => void; - - // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially - // inaccurate) packet timestamps. - currentTimestamp: number | null = null; - - constructor( - onSample: (sample: AudioSample) => unknown, - onError: (error: unknown) => unknown, - public decoderConfig: AudioDecoderConfig, - ) { - super(onSample, onError); - - assert((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)); - this.codec = decoderConfig.codec as PcmAudioCodec; - - const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec); - this.inputSampleSize = sampleSize; - - switch (sampleSize) { - case 1: { - if (dataType === 'unsigned') { - this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7; - } else if (dataType === 'signed') { - this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset); - } else if (dataType === 'ulaw') { - this.readInputValue = (view, byteOffset) => fromUlaw(view.getUint8(byteOffset)); - } else if (dataType === 'alaw') { - this.readInputValue = (view, byteOffset) => fromAlaw(view.getUint8(byteOffset)); - } else { - assert(false); - } - }; break; - case 2: { - if (dataType === 'unsigned') { - this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15; - } else if (dataType === 'signed') { - this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian); - } else { - assert(false); - } - }; break; - case 3: { - if (dataType === 'unsigned') { - this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23; - } else if (dataType === 'signed') { - this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian); - } else { - assert(false); - } - }; break; - case 4: { - if (dataType === 'unsigned') { - this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31; - } else if (dataType === 'signed') { - this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian); - } else if (dataType === 'float') { - this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian); - } else { - assert(false); - } - }; break; - case 8: { - if (dataType === 'float') { - this.readInputValue = (view, byteOffset) => view.getFloat64(byteOffset, littleEndian); - } else { - assert(false); - } - }; break; - default: { - assertNever(sampleSize); - assert(false); - }; - } - - switch (sampleSize) { - case 1: { - if (dataType === 'ulaw' || dataType === 'alaw') { - this.outputSampleSize = 2; - this.outputFormat = 's16'; - this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true); - } else { - this.outputSampleSize = 1; - this.outputFormat = 'u8'; - this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7); - } - }; break; - case 2: { - this.outputSampleSize = 2; - this.outputFormat = 's16'; - this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true); - }; break; - case 3: { - this.outputSampleSize = 4; - this.outputFormat = 's32'; - // From https://www.w3.org/TR/webcodecs: - // AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are - // stored in s32, each sample MUST be left-shifted by 8 bits. - this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true); - }; break; - case 4: { - this.outputSampleSize = 4; - - if (dataType === 'float') { - this.outputFormat = 'f32'; - this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true); - } else { - this.outputFormat = 's32'; - this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true); - } - }; break; - case 8: { - this.outputSampleSize = 4; - - this.outputFormat = 'f32'; - this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true); - }; break; - default: { - assertNever(sampleSize); - assert(false); - }; - }; - } - - getDecodeQueueSize() { - return 0; - } - - decode(packet: EncodedPacket) { - const inputView = toDataView(packet.data); - - const numberOfFrames = packet.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize; - - const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize; - const outputBuffer = new ArrayBuffer(outputBufferSize); - const outputView = new DataView(outputBuffer); - - for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) { - const inputIndex = i * this.inputSampleSize; - const outputIndex = i * this.outputSampleSize; - - const value = this.readInputValue(inputView, inputIndex); - this.writeOutputValue(outputView, outputIndex, value); - } - - const preciseDuration = numberOfFrames / this.decoderConfig.sampleRate; - if (this.currentTimestamp === null || Math.abs(packet.timestamp - this.currentTimestamp) >= preciseDuration) { - // We need to sync with the packet timestamp again - this.currentTimestamp = packet.timestamp; - } - - const preciseTimestamp = this.currentTimestamp; - this.currentTimestamp += preciseDuration; - - const audioSample = new AudioSample({ - format: this.outputFormat, - data: outputBuffer, - numberOfChannels: this.decoderConfig.numberOfChannels, - sampleRate: this.decoderConfig.sampleRate, - numberOfFrames, - timestamp: preciseTimestamp, - }); - - this.onSample(audioSample); - } - - async flush() { - // Do nothing - } - - close() { - // Do nothing - } -} - -/** - * Sink for retrieving decoded audio samples from an audio track. - * @group Media sinks - * @public - */ -export class AudioSampleSink extends BaseMediaSampleSink { - /** @internal */ - _track: InputAudioTrack; - - /** Creates a new {@link AudioSampleSink} for the given {@link InputAudioTrack}. */ - constructor(audioTrack: InputAudioTrack) { - if (!(audioTrack instanceof InputAudioTrack)) { - throw new TypeError('audioTrack must be an InputAudioTrack.'); - } - - super(); - - this._track = audioTrack; - } - - /** @internal */ - async _createDecoder( - onSample: (sample: AudioSample) => unknown, - onError: (error: unknown) => unknown, - ) { - if (!(await this._track.canDecode())) { - if (typeof AudioDecoder === 'undefined') { - throw new Error(missingWebCodecsClassMessage('AudioDecoder')); - } - - throw new Error( - 'This audio track cannot be decoded in this environment. Make sure to check decodability before using' - + ' a track.', - ); - } - - const codec = await this._track.getCodec(); - const decoderConfig = await this._track.getDecoderConfig(); - assert(codec && decoderConfig); - - if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) { - return new PcmAudioDecoderWrapper(onSample, onError, decoderConfig); - } else { - return new AudioDecoderWrapper(onSample, onError, codec, decoderConfig); - } - } - - /** @internal */ - _createPacketSink() { - return new EncodedPacketSink(this._track); - } - - /** - * Retrieves the audio sample corresponding to the given timestamp, in seconds. More specifically, returns - * the last audio sample (in presentation order) with a start timestamp less than or equal to the given timestamp. - * Returns null if the timestamp is before the track's first timestamp. - * - * @param timestamp - The timestamp used for retrieval, in seconds. - * @param options - Options used for the underlying packet retrieval. - */ - async getSample(timestamp: number, options: PacketRetrievalOptions = {}) { - validateTimestamp(timestamp); - - for await (const sample of this.mediaSamplesAtTimestamps([timestamp], options)) { - return sample; - } - throw new Error('Internal error: Iterator returned nothing.'); - } - - /** - * Creates an async iterator that yields the audio samples of this track in presentation order. This method - * will intelligently pre-decode a few samples ahead to enable fast iteration. - * - * @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive). - * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive). - * @param options - Options used for the underlying packet retrieval. - */ - samples(startTimestamp?: number, endTimestamp?: number, options: PacketRetrievalOptions = {}) { - return this.mediaSamplesInRange(startTimestamp, endTimestamp, options); - } - - /** - * Creates an async iterator that yields an audio sample for each timestamp in the argument. This method - * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most - * once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may - * yield null if no sample is available for a given timestamp. - * - * This method is good for sparse access of media data. If you want primarily sequential media access, prefer - * {@link AudioSampleSink.samples} instead. - * - * @param timestamps - An iterable or async iterable of timestamps in seconds. - * @param options - Options used for the underlying packet retrieval. - */ - samplesAtTimestamps(timestamps: AnyIterable, options: PacketRetrievalOptions = {}) { - return this.mediaSamplesAtTimestamps(timestamps, options); - } -} - -/** - * An AudioBuffer with additional timing information (timestamp & duration). - * @group Media sinks - * @public - */ -export type WrappedAudioBuffer = { - /** An AudioBuffer. */ - buffer: AudioBuffer; - /** The timestamp of the corresponding audio sample, in seconds. */ - timestamp: number; - /** The duration of the corresponding audio sample, in seconds. */ - duration: number; -}; - -/** - * A sink that retrieves decoded audio samples from an audio track and converts them to `AudioBuffer` instances. This is - * often more useful than directly retrieving audio samples, as audio buffers can be directly used with the - * Web Audio API. - * @group Media sinks - * @public - */ -export class AudioBufferSink { - /** @internal */ - _audioSampleSink: AudioSampleSink; - - /** Creates a new {@link AudioBufferSink} for the given {@link InputAudioTrack}. */ - constructor(audioTrack: InputAudioTrack) { - if (!(audioTrack instanceof InputAudioTrack)) { - throw new TypeError('audioTrack must be an InputAudioTrack.'); - } - - this._audioSampleSink = new AudioSampleSink(audioTrack); - } - - /** @internal */ - _audioSampleToWrappedArrayBuffer(sample: AudioSample): WrappedAudioBuffer { - const result: WrappedAudioBuffer = { - buffer: sample.toAudioBuffer(), - timestamp: sample.timestamp, - duration: sample.duration, - }; - - sample.close(); - return result; - } - - /** - * Retrieves the audio buffer corresponding to the given timestamp, in seconds. More specifically, returns - * the last audio buffer (in presentation order) with a start timestamp less than or equal to the given timestamp. - * Returns null if the timestamp is before the track's first timestamp. - * - * @param timestamp - The timestamp used for retrieval, in seconds. - * @param options - Options used for the underlying packet retrieval. - */ - async getBuffer(timestamp: number, options?: PacketRetrievalOptions) { - validateTimestamp(timestamp); - - const data = await this._audioSampleSink.getSample(timestamp, options); - return data && this._audioSampleToWrappedArrayBuffer(data); - } - - /** - * Creates an async iterator that yields audio buffers of this track in presentation order. This method - * will intelligently pre-decode a few buffers ahead to enable fast iteration. - * - * @param startTimestamp - The timestamp in seconds at which to start yielding buffers (inclusive). - * @param endTimestamp - The timestamp in seconds at which to stop yielding buffers (exclusive). - * @param options - Options used for the underlying packet retrieval. - */ - buffers(startTimestamp?: number, endTimestamp?: number, options?: PacketRetrievalOptions) { - return mapAsyncGenerator( - this._audioSampleSink.samples(startTimestamp, endTimestamp, options), - data => this._audioSampleToWrappedArrayBuffer(data), - ); - } - - /** - * Creates an async iterator that yields an audio buffer for each timestamp in the argument. This method - * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most - * once, and is therefore more efficient than manually getting the buffer for every timestamp. The iterator may - * yield null if no buffer is available for a given timestamp. - * - * @param timestamps - An iterable or async iterable of timestamps in seconds. - * @param options - Options used for the underlying packet retrieval. - */ - buffersAtTimestamps(timestamps: AnyIterable, options?: PacketRetrievalOptions) { - return mapAsyncGenerator( - this._audioSampleSink.samplesAtTimestamps(timestamps, options), - data => data && this._audioSampleToWrappedArrayBuffer(data), - ); - } -} diff --git a/src/media-source.ts b/src/media-source.ts index dad5832fb..973abcbd4 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -24,7 +24,7 @@ import { assert, assertNever, binarySearchLessOrEqual, - CallSerializer, + NaiveCallSerializer, clamp, clearIntervalUnthrottled, floorToDivisor, @@ -266,7 +266,7 @@ class VideoEncoderWrapper { }[] = []; private customEncoder: CustomVideoEncoder | null = null; - private customEncoderCallSerializer = new CallSerializer(); + private customEncoderCallSerializer = new NaiveCallSerializer(); private customEncoderQueueSize = 0; // Set when the encoder uses quantizer-based rate control; carries the quantizer value applied to each frame @@ -299,8 +299,6 @@ class VideoEncoderWrapper { constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {} async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) { - const originalSample = videoSample; - try { this.checkForEncoderError(); this.source._ensureValidAdd(); @@ -402,14 +400,17 @@ class VideoEncoderWrapper { } } - // Clone if the sample is still the user's, to avoid mutating externally-owned data - if (videoSample === originalSample) { - videoSample = videoSample.clone(); - shouldClose = true; + const adjusted = videoSample.clone({ + timestamp: alignedTimestamp, + duration: 1 / frameRate, + }); + + if (shouldClose) { + videoSample.close(); } - videoSample.setTimestamp(alignedTimestamp); - videoSample.setDuration(1 / frameRate); + videoSample = adjusted; + shouldClose = true; this.frameRateLastSample?.close(); this.frameRateLastSample = videoSample.clone(); @@ -656,9 +657,10 @@ class VideoEncoderWrapper { const frameDifference = Math.round((until - this.frameRateLastTimestamp!) * frameRate); for (let i = 1; i < frameDifference; i++) { - using sample = this.frameRateLastSample.clone(); - sample.setTimestamp(this.frameRateLastTimestamp! + i / frameRate); - sample.setDuration(1 / frameRate); + using sample = this.frameRateLastSample.clone({ + timestamp: this.frameRateLastTimestamp! + i / frameRate, + duration: 1 / frameRate, + }); await this.processAndEncode(sample, encodeOptions); } } @@ -1902,7 +1904,7 @@ class AudioEncoderWrapper { private writeOutputValue: ((view: DataView, byteOffset: number, value: number) => void) | null = null; private customEncoder: CustomAudioEncoder | null = null; - private customEncoderCallSerializer = new CallSerializer(); + private customEncoderCallSerializer = new NaiveCallSerializer(); private customEncoderQueueSize = 0; private lastEndSampleIndex: number | null = null; @@ -2806,7 +2808,8 @@ export class MediaStreamAudioTrackSource extends AudioSource { return; } - audioSample.setTimestamp(currentTimestamp + timestampOffset); + // @ts-expect-error Readonly, kind of a hack + audioSample.timestamp = currentTimestamp + timestampOffset; void this._encoder.add(audioSample, true) .catch((error) => { diff --git a/src/metadata.ts b/src/metadata.ts index d452b855b..561d44e6b 100644 --- a/src/metadata.ts +++ b/src/metadata.ts @@ -81,7 +81,7 @@ export type MetadataTags = { * Additionally, the `'vendor'` key refers to the vendor string within this header. If a key exists more than once, * a string array is used instead. * - WAVE: The individual metadata chunks within the RIFF INFO chunk. Values are always ISO 8859-1 strings. - * - FLAC: The key-value string pairs from the vorbis metadata block (see RFC 9639, Section D.2.3). + * - FLAC: The key-value string pairs from the Vorbis metadata block (see RFC 9639, Section D.2.3). * Additionally, the `'vendor'` key refers to the vendor string within this header. If a key exists more than once, * a string array is used instead. If ID3v2 tags appear at the start of the file, their content is stored just like * for MP3. diff --git a/src/misc.ts b/src/misc.ts index b87cd147b..50b4ca469 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -156,6 +156,19 @@ export const composeRotationAndFlip = ( }; }; +/** The inverse of {@link composeRotationAndFlip}: what needs to be composed onto the base to arrive at the target. */ +export const relativeRotationAndFlip = ( + baseRotation: Rotation, + baseFlip: boolean, + targetRotation: Rotation, + targetFlip: boolean, +) => { + return { + rotation: normalizeRotation(baseFlip ? baseRotation - targetRotation : targetRotation - baseRotation), + flip: baseFlip !== targetFlip, + }; +}; + export const last = (arr: T[]) => { return arr && arr[arr.length - 1]; }; @@ -485,36 +498,6 @@ export const isAllowSharedBufferSource = (x: unknown) => { ); }; -export class AsyncMutex { - currentPromise = Promise.resolve(); - pending = 0; - - async acquire() { - let resolver: () => void; - const nextPromise = new Promise((resolve) => { - let resolved = false; - - resolver = () => { - if (resolved) { - return; - } - - resolve(); - this.pending--; - resolved = true; - }; - }); - - const currentPromiseAlias = this.currentPromise; - this.currentPromise = nextPromise; - this.pending++; - - await currentPromiseAlias; - - return resolver!; - } -} - export const HEX_STRING_REGEX = /^[0-9a-fA-F]+$/; export const bytesToHexString = (bytes: Uint8Array) => { @@ -600,6 +583,35 @@ export const promiseWithResolvers = () => { return { promise, resolve: resolve!, reject: reject! }; }; +export const promiseAllEnsureOrder = async (promises: T[]) => { + const results: Awaited[] = []; + const { promise, resolve, reject } = promiseWithResolvers(); + + const onValue = (value: Awaited, i: number) => { + if (results.length === i) { + results.push(value); + + if (results.length === promises.length) { + resolve(); + } + } else { + reject(new Error('Order violation')); + } + }; + + for (let i = 0; i < promises.length; i++) { + const value = promises[i]!; + if (isThenable(value)) { + void value.then(x => onValue(x as Awaited, i)); + } else { + onValue(value as Awaited, i); + } + } + + await promise; + return results; +}; + export const removeItem = (arr: T[], item: T) => { const index = arr.indexOf(item); if (index !== -1) { @@ -782,6 +794,37 @@ export const floorToDivisor = (value: number, multiple: number) => { return Math.floor(value * multiple) / multiple; }; +const nextDownView = /* #__PURE__ */ new DataView(new ArrayBuffer(8)); + +/** Returns the largest number smaller than the given number. */ +export const nextDown = (value: number) => { + if (value === 0) { + return -Number.MIN_VALUE; + } + + nextDownView.setFloat64(0, value); + const high = nextDownView.getUint32(0); + const low = nextDownView.getUint32(4); + + if (value > 0) { + if (low === 0) { + nextDownView.setUint32(0, high - 1); + nextDownView.setUint32(4, 0xffffffff); + } else { + nextDownView.setUint32(4, low - 1); + } + } else { + if (low === 0xffffffff) { + nextDownView.setUint32(0, high + 1); + nextDownView.setUint32(4, 0); + } else { + nextDownView.setUint32(4, low + 1); + } + } + + return nextDownView.getFloat64(0); +}; + export const ilog = (x: number) => { let ret = 0; while (x) { @@ -973,14 +1016,6 @@ export const computeRationalApproximation = (x: number, maxDenominator: number): }; }; -export class CallSerializer { - currentPromise = Promise.resolve(); - - call(fn: () => Promise | void) { - return this.currentPromise = this.currentPromise.then(fn); - } -} - let isWebKitCache: boolean | null = null; export const isWebKit = () => { if (isWebKitCache !== null) { @@ -1171,12 +1206,217 @@ export const polyfillSymbolDispose = () => { // https://www.typescriptlang.org/docs/handbook/release-notes/typescript-5-2.html // @ts-expect-error Readonly Symbol.dispose ??= Symbol('Symbol.dispose'); + // @ts-expect-error Readonly + Symbol.asyncDispose ??= Symbol('Symbol.asyncDispose'); }; export const isNumber = (x: unknown) => { return typeof x === 'number' && !Number.isNaN(x); }; +// We use a unique symbol to ensure that any function using the ResultValue system actually returns when setting it, +// instead of simply setting it and continuing on. +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const returnSymbol: unique symbol = Symbol(); +export type ReturnSymbol = typeof returnSymbol; +export type MaybeRelevantPromise = Promise; + +/** + * Represents a wrapper that holds a function's return value. Instead of the function returning its return value, it + * instead gets passed a ResultValue instance and writes the return value into it. This allows async functions that + * don't hit any async path to synchronously expose their return value, allowing the caller to ignore the returned + * promise if possible. This allows for "optinally-asynchronous code". + */ +export class ResultValue { + value!: T; + pending = true; + + // @ts-expect-error Return value just for the types + set(value: T): ReturnSymbol { + this.value = value; + this.pending = false; + } + + reset() { + this.pending = true; + } +} + +export class AsyncMutex { + locked = false; + resolverQueue: (() => void)[] = []; + + lock() { + if (!this.locked) { + // Fast path + this.locked = true; + return new AsyncMutexLock(this, false, null); + } + + const { promise, resolve } = promiseWithResolvers(); + this.resolverQueue.push(resolve); + + return new AsyncMutexLock(this, true, promise); + } + + dispatch() { + if (this.resolverQueue.length > 0) { + const resolve = this.resolverQueue.shift()!; + resolve(); + } else { + this.locked = false; + } + } + + async waitForUnlock() { + const lock = this.lock(); + await lock.ready; + lock.release(); + } +} + +export class AsyncMutexLock implements Disposable { + private released = false; + + constructor( + private readonly mutex: AsyncMutex, + public readonly pending: boolean, + public readonly ready: Promise | null, + ) {} + + release() { + if (this.released) { + return; + } + + this.released = true; + this.mutex.dispatch(); + } + + [Symbol.dispose]() { + this.release(); + } +} + +/** + * A simple call serializer that works by chaining promises. When one callback throws, the serializer becomes bricked, + * meaning all future calls will also throw. + */ +export class NaiveCallSerializer { + currentPromise = Promise.resolve(); + errored = false; + + call(fn: () => Promise | void) { + return this.currentPromise = this.currentPromise + .then(fn) + .catch((error) => { + this.errored = true; + throw error; + }); + } +} + +/** + * A more complex call serializer implementation that works with optionally asynchronous functions. It is forgiving + * in the sense that when a call throws, the error is surfaced but subsequent calls will go through again. So, it + * recovers. + */ +export class ForgivingCallSerializer { + private currentPromise: Promise | null = null; + private queuedCalls = 0; + + call(fn: () => T) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + type ReturnType = T extends Promise ? T : T | Promise; + + if (this.currentPromise) { + this.queuedCalls++; + + return (this.currentPromise = this.currentPromise + .catch(() => {}) + .then(() => { + this.queuedCalls--; + return fn(); + }) + .finally(() => { + if (this.queuedCalls === 0) { + this.currentPromise = null; + } + })) as unknown as ReturnType; + } else { + const result = fn(); + + if (isThenable(result)) { + this.currentPromise = result + .catch(() => {}) + .finally(() => { + if (this.queuedCalls === 0) { + this.currentPromise = null; + } + }); + } + + return result as unknown as ReturnType; + } + } + + waitUntilIdle() { + if (this.currentPromise) { + return this.currentPromise + .catch(() => {}) + .then(() => {}); + } else { + return null; + } + } + + isIdle() { + return !this.currentPromise; + } +} + +export class AsyncGate { + resolvers: (() => void)[] = []; + + wait() { + const { promise, resolve } = promiseWithResolvers(); + + this.resolvers.push(resolve); + return promise; + } + + open() { + if (this.resolvers.length > 0) { + this.resolvers.forEach(fn => fn()); + this.resolvers.length = 0; + } + } +} + +export const defer = (callback: () => void) => { + let executed = false; + + return { + execute() { + if (executed) { + return; + } + + executed = true; + callback(); + }, + [Symbol.dispose]() { + this.execute(); + }, + }; +}; + +export const promiseIterateAll = async function* (iterable: Iterable) { + for (const promise of iterable) { + yield await promise; + } +}; + /** * A path to a file. File paths can be relative or absolute, and be local paths or full URLs. Paths must be POSIX-like, * using `/` as the separator. @@ -1392,6 +1632,13 @@ export const validateRectangle = (rect: Rectangle, propertyPath: string) => { } }; +export const rectangleToDOMRectInit = (rectangle: Rectangle): DOMRectInit => ({ + x: rectangle.left, + y: rectangle.top, + width: rectangle.width, + height: rectangle.height, +}); + export type NonFunctionKeys = { [K in keyof T]-?: T[K] extends ((...args: never[]) => unknown) ? never : K }[keyof T]; diff --git a/src/mp3/mp3-demuxer.ts b/src/mp3/mp3-demuxer.ts index a89d22203..15b682f85 100644 --- a/src/mp3/mp3-demuxer.ts +++ b/src/mp3/mp3-demuxer.ts @@ -11,17 +11,17 @@ import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrackBacking } from '../input-track'; import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; -import { PacketRetrievalOptions } from '../media-sink'; import { assert, AsyncMutex, - binarySearchExact, binarySearchLessOrEqual, isThenable, toDataView, + MaybeRelevantPromise, + ResultValue, UNDETERMINED_LANGUAGE, } from '../misc'; -import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; +import { EncodedPacket, PLACEHOLDER_DATA, PacketRetrievalOptions, PacketRetrievalResult } from '../packet'; import { Mp3FrameHeader, getXingOffset, @@ -80,7 +80,9 @@ export class Mp3Demuxer extends Demuxer { return this.metadataPromise ??= (async () => { // Keep loading until we find the first frame header while (!this.firstFrameHeader && !this.lastSampleLoaded) { - await this.advanceReader(); + const result = new ResultValue(); + const promise = this.advanceReader(result); + if (result.pending) await promise; } if (!this.firstFrameHeader && this.xingFrameHeader) { @@ -98,7 +100,7 @@ export class Mp3Demuxer extends Demuxer { })(); } - async advanceReader() { + async advanceReader(res: ResultValue): MaybeRelevantPromise { if (this.lastLoadedPos === 0) { // Let's skip all ID3v2 tags at the start of the file while (true) { @@ -107,7 +109,7 @@ export class Mp3Demuxer extends Demuxer { if (!slice) { this.lastSampleLoaded = true; - return; + return res.set(); } const id3V2Header = readId3V2Header(slice); @@ -119,24 +121,31 @@ export class Mp3Demuxer extends Demuxer { } } - const result = await readNextMp3FrameHeader( + const result = new ResultValue<{ + header: Mp3FrameHeader; + startPos: number; + } | null>(); + const promise = readNextMp3FrameHeader( + result, this.reader, this.lastLoadedPos, this.reader.fileSize, this.firstFrameHeader, ); - if (!result) { + if (result.pending) await promise; + + if (!result.value) { this.lastSampleLoaded = true; - return; + return res.set(); } - const header = result.header; + const header = result.value.header; - this.lastLoadedPos = result.startPos + header.totalSize - 1; // -1 in case the frame is 1 byte too short + this.lastLoadedPos = result.value.startPos + header.totalSize - 1; // -1 in case the frame is 1 byte too short const xingOffset = getXingOffset(header.mpegVersionId, header.channel); - let slice = this.reader.requestSlice(result.startPos + xingOffset, 4); + let slice = this.reader.requestSlice(result.value.startPos + xingOffset, 4); if (isThenable(slice)) slice = await slice; if (slice) { const word = readU32Be(slice); @@ -147,11 +156,11 @@ export class Mp3Demuxer extends Demuxer { if (!this.xingFrameHeader) { this.xingFrameHeader = header; - this.xingFrameHeaderPos = result.startPos; + this.xingFrameHeaderPos = result.value.startPos; } if (!this.xingData) { - let xingDataSlice = this.reader.requestSlice(result.startPos + xingOffset + 4, 12); + let xingDataSlice = this.reader.requestSlice(result.value.startPos + xingOffset + 4, 12); if (isThenable(xingDataSlice)) xingDataSlice = await xingDataSlice; if (xingDataSlice) { const xingData = readBytes(xingDataSlice, 12); @@ -169,27 +178,27 @@ export class Mp3Demuxer extends Demuxer { } } - return; + return res.set(); } } if (!this.firstFrameHeader) { this.firstFrameHeader = header; - this.firstFrameHeaderPos = result.startPos; + this.firstFrameHeaderPos = result.value.startPos; } const sampleDuration = header.audioSamplesInFrame / this.firstFrameHeader.sampleRate; const sample: Sample = { timestamp: this.nextTimestampInSamples / this.firstFrameHeader.sampleRate, duration: sampleDuration, - dataStart: result.startPos, + dataStart: result.value.startPos, dataSize: header.totalSize, }; this.loadedSamples.push(sample); this.nextTimestampInSamples += header.audioSamplesInFrame; - return; + return res.set(); } async getMimeType() { @@ -202,56 +211,53 @@ export class Mp3Demuxer extends Demuxer { } async getMetadataTags() { - const release = await this.readingMutex.acquire(); + using lock = this.readingMutex.lock(); + if (lock.pending) await lock.ready; - try { - await this.readMetadata(); + await this.readMetadata(); - if (this.metadataTags) { - return this.metadataTags; - } + if (this.metadataTags) { + return this.metadataTags; + } - this.metadataTags = {}; - let currentPos = 0; - let id3V2HeaderFound = false; + this.metadataTags = {}; + let currentPos = 0; + let id3V2HeaderFound = false; - while (true) { - let headerSlice = this.reader.requestSlice(currentPos, ID3_V2_HEADER_SIZE); - if (isThenable(headerSlice)) headerSlice = await headerSlice; - if (!headerSlice) break; + while (true) { + let headerSlice = this.reader.requestSlice(currentPos, ID3_V2_HEADER_SIZE); + if (isThenable(headerSlice)) headerSlice = await headerSlice; + if (!headerSlice) break; - const id3V2Header = readId3V2Header(headerSlice); - if (!id3V2Header) { - break; - } + const id3V2Header = readId3V2Header(headerSlice); + if (!id3V2Header) { + break; + } - id3V2HeaderFound = true; + id3V2HeaderFound = true; - let contentSlice = this.reader.requestSlice(headerSlice.filePos, id3V2Header.size); - if (isThenable(contentSlice)) contentSlice = await contentSlice; - if (!contentSlice) break; + let contentSlice = this.reader.requestSlice(headerSlice.filePos, id3V2Header.size); + if (isThenable(contentSlice)) contentSlice = await contentSlice; + if (!contentSlice) break; - parseId3V2Tag(contentSlice, id3V2Header, this.metadataTags); + parseId3V2Tag(contentSlice, id3V2Header, this.metadataTags); - currentPos = headerSlice.filePos + id3V2Header.size; - } + currentPos = headerSlice.filePos + id3V2Header.size; + } - if (!id3V2HeaderFound && this.reader.fileSize !== null && this.reader.fileSize >= ID3_V1_TAG_SIZE) { - // Try reading an ID3v1 tag at the end of the file - let slice = this.reader.requestSlice(this.reader.fileSize - ID3_V1_TAG_SIZE, ID3_V1_TAG_SIZE); - if (isThenable(slice)) slice = await slice; - assert(slice); + if (!id3V2HeaderFound && this.reader.fileSize !== null && this.reader.fileSize >= ID3_V1_TAG_SIZE) { + // Try reading an ID3v1 tag at the end of the file + let slice = this.reader.requestSlice(this.reader.fileSize - ID3_V1_TAG_SIZE, ID3_V1_TAG_SIZE); + if (isThenable(slice)) slice = await slice; + assert(slice); - const tag = readAscii(slice, 3); - if (tag === 'TAG') { - parseId3V1Tag(slice, this.metadataTags); - } + const tag = readAscii(slice, 3); + if (tag === 'TAG') { + parseId3V1Tag(slice, this.metadataTags); } - - return this.metadataTags; - } finally { - release(); } + + return this.metadataTags; } } @@ -374,14 +380,18 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking { }; } - async getPacketAtIndex(sampleIndex: number, options: PacketRetrievalOptions) { + async getPacketAtIndex( + res: ResultValue, + sampleIndex: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { if (sampleIndex === -1) { - return null; + return res.set(new PacketRetrievalResult(null)); } const rawSample = this.demuxer.loadedSamples[sampleIndex]; if (!rawSample) { - return null; + return res.set(new PacketRetrievalResult(null)); } let data: Uint8Array; @@ -392,93 +402,102 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking { if (isThenable(slice)) slice = await slice; if (!slice) { - return null; // Data didn't fit into the rest of the file + return res.set(new PacketRetrievalResult(null)); // Data didn't fit into the rest of the file } data = readBytes(slice, rawSample.dataSize); } - return new EncodedPacket( + return res.set(new PacketRetrievalResult(new EncodedPacket( data, 'key', rawSample.timestamp, rawSample.duration, sampleIndex, rawSample.dataSize, - ); + ))); } - getFirstPacket(options: PacketRetrievalOptions) { - return this.getPacketAtIndex(0, options); + getFirstPacket(res: ResultValue, options: PacketRetrievalOptions) { + return this.getPacketAtIndex(res, 0, options); } - async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - const release = await this.demuxer.readingMutex.acquire(); + async getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + using lock = this.demuxer.readingMutex.lock(); + if (lock.pending) await lock.ready; + + const sampleIndex = packet.sequenceNumber; + assert(sampleIndex >= 0); + + const nextIndex = sampleIndex + 1; + + // Ensure the next sample exists + while ( + nextIndex >= this.demuxer.loadedSamples.length + && !this.demuxer.lastSampleLoaded + ) { + const result = new ResultValue(); + const promise = this.demuxer.advanceReader(result); + if (result.pending) await promise; + } + + return this.getPacketAtIndex(res, nextIndex, options); + } - try { - const sampleIndex = binarySearchExact( + async getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + using lock = this.demuxer.readingMutex.lock(); + if (lock.pending) await lock.ready; + + while (true) { + const index = binarySearchLessOrEqual( this.demuxer.loadedSamples, - packet.timestamp, + timestamp, x => x.timestamp, ); - if (sampleIndex === -1) { - throw new Error('Packet was not created from this track.'); - } - const nextIndex = sampleIndex + 1; - // Ensure the next sample exists - while ( - nextIndex >= this.demuxer.loadedSamples.length - && !this.demuxer.lastSampleLoaded - ) { - await this.demuxer.advanceReader(); + if (index === -1 && this.demuxer.loadedSamples.length > 0) { + // We're before the first sample + return res.set(new PacketRetrievalResult(null)); } - return this.getPacketAtIndex(nextIndex, options); - } finally { - release(); - } - } - - async getPacket(timestamp: number, options: PacketRetrievalOptions) { - const release = await this.demuxer.readingMutex.acquire(); - - try { - while (true) { - const index = binarySearchLessOrEqual( - this.demuxer.loadedSamples, - timestamp, - x => x.timestamp, - ); - - if (index === -1 && this.demuxer.loadedSamples.length > 0) { - // We're before the first sample - return null; - } - - if (this.demuxer.lastSampleLoaded) { - // All data is loaded, return what we found - return this.getPacketAtIndex(index, options); - } - - if (index >= 0 && index + 1 < this.demuxer.loadedSamples.length) { - // The next packet also exists, we're done - return this.getPacketAtIndex(index, options); - } + if (this.demuxer.lastSampleLoaded) { + // All data is loaded, return what we found + return this.getPacketAtIndex(res, index, options); + } - // Otherwise, keep loading data - await this.demuxer.advanceReader(); + if (index >= 0 && index + 1 < this.demuxer.loadedSamples.length) { + // The next packet also exists, we're done + return this.getPacketAtIndex(res, index, options); } - } finally { - release(); + + // Otherwise, keep loading data + const result = new ResultValue(); + const promise = this.demuxer.advanceReader(result); + if (result.pending) await promise; } } - getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { - return this.getPacket(timestamp, options); + getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this.getPacket(res, timestamp, options); } - getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - return this.getNextPacket(packet, options); + getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this.getNextPacket(res, packet, options); } } diff --git a/src/mp3/mp3-muxer.ts b/src/mp3/mp3-muxer.ts index 4fc1763f7..d07d66500 100644 --- a/src/mp3/mp3-muxer.ts +++ b/src/mp3/mp3-muxer.ts @@ -35,7 +35,8 @@ export class Mp3Muxer extends Muxer { } async start() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.writer = await this.output._getRootWriter(!this.writeXingHeader); this.mp3Writer = new Mp3Writer(this.writer); @@ -44,8 +45,6 @@ export class Mp3Muxer extends Muxer { const id3Writer = new Id3V2Writer(this.writer); id3Writer.writeId3V2Tag(this.output._metadataTags); } - - release(); } async getMimeType() { @@ -60,69 +59,66 @@ export class Mp3Muxer extends Muxer { track: OutputAudioTrack, packet: EncodedPacket, ) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - if (!this.xingFrameData && this.writeXingHeader) { - const view = toDataView(packet.data); - if (view.byteLength < 4) { - throw new Error('Invalid MP3 header in sample.'); - } + if (!this.xingFrameData && this.writeXingHeader) { + const view = toDataView(packet.data); + if (view.byteLength < 4) { + throw new Error('Invalid MP3 header in sample.'); + } - const word = view.getUint32(0, false); - const header = readMp3FrameHeader(word, null).header; - if (!header) { - throw new Error('Invalid MP3 header in sample.'); - } + const word = view.getUint32(0, false); + const header = readMp3FrameHeader(word, null).header; + if (!header) { + throw new Error('Invalid MP3 header in sample.'); + } - const xingOffset = getXingOffset(header.mpegVersionId, header.channel); - if (view.byteLength >= xingOffset + 4) { - const word = view.getUint32(xingOffset, false); - const isXing = word === XING || word === INFO; + const xingOffset = getXingOffset(header.mpegVersionId, header.channel); + if (view.byteLength >= xingOffset + 4) { + const word = view.getUint32(xingOffset, false); + const isXing = word === XING || word === INFO; - if (isXing) { - // This is not a data frame, so let's completely ignore this sample - return; - } + if (isXing) { + // This is not a data frame, so let's completely ignore this sample + return; } - - this.xingFrameData = { - mpegVersionId: header.mpegVersionId, - layer: header.layer, - frequencyIndex: header.frequencyIndex, - sampleRate: header.sampleRate, - channel: header.channel, - modeExtension: header.modeExtension, - copyright: header.copyright, - original: header.original, - emphasis: header.emphasis, - - frameCount: null, - fileSize: null, - toc: null, - }; - - // Write a Xing frame because this muxer doesn't make any bitrate constraints, meaning we don't know if - // this will be a constant or variable bitrate file. Therefore, always write the Xing frame. - this.xingFramePos = this.writer.getPos(); - this.mp3Writer.writeXingFrame(this.xingFrameData); - - this.frameCount++; } - this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); - - if (this.writeXingHeader) { - this.framePositions.push(this.writer.getPos()); - } + this.xingFrameData = { + mpegVersionId: header.mpegVersionId, + layer: header.layer, + frequencyIndex: header.frequencyIndex, + sampleRate: header.sampleRate, + channel: header.channel, + modeExtension: header.modeExtension, + copyright: header.copyright, + original: header.original, + emphasis: header.emphasis, + + frameCount: null, + fileSize: null, + toc: null, + }; + + // Write a Xing frame because this muxer doesn't make any bitrate constraints, meaning we don't know if + // this will be a constant or variable bitrate file. Therefore, always write the Xing frame. + this.xingFramePos = this.writer.getPos(); + this.mp3Writer.writeXingFrame(this.xingFrameData); - this.writer.write(packet.data); this.frameCount++; + } + + this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); - await this.writer.flush(); - } finally { - release(); + if (this.writeXingHeader) { + this.framePositions.push(this.writer.getPos()); } + + this.writer.write(packet.data); + this.frameCount++; + + await this.writer.flush(); } async addSubtitleCue() { @@ -130,7 +126,8 @@ export class Mp3Muxer extends Muxer { } async finalize() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; const isEmpty = this.frameCount === 0; @@ -262,7 +259,5 @@ export class Mp3Muxer extends Muxer { this.format._options.onXingFrame(data, start); } } - - release(); } } diff --git a/src/mp3/mp3-reader.ts b/src/mp3/mp3-reader.ts index 29c4689fd..ce0a86ae0 100644 --- a/src/mp3/mp3-reader.ts +++ b/src/mp3/mp3-reader.ts @@ -6,19 +6,21 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import { MaybeRelevantPromise, ResultValue } from '../misc'; import { MP3_FRAME_HEADER_SIZE, getMp3ChannelCount, Mp3FrameHeader, readMp3FrameHeader } from '../../shared/mp3-misc'; import { isThenable } from '../misc'; import { Reader, readU32Be } from '../reader'; export const readNextMp3FrameHeader = async ( + res: ResultValue<{ + header: Mp3FrameHeader; + startPos: number; + } | null>, reader: Reader, startPos: number, until: number | null, ref: Mp3FrameHeader | null = null, -): Promise<{ - header: Mp3FrameHeader; - startPos: number; -} | null> => { +): MaybeRelevantPromise => { const CHUNK_SIZE = 2 ** 16; // So we don't need to grab thousands of slices let currentPos = startPos; @@ -50,7 +52,7 @@ export const readNextMp3FrameHeader = async ( && getMp3ChannelCount(result.header.channel) === getMp3ChannelCount(ref.channel) )) ) { - return { header: result.header, startPos: currentPos }; + return res.set({ header: result.header, startPos: currentPos }); } slice.filePos = posBeforeRead + result.bytesAdvanced; @@ -58,5 +60,5 @@ export const readNextMp3FrameHeader = async ( } } - return null; + return res.set(null); }; diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index b1943b716..40540b3f2 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -54,7 +54,6 @@ import { InputTrackBacking, InputVideoTrackBacking, } from '../input-track'; -import { PacketRetrievalOptions } from '../media-sink'; import { DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata'; import { assert, @@ -67,7 +66,9 @@ import { IDENTITY_MATRIX, last, MATRIX_COEFFICIENTS_MAP_INVERSE, + MaybeRelevantPromise, readExpGolomb, + ResultValue, roundIfAlmostInteger, toDataView, TRANSFER_CHARACTERISTICS_MAP_INVERSE, @@ -79,7 +80,13 @@ import { getMp3ChannelCount, readMp3FrameHeader, } from '../../shared/mp3-misc'; -import { EncodedPacket, PacketType, PLACEHOLDER_DATA } from '../packet'; +import { + EncodedPacket, + PacketType, + PLACEHOLDER_DATA, + PacketRetrievalOptions, + PacketRetrievalResult, +} from '../packet'; import { FileSlice, readBytes, Reader, readU16Be, readU32Be, readU8 } from '../reader'; import { buildMpegTsMimeType, @@ -232,8 +239,15 @@ export class MpegTsDemuxer extends Demuxer { let hasProgramAssociationTable = false; let hasProgramMap = false; + const packetHeaderResult = new ResultValue(); + const sectionResult = new ResultValue
(); + while (true) { - const packetHeader = await this.readPacketHeader(currentPos); + packetHeaderResult.reset(); + const packetHeaderPromise = this.readPacketHeader(packetHeaderResult, currentPos); + if (packetHeaderResult.pending) await packetHeaderPromise; + + const packetHeader = packetHeaderResult.value; if (!packetHeader) { break; } @@ -250,11 +264,16 @@ export class MpegTsDemuxer extends Demuxer { continue; } - const section = await this.readSection( + sectionResult.reset(); + const sectionPromise = this.readSection( + sectionResult, currentPos, true, !hasProgramMap, // Expect contiguous sections as long as we don't have the PMT ); + if (sectionResult.pending) await sectionPromise; + + const section = sectionResult.value; if (!section) { break; } @@ -544,13 +563,16 @@ export class MpegTsDemuxer extends Demuxer { } const context = new PacketReadingContext(elementaryStream, pesPacket); + const markResult = new ResultValue(); if (elementaryStream.info.type === 'video') { // We loop because in some files, the video parameters are not in the first packet while (true) { const contextAlias = context; // TyyyyypeScript 😩 contextAlias.suppliedPacket = null; - await context.markNextPacket(); + markResult.reset(); + const markPromise = context.markNextPacket(markResult); + if (markResult.pending) await markPromise; if (elementaryStream.info.codec === 'avc') { if (!context.suppliedPacket) { @@ -695,7 +717,9 @@ export class MpegTsDemuxer extends Demuxer { elementaryStream.initialized = true; } else { - await context.markNextPacket(); + const markPromise = context.markNextPacket(markResult); + if (markResult.pending) await markPromise; + if (!context.suppliedPacket) { throw new Error( `Couldn't parse first media packet for Elementary Stream with` @@ -854,7 +878,12 @@ export class MpegTsDemuxer extends Demuxer { return buildMpegTsMimeType(codecStrings); } - async readSection(startPos: number, full: boolean, contiguous = false): Promise
{ + async readSection( + res: ResultValue
, + startPos: number, + full: boolean, + contiguous = false, + ): MaybeRelevantPromise { let endPos = startPos; let currentPos = startPos; const chunks: Uint8Array[] = []; @@ -863,8 +892,14 @@ export class MpegTsDemuxer extends Demuxer { let mustAddSectionEnd = true; let randomAccessIndicator = 0; + const packetResult = new ResultValue(); + while (true) { - const packet = await this.readPacket(currentPos); + packetResult.reset(); + const promise = this.readPacket(packetResult, currentPos); + if (packetResult.pending) await promise; + + const packet = packetResult.value; currentPos += this.packetStride; if (!packet) { @@ -936,7 +971,7 @@ export class MpegTsDemuxer extends Demuxer { } if (!firstPacket) { - return null; + return res.set(null); } let merged: Uint8Array; @@ -952,21 +987,21 @@ export class MpegTsDemuxer extends Demuxer { } } - return { + return res.set({ startPos, endPos: full ? endPos : null, pid: firstPacket.pid, payload: merged, randomAccessIndicator, - }; + }); } - async readPacketHeader(pos: number): Promise { + async readPacketHeader(res: ResultValue, pos: number): MaybeRelevantPromise { let slice = this.reader.requestSlice(pos, 4); if (isThenable(slice)) slice = await slice; if (!slice) { - return null; + return res.set(null); } const syncByte = readU8(slice); @@ -989,20 +1024,20 @@ export class MpegTsDemuxer extends Demuxer { // eslint-disable-next-line @typescript-eslint/no-unused-vars const continuityCounter = nextByte & 0xF; - return { + return res.set({ payloadUnitStartIndicator, pid, adaptationFieldControl, - }; + }); } - async readPacket(pos: number): Promise { + async readPacket(res: ResultValue, pos: number): MaybeRelevantPromise { // Code in here is duplicated from readPacketHeader for performance reasons let slice = this.reader.requestSlice(pos, TS_PACKET_SIZE); if (isThenable(slice)) slice = await slice; if (!slice) { - return null; + return res.set(null); } const bytes = readBytes(slice, TS_PACKET_SIZE); @@ -1027,12 +1062,12 @@ export class MpegTsDemuxer extends Demuxer { // eslint-disable-next-line @typescript-eslint/no-unused-vars const continuityCounter = nextByte & 0xF; - return { + return res.set({ payloadUnitStartIndicator, pid, adaptationFieldControl, body: bytes.subarray(4), - }; + }); } normalizeTimestamp(timestamp: number) { @@ -1185,11 +1220,13 @@ const readPesPacket = ( } as T extends true ? TimestampedPesPacket : PesPacket; }; -abstract class MpegTsTrackBacking implements InputTrackBacking { - packetBuffers = new WeakMap(); - /** Used for recreating PacketBuffers if necessary. */ - packetSectionStarts = new WeakMap(); +type EncodedPacketMetadata = { + buffer: PacketBuffer | null; + /** Used for recreating the PacketBuffer if necessary. */ + sectionStartPos: number; +}; +abstract class MpegTsTrackBacking implements InputTrackBacking { constructor(public elementaryStream: ElementaryStream) {} abstract getType(): TrackType; @@ -1279,6 +1316,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { createEncodedPacket( suppliedPacket: SuppliedPacket, duration: number, + buffer: PacketBuffer, options: PacketRetrievalOptions, ) { let packetType: PacketType; @@ -1291,7 +1329,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { : 'delta'; } - return new EncodedPacket( + const packet = new EncodedPacket( options.metadataOnly ? PLACEHOLDER_DATA : suppliedPacket.data, packetType, suppliedPacket.pts / TIMESCALE, @@ -1299,9 +1337,18 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { suppliedPacket.sequenceNumber, suppliedPacket.data.byteLength, ); + packet._internal = { + buffer, + sectionStartPos: suppliedPacket.sectionStartPos, + } satisfies EncodedPacketMetadata; + + return packet; } - async getFirstPacket(options: PacketRetrievalOptions): Promise { + async getFirstPacket( + res: ResultValue, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { const section = this.elementaryStream.firstSection; assert(section); @@ -1311,95 +1358,123 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { const context = new PacketReadingContext(this.elementaryStream, pesPacket); const buffer = new PacketBuffer(this, context); - const result = await buffer.readNext(); + const readResult = new ResultValue(); + const promise = buffer.readNext(readResult); + if (readResult.pending) await promise; + + const result = readResult.value; if (!result) { - return null; + return res.set(new PacketRetrievalResult(null)); } - const packet = this.createEncodedPacket(result.packet, result.duration, options); - this.packetBuffers.set(packet, buffer); - this.packetSectionStarts.set(packet, result.packet.sectionStartPos); - - return packet; + const packet = this.createEncodedPacket(result.packet, result.duration, buffer, options); + return res.set(new PacketRetrievalResult(packet)); } - async getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { - let buffer = this.packetBuffers.get(packet); + async getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + const metadata = packet._internal as EncodedPacketMetadata | undefined; + assert(metadata); + + const readResult = new ResultValue(); - if (buffer) { + if (metadata.buffer) { // Fast path - const result = await buffer.readNext(); + const buffer = metadata.buffer; + const promise = buffer.readNext(readResult); + if (readResult.pending) await promise; + + const result = readResult.value; if (!result) { - return null; + return res.set(new PacketRetrievalResult(null)); } - // Remove PacketBuffer access from the old packet, it belongs to the next packet now - this.packetBuffers.delete(packet); + // Remove PacketBuffer access from the old packet (and all its clones), it belongs to the next packet now + metadata.buffer = null; - const newPacket = this.createEncodedPacket(result.packet, result.duration, options); - this.packetBuffers.set(newPacket, buffer); - this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos); - - return newPacket; + const newPacket = this.createEncodedPacket(result.packet, result.duration, buffer, options); + return res.set(new PacketRetrievalResult(newPacket)); } // No buffer, we gotta do some rereading - const sectionStartPos = this.packetSectionStarts.get(packet); - if (sectionStartPos === undefined) { - throw new Error('Packet was not created from this track.'); - } - const demuxer = this.elementaryStream.demuxer; - const section = await demuxer.readSection(sectionStartPos, true); + const sectionResult = new ResultValue
(); + const sectionPromise = demuxer.readSection(sectionResult, metadata.sectionStartPos, true); + if (sectionResult.pending) await sectionPromise; + + const section = sectionResult.value; assert(section); const pesPacket = readPesPacket(demuxer, section, true); assert(pesPacket); const context = new PacketReadingContext(this.elementaryStream, pesPacket); - buffer = new PacketBuffer(this, context); + const buffer = new PacketBuffer(this, context); // Advance until we pass the current packet's sequence number const targetSequenceNumber = packet.sequenceNumber; while (true) { - const result = await buffer.readNext(); + readResult.reset(); + const promise = buffer.readNext(readResult); + if (readResult.pending) await promise; + + const result = readResult.value; if (!result) { - return null; + return res.set(new PacketRetrievalResult(null)); } if (result.packet.sequenceNumber > targetSequenceNumber) { // We found the next packet! - const newPacket = this.createEncodedPacket(result.packet, result.duration, options); - this.packetBuffers.set(newPacket, buffer); - this.packetSectionStarts.set(newPacket, result.packet.sectionStartPos); - return newPacket; + const newPacket = this.createEncodedPacket(result.packet, result.duration, buffer, options); + return res.set(new PacketRetrievalResult(newPacket)); } } } - async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { + async getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { let currentPacket: EncodedPacket | null = packet; + const nextResult = new ResultValue(); + // Just loop until we hit one while (true) { - currentPacket = await this.getNextPacket(currentPacket, options); + nextResult.reset(); + const promise = this.getNextPacket(nextResult, currentPacket, options); + if (nextResult.pending) await promise; + + currentPacket = nextResult.value.packet; if (!currentPacket) { - return null; + return res.set(nextResult.value); } if (currentPacket.type === 'key') { - return currentPacket; + return res.set(nextResult.value); } } } - getPacket(timestamp: number, options: PacketRetrievalOptions): Promise { - return this.doPacketLookup(timestamp, false, options); + getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this.doPacketLookup(res, timestamp, false, options); } - getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise { - return this.doPacketLookup(timestamp, true, options); + getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this.doPacketLookup(res, timestamp, true, options); } /** @@ -1408,48 +1483,64 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { * make it more performant for small files and over high-latency readers such as the network. */ async doPacketLookup( + res: ResultValue, timestamp: number, keyframesOnly: boolean, options: PacketRetrievalOptions, - ): Promise { + ): MaybeRelevantPromise { const searchPts = roundIfAlmostInteger(timestamp * TIMESCALE); const demuxer = this.elementaryStream.demuxer; const { reader, seekChunkSize } = demuxer; const pid = this.elementaryStream.pid; + type ChunkSearchResult = { pesPacketHeader: TimestampedPesPacketHeader; section: Section } | null; + + const packetHeaderResult = new ResultValue(); + const sectionResult = new ResultValue
(); + const chunkSearchResult = new ResultValue(); + const findFirstPesPacketHeaderInChunk = async ( + res: ResultValue, startPos: number, endPos: number, readSectionInFull: boolean, - ) => { + ): MaybeRelevantPromise => { let currentPos = startPos; while (currentPos < endPos) { - const packetHeader = await demuxer.readPacketHeader(currentPos); + packetHeaderResult.reset(); + const packetHeaderPromise = demuxer.readPacketHeader(packetHeaderResult, currentPos); + if (packetHeaderResult.pending) await packetHeaderPromise; + + const packetHeader = packetHeaderResult.value; if (!packetHeader) { - return null; + return res.set(null); } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { - const section = await demuxer.readSection(currentPos, readSectionInFull); + sectionResult.reset(); + const sectionPromise = demuxer.readSection(sectionResult, currentPos, readSectionInFull); + if (sectionResult.pending) await sectionPromise; + + const section = sectionResult.value; if (!section) { - return null; + return res.set(null); } const pesPacketHeader = readPesPacketHeader(demuxer, section, false); if (pesPacketHeader && pesPacketHeader.pts !== null) { - return { + return res.set({ pesPacketHeader: pesPacketHeader as TimestampedPesPacketHeader, section, - }; + }); } } currentPos += demuxer.packetStride; } - return null; + return res.set(null); }; // Get the first PES packet of the track @@ -1460,7 +1551,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { if (searchPts < firstPesPacketHeader.pts) { // We're before the first packet, definitely nothing here - return null; + return res.set(new PacketRetrievalResult(null)); } let scanStartPos: number; @@ -1489,7 +1580,16 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { + firstPesPacketHeader.sectionStartPos; const chunkEndPos = chunkStartPos + seekChunkSize; - const result = await findFirstPesPacketHeaderInChunk(chunkStartPos, chunkEndPos, false); + chunkSearchResult.reset(); + const promise = findFirstPesPacketHeaderInChunk( + chunkSearchResult, + chunkStartPos, + chunkEndPos, + false, + ); + if (chunkSearchResult.pending) await promise; + + const result = chunkSearchResult.value; if (!result) { // No PES packet found in this chunk, search left @@ -1516,11 +1616,16 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { } // Find the first PES packet at or after scanStartPos - const result = await findFirstPesPacketHeaderInChunk( + chunkSearchResult.reset(); + const scanPromise = findFirstPesPacketHeaderInChunk( + chunkSearchResult, scanStartPos, reader.fileSize ?? Infinity, false, ); + if (chunkSearchResult.pending) await scanPromise; + + const result = chunkSearchResult.value; let currentPesHeader = result?.pesPacketHeader ?? null; if (!currentPesHeader) { @@ -1533,9 +1638,13 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { const retrieveEncodedPacket = async ( sectionStartPos: number, predicate: (packet: SuppliedPacket) => boolean, - ) => { + ): MaybeRelevantPromise => { // Load the relevant section in full - const section = await demuxer.readSection(sectionStartPos, true); + sectionResult.reset(); + const sectionPromise = demuxer.readSection(sectionResult, sectionStartPos, true); + if (sectionResult.pending) await sectionPromise; + + const section = sectionResult.value; assert(section); const pesPacket = readPesPacket(demuxer, section, true); @@ -1544,6 +1653,8 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { const context = new PacketReadingContext(this.elementaryStream, pesPacket); const buffer = new PacketBuffer(this, context); + const didReadResult = new ResultValue(); + // Advance until the top-most presentation timestamp crosses or equals searchPts while (true) { const topPts = last(buffer.presentationOrderPackets)?.pts ?? -Infinity; @@ -1551,15 +1662,18 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { break; } - const didRead = await buffer.readNextPacket(); - if (!didRead) { + didReadResult.reset(); + const promise = buffer.readNextPacket(didReadResult); + if (didReadResult.pending) await promise; + + if (!didReadResult.value) { break; } } const targetIndex = findLastIndex(buffer.presentationOrderPackets, predicate); if (targetIndex === -1) { - return null; + return res.set(new PacketRetrievalResult(null)); } const targetPacket = buffer.presentationOrderPackets[targetIndex]!; @@ -1573,14 +1687,15 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { } buffer.lastDuration = lastDuration; // Kinda ugly but necessary fix - const result = await buffer.readNext(); - assert(result); + const readResult = new ResultValue(); + const readPromise = buffer.readNext(readResult); + if (readResult.pending) await readPromise; - const packet = this.createEncodedPacket(result.packet, result.duration, options); - this.packetBuffers.set(packet, buffer); - this.packetSectionStarts.set(packet, result.packet.sectionStartPos); + const result = readResult.value; + assert(result); - return packet; + const packet = this.createEncodedPacket(result.packet, result.duration, buffer, options); + return res.set(new PacketRetrievalResult(packet)); }; if (!keyframesOnly || this.allPacketsAreKeyPackets()) { @@ -1594,13 +1709,21 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { let currentPos = currentPesHeader.sectionStartPos + demuxer.packetStride; while (true) { - const packetHeader = await demuxer.readPacketHeader(currentPos); + packetHeaderResult.reset(); + const packetHeaderPromise = demuxer.readPacketHeader(packetHeaderResult, currentPos); + if (packetHeaderResult.pending) await packetHeaderPromise; + + const packetHeader = packetHeaderResult.value; if (!packetHeader) { break outer; // End of file } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { - const section = await demuxer.readSection(currentPos, false); + sectionResult.reset(); + const sectionPromise = demuxer.readSection(sectionResult, currentPos, false); + if (sectionResult.pending) await sectionPromise; + + const section = sectionResult.value; if (section) { const nextPesHeader = readPesPacketHeader(demuxer, section, false); if (nextPesHeader && nextPesHeader.pts !== null) { @@ -1626,13 +1749,21 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { let pos = currentPesHeader.sectionStartPos - demuxer.packetStride; while (pos >= demuxer.packetOffset) { - const packetHeader = await demuxer.readPacketHeader(pos); + packetHeaderResult.reset(); + const packetHeaderPromise = demuxer.readPacketHeader(packetHeaderResult, pos); + if (packetHeaderResult.pending) await packetHeaderPromise; + + const packetHeader = packetHeaderResult.value; if (!packetHeader) { break outer; } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { - const section = await demuxer.readSection(pos, false); + sectionResult.reset(); + const sectionPromise = demuxer.readSection(sectionResult, pos, false); + if (sectionResult.pending) await sectionPromise; + + const section = sectionResult.value; if (section) { const header = readPesPacketHeader(demuxer, section, false); if (header && header.pts !== null) { @@ -1655,6 +1786,7 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { let nextChunkStartPos: number | null = null; // "next" as in later in the file, even tho we scan backwards const readSectionsInFull = !this.elementaryStream.canBeTrustedWithKeyPackets; + const markResult = new ResultValue(); while (true) { let bestKeyPesHeader: TimestampedPesPacketHeader | null = null; @@ -1668,11 +1800,16 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { pesHeader = firstPesPacketHeader; pesHeaderSection = firstSection; } else { - const result = await findFirstPesPacketHeaderInChunk( + chunkSearchResult.reset(); + const promise = findFirstPesPacketHeaderInChunk( + chunkSearchResult, currentChunkStartPos, reader.fileSize ?? Infinity, readSectionsInFull, ); + if (chunkSearchResult.pending) await promise; + + const result = chunkSearchResult.value; pesHeader = result?.pesPacketHeader ?? null; pesHeaderSection = result?.section ?? null; @@ -1698,7 +1835,9 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { assert(pesPacket); const context = new PacketReadingContext(this.elementaryStream, pesPacket); - await context.markNextPacket(); + markResult.reset(); + const markPromise = context.markNextPacket(markResult); + if (markResult.pending) await markPromise; isKeyPacket = context.suppliedPacket?.randomAccessIndicator === 1; } @@ -1725,13 +1864,21 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { let currentPos = pesHeader.sectionStartPos + demuxer.packetStride; while (true) { - const packetHeader = await demuxer.readPacketHeader(currentPos); + packetHeaderResult.reset(); + const packetHeaderPromise = demuxer.readPacketHeader(packetHeaderResult, currentPos); + if (packetHeaderResult.pending) await packetHeaderPromise; + + const packetHeader = packetHeaderResult.value; if (!packetHeader) { break outer; // End of file } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { - const section = await demuxer.readSection(currentPos, readSectionsInFull); + sectionResult.reset(); + const sectionPromise = demuxer.readSection(sectionResult, currentPos, readSectionsInFull); + if (sectionResult.pending) await sectionPromise; + + const section = sectionResult.value; if (section) { const nextPesHeader = readPesPacketHeader(demuxer, section, false); @@ -1759,13 +1906,21 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { let pos = startPesHeader.sectionStartPos - demuxer.packetStride; while (pos >= demuxer.packetOffset) { - const packetHeader = await demuxer.readPacketHeader(pos); + packetHeaderResult.reset(); + const packetHeaderPromise = demuxer.readPacketHeader(packetHeaderResult, pos); + if (packetHeaderResult.pending) await packetHeaderPromise; + + const packetHeader = packetHeaderResult.value; if (!packetHeader) { break outer; } if (packetHeader.pid === pid && packetHeader.payloadUnitStartIndicator === 1) { - const section = await demuxer.readSection(pos, readSectionsInFull); + sectionResult.reset(); + const sectionPromise = demuxer.readSection(sectionResult, pos, readSectionsInFull); + if (sectionResult.pending) await sectionPromise; + + const section = sectionResult.value; if (section) { const header = readPesPacketHeader(demuxer, section, false); if (header && header.pts !== null) { @@ -1780,17 +1935,15 @@ abstract class MpegTsTrackBacking implements InputTrackBacking { } } - const encodedPacket = await retrieveEncodedPacket( + // There must be one that matches + return retrieveEncodedPacket( startPesHeader.sectionStartPos, p => p.pts <= searchPts && p.randomAccessIndicator === 1, ); - assert(encodedPacket); // There must be one - - return encodedPacket; } if (isFirstChunk) { - return null; + return res.set(new PacketRetrievalResult(null)); } // No key frame found in this chunk, move one chunk to the left @@ -1945,6 +2098,11 @@ type SuppliedPacket = { randomAccessIndicator: number; }; +type PacketWithDuration = { + packet: SuppliedPacket; + duration: number; +}; + /** Stateful context used to extract exact encoded packets from the underlying data stream. */ class PacketReadingContext { elementaryStream: ElementaryStream; @@ -1969,25 +2127,15 @@ class PacketReadingContext { this.startingPesPacket = startingPesPacket; } - ensureBuffered(length: number) { - const remaining = this.endPos - this.currentPos; - if (remaining >= length) { - return length; - } - - return this.bufferData(length - remaining) - .then(() => Math.min(this.endPos - this.currentPos, length)); - } + async ensureBuffered(res: ResultValue, length: number): MaybeRelevantPromise { + const targetEndPos = this.currentPos + length; - getCurrentPesPacket() { - const packet = this.pesPackets[this.currentPesPacketIndex]; - assert(packet); - - return packet; - } + if (this.endPos >= targetEndPos) { + return res.set(length); + } - async bufferData(length: number): Promise { - const targetEndPos = this.endPos + length; + const packetHeaderResult = new ResultValue(); + const sectionResult = new ResultValue
(); while (this.endPos < targetEndPos) { let pesPacket: PesPacket; @@ -1999,15 +2147,23 @@ class PacketReadingContext { assert(currentPos !== null); while (true) { - const packetHeader = await this.demuxer.readPacketHeader(currentPos); + packetHeaderResult.reset(); + const packetHeaderPromise = this.demuxer.readPacketHeader(packetHeaderResult, currentPos); + if (packetHeaderResult.pending) await packetHeaderPromise; + + const packetHeader = packetHeaderResult.value; if (!packetHeader) { - return; + return res.set(Math.min(this.endPos - this.currentPos, length)); } if (packetHeader.pid === this.pid) { - const nextSection = await this.demuxer.readSection(currentPos, true); + sectionResult.reset(); + const sectionPromise = this.demuxer.readSection(sectionResult, currentPos, true); + if (sectionResult.pending) await sectionPromise; + + const nextSection = sectionResult.value; if (!nextSection) { - return; + return res.set(Math.min(this.endPos - this.currentPos, length)); } const nextPesPacket = readPesPacket(this.demuxer, nextSection, false); @@ -2024,6 +2180,15 @@ class PacketReadingContext { this.pesPackets.push(pesPacket); this.endPos += pesPacket.data.byteLength; } + + return res.set(length); + } + + getCurrentPesPacket() { + const packet = this.pesPackets[this.currentPesPacketIndex]; + assert(packet); + + return packet; } readBytes(length: number) { @@ -2116,10 +2281,11 @@ class PacketReadingContext { this.currentPesPacketIndex++; } - async markNextPacket() { + async markNextPacket(res: ResultValue): MaybeRelevantPromise { assert(!this.suppliedPacket); const elementaryStream = this.elementaryStream; + const bufferedResult = new ResultValue(); if (elementaryStream.info.type === 'video') { // Our job here is to separate the video stream into access units. Sometimes this is easy (like when AUDs @@ -2138,8 +2304,11 @@ class PacketReadingContext { let lastFirstMacroblockInSlice = 0; while (true) { - let remaining = this.ensureBuffered(CHUNK_SIZE); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const bufferedPromise = this.ensureBuffered(bufferedResult, CHUNK_SIZE); + if (bufferedResult.pending) await bufferedPromise; + + const remaining = bufferedResult.value; if (remaining === 0) { break; @@ -2269,7 +2438,8 @@ class PacketReadingContext { // End the packet at this start code (the next frame begins here) const packetLength = startCodePos - packetStartPos; this.seekTo(packetStartPos); - return this.supplyPacket(packetLength, 0); + this.supplyPacket(packetLength, 0); + return res.set(); } i += startCodeLength; @@ -2285,15 +2455,20 @@ class PacketReadingContext { if (packetStartPos !== null && this.endPos > packetStartPos) { const packetLength = this.endPos - packetStartPos; this.seekTo(packetStartPos); - return this.supplyPacket(packetLength, 0); + this.supplyPacket(packetLength, 0); } + + return res.set(); } else { const codec = elementaryStream.info.codec; const CHUNK_SIZE = 128; while (true) { - let remaining = this.ensureBuffered(CHUNK_SIZE); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const bufferedPromise = this.ensureBuffered(bufferedResult, CHUNK_SIZE); + if (bufferedResult.pending) await bufferedPromise; + + const remaining = bufferedResult.value; const startPos = this.currentPos; @@ -2308,11 +2483,12 @@ class PacketReadingContext { this.skip(-1); const possibleHeaderStartPos = this.currentPos; - let remaining = this.ensureBuffered(MAX_ADTS_FRAME_HEADER_SIZE); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const headerPromise = this.ensureBuffered(bufferedResult, MAX_ADTS_FRAME_HEADER_SIZE); + if (bufferedResult.pending) await headerPromise; - if (remaining < MAX_ADTS_FRAME_HEADER_SIZE) { - return; + if (bufferedResult.value < MAX_ADTS_FRAME_HEADER_SIZE) { + return res.set(); } const headerBytes = this.readBytes(MAX_ADTS_FRAME_HEADER_SIZE); @@ -2321,13 +2497,15 @@ class PacketReadingContext { if (header) { this.seekTo(possibleHeaderStartPos); - let remaining = this.ensureBuffered(header.frameLength); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const framePromise = this.ensureBuffered(bufferedResult, header.frameLength); + if (bufferedResult.pending) await framePromise; - return this.supplyPacket( - remaining, + this.supplyPacket( + bufferedResult.value, Math.round(SAMPLES_PER_AAC_FRAME * TIMESCALE / elementaryStream.info.sampleRate), ); + return res.set(); } else { this.seekTo(possibleHeaderStartPos + 1); } @@ -2339,11 +2517,12 @@ class PacketReadingContext { this.skip(-1); const possibleHeaderStartPos = this.currentPos; - let remaining = this.ensureBuffered(MP3_FRAME_HEADER_SIZE); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const headerPromise = this.ensureBuffered(bufferedResult, MP3_FRAME_HEADER_SIZE); + if (bufferedResult.pending) await headerPromise; - if (remaining < MP3_FRAME_HEADER_SIZE) { - return; + if (bufferedResult.value < MP3_FRAME_HEADER_SIZE) { + return res.set(); } const headerBytes = this.readBytes(MP3_FRAME_HEADER_SIZE); @@ -2353,12 +2532,14 @@ class PacketReadingContext { if (result.header) { this.seekTo(possibleHeaderStartPos); - let remaining = this.ensureBuffered(result.header.totalSize); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const framePromise = this.ensureBuffered(bufferedResult, result.header.totalSize); + if (bufferedResult.pending) await framePromise; const duration = result.header.audioSamplesInFrame * TIMESCALE / elementaryStream.info.sampleRate; - return this.supplyPacket(remaining, Math.round(duration)); + this.supplyPacket(bufferedResult.value, Math.round(duration)); + return res.set(); } else { this.seekTo(possibleHeaderStartPos + 1); } @@ -2371,11 +2552,12 @@ class PacketReadingContext { const possibleSyncPos = this.currentPos; // Need at least 5 bytes for sync word + CRC + fscod/frmsizecod - let remaining = this.ensureBuffered(5); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const headerPromise = this.ensureBuffered(bufferedResult, 5); + if (bufferedResult.pending) await headerPromise; - if (remaining < 5) { - return; + if (bufferedResult.value < 5) { + return res.set(); } const headerBytes = this.readBytes(5); @@ -2400,13 +2582,15 @@ class PacketReadingContext { this.seekTo(possibleSyncPos); - remaining = this.ensureBuffered(frameSize); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const framePromise = this.ensureBuffered(bufferedResult, frameSize); + if (bufferedResult.pending) await framePromise; const duration = Math.round( AC3_SAMPLES_PER_FRAME * TIMESCALE / elementaryStream.info.sampleRate, ); - return this.supplyPacket(remaining, duration); + this.supplyPacket(bufferedResult.value, duration); + return res.set(); } else if (codec === 'eac3') { if (byte !== 0x0b) { continue; @@ -2416,11 +2600,12 @@ class PacketReadingContext { const possibleSyncPos = this.currentPos; // Need at least 5 bytes for E-AC-3 header parsing (sync word + frmsiz + fscod/numblkscod) - let remaining = this.ensureBuffered(5); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const headerPromise = this.ensureBuffered(bufferedResult, 5); + if (bufferedResult.pending) await headerPromise; - if (remaining < 5) { - return; + if (bufferedResult.value < 5) { + return res.set(); } const headerBytes = this.readBytes(5); @@ -2438,15 +2623,17 @@ class PacketReadingContext { this.seekTo(possibleSyncPos); - remaining = this.ensureBuffered(frameSize); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const framePromise = this.ensureBuffered(bufferedResult, frameSize); + if (bufferedResult.pending) await framePromise; // Duration = numblks * 256 samples per block const samplesPerFrame = numblks * 256; const duration = Math.round( samplesPerFrame * TIMESCALE / elementaryStream.info.sampleRate, ); - return this.supplyPacket(remaining, duration); + this.supplyPacket(bufferedResult.value, duration); + return res.set(); } else if (codec === 'dts') { if (byte !== 0x7f && byte !== 0x64) { continue; @@ -2455,11 +2642,12 @@ class PacketReadingContext { this.skip(-1); const possibleSyncPos = this.currentPos; - let remaining = this.ensureBuffered(DTS_CORE_FRAME_HEADER_SIZE); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const headerPromise = this.ensureBuffered(bufferedResult, DTS_CORE_FRAME_HEADER_SIZE); + if (bufferedResult.pending) await headerPromise; - if (remaining < DTS_CORE_FRAME_HEADER_SIZE) { - return; + if (bufferedResult.value < DTS_CORE_FRAME_HEADER_SIZE) { + return res.set(); } const headerBytes = this.readBytes(DTS_CORE_FRAME_HEADER_SIZE); @@ -2477,10 +2665,11 @@ class PacketReadingContext { this.seekTo(possibleSyncPos); const headerBound = Math.min(leadingExss.frameSize, DTS_EXSS_MAX_HEADER_SIZE); - let remaining = this.ensureBuffered(headerBound); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const boundPromise = this.ensureBuffered(bufferedResult, headerBound); + if (bufferedResult.pending) await boundPromise; - leadingExss = parseDtsExssHeader(this.readBytes(remaining)) ?? leadingExss; + leadingExss = parseDtsExssHeader(this.readBytes(bufferedResult.value)) ?? leadingExss; } let frameSize = core ? core.frameSize : leadingExss!.frameSize; @@ -2495,10 +2684,11 @@ class PacketReadingContext { this.seekTo(possibleSyncPos); const neededBytes = nextSubstreamPos + DTS_EXSS_HEADER_PREFIX_SIZE; - let remaining = this.ensureBuffered(neededBytes); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const neededPromise = this.ensureBuffered(bufferedResult, neededBytes); + if (bufferedResult.pending) await neededPromise; - if (remaining < neededBytes) { + if (bufferedResult.value < neededBytes) { break; } @@ -2523,13 +2713,15 @@ class PacketReadingContext { this.seekTo(possibleSyncPos); - remaining = this.ensureBuffered(frameSize); - if (isThenable(remaining)) remaining = await remaining; + bufferedResult.reset(); + const framePromise = this.ensureBuffered(bufferedResult, frameSize); + if (bufferedResult.pending) await framePromise; const duration = Math.round( sampleCount * TIMESCALE / elementaryStream.info.sampleRate, ); - return this.supplyPacket(remaining, duration); + this.supplyPacket(bufferedResult.value, duration); + return res.set(); } else { throw new Error('Unhandled.'); } @@ -2539,6 +2731,8 @@ class PacketReadingContext { break; } } + + return res.set(); } } @@ -2624,19 +2818,39 @@ class PacketBuffer { assert(this.reorderSize >= 0); } - async readNext(): Promise<{ packet: SuppliedPacket; duration: number } | null> { + async readNext(res: ResultValue): MaybeRelevantPromise { + const readResult = new ResultValue(); + if (this.decodeOrderPackets.length === 0) { // We need the next packet - const didRead = await this.readNextPacket(); - if (!didRead) { - return null; + const promise = this.readNextPacket(readResult); + if (readResult.pending) await promise; + + if (!readResult.value) { + return res.set(null); } } + const packet = this.decodeOrderPackets[0]; + assert(packet); + // Ensure we know the next packet in presentation order so we can compute the current packet's duration - await this.ensureCurrentPacketHasNext(); + while (true) { + const presentationIndex = this.presentationOrderPackets.indexOf(packet); - const packet = this.decodeOrderPackets[0]!; + // Check if the current packet has a next packet + if (presentationIndex !== -1 && presentationIndex <= this.presentationOrderPackets.length - 2) { + break; + } + + readResult.reset(); + const promise = this.readNextPacket(readResult); + if (readResult.pending) await promise; + + if (!readResult.value) { + break; + } + } // Let's compute the duration const presentationIndex = this.presentationOrderPackets.indexOf(packet); @@ -2663,12 +2877,12 @@ class PacketBuffer { this.presentationOrderPackets.shift(); } - return { packet, duration }; + return res.set({ packet, duration }); } - async readNextPacket() { + async readNextPacket(res: ResultValue): MaybeRelevantPromise { if (this.reachedEnd) { - return false; + return res.set(false); } let suppliedPacket: SuppliedPacket | null; @@ -2676,7 +2890,10 @@ class PacketBuffer { // Small optimization: there was already a supplied packet in the context, so let's first use that one suppliedPacket = this.context.suppliedPacket; } else { - await this.context.markNextPacket(); + const markResult = new ResultValue(); + const promise = this.context.markNextPacket(markResult); + if (markResult.pending) await promise; + suppliedPacket = this.context.suppliedPacket; } this.context.suppliedPacket = null; @@ -2685,32 +2902,13 @@ class PacketBuffer { this.reachedEnd = true; this.flushReorderBuffer(); - return false; + return res.set(false); } this.decodeOrderPackets.push(suppliedPacket); this.processPacketThroughReorderBuffer(suppliedPacket); - return true; - } - - async ensureCurrentPacketHasNext() { - const current = this.decodeOrderPackets[0]; - assert(current); - - while (true) { - const presentationIndex = this.presentationOrderPackets.indexOf(current); - - // Check if current packet has a next packet - if (presentationIndex !== -1 && presentationIndex <= this.presentationOrderPackets.length - 2) { - break; - } - - const didRead = await this.readNextPacket(); - if (!didRead) { - break; - } - } + return res.set(true); } processPacketThroughReorderBuffer(packet: SuppliedPacket) { diff --git a/src/mpeg-ts/mpeg-ts-muxer.ts b/src/mpeg-ts/mpeg-ts-muxer.ts index 4edb9430a..cec8dc3d4 100644 --- a/src/mpeg-ts/mpeg-ts-muxer.ts +++ b/src/mpeg-ts/mpeg-ts-muxer.ts @@ -100,11 +100,10 @@ export class MpegTsMuxer extends Muxer { } async start() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.writer = await this.output._getRootWriter(true); - - release(); } async getMimeType() { @@ -233,32 +232,29 @@ export class MpegTsMuxer extends Muxer { packet: EncodedPacket, meta?: EncodedVideoChunkMetadata, ) { - const release = await this.mutex.acquire(); - - try { - const trackData = this.getVideoTrackData(track, meta); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - this.validateTimestamp( - trackData.track, - packet.timestamp, - packet.type === 'key', - ); + const trackData = this.getVideoTrackData(track, meta); - const preparedData = this.prepareVideoPacket(trackData, packet, meta); + this.validateTimestamp( + trackData.track, + packet.timestamp, + packet.type === 'key', + ); - if (packet.type === 'key') { - await this.flushTimestampQueue(trackData); - } + const preparedData = this.prepareVideoPacket(trackData, packet, meta); - trackData.timestampProcessingQueue.push({ - data: preparedData, - presentationTimestamp: packet.timestamp, - decodeTimestamp: null, - isKeyframe: packet.type === 'key', - }); - } finally { - release(); + if (packet.type === 'key') { + await this.flushTimestampQueue(trackData); } + + trackData.timestampProcessingQueue.push({ + data: preparedData, + presentationTimestamp: packet.timestamp, + decodeTimestamp: null, + isKeyframe: packet.type === 'key', + }); } async addEncodedAudioPacket( @@ -266,32 +262,29 @@ export class MpegTsMuxer extends Muxer { packet: EncodedPacket, meta?: EncodedAudioChunkMetadata, ) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - const trackData = this.getAudioTrackData(track, meta); + const trackData = this.getAudioTrackData(track, meta); - this.validateTimestamp( - trackData.track, - packet.timestamp, - packet.type === 'key', - ); + this.validateTimestamp( + trackData.track, + packet.timestamp, + packet.type === 'key', + ); - const preparedData = this.prepareAudioPacket(trackData, packet, meta); + const preparedData = this.prepareAudioPacket(trackData, packet, meta); - if (packet.type === 'key') { - await this.flushTimestampQueue(trackData); - } - - trackData.timestampProcessingQueue.push({ - data: preparedData, - presentationTimestamp: packet.timestamp, - decodeTimestamp: null, - isKeyframe: packet.type === 'key', - }); - } finally { - release(); + if (packet.type === 'key') { + await this.flushTimestampQueue(trackData); } + + trackData.timestampProcessingQueue.push({ + data: preparedData, + presentationTimestamp: packet.timestamp, + decodeTimestamp: null, + isKeyframe: packet.type === 'key', + }); } async addSubtitleCue(): Promise { @@ -729,7 +722,8 @@ export class MpegTsMuxer extends Muxer { // eslint-disable-next-line @typescript-eslint/no-misused-promises override async onTrackClose(track: OutputTrack) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; const trackData = this.trackDatas.find(x => x.track === track); if (trackData) { @@ -742,12 +736,11 @@ export class MpegTsMuxer extends Muxer { } await this.interleavePackets(); - - release(); } async finalize() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.allTracksKnown.resolve(); @@ -757,8 +750,6 @@ export class MpegTsMuxer extends Muxer { } await this.interleavePackets(true); - - release(); } } diff --git a/src/ogg/ogg-demuxer.ts b/src/ogg/ogg-demuxer.ts index 0d6c5aaae..dc2ba90f4 100644 --- a/src/ogg/ogg-demuxer.ts +++ b/src/ogg/ogg-demuxer.ts @@ -11,7 +11,6 @@ import { parseModesFromVorbisSetupPacket, parseOpusIdentificationHeader, readVor import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrackBacking } from '../input-track'; -import { PacketRetrievalOptions } from '../media-sink'; import { DEFAULT_TRACK_DISPOSITION, MetadataTags, TrackDisposition } from '../metadata'; import { assert, @@ -21,11 +20,13 @@ import { findLast, isThenable, last, + MaybeRelevantPromise, + ResultValue, roundIfAlmostInteger, toDataView, UNDETERMINED_LANGUAGE, } from '../misc'; -import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; +import { EncodedPacket, PacketRetrievalOptions, PacketRetrievalResult, PLACEHOLDER_DATA } from '../packet'; import { readBytes, Reader } from '../reader'; import { buildOggMimeType, computeOggPageCrc, extractSampleMetadata, OggCodecInfo } from './ogg-misc'; import { @@ -54,6 +55,11 @@ type Packet = { endSegmentIndex: number; }; +type PacketStart = { + startPage: Page; + startSegmentIndex: number; +}; + export class OggDemuxer extends Demuxer { reader: Reader; @@ -107,7 +113,10 @@ export class OggDemuxer extends Demuxer { } for (const bitstream of this.bitstreams) { - const firstPacket = await this.readPacket(bitstream.bosPage, 0); + const packetResult = new ResultValue(); + await this.readPacket(packetResult, bitstream.bosPage, 0); + + const firstPacket = packetResult.value; if (!firstPacket) { continue; } @@ -147,22 +156,32 @@ export class OggDemuxer extends Demuxer { } async readVorbisMetadata(firstPacket: Packet, bitstream: LogicalBitstream) { - let nextPacketPosition = await this.findNextPacketStart(firstPacket); + const positionResult = new ResultValue(); + await this.findNextPacketStart(positionResult, firstPacket); + + let nextPacketPosition = positionResult.value; if (!nextPacketPosition) { return; } - const secondPacket = await this.readPacket(nextPacketPosition.startPage, nextPacketPosition.startSegmentIndex); + const packetResult = new ResultValue(); + await this.readPacket(packetResult, nextPacketPosition.startPage, nextPacketPosition.startSegmentIndex); + + const secondPacket = packetResult.value; if (!secondPacket) { return; } - nextPacketPosition = await this.findNextPacketStart(secondPacket); + await this.findNextPacketStart(positionResult, secondPacket); + + nextPacketPosition = positionResult.value; if (!nextPacketPosition) { return; } - const thirdPacket = await this.readPacket(nextPacketPosition.startPage, nextPacketPosition.startSegmentIndex); + await this.readPacket(packetResult, nextPacketPosition.startPage, nextPacketPosition.startSegmentIndex); + + const thirdPacket = packetResult.value; if (!thirdPacket) { return; } @@ -230,15 +249,19 @@ export class OggDemuxer extends Demuxer { // From https://datatracker.ietf.org/doc/html/rfc7845#section-5: // "An Ogg Opus logical stream contains exactly two mandatory header packets: an identification header and a // comment header." - const nextPacketPosition = await this.findNextPacketStart(firstPacket); + + const positionResult = new ResultValue(); + await this.findNextPacketStart(positionResult, firstPacket); + + const nextPacketPosition = positionResult.value; if (!nextPacketPosition) { return; } - const secondPacket = await this.readPacket( - nextPacketPosition.startPage, - nextPacketPosition.startSegmentIndex, - ); + const packetResult = new ResultValue(); + await this.readPacket(packetResult, nextPacketPosition.startPage, nextPacketPosition.startSegmentIndex); + + const secondPacket = packetResult.value; if (!secondPacket) { return; } @@ -258,7 +281,11 @@ export class OggDemuxer extends Demuxer { readVorbisComments(secondPacket.data.subarray(8), this.metadataTags); // Skip 'OpusTags' } - async readPacket(startPage: Page, startSegmentIndex: number): Promise { + async readPacket( + res: ResultValue, + startPage: Page, + startSegmentIndex: number, + ): MaybeRelevantPromise { assert(startSegmentIndex < startPage.lacingValues.length); let startDataOffset = 0; @@ -303,12 +330,12 @@ export class OggDemuxer extends Demuxer { let headerSlice = this.reader.requestSliceRange(currentPos, MIN_PAGE_HEADER_SIZE, MAX_PAGE_HEADER_SIZE); if (isThenable(headerSlice)) headerSlice = await headerSlice; if (!headerSlice) { - return null; + return res.set(null); } const nextPage = readPageHeader(headerSlice); if (!nextPage) { - return null; + return res.set(null); } currentPage = nextPage; @@ -325,35 +352,43 @@ export class OggDemuxer extends Demuxer { const totalPacketSize = chunks.reduce((sum, chunk) => sum + chunk.length, 0); if (totalPacketSize === 0) { - return null; // Invalid packet, treat it as end of stream + return res.set(null); // Invalid packet, treat it as end of stream } - const packetData = new Uint8Array(totalPacketSize); + let packetData: Uint8Array; - let offset = 0; - for (let i = 0; i < chunks.length; i++) { - const chunk = chunks[i]!; - packetData.set(chunk, offset); - offset += chunk.length; + if (chunks.length === 1) { + // Fast path, no need for an allocation. Also typically the common path! + packetData = chunks[0]!; + } else { + const totalPacketSize = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + packetData = new Uint8Array(totalPacketSize); + + let offset = 0; + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]!; + packetData.set(chunk, offset); + offset += chunk.length; + } } - return { + return res.set({ data: packetData, endPage: currentPage, endSegmentIndex: currentSegmentIndex, - }; + }); } - async findNextPacketStart(lastPacket: Packet) { + async findNextPacketStart(res: ResultValue, lastPacket: Packet): MaybeRelevantPromise { // If there's another segment in the same page, return it if (lastPacket.endSegmentIndex < lastPacket.endPage.lacingValues.length - 1) { - return { startPage: lastPacket.endPage, startSegmentIndex: lastPacket.endSegmentIndex + 1 }; + return res.set({ startPage: lastPacket.endPage, startSegmentIndex: lastPacket.endSegmentIndex + 1 }); } const isEos = !!(lastPacket.endPage.headerType & 0x04); if (isEos) { // The page is marked as the last page of the logical bitstream, so we won't find anything beyond it - return null; + return res.set(null); } // Otherwise, search for the next page belonging to the same bitstream @@ -362,16 +397,16 @@ export class OggDemuxer extends Demuxer { let slice = this.reader.requestSliceRange(currentPos, MIN_PAGE_HEADER_SIZE, MAX_PAGE_HEADER_SIZE); if (isThenable(slice)) slice = await slice; if (!slice) { - return null; + return res.set(null); } const nextPage = readPageHeader(slice); if (!nextPage) { - return null; + return res.set(null); } if (nextPage.serialNumber === lastPacket.endPage.serialNumber) { - return { startPage: nextPage, startSegmentIndex: 0 }; + return res.set({ startPage: nextPage, startSegmentIndex: 0 }); } currentPos = nextPage.headerStartPos + nextPage.totalSize; @@ -411,10 +446,9 @@ type EncodedPacketMetadata = { class OggAudioTrackBacking implements InputAudioTrackBacking { internalSampleRate: number; - encodedPacketToMetadata = new WeakMap(); sequentialScanCache: EncodedPacketMetadata[] = []; sequentialScanMutex = new AsyncMutex(); - firstTimestampInSamplesPromise: Promise | null = null; + firstTimestampInSamples: number | null = null; constructor(public bitstream: LogicalBitstream, public demuxer: OggDemuxer) { // Opus always uses a fixed sample rate for its internal calculations, even if the actual rate is different @@ -565,133 +599,186 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { packet.data.byteLength, ); - this.encodedPacketToMetadata.set(encodedPacket, { + encodedPacket._internal = { packet, timestampInSamples: additional.timestampInSamples, durationInSamples, vorbisLastBlockSize: additional.vorbisLastBlocksize, vorbisBlockSize, - }); + } satisfies EncodedPacketMetadata; + return encodedPacket; } - getFirstTimestampInSamples() { - return this.firstTimestampInSamplesPromise ??= (async () => { - const defaultTimestampInSamples = this.granulePositionToTimestampInSamples(0); + async getFirstTimestampInSamples(res: ResultValue): MaybeRelevantPromise { + if (this.firstTimestampInSamples !== null) { + return res.set(this.firstTimestampInSamples); + } - assert(this.bitstream.lastMetadataPacket); - let position = await this.demuxer.findNextPacketStart(this.bitstream.lastMetadataPacket); - if (!position) { - return defaultTimestampInSamples; - } + const defaultTimestampInSamples = this.granulePositionToTimestampInSamples(0); + + assert(this.bitstream.lastMetadataPacket); - // The first page on which a packet completes may have a granule position larger than the number of - // samples in the packets completing on it, meaning the stream starts at a later time. So, let's sum up the - // durations of these packets and compare. - let firstEndPage: Page | null = null; - let totalDurationInSamples = 0; - let vorbisLastBlocksize: number | null = null; + const positionResult = new ResultValue(); + let promise = this.demuxer.findNextPacketStart(positionResult, this.bitstream.lastMetadataPacket); + if (positionResult.pending) await promise; - while (position) { - const packet = await this.demuxer.readPacket(position.startPage, position.startSegmentIndex); - if (!packet) { - break; - } + let position = positionResult.value; + if (!position) { + return res.set(this.firstTimestampInSamples = defaultTimestampInSamples); + } - firstEndPage ??= packet.endPage; - if (packet.endPage.headerStartPos !== firstEndPage.headerStartPos) { - break; - } + // The first page on which a packet completes may have a granule position larger than the number of samples in + // the packets completing on it, meaning the stream starts at a later time. So, let's sum up the durations of + // these packets and compare. + let firstEndPage: Page | null = null; + let totalDurationInSamples = 0; + let vorbisLastBlocksize: number | null = null; + const packetResult = new ResultValue(); - const { durationInSamples, vorbisBlockSize } = extractSampleMetadata( - packet.data, - this.bitstream.codecInfo, - vorbisLastBlocksize, - ); - totalDurationInSamples += durationInSamples; - vorbisLastBlocksize = vorbisBlockSize; + while (position) { + packetResult.reset(); + promise = this.demuxer.readPacket(packetResult, position.startPage, position.startSegmentIndex); + if (packetResult.pending) await promise; - position = await this.demuxer.findNextPacketStart(packet); + const packet = packetResult.value; + if (!packet) { + break; } - if (!firstEndPage) { - return defaultTimestampInSamples; + firstEndPage ??= packet.endPage; + if (packet.endPage.headerStartPos !== firstEndPage.headerStartPos) { + break; } - // A smaller granule position is only legal on the final page, where it signals end trimming - return defaultTimestampInSamples + Math.max(firstEndPage.granulePosition - totalDurationInSamples, 0); - })(); + const { durationInSamples, vorbisBlockSize } = extractSampleMetadata( + packet.data, + this.bitstream.codecInfo, + vorbisLastBlocksize, + ); + totalDurationInSamples += durationInSamples; + vorbisLastBlocksize = vorbisBlockSize; + + positionResult.reset(); + promise = this.demuxer.findNextPacketStart(positionResult, packet); + if (positionResult.pending) await promise; + + position = positionResult.value; + } + + if (!firstEndPage) { + return res.set(this.firstTimestampInSamples = defaultTimestampInSamples); + } + + // A smaller granule position is only legal on the final page, where it signals end trimming + this.firstTimestampInSamples = defaultTimestampInSamples + + Math.max(firstEndPage.granulePosition - totalDurationInSamples, 0); + return res.set(this.firstTimestampInSamples); } - async getFirstPacket(options: PacketRetrievalOptions) { + async getFirstPacket( + res: ResultValue, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { assert(this.bitstream.lastMetadataPacket); - const packetPosition = await this.demuxer.findNextPacketStart(this.bitstream.lastMetadataPacket); + + const positionResult = new ResultValue(); + let promise = this.demuxer.findNextPacketStart(positionResult, this.bitstream.lastMetadataPacket); + if (positionResult.pending) await promise; + + const packetPosition = positionResult.value; + if (!packetPosition) { - return null; + return res.set(new PacketRetrievalResult(null)); } - const timestampInSamples = await this.getFirstTimestampInSamples(); - const packet = await this.demuxer.readPacket(packetPosition.startPage, packetPosition.startSegmentIndex); + const firstTimestampResult = new ResultValue(); + promise = this.getFirstTimestampInSamples(firstTimestampResult); + if (firstTimestampResult.pending) await promise; - return this.createEncodedPacketFromOggPacket( - packet, + const timestampInSamples = firstTimestampResult.value; + + const packetResult = new ResultValue(); + promise = this.demuxer.readPacket(packetResult, packetPosition.startPage, packetPosition.startSegmentIndex); + if (packetResult.pending) await promise; + + return res.set(new PacketRetrievalResult(this.createEncodedPacketFromOggPacket( + packetResult.value, { timestampInSamples, vorbisLastBlocksize: null, }, options, - ); + ))); } - async getNextPacket(prevPacket: EncodedPacket, options: PacketRetrievalOptions) { - const prevMetadata = this.encodedPacketToMetadata.get(prevPacket); - if (!prevMetadata) { - throw new Error('Packet was not created from this track.'); - } + async getNextPacket( + res: ResultValue, + prevPacket: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + const prevMetadata = prevPacket._internal as EncodedPacketMetadata | undefined; + assert(prevMetadata); + + const positionResult = new ResultValue(); + let promise = this.demuxer.findNextPacketStart(positionResult, prevMetadata.packet); + if (positionResult.pending) await promise; - const packetPosition = await this.demuxer.findNextPacketStart(prevMetadata.packet); + const packetPosition = positionResult.value; if (!packetPosition) { - return null; + return res.set(new PacketRetrievalResult(null)); } const timestampInSamples = prevMetadata.timestampInSamples + prevMetadata.durationInSamples; - const packet = await this.demuxer.readPacket( - packetPosition.startPage, - packetPosition.startSegmentIndex, - ); + const packetResult = new ResultValue(); + promise = this.demuxer.readPacket(packetResult, packetPosition.startPage, packetPosition.startSegmentIndex); + if (packetResult.pending) await promise; - return this.createEncodedPacketFromOggPacket( - packet, + return res.set(new PacketRetrievalResult(this.createEncodedPacketFromOggPacket( + packetResult.value, { timestampInSamples, vorbisLastBlocksize: prevMetadata.vorbisBlockSize, }, options, - ); + ))); } - async getPacket(timestamp: number, options: PacketRetrievalOptions) { + async getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { if (this.demuxer.reader.fileSize === null) { // No file size known, can't do binary search, but fall back to sequential algo instead - return this.getPacketSequential(timestamp, options); + return this.getPacketSequential(res, timestamp, options); } const timestampInSamples = roundIfAlmostInteger(timestamp * this.internalSampleRate); - const firstTimestampInSamples = await this.getFirstTimestampInSamples(); + const firstTimestampResult = new ResultValue(); + const firstTimestampPromise = this.getFirstTimestampInSamples(firstTimestampResult); + if (firstTimestampResult.pending) await firstTimestampPromise; + + const firstTimestampInSamples = firstTimestampResult.value; if (timestampInSamples === Math.max(firstTimestampInSamples, 0)) { // Fast path for the first timestamp - avoids binary search when playing back from the start - return this.getFirstPacket(options); + return this.getFirstPacket(res, options); } if (timestampInSamples < Math.max(firstTimestampInSamples, 0)) { // There's nothing here - return null; + return res.set(new PacketRetrievalResult(null)); } assert(this.bitstream.lastMetadataPacket); - const startPosition = await this.demuxer.findNextPacketStart(this.bitstream.lastMetadataPacket); + + const positionResult = new ResultValue(); + const promise = this.demuxer.findNextPacketStart(positionResult, this.bitstream.lastMetadataPacket); + if (positionResult.pending) await promise; + + const startPosition = positionResult.value; if (!startPosition) { - return null; + return res.set(new PacketRetrievalResult(null)); } let lowPage = startPosition.startPage; @@ -878,7 +965,12 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { endPage, endSegmentIndex, }; - const nextPosition = await this.demuxer.findNextPacketStart(pseudopacket); + + positionResult.reset(); + const promise = this.demuxer.findNextPacketStart(positionResult, pseudopacket); + if (positionResult.pending) await promise; + + const nextPosition = positionResult.value; if (nextPosition) { // Let's rewind a single step (packet) - this previous packet ensures that we'll correctly compute @@ -927,12 +1019,18 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { let lastEncodedPacket: EncodedPacket | null = null; let lastEncodedPacketMetadata: EncodedPacketMetadata | null = null; + const packetResult = new ResultValue(); + // Alright, now it's time for the final, granular seek: We keep iterating over packets until we've found the // one with the correct timestamp - i.e., the last one with a timestamp <= the timestamp we're looking for. while (currentPage !== null) { assert(currentSegmentIndex !== null); - const packet = await this.demuxer.readPacket(currentPage, currentSegmentIndex); + packetResult.reset(); + let promise = this.demuxer.readPacket(packetResult, currentPage, currentSegmentIndex); + if (packetResult.pending) await promise; + + const packet = packetResult.value; if (!packet) { break; } @@ -952,7 +1050,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { ); assert(encodedPacket); - let encodedPacketMetadata = this.encodedPacketToMetadata.get(encodedPacket); + let encodedPacketMetadata = encodedPacket._internal as EncodedPacketMetadata | undefined; assert(encodedPacketMetadata); if ( @@ -977,7 +1075,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { ); assert(encodedPacket); - encodedPacketMetadata = this.encodedPacketToMetadata.get(encodedPacket); + encodedPacketMetadata = encodedPacket._internal as EncodedPacketMetadata | undefined; assert(encodedPacketMetadata); } else { currentTimestampInSamples += encodedPacketMetadata.durationInSamples; @@ -999,7 +1097,11 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { } } - const nextPosition = await this.demuxer.findNextPacketStart(packet); + positionResult.reset(); + promise = this.demuxer.findNextPacketStart(positionResult, packet); + if (positionResult.pending) await promise; + + const nextPosition = positionResult.value; if (!nextPosition) { break; } @@ -1008,81 +1110,93 @@ class OggAudioTrackBacking implements InputAudioTrackBacking { currentSegmentIndex = nextPosition.startSegmentIndex; } - return lastEncodedPacket; + return res.set(new PacketRetrievalResult(lastEncodedPacket)); } // A slower but simpler and sequential algorithm for finding a packet in a file - async getPacketSequential(timestamp: number, options: PacketRetrievalOptions) { - const release = await this.sequentialScanMutex.acquire(); // Requires exclusivity because we write to a cache + async getPacketSequential( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + using lock = this.sequentialScanMutex.lock(); // Requires exclusivity because we write to a cache + if (lock.pending) await lock.ready; + + const timestampInSamples = roundIfAlmostInteger(timestamp * this.internalSampleRate); + timestamp = timestampInSamples / this.internalSampleRate; - try { - const timestampInSamples = roundIfAlmostInteger(timestamp * this.internalSampleRate); - timestamp = timestampInSamples / this.internalSampleRate; + const index = binarySearchLessOrEqual( + this.sequentialScanCache, + timestampInSamples, + x => x.timestampInSamples, + ); - const index = binarySearchLessOrEqual( - this.sequentialScanCache, - timestampInSamples, - x => x.timestampInSamples, + const result = new ResultValue(); + let currentPacket: EncodedPacket | null; + + if (index !== -1) { + // We don't need to start from the beginning, we can start at a previous scan point + const cacheEntry = this.sequentialScanCache[index]!; + currentPacket = this.createEncodedPacketFromOggPacket( + cacheEntry.packet, + { + timestampInSamples: cacheEntry.timestampInSamples, + vorbisLastBlocksize: cacheEntry.vorbisLastBlockSize, + }, + options, ); + } else { + const promise = this.getFirstPacket(result, options); + if (result.pending) await promise; - let currentPacket: EncodedPacket | null; - if (index !== -1) { - // We don't need to start from the beginning, we can start at a previous scan point - const cacheEntry = this.sequentialScanCache[index]!; - currentPacket = this.createEncodedPacketFromOggPacket( - cacheEntry.packet, - { - timestampInSamples: cacheEntry.timestampInSamples, - vorbisLastBlocksize: cacheEntry.vorbisLastBlockSize, - }, - options, - ); - } else { - currentPacket = await this.getFirstPacket(options); - if (currentPacket && currentPacket.timestamp > timestamp) { - // The stream starts after the requested timestamp - return null; - } + currentPacket = result.value.packet; + if (currentPacket && currentPacket.timestamp > timestamp) { + // The stream starts after the requested timestamp + return res.set(new PacketRetrievalResult(null)); } + } - let i = 0; + let i = 0; - while (currentPacket && currentPacket.timestamp < timestamp) { - const nextPacket = await this.getNextPacket(currentPacket, options); - if (!nextPacket || nextPacket.timestamp > timestamp) { - break; - } + while (currentPacket && currentPacket.timestamp < timestamp) { + result.reset(); + const promise = this.getNextPacket(result, currentPacket, options); + if (result.pending) await promise; - currentPacket = nextPacket; - i++; + const nextPacket = result.value.packet; + if (!nextPacket || nextPacket.timestamp > timestamp) { + break; + } - if (i === 100) { - // Add "checkpoints" every once in a while to speed up subsequent random accesses - i = 0; - const metadata = this.encodedPacketToMetadata.get(currentPacket); - assert(metadata); + currentPacket = nextPacket; + i++; - if (this.sequentialScanCache.length > 0) { - // If we reach this case, we must be at the end of the cache - assert(last(this.sequentialScanCache)!.timestampInSamples <= metadata.timestampInSamples); - } + if (i === 100) { + // Add "checkpoints" every once in a while to speed up subsequent random accesses + i = 0; + const metadata = currentPacket._internal as EncodedPacketMetadata | undefined; + assert(metadata); - this.sequentialScanCache.push(metadata); + if (this.sequentialScanCache.length > 0) { + // If we reach this case, we must be at the end of the cache + assert(last(this.sequentialScanCache)!.timestampInSamples <= metadata.timestampInSamples); } - } - return currentPacket; - } finally { - release(); + this.sequentialScanCache.push(metadata); + } } + + return res.set(new PacketRetrievalResult(currentPacket)); } - getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { - return this.getPacket(timestamp, options); + getKeyPacket(res: ResultValue, timestamp: number, options: PacketRetrievalOptions) { + // Correct since only audio codecs are supported + return this.getPacket(res, timestamp, options); } - getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - return this.getNextPacket(packet, options); + getNextKeyPacket(res: ResultValue, packet: EncodedPacket, options: PacketRetrievalOptions) { + // Correct since only audio codecs are supported + return this.getNextPacket(res, packet, options); } } diff --git a/src/ogg/ogg-muxer.ts b/src/ogg/ogg-muxer.ts index b2a7e0f45..a6d0c7900 100644 --- a/src/ogg/ogg-muxer.ts +++ b/src/ogg/ogg-muxer.ts @@ -81,7 +81,8 @@ export class OggMuxer extends Muxer { } async start() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.writer = await this.output._getRootWriter(true); // Ogg is always monotonically written! @@ -91,8 +92,6 @@ export class OggMuxer extends Muxer { this.getTrackData(track, { decoderConfig: track.metadata.decoderConfig }); } } - - release(); } async getMimeType() { @@ -280,59 +279,56 @@ export class OggMuxer extends Muxer { } async addEncodedAudioPacket(track: OutputAudioTrack, packet: EncodedPacket, meta?: EncodedAudioChunkMetadata) { - const release = await this.mutex.acquire(); - - try { - const trackData = this.getTrackData(track, meta); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - this.validateTimestamp(trackData.track, packet.timestamp, packet.type === 'key'); + const trackData = this.getTrackData(track, meta); - if (trackData.startTimestampInSamples === null) { - // The first packet determines where the stream begins. A packet at zero means the stream starts at - // zero, with the pre-skip preceding it. A later timestamp is treated as the start of the packet's - // decoded output, which the first page's granule position then signals as a start offset. - const timestampInSamples = Math.round(packet.timestamp * trackData.internalSampleRate); - const preSkip = trackData.codecInfo.opusInfo?.preSkip ?? 0; - - trackData.startTimestampInSamples = timestampInSamples > 0 - ? timestampInSamples + preSkip - : 0; - trackData.currentTimestampInSamples = trackData.startTimestampInSamples; - } + this.validateTimestamp(trackData.track, packet.timestamp, packet.type === 'key'); - const currentTimestampInSamples = trackData.currentTimestampInSamples; + if (trackData.startTimestampInSamples === null) { + // The first packet determines where the stream begins. A packet at zero means the stream starts at + // zero, with the pre-skip preceding it. A later timestamp is treated as the start of the packet's + // decoded output, which the first page's granule position then signals as a start offset. + const timestampInSamples = Math.round(packet.timestamp * trackData.internalSampleRate); + const preSkip = trackData.codecInfo.opusInfo?.preSkip ?? 0; - const { durationInSamples, vorbisBlockSize } = extractSampleMetadata( - packet.data, - trackData.codecInfo, - trackData.vorbisLastBlocksize, - ); - trackData.currentTimestampInSamples += durationInSamples; - trackData.vorbisLastBlocksize = vorbisBlockSize; - - // A shorter packet duration signals trailing samples to discard, which only the final packet can express - const trimmedDurationInSamples = packet.duration > 0 - ? clamp(Math.round(packet.duration * trackData.internalSampleRate), 0, durationInSamples) - : durationInSamples; + trackData.startTimestampInSamples = timestampInSamples > 0 + ? timestampInSamples + preSkip + : 0; + trackData.currentTimestampInSamples = trackData.startTimestampInSamples; + } - // With a start offset, the second audio packet flushes the page. This keeps the offset from sharing a page - // with end trimming, as both are signaled through the granule position and would be indistinguishable. - // This mirrors a requirement of the Vorbis spec. - const forcePageFlush = trackData.startTimestampInSamples > 0 && trackData.audioPacketCount === 1; - trackData.audioPacketCount++; + const currentTimestampInSamples = trackData.currentTimestampInSamples; - trackData.packetQueue.push({ - data: packet.data, - timestampInSamples: currentTimestampInSamples, - durationInSamples, - trimmedDurationInSamples, - forcePageFlush, - }); + const { durationInSamples, vorbisBlockSize } = extractSampleMetadata( + packet.data, + trackData.codecInfo, + trackData.vorbisLastBlocksize, + ); + trackData.currentTimestampInSamples += durationInSamples; + trackData.vorbisLastBlocksize = vorbisBlockSize; + + // A shorter packet duration signals trailing samples to discard, which only the final packet can express + const trimmedDurationInSamples = packet.duration > 0 + ? clamp(Math.round(packet.duration * trackData.internalSampleRate), 0, durationInSamples) + : durationInSamples; + + // With a start offset, the second audio packet flushes the page. This keeps the offset from sharing a page + // with end trimming, as both are signaled through the granule position and would be indistinguishable. + // This mirrors a requirement of the Vorbis spec. + const forcePageFlush = trackData.startTimestampInSamples > 0 && trackData.audioPacketCount === 1; + trackData.audioPacketCount++; + + trackData.packetQueue.push({ + data: packet.data, + timestampInSamples: currentTimestampInSamples, + durationInSamples, + trimmedDurationInSamples, + forcePageFlush, + }); - await this.interleavePages(); - } finally { - release(); - } + await this.interleavePages(); } addSubtitleCue(): never { @@ -540,7 +536,8 @@ export class OggMuxer extends Muxer { // eslint-disable-next-line @typescript-eslint/no-misused-promises override async onTrackClose(track: OutputTrack) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; const trackData = this.trackDatas.find(x => x.track === track); if (trackData) { @@ -553,12 +550,11 @@ export class OggMuxer extends Muxer { // Since a track is now closed, we may be able to write out chunks that were previously waiting await this.interleavePages(); - - release(); } async finalize() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.allTracksKnown.resolve(); @@ -573,7 +569,5 @@ export class OggMuxer extends Muxer { this.writePage(trackData, true); } } - - release(); } } diff --git a/src/output.ts b/src/output.ts index 10e79085e..919699d1d 100644 --- a/src/output.ts +++ b/src/output.ts @@ -221,7 +221,7 @@ export type BaseTrackMetadata = { /** The track's disposition, i.e. information about its intended usage. */ disposition?: Partial; /** - * The maximum amount of encoded packets that will be added to this track. Setting this field provides the muxer + * The maximum number of encoded packets that will be added to this track. Setting this field provides the muxer * with an additional signal that it can use to preallocate space in the file. * * When this field is set, it is an error to provide more packets than whatever this field specifies. @@ -938,17 +938,13 @@ export class Output< return this._startPromise = (async () => { this.state = 'started'; - // We want to call muxer.start immediately, so we avoid using an await here - const releasePromise = this._mutex.acquire(); + using lock = this._mutex.lock(); + if (lock.pending) await lock.ready; - try { - await this._muxer.start(); + await this._muxer.start(); - const promises = this.tracks.map(track => track.source._start()); - await Promise.all(promises); - } finally { - (await releasePromise)(); - } + const promises = this.tracks.map(track => track.source._start()); + await Promise.all(promises); })(); } @@ -983,17 +979,14 @@ export class Output< return this._cancelPromise = (async () => { this.state = 'canceled'; - const release = await this._mutex.acquire(); + using lock = this._mutex.lock(); + if (lock.pending) await lock.ready; - try { - const promises = this.tracks.map(x => x.source._flushOrWaitForOngoingClose(true)); // Force close - await Promise.all(promises); + const promises = this.tracks.map(x => x.source._flushOrWaitForOngoingClose(true)); // Force close + await Promise.all(promises); - await Promise.all([...this._unfinalizedTargets].map(target => target._close())); - this._unfinalizedTargets.clear(); - } finally { - release(); - } + await Promise.all([...this._unfinalizedTargets].map(target => target._close())); + this._unfinalizedTargets.clear(); })(); } @@ -1016,7 +1009,8 @@ export class Output< return this._finalizePromise = (async () => { this.state = 'finalizing'; - const release = await this._mutex.acquire(); + using lock = this._mutex.lock(); + if (lock.pending) await lock.ready; try { const promises = this.tracks.map(x => x.source._flushOrWaitForOngoingClose(false)); @@ -1043,8 +1037,6 @@ export class Output< } finally { await Promise.all([...this._unfinalizedTargets].map(target => target._close().catch(() => {}))); this._unfinalizedTargets.clear(); - - release(); } })(); } diff --git a/src/packet.ts b/src/packet.ts index f345b5d8e..29f61dc3a 100644 --- a/src/packet.ts +++ b/src/packet.ts @@ -6,7 +6,20 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { SECOND_TO_MICROSECOND_FACTOR } from './misc'; +import { InputDisposedError } from './input'; +import { InputTrack } from './input-track'; +import { + assert, + binarySearchLessOrEqual, + isNumber, + isThenable, + MaybePromise, + MaybeRelevantPromise, + nextDown, + removeItem, + ResultValue, + SECOND_TO_MICROSECOND_FACTOR, +} from './misc'; export const PLACEHOLDER_DATA = /* #__PURE__ */ new Uint8Array(0); @@ -54,6 +67,18 @@ export class EncodedPacket { /** Additional data carried with this packet. */ readonly sideData: EncodedPacketSideData; + /** + * Data that demuxers can populate for whatever internal use they have. + * @internal + */ + _internal: unknown = undefined; + + /** + * The unique ID of the track this packet was retrieved from. + * @internal + */ + _ownerId: number | null = null; + /** Creates a new {@link EncodedPacket} from raw bytes and timing information. */ constructor( /** @@ -71,7 +96,7 @@ export class EncodedPacket { /** The duration of this packet in seconds. */ public readonly duration: number, /** - * The sequence number indicates the decode order of the packets. Packet A must be decoded before packet B if A + * The sequence number indicates the decode order of the packets. Packet A must be decoded before packet B if A * has a lower sequence number than B. If two packets have the same sequence number, they are the same packet. * Otherwise, sequence numbers are arbitrary and are not guaranteed to have any meaning besides their relative * ordering. Negative sequence numbers mean the sequence number is undefined. @@ -284,4 +309,1378 @@ export class EncodedPacket { options?.sideData ?? this.sideData, ); } + + /** @internal */ + _cloneInternal(options?: Parameters[0]) { + const packet = this.clone(options); + packet._internal = this._internal; + packet._ownerId = this._ownerId; + + return packet; + } + + /** @internal */ + _toMetadataOnly() { + if (this.isMetadataOnly) { + return this; + } + + return this._cloneInternal({ + data: PLACEHOLDER_DATA, + sideData: { + ...this.sideData, + alpha: this.sideData.alpha ? PLACEHOLDER_DATA : undefined, + }, + }); + } +} + +/** + * Additional options for controlling packet retrieval. + * @group Media sinks + * @public + */ +export type PacketRetrievalOptions = { + /** + * When set to `true`, only packet metadata (like timestamp) will be retrieved - the actual packet data will not + * be loaded. + */ + metadataOnly?: boolean; + + /** + * When set to true, key packets will be verified upon retrieval by looking into the packet's bitstream. + * If not enabled, the packet types will be determined solely by what's stored in the containing file and may be + * incorrect, potentially leading to decoder errors. Since determining a packet's actual type requires looking into + * its data, this option cannot be enabled together with `metadataOnly`. + */ + verifyKeyPackets?: boolean; + + /** + * When querying packets in live media that are in the future relative to the current live edge, Mediabunny will, + * by default, wait for the stream to advance until the query can be satisfied. In a sense, Mediabunny simply treats + * live streams as media files that are still being written, and any read that depends on future information will + * wait until it can be fulfilled. + * + * If you want to query packets based only on the currently known information, set this field to `true` - this way, + * Mediabunny will never wait for the live stream to catch up. + * + * For non-live media, this field has no effect. + */ + skipLiveWait?: boolean; +}; + +const retrievalOptionsAreEqual = (a: PacketRetrievalOptions, b: PacketRetrievalOptions) => { + return !!a.metadataOnly === !!b.metadataOnly + && !!a.verifyKeyPackets === !!b.verifyKeyPackets + && !!a.skipLiveWait === !!b.skipLiveWait; +}; + +export const validatePacketRetrievalOptions = (options: PacketRetrievalOptions, path = 'options') => { + if (!options || typeof options !== 'object') { + throw new TypeError(`${path} must be an object.`); + } + if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') { + throw new TypeError(`${path}.metadataOnly, when defined, must be a boolean.`); + } + if (options.verifyKeyPackets !== undefined && typeof options.verifyKeyPackets !== 'boolean') { + throw new TypeError(`${path}.verifyKeyPackets, when defined, must be a boolean.`); + } + if (options.verifyKeyPackets && options.metadataOnly) { + throw new TypeError(`${path}.verifyKeyPackets and options.metadataOnly cannot be enabled together.`); + } + if (options.skipLiveWait !== undefined && typeof options.skipLiveWait !== 'boolean') { + throw new TypeError(`${path}.skipLiveWait, when defined, must be a boolean.`); + } +}; + +export const validateTimestamp = (timestamp: number) => { + if (!isNumber(timestamp)) { + throw new TypeError('timestamp must be a number.'); // It can be non-finite, that's fine + } +}; + +export class PacketRetrievalResult { + packet: EncodedPacket | null; + provisional = false; + + constructor(packet: EncodedPacket | null) { + this.packet = packet; + } +} + +export type PacketReaderOptions = { + cache?: PacketCache; +}; + +export class PacketReader { + track: T; + cache: PacketCache | null = null; + /** @internal */ + _timeResolution: number | null = null; + + constructor(track: T, options: PacketReaderOptions = {}) { + if (!(track instanceof InputTrack)) { + throw new TypeError('track must be an InputTrack.'); + } + if (typeof options !== 'object' || !options) { + throw new TypeError('options must be an object.'); + } + if (options.cache !== undefined && !(options.cache instanceof PacketCache)) { + throw new TypeError('options.cache, when provided, must be a PacketCache.'); + } + + this.track = track; + this.cache = options.cache ?? null; + } + + getFirst(options: PacketRetrievalOptions = {}): MaybePromise { + validatePacketRetrievalOptions(options); + + if (this.track.input._disposed) { + throw new InputDisposedError(); + } + + const cacheTrackInfo = this.cache?._getTrackInfo(this.track); + + if (cacheTrackInfo) { + const cacheResult = this.cache!._getFirst(cacheTrackInfo, options); + if (cacheResult !== undefined) { + return cacheResult; + } + + const pending = this.cache!._getPendingFirst(cacheTrackInfo, options); + if (pending) { + const retry = () => this.getFirst(options); + return pending.then(retry, retry); + } + } + + const result = new ResultValue(); + const promise = this.track._backing.getFirstPacket(result, options); + + const run = () => { + const packet = result.value.packet; + const cache = result.value.provisional ? null : this.cache; + + if (packet) { + packet._ownerId = this.track._uniqueId; + } + + if (!options.verifyKeyPackets || !packet || packet.type === 'delta') { + cache?._insertFirst(cacheTrackInfo!, packet, undefined); + return packet; + } + + return this.track.determinePacketType(packet).then((determinedType) => { + cache?._insertFirst(cacheTrackInfo!, packet, determinedType); + + if (determinedType === 'delta') { + return packet._cloneInternal({ type: 'delta' }); + } else { + return packet; + } + }); + }; + + const finalResult = result.pending + ? promise.then(() => run()) + : run(); + + if (cacheTrackInfo && isThenable(finalResult)) { + this.cache!._addPendingFirst(cacheTrackInfo, options, finalResult); + } + + return finalResult; + } + + getFirstKey(options: PacketRetrievalOptions = {}): MaybePromise { + const result = this.getFirst(options); + + const onPacket = (packet: EncodedPacket | null): MaybePromise => { + if (!packet || packet.type === 'key') { + return packet; + } + + return this.getNextKey(packet, options); + }; + + if (isThenable(result)) { + return result.then(onPacket); + } else { + return onPacket(result); + } + } + + getAt(timestamp: number, options: PacketRetrievalOptions = {}): MaybePromise { + validateTimestamp(timestamp); + validatePacketRetrievalOptions(options); + + if (this.track.input._disposed) { + throw new InputDisposedError(); + } + + const cacheTrackInfo = this.cache?._getTrackInfo(this.track); + + if (cacheTrackInfo) { + const cacheResult = this.cache!._getAt(cacheTrackInfo, timestamp, options); + if (cacheResult !== undefined) { + return cacheResult; + } + + const pending = this.cache!._getPending(cacheTrackInfo.pendingAtCalls, timestamp, options); + if (pending) { + const retry = () => this.getAt(timestamp, options); + return pending.then(retry, retry); + } + } + + const result = new ResultValue(); + const promise = this.track._backing.getPacket(result, timestamp, options); + + const run = () => { + const packet = result.value.packet; + const cache = result.value.provisional ? null : this.cache; + + if (packet) { + packet._ownerId = this.track._uniqueId; + } + + if (!options.verifyKeyPackets || !packet || packet.type === 'delta') { + cache?._insertAt(cacheTrackInfo!, timestamp, packet, undefined); + return packet; + } + + return this.track.determinePacketType(packet).then((determinedType) => { + cache?._insertAt(cacheTrackInfo!, timestamp, packet, determinedType); + + if (determinedType === 'delta') { + return packet._cloneInternal({ type: 'delta' }); + } else { + return packet; + } + }); + }; + + const finalResult = result.pending + ? promise.then(() => run()) + : run(); + + if (cacheTrackInfo && isThenable(finalResult)) { + this.cache!._addPending(cacheTrackInfo.pendingAtCalls, timestamp, options, finalResult); + } + + return finalResult; + } + + getKeyAt(timestamp: number, options: PacketRetrievalOptions = {}): MaybePromise { + validateTimestamp(timestamp); + validatePacketRetrievalOptions(options); + + if (this.track.input._disposed) { + throw new InputDisposedError(); + } + + const cacheTrackInfo = this.cache?._getTrackInfo(this.track); + + if (cacheTrackInfo) { + const pending = this.cache!._getPending(cacheTrackInfo.pendingKeyAtCalls, timestamp, options); + if (pending) { + const retry = () => this.getKeyAt(timestamp, options); + return pending.then(retry, retry); + } + } + + const result = new ResultValue(); + const promise = this._getKeyAtInternal(result, cacheTrackInfo, timestamp, options); + + if (!result.pending) { + return result.value; + } + + const finalResult = promise.then(() => result.value); + if (cacheTrackInfo) { + this.cache!._addPending(cacheTrackInfo.pendingKeyAtCalls, timestamp, options, finalResult); + } + + return finalResult; + } + + /** @internal */ + async _getKeyAtInternal( + res: ResultValue, + cacheTrackInfo: PacketCacheTrackInfo | undefined, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + while (true) { + if (cacheTrackInfo) { + let cacheResult = this.cache!._getKeyAt(cacheTrackInfo, timestamp, options); + if (isThenable(cacheResult)) cacheResult = await cacheResult; + + if (cacheResult !== undefined) { + if (cacheResult === null || cacheResult.type === 'key') { + return res.set(cacheResult); + } + + let timeResolution = this._getTimeResolution(); + if (isThenable(timeResolution)) timeResolution = await timeResolution; + + // Turned out to be a delta packet, so try the key packet before it + timestamp = cacheResult.timestamp - 1 / timeResolution; + continue; + } + } + + const result = new ResultValue(); + const promise = this.track._backing.getKeyPacket(result, timestamp, options); + if (result.pending) await promise; + + const packet = result.value.packet; + const cache = result.value.provisional ? null : this.cache; + + if (packet) { + packet._ownerId = this.track._uniqueId; + } + + if (!options.verifyKeyPackets || !packet) { + cache?._insertKeyAt(cacheTrackInfo!, timestamp, packet, undefined); + return res.set(packet); + } + + const determinedType = await this.track.determinePacketType(packet); + cache?._insertKeyAt(cacheTrackInfo!, timestamp, packet, determinedType); + + if (determinedType !== 'delta') { + return res.set(packet); + } + + let timeResolution = this._getTimeResolution(); + if (isThenable(timeResolution)) timeResolution = await timeResolution; + + // Try the previous key packet instead (in hopes that it's actually a key packet) + timestamp = packet.timestamp - 1 / timeResolution; + } + } + + /** @internal */ + _getTimeResolution(): MaybePromise { + if (this._timeResolution !== null) { + return this._timeResolution; + } + + const timeResolution = this.track._backing.getTimeResolution(); + if (isThenable(timeResolution)) { + return timeResolution.then((value) => { + this._timeResolution = value; + return value; + }); + } + + this._timeResolution = timeResolution; + return timeResolution; + } + + getNext(packet: EncodedPacket, options: PacketRetrievalOptions = {}): MaybePromise { + if (!(packet instanceof EncodedPacket)) { + throw new TypeError('packet must be an EncodedPacket.'); + } + if (packet._ownerId !== this.track._uniqueId) { + throw new Error('Packet was not created from this track.'); + } + validatePacketRetrievalOptions(options); + + if (this.track.input._disposed) { + throw new InputDisposedError(); + } + + const cacheTrackInfo = this.cache?._getTrackInfo(this.track); + + if (cacheTrackInfo) { + const cacheResult = this.cache!._getNext(cacheTrackInfo, packet, options); + if (cacheResult !== undefined) { + return cacheResult; + } + + const pending = this.cache!._getPending(cacheTrackInfo.pendingNextCalls, packet.sequenceNumber, options); + if (pending) { + const retry = () => this.getNext(packet, options); + return pending.then(retry, retry); + } + } + + const result = new ResultValue(); + const promise = this.track._backing.getNextPacket(result, packet, options); + + const run = () => { + const nextPacket = result.value.packet; + const cache = result.value.provisional ? null : this.cache; + + if (nextPacket) { + nextPacket._ownerId = this.track._uniqueId; + } + + if (!options.verifyKeyPackets || !nextPacket || nextPacket.type === 'delta') { + cache?._insertNext(cacheTrackInfo!, packet, nextPacket, undefined); + return nextPacket; + } + + return this.track.determinePacketType(nextPacket).then((determinedType) => { + cache?._insertNext(cacheTrackInfo!, packet, nextPacket, determinedType); + + if (determinedType === 'delta') { + return nextPacket._cloneInternal({ type: 'delta' }); + } else { + return nextPacket; + } + }); + }; + + const finalResult = result.pending + ? promise.then(() => run()) + : run(); + + if (cacheTrackInfo && isThenable(finalResult)) { + this.cache!._addPending(cacheTrackInfo.pendingNextCalls, packet.sequenceNumber, options, finalResult); + } + + return finalResult; + } + + getNextKey(packet: EncodedPacket, options: PacketRetrievalOptions = {}): MaybePromise { + if (!(packet instanceof EncodedPacket)) { + throw new TypeError('packet must be an EncodedPacket.'); + } + if (packet._ownerId !== this.track._uniqueId) { + throw new Error('Packet was not created from this track.'); + } + validatePacketRetrievalOptions(options); + + if (this.track.input._disposed) { + throw new InputDisposedError(); + } + + const cacheTrackInfo = this.cache?._getTrackInfo(this.track); + + if (cacheTrackInfo) { + const pending = this.cache!._getPending(cacheTrackInfo.pendingNextKeyCalls, packet.sequenceNumber, options); + if (pending) { + const retry = () => this.getNextKey(packet, options); + return pending.then(retry, retry); + } + } + + const result = new ResultValue(); + const promise = this._getNextKeyInternal(result, cacheTrackInfo, packet, options); + + if (!result.pending) { + return result.value; + } + + const finalResult = promise.then(() => result.value); + if (cacheTrackInfo) { + this.cache!._addPending(cacheTrackInfo.pendingNextKeyCalls, packet.sequenceNumber, options, finalResult); + } + + return finalResult; + } + + /** @internal */ + async _getNextKeyInternal( + res: ResultValue, + cacheTrackInfo: PacketCacheTrackInfo | undefined, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + while (true) { + if (cacheTrackInfo) { + let cacheResult = this.cache!._getNextKey(cacheTrackInfo, packet, options); + if (isThenable(cacheResult)) cacheResult = await cacheResult; + + if (cacheResult !== undefined) { + if (cacheResult === null || cacheResult.type === 'key') { + return res.set(cacheResult); + } + + // Turned out to be a delta packet, so try the key packet after it + packet = cacheResult; + continue; + } + } + + const result = new ResultValue(); + const promise = this.track._backing.getNextKeyPacket(result, packet, options); + if (result.pending) await promise; + + const nextKeyPacket = result.value.packet; + const cache = result.value.provisional ? null : this.cache; + + if (nextKeyPacket) { + nextKeyPacket._ownerId = this.track._uniqueId; + } + + if (!options.verifyKeyPackets || !nextKeyPacket) { + cache?._insertNextKey(cacheTrackInfo!, packet, nextKeyPacket, undefined); + return res.set(nextKeyPacket); + } + + const determinedType = await this.track.determinePacketType(nextKeyPacket); + cache?._insertNextKey(cacheTrackInfo!, packet, nextKeyPacket, determinedType); + + if (determinedType !== 'delta') { + return res.set(nextKeyPacket); + } + + // Try the next key packet instead (in hopes that it's actually a key packet) + packet = nextKeyPacket; + } + } +} + +type PacketCacheTrackInfo = { + track: InputTrack; + // Contains exactly the cached packets, keyed by sequence number + entries: Map; + sortedEntries: CacheEntry[]; // Sorted by timestamp, then by sequence number + first: CacheEntry | null | undefined; + minTimestamp: number; + minKeyTimestamp: number; + + pendingFirstCalls: PendingCall[]; + pendingAtCalls: Map; + pendingKeyAtCalls: Map; + pendingNextCalls: Map; + pendingNextKeyCalls: Map; +}; + +type PendingCall = { + options: PacketRetrievalOptions; + promise: Promise; +}; + +// Everything the cache knows about a cached packet +type CacheEntry = { + trackInfo: PacketCacheTrackInfo; + // Set to null once evicted, so that lingering references to the entry don't keep the packet alive + packet: EncodedPacket | null; + // Copied from the packet since it's still needed after eviction + timestamp: number; + + // For the links, undefined means unknown and null means there is none + next: CacheEntry | null | undefined; + prev: CacheEntry | null | undefined; + nextKey: CacheEntry | null | undefined; + // The inclusive timestamp up to which getAt() queries landing on this packet are valid + seekValidityEndpoint: number | null; + // The inclusive timestamp up to which getKeyAt() queries landing on this (key) packet are valid + keySeekValidityEndpoint: number | null; + // For key packets, the minimum timestamp of all packets within their GOP, if known + gopMinTimestamp: number | null; + // The key packet that starts this packet's GOP, if known + gopKey: CacheEntry | null; + determinedType: PacketType | null | undefined; + + size: number; + lruPrev: CacheEntry | null; // More recently used + lruNext: CacheEntry | null; // Less recently used +}; + +const DEFAULT_MAX_CACHE_SIZE = 64 * 1024 * 1024; + +// To make eviction batched, which improves performance: +const EVICTION_HIGH_WATERMARK = 1.1; +const EVICTION_LOW_WATERMARK = 0.9; + +/** Rough estimate of the memory a cached packet takes up besides its data. */ +export const PACKET_SIZE_OVERHEAD = 200; + +export type PacketCacheOptions = { + maxCacheSize?: number; + autoEvict?: boolean; +}; + +export class PacketCache { + /** @internal */ + _trackInfos = new Map(); + /** @internal */ + _maxCacheSize: number; + /** @internal */ + _autoEvict: boolean; + /** @internal */ + _evictionEnabled: boolean; + /** @internal */ + _cacheSize = 0; + /** @internal */ + _lruHead: CacheEntry | null = null; + /** @internal */ + _lruTail: CacheEntry | null = null; + + constructor(options: PacketCacheOptions = {}) { + if (!options || typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if (options.maxCacheSize !== undefined && (!isNumber(options.maxCacheSize) || options.maxCacheSize < 0)) { + throw new TypeError('options.maxCacheSize, when provided, must be a non-negative number.'); + } + if (options.autoEvict !== undefined && typeof options.autoEvict !== 'boolean') { + throw new TypeError('options.autoEvict, when provided, must be a boolean.'); + } + + this._maxCacheSize = options.maxCacheSize ?? DEFAULT_MAX_CACHE_SIZE; + this._autoEvict = options.autoEvict ?? true; + this._evictionEnabled = this._maxCacheSize !== Infinity; + } + + clear() { + // Emptied in place since in-flight requests may still be holding on to them + for (const trackInfo of this._trackInfos.values()) { + for (const entry of trackInfo.sortedEntries) { + entry.packet = null; // Marks the entry as evicted for anyone still holding on to it + } + + trackInfo.entries.clear(); + trackInfo.sortedEntries.length = 0; + trackInfo.first = undefined; + trackInfo.minTimestamp = -Infinity; + trackInfo.minKeyTimestamp = -Infinity; + } + + this._lruHead = null; + this._lruTail = null; + this._cacheSize = 0; + } + + evict() { + if (this._cacheSize <= EVICTION_HIGH_WATERMARK * this._maxCacheSize) { + return; + } + + const targetSize = EVICTION_LOW_WATERMARK * this._maxCacheSize; + const affectedTrackInfos = new Set(); + + while (this._cacheSize > targetSize) { + const entry = this._lruTail!; + this._unlinkLruEntry(entry); + this._cacheSize -= entry.size; + affectedTrackInfos.add(entry.trackInfo); + this._removeEntry(entry); + } + + // Now, remove the evicted entries from the sorted lists in a single pass each + for (const trackInfo of affectedTrackInfos) { + const sortedEntries = trackInfo.sortedEntries; + let writeIndex = 0; + let previousWasKept = false; + + for (let i = 0; i < sortedEntries.length; i++) { + const entry = sortedEntries[i]!; + if (entry.packet !== null) { + sortedEntries[writeIndex++] = entry; + previousWasKept = true; + continue; + } + + if (previousWasKept) { + // Lookups that used to land on this run of evicted packets now land on the kept packet before it, + // which therefore mustn't claim validity up to the run anymore + const predecessor = sortedEntries[writeIndex - 1]!; + if (predecessor.seekValidityEndpoint !== null) { + predecessor.seekValidityEndpoint = Math.min( + predecessor.seekValidityEndpoint, + nextDown(entry.timestamp), + ); + } + } + + previousWasKept = false; + } + + sortedEntries.length = writeIndex; + } + } + + /** @internal */ + _getTrackInfo(track: InputTrack) { + let info = this._trackInfos.get(track); + if (!info) { + info = { + track, + entries: new Map(), + sortedEntries: [], + first: undefined, + minTimestamp: -Infinity, + minKeyTimestamp: -Infinity, + + pendingFirstCalls: [], + pendingAtCalls: new Map(), + pendingKeyAtCalls: new Map(), + pendingNextCalls: new Map(), + pendingNextKeyCalls: new Map(), + }; + this._trackInfos.set(track, info); + } + + return info; + } + + /** @internal */ + _insertFirst( + trackInfo: PacketCacheTrackInfo, + packet: EncodedPacket | null, + determinedType: PacketType | null | undefined, + ) { + if (!packet) { + trackInfo.first = null; + return; + } + + const entry = this._insertPacket(trackInfo, packet); + trackInfo.first = entry; + entry.prev = null; + + if (entry.packet!.type === 'key') { + // Under the GOP rule, no later key packet can have a smaller timestamp than the first one + trackInfo.minKeyTimestamp = Math.max(trackInfo.minKeyTimestamp, nextDown(entry.timestamp)); + } + + // Knowing that nothing comes before this packet may be what completes the first GOP + this._finalizeGop(entry); + + if (determinedType !== undefined) { + entry.determinedType = determinedType; + } + + if (this._autoEvict) { + this.evict(); + } + } + + /** @internal */ + _insertAt( + trackInfo: PacketCacheTrackInfo, + timestamp: number, + packet: EncodedPacket | null, + determinedType: PacketType | null | undefined, + ) { + if (!packet) { + trackInfo.minTimestamp = Math.max(trackInfo.minTimestamp, timestamp); + return; + } + + const entry = this._insertPacket(trackInfo, packet); + + entry.seekValidityEndpoint = Math.max(timestamp, entry.seekValidityEndpoint ?? -Infinity); + + if (entry.gopKey && !entry.gopKey.packet) { + entry.gopKey = null; // The GOP's key packet has been evicted + } + + const gopKey = entry.packet!.type === 'key' ? entry : entry.gopKey; + if (gopKey) { + // There's no packet at all between this one and the timestamp, so certainly no key packet either. And since + // any key packet after this one can't have a smaller timestamp, the key packet of this packet's GOP is + // valid up to the timestamp too. + gopKey.keySeekValidityEndpoint = Math.max(timestamp, gopKey.keySeekValidityEndpoint ?? -Infinity); + } + + if (determinedType !== undefined) { + entry.determinedType = determinedType; + } + + if (this._autoEvict) { + this.evict(); + } + } + + /** @internal */ + _insertKeyAt( + trackInfo: PacketCacheTrackInfo, + timestamp: number, + packet: EncodedPacket | null, + determinedType: PacketType | null | undefined, + ) { + if (!packet) { + trackInfo.minKeyTimestamp = Math.max(trackInfo.minKeyTimestamp, timestamp); + return; + } + + const entry = this._insertPacket(trackInfo, packet); + + entry.keySeekValidityEndpoint = Math.max(timestamp, entry.keySeekValidityEndpoint ?? -Infinity); + + if (determinedType !== undefined) { + entry.determinedType = determinedType; + } + + if (this._autoEvict) { + this.evict(); + } + } + + /** @internal */ + _insertNext( + trackInfo: PacketCacheTrackInfo, + packet: EncodedPacket, + next: EncodedPacket | null, + determinedType: PacketType | null | undefined, + ) { + const nextEntry = next ? this._insertPacket(trackInfo, next) : null; + if (nextEntry && determinedType !== undefined) { + nextEntry.determinedType = determinedType; + } + + // Links only ever connect two packets in the cache + const entry = trackInfo.entries.get(packet.sequenceNumber); + if (!entry) { + if (this._autoEvict) { + this.evict(); + } + + return; + } + + entry.next = nextEntry; + if (nextEntry) { + nextEntry.prev = entry; + } + + // Only these links can complete a GOP: either they close it off, or they join onto an existing chain that + // leads to its end. Trying to finalize on every link would be quadratic in the GOP size. + if (!nextEntry || nextEntry.packet!.type === 'key' || nextEntry.next !== undefined) { + this._finalizeGop(entry); + } + + // Under the GOP rule, a future key packet can't have a smaller timestamp than any packet before it. So, as the + // chain following a key packet grows, so does the range in which no other key packet can lie. + if (entry.gopKey && !entry.gopKey.packet) { + entry.gopKey = null; // The GOP's key packet has been evicted + } + const gopKey = entry.packet!.type === 'key' ? entry : entry.gopKey; + if (gopKey) { + let keyEndpoint = gopKey.keySeekValidityEndpoint ?? -Infinity; + + let currentEntry = entry; + while (true) { + const followingEntry = currentEntry.next; + if (followingEntry === undefined) { + break; + } + if (followingEntry === null) { + keyEndpoint = Infinity; + break; + } + if (followingEntry.packet!.type === 'key') { + // Exclusive, so that this stays true even once the next key packet is no longer cached + keyEndpoint = Math.max(keyEndpoint, nextDown(followingEntry.timestamp)); + break; + } + if (followingEntry.gopKey === gopKey) { + break; // We've already been here + } + + followingEntry.gopKey = gopKey; + // A future key packet may share this packet's timestamp, so we can only go right up to it + keyEndpoint = Math.max(keyEndpoint, nextDown(followingEntry.timestamp)); + + // The packet may have learned about its next key packet before joining the GOP. Its timestamp stays + // true even if it has been evicted since. + if (followingEntry.nextKey !== undefined) { + keyEndpoint = Math.max( + keyEndpoint, + followingEntry.nextKey ? nextDown(followingEntry.nextKey.timestamp) : Infinity, + ); + } + + currentEntry = followingEntry; + } + + gopKey.keySeekValidityEndpoint = keyEndpoint; + } + + if (this._autoEvict) { + this.evict(); + } + } + + /** @internal */ + _insertNextKey( + trackInfo: PacketCacheTrackInfo, + packet: EncodedPacket, + nextKey: EncodedPacket | null, + determinedType: PacketType | null | undefined, + ) { + const nextKeyEntry = nextKey ? this._insertPacket(trackInfo, nextKey) : null; + if (nextKeyEntry && determinedType !== undefined) { + nextKeyEntry.determinedType = determinedType; + } + + // Like with next(), only remember the link if the packet we came from is also cached + const entry = trackInfo.entries.get(packet.sequenceNumber); + if (entry) { + entry.nextKey = nextKeyEntry; + + // If we know the key packet starting this packet's GOP, then we now also know the key packet following it + if (entry.gopKey && !entry.gopKey.packet) { + entry.gopKey = null; // The GOP's key packet has been evicted + } + + const gopKey = entry.packet!.type === 'key' ? entry : entry.gopKey; + if (gopKey) { + const newEndpoint = nextKeyEntry ? nextDown(nextKeyEntry.timestamp) : Infinity; + gopKey.keySeekValidityEndpoint = Math.max(newEndpoint, gopKey.keySeekValidityEndpoint ?? -Infinity); + } + } + + if (this._autoEvict) { + this.evict(); + } + } + + /** @internal */ + _finalizeGop(entry: CacheEntry) { + // When we know an entire GOP, we can conclude a bunch of additional information about it and make the packets + // eligible for being returned by the .getAt method. This is because we make the assumption that the "GOP rule" + // holds for the streams we read. This rule says that when a key frame occurs, no timestamp after it can be + // less than any timestamp before it. This rule still allows for open GOPs but it puts a reasonable clamp on + // timestamp monotonicity. + + // Walk forward to the last packet of the GOP; what follows it is either the next GOP's key packet or the end + let lastGopEntry = entry; + let gopEnd: CacheEntry | null; + while (true) { + const nextEntry = lastGopEntry.next; + if (nextEntry === undefined) { + return; // Incomplete + } + if (nextEntry === null || nextEntry.packet!.type === 'key') { + gopEnd = nextEntry; + break; + } + + lastGopEntry = nextEntry; + } + + // Walk backward to the start of the GOP + let minTimestamp = Infinity; + let endpoint = -Infinity; + let gopStart = lastGopEntry; + while (true) { + minTimestamp = Math.min(minTimestamp, gopStart.timestamp); + endpoint = Math.max(endpoint, gopStart.timestamp, gopStart.seekValidityEndpoint ?? -Infinity); + if (gopStart.packet!.type === 'key') { + break; + } + + const prevEntry = gopStart.prev; + if (prevEntry === undefined) { + return; // Incomplete + } + if (prevEntry === null) { + break; // The first packet always starts a GOP + } + + gopStart = prevEntry; + } + + if (gopEnd) { + // If we already know the next GOP's minimum timestamp, nothing can lie between this GOP and it. Exclusive, + // since the packet at that timestamp may no longer be cached. + if (gopEnd.gopMinTimestamp !== null) { + endpoint = Math.max(endpoint, nextDown(gopEnd.gopMinTimestamp)); + } + } else { + endpoint = Infinity; + } + + let currentEntry = gopStart; + while (true) { + currentEntry.seekValidityEndpoint = endpoint; + if (currentEntry === lastGopEntry) { + break; + } + + currentEntry = currentEntry.next!; + } + + gopStart.gopMinTimestamp = minTimestamp; + + if (gopStart.prev === null) { + // Nothing after the first GOP can go below anything in it, so this is the smallest timestamp of the track + const trackInfo = gopStart.trackInfo; + trackInfo.minTimestamp = Math.max(trackInfo.minTimestamp, nextDown(minTimestamp)); + + if (gopStart.packet!.type !== 'key') { + // The track starts with a delta packet, so the first key packet is the one ending this GOP + const firstKeyEndpoint = gopEnd ? nextDown(gopEnd.timestamp) : Infinity; + trackInfo.minKeyTimestamp = Math.max(trackInfo.minKeyTimestamp, firstKeyEndpoint); + } + } + + // Same thing the other way around: if the previous GOP is complete, it now extends up to our minimum timestamp + let previousGopStart = gopStart; + while (true) { + const prevEntry = previousGopStart.prev; + if (prevEntry === undefined) { + return; // Incomplete + } + if (prevEntry === null) { + break; // We've reached the first packet + } + + previousGopStart = prevEntry; + if (prevEntry.packet!.type === 'key') { + break; + } + } + + currentEntry = previousGopStart; + while (currentEntry !== gopStart) { + currentEntry.seekValidityEndpoint = Math.max(minTimestamp, currentEntry.seekValidityEndpoint ?? -Infinity); + currentEntry = currentEntry.next!; + } + } + + /** @internal */ + _insertPacket(trackInfo: PacketCacheTrackInfo, packet: EncodedPacket) { + const existingEntry = trackInfo.entries.get(packet.sequenceNumber); + if (existingEntry) { + const existingPacket = existingEntry.packet!; + if (existingPacket.isMetadataOnly && !packet.isMetadataOnly) { + // Upgrade in place so that everything referencing the packet gets the data too + // @ts-expect-error Technically readonly + existingPacket.data = packet.data; + // @ts-expect-error Technically readonly + existingPacket.sideData = packet.sideData; + + if (this._evictionEnabled) { + const size = getCachedPacketSize(existingPacket); + this._cacheSize += size - existingEntry.size; + existingEntry.size = size; + } + } + + if (this._evictionEnabled) { + this._unlinkLruEntry(existingEntry); + this._linkLruEntryAtHead(existingEntry); + } + + return existingEntry; + } + + const sortedEntries = trackInfo.sortedEntries; + let index = binarySearchLessOrEqual(sortedEntries, packet.timestamp, x => x.timestamp); + + // Packets with equal timestamps are ordered by sequence number + while ( + index !== -1 + && sortedEntries[index]!.timestamp === packet.timestamp + && sortedEntries[index]!.packet!.sequenceNumber > packet.sequenceNumber + ) { + index--; + } + + // Metadata-only packets may get upgraded in place later, which must not affect the instance we were handed + const storedPacket = packet.isMetadataOnly ? packet._cloneInternal() : packet; + + const entry: CacheEntry = { + trackInfo, + packet: storedPacket, + timestamp: storedPacket.timestamp, + next: undefined, + prev: undefined, + nextKey: undefined, + seekValidityEndpoint: null, + keySeekValidityEndpoint: null, + gopMinTimestamp: null, + gopKey: null, + determinedType: undefined, + size: 0, + lruPrev: null, + lruNext: null, + }; + sortedEntries.splice(index + 1, 0, entry); + trackInfo.entries.set(storedPacket.sequenceNumber, entry); + + if (this._evictionEnabled) { + entry.size = getCachedPacketSize(storedPacket); + this._cacheSize += entry.size; + this._linkLruEntryAtHead(entry); + } + + return entry; + } + + /** @internal */ + _removeEntry(entry: CacheEntry) { + const trackInfo = entry.trackInfo; + + if (entry.prev) { + entry.prev.next = undefined; + } + if (entry.next) { + entry.next.prev = undefined; + } + if (trackInfo.first === entry) { + trackInfo.first = undefined; + } + + trackInfo.entries.delete(entry.packet!.sequenceNumber); + + // Other entries may still reference this one, so mark it as evicted and make sure it doesn't keep + // anything alive + entry.packet = null; + entry.next = undefined; + entry.prev = undefined; + entry.nextKey = undefined; + entry.gopKey = null; + } + + /** @internal */ + _getFirst(trackInfo: PacketCacheTrackInfo, options: PacketRetrievalOptions) { + if (trackInfo.first === undefined || trackInfo.first === null) { + return trackInfo.first; + } + + return this._checkAgainstRetrievalOptions(trackInfo.first, options); + } + + /** @internal */ + _getPendingFirst(trackInfo: PacketCacheTrackInfo, options: PacketRetrievalOptions) { + const pendingCall = trackInfo.pendingFirstCalls.find(x => retrievalOptionsAreEqual(x.options, options)); + return pendingCall?.promise ?? null; + } + + /** @internal */ + _addPendingFirst(trackInfo: PacketCacheTrackInfo, options: PacketRetrievalOptions, promise: Promise) { + const pendingCall = { + options, + promise, + }; + trackInfo.pendingFirstCalls.push(pendingCall); + + const remove = () => { + removeItem(trackInfo.pendingFirstCalls, pendingCall); + }; + void promise.then(remove, remove); + } + + /** @internal */ + _getPending(pendingCalls: Map, key: number, options: PacketRetrievalOptions) { + const pendingCall = pendingCalls.get(key)?.find(x => retrievalOptionsAreEqual(x.options, options)); + return pendingCall?.promise ?? null; + } + + /** @internal */ + _addPending( + pendingCalls: Map, + key: number, + options: PacketRetrievalOptions, + promise: Promise, + ) { + const calls = pendingCalls.get(key) ?? []; + pendingCalls.set(key, calls); + + const pendingCall = { + options, + promise, + }; + calls.push(pendingCall); + + const remove = () => { + removeItem(calls, pendingCall); + if (calls.length === 0) { + pendingCalls.delete(key); + } + }; + void promise.then(remove, remove); + } + + /** @internal */ + _getAt(trackInfo: PacketCacheTrackInfo, timestamp: number, options: PacketRetrievalOptions) { + if (timestamp <= trackInfo.minTimestamp) { + return null; // Nothing here + } + + const index = binarySearchLessOrEqual(trackInfo.sortedEntries, timestamp, x => x.timestamp); + + if (index === -1) { + return undefined; + } + + const entry = trackInfo.sortedEntries[index]!; + if (entry.seekValidityEndpoint === null || timestamp > entry.seekValidityEndpoint) { + return undefined; + } + + return this._checkAgainstRetrievalOptions(entry, options); + } + + /** @internal */ + _getKeyAt(trackInfo: PacketCacheTrackInfo, timestamp: number, options: PacketRetrievalOptions) { + if (timestamp <= trackInfo.minTimestamp || timestamp <= trackInfo.minKeyTimestamp) { + return null; // Nothing here + } + + // Walk back to the latest key packet we know of + const sortedEntries = trackInfo.sortedEntries; + let index = binarySearchLessOrEqual(sortedEntries, timestamp, x => x.timestamp); + while (index !== -1 && sortedEntries[index]!.packet!.type !== 'key') { + index--; + } + + if (index === -1) { + return undefined; + } + + const entry = sortedEntries[index]!; + if (entry.keySeekValidityEndpoint === null || timestamp > entry.keySeekValidityEndpoint) { + return undefined; + } + + return this._checkAgainstRetrievalOptions(entry, options); + } + + /** @internal */ + _getNext(trackInfo: PacketCacheTrackInfo, packet: EncodedPacket, options: PacketRetrievalOptions) { + const nextEntry = trackInfo.entries.get(packet.sequenceNumber)?.next; + if (nextEntry === undefined) { + return undefined; + } + + if (nextEntry === null) { + return null; + } + + return this._checkAgainstRetrievalOptions(nextEntry, options); + } + + /** @internal */ + _getNextKey(trackInfo: PacketCacheTrackInfo, packet: EncodedPacket, options: PacketRetrievalOptions) { + const entry = trackInfo.entries.get(packet.sequenceNumber); + if (!entry) { + return undefined; + } + + // Delta packets share the next key packet of the packet before them, so walk back through the GOP until some + // packet knows it + let currentEntry = entry; + while (true) { + // The next key packet itself may have been evicted in the meantime + if (currentEntry.nextKey && !currentEntry.nextKey.packet) { + currentEntry.nextKey = undefined; + } + if (currentEntry.nextKey === undefined && currentEntry.keySeekValidityEndpoint === Infinity) { + currentEntry.nextKey = null; // It's the last key packet + } + if (currentEntry.nextKey !== undefined) { + entry.nextKey = currentEntry.nextKey; // Remember it so the next lookup doesn't need to walk again + if (entry.nextKey === null) { + return null; + } + + return this._checkAgainstRetrievalOptions(entry.nextKey, options); + } + if (currentEntry.packet!.type === 'key' || !currentEntry.prev) { + break; + } + + currentEntry = currentEntry.prev; + } + + // No direct information, but maybe we can find it by following the chain + currentEntry = entry; + while (true) { + const nextEntry = currentEntry.next; + if (nextEntry === undefined) { + return undefined; + } + if (nextEntry === null) { + return null; + } + if (nextEntry.packet!.type === 'key') { + return this._checkAgainstRetrievalOptions(nextEntry, options); + } + + currentEntry = nextEntry; + } + } + + /** @internal */ + _checkAgainstRetrievalOptions(entry: CacheEntry, options: PacketRetrievalOptions) { + const packet = entry.packet!; + if (packet.isMetadataOnly && !options.metadataOnly) { + return undefined; + } + + if (this._evictionEnabled) { + // The packet gets returned, so it's now the most recently used one + this._unlinkLruEntry(entry); + this._linkLruEntryAtHead(entry); + } + + if (!packet.isMetadataOnly && options.metadataOnly) { + return packet._toMetadataOnly(); + } + + if (options.verifyKeyPackets && packet.type === 'key') { + assert(!packet.isMetadataOnly); // Can't be + + if (entry.determinedType !== undefined) { + if (entry.determinedType === 'delta') { + return packet._cloneInternal({ type: 'delta' }); + } else { + return packet._cloneInternal(); + } + } + + return entry.trackInfo.track.determinePacketType(packet).then((determinedType) => { + // The packet may have been evicted while we were waiting + if (entry.packet) { + entry.determinedType = determinedType; + } + + if (determinedType === 'delta') { + return packet._cloneInternal({ type: 'delta' }); + } else { + return packet._cloneInternal(); + } + }); + } + + return packet._cloneInternal(); + } + + /** @internal */ + _unlinkLruEntry(entry: CacheEntry) { + if (entry.lruPrev) { + entry.lruPrev.lruNext = entry.lruNext; + } else { + this._lruHead = entry.lruNext; + } + + if (entry.lruNext) { + entry.lruNext.lruPrev = entry.lruPrev; + } else { + this._lruTail = entry.lruPrev; + } + + entry.lruPrev = null; + entry.lruNext = null; + } + + /** @internal */ + _linkLruEntryAtHead(entry: CacheEntry) { + entry.lruNext = this._lruHead; + + if (this._lruHead) { + this._lruHead.lruPrev = entry; + } else { + this._lruTail = entry; + } + + this._lruHead = entry; + } } + +const getCachedPacketSize = (packet: EncodedPacket) => { + return PACKET_SIZE_OVERHEAD + packet.data.byteLength + (packet.sideData.alpha?.byteLength ?? 0); +}; diff --git a/src/reader.ts b/src/reader.ts index 586529510..8f13fb4c8 100644 --- a/src/reader.ts +++ b/src/reader.ts @@ -232,7 +232,7 @@ export const readBytes = (slice: FileSlice, length: number) => { export const readU8 = (slice: FileSlice) => { checkIsInRange(slice, 1); - return slice.view.getUint8(slice.bufferPos++); + return slice.bytes[slice.bufferPos++]!; }; export const readU16 = (slice: FileSlice, littleEndian: boolean) => { diff --git a/src/sample.ts b/src/sample.ts index bd2814a9e..f1fa2f70b 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -26,8 +26,11 @@ import { Rational, simplifyRational, Rectangle, + rectangleToDOMRectInit, validateRectangle, + normalizeRotation, composeRotationAndFlip, + relativeRotationAndFlip, roundToMultiple, arrayArgmin, MaybePromise, @@ -37,6 +40,24 @@ import { Logging } from './logging'; polyfillSymbolDispose(); +// Patch the type +declare global { + interface VideoFrame { + rotation?: number; + flip?: boolean; + } + + interface VideoFrameInit { + rotation?: number; + flip?: boolean; + } + + interface VideoFrameBufferInit { + rotation?: number; + flip?: boolean; + } +} + type FinalizationRegistryValue = { type: 'video'; data: VideoFrame | OffscreenCanvas | Uint8Array | VideoSampleResource; @@ -251,6 +272,9 @@ export type VideoSampleInit = { * @public */ export class VideoSample implements Disposable { + /** @internal */ + static _openSampleCount = 0; + /** @internal */ _data!: VideoFrame | OffscreenCanvas | Uint8Array | VideoSampleResource | null; /** @@ -338,6 +362,14 @@ export class VideoSample implements Disposable { return this.format && this.format.includes('A'); } + /** + * Whether this sample is closed, meaning its underlying data has been discarded. When a sample is closed, most + * operations will fail. + */ + get closed() { + return this._closed; + } + /** * Creates a new {@link VideoSample} from a * [`VideoFrame`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame). This is essentially a near zero-cost @@ -513,20 +545,21 @@ export class VideoSample implements Disposable { this._layout = null; this.format = data.format; + this.visibleRect = { left: data.visibleRect?.x ?? 0, top: data.visibleRect?.y ?? 0, width: data.visibleRect?.width ?? data.codedWidth, height: data.visibleRect?.height ?? data.codedHeight, }; - // The VideoFrame's rotation and flip are ignored here. They're still new fields, and I'm not sure of any - // application where the browser makes use of them. If a case gets found, I'll add it. - this.rotation = init?.rotation ?? 0; - this.flip = init?.flip ?? false; - // Assuming no innate VideoFrame rotation here - this.squarePixelWidth = data.displayWidth; - this.squarePixelHeight = data.displayHeight; + const frameRotation = normalizeRotation(data.rotation ?? 0); + this.rotation = init?.rotation ?? frameRotation; + this.flip = init?.flip ?? data.flip ?? false; + + // data.displayWidth/Height are post-rotation, but the square pixel dimensions are pre-rotation + this.squarePixelWidth = frameRotation % 180 === 0 ? data.displayWidth : data.displayHeight; + this.squarePixelHeight = frameRotation % 180 === 0 ? data.displayHeight : data.displayWidth; this.timestamp = init?.timestamp ?? data.timestamp / 1e6; this.duration = init?.duration ?? (data.duration ?? 0) / 1e6; @@ -694,31 +727,44 @@ export class VideoSample implements Disposable { den: this.squarePixelHeight * this.codedWidth, }); finalizationRegistry?.register(this, { type: 'video', data: this._data }, this); + VideoSample._openSampleCount++; } /** Clones this video sample. */ - clone() { + clone(override?: { + timestamp?: number; + duration?: number; + rotation?: Rotation; + flip?: boolean; + encodeOptions?: VideoEncoderEncodeOptions; + }) { if (this._closed) { throw new Error('VideoSample is closed.'); } assert(this._data !== null); + const timestamp = override?.timestamp ?? this.timestamp; + const duration = override?.duration ?? this.duration; + const rotation = override?.rotation ?? this.rotation; + const flip = override?.flip ?? this.flip; + const encodeOptions = override?.encodeOptions ?? this.encodeOptions; + if (this._data instanceof VideoSampleResource) { return new VideoSample(this._data, { - timestamp: this.timestamp, - duration: this.duration, - rotation: this.rotation, - flip: this.flip, - encodeOptions: this.encodeOptions, + timestamp, + duration, + rotation, + flip, + encodeOptions, }); } else if (isVideoFrame(this._data)) { return new VideoSample(this._data.clone(), { - timestamp: this.timestamp, - duration: this.duration, - rotation: this.rotation, - flip: this.flip, - encodeOptions: this.encodeOptions, + timestamp, + duration, + rotation, + flip, + encodeOptions, }); } else if (this._data instanceof Uint8Array) { assert(this._layout); @@ -728,15 +774,16 @@ export class VideoSample implements Disposable { layout: this._layout, codedWidth: this.codedWidth, codedHeight: this.codedHeight, - timestamp: this.timestamp, - duration: this.duration, + timestamp, + duration, colorSpace: this.colorSpace, - rotation: this.rotation, - flip: this.flip, + rotation, + flip, visibleRect: this.visibleRect, - displayWidth: this.displayWidth, - displayHeight: this.displayHeight, - encodeOptions: this.encodeOptions, + // Display dimensions must be given in terms of the clone's rotation, which may differ from ours + displayWidth: rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight, + displayHeight: rotation % 180 === 0 ? this.squarePixelHeight : this.squarePixelWidth, + encodeOptions, // It's already been copied, if we copy it again we make the clone unnecessarily expensive _doNotCopy: true, @@ -746,15 +793,16 @@ export class VideoSample implements Disposable { format: this.format!, codedWidth: this.codedWidth, codedHeight: this.codedHeight, - timestamp: this.timestamp, - duration: this.duration, + timestamp, + duration, colorSpace: this.colorSpace, - rotation: this.rotation, - flip: this.flip, + rotation, + flip, visibleRect: this.visibleRect, - displayWidth: this.displayWidth, - displayHeight: this.displayHeight, - encodeOptions: this.encodeOptions, + // Display dimensions must be given in terms of the clone's rotation, which may differ from ours + displayWidth: rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight, + displayHeight: rotation % 180 === 0 ? this.squarePixelHeight : this.squarePixelWidth, + encodeOptions, }); } } @@ -782,6 +830,7 @@ export class VideoSample implements Disposable { } this._closed = true; + VideoSample._openSampleCount--; } /** @@ -1083,16 +1132,29 @@ export class VideoSample implements Disposable { codedHeight: this.codedHeight, timestamp: this.microsecondTimestamp, duration: this.microsecondDuration, + rotation: this.rotation, + flip: this.flip, colorSpace: this.colorSpace, - visibleRect: this.visibleRect, - displayWidth: this.squarePixelWidth, // Not display* since we're not passing rotation - displayHeight: this.squarePixelHeight, + visibleRect: rectangleToDOMRectInit(this.visibleRect), + displayWidth: this.displayWidth, + displayHeight: this.displayHeight, }); } else if (isVideoFrame(this._data)) { - return new VideoFrame(this._data, { + // The rotation and flip fields here compose with the frame's existing ones, but we want them to act like an + // override instead, so we need to pass the relative orientation + const { rotation, flip } = relativeRotationAndFlip( + normalizeRotation(this._data.rotation ?? 0), + this._data.flip ?? false, + this.rotation, + this.flip, + ); + const init: VideoFrameInit = { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, // Drag 0 duration to undefined, glitches some codecs - }); + rotation, + flip, + }; + return new VideoFrame(this._data, init); } else if (this._data instanceof Uint8Array) { assert(this._layout); @@ -1104,15 +1166,20 @@ export class VideoSample implements Disposable { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, colorSpace: this.colorSpace, - visibleRect: this.visibleRect, - displayWidth: this.squarePixelWidth, // Not display* since we're not passing rotation - displayHeight: this.squarePixelHeight, + rotation: this.rotation, + flip: this.flip, + visibleRect: rectangleToDOMRectInit(this.visibleRect), + displayWidth: this.displayWidth, + displayHeight: this.displayHeight, }); } else { - return new VideoFrame(this._data, { + const init: VideoFrameInit = { timestamp: this.microsecondTimestamp, duration: this.microsecondDuration || undefined, - }); + rotation: this.rotation, + flip: this.flip, + }; + return new VideoFrame(this._data, init); } } @@ -1239,10 +1306,28 @@ export class VideoSample implements Disposable { throw new Error('VideoSample is closed.'); } - ({ sx, sy, sWidth, sHeight } = this._unmapSourceRegion(sx, sy, sWidth, sHeight, this.rotation, this.flip)); - const source = this.toCanvasImageSource(); + // Relative to the innate rotation and flip of the source + const { rotation: relativeRotation, flip: relativeFlip } + = typeof VideoFrame !== 'undefined' && source instanceof VideoFrame + ? relativeRotationAndFlip( + normalizeRotation(source.rotation ?? 0), + source.flip ?? false, + this.rotation, + this.flip, + ) + : { rotation: this.rotation, flip: this.flip }; + + ({ sx, sy, sWidth, sHeight } = this._unmapSourceRegion( + sx, + sy, + sWidth, + sHeight, + relativeRotation, + relativeFlip, + )); + context.save(); const centerX = dx + dWidth / 2; @@ -1252,12 +1337,12 @@ export class VideoSample implements Disposable { // Canvas transforms apply to the image in reverse call order, so to rotate first and then flip, the flip has // to be set up before the rotation - if (this.flip) { + if (relativeFlip) { context.scale(-1, 1); } - context.rotate(this.rotation * Math.PI / 180); + context.rotate(relativeRotation * Math.PI / 180); - const aspectRatioChange = this.rotation % 180 === 0 ? 1 : dWidth / dHeight; + const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : dWidth / dHeight; // Scale to compensate for aspect ratio changes when rotated context.scale(1 / aspectRatioChange, aspectRatioChange); @@ -1334,6 +1419,14 @@ export class VideoSample implements Disposable { const rotation = options.rotation ?? this.rotation; const flip = options.flip ?? this.flip; + const source = this.toCanvasImageSource(); + + // Relative to the innate rotation and flip of the source + const { rotation: relativeRotation, flip: relativeFlip } + = typeof VideoFrame !== 'undefined' && source instanceof VideoFrame + ? relativeRotationAndFlip(normalizeRotation(source.rotation ?? 0), source.flip ?? false, rotation, flip) + : { rotation, flip }; + const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 ? [this.squarePixelWidth, this.squarePixelHeight] : [this.squarePixelHeight, this.squarePixelWidth]; @@ -1350,12 +1443,12 @@ export class VideoSample implements Disposable { let newHeight: number; const { sx, sy, sWidth, sHeight } = this._unmapSourceRegion( - options.crop?.left ?? 0, - options.crop?.top ?? 0, - options.crop?.width ?? rotatedWidth, - options.crop?.height ?? rotatedHeight, - rotation, - flip, + finalCrop?.left ?? 0, + finalCrop?.top ?? 0, + finalCrop?.width ?? rotatedWidth, + finalCrop?.height ?? rotatedHeight, + relativeRotation, + relativeFlip, ); if (options.fit === 'fill') { @@ -1364,8 +1457,8 @@ export class VideoSample implements Disposable { newWidth = canvasWidth; newHeight = canvasHeight; } else { - const [sampleWidth, sampleHeight] = options.crop - ? [options.crop.width, options.crop.height] + const [sampleWidth, sampleHeight] = finalCrop + ? [finalCrop.width, finalCrop.height] : [rotatedWidth, rotatedHeight]; const scale = options.fit === 'contain' @@ -1379,14 +1472,14 @@ export class VideoSample implements Disposable { context.save(); - const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight; + const aspectRatioChange = relativeRotation % 180 === 0 ? 1 : newWidth / newHeight; context.translate(canvasWidth / 2, canvasHeight / 2); // Canvas transforms apply to the image in reverse call order, so to rotate first and then flip, the flip has // to be set up before the rotation - if (flip) { + if (relativeFlip) { context.scale(-1, 1); } - context.rotate(rotation * Math.PI / 180); + context.rotate(relativeRotation * Math.PI / 180); // This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and // don't need to think about how those dimensions change after the rotation context.scale(1 / aspectRatioChange, aspectRatioChange); @@ -1394,36 +1487,54 @@ export class VideoSample implements Disposable { // Important that we don't use .draw() here since that would take rotation and flip into account, but we wanna // handle them ourselves here - context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight); + context.drawImage(source, sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight); context.restore(); } /** @internal */ - _unmapSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: number, flip: boolean) { + _unmapSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: Rotation, flip: boolean) { + // The formulas below need the intrinsic dimensions of the image source that ends up in drawImage + let sourceWidth: number; + let sourceHeight: number; + + if (typeof VideoFrame !== 'undefined' && this._data instanceof VideoFrame) { + // Kinda dirty but has to be done + sourceWidth = this._data.displayWidth; + sourceHeight = this._data.displayHeight; + } else if (this._data instanceof Uint8Array || this._data instanceof VideoSampleResource) { + // toCanvasImageSource() bakes this sample's rotation and flip into the VideoFrame it creates + sourceWidth = this.displayWidth; + sourceHeight = this.displayHeight; + } else { + // Canvas-backed; the canvas dimensions equal the square pixel dimensions + sourceWidth = this.squarePixelWidth; + sourceHeight = this.squarePixelHeight; + } + // The provided sx,sy,sWidth,sHeight refer to the final rotated and flipped image, but that's not actually how // the image is stored. Therefore, we must map these back onto the original image. Since the flip is applied // last, we undo it first, mirroring within the rotated width. if (flip) { - const rotatedWidth = rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight; + const rotatedWidth = rotation % 180 === 0 ? sourceWidth : sourceHeight; sx = rotatedWidth - sx - sWidth; } if (rotation === 90) { [sx, sy, sWidth, sHeight] = [ sy, - this.squarePixelHeight - sx - sWidth, + sourceHeight - sx - sWidth, sHeight, sWidth, ]; } else if (rotation === 180) { [sx, sy] = [ - this.squarePixelWidth - sx - sWidth, - this.squarePixelHeight - sy - sHeight, + sourceWidth - sx - sWidth, + sourceHeight - sy - sHeight, ]; } else if (rotation === 270) { [sx, sy, sWidth, sHeight] = [ - this.squarePixelWidth - sy - sHeight, + sourceWidth - sy - sHeight, sx, sHeight, sWidth, @@ -1680,59 +1791,10 @@ export class VideoSample implements Disposable { // Any previous rotation and flip are now baked in rotation: 0, flip: false, + encodeOptions: this.encodeOptions, }); } - /** Sets the rotation metadata of this video sample. */ - setRotation(newRotation: Rotation) { - if (![0, 90, 180, 270].includes(newRotation)) { - throw new TypeError('newRotation must be 0, 90, 180, or 270.'); - } - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - (this.rotation as Rotation) = newRotation; - } - - /** Sets the flip metadata of this video sample. */ - setFlip(newFlip: boolean) { - if (typeof newFlip !== 'boolean') { - throw new TypeError('newFlip must be a boolean.'); - } - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - (this.flip as boolean) = newFlip; - } - - /** Sets the presentation timestamp of this video sample, in seconds. */ - setTimestamp(newTimestamp: number) { - if (!Number.isFinite(newTimestamp)) { - throw new TypeError('newTimestamp must be a number.'); - } - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - (this.timestamp as number) = newTimestamp; - } - - /** Sets the duration of this video sample, in seconds. */ - setDuration(newDuration: number) { - if (!Number.isFinite(newDuration) || newDuration < 0) { - throw new TypeError('newDuration must be a non-negative number.'); - } - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - (this.duration as number) = newDuration; - } - - /** Sets the encode options used when this sample is passed to an encoder. */ - setEncodeOptions(newEncodeOptions: VideoEncoderEncodeOptions) { - if (!newEncodeOptions || typeof newEncodeOptions !== 'object') { - throw new TypeError('newEncodeOptions must be an object.'); - } - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - (this.encodeOptions as DeepReadonly) = newEncodeOptions; - } - /** Calls `.close()`. */ [Symbol.dispose]() { this.close(); @@ -2532,6 +2594,9 @@ export type AudioSampleCopyToOptions = { * @public */ export class AudioSample implements Disposable { + /** @internal */ + static _openSampleCount = 0; + /** @internal */ _data: AudioData | Uint8Array | AudioSampleResource; /** @internal */ @@ -2568,6 +2633,14 @@ export class AudioSample implements Disposable { return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.duration); } + /** + * Whether this sample is closed, meaning its underlying data has been discarded. When a sample is closed, most + * operations will fail. + */ + get closed() { + return this._closed; + } + /** * Creates a new {@link AudioSample}, either from an existing * [`AudioData`](https://developer.mozilla.org/en-US/docs/Web/API/AudioData) or from raw bytes specified in @@ -2667,6 +2740,7 @@ export class AudioSample implements Disposable { this._data = dataBuffer; } + AudioSample._openSampleCount++; finalizationRegistry?.register(this, { type: 'audio', data: this._data }, this); } @@ -2928,19 +3002,27 @@ export class AudioSample implements Disposable { } /** Clones this audio sample. */ - clone(): AudioSample { + clone(override?: { + timestamp?: number; + }): AudioSample { if (this._closed) { throw new Error('AudioSample is closed.'); } + const timestamp = override?.timestamp ?? this.timestamp; + if (this._data instanceof AudioSampleResource) { const sample = new AudioSample(this._data); - sample.setTimestamp(this.timestamp); // Make sure the timestamp is correct + + // @ts-expect-error Readonly + sample.timestamp = timestamp; // Make sure the timestamp is precise (beyond microsecond accuracy) return sample; } else if (isAudioData(this._data)) { const sample = new AudioSample(this._data.clone()); - sample.setTimestamp(this.timestamp); + + // @ts-expect-error Readonly + sample.timestamp = timestamp; // Make sure the timestamp is precise (beyond microsecond accuracy) return sample; } else { @@ -3046,6 +3128,7 @@ export class AudioSample implements Disposable { } this._closed = true; + AudioSample._openSampleCount--; } /** @@ -3137,16 +3220,6 @@ export class AudioSample implements Disposable { return audioBuffer; } - /** Sets the presentation timestamp of this audio sample, in seconds. */ - setTimestamp(newTimestamp: number) { - if (!Number.isFinite(newTimestamp)) { - throw new TypeError('newTimestamp must be a number.'); - } - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - (this.timestamp as number) = newTimestamp; - } - /** Calls `.close()`. */ [Symbol.dispose]() { this.close(); diff --git a/src/segmented-input.ts b/src/segmented-input.ts index 2c96e8a03..ce5056a1e 100644 --- a/src/segmented-input.ts +++ b/src/segmented-input.ts @@ -18,9 +18,16 @@ import { InputVideoTrack, InputVideoTrackBacking, } from './input-track'; -import { PacketRetrievalOptions } from './media-sink'; -import { arrayCount, assert, MaybePromise, roundToDivisor } from './misc'; -import { EncodedPacket } from './packet'; +import { + arrayCount, + assert, + isThenable, + MaybePromise, + MaybeRelevantPromise, + ResultValue, + roundToDivisor, +} from './misc'; +import { EncodedPacket, PacketRetrievalOptions, PacketRetrievalResult } from './packet'; export type SegmentedInputMetadata = { name: string | null; @@ -54,6 +61,15 @@ export type SegmentRetrievalOptions = { skipLiveWait?: boolean; }; +export class SegmentRetrievalResult { + segment: Segment | null; + provisional = false; + + constructor(segment: Segment | null) { + this.segment = segment; + } +} + export type SegmentedInputTrackDeclaration = { id: number; type: TrackType; @@ -83,18 +99,37 @@ export abstract class SegmentedInput { this.trackDeclarations = trackDeclarations; } - abstract getFirstSegment(options: SegmentRetrievalOptions): Promise; - abstract getSegmentAt(timestamp: number, options: SegmentRetrievalOptions): Promise; - abstract getNextSegment(segment: Segment, options: SegmentRetrievalOptions): Promise; - abstract getPreviousSegment(segment: Segment, options: SegmentRetrievalOptions): Promise; + abstract getFirstSegment( + res: ResultValue, + options: SegmentRetrievalOptions, + ): MaybeRelevantPromise; + abstract getSegmentAt( + res: ResultValue, + timestamp: number, + options: SegmentRetrievalOptions, + ): MaybeRelevantPromise; + abstract getNextSegment( + res: ResultValue, + segment: Segment, + options: SegmentRetrievalOptions, + ): MaybeRelevantPromise; + abstract getPreviousSegment( + res: ResultValue, + segment: Segment, + options: SegmentRetrievalOptions, + ): MaybeRelevantPromise; abstract getInputForSegment(segment: Segment): Input; abstract getLiveRefreshInterval(): Promise; async getDurationFromMetadata(options: DurationMetadataRequestOptions) { - const lastSegment = await this.getSegmentAt(Infinity, { + const segmentResult = new ResultValue(); + const promise = this.getSegmentAt(segmentResult, Infinity, { skipLiveWait: options.skipLiveWait, }); + if (segmentResult.pending) await promise; + + const lastSegment = segmentResult.value.segment; if (!lastSegment) { return null; } @@ -103,8 +138,19 @@ export abstract class SegmentedInput { } async getUnixTimeForTimestamp(timestamp: number): Promise { - let segment = await this.getSegmentAt(timestamp, {}); - segment ??= await this.getFirstSegment({}); + const segmentResult = new ResultValue(); + const promise = this.getSegmentAt(segmentResult, timestamp, {}); + if (segmentResult.pending) await promise; + + let segment = segmentResult.value.segment; + if (!segment) { + // Default to the first segment + segmentResult.reset(); + const promise = this.getFirstSegment(segmentResult, {}); + if (segmentResult.pending) await promise; + + segment = segmentResult.value.segment; + } if (!segment || segment.unixEpochTimestamp === null) { return null; @@ -136,7 +182,11 @@ export abstract class SegmentedInput { } } else { // There are no declarations, we must determine the tracks from the first segment - this.firstSegment = await this.getFirstSegment({}); + const firstSegmentResult = new ResultValue(); + const promise = this.getFirstSegment(firstSegmentResult, {}); + if (firstSegmentResult.pending) await promise; + + this.firstSegment = firstSegmentResult.value.segment; if (!this.firstSegment) { return []; } @@ -172,19 +222,19 @@ export abstract class SegmentedInput { } // This operation is done a lot and can be semi-expensive, so it's good to have a cache for it - async getFirstTimestampForInput(input: Input) { + getFirstTimestampForInput(input: Input): MaybePromise { const existing = this.firstTimestampCache.get(input); if (existing !== undefined) { return existing; } - const firstTimestamp = await input.getFirstTimestamp(); - this.firstTimestampCache.set(input, firstTimestamp); - - return firstTimestamp; + return input.getFirstTimestamp().then((firstTimestamp) => { + this.firstTimestampCache.set(input, firstTimestamp); + return firstTimestamp; + }); } - async getMediaOffset(segment: Segment, input: Input) { + async getMediaOffset(res: ResultValue, segment: Segment, input: Input): MaybeRelevantPromise { const firstSegment = segment.firstSegment ?? segment; let firstSegmentFirstTimestamp: number; @@ -192,29 +242,34 @@ export abstract class SegmentedInput { firstSegmentFirstTimestamp = this.firstSegmentFirstTimestamps.get(firstSegment)!; } else { const firstInput = this.getInputForSegment(firstSegment); - firstSegmentFirstTimestamp = await this.getFirstTimestampForInput(firstInput); + let firstTimestamp = this.getFirstTimestampForInput(firstInput); + if (isThenable(firstTimestamp)) firstTimestamp = await firstTimestamp; + + firstSegmentFirstTimestamp = firstTimestamp; this.firstSegmentFirstTimestamps.set(firstSegment, firstSegmentFirstTimestamp); } if (firstSegment === segment) { - return firstSegment.timestamp - firstSegmentFirstTimestamp; + return res.set(firstSegment.timestamp - firstSegmentFirstTimestamp); } - const segmentFirstTimestamp = await this.getFirstTimestampForInput(input); + let segmentFirstTimestamp = this.getFirstTimestampForInput(input); + if (isThenable(segmentFirstTimestamp)) segmentFirstTimestamp = await segmentFirstTimestamp; + const segmentElapsed = segment.timestamp - firstSegment.timestamp; const inputElapsed = segmentFirstTimestamp - firstSegmentFirstTimestamp; const difference = inputElapsed - segmentElapsed; if (Math.abs(difference) <= Math.min(0.25, segmentElapsed)) { // Heuristic // We're close enough - return firstSegment.timestamp - firstSegmentFirstTimestamp; + return res.set(firstSegment.timestamp - firstSegmentFirstTimestamp); } else { // Ideally, each segment has absolute timestamps that are relative to some outside clock which is // consistent across segments. This is often the case, but not always. Either the container format used is // not timestamped at all (like ADTS), or the segments are just fucky. In this case, use the segment's // relative timestamp to determine where we are, and completely offset out the segment's input start // timestamp. - return segment.timestamp - segmentFirstTimestamp; + return res.set(segment.timestamp - segmentFirstTimestamp); } } @@ -236,7 +291,6 @@ class SegmentedInputInputTrackBacking implements InputTrackBacking { segmentedInput: SegmentedInput; decl: SegmentedInputTrackDeclaration; number: number; - packetInfos = new WeakMap(); hydrationPromise: Promise | null = null; firstInputTrack: InputTrack | null = null; @@ -250,13 +304,21 @@ class SegmentedInputInputTrackBacking implements InputTrackBacking { hydrate() { return this.hydrationPromise ??= (async () => { - this.segmentedInput.firstSegment ??= await this.segmentedInput.getFirstSegment({}); + if (!this.segmentedInput.firstSegment) { + const firstSegmentResult = new ResultValue(); + const promise = this.segmentedInput.getFirstSegment(firstSegmentResult, {}); + if (firstSegmentResult.pending) await promise; + + this.segmentedInput.firstSegment = firstSegmentResult.value.segment; + } + if (!this.segmentedInput.firstSegment) { throw new Error('Missing first segment, can\'t retrieve track.'); } let currentSegment: Segment | null = this.segmentedInput.firstSegment; let track: InputTrack | null = null; + const segmentResult = new ResultValue(); // For playlists with sparse tracks (rare af!!), not every segment has every track, so we need to loop to // find the first segment that actually contains the track we want. @@ -269,7 +331,11 @@ class SegmentedInputInputTrackBacking implements InputTrackBacking { break; } - currentSegment = await this.segmentedInput.getNextSegment(currentSegment, {}); + segmentResult.reset(); + const promise = this.segmentedInput.getNextSegment(segmentResult, currentSegment, {}); + if (segmentResult.pending) await promise; + + currentSegment = segmentResult.value.segment; } if (!track) { @@ -365,104 +431,164 @@ class SegmentedInputInputTrackBacking implements InputTrackBacking { return this.segmentedInput.getLiveRefreshInterval(); } - async createAdjustedPacket(packet: EncodedPacket, segment: Segment, track: InputTrack) { + async createAdjustedPacket( + res: ResultValue, + packet: EncodedPacket, + segment: Segment, + track: InputTrack, + provisional: boolean, + ): MaybeRelevantPromise { assert(packet.sequenceNumber >= 0); assert(this.segmentedInput.firstSegment); - const mediaOffset = await this.segmentedInput.getMediaOffset(segment, track.input); + const mediaOffsetResult = new ResultValue(); + const mediaOffsetPromise = this.segmentedInput.getMediaOffset(mediaOffsetResult, segment, track.input); + if (mediaOffsetResult.pending) await mediaOffsetPromise; + + const mediaOffset = mediaOffsetResult.value; // If we didn't do this then sequence numbers would exceed Number.MAX_SAFE_INTEGER for Unix-timestamped segments const segmentTimestampRelativeToFirst = segment.timestamp - this.segmentedInput.firstSegment.timestamp; + let timeResolution = track._backing.getTimeResolution(); + if (isThenable(timeResolution)) timeResolution = await timeResolution; + const modified = packet.clone({ timestamp: roundToDivisor( packet.timestamp + mediaOffset, - await track.getTimeResolution(), + timeResolution, ), // The 1e8 assumes a max of 100 MB per second, highly unlikely to be hit, so this should guarantee // monotonically increasing sequence numbers across segments. sequenceNumber: Math.floor(1e8 * segmentTimestampRelativeToFirst) + packet.sequenceNumber, }); - this.packetInfos.set(modified, { + // Replaces the inner packet's internal data the clone carried over; it stays reachable via sourcePacket + modified._internal = { segment, track, sourcePacket: packet, - }); + } satisfies PacketInfo; - return modified; + const result = new PacketRetrievalResult(modified); + result.provisional = provisional; + + return res.set(result); } - async getFirstPacket(options: PacketRetrievalOptions): Promise { - await this.hydrate(); + async getFirstPacket( + res: ResultValue, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + if (!this.firstInputTrack) { + await this.hydrate(); + } assert(this.firstInputTrack); assert(this.firstSegment); let currentTrack: InputTrack | null = this.firstInputTrack; let currentSegment: Segment | null = this.firstSegment; + const packetResult = new ResultValue(); + const segmentResult = new ResultValue(); // Loop until we found a segment with a packet (segments may contain zero packets in rare cases) while (true) { if (currentTrack) { - const packet = await currentTrack._backing.getFirstPacket(options); + packetResult.reset(); + const promise = currentTrack._backing.getFirstPacket(packetResult, options); + if (packetResult.pending) await promise; + + const packet = packetResult.value.packet; if (packet) { - return this.createAdjustedPacket(packet, currentSegment, currentTrack); + return this.createAdjustedPacket(res, packet, currentSegment, currentTrack, false); } } - currentSegment = await this.segmentedInput.getNextSegment(currentSegment, { + segmentResult.reset(); + const segmentPromise = this.segmentedInput.getNextSegment(segmentResult, currentSegment, { skipLiveWait: options.skipLiveWait, }); + if (segmentResult.pending) await segmentPromise; + + currentSegment = segmentResult.value.segment; if (!currentSegment) { break; } const nextInput = this.segmentedInput.getInputForSegment(currentSegment); - const nextTracks = await nextInput.getTracks(); + let nextTracks = getTracksMaybeSync(nextInput); + if (isThenable(nextTracks)) nextTracks = await nextTracks; + currentTrack = nextTracks.find(t => ( t.type === this.firstInputTrack!.type && t.number === this.firstInputTrack!.number )) ?? null; } - return null; + const result = new PacketRetrievalResult(null); + result.provisional = segmentResult.value.provisional; + + return res.set(result); } - getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { - return this._getNextInternal(packet, options, false); + getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this._getNextInternal(res, packet, options, false); } - getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions): Promise { - return this._getNextInternal(packet, options, true); + getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this._getNextInternal(res, packet, options, true); } async _getNextInternal( + res: ResultValue, packet: EncodedPacket, options: PacketRetrievalOptions, keyframesOnly: boolean, - ): Promise { - const info = this.packetInfos.get(packet); - if (!info) { - throw new Error('Packet was not created from this track.'); - } + ): MaybeRelevantPromise { + const info = packet._internal as PacketInfo | undefined; + assert(info); + + const packetResult = new ResultValue(); + const promise = keyframesOnly + ? info.track._backing.getNextKeyPacket(packetResult, info.sourcePacket, options) + : info.track._backing.getNextPacket(packetResult, info.sourcePacket, options); + if (packetResult.pending) await promise; + + const nextPacket = packetResult.value.packet; - const nextPacket = keyframesOnly - ? await info.track._backing.getNextKeyPacket(info.sourcePacket, options) - : await info.track._backing.getNextPacket(info.sourcePacket, options); if (nextPacket) { - return this.createAdjustedPacket(nextPacket, info.segment, info.track); + return this.createAdjustedPacket(res, nextPacket, info.segment, info.track, false); } let currentSegment: Segment | null = info.segment; + const segmentResult = new ResultValue(); + while (true) { - const nextSegment = await this.segmentedInput.getNextSegment(currentSegment, { + segmentResult.reset(); + const segmentPromise = this.segmentedInput.getNextSegment(segmentResult, currentSegment, { skipLiveWait: options.skipLiveWait, }); + if (segmentResult.pending) await segmentPromise; + + const nextSegment = segmentResult.value.segment; if (!nextSegment) { - return null; + const result = new PacketRetrievalResult(null); + result.provisional = segmentResult.value.provisional; + + return res.set(result); } const nextInput = this.segmentedInput.getInputForSegment(nextSegment); - const nextTracks = await nextInput.getTracks(); + let nextTracks = getTracksMaybeSync(nextInput); + if (isThenable(nextTracks)) nextTracks = await nextTracks; + const nextTrack = nextTracks.find(t => t.type === info.track.type && t.number === info.track.number); if (!nextTrack) { @@ -470,74 +596,131 @@ class SegmentedInputInputTrackBacking implements InputTrackBacking { continue; } - const firstPacket = await nextTrack._backing.getFirstPacket(options); + packetResult.reset(); + const promise = nextTrack._backing.getFirstPacket(packetResult, options); + if (packetResult.pending) await promise; + + const firstPacket = packetResult.value.packet; + if (!firstPacket) { - return null; + return res.set(new PacketRetrievalResult(null)); } - return this.createAdjustedPacket(firstPacket, nextSegment, nextTrack); + return this.createAdjustedPacket(res, firstPacket, nextSegment, nextTrack, false); } } - getPacket(timestamp: number, options: PacketRetrievalOptions): Promise { - return this._getPacketInternal(timestamp, options, false); + getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this._getPacketInternal(res, timestamp, options, false); } - getKeyPacket(timestamp: number, options: PacketRetrievalOptions): Promise { - return this._getPacketInternal(timestamp, options, true); + getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this._getPacketInternal(res, timestamp, options, true); } async _getPacketInternal( + res: ResultValue, timestamp: number, options: PacketRetrievalOptions, keyframesOnly: boolean, - ): Promise { - let currentSegment = await this.segmentedInput.getSegmentAt(timestamp, { + ): MaybeRelevantPromise { + const segmentResult = new ResultValue(); + const segmentPromise = this.segmentedInput.getSegmentAt(segmentResult, timestamp, { skipLiveWait: options.skipLiveWait, }); + if (segmentResult.pending) await segmentPromise; + + // If the segment lookup is provisional, then so is anything we derive from it + const provisional = segmentResult.value.provisional; + + let currentSegment = segmentResult.value.segment; if (!currentSegment) { - return null; + const result = new PacketRetrievalResult(null); + result.provisional = provisional; + + return res.set(result); } - await this.hydrate(); + if (!this.firstInputTrack) { + await this.hydrate(); + } + + const packetResult = new ResultValue(); + const mediaOffsetResult = new ResultValue(); while (currentSegment) { const input = this.segmentedInput.getInputForSegment(currentSegment); - const tracks = await input.getTracks(); + let tracks = getTracksMaybeSync(input); + if (isThenable(tracks)) tracks = await tracks; + const track = tracks.find(t => ( t.type === this.firstInputTrack!.type && t.number === this.firstInputTrack!.number )); if (!track) { // Search the previous segment - currentSegment = await this.segmentedInput.getPreviousSegment(currentSegment, { + segmentResult.reset(); + const prevSegmentPromise = this.segmentedInput.getPreviousSegment(segmentResult, currentSegment, { skipLiveWait: options.skipLiveWait, }); + if (segmentResult.pending) await prevSegmentPromise; + + currentSegment = segmentResult.value.segment; continue; } - const mediaOffset = await this.segmentedInput.getMediaOffset(currentSegment, input); + mediaOffsetResult.reset(); + const mediaOffsetPromise = this.segmentedInput.getMediaOffset(mediaOffsetResult, currentSegment, input); + if (mediaOffsetResult.pending) await mediaOffsetPromise; - const offsetTimestamp = timestamp - mediaOffset; - const packet = keyframesOnly - ? await track._backing.getKeyPacket(offsetTimestamp, options) - : await track._backing.getPacket(offsetTimestamp, options); + const offsetTimestamp = timestamp - mediaOffsetResult.value; + packetResult.reset(); + const packetPromise = keyframesOnly + ? track._backing.getKeyPacket(packetResult, offsetTimestamp, options) + : track._backing.getPacket(packetResult, offsetTimestamp, options); + if (packetResult.pending) await packetPromise; + + const packet = packetResult.value.packet; if (!packet) { // Search the previous segment - currentSegment = await this.segmentedInput.getPreviousSegment(currentSegment, { + segmentResult.reset(); + const prevSegmentPromise = this.segmentedInput.getPreviousSegment(segmentResult, currentSegment, { skipLiveWait: options.skipLiveWait, }); + if (segmentResult.pending) await prevSegmentPromise; + + currentSegment = segmentResult.value.segment; continue; } - return this.createAdjustedPacket(packet, currentSegment, track); + return this.createAdjustedPacket(res, packet, currentSegment, track, provisional); } - return null; + const result = new PacketRetrievalResult(null); + result.provisional = provisional; + + return res.set(result); } } +/** Retrieves the tracks of an input, synchronously if the input's tracks have already been determined. */ +const getTracksMaybeSync = (input: Input): MaybePromise => { + if (input._trackBackingsCache) { + return input._trackBackingsCache.map(x => input._wrapBackingAsTrack(x)); + } + + return input.getTracks(); +}; + class SegmentedInputInputVideoTrackBacking extends SegmentedInputInputTrackBacking implements InputVideoTrackBacking { diff --git a/src/wave/wave-demuxer.ts b/src/wave/wave-demuxer.ts index 13e889281..ae6db904e 100644 --- a/src/wave/wave-demuxer.ts +++ b/src/wave/wave-demuxer.ts @@ -10,10 +10,9 @@ import { AudioCodec } from '../codec'; import { Demuxer } from '../demuxer'; import { Input } from '../input'; import { InputAudioTrackBacking } from '../input-track'; -import { PacketRetrievalOptions } from '../media-sink'; import { DEFAULT_TRACK_DISPOSITION, MetadataTags } from '../metadata'; -import { assert, isThenable, UNDETERMINED_LANGUAGE } from '../misc'; -import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; +import { assert, isThenable, MaybeRelevantPromise, ResultValue, UNDETERMINED_LANGUAGE } from '../misc'; +import { EncodedPacket, PacketRetrievalOptions, PacketRetrievalResult, PLACEHOLDER_DATA } from '../packet'; import { readAscii, readBytes, Reader, readU16, readU32, readU64 } from '../reader'; import { ID3_V2_HEADER_SIZE, parseId3V2Tag, readId3V2Header } from '../id3'; @@ -467,15 +466,16 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { } private async getPacketAtIndex( + res: ResultValue, packetIndex: number, options: PacketRetrievalOptions, - ): Promise { + ): MaybeRelevantPromise { assert(packetIndex >= 0); assert(this.demuxer.audioInfo); const startOffset = packetIndex * PACKET_SIZE_IN_FRAMES * this.demuxer.audioInfo.blockSizeInBytes; if (startOffset >= this.demuxer.dataSize) { - return null; + return res.set(new PacketRetrievalResult(null)); } const sizeInBytes = Math.min( @@ -492,7 +492,7 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { if (isThenable(slice)) slice = await slice; if (!slice) { - return null; + return res.set(new PacketRetrievalResult(null)); } } @@ -515,21 +515,25 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { this.demuxer.lastKnownPacketIndex, ); - return new EncodedPacket( + return res.set(new PacketRetrievalResult(new EncodedPacket( data, 'key', timestamp, duration, packetIndex, sizeInBytes, - ); + ))); } - getFirstPacket(options: PacketRetrievalOptions) { - return this.getPacketAtIndex(0, options); + getFirstPacket(res: ResultValue, options: PacketRetrievalOptions): MaybeRelevantPromise { + return this.getPacketAtIndex(res, 0, options); } - async getPacket(timestamp: number, options: PacketRetrievalOptions) { + async getPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { assert(this.demuxer.audioInfo); const packetIndex = Math.floor(Math.min( @@ -537,25 +541,36 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { (this.demuxer.dataSize - 1) / (PACKET_SIZE_IN_FRAMES * this.demuxer.audioInfo.blockSizeInBytes), )); if (packetIndex < 0) { - return null; + return res.set(new PacketRetrievalResult(null)); } - const packet = await this.getPacketAtIndex(packetIndex, options); - if (packet) { - return packet; + const result = new ResultValue(); + let promise = this.getPacketAtIndex(result, packetIndex, options); + if (result.pending) await promise; + + if (result.value.packet) { + return res.set(result.value); } if (packetIndex === 0) { - return null; // Empty data chunk + return res.set(new PacketRetrievalResult(null)); // Empty data chunk } assert(this.demuxer.reader.fileSize === null); // The file is shorter than we thought, meaning the packet we were looking for doesn't exist. So, let's find // the last packet by doing a sequential scan, instead. - let currentPacket = await this.getPacketAtIndex(this.demuxer.lastKnownPacketIndex, options); + result.reset(); + promise = this.getPacketAtIndex(result, this.demuxer.lastKnownPacketIndex, options); + if (result.pending) await promise; + + let currentPacket = result.value.packet; while (currentPacket) { - const nextPacket = await this.getNextPacket(currentPacket, options); + result.reset(); + promise = this.getNextPacket(result, currentPacket, options); + if (result.pending) await promise; + + const nextPacket = result.value.packet; if (!nextPacket) { break; } @@ -563,21 +578,35 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking { currentPacket = nextPacket; } - return currentPacket; + return res.set(new PacketRetrievalResult(currentPacket)); } - getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { + getNextPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { assert(this.demuxer.audioInfo); - const packetIndex = Math.round(packet.timestamp * this.demuxer.audioInfo.sampleRate / PACKET_SIZE_IN_FRAMES); - return this.getPacketAtIndex(packetIndex + 1, options); + const packetIndex = packet.sequenceNumber; + assert(packetIndex >= 0); + + return this.getPacketAtIndex(res, packetIndex + 1, options); } - getKeyPacket(timestamp: number, options: PacketRetrievalOptions) { - return this.getPacket(timestamp, options); + getKeyPacket( + res: ResultValue, + timestamp: number, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this.getPacket(res, timestamp, options); } - getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions) { - return this.getNextPacket(packet, options); + getNextKeyPacket( + res: ResultValue, + packet: EncodedPacket, + options: PacketRetrievalOptions, + ): MaybeRelevantPromise { + return this.getNextPacket(res, packet, options); } } diff --git a/src/wave/wave-muxer.ts b/src/wave/wave-muxer.ts index c13929d2b..2649a90bd 100644 --- a/src/wave/wave-muxer.ts +++ b/src/wave/wave-muxer.ts @@ -43,7 +43,8 @@ export class WaveMuxer extends Muxer { } async start() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; this.writer = await this.output._getRootWriter(false); this.riffWriter = new RiffWriter(this.writer); @@ -60,8 +61,6 @@ export class WaveMuxer extends Muxer { this.sampleRate = track.metadata.decoderConfig.sampleRate; this.headerWritten = true; } - - release(); } async getMimeType() { @@ -77,37 +76,34 @@ export class WaveMuxer extends Muxer { packet: EncodedPacket, meta?: EncodedAudioChunkMetadata, ) { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; - try { - if (!this.headerWritten) { - validateAudioChunkMetadata(meta, track.source._codec); + if (!this.headerWritten) { + validateAudioChunkMetadata(meta, track.source._codec); - assert(meta); - assert(meta.decoderConfig); + assert(meta); + assert(meta.decoderConfig); - this.writeHeader(track, meta.decoderConfig); - this.sampleRate = meta.decoderConfig.sampleRate; - this.headerWritten = true; - } + this.writeHeader(track, meta.decoderConfig); + this.sampleRate = meta.decoderConfig.sampleRate; + this.headerWritten = true; + } - this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); + this.validateTimestamp(track, packet.timestamp, packet.type === 'key'); - if (!this.isRf64 && this.writer.getPos() + packet.data.byteLength >= 2 ** 32) { - throw new Error( - 'Adding more audio data would exceed the maximum RIFF size of 4 GiB. To write larger files, use' - + ' RF64 by setting `large: true` in the WavOutputFormatOptions.', - ); - } + if (!this.isRf64 && this.writer.getPos() + packet.data.byteLength >= 2 ** 32) { + throw new Error( + 'Adding more audio data would exceed the maximum RIFF size of 4 GiB. To write larger files, use' + + ' RF64 by setting `large: true` in the WavOutputFormatOptions.', + ); + } - this.writer.write(packet.data); - this.dataSize += packet.data.byteLength; - this.sampleCount += Math.round(packet.duration * this.sampleRate!); + this.writer.write(packet.data); + this.dataSize += packet.data.byteLength; + this.sampleCount += Math.round(packet.duration * this.sampleRate!); - await this.writer.flush(); - } finally { - release(); - } + await this.writer.flush(); } async addSubtitleCue() { @@ -351,7 +347,8 @@ export class WaveMuxer extends Muxer { } async finalize() { - const release = await this.mutex.acquire(); + using lock = this.mutex.lock(); + if (lock.pending) await lock.ready; if (!this.headerWritten) { throw new Error( @@ -388,7 +385,5 @@ export class WaveMuxer extends Muxer { this.writer.seek(this.dataSizePos); this.riffWriter.writeU32(this.dataSize); } - - release(); } } diff --git a/test/browser/adts-demuxing.test.ts b/test/browser/adts-demuxing.test.ts index 02905c884..adcfca6b8 100644 --- a/test/browser/adts-demuxing.test.ts +++ b/test/browser/adts-demuxing.test.ts @@ -2,8 +2,8 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { UrlSource } from '../../src/source.js'; import { ADTS, ALL_FORMATS } from '../../src/input-format.js'; -import { AudioSampleSink, EncodedPacketSink } from '../../src/media-sink.js'; import { assert } from '../../src/misc.js'; +import { AudioSampleCursor, PacketCursor } from '../../src/cursors.js'; test('ADTS demuxing', async () => { using input = new Input({ @@ -30,16 +30,16 @@ test('ADTS demuxing', async () => { // No description }); - const sink = new EncodedPacketSink(audioTrack); + const cursor = new PacketCursor(audioTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await cursor.seekToFirst(); assert(firstPacket); expect(firstPacket.data[0]).toBe(0xff); expect((firstPacket.data[1]! & 0xf0)).toBe(0xf0); // Second nibble is also all 1s expect(firstPacket.type).toBe('key'); - const secondPacket = await sink.getNextPacket(firstPacket); + const secondPacket = await cursor.next(); assert(secondPacket); expect(secondPacket.data[0]).toBe(0xff); @@ -56,11 +56,11 @@ test('ADTS packet decodability', async () => { const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); - const sink = new AudioSampleSink(audioTrack); + const cursor = new AudioSampleCursor(audioTrack); let count = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (using sample of sink.samples()) { + for await (using sample of cursor) { count++; } diff --git a/test/browser/adts-muxing.test.ts b/test/browser/adts-muxing.test.ts index 33e5a5a90..7255d9a9e 100644 --- a/test/browser/adts-muxing.test.ts +++ b/test/browser/adts-muxing.test.ts @@ -2,7 +2,7 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { BufferSource, UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; import { EncodedAudioPacketSource } from '../../src/media-source.js'; import { Output } from '../../src/output.js'; import { StreamTarget, type StreamTargetChunk } from '../../src/target.js'; @@ -56,10 +56,10 @@ test('ADTS with metadata over StreamTarget', async () => { const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); - const sink = new EncodedPacketSink(audioTrack); + const cursor = new PacketCursor(audioTrack); let isFirst = true; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { await audioSource.add(packet, { decoderConfig: isFirst ? (await audioTrack.getDecoderConfig())! : undefined, }); @@ -113,11 +113,11 @@ test('StreamTarget write errors surface directly', async () => { const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); - const sink = new EncodedPacketSink(audioTrack); + const cursor = new PacketCursor(audioTrack); const run = async () => { let isFirst = true; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { await audioSource.add(packet, { decoderConfig: isFirst ? (await audioTrack.getDecoderConfig())! : undefined, }); diff --git a/test/browser/color-space.test.ts b/test/browser/color-space.test.ts index c54046d6d..af2d272e9 100644 --- a/test/browser/color-space.test.ts +++ b/test/browser/color-space.test.ts @@ -2,7 +2,7 @@ import { expect, test } from 'vitest'; import { VideoCodec } from '../../src/codec.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { Input } from '../../src/input.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; import { EncodedVideoPacketSource } from '../../src/media-source.js'; import { assert, colorSpaceIsComplete } from '../../src/misc.js'; import { Output } from '../../src/output.js'; @@ -113,10 +113,10 @@ const readPackets = async (path: string) => { const decoderConfig = await track.getDecoderConfig(); assert(decoderConfig); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); const packets: EncodedPacket[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { packets.push(packet); if (packets.length === 10) { diff --git a/test/browser/conversion.test.ts b/test/browser/conversion.test.ts index a3a557cb7..464e7e69d 100644 --- a/test/browser/conversion.test.ts +++ b/test/browser/conversion.test.ts @@ -18,8 +18,8 @@ import { assert, uint8ArraysAreEqual } from '../../src/misc.js'; import { InputVideoTrack } from '../../src/input-track.js'; import { CanvasSource, EncodedAudioPacketSource } from '../../src/media-source.js'; import { Quality } from '../../src/encode.js'; -import { EncodedPacket } from '../../src/packet.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { EncodedPacket, PacketReader } from '../../src/packet.js'; +import { PacketCursor } from '../../src/cursors.js'; test('Rotation is baked in when rerendering', async () => { using input = new Input({ @@ -1131,8 +1131,8 @@ const testCopy = async (options: { const audioTrack = await input.getPrimaryAudioTrack(); assert(videoTrack); assert(audioTrack); - const videoSink = new EncodedPacketSink(videoTrack); - const audioSink = new EncodedPacketSink(audioTrack); + const videoPacketReader = new PacketReader(videoTrack); + const audioPacketReader = new PacketReader(audioTrack); const conversion = await Conversion.init({ input, @@ -1150,14 +1150,14 @@ const testCopy = async (options: { const newAudioTrack = await newInput.getPrimaryAudioTrack(); if (newVideoTrack) { - const newVideoSink = new EncodedPacketSink(newVideoTrack); + const newVideoCursor = new PacketCursor(newVideoTrack); expect(isCloseTo(await newVideoTrack.getFirstTimestamp(), options.videoStartTimestamp!)).toBe(true); expect(isCloseTo(await newVideoTrack.computeDuration(), options.videoEndTimestamp!)).toBe(true); if (options.compareVideoPackets ?? true) { - for await (const newPacket of newVideoSink.packets()) { - const oldPacket = await videoSink.getPacket( + for await (const newPacket of newVideoCursor) { + const oldPacket = await videoPacketReader.getAt( newPacket.timestamp + options.expectedTimeOffset + precision, ); assert(oldPacket); @@ -1169,13 +1169,15 @@ const testCopy = async (options: { } if (newAudioTrack) { - const newAudioSink = new EncodedPacketSink(newAudioTrack); + const newAudioCursor = new PacketCursor(newAudioTrack); expect(isCloseTo(await newAudioTrack.getFirstTimestamp(), options.audioStartTimestamp)).toBe(true); expect(isCloseTo(await newAudioTrack.computeDuration(), options.audioEndTimestamp)).toBe(true); - for await (const newPacket of newAudioSink.packets()) { - const oldPacket = await audioSink.getPacket(newPacket.timestamp + options.expectedTimeOffset + precision); + for await (const newPacket of newAudioCursor) { + const oldPacket = await audioPacketReader.getAt( + newPacket.timestamp + options.expectedTimeOffset + precision, + ); assert(oldPacket); const process = options.processNewAudioPacketData ?? (x => x); diff --git a/test/browser/decode.test.ts b/test/browser/decode.test.ts new file mode 100644 index 000000000..5eeaf36e3 --- /dev/null +++ b/test/browser/decode.test.ts @@ -0,0 +1,22 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { UrlSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { assert } from '../../src/misc.js'; +import { AudioSampleCursor } from '../../src/cursors.js'; + +test('MP3 decoding', async () => { + using input = new Input({ + source: new UrlSource('/AudacityTest1.mp3'), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const cursor = new AudioSampleCursor(audioTrack); + const sample = await cursor.seekToFirst(); + expect(sample).not.toBe(null); + expect(sample!.timestamp).toBe(0); + expect(sample!.duration).toBeCloseTo(1152 / await audioTrack.getSampleRate()); +}); diff --git a/test/browser/flac-encoder-extension.test.ts b/test/browser/flac-encoder-extension.test.ts index 519e5a811..d67c88e2e 100644 --- a/test/browser/flac-encoder-extension.test.ts +++ b/test/browser/flac-encoder-extension.test.ts @@ -2,7 +2,6 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { AudioSampleSource } from '../../src/media-source.js'; -import { AudioSampleSink } from '../../src/media-sink.js'; import { assert } from '../../src/misc.js'; import { Output } from '../../src/output.js'; import { FlacOutputFormat } from '../../src/output-format.js'; @@ -10,6 +9,7 @@ import { AudioSample } from '../../src/sample.js'; import { BufferSource } from '../../src/source.js'; import { BufferTarget } from '../../src/target.js'; import { registerFlacEncoder } from '@mediabunny/flac-encoder'; +import { EncodedPacket, PacketReader } from '../../src/packet.js'; test('FLAC encoder, 24-bit', async () => { registerFlacEncoder(); @@ -19,18 +19,17 @@ test('FLAC encoder, 24-bit', async () => { const durationSeconds = 2; const data = createF32SineWave(sampleRate, channels, durationSeconds); - const result = await encodeAndDecodeFirstSample(new AudioSample({ + const result = await encodeSample(new AudioSample({ data, format: 'f32', numberOfChannels: channels, sampleRate, timestamp: 0, })); - using sample = result.sample; expect(result.size).toBeGreaterThan(90_000); - expect(sample.format).toBe('s32'); + expect(getBitDepthFromFlacPacket(result.packet!)).toBe(0b110); // 0b110 = 24 bit }); test('FLAC encoder, 16-bit', async () => { @@ -41,20 +40,17 @@ test('FLAC encoder, 16-bit', async () => { const durationSeconds = 2; const data = createS16SineWave(sampleRate, channels, durationSeconds); - const result = await encodeAndDecodeFirstSample(new AudioSample({ + const result = await encodeSample(new AudioSample({ data, format: 's16', numberOfChannels: channels, sampleRate, timestamp: 0, })); - using sample = result.sample; expect(result.size).toBeLessThan(50_000); // Shit just uses less data - // Really, this should only have s16 but decoders can differ and be flaky here. Trust me, I have tested that the - // encoder does in fact encode s16. - expect(['s16', 's32'].includes(sample.format)).toBe(true); + expect(getBitDepthFromFlacPacket(result.packet!)).toBe(0b100); // 0b100 = 16 bit }); const createF32SineWave = (sampleRate: number, channels: number, durationSeconds: number) => { @@ -85,7 +81,7 @@ const createS16SineWave = (sampleRate: number, channels: number, durationSeconds return data; }; -const encodeAndDecodeFirstSample = async (audioSample: AudioSample) => { +const encodeSample = async (audioSample: AudioSample) => { const output = new Output({ format: new FlacOutputFormat(), target: new BufferTarget(), @@ -108,9 +104,12 @@ const encodeAndDecodeFirstSample = async (audioSample: AudioSample) => { const track = await input.getPrimaryAudioTrack(); assert(track); - const sink = new AudioSampleSink(track); - const sample = await sink.getSample(0); - assert(sample); + const reader = new PacketReader(track); + const packet = await reader.getFirst(); - return { sample, size: output.target.buffer!.byteLength }; + return { packet, size: output.target.buffer!.byteLength }; +}; + +const getBitDepthFromFlacPacket = (packet: EncodedPacket) => { + return (packet.data[3]! & 0b1110) >> 1; }; diff --git a/test/browser/flac.test.ts b/test/browser/flac.test.ts index c4ccf5d26..7d22f9cd0 100644 --- a/test/browser/flac.test.ts +++ b/test/browser/flac.test.ts @@ -2,12 +2,12 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { BufferSource, UrlSource } from '../../src/source.js'; import { FLAC, WAVE } from '../../src/input-format.js'; -import { AudioSampleSink } from '../../src/media-sink.js'; import { assert } from '../../src/misc.js'; import { Output } from '../../src/output.js'; import { WavOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; +import { AudioSampleCursor } from '../../src/cursors.js'; test('can decode samples from a FLAC file', async () => { using input = new Input({ @@ -17,9 +17,9 @@ test('can decode samples from a FLAC file', async () => { const track = await input.getPrimaryAudioTrack(); assert(track); - const sink = new AudioSampleSink(track); + await using cursor = new AudioSampleCursor(track); - using sample = await sink.getSample(1); + using sample = await cursor.seekTo(1); assert(sample); expect(sample.timestamp).toBe(0.9287981859410431); }); diff --git a/test/browser/media-sinks.test.ts b/test/browser/media-sinks.test.ts index a542ffd94..6257a3f3e 100644 --- a/test/browser/media-sinks.test.ts +++ b/test/browser/media-sinks.test.ts @@ -3,7 +3,7 @@ import { Input } from '../../src/input.js'; import { BufferSource, UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { assert } from '../../src/misc.js'; -import { AudioSampleSink, VideoSampleSink } from '../../src/media-sink.js'; +import { AudioSampleCursor, VideoSampleCursor } from '../../src/cursors.js'; import { Output } from '../../src/output.js'; import { MpegTsOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; @@ -21,9 +21,9 @@ test('Negative audio timestamps are preserved', async () => { expect(await track.getFirstTimestamp()).toBeLessThan(0); - const sink = new AudioSampleSink(track); + await using cursor = new AudioSampleCursor(track); - for await (using sample of sink.samples()) { + for await (const sample of cursor) { expect(sample.timestamp).toBe(await track.getFirstTimestamp()); break; } @@ -38,13 +38,13 @@ test('No B-frames are skipped when software-decoding AVC', async () => { const track = await input.getPrimaryVideoTrack(); assert(track); - const sink = new VideoSampleSink(track, { + await using cursor = new VideoSampleCursor(track, { hardwareAcceleration: 'prefer-software', }); let count = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (using sample of sink.samples()) { + for await (const sample of cursor) { count++; } @@ -74,13 +74,13 @@ test('No B-frames are skipped when software-decoding AVC, Annex B edition', asyn const track = await newInput.getPrimaryVideoTrack(); assert(track); - const sink = new VideoSampleSink(track, { + await using cursor = new VideoSampleCursor(track, { hardwareAcceleration: 'prefer-software', }); let count = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (using sample of sink.samples()) { + for await (const sample of cursor) { count++; } diff --git a/test/browser/media-sources.test.ts b/test/browser/media-sources.test.ts index cbe4d480d..aad2c3b1b 100644 --- a/test/browser/media-sources.test.ts +++ b/test/browser/media-sources.test.ts @@ -8,7 +8,7 @@ import { Quality } from '../../src/encode.js'; import { Input } from '../../src/input.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { BufferSource } from '../../src/source.js'; -import { VideoSampleSink } from '../../src/media-sink.js'; +import { VideoSampleCursor } from '../../src/cursors.js'; import { assert, Rotation } from '../../src/misc.js'; import { InputAudioTrack, InputVideoTrack } from '../../src/input-track.js'; @@ -214,8 +214,8 @@ test('VideoSampleSource, flip is baked in via transform', async () => { // After the flip, the blue half is on the right const { input, track } = await readBackTrack(buffer); - const sink = new VideoSampleSink(track); - using sample = (await sink.getSample(0))!; + await using cursor = new VideoSampleCursor(track); + const sample = (await cursor.seekTo(0))!; expect(sample.flip).toBe(false); const canvas = new OffscreenCanvas(200, 100); @@ -236,8 +236,8 @@ test('VideoSampleSource, encoding flipped video frames with forced transform', a ); const { input, track } = await readBackTrack(buffer); - const sink = new VideoSampleSink(track); - using sample = (await sink.getSample(0))!; + await using cursor = new VideoSampleCursor(track); + const sample = (await cursor.seekTo(0))!; const canvas = new OffscreenCanvas(200, 100); const ctx = canvas.getContext('2d', { willReadFrequently: true })!; @@ -368,12 +368,8 @@ test('VideoSampleSource, transform.process expands every frame into two', async process: (sample) => { const t = sample.timestamp; const d = sample.duration; - const clone = sample.clone(); - clone.setTimestamp(2 * t); - clone.setDuration(d); - const clone2 = sample.clone(); - clone2.setTimestamp(2 * t + d); - clone2.setDuration(d); + const clone = sample.clone({ timestamp: 2 * t, duration: d }); + const clone2 = sample.clone({ timestamp: 2 * t + d, duration: d }); return [clone, clone2]; }, }, @@ -603,10 +599,10 @@ const readBackTrack = async (buffer: ArrayBuffer) => { const readBackSamples = async (buffer: ArrayBuffer) => { const { input, track } = await readBackTrack(buffer); - const sink = new VideoSampleSink(track); + await using cursor = new VideoSampleCursor(track); const samples: { codedWidth: number; codedHeight: number; timestamp: number; duration: number }[] = []; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { samples.push({ codedWidth: sample.codedWidth, codedHeight: sample.codedHeight, diff --git a/test/browser/mp3.test.ts b/test/browser/mp3.test.ts index 2c3af54a6..c4eaf579d 100644 --- a/test/browser/mp3.test.ts +++ b/test/browser/mp3.test.ts @@ -3,7 +3,7 @@ import { Input } from '../../src/input.js'; import { UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { assert } from '../../src/misc.js'; -import { AudioSampleSink } from '../../src/media-sink.js'; +import { AudioSampleCursor } from '../../src/cursors.js'; // "joined" in the sense that it was two separate MP3s that were spliced together (I think) test('Can decode malformed joined MP3', async () => { @@ -15,7 +15,7 @@ test('Can decode malformed joined MP3', async () => { const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); - const sink = new AudioSampleSink(audioTrack); + await using cursor = new AudioSampleCursor(audioTrack); // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (using sample of sink.samples()); + for await (const sample of cursor); }); diff --git a/test/browser/mpeg-ts-decoding.test.ts b/test/browser/mpeg-ts-decoding.test.ts index 0462bf577..0f6d3d964 100644 --- a/test/browser/mpeg-ts-decoding.test.ts +++ b/test/browser/mpeg-ts-decoding.test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { Input } from '../../src/input.js'; import { UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { VideoSampleSink, AudioSampleSink } from '../../src/media-sink.js'; +import { VideoSampleCursor, AudioSampleCursor } from '../../src/cursors.js'; import { assert } from '../../src/misc.js'; test('MPEG-TS video samples are decodable', async () => { @@ -14,11 +14,11 @@ test('MPEG-TS video samples are decodable', async () => { const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new VideoSampleSink(videoTrack); + await using cursor = new VideoSampleCursor(videoTrack); let count = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (using sample of sink.samples()) { + for await (const sample of cursor) { count++; } @@ -34,11 +34,11 @@ test('MPEG-TS audio samples are decodable', async () => { const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); - const sink = new AudioSampleSink(audioTrack); + await using cursor = new AudioSampleCursor(audioTrack); let count = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (using sample of sink.samples()) { + for await (const sample of cursor) { count++; } diff --git a/test/browser/mpeg-ts-muxing.test.ts b/test/browser/mpeg-ts-muxing.test.ts index fea47729c..96297fab8 100644 --- a/test/browser/mpeg-ts-muxing.test.ts +++ b/test/browser/mpeg-ts-muxing.test.ts @@ -7,7 +7,8 @@ import { MpegTsOutputFormat } from '../../src/output-format.js'; import { BufferTarget, StreamTarget, StreamTargetChunk } from '../../src/target.js'; import { CanvasSource, EncodedAudioPacketSource, EncodedVideoPacketSource } from '../../src/media-source.js'; import { Quality } from '../../src/encode.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { assert } from '../../src/misc.js'; import { Conversion } from '../../src/conversion.js'; @@ -86,10 +87,10 @@ test('MPEG-TS muxing with AVC and AAC', async () => { const aacTrack = await aacInput.getPrimaryAudioTrack(); assert(aacTrack); - const aacSink = new EncodedPacketSink(aacTrack); + const aacCursor = new PacketCursor(aacTrack); let isFirst = true; - for await (const packet of aacSink.packets()) { + for await (const packet of aacCursor) { if (packet.timestamp >= duration) break; await audioSource.add(packet, { @@ -147,48 +148,48 @@ test('MPEG-TS muxing with AVC and AAC', async () => { expect(audioDecoderConfig.description).toBeUndefined(); // ADTS, no description // Verify video packets are Annex B - const videoSink = new EncodedPacketSink(videoTrack); + const videoReader = new PacketReader(videoTrack); let videoPacketCount = 0; - const firstVideoPacket = await videoSink.getFirstPacket(); + const firstVideoPacket = await videoReader.getFirst(); assert(firstVideoPacket); expect(firstVideoPacket.type).toBe('key'); - const secondVideoPacket = await videoSink.getNextPacket(firstVideoPacket); + const secondVideoPacket = await videoReader.getNext(firstVideoPacket); assert(secondVideoPacket); - for await (const packet of videoSink.packets()) { + for await (const packet of new PacketCursor(videoTrack)) { expect(packet.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); // Annex B start code videoPacketCount++; } // Check that seeking works - const middlePacket = await videoSink.getPacket(1); + const middlePacket = await videoReader.getAt(1); assert(middlePacket); expect(middlePacket.timestamp).toBeCloseTo(1); expect(videoPacketCount).toBe(frameCount); // Verify audio packets are ADTS - const audioSink = new EncodedPacketSink(audioTrack); + const audioReader = new PacketReader(audioTrack); let audioPacketCount = 0; - const firstAudioPacket = await audioSink.getFirstPacket(); + const firstAudioPacket = await audioReader.getFirst(); assert(firstAudioPacket); expect(firstAudioPacket.type).toBe('key'); - const secondAudioPacket = await audioSink.getNextPacket(firstAudioPacket); + const secondAudioPacket = await audioReader.getNext(firstAudioPacket); assert(secondAudioPacket); expect(secondAudioPacket.type).toBe('key'); - for await (const packet of audioSink.packets()) { + for await (const packet of new PacketCursor(audioTrack)) { expect(packet.data[0]).toBe(0xff); // ADTS sync word expect(packet.data[1]! & 0xf0).toBe(0xf0); // ADTS sync word continued audioPacketCount++; } // Check that seeking works - const audioMiddlePacket = await audioSink.getPacket(1); + const audioMiddlePacket = await audioReader.getAt(1); assert(audioMiddlePacket); expect(audioMiddlePacket.timestamp).toBeCloseTo(1, 1); @@ -233,11 +234,11 @@ test('MPEG-TS muxing with HEVC and MP3', async () => { const hevcTrack = await hevcInput.getPrimaryVideoTrack(); assert(hevcTrack); - const hevcSink = new EncodedPacketSink(hevcTrack); + const hevcCursor = new PacketCursor(hevcTrack); let isFirstVideo = true; let videoPacketCountWritten = 0; - for await (const packet of hevcSink.packets()) { + for await (const packet of hevcCursor) { if (packet.timestamp >= duration) break; await videoSource.add(packet, { @@ -258,10 +259,10 @@ test('MPEG-TS muxing with HEVC and MP3', async () => { const mp3Track = await mp3Input.getPrimaryAudioTrack(); assert(mp3Track); - const mp3Sink = new EncodedPacketSink(mp3Track); + const mp3Cursor = new PacketCursor(mp3Track); let isFirstAudio = true; - for await (const packet of mp3Sink.packets()) { + for await (const packet of mp3Cursor) { if (packet.timestamp >= duration) break; await audioSource.add(packet, { @@ -313,10 +314,10 @@ test('MPEG-TS muxing with HEVC and MP3', async () => { expect(audioDecoderConfig.description).toBeUndefined(); // MP3 has no description // Verify video packets are Annex B - const videoSink = new EncodedPacketSink(videoTrack); + const videoCursor = new PacketCursor(videoTrack); let videoPacketCount = 0; - for await (const packet of videoSink.packets()) { + for await (const packet of videoCursor) { expect(packet.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); // Annex B start code videoPacketCount++; } @@ -324,10 +325,10 @@ test('MPEG-TS muxing with HEVC and MP3', async () => { expect(videoPacketCount).toBe(videoPacketCountWritten); // Verify audio packets are MP3 frames - const audioSink = new EncodedPacketSink(audioTrack); + const audioCursor = new PacketCursor(audioTrack); let audioPacketCount = 0; - for await (const packet of audioSink.packets()) { + for await (const packet of audioCursor) { expect(packet.data[0]).toBe(0xff); // MP3 sync word audioPacketCount++; } @@ -412,10 +413,10 @@ test('MPEG-TS muxing with video only', async () => { const audioTrack = await input.getPrimaryAudioTrack(); expect(audioTrack).toBeNull(); - const videoSink = new EncodedPacketSink(videoTrack); + const videoCursor = new PacketCursor(videoTrack); let videoPacketCount = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const packet of videoSink.packets()) { + for await (const packet of videoCursor) { videoPacketCount++; } expect(videoPacketCount).toBe(frameCount); @@ -442,10 +443,10 @@ test('MPEG-TS muxing with audio only', async () => { const aacTrack = await aacInput.getPrimaryAudioTrack(); assert(aacTrack); - const aacSink = new EncodedPacketSink(aacTrack); + const aacCursor = new PacketCursor(aacTrack); let isFirst = true; - for await (const packet of aacSink.packets()) { + for await (const packet of aacCursor) { if (packet.timestamp >= duration) break; await audioSource.add(packet, { @@ -472,10 +473,10 @@ test('MPEG-TS muxing with audio only', async () => { assert(audioTrack); expect(await audioTrack.getCodec()).toBe('aac'); - const audioSink = new EncodedPacketSink(audioTrack); + const audioCursor = new PacketCursor(audioTrack); let audioPacketCount = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const packet of audioSink.packets()) { + for await (const packet of audioCursor) { audioPacketCount++; } expect(audioPacketCount).toBeGreaterThan(0); @@ -505,11 +506,11 @@ test('MPEG-TS muxing with two video tracks', async () => { const hevcTrack = await hevcInput.getPrimaryVideoTrack(); assert(hevcTrack); - const hevcSink = new EncodedPacketSink(hevcTrack); + const hevcCursor = new PacketCursor(hevcTrack); const hevcDecoderConfig = await hevcTrack.getDecoderConfig(); let isFirst = true; - for await (const packet of hevcSink.packets()) { + for await (const packet of hevcCursor) { if (packet.timestamp >= duration) break; const meta = { decoderConfig: isFirst ? hevcDecoderConfig! : undefined }; @@ -561,13 +562,11 @@ test('MPEG-TS muxing with two audio tracks', async () => { const aacTrack = await aacInput.getPrimaryAudioTrack(); assert(aacTrack); - const aacSink = new EncodedPacketSink(aacTrack); - const decoderConfig = await aacTrack.getDecoderConfig(); assert(decoderConfig); let isFirst1 = true; - for await (const packet of aacSink.packets()) { + for await (const packet of new PacketCursor(aacTrack)) { if (packet.timestamp >= duration) break; await audioSource1.add(packet, { @@ -577,7 +576,7 @@ test('MPEG-TS muxing with two audio tracks', async () => { } let isFirst2 = true; - for await (const packet of aacSink.packets()) { + for await (const packet of new PacketCursor(aacTrack)) { if (packet.timestamp >= duration) break; await audioSource2.add(packet, { @@ -650,14 +649,14 @@ test('MPEG-TS transmux (Annex B and ADTS passthrough)', async () => { expect(await outputAudioTrack.getCodec()).toBe(await inputAudioTrack.getCodec()); // Verify video packets are Annex B - const videoSink = new EncodedPacketSink(outputVideoTrack); - const firstVideoPacket = await videoSink.getFirstPacket(); + const videoReader = new PacketReader(outputVideoTrack); + const firstVideoPacket = await videoReader.getFirst(); assert(firstVideoPacket); expect(firstVideoPacket.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); // Verify audio packets are ADTS - const audioSink = new EncodedPacketSink(outputAudioTrack); - const firstAudioPacket = await audioSink.getFirstPacket(); + const audioReader = new PacketReader(outputAudioTrack); + const firstAudioPacket = await audioReader.getFirst(); assert(firstAudioPacket); expect(firstAudioPacket.data[0]).toBe(0xff); expect(firstAudioPacket.data[1]! & 0xf0).toBe(0xf0); @@ -730,10 +729,10 @@ test('MPEG-TS muxing with StreamTarget', async () => { assert(videoTrack); expect(await videoTrack.getCodec()).toBe('avc'); - const videoSink = new EncodedPacketSink(videoTrack); + const videoCursor = new PacketCursor(videoTrack); let videoPacketCount = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const packet of videoSink.packets()) { + for await (const packet of videoCursor) { videoPacketCount++; } expect(videoPacketCount).toBe(frameCount); @@ -779,10 +778,10 @@ const testNegativeTimestampRoundTrip = async (timestamps: number[], duration: nu const track = await input.getPrimaryVideoTrack(); assert(track); - const sink = new EncodedPacketSink(track); + const reader = new PacketReader(track); const outputPackets = []; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(track)) { outputPackets.push(packet); } @@ -793,7 +792,7 @@ const testNegativeTimestampRoundTrip = async (timestamps: number[], duration: nu }))).toEqual(expectedPackets); for (const expectedPacket of expectedPackets) { - const outputPacket = await sink.getPacket(expectedPacket.timestamp); + const outputPacket = await reader.getAt(expectedPacket.timestamp); assert(outputPacket); expect({ diff --git a/test/browser/ogg-demuxer.test.ts b/test/browser/ogg-demuxer.test.ts index 388da4447..16b4d0f18 100644 --- a/test/browser/ogg-demuxer.test.ts +++ b/test/browser/ogg-demuxer.test.ts @@ -2,7 +2,7 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { AudioBufferSink } from '../../src/media-sink.js'; +import { AudioSampleCursor } from '../../src/cursors.js'; import { assert } from '../../src/misc.js'; // VLC creates OGG files with an empty EOS page, which previously caused decoding errors @@ -15,11 +15,17 @@ test('can decode OGG Vorbis file with empty EOS page', async () => { const track = await input.getPrimaryAudioTrack(); assert(track); - const sink = new AudioBufferSink(track); + await using cursor = new AudioSampleCursor(track); const buffers: AudioBuffer[] = []; - for await (const { buffer } of sink.buffers(4, 10)) { - buffers.push(buffer); + await cursor.seekTo(4); + + for await (const sample of cursor) { + if (sample.timestamp >= 10) { + break; + } + + buffers.push(sample.toAudioBuffer()); } expect(buffers.length).toBeGreaterThan(0); diff --git a/test/browser/ogg-muxer.test.ts b/test/browser/ogg-muxer.test.ts index 5d7aeb924..b7a6dc693 100644 --- a/test/browser/ogg-muxer.test.ts +++ b/test/browser/ogg-muxer.test.ts @@ -3,13 +3,13 @@ import { Output } from '../../src/output.js'; import { OggOutputFormat } from '../../src/output-format.js'; import { BufferTarget, NullTarget } from '../../src/target.js'; import { AudioBufferSource, EncodedAudioPacketSource } from '../../src/media-source.js'; -import { EncodedPacket } from '../../src/packet.js'; +import { EncodedPacket, PacketReader } from '../../src/packet.js'; import { Quality } from '../../src/encode.js'; import { assert } from '../../src/misc.js'; import { Input } from '../../src/input.js'; import { BufferSource } from '../../src/source.js'; import { ALL_FORMATS, OggInputFormat } from '../../src/input-format.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; test('maximumPageDuration option', async () => { const sampleRate = 48000; @@ -136,12 +136,12 @@ test('Multi-frame Opus packets', async () => { expect(await input.getFormat()).toBeInstanceOf(OggInputFormat); - const sink = new EncodedPacketSink((await input.getPrimaryAudioTrack())!); - const firstPacket = await sink.getFirstPacket(); + const packetReader = new PacketReader((await input.getPrimaryAudioTrack())!); + const firstPacket = await packetReader.getFirst(); assert(firstPacket); // The first packet is shortened by the pre-skip, so check the second one - const packet = await sink.getNextPacket(firstPacket); + const packet = await packetReader.getNext(firstPacket); expect(packet?.duration).toBe(packetDuration); }); @@ -196,7 +196,7 @@ test('Opus start offset and end trimming survive a demux-mux roundtrip', async ( assert(track); const packets: EncodedPacket[] = []; - for await (const packet of new EncodedPacketSink(track).packets()) { + for await (const packet of new PacketCursor(track)) { packets.push(packet); } diff --git a/test/browser/par.test.ts b/test/browser/par.test.ts index edf760058..9829692c5 100644 --- a/test/browser/par.test.ts +++ b/test/browser/par.test.ts @@ -2,7 +2,7 @@ import { expect, test } from 'vitest'; import { Conversion } from '../../src/conversion.js'; import { ALL_FORMATS, MATROSKA, MP4, MPEG_TS } from '../../src/input-format.js'; import { Input } from '../../src/input.js'; -import { VideoSampleSink } from '../../src/media-sink.js'; +import { VideoSampleCursor } from '../../src/cursors.js'; import { assert, Rational } from '../../src/misc.js'; import { Output } from '../../src/output.js'; import { MkvOutputFormat, Mp4OutputFormat, MpegTsOutputFormat } from '../../src/output-format.js'; @@ -30,8 +30,8 @@ test('Pixel aspect ratio reading', async () => { expect(decoderConfig.displayAspectWidth).toBe(await videoTrack.getSquarePixelWidth()); expect(decoderConfig.displayAspectHeight).toBe(await videoTrack.getSquarePixelHeight()); - const sink = new VideoSampleSink(videoTrack); - using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + await using cursor = new VideoSampleCursor(videoTrack); + const sample = (await cursor.seekToFirst())!; expect(sample.rotation).toBe(0); expect(sample.visibleRect.width).toBe(sample.codedWidth); diff --git a/test/browser/read-mp4.test.ts b/test/browser/read-mp4.test.ts index de0aa4a2f..caea7cbb7 100644 --- a/test/browser/read-mp4.test.ts +++ b/test/browser/read-mp4.test.ts @@ -2,7 +2,7 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { AudioSampleSink } from '../../src/media-sink.js'; +import { AudioSampleCursor } from '../../src/cursors.js'; import { assert } from '../../src/misc.js'; test('Encrypted MP4 without senc', async () => { @@ -20,8 +20,8 @@ test('Encrypted MP4 without senc', async () => { assert(audioTrack); // Test that it can decode - const sink = new AudioSampleSink(audioTrack); - using firstSample = await sink.getSample(0); + await using cursor = new AudioSampleCursor(audioTrack); + const firstSample = await cursor.seekTo(0); assert(firstSample); expect(firstSample.timestamp).toBe(0); }); diff --git a/test/browser/sample-cursor.test.ts b/test/browser/sample-cursor.test.ts new file mode 100644 index 000000000..14dda4752 --- /dev/null +++ b/test/browser/sample-cursor.test.ts @@ -0,0 +1,1305 @@ +import { afterEach, beforeEach, expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { BufferSource, UrlSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { + AudioSampleCursor, + canvasTransformer, + VideoSampleCursor, + WrappedCanvas, +} from '../../src/cursors.js'; +import { AudioSample, VideoSample } from '../../src/sample.js'; +import { promiseAllEnsureOrder, promiseIterateAll } from '../../src/misc.js'; +import { Output } from '../../src/output.js'; +import { WavOutputFormat } from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { EncodedAudioPacketSource } from '../../src/media-source.js'; +import { EncodedPacket } from '../../src/packet.js'; + +beforeEach(() => { + VideoSample._openSampleCount = 0; + AudioSample._openSampleCount = 0; +}); + +afterEach(() => { + expect(VideoSample._openSampleCount).toBe(0); + expect(AudioSample._openSampleCount).toBe(0); +}); + +test('Sample cursor seeking', { timeout: 30_000 }, async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + await using cursor = new VideoSampleCursor(videoTrack); + cursor._debug.enabled = true; + + expect(cursor.current).toBe(null); + expect(cursor._debug.pumpsStarted).toBe(0); + + // Seek to start + const seekToResult1 = cursor.seekToFirst(); + expect(seekToResult1).instanceOf(Promise); + const sample1 = (await seekToResult1)!; + expect(sample1).not.toBe(null); + expect(sample1).toBe(cursor.current); + expect(sample1.timestamp).toBeLessThanOrEqual(0); + expect(sample1.closed).toBe(false); + + expect(cursor._debug.pumpsStarted).toBe(1); + + // Seek to a frame in the current GOP + const seekToResult2 = cursor.seekTo(0.5); + expect(seekToResult2).instanceOf(Promise); + const sample2 = (await seekToResult2)!; + expect(sample2).not.toBe(null); + expect(sample2).toBe(cursor.current); + expect(sample2.timestamp).toBeGreaterThan(0); + expect(sample2.timestamp).toBeLessThanOrEqual(0.5); + expect(sample2.closed).toBe(false); + expect(sample1.closed).toBe(true); + + // Wait a little bit so the next sample is most definitely decoded and waiting in the queue + await new Promise(resolve => setTimeout(resolve, 200)); + + const seekToResult3 = cursor.seekTo(0.55); + expect(seekToResult3).instanceOf(VideoSample); // No promise this time + const sample3 = (await seekToResult3)!; + expect(sample3).toBe(cursor.current); + expect(sample3.timestamp).toBeGreaterThan(0.5); + expect(sample3.timestamp).toBeLessThanOrEqual(0.55); + expect(sample3.closed).toBe(false); + expect(sample2.closed).toBe(true); + + // Let's get the same sample again + const seekToResult4 = cursor.seekTo(0.55); + expect(seekToResult4).instanceOf(VideoSample); + const sample4 = (await seekToResult4)!; + expect(sample3).toBe(sample4); + + expect(cursor._debug.pumpsStarted).toBe(1); + + // Seek to a different GOP + const seekToResult5 = cursor.seekTo(2); + expect(seekToResult5).instanceOf(Promise); + const sample5 = (await seekToResult5)!; + expect(sample5).toBe(cursor.current); + expect(sample5.timestamp).toBeGreaterThan(0.55); + expect(sample5.timestamp).toBeLessThanOrEqual(2); + expect(sample5.closed).toBe(false); + expect(sample3.closed).toBe(true); + + expect(cursor._debug.pumpsStarted).toBe(2); + + // Seek to a frame in the current GOP + const seekToResult6 = cursor.seekTo(2.5); + expect(seekToResult6).instanceOf(Promise); + const sample6 = (await seekToResult6)!; + expect(sample6).toBe(cursor.current); + expect(sample6.timestamp).toBeGreaterThan(2); + expect(sample6.timestamp).toBeLessThanOrEqual(2.5); + expect(sample6.closed).toBe(false); + expect(sample5.closed).toBe(true); + + // Seek to a frame in the current GOP, but backwards, requiring decoding to start over + const seekToResult7 = cursor.seekTo(2.4); + expect(seekToResult7).instanceOf(Promise); + const sample7 = (await seekToResult7)!; + expect(sample7).toBe(cursor.current); + expect(sample7.timestamp).toBeGreaterThan(2); + expect(sample7.timestamp).toBeLessThanOrEqual(2.4); + expect(sample7.closed).toBe(false); + expect(sample6.closed).toBe(true); + + expect(cursor._debug.pumpsStarted).toBe(3); + + // Seek to a previous GOP + const seekToResult8 = cursor.seekTo(1); + expect(seekToResult8).instanceOf(Promise); + const sample8 = (await seekToResult8)!; + expect(sample8).toBe(cursor.current); + expect(sample8.timestamp).toBeLessThanOrEqual(1); + expect(sample8.closed).toBe(false); + expect(sample7.closed).toBe(true); + + expect(cursor._debug.pumpsStarted).toBe(4); + + // Seek to past the end + const seekToResult9 = cursor.seekTo(Infinity); + expect(seekToResult9).instanceOf(Promise); + const sample9 = (await seekToResult9)!; + expect(sample9).toBe(cursor.current); + expect(sample9.timestamp).toBe(5); // The length of the video + expect(sample9.closed).toBe(false); + expect(sample8.closed).toBe(true); + + expect(cursor._debug.pumpsStarted).toBe(5); + + await new Promise(resolve => setTimeout(resolve, 200)); + expect(cursor.current).toBe(sample9); + + // Seek to the end again + const seekToResult10 = cursor.seekTo(Infinity); + expect(seekToResult10).not.toBeInstanceOf(Promise); + const sample10 = await seekToResult10; + expect(sample9).toBe(sample10); + + await new Promise(resolve => setTimeout(resolve, 200)); + expect(cursor._debug.pumpsStarted).toBe(5); + + // Seek to before the start + const seekToResult11 = cursor.seekTo(-Infinity); + expect(seekToResult11).toBe(null); + expect(sample9.closed).toBe(true); + expect(cursor.current).toBe(null); + + const seekToResult12 = cursor.seekToKey(2.5); + expect(seekToResult12).toBeInstanceOf(Promise); + const sample12 = (await seekToResult12)!; + expect(sample12).toBe(cursor.current); + expect(sample12.timestamp).toBe(2); + expect(sample12.closed).toBe(false); + + await cursor.close(); + + expect(sample12.closed).toBe(true); + expect(cursor.current).toBe(null); + expect(cursor._debug.pumpsStarted).toBe(6); + + await cursor.close(); +}); + +test('Sample cursor advancing', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const cursor = new VideoSampleCursor(videoTrack); + cursor._debug.enabled = true; + + expect(cursor.current).toBe(null); + expect(cursor._debug.pumpsStarted).toBe(0); + + const firstSample = (await cursor.seekToFirst())!; + const secondSample = (await cursor.next())!; + const thirdSample = (await cursor.next())!; + + expect(secondSample.timestamp).toBeGreaterThan(firstSample.timestamp); + expect(thirdSample.timestamp).toBeGreaterThan(secondSample.timestamp); + expect(firstSample.closed).toBe(true); + expect(secondSample.closed).toBe(true); + expect(thirdSample.closed).toBe(false); + expect(cursor.current).toBe(thirdSample); + + await new Promise(resolve => setTimeout(resolve, 200)); + + const fourthSampleResult = cursor.next(); // It's available instantly + expect(fourthSampleResult).not.toBeInstanceOf(Promise); + + const fourthSample = fourthSampleResult as VideoSample; + expect(fourthSample.timestamp).toBeGreaterThan(thirdSample.timestamp); + + const lastSample = await cursor.seekTo(5); + expect(lastSample).not.toBe(null); + + const nextSample = await cursor.next(); + expect(nextSample).toBe(null); + const nextNextSample = await cursor.next(); + expect(nextNextSample).toBe(null); + + const middleSample = (await cursor.seekTo(3))!; + const sampleAfterMiddle = (await cursor.next())!; + expect(sampleAfterMiddle.timestamp).toBeGreaterThan(middleSample.timestamp); + + await cursor.seekTo(-Infinity); + const firstSampleAgain = (await cursor.next())!; + expect(firstSampleAgain.timestamp).toBe(firstSample.timestamp); + + let total = 0; + let lastTimestamp = -Infinity; + await cursor.iterate((sample) => { + total++; + + expect(sample.timestamp).toBeGreaterThan(lastTimestamp); + lastTimestamp = sample.timestamp; + }); + expect(total).toBe(121); + expect(cursor.current).toBe(null); + + total = 0; + await cursor.iterate(() => void total++); + expect(total).toBe(0); // Since we're at the end + + await cursor.seekToFirst(); ; + total = 0; + await cursor.iterate((sample) => { + if (sample.timestamp === 1) { + return false; + } + + total++; + }); + expect(total).toBe(24); + expect(cursor.current!.timestamp).toBe(1); + + total = 0; + for await (const sample of cursor) { + if (total === 0) { + expect(sample.timestamp).toBe(1); + } + + total++; + } + + expect(total).toBe(97); + + await cursor.close(); + + expect(cursor._debug.pumpsStarted).toBe(5); +}); + +test('Sample cursor advancing, cold start', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const cursor = new VideoSampleCursor(videoTrack); + cursor._debug.enabled = true; + + let firstSample = (await cursor.next())!; + expect(firstSample).not.toBe(null); + expect(firstSample.timestamp).toBe(0); + + const secondSample = (await cursor.next())!; + expect(secondSample.timestamp).toBeGreaterThan(firstSample.timestamp); + + await cursor.seekTo(-Infinity); + + firstSample = (await cursor.next())!; + expect(firstSample).not.toBe(null); + expect(firstSample.timestamp).toBe(0); + + await cursor.seekTo(-Infinity); + + firstSample = (await cursor.nextKey())!; + expect(firstSample).not.toBe(null); + expect(firstSample.timestamp).toBe(0); + + void cursor.next(); + void cursor.seekTo(2); + + await cursor.close(); + + // Ensure the calls were serialized correctly + expect(cursor._debug.seekPackets.map(x => x?.timestamp ?? null)).toEqual([0, null, 0, null, 0, 2]); + + const cursor2 = new VideoSampleCursor(videoTrack); + for await (const sample of cursor2) { + expect(sample.timestamp).toBe(0); + break; + } + + await cursor2.iterate((sample) => { + expect(sample.timestamp).toBe(0); + return false; + }); + + await cursor2.close(); +}); + +test('Sample cursor advancing, nextKey', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + await using cursor = new VideoSampleCursor(videoTrack); + + await cursor.seekToFirst(); + + let sample = await cursor.nextKey(); + expect(sample!.timestamp).toBe(1); + + sample = await cursor.nextKey(); + expect(sample!.timestamp).toBe(2); + + void cursor.nextKey(); + void cursor.nextKey(); + await cursor.nextKey(); + + expect(cursor.current!.timestamp).toBe(5); + + sample = await cursor.nextKey(); + expect(sample).toBe(null); + + await cursor.seekTo(0.5); + + sample = await cursor.nextKey(); + expect(sample!.timestamp).toBe(1); + + void cursor.seekTo(3.5); + sample = await cursor.nextKey(); + expect(sample!.timestamp).toBe(4); + + await cursor.seekTo(3.9); + void cursor.next(); + void cursor.next(); + void cursor.next(); + sample = await cursor.nextKey(); + expect(sample!.timestamp).toBe(5); +}); + +test('Sample cursor sample reuse', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const cursor1 = new VideoSampleCursor(videoTrack); + cursor1._debug.enabled = true; + + const sample1 = await cursor1.seekToFirst(); + const sample2 = await cursor1.seekToFirst(); + + expect(cursor1._debug.pumpsStarted).toBe(1); + expect(sample1!.timestamp).toBe(sample2!.timestamp); + expect(sample1).toBe(sample2); + + await cursor1.next(); + expect(sample1!.closed).toBe(true); + expect(sample2!.closed).toBe(true); + + const cursor2 = new VideoSampleCursor(videoTrack, { + closeSamples: false, + }); + + const sample3 = await cursor2.seekToFirst(); + const sample4 = await cursor2.seekToFirst(); + expect(sample3!.timestamp).toBe(sample4!.timestamp); + expect(sample3).not.toBe(sample4); + + sample3!.close(); + expect(sample3!.closed).toBe(true); + expect(sample4!.closed).toBe(false); + sample4!.close(); + + let count = 0; + const cursor3 = new VideoSampleCursor(videoTrack, { + transform: () => count++, + }); + + await cursor3.seekToFirst(); + await cursor3.seekToFirst(); + + expect(count).toBe(1); + + count = 0; + const cursor4 = new VideoSampleCursor(videoTrack, { + closeSamples: false, + transform: sample => (sample.close(), count++), + }); + + await cursor4.seekToFirst(); + await cursor4.seekToFirst(); + + expect(count).toBe(2); + + await cursor1.close(); + await cursor2.close(); + await cursor3.close(); + await cursor4.close(); +}); + +test('Sample cursor reset', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + await using cursor = new VideoSampleCursor(videoTrack); + + expect(cursor.closed).toBe(false); + + await cursor.close(); + expect(cursor.closed).toBe(true); + + expect(() => cursor.seekToFirst()).toThrow('cursor has been closed'); + + await cursor.reset(); + expect(cursor.closed).toBe(false); + + const firstSample = await cursor.seekToFirst(); + expect(firstSample).not.toBe(null); + + await cursor.close(); + await cursor.reset(); + + const nextSample = await cursor.next(); + expect(nextSample).not.toBe(null); + expect(nextSample!.timestamp).toBe(firstSample!.timestamp); + + // Absolutely deranged usage, but it's gotta work! + const commands = [ + cursor.next(), + cursor.close(), + cursor.reset(), + cursor.next(), + cursor.next(), + cursor.reset(), + cursor.next(), + ]; + + const results = await promiseAllEnsureOrder(commands); + + expect(cursor.closed).toBe(false); + expect(results[0]!.timestamp).toBeGreaterThan(firstSample!.timestamp); + expect(results[3]!.timestamp).toBe(firstSample!.timestamp); + expect(results[4]!.timestamp).toBe(results[0]!.timestamp); + expect(results[6]!.timestamp).toBe(firstSample!.timestamp); + + await using cursor2 = new VideoSampleCursor(videoTrack); + cursor2._debug.enabled = true; + + // Test if queueing a reset makes the decoder decode minimally many packets + const commands2 = [ + cursor2.seekToFirst(), + cursor2.reset(), + ]; + + await promiseAllEnsureOrder(commands2); + + expect(cursor2._debug.decodedPackets).toHaveLength(1); +}); + +test('Decoder setup error & reset', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + await using cursor1 = new VideoSampleCursor(videoTrack); + cursor1._debug.enabled = true; + cursor1._debug.throwInDecoderInit = true; + expect(cursor1.closed).toBe(false); + + await expect(cursor1.seekToFirst()).rejects.toThrow('Fake decoder init error'); + + expect(cursor1.closed).toBe(true); + expect(cursor1.errored).toBe(true); + + expect(() => cursor1.seekToFirst()).toThrow('Fake decoder init error'); // Bricked + + await expect(cursor1.reset()).rejects.toThrow('Fake decoder init error'); + expect(cursor1.closed).toBe(true); + + cursor1._debug.throwInDecoderInit = false; + await cursor1.reset(); + expect(cursor1.closed).toBe(false); + expect(cursor1.errored).toBe(false); + + const firstSample = (await cursor1.seekToFirst())!; + expect(firstSample).not.toBe(null); + + // Let's test directly closing after opening + const cursor2 = new VideoSampleCursor(videoTrack); + cursor2._debug.enabled = true; + cursor2._debug.throwInDecoderInit = true; + await cursor2.close(); +}); + +test('Decoder pump error handling & reset', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + await using cursor = new VideoSampleCursor(videoTrack); + cursor._debug.enabled = true; + + cursor._debug.throwInPump = true; + await expect(async () => cursor.seekToFirst()).rejects.toThrow('Fake pump error'); + + expect(cursor.closed).toBe(true); + expect(cursor.errored).toBe(true); + expect(cursor.current).toBe(null); + + cursor._debug.throwInPump = false; + await expect(async () => cursor.seekToFirst()).rejects.toThrow('Fake pump error'); // It's bricked + + await cursor.reset(); + + const firstSample = await cursor.seekToFirst(); + expect(firstSample!.timestamp).toBe(0); +}); + +test('Decoder errors & reset', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + + const cursor1 = new VideoSampleCursor(videoTrack); + cursor1._debug.enabled = true; + cursor1._debug.throwDecoderError = true; + + await expect(cursor1.seekToFirst()).rejects.toThrow('Fake decoder error'); + expect(cursor1.closed).toBe(true); + expect(cursor1.errored).toBe(true); + + cursor1._debug.throwDecoderError = false; + expect(() => cursor1.seekToFirst()).toThrow('Fake decoder error'); // Bricked + + await cursor1.reset(); + + const firstSample = await cursor1.seekToFirst(); + expect(firstSample!.timestamp).toBe(0); + + const cursor2 = new VideoSampleCursor(videoTrack); + cursor2._debug.enabled = true; + + await cursor2.seekToFirst(); + + cursor2._debug.throwDecoderError = true; + await new Promise(resolve => setTimeout(resolve, 200)); + + expect(() => cursor2.next()).toThrow('Fake decoder error'); + expect(cursor2.closed).toBe(true); +}); + +test('Use after close', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + + const cursor1 = new VideoSampleCursor(videoTrack); + + expect(cursor1.closed).toBe(false); + await cursor1.close(); + expect(cursor1.closed).toBe(true); + + await expect(async () => await cursor1.seekToFirst()).rejects.toThrow('cursor has been closed'); + await expect(async () => await cursor1.seekTo(0)).rejects.toThrow('cursor has been closed'); + await expect(async () => await cursor1.seekToKey(0)).rejects.toThrow('cursor has been closed'); + await expect(async () => await cursor1.next()).rejects.toThrow('cursor has been closed'); + + const cursor2 = new VideoSampleCursor(videoTrack); + const commands2 = [ + cursor2.seekToFirst(), + cursor2.next(), + cursor2.close(), + cursor2.seekTo(1), + ]; + + await expect(commands2[0]).resolves.toBeInstanceOf(VideoSample); + await expect(commands2[1]).resolves.toBeInstanceOf(VideoSample); + await expect(commands2[2]).resolves.toBeUndefined(); + await expect(commands2[3]).rejects.toThrow('cursor has been closed'); + + expect(cursor2._pumpRunning).toBe(false); + expect(cursor2._decoder).toBe(null); + + const cursor3 = new VideoSampleCursor(videoTrack); + const commands3 = [ + cursor3.seekToFirst(), + cursor3.next(), + cursor3.close(), + cursor3.next(), + ]; + + await expect(commands3[0]).resolves.toBeInstanceOf(VideoSample); + await expect(commands3[1]).resolves.toBeInstanceOf(VideoSample); + await expect(commands3[2]).resolves.toBeUndefined(); + await expect(commands3[3]).rejects.toThrow('cursor has been closed'); +}); + +test('Wait until idle', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov', { + maxCacheSize: 0, // HORRENDOUS for performance, but forces promises + }), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const cursor = new VideoSampleCursor(videoTrack); + + const promise1 = cursor.waitUntilIdle(); + expect(promise1).toBeInstanceOf(Promise); // The decoder is setting up + expect(cursor.isIdle()).toBe(false); + await promise1; + + expect(cursor.waitUntilIdle()).toBe(null); + expect(cursor.isIdle()).toBe(true); + + const promise2 = cursor.seekToFirst(); + const promise3 = cursor.waitUntilIdle(); + expect(cursor.isIdle()).toBe(false); + expect(promise2).toBeInstanceOf(Promise); + expect(promise3).toBeInstanceOf(Promise); + + expect(cursor.current).toBe(null); + await promise3; + expect(cursor.current!.timestamp).toBe(0); + expect(cursor.isIdle()).toBe(true); + + expect(cursor.waitUntilIdle()).toBe(null); + + const promise4 = cursor.reset(); + const promise5 = cursor.waitUntilIdle(); + expect(promise4).toBeInstanceOf(Promise); + expect(promise5).toBeInstanceOf(Promise); + + await promise5; + + expect(cursor.current).toBe(null); + + void cursor.seekToFirst(); + const promise6 = cursor.close(); + const promise7 = cursor.waitUntilIdle(); + expect(promise6).toBeInstanceOf(Promise); + expect(promise7).toBeInstanceOf(Promise); + + expect(cursor.closed).toBe(false); + await promise7; + expect(cursor.closed).toBe(true); + expect(cursor.isIdle()).toBe(true); +}); + +test('Command queuing', async () => { + using input = new Input({ + // Fetch the data into RAM to avoid packet lookups causing flaky timing + source: new BufferSource(await fetch('/trim-buck-bunny.mov').then(x => x.arrayBuffer())), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + + const cursor0 = new VideoSampleCursor(videoTrack); + cursor0._debug.enabled = true; + + const commands0 = [ + cursor0.seekToFirst(), + cursor0.seekToFirst(), + cursor0.close(), + ]; + expect(commands0.every(x => x instanceof Promise)).toBe(true); + + await promiseAllEnsureOrder(commands0); + + expect(cursor0._debug.pumpsStarted).toBe(1); + + const cursor1 = new VideoSampleCursor(videoTrack); + cursor1._debug.enabled = true; + + const commands1 = [ + cursor1.seekToFirst(), + cursor1.close(), + ]; + expect(commands1.every(x => x instanceof Promise)).toBe(true); + + const results1 = await promiseAllEnsureOrder(commands1); + expect(results1[0]!.timestamp).toBe(0); + expect(cursor1._debug.decodedPackets.map(x => x.timestamp)).toEqual([0]); + + const cursor2 = new VideoSampleCursor(videoTrack); + cursor2._debug.enabled = true; + + const commands2 = [ + cursor2.seekTo(0), + cursor2.seekTo(1), + cursor2.seekTo(2), + cursor2.waitUntilIdle()!.then(() => cursor2.current), + cursor2.seekTo(3), + cursor2.seekTo(4), + cursor2.seekTo(5), + cursor2.close(), + ]; + expect(commands2.every(x => x instanceof Promise)).toBe(true); + + const results2 = await promiseAllEnsureOrder(commands2); + expect(results2[0]!.timestamp).toBe(0); + expect(results2[1]!.timestamp).toBe(1); + expect(results2[2]!.timestamp).toBe(2); + expect(results2[3]!.timestamp).toBe(2); + expect(results2[4]!.timestamp).toBe(3); + expect(results2[5]!.timestamp).toBe(4); + expect(results2[6]!.timestamp).toBe(5); + expect(cursor2._debug.decodedPackets.map(x => x.timestamp)).toEqual([ + 0, 1, 2, 3, 4, 5, + ]); + expect(cursor2._debug.pumpsStarted).toBe(6); + + const cursor3 = new VideoSampleCursor(videoTrack); + cursor3._debug.enabled = true; + + const commands3 = [ + cursor3.seekTo(0.5), + cursor3.seekTo(0.4), + cursor3.waitUntilIdle()!.then(() => cursor3.current), + cursor3.seekTo(0.3), + cursor3.seekTo(0.2), + cursor3.seekTo(0.1), + cursor3.seekTo(0), + cursor3.close(), + ]; + expect(commands3.every(x => x instanceof Promise)).toBe(true); + + const results3 = await promiseAllEnsureOrder(commands3); + + expect(results3[0]!.timestamp).toBeLessThanOrEqual(0.5); + expect(results3[1]!.timestamp).toBeLessThanOrEqual(0.4); + expect(results3[2]!.timestamp).toBeLessThanOrEqual(0.4); + expect(results3[3]!.timestamp).toBeLessThanOrEqual(0.3); + expect(results3[4]!.timestamp).toBeLessThanOrEqual(0.2); + expect(results3[5]!.timestamp).toBeLessThanOrEqual(0.1); + expect(results3[6]!.timestamp).toBe(0); + expect(cursor3._debug.decodedPackets.every(x => x.timestamp <= 0.5)).toBe(true); + expect(cursor3._debug.pumpsStarted).toBe(6); + + const cursor4 = new VideoSampleCursor(videoTrack); + cursor4._debug.enabled = true; + + const commands4 = [ + cursor4.seekToFirst(), + cursor4.next(), + cursor4.next(), + cursor4.nextKey(), + cursor4.close(), + ]; + + expect(commands4.every(x => x instanceof Promise)).toBe(true); + + const results4 = await promiseAllEnsureOrder(commands4); + + expect(results4[0]!.timestamp).toBe(0); + expect(results4[1]!.timestamp).toBeGreaterThan(results4[0]!.timestamp); + expect(results4[2]!.timestamp).toBeGreaterThan(results4[1]!.timestamp); + expect(results4[3]!.timestamp).toBe(1); + expect(cursor4._debug.decodedPackets.length).toBeGreaterThan(3); // Because .next() goes into "sequential mode" + + const cursor5 = new VideoSampleCursor(videoTrack); + cursor5._debug.enabled = true; + + const commands5 = [ + cursor5.seekTo(0), + cursor5.next(), + cursor5.next(), + cursor5.seekTo(0.4), + cursor5.next(), + cursor5.next(), + cursor5.seekTo(0.8), + cursor5.next(), + cursor5.next(), + cursor5.seekTo(0), + cursor5.next(), + cursor5.close(), + ]; + + const results5 = await promiseAllEnsureOrder(commands5); + + expect(results5[0]!.timestamp).toBe(0); + expect(results5[1]!.timestamp).toBeGreaterThan(results5[0]!.timestamp); + expect(results5[2]!.timestamp).toBeGreaterThan(results5[1]!.timestamp); + expect(results5[3]!.timestamp).toBeLessThanOrEqual(0.4); + expect(results5[3]!.timestamp).toBeGreaterThan(results5[2]!.timestamp); + expect(results5[4]!.timestamp).toBeGreaterThan(results5[3]!.timestamp); + expect(results5[5]!.timestamp).toBeGreaterThan(results5[4]!.timestamp); + expect(results5[6]!.timestamp).toBeLessThanOrEqual(0.8); + expect(results5[6]!.timestamp).toBeGreaterThan(results5[5]!.timestamp); + expect(results5[7]!.timestamp).toBeGreaterThan(results5[6]!.timestamp); + expect(results5[8]!.timestamp).toBeGreaterThan(results5[7]!.timestamp); + expect(results5[9]!.timestamp).toBe(0); + expect(results5[10]!.timestamp).toBeGreaterThan(results5[9]!.timestamp); + + expect(cursor5._debug.pumpsStarted).toBe(2); + + const cursor6 = new VideoSampleCursor(videoTrack); + cursor6._debug.enabled = true; + + const commands6 = [ + cursor6.seekTo(0), + cursor6.seekTo(0.4), + cursor6.seekTo(0.8), + cursor6.seekTo(3.4), + cursor6.waitUntilIdle()!.then(() => cursor6.current), + cursor6.seekTo(3.8), + cursor6.seekTo(5), + cursor6.close(), + ]; + + const results6 = await promiseAllEnsureOrder(commands6); + + expect(results6[0]!.timestamp).toBe(0); + expect(results6[1]!.timestamp).toBeLessThanOrEqual(0.4); + expect(results6[2]!.timestamp).toBeLessThanOrEqual(0.8); + expect(results6[3]!.timestamp).toBeLessThanOrEqual(3.4); + expect(results6[4]!.timestamp).toBeLessThanOrEqual(3.4); + expect(results6[5]!.timestamp).toBeLessThanOrEqual(3.8); + expect(results6[6]!.timestamp).toBe(5); + + expect(cursor6._debug.pumpsStarted).toBe(3); + + const cursor7 = new VideoSampleCursor(videoTrack); + const commands7 = [ + cursor7.close(), + cursor7.close(), + ]; + expect(commands7[0]).toBe(commands7[1]); // Same Promise + + await promiseAllEnsureOrder(commands7); + + const cursor8 = new VideoSampleCursor(videoTrack, { closeSamples: false }); + + const firstSample = await cursor8.seekToFirst(); + firstSample!.close(); + await new Promise(resolve => setTimeout(resolve, 200)); + + const commands8 = [ + cursor8.next(), + cursor8.next(), + cursor8.next(), + ]; + expect(commands8.every(x => !(x instanceof Promise))).toBe(true); + + for await (using sample of promiseIterateAll(commands8)) { + expect(sample!.closed).toBe(false); + } + + await cursor8.close(); + + const cursor9 = new VideoSampleCursor(videoTrack, { closeSamples: false }); + + const commands9 = [ + cursor9.seekToFirst(), + cursor9.seekToFirst(), + cursor9.seekToFirst(), + ]; + expect(commands9.every(x => x instanceof Promise)).toBe(true); + + for await (using sample of promiseIterateAll(commands9)) { + expect(sample!.timestamp).toBe(0); + expect(sample!.closed).toBe(false); + } + + await cursor9.close(); + + const cursor10 = new VideoSampleCursor(videoTrack); + cursor10._debug.enabled = true; + + const commands10 = [ + cursor10.seekToFirst(), + cursor10.next(), + cursor10.seekToFirst(), + cursor10.next(), + cursor10.close(), + ]; + + const results10 = await promiseAllEnsureOrder(commands10); + + expect(results10[0]!.timestamp).toBe(0); + expect(results10[1]!.timestamp).toBeGreaterThan(0); + expect(results10[2]!.timestamp).toBe(0); + expect(results10[3]!.timestamp).toBe(results10[1]!.timestamp); + expect(cursor10._debug.pumpsStarted).toBe(2); +}); + +test('Automatic cursor disposal', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + + const cursor = new VideoSampleCursor(videoTrack); + await cursor.seekToFirst(); + + // No cursor.close() here, but the disposed Input closes the cursor + input.dispose(); + + expect(cursor.closed).toBe(true); + + // Make sure the close is actually complete + using lock = cursor._mutex.lock(); + if (lock.pending) await lock.ready; +}); + +test('Video with stubborn first sample emit', async () => { + using input = new Input({ + // For some reason, this video is stubborn in the sense that it takes quite a lot of packets for the decoder to + // emit its first samples. This used to cause issues in the past where the decoder got stuck, so good to have it + // tested. + source: new UrlSource('/sylvie-trimmed.mp4'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + await using cursor = new VideoSampleCursor(videoTrack); + + const firstSample = (await cursor.seekToFirst())!; + expect(firstSample).not.toBe(null); + expect(firstSample.timestamp).toBe(0); +}); + +test('AudioSampleCursor', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const audioTrack = (await input.getPrimaryAudioTrack())!; + const cursor = new AudioSampleCursor(audioTrack); + cursor._debug.enabled = true; + + const firstSample = (await cursor.seekToFirst())!; + expect(firstSample).not.toBe(null); + expect(firstSample.timestamp).toBe(0); + expect(firstSample).toBeInstanceOf(AudioSample); + + const secondSample = (await cursor.next())!; + expect(secondSample.timestamp).toBeCloseTo(firstSample.timestamp + firstSample.duration); + + const thirdSample = (await cursor.seekTo(secondSample.timestamp + 0.05))!; + expect(thirdSample.timestamp).toBeGreaterThan(secondSample.timestamp); + + expect(cursor._debug.pumpsStarted).toBe(1); + + await cursor.seekToFirst(); + + let lastTimestamp = -Infinity; + let total = 0; + for await (const sample of cursor) { + if (total === 0) { + expect(sample.timestamp).toBe(0); + } + + expect(sample.timestamp).toBeGreaterThan(lastTimestamp); + lastTimestamp = sample.timestamp; + total++; + } + + expect(total).toBe(235); + expect(lastTimestamp).toBeCloseTo(5, 1); + + expect(cursor._debug.pumpsStarted).toBe(2); + + const middleSample = (await cursor.seekTo(2.5))!; + expect(middleSample.timestamp).toBeLessThanOrEqual(2.5); + expect(middleSample.timestamp).toBeGreaterThan(2.4); + + expect(cursor._debug.pumpsStarted).toBe(3); + + const commands = [ + cursor.seekTo(0), + cursor.seekTo(0.05), + cursor.seekTo(0.1), + cursor.seekTo(0.15), + cursor.seekTo(0.2), + ]; + + const result = await promiseAllEnsureOrder(commands); + + expect(result[0]!.timestamp).toBeLessThanOrEqual(0); + expect(result[1]!.timestamp).toBeLessThanOrEqual(0.05); + expect(result[2]!.timestamp).toBeLessThanOrEqual(0.1); + expect(result[3]!.timestamp).toBeLessThanOrEqual(0.15); + expect(result[4]!.timestamp).toBeLessThanOrEqual(0.2); + + // One pump was used for all the above commands, even tho they all seek to different key packets + expect(cursor._debug.pumpsStarted).toBe(4); + + await cursor.seekToFirst(); + const nextSample = (await cursor.nextKey())!; // nextKey acts like next for audio tracks + const nextNextSample = (await cursor.nextKey())!; + expect(nextSample.timestamp).toBeCloseTo(firstSample.timestamp + firstSample.duration); + expect(nextNextSample.timestamp).toBeCloseTo(nextSample.timestamp + firstSample.duration); + + await cursor.seekToFirst(); + const actualNextSample = (await cursor.next())!; + expect(nextSample.timestamp).toBe(actualNextSample.timestamp); + + expect(cursor._debug.pumpsStarted).toBe(6); + + await cursor.close(); +}); + +test('AudioSampleCursor with PCM', async () => { + using input = new Input({ + source: new UrlSource('/glitch-hop-is-dead.wav'), + formats: ALL_FORMATS, + }); + + const audioTrack = (await input.getPrimaryAudioTrack())!; + expect(await audioTrack.getCodec()).toBe('pcm-s16'); + await using cursor = new AudioSampleCursor(audioTrack); + + let frames = 0; + for await (const sample of cursor) { + expect(sample.format).toBe('s16'); + frames += sample.numberOfFrames; + } + + expect(frames).toBe(425000); +}); + +test('Sample mapping', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + + let callCount = 0; + await using cursor = new VideoSampleCursor(videoTrack, { + transform: (sample) => { + callCount++; + + return { + original: sample, + timestampMs: sample.timestamp * 1000, + }; + }, + }); + + const firstSample = (await cursor.seekToFirst())!; + expect(firstSample.original).toBeInstanceOf(VideoSample); + expect(firstSample.timestampMs).toBe(0); + expect(firstSample.original.closed).toBe(false); + + const secondSample = (await cursor.next())!; + expect(secondSample.original).toBeInstanceOf(VideoSample); + expect(secondSample.timestampMs).toBeCloseTo(firstSample.timestampMs + firstSample.original.duration * 1000); + expect(firstSample.original.closed).toBe(true); + expect(secondSample.original.closed).toBe(false); + + const thirdSample = (await cursor.seekTo(0.5))!; + expect(thirdSample).not.toBe(null); + + expect(callCount).toBe(3); +}); + +test('Canvas transformer', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + + const cursor1 = new VideoSampleCursor(videoTrack, { + transform: canvasTransformer(), + }); + + const firstSample = (await cursor1.seekToFirst())!; + expect(firstSample).toBeInstanceOf(WrappedCanvas); + expect(firstSample.canvas).toBeInstanceOf(HTMLCanvasElement); + expect(firstSample.canvas.width).toBe(await videoTrack.getDisplayWidth()); + expect(firstSample.canvas.height).toBe(await videoTrack.getDisplayHeight()); + expect(firstSample.timestamp).toBe(0); + expect(firstSample.duration).toBeGreaterThan(0); + + const nextSample = (await cursor1.next())!; + expect(nextSample.canvas).toBeInstanceOf(HTMLCanvasElement); + expect(nextSample.timestamp).toBeGreaterThan(firstSample.timestamp); + expect(nextSample.canvas).not.toBe(firstSample.canvas); + + await cursor1.close(); + + const cursor2 = new VideoSampleCursor(videoTrack, { + transform: canvasTransformer({ + width: 320, + poolSize: 2, + }), + }); + + const sample1 = (await cursor2.seekToFirst())!; + expect(sample1.canvas.width).toBe(320); + expect(sample1.canvas.height).toBe(180); + + const sample2 = (await cursor2.next())!; + expect(sample2.canvas.width).toBe(320); + expect(sample2.canvas.height).toBe(180); + expect(sample2.canvas).not.toBe(sample1.canvas); + + const sample3 = (await cursor2.next())!; + const sample4 = (await cursor2.next())!; + + expect(sample3.canvas).toBe(sample1.canvas); + expect(sample2.canvas).toBe(sample4.canvas); + + await cursor2.close(); +}); + +test('Unthrottled decoder', async () => { + // While the decoder is throttled in practice, these tests simulate what happens when the decoder naturally reaches + // the end of the encodable packets without having been stopped; in this case, we still expect the cursor to keep + // functioning normally, just that requests are now fully resolved through the sample queue only (when possible). + + using input = new Input({ + source: new BufferSource(await fetch('/trim-buck-bunny.mov').then(x => x.arrayBuffer())), + formats: ALL_FORMATS, + }); + + const audioTrack = (await input.getPrimaryAudioTrack())!; + + const cursor1 = new AudioSampleCursor(audioTrack); + cursor1._debug.enabled = true; + cursor1._debug.unthrottledPump = true; + + const first = await cursor1.seekToFirst(); + expect(first!.timestamp).toBe(0); + + await cursor1._debug.pumpEnded.wait(); + expect(cursor1._pumpRunning).toBe(false); + + // All of these requests can be fully satisfied only by using the sample queue: + + const next = await cursor1.next(); + expect(next).not.toBe(null); + expect(next!.timestamp).toBeCloseTo(first!.timestamp + first!.duration); + + const nextNext = await cursor1.next(); + expect(nextNext).not.toBe(null); + expect(nextNext!.timestamp).toBeCloseTo(next!.timestamp + next!.duration); + + const seeked = await cursor1.seekTo(1); + expect(seeked).not.toBe(null); + expect(seeked!.timestamp).toBeGreaterThan(0.9); + expect(seeked!.timestamp).toBeLessThanOrEqual(1); + + const nextKey = await cursor1.nextKey(); + expect(nextKey).not.toBe(null); + expect(nextKey!.timestamp).toBeCloseTo(seeked!.timestamp + seeked!.duration); + + const theLast = await cursor1.seekTo(Infinity); + expect(theLast).not.toBe(null); + + const pastTheLast = await cursor1.next(); + expect(pastTheLast).toBe(null); + + expect(cursor1._debug.pumpsStarted).toBe(1); + + const someRandomSample = await cursor1.seekTo(0.5); + expect(someRandomSample).not.toBe(null); + expect(someRandomSample!.timestamp).toBeGreaterThan(0.4); + expect(someRandomSample!.timestamp).toBeLessThanOrEqual(0.5); + + expect(cursor1._debug.pumpsStarted).toBe(2); + + await cursor1.close(); + + const cursor2 = new AudioSampleCursor(audioTrack); + cursor2._debug.enabled = true; + cursor2._debug.unthrottledPump = true; + + await cursor2.seekToFirst(); + + await cursor2._debug.pumpEnded.wait(); + expect(cursor2._pumpRunning).toBe(false); + + // Test that all samples in the queue get closed now + await cursor2.close(); +}); + +test('Seeking to a consumed sample after the pump ended', async () => { + const output = new Output({ + format: new WavOutputFormat(), + target: new BufferTarget(), + }); + const audioSource = new EncodedAudioPacketSource('pcm-s16'); + output.addAudioTrack(audioSource); + await output.start(); + + const data = new Uint8Array(1600); + new Int16Array(data.buffer).fill(1234); + const decoderConfig = { + codec: 'pcm-s16', + numberOfChannels: 1, + sampleRate: 8000, + }; + await audioSource.add(new EncodedPacket(data, 'key', 0, 0.1), { decoderConfig }); + await audioSource.add(new EncodedPacket(data, 'key', 0.1, 0.1), { decoderConfig }); + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const audioTrack = (await input.getPrimaryAudioTrack())!; + await using cursor = new AudioSampleCursor(audioTrack); + cursor._debug.enabled = true; + const pumpEnded = cursor._debug.pumpEnded.wait(); + + const first = (await cursor.next())!; + expect(first.timestamp).toBe(0); + const decoded = new Int16Array(first.numberOfFrames); + first.copyTo(decoded, { planeIndex: 0, format: 's16' }); + expect(decoded[0]).toBe(1234); + + // Closing it rules out reusing the current sample, so it has to be decoded again + first.close(); + + await pumpEnded; + expect(cursor._pumpRunning).toBe(false); + + const again = (await cursor.seekTo(0))!; + expect(again.timestamp).toBe(0); + expect(cursor._debug.pumpsStarted).toBe(2); + + let frames = again.numberOfFrames; + for (let sample = await cursor.next(); sample; sample = await cursor.next()) { + frames += sample.numberOfFrames; + } + + expect(frames).toBe(1600); +}); + +test('hasNext', async () => { + using input = new Input({ + source: new UrlSource('/trim-buck-bunny.mov'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + await using cursor = new VideoSampleCursor(videoTrack); + + expect(cursor.current).toBe(null); + expect(await cursor.hasNext()).toBe(true); + + await cursor.seekToFirst(); + expect(await cursor.hasNext()).toBe(true); + + await cursor.seekTo(Infinity); + expect(cursor.current).not.toBe(null); + expect(await cursor.hasNext()).toBe(false); + + await cursor.next(); + expect(cursor.current).toBe(null); + expect(await cursor.hasNext()).toBe(false); + + void cursor.seekTo(2.5); + expect(await cursor.hasNext()).toBe(true); + + void cursor.seekTo(Infinity); + expect(await cursor.hasNext()).toBe(false); + + expect(cursor.hasNext()).not.toBeInstanceOf(Promise); +}); diff --git a/test/browser/transparency.test.ts b/test/browser/transparency.test.ts index 7d1cf5f13..b1a4c243d 100644 --- a/test/browser/transparency.test.ts +++ b/test/browser/transparency.test.ts @@ -2,7 +2,6 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { BufferSource, UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; -import { CanvasSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; import { Output } from '../../src/output.js'; import { WebMOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; @@ -10,6 +9,8 @@ import { CanvasSource, VideoSampleSource } from '../../src/media-source.js'; import { canEncodeVideo, Quality } from '../../src/encode.js'; import { VideoSample } from '../../src/sample.js'; import { Conversion } from '../../src/conversion.js'; +import { canvasTransformer, PacketCursor, VideoSampleCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; const decodeTransparentVideoTest = async () => { using input = new Input({ @@ -20,8 +21,8 @@ const decodeTransparentVideoTest = async () => { const videoTrack = (await input.getPrimaryVideoTrack())!; expect(await videoTrack.canBeTransparent()).toBe(true); - const sink = new VideoSampleSink(videoTrack); - using sample = (await sink.getSample(0.5))!; + await using cursor = new VideoSampleCursor(videoTrack); + using sample = (await cursor.seekTo(0.5))!; expect(sample.format).toContain('A'); // Probably RGBA expect(sample.hasAlpha).toBe(true); @@ -46,15 +47,15 @@ test('Can decode faulty transparent video and behaves gracefully', async () => { }); const videoTrack = (await input.getPrimaryVideoTrack())!; - const packetSink = new EncodedPacketSink(videoTrack); - const secondKeyPacket = (await packetSink.getNextKeyPacket((await packetSink.getFirstPacket())!))!; + const reader = new PacketReader(videoTrack); + const secondKeyPacket = (await reader.getNextKey((await reader.getFirst())!))!; - const sink = new VideoSampleSink(videoTrack); + await using cursor = new VideoSampleCursor(videoTrack); - using startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using startSample = (await cursor.seekToFirst())!; expect(startSample.format).toContain('A'); - using secondSample = (await sink.getSample(secondKeyPacket.timestamp))!; + using secondSample = (await cursor.seekTo(secondKeyPacket.timestamp))!; expect(secondSample.format).not.toContain('A'); // There was no alpha key frame for this one expect(secondSample.hasAlpha).toBe(false); }); @@ -66,8 +67,11 @@ test('Can extract transparent frames via CanvasSink', async () => { }); const videoTrack = (await input.getPrimaryVideoTrack())!; - const sink = new CanvasSink(videoTrack, { alpha: true }); - const wrappedCanvas = (await sink.getCanvas(await videoTrack.getFirstTimestamp()))!; + const cursor = new VideoSampleCursor(videoTrack, { + transform: canvasTransformer({ alpha: true }), + }); + const wrappedCanvas = (await cursor.seekToFirst())!; + await cursor.close(); const canvas = new OffscreenCanvas(wrappedCanvas.canvas.width, wrappedCanvas.canvas.height); const context = canvas.getContext('2d')!; @@ -76,8 +80,11 @@ test('Can extract transparent frames via CanvasSink', async () => { let imageData = context.getImageData(0, 0, canvas.width, canvas.height); expect(imageData.data[3]).toBeLessThan(255); // Check that there's actually transparent pixels - const opaqueSink = new CanvasSink(videoTrack); // Default is alpha: false - const opaqueWrappedCanvas = (await opaqueSink.getCanvas(await videoTrack.getFirstTimestamp()))!; + const opaqueCursor = new VideoSampleCursor(videoTrack, { + transform: canvasTransformer(), // Default is alpha: false + }); + const opaqueWrappedCanvas = (await opaqueCursor.seekToFirst())!; + await opaqueCursor.close(); context.drawImage(opaqueWrappedCanvas.canvas, 0, 0); @@ -166,9 +173,9 @@ const encodeTransparentVideoTest = async () => { const videoTrack = (await input.getPrimaryVideoTrack())!; expect(await videoTrack.canBeTransparent()).toBe(true); - const sink = new VideoSampleSink(videoTrack); + await using cursor = new VideoSampleCursor(videoTrack); - using firstSample = (await sink.getSample(0))!; + using firstSample = (await cursor.seekToFirst())!; expect(firstSample.format).toContain('A'); probeContext.clearRect(0, 0, probeCanvas.width, probeCanvas.height); @@ -229,10 +236,10 @@ test('Can encode video with alternating transparency', async () => { }); const videoTrack = (await input.getPrimaryVideoTrack())!; - const packetSink = new EncodedPacketSink(videoTrack); + const packetCursor = new PacketCursor(videoTrack); let i = 0; - for await (const packet of packetSink.packets()) { + for await (const packet of packetCursor) { if (i % 2) { expect(packet.sideData.alpha).toBeUndefined(); } else { @@ -242,10 +249,10 @@ test('Can encode video with alternating transparency', async () => { i++; } - const sampleSink = new VideoSampleSink(videoTrack); + await using sampleCursor = new VideoSampleCursor(videoTrack); i = 0; - for await (using sample of sampleSink.samples()) { + for await (const sample of sampleCursor) { if (i % 2) { expect(sample.format).not.toContain('A'); } else { @@ -309,8 +316,8 @@ test('Can transmux transparent video, discards alpha by default', async () => { const videoTrack = (await outputInput.getPrimaryVideoTrack())!; expect(await videoTrack.canBeTransparent()).toBe(false); - const sink = new VideoSampleSink(videoTrack); - using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + await using cursor = new VideoSampleCursor(videoTrack); + using sample = (await cursor.seekToFirst())!; expect(sample.hasAlpha).toBe(false); }); @@ -341,8 +348,8 @@ test('Can transmux transparent video, can keep alpha', async () => { const videoTrack = (await outputInput.getPrimaryVideoTrack())!; expect(await videoTrack.canBeTransparent()).toBe(true); - const sink = new VideoSampleSink(videoTrack); - using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + await using cursor = new VideoSampleCursor(videoTrack); + using sample = (await cursor.seekToFirst())!; expect(sample.format).toContain('A'); expect(sample.hasAlpha).toBe(true); }); @@ -380,8 +387,8 @@ test('Can reencode transparent video, keeping alpha', async () => { expect(await videoTrack.canBeTransparent()).toBe(true); expect(await videoTrack.getDisplayWidth()).toBe(320); - const sink = new VideoSampleSink(videoTrack); - using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + const cursor = new VideoSampleCursor(videoTrack); + using sample = (await cursor.seekToFirst())!; expect(sample.format).toContain('A'); expect(sample.hasAlpha).toBe(true); }); diff --git a/test/browser/video-sample.test.ts b/test/browser/video-sample.test.ts new file mode 100644 index 000000000..536f6c45a --- /dev/null +++ b/test/browser/video-sample.test.ts @@ -0,0 +1,189 @@ +import { expect, test } from 'vitest'; +import { VideoSample } from '../../src/sample.js'; + +const createCanvas = () => { + const canvas = document.createElement('canvas'); + canvas.width = 300; + canvas.height = 150; + + const ctx = canvas.getContext('2d')!; + + ctx.fillStyle = '#0000ff'; + ctx.fillRect(0, 0, canvas.width, canvas.height); + + ctx.fillStyle = '#ff0000'; + ctx.fillRect(0, 0, 100, 50); + + return canvas; +}; + +type Color = { + r: number; + g: number; + b: number; +}; + +const colorDistance = (c1: Color, c2: Color) => { + return Math.hypot(c1.r - c2.r, c1.g - c2.g, c1.b - c2.b); +}; + +const sampleCanvasColor = (ctx: CanvasRenderingContext2D, x: number, y: number): Color => { + const pixel = ctx.getImageData(x, y, 1, 1).data; + + return { + r: pixel[0]!, + g: pixel[1]!, + b: pixel[2]!, + }; +}; + +test('Can create VideoSample from VideoFrame and modify rotation', () => { + const frame = new VideoFrame(createCanvas(), { timestamp: 0 }); + using sample = new VideoSample(frame); + + expect(frame.rotation).toBe(0); + expect(sample.rotation).toBe(0); + + const canvas = document.createElement('canvas'); + canvas.width = 300; + canvas.height = 150; + const ctx = canvas.getContext('2d')!; + + sample.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + using rotated = sample.clone({ rotation: 90 }); + expect(rotated.rotation).toBe(90); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + rotated.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 95, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 145, 110), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + const extracted = rotated.toVideoFrame(); + expect(extracted.rotation).toBe(90); // It was changed + + ctx.clearRect(0, 0, canvas.width, canvas.height); + rotated.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 295, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 295, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); +}); + +test('Can create VideoSample from rotated VideoFrame', () => { + const frame = new VideoFrame(createCanvas(), { timestamp: 0, rotation: 90 }); + using sample = new VideoSample(frame); + + expect(frame.rotation).toBe(90); + expect(sample.rotation).toBe(90); + + const canvas = document.createElement('canvas'); + canvas.width = 300; + canvas.height = 150; + const ctx = canvas.getContext('2d')!; + + sample.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 95, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 145, 110), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + sample.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 295, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 295, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + using unrotated = sample.clone({ rotation: 0 }); + expect(unrotated.rotation).toBe(0); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + unrotated.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + unrotated.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); +}); + +test('Can create VideoSample from rotated and flipped VideoFrame', () => { + const frame = new VideoFrame(createCanvas(), { timestamp: 0, rotation: 90, flip: true }); + using sample = new VideoSample(frame); + + expect(frame.rotation).toBe(90); + expect(frame.flip).toBe(true); + expect(sample.rotation).toBe(90); + expect(sample.flip).toBe(true); + + const canvas = document.createElement('canvas'); + canvas.width = 150; + canvas.height = 300; + const ctx = canvas.getContext('2d')!; + + // Rotating moves the red corner to the top right, flipping then moves it back to the top left + sample.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 150), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + sample.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 150), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + using unflipped = sample.clone({ flip: false }); + expect(unflipped.rotation).toBe(90); + expect(unflipped.flip).toBe(false); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + unflipped.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + unflipped.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 145, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + const extracted = unflipped.toVideoFrame(); + expect(extracted.rotation).toBe(90); + expect(extracted.flip).toBe(false); // It was changed + extracted.close(); + + using upright = sample.clone({ rotation: 0, flip: false }); + + canvas.width = 300; + canvas.height = 150; + upright.draw(ctx, 0, 0); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 5, 55), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + ctx.clearRect(0, 0, canvas.width, canvas.height); + upright.drawWithFit(ctx, { fit: 'fill' }); + + expect(colorDistance(sampleCanvasColor(ctx, 5, 5), { r: 255, g: 0, b: 0 })).toBeLessThan(10); + expect(colorDistance(sampleCanvasColor(ctx, 110, 5), { r: 0, g: 0, b: 255 })).toBeLessThan(10); + + const uprightExtracted = upright.toVideoFrame(); + expect(uprightExtracted.rotation).toBe(0); + expect(uprightExtracted.flip).toBe(false); + uprightExtracted.close(); +}); diff --git a/test/browser/video-samples.test.ts b/test/browser/video-samples.test.ts index 1fbf9819c..7317a7844 100644 --- a/test/browser/video-samples.test.ts +++ b/test/browser/video-samples.test.ts @@ -543,13 +543,13 @@ test('encodeOptions', () => { using a = new VideoSample(canvas, { timestamp: 0 }); expect(a.encodeOptions).toEqual({}); - // It can be overridden via the setter - a.setEncodeOptions({ keyFrame: true }); - expect(a.encodeOptions).toEqual({ keyFrame: true }); + // It can be overridden when cloning + using aClone = a.clone({ encodeOptions: { keyFrame: true } }); + expect(aClone.encodeOptions).toEqual({ keyFrame: true }); // Cloning carries the encode options over - using aClone = a.clone(); - expect(aClone.encodeOptions).toEqual({ keyFrame: true }); + using aCloneClone = aClone.clone(); + expect(aCloneClone.encodeOptions).toEqual({ keyFrame: true }); // It can be set via the constructor using b = new VideoSample(canvas, { timestamp: 0, encodeOptions: { keyFrame: true } }); diff --git a/test/node/aac-encoder-extension.test.ts b/test/node/aac-encoder-extension.test.ts index f1b1efe0e..4d3fd92e7 100644 --- a/test/node/aac-encoder-extension.test.ts +++ b/test/node/aac-encoder-extension.test.ts @@ -6,7 +6,8 @@ import { Output } from '../../src/output.js'; import { BufferTarget } from '../../src/target.js'; import { canEncode, Quality } from '../../src/encode.js'; import { AudioSampleSource } from '../../src/media-source.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { Mp4OutputFormat } from '../../src/output-format.js'; import { AudioSample } from '../../src/sample.js'; import { registerAacEncoder } from '@mediabunny/aac-encoder'; @@ -73,9 +74,9 @@ test('AAC encoding', async () => { expect(await track.getSampleRate()).toBe(sampleRate); expect(await track.getNumberOfChannels()).toBe(channels); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { expect(packet.type).toBe('key'); packetCount++; } @@ -120,8 +121,8 @@ test('AAC with huge timestamps', async () => { }); const track = (await input.getPrimaryAudioTrack())!; - const sink = new EncodedPacketSink(track); - const firstPacket = await sink.getFirstPacket(); + const reader = new PacketReader(track); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBe(timestamp); diff --git a/test/node/ac3.test.ts b/test/node/ac3.test.ts index 9383a3929..199aead87 100644 --- a/test/node/ac3.test.ts +++ b/test/node/ac3.test.ts @@ -10,7 +10,8 @@ import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; import { AC3_REGISTRATION_DESCRIPTOR, EAC3_REGISTRATION_DESCRIPTOR } from '../../src/codec-data.js'; import { canEncode, Quality } from '../../src/encode.js'; -import { AudioSampleSink, EncodedPacketSink } from '../../src/media-sink.js'; +import { AudioSampleCursor, PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { AudioSampleSource } from '../../src/media-source.js'; import { Mp4OutputFormat } from '../../src/output-format.js'; import { AudioSample } from '../../src/sample.js'; @@ -172,12 +173,12 @@ test('AC-3 decoding', async () => { const { packetCount } = await track.computePacketStats(); const trackNumberOfChannels = await track.getNumberOfChannels(); const trackSampleRate = await track.getSampleRate(); - const sink = new AudioSampleSink(track); + await using cursor = new AudioSampleCursor(track); let sampleCount = 0; let nextTimestamp = 0; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { expect(sample.timestamp).toBeCloseTo(nextTimestamp); expect(sample.duration).toBe(0.032); expect(sample.format).toBe('f32-planar'); @@ -203,12 +204,12 @@ test('E-AC-3 decoding', async () => { const { packetCount } = await track.computePacketStats(); const trackNumberOfChannels = await track.getNumberOfChannels(); const trackSampleRate = await track.getSampleRate(); - const sink = new AudioSampleSink(track); + await using cursor = new AudioSampleCursor(track); let sampleCount = 0; let nextTimestamp = 0; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { expect(sample.timestamp).toBeCloseTo(nextTimestamp); expect(sample.duration).toBe(0.032); expect(sample.format).toBe('f32-planar'); @@ -275,9 +276,9 @@ test('AC-3 encoding', async () => { expect(await track.getSampleRate()).toBe(sampleRate); expect(await track.getNumberOfChannels()).toBe(channels); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { expect(packet.type).toBe('key'); packetCount++; } @@ -325,9 +326,9 @@ test('E-AC-3 encoding', async () => { expect(await track.getSampleRate()).toBe(sampleRate); expect(await track.getNumberOfChannels()).toBe(channels); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { expect(packet.type).toBe('key'); packetCount++; } @@ -373,15 +374,14 @@ test('AC-3 with huge timestamps', async () => { }); const track = (await input.getPrimaryAudioTrack())!; - const packetSink = new EncodedPacketSink(track); - const firstPacket = await packetSink.getFirstPacket(); + const packetReader = new PacketReader(track); + const firstPacket = await packetReader.getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBe(timestamp); - const sampleSink = new AudioSampleSink(track); - const iterator = sampleSink.samples(timestamp); - const firstSample = (await iterator.next()).value; + await using sampleCursor = new AudioSampleCursor(track); + const firstSample = await sampleCursor.seekTo(timestamp); assert(firstSample); expect(firstSample.timestamp).toBe(timestamp); @@ -424,15 +424,14 @@ test('E-AC-3 with huge timestamps', async () => { }); const track = (await input.getPrimaryAudioTrack())!; - const packetSink = new EncodedPacketSink(track); - const firstPacket = await packetSink.getFirstPacket(); + const packetReader = new PacketReader(track); + const firstPacket = await packetReader.getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBe(timestamp); - const sampleSink = new AudioSampleSink(track); - const iterator = sampleSink.samples(timestamp); - const firstSample = (await iterator.next()).value; + await using sampleCursor = new AudioSampleCursor(track); + const firstSample = await sampleCursor.seekTo(timestamp); assert(firstSample); expect(firstSample.timestamp).toBe(timestamp); diff --git a/test/node/adts-muxer.test.ts b/test/node/adts-muxer.test.ts index ddbcb9c98..f78c05679 100644 --- a/test/node/adts-muxer.test.ts +++ b/test/node/adts-muxer.test.ts @@ -4,7 +4,8 @@ import { fileURLToPath } from 'node:url'; import { Input } from '../../src/input.js'; import { BufferSource, FilePathSource } from '../../src/source.js'; import { ADTS, ALL_FORMATS } from '../../src/input-format.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { Output } from '../../src/output.js'; import { BufferTarget } from '../../src/target.js'; import { AdtsOutputFormat } from '../../src/output-format.js'; @@ -50,10 +51,10 @@ test('ADTS muxer with raw AAC input', async () => { const outputDecoderConfig = await outputTrack.getDecoderConfig(); expect(outputDecoderConfig!.description).toBeUndefined(); // ADTS has no description - const outputSink = new EncodedPacketSink(outputTrack); + const outputCursor = new PacketCursor(outputTrack); let count = 0; - for await (const packet of outputSink.packets()) { + for await (const packet of outputCursor) { // All packets should be ADTS frames now (start with 0xfff sync word) expect(packet.data[0]).toBe(0xff); expect((packet.data[1]! & 0xf0)).toBe(0xf0); @@ -93,11 +94,11 @@ test('ADTS muxer with ADTS input (passthrough)', { timeout: 15_000 }, async () = const outputTrack = await outputAsInput.getPrimaryAudioTrack(); assert(outputTrack); - const inputSink = new EncodedPacketSink(inputTrack); - const outputSink = new EncodedPacketSink(outputTrack); + const inputReader = new PacketReader(inputTrack); + const outputReader = new PacketReader(outputTrack); - let inputPacket = await inputSink.getFirstPacket(); - let outputPacket = await outputSink.getFirstPacket(); + let inputPacket = await inputReader.getFirst(); + let outputPacket = await outputReader.getFirst(); let count = 0; while (inputPacket && outputPacket) { @@ -106,8 +107,8 @@ test('ADTS muxer with ADTS input (passthrough)', { timeout: 15_000 }, async () = expect(outputPacket.timestamp).toBe(inputPacket.timestamp); expect(outputPacket.duration).toBe(inputPacket.duration); - inputPacket = await inputSink.getNextPacket(inputPacket); - outputPacket = await outputSink.getNextPacket(outputPacket); + inputPacket = await inputReader.getNext(inputPacket); + outputPacket = await outputReader.getNext(outputPacket); count++; } diff --git a/test/node/annex-b-conversion.test.ts b/test/node/annex-b-conversion.test.ts index 79c1e420d..8e96672df 100644 --- a/test/node/annex-b-conversion.test.ts +++ b/test/node/annex-b-conversion.test.ts @@ -8,8 +8,8 @@ import { Output } from '../../src/output.js'; import { Mp4OutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; import { iterateAvcNalUnits } from '../../src/codec-data.js'; +import { PacketReader } from '../../src/packet.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -23,8 +23,8 @@ test('Annex B to length-prefixed conversion, MP4', async () => { expect(originalDecoderConfig.description).toBeUndefined(); expect(await originalVideoTrack.getCodec()).toBe('avc'); - const originalSink = new EncodedPacketSink(originalVideoTrack); - const originalFirstPacket = await originalSink.getFirstPacket(); + const originalReader = new PacketReader(originalVideoTrack); + const originalFirstPacket = await originalReader.getFirst(); expect([...originalFirstPacket!.data.slice(0, 4)]).toEqual([0, 0, 0, 1]); const originalNalUnits = [...iterateAvcNalUnits(originalFirstPacket!.data, originalDecoderConfig)] @@ -47,8 +47,8 @@ test('Annex B to length-prefixed conversion, MP4', async () => { expect(newDecoderConfig.description).toBeDefined(); expect(await newVideoTrack.getCodec()).toBe('avc'); - const newSink = new EncodedPacketSink(newVideoTrack); - const newFirstPacket = await newSink.getFirstPacket(); + const newReader = new PacketReader(newVideoTrack); + const newFirstPacket = await newReader.getFirst(); expect([...newFirstPacket!.data.slice(0, 4)]).not.toEqual([0, 0, 0, 1]); // Successfully converted const newNalUnits = [...iterateAvcNalUnits(newFirstPacket!.data, newDecoderConfig)] diff --git a/test/node/demux.test.ts b/test/node/demux.test.ts new file mode 100644 index 000000000..6be44fca5 --- /dev/null +++ b/test/node/demux.test.ts @@ -0,0 +1,354 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { BufferSource, FilePathSource } from '../../src/source.js'; +import path from 'node:path'; +import fs from 'node:fs'; +import { ADTS, ALL_FORMATS, FLAC, MP3, MP4, OGG, QTFF, WAVE, WEBM } from '../../src/input-format.js'; +import { InputAudioTrack, InputTrack } from '../../src/input-track.js'; +import { assert } from '../../src/misc.js'; +import { EncodedPacket, PacketReader } from '../../src/packet.js'; + +const __dirname = new URL('.', import.meta.url).pathname; + +const testBasicPacketReading = async (track: InputTrack) => { + const reader = new PacketReader(track); + const first = await reader.getFirst(); + expect(first).not.toBe(null); + expect(first!.timestamp).toBe(0); + + const next = await reader.getNext(first!); + expect(next).not.toBe(null); + expect(next!.timestamp).toBeCloseTo(first!.timestamp + first!.duration); + expect(next!.sequenceNumber).toBeGreaterThan(first!.sequenceNumber); + + const nextKey = await reader.getNextKey(first!); + expect(nextKey).not.toBe(null); + expect(nextKey!.type).toBe('key'); + expect(nextKey!.timestamp).toBeGreaterThanOrEqual(next!.timestamp); + + const seeked = await reader.getAt(1); + expect(seeked).not.toBe(null); + expect(seeked!.timestamp).toBeGreaterThan(0.9); + expect(seeked!.timestamp).toBeLessThanOrEqual(1); + expect(seeked!.sequenceNumber).toBeGreaterThan(next!.sequenceNumber); + + const seekedKey = await reader.getKeyAt(1); + expect(seekedKey).not.toBe(null); + expect(seekedKey!.type).toBe('key'); + expect(seekedKey!.sequenceNumber).toBeGreaterThanOrEqual(first!.sequenceNumber); + expect(seekedKey!.sequenceNumber).toBeLessThanOrEqual(seeked!.sequenceNumber); + + const last = await reader.getAt(Infinity); + expect(last).not.toBe(null); + expect(last!.sequenceNumber).toBeGreaterThan(seeked!.sequenceNumber); + + const afterLast = await reader.getNext(last!); + expect(afterLast).toBe(null); +}; + +const testSyncPacketReading = (track: InputTrack) => { + const reader = new PacketReader(track); + + const seeked = reader.getAt(1) as EncodedPacket | null; + expect(seeked).toBeInstanceOf(EncodedPacket); + + let current = reader.getFirst() as EncodedPacket | null; + expect(current).toBeInstanceOf(EncodedPacket); + + let count = 0; + while (current) { + current = reader.getNext(current) as EncodedPacket | null; + expect(current instanceof EncodedPacket || current === null).toBe(true); + count++; + } + + expect(count).toBeGreaterThan(0); + + return count; +}; + +test('Regular ISOBMFF demuxing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/trim-buck-bunny.mov')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(QTFF); + expect(await input.getMimeType()).toBe('video/quicktime; codecs="avc1.4d4029, mp4a.40.2"'); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(2); + + const videoTrack = await input.getPrimaryAudioTrack(); + assert(videoTrack); + + await testBasicPacketReading(videoTrack); + + expect(await input.computeDuration()).toBeCloseTo(5.041666666666667); +}); + +test('Regular ISOBMFF sync reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/trim-buck-bunny.mov'))), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const count = testSyncPacketReading(videoTrack); + expect(count).toBe(121); +}); + +test('Fragmented ISOBMFF demuxing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/frag-buck-bunny.mp4')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MP4); + expect(await input.getMimeType()).toBe('video/mp4; codecs="avc1.640014, mp4a.40.2"'); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(2); + + const videoTrack = await input.getPrimaryAudioTrack(); + assert(videoTrack); + + await testBasicPacketReading(videoTrack); + + expect(await input.computeDuration()).toBeCloseTo(5); +}); + +test('Fragmented ISOBMFF sync reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/frag-buck-bunny.mp4'))), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const count = testSyncPacketReading(videoTrack); + expect(count).toBe(120); +}); + +test('Matroska demuxing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/trim-buck-bunny.webm')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(WEBM); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(2); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + await testBasicPacketReading(videoTrack); + + expect(await input.computeDuration()).toBeCloseTo(5); +}); + +test('Matroska sync reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/trim-buck-bunny.webm'))), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + const count = testSyncPacketReading(videoTrack); + expect(count).toBe(120); +}); + +test('MP3 demuxing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/AudacityTest1.mp3')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MP3); + expect(await input.getMimeType()).toBe('audio/mpeg'); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + const audioTrack = tracks[0] as InputAudioTrack; + expect(audioTrack).toBeInstanceOf(InputAudioTrack); + + expect(await audioTrack.getNumberOfChannels()).toBe(1); + expect(await audioTrack.getSampleRate()).toBe(44100); + + const decoderConfig = await audioTrack.getDecoderConfig(); + expect(decoderConfig!.codec).toBe('mp3'); + expect(decoderConfig!.numberOfChannels).toBe(await audioTrack.getNumberOfChannels()); + expect(decoderConfig!.sampleRate).toBe(await audioTrack.getSampleRate()); + expect(decoderConfig!.description).toBeUndefined(); + + await testBasicPacketReading(audioTrack); + + const firstTimestamp = await audioTrack.getFirstTimestamp(); + expect(firstTimestamp).toBe(0); + + const duration = await audioTrack.computeDuration(); + expect(duration).toBeGreaterThan(0); +}); + +test('MP3 sync reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/AudacityTest1.mp3'))), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const count = testSyncPacketReading(audioTrack); + expect(count).toBe(475); +}); + +test('WAVE demuxing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/glitch-hop-is-dead.wav')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(WAVE); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + await testBasicPacketReading(audioTrack); + + expect(await input.computeDuration()).toBeCloseTo(9.637188208616779); +}); + +test('WAVE sync reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/glitch-hop-is-dead.wav'))), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const count = testSyncPacketReading(audioTrack); + expect(count).toBe(208); +}); + +test('ADTS demuxing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/trim-buck-bunny.aac')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(ADTS); + expect(await input.getMimeType()).toBe('audio/aac'); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + const audioTrack = tracks[0] as InputAudioTrack; + expect(audioTrack).toBeInstanceOf(InputAudioTrack); + + expect(await audioTrack.getCodec()).toBe('aac'); + expect(await audioTrack.getNumberOfChannels()).toBeGreaterThan(0); + expect(await audioTrack.getSampleRate()).toBeGreaterThan(0); + + await testBasicPacketReading(audioTrack); + + const duration = await audioTrack.computeDuration(); + expect(duration).toBeCloseTo(4.992); +}); + +test('ADTS sync reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/trim-buck-bunny.aac'))), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const count = testSyncPacketReading(audioTrack); + expect(count).toBe(234); +}); + +test('Ogg demuxing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/beach-party.ogg')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(OGG); + expect(await input.getMimeType()).toBe('audio/ogg; codecs="vorbis"'); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + const audioTrack = tracks[0] as InputAudioTrack; + expect(audioTrack).toBeInstanceOf(InputAudioTrack); + + expect(await audioTrack.getCodec()).toBe('vorbis'); + expect(await audioTrack.getNumberOfChannels()).toBeGreaterThan(0); + expect(await audioTrack.getSampleRate()).toBeGreaterThan(0); + + await testBasicPacketReading(audioTrack); + + const duration = await audioTrack.computeDuration(); + expect(duration).toBeCloseTo(57.31360544217687); +}); + +test('Ogg sync reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/beach-party.ogg'))), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const count = testSyncPacketReading(audioTrack); + expect(count).toBe(5041); +}); + +test('FLAC demuxing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/sample.flac')), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(FLAC); + expect(await input.getMimeType()).toBe('audio/flac'); + + const tracks = await input.getTracks(); + expect(tracks).toHaveLength(1); + const audioTrack = tracks[0] as InputAudioTrack; + expect(audioTrack).toBeInstanceOf(InputAudioTrack); + + expect(await audioTrack.getCodec()).toBe('flac'); + expect(await audioTrack.getNumberOfChannels()).toBeGreaterThan(0); + expect(await audioTrack.getSampleRate()).toBeGreaterThan(0); + + await testBasicPacketReading(audioTrack); + + const duration = await audioTrack.computeDuration(); + expect(duration).toBeCloseTo(19.71428571428571); +}); + +test('FLAC sync reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/sample.flac'))), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const count = testSyncPacketReading(audioTrack); + expect(count).toBe(213); +}); diff --git a/test/node/dts.test.ts b/test/node/dts.test.ts index 76c256ffe..835c89b4b 100644 --- a/test/node/dts.test.ts +++ b/test/node/dts.test.ts @@ -8,10 +8,9 @@ import { Output } from '../../src/output.js'; import { MkvOutputFormat, Mp4OutputFormat, MpegTsOutputFormat, OutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; -import { EncodedPacket } from '../../src/packet.js'; +import { EncodedPacket, PacketReader } from '../../src/packet.js'; import { assert, uint8ArraysAreEqual } from '../../src/misc.js'; -import { AudioSampleSink } from '../../src/media-sink.js'; +import { AudioSampleCursor, PacketCursor } from '../../src/cursors.js'; import { AudioSampleSource } from '../../src/media-source.js'; import { AudioSample } from '../../src/sample.js'; import { canEncode, Quality } from '../../src/encode.js'; @@ -126,12 +125,12 @@ test('Decode with the extension', async () => { assert(track); const { packetCount } = await track.computePacketStats(); - const sink = new AudioSampleSink(track); + await using cursor = new AudioSampleCursor(track); let sampleCount = 0; let nextTimestamp = 0; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { expect(sample.timestamp).toBeCloseTo(nextTimestamp); expect(sample.duration).toBeCloseTo(512 / 24000); expect(sample.format).toBe('f32-planar'); @@ -167,10 +166,10 @@ test('Encode with the extension', async () => { expect(decoderConfig.codec).toBe('dtsc'); expect(decoderConfig.description).toBeUndefined(); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { expect(packet.type).toBe('key'); packetCount++; } @@ -193,11 +192,11 @@ test('Round-trip through the extension', async () => { const track = await input.getPrimaryAudioTrack(); assert(track); - const sink = new AudioSampleSink(track); + await using cursor = new AudioSampleCursor(track); const chunks: Float32Array[] = []; let decodedFrames = 0; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { expect(sample.numberOfChannels).toBe(ENCODE_CHANNELS); expect(sample.sampleRate).toBe(ENCODE_SAMPLE_RATE); decodedFrames += sample.numberOfFrames; @@ -236,14 +235,14 @@ test('Encode with huge timestamps', async () => { const track = await input.getPrimaryAudioTrack(); assert(track); - const packetSink = new EncodedPacketSink(track); - const firstPacket = await packetSink.getFirstPacket(); + const packetReader = new PacketReader(track); + const firstPacket = await packetReader.getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBe(timestamp); - const sampleSink = new AudioSampleSink(track); - const firstSample = (await sampleSink.samples(timestamp).next()).value; + await using sampleCursor = new AudioSampleCursor(track); + const firstSample = await sampleCursor.seekTo(timestamp); assert(firstSample); expect(firstSample.timestamp).toBe(timestamp); @@ -320,10 +319,10 @@ const readAllPackets = async (input: Input) => { const track = await input.getPrimaryAudioTrack(); assert(track); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); const packets: EncodedPacket[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { packets.push(packet); } diff --git a/test/node/flac-encoder-extension.test.ts b/test/node/flac-encoder-extension.test.ts index b39a1cef3..03de24453 100644 --- a/test/node/flac-encoder-extension.test.ts +++ b/test/node/flac-encoder-extension.test.ts @@ -6,7 +6,8 @@ import { Output } from '../../src/output.js'; import { BufferTarget } from '../../src/target.js'; import { canEncode } from '../../src/encode.js'; import { AudioSampleSource } from '../../src/media-source.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { FlacOutputFormat, Mp4OutputFormat } from '../../src/output-format.js'; import { AudioSample } from '../../src/sample.js'; import { registerFlacEncoder } from '@mediabunny/flac-encoder'; @@ -73,9 +74,9 @@ test('FLAC encoding', async () => { expect(await track.getSampleRate()).toBe(sampleRate); expect(await track.getNumberOfChannels()).toBe(channels); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { expect(packet.type).toBe('key'); packetCount++; } @@ -121,8 +122,8 @@ test('FLAC with huge timestamps', async () => { }); const track = (await input.getPrimaryAudioTrack())!; - const sink = new EncodedPacketSink(track); - const firstPacket = await sink.getFirstPacket(); + const reader = new PacketReader(track); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBe(timestamp); diff --git a/test/node/flac.test.ts b/test/node/flac.test.ts index adac88de8..b4b6b3cfb 100644 --- a/test/node/flac.test.ts +++ b/test/node/flac.test.ts @@ -6,11 +6,12 @@ import { assert, toUint8Array } from '../../src/misc.js'; import { Input } from '../../src/input.js'; import { BufferSource, FilePathSource } from '../../src/source.js'; import { ALL_FORMATS, FLAC } from '../../src/input-format.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; import { Output } from '../../src/output.js'; import { BufferTarget } from '../../src/target.js'; import { FlacOutputFormat } from '../../src/output-format.js'; import { Conversion } from '../../src/conversion.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { VERSIONED_NAME } from '../../src/version.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -39,10 +40,11 @@ test('can loop over all samples', async () => { expect(await track.getTimeResolution()).toEqual(44100); expect(await input.getMimeType()).toEqual('audio/flac'); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); + let samples = 0; let lastSampleTimestamp = 0; - for await (const sample of sink.packets()) { + for await (const sample of cursor) { samples++; lastSampleTimestamp = sample.timestamp; if (sample.sequenceNumber === 212) { @@ -66,23 +68,24 @@ test('can do random access', async () => { const track = await input.getPrimaryAudioTrack(); assert(track); - const packetSink = new EncodedPacketSink(track); - const packet = await packetSink.getPacket(10); + const reader = new PacketReader(track); + + const packet = await reader.getAt(10); assert(packet); expect(packet.timestamp).toBe(9.93814058956916); expect(packet.data.byteLength).toBe(8345); expect(packet.sequenceNumber).toBe(107); expect(packet.duration).toBe(0.09287981859410431); - const nextPacket = await packetSink.getNextPacket(packet); + const nextPacket = await reader.getNext(packet); assert(nextPacket); expect(nextPacket.timestamp).toBe(10.031020408163265); expect(nextPacket.data.byteLength).toBe(8988); expect(nextPacket.sequenceNumber).toBe(108); expect(nextPacket.duration).toBe(0.09287981859410431); - const priorPacket = await packetSink.getPacket(3); + const priorPacket = await reader.getAt(3); assert(priorPacket); expect(priorPacket.timestamp).toBe(2.972154195011338); expect(priorPacket.data.byteLength).toBe(6877); @@ -99,9 +102,10 @@ test('can get metadata-only packets', async () => { const track = await input.getPrimaryAudioTrack(); assert(track); - const packetSink = new EncodedPacketSink(track); - const packet = await packetSink.getPacket(10, { metadataOnly: true }); + const reader = new PacketReader(track); + + const packet = await reader.getAt(10, { metadataOnly: true }); assert(packet); expect(packet.timestamp).toBe(9.93814058956916); expect(packet.isMetadataOnly).toBe(true); @@ -213,11 +217,13 @@ test('can re-mux a .flac', async () => { images: inputImages, }); - const inputPacketSink = new EncodedPacketSink(inputTrack); - const outputPacketSink = new EncodedPacketSink(outputTrack); + const inputReader = new PacketReader(inputTrack); + const outputReader = new PacketReader(outputTrack); + const outputCursor = new PacketCursor(outputTrack); + let packets = 0; let timestamp = 0; - for await (const packet of outputPacketSink.packets()) { + for await (const packet of outputCursor) { packets++; timestamp = packet.timestamp; } @@ -226,14 +232,14 @@ test('can re-mux a .flac', async () => { expect(timestamp).toBe(19.690521541950112); // Test that packets are byte-identical - const inputPacket = await inputPacketSink.getPacket(10); - const outputPacket = await outputPacketSink.getPacket(10); + const inputPacket = await inputReader.getAt(10); + const outputPacket = await outputReader.getAt(10); assert(inputPacket); assert(outputPacket); - const { data: inputPacketData, ...otherInputPacket } = inputPacket; - const { data: outputPacketData, ...otherOutputPacket } = outputPacket; + const { data: inputPacketData, _ownerId: _inputOwnerId, ...otherInputPacket } = inputPacket; + const { data: outputPacketData, _ownerId: _outputOwnerId, ...otherOutputPacket } = outputPacket; expect(otherInputPacket).toEqual(otherOutputPacket); expect(inputPacketData).toEqual(outputPacketData); @@ -334,7 +340,7 @@ test('can read a FLAC file with leading ID3v2 tags', async () => { assert(track); expect(await track.getDurationFromMetadata()).toEqual(19.714285714285715); - const firstPacket = await new EncodedPacketSink(track).getPacket(0); + const firstPacket = await new PacketReader(track).getAt(0); assert(firstPacket); expect(firstPacket.sequenceNumber).toBe(0); expect(firstPacket.timestamp).toBe(0); diff --git a/test/node/forgiving-call-serializer.test.ts b/test/node/forgiving-call-serializer.test.ts new file mode 100644 index 000000000..6be1b5671 --- /dev/null +++ b/test/node/forgiving-call-serializer.test.ts @@ -0,0 +1,71 @@ +import { expect, test } from 'vitest'; +import { ForgivingCallSerializer } from '../../src/misc.js'; + +const executeDelayed = async (fn: () => T) => { + await new Promise(resolve => setTimeout(resolve, 10)); + return fn(); +}; + +test('Call serialization and return values', async () => { + const numbers: number[] = []; + const serializer = new ForgivingCallSerializer(); + + const first = serializer.call(() => numbers.push(1)); + const second = serializer.call(() => executeDelayed(() => numbers.push(2))); + const third = serializer.call(() => numbers.push(3)); + const fourth = serializer.call(() => executeDelayed(() => numbers.push(4))); + + await fourth; + + expect(numbers).toEqual([1, 2, 3, 4]); + expect(await first).toBe(1); + expect(await second).toBe(2); + expect(await third).toBe(3); + expect(await fourth).toBe(4); +}); + +test('Synchronous return value', async () => { + const serializer = new ForgivingCallSerializer(); + + const first = serializer.call(() => {}); + expect(first).not.toBeInstanceOf(Promise); + + const second = serializer.call(async () => {}); + const third = serializer.call(() => {}); + expect(second).toBeInstanceOf(Promise); + expect(third).toBeInstanceOf(Promise); + + await third; + + const fourth = serializer.call(() => {}); + expect(fourth).not.toBeInstanceOf(Promise); +}); + +test('Error handling', async () => { + const serializer = new ForgivingCallSerializer(); + + expect(() => serializer.call(() => { + throw new Error('yo'); + })).toThrow(); + + const second = serializer.call(() => executeDelayed(() => { + throw new Error('yo'); + })); + const third = serializer.call(() => executeDelayed(() => 1 + 2)); + + await expect(second).rejects.toThrow(); + expect(await third).toBe(3); +}); + +test('Dangling rejected call', async () => { + const serializer = new ForgivingCallSerializer(); + + const first = serializer.call(() => executeDelayed(() => { + throw new Error('yo'); + })); + + await expect(first).rejects.toThrow(); + + await new Promise(resolve => setTimeout(resolve, 20)); + // No error +}); diff --git a/test/node/frame-rate.test.ts b/test/node/frame-rate.test.ts index dad6b4250..d6c087b49 100644 --- a/test/node/frame-rate.test.ts +++ b/test/node/frame-rate.test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { ALL_FORMATS, Input, FilePathSource, EncodedPacketSink } from '../../src/index.js'; +import { ALL_FORMATS, Input, FilePathSource, PacketCursor } from '../../src/index.js'; import { findUnderlyingFrameRate } from '../../src/input-track.js'; import { assert } from '../../src/misc.js'; @@ -147,10 +147,10 @@ const getSortedTrackTicks = async (filePath: string) => { assert(videoTrack); const timeResolution = await videoTrack.getTimeResolution(); - const sink = new EncodedPacketSink(videoTrack); + const cursor = new PacketCursor(videoTrack, { options: { metadataOnly: true } }); const ticks: number[] = []; - for await (const packet of sink.packets(undefined, undefined, { metadataOnly: true })) { + for await (const packet of cursor) { ticks.push(Math.round(packet.timestamp * timeResolution)); } diff --git a/test/node/hls-input.test.ts b/test/node/hls-input.test.ts index 90a578b21..ec98bbc96 100644 --- a/test/node/hls-input.test.ts +++ b/test/node/hls-input.test.ts @@ -1,5 +1,14 @@ /* eslint-disable @stylistic/max-len */ -import { ALL_FORMATS, BufferSource, EncodedPacketSink, Input, InputAudioTrack, InputVideoTrack, UrlSource } from 'mediabunny'; +import { + ALL_FORMATS, + BufferSource, + Input, + InputAudioTrack, + InputVideoTrack, + PacketCursor, + PacketReader, + UrlSource, +} from 'mediabunny'; import { expect, test, vi } from 'vitest'; import { HLS, HLS_FORMATS, HlsInputFormat, MP4 } from '../../src/input-format.js'; import { assert, hexStringToBytes, rejectAfter } from '../../src/misc.js'; @@ -169,20 +178,20 @@ test.concurrent('Big Buck Bunny', { timeout: 15_000 }, async () => { expect(Math.min(await videoTracks[0]!.getFirstTimestamp(), await audioTracks[0]!.getFirstTimestamp())).toBe(0); - const sink = new EncodedPacketSink(videoTracks[0]!); - const lastPacket = await sink.getPacket(Infinity); + const reader = new PacketReader(videoTracks[0]!); + const lastPacket = await reader.getAt(Infinity); expect(lastPacket!.timestamp).toBeCloseTo(634.5899); expect(sourceCount).toBe(1 + 5 + 5 + 1); - const middlePacket = await sink.getPacket(300); + const middlePacket = await reader.getAt(300); expect(middlePacket!.timestamp).toBeCloseTo(299.9899); expect(sourceCount).toBe(1 + 5 + 5 + 1 + 1); let lastSequenceNumber = -Infinity; let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(videoTracks[0]!)) { expect(packet.sequenceNumber).toBeGreaterThan(lastSequenceNumber); lastSequenceNumber = packet.sequenceNumber; @@ -302,14 +311,17 @@ test.concurrent('AES and discontinuities', { timeout: 15_000 }, async () => { expect(sourceCount).toBe(3); // Entry, first segment, and encryption key - const sink = new EncodedPacketSink(videoTrack); - const firstPacket = await sink.getFirstPacket(); + const reader = new PacketReader(videoTrack); + const firstPacket = await reader.getFirst(); assert(firstPacket); let packetCount = 0; + const cursor = new PacketCursor(videoTrack); + await cursor.seekTo(35); + // Loop over the discontunity boundary - for await (const packet of sink.packets((await sink.getPacket(35))!)) { + for await (const packet of cursor) { packetCount++; if (packet.timestamp >= 40) { @@ -335,12 +347,12 @@ test.concurrent('Range requests', { timeout: 15_000 }, async () => { const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new EncodedPacketSink(videoTrack); - const firstPacket = await sink.getFirstPacket(); + const reader = new PacketReader(videoTrack); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBeCloseTo(0, 1); - const lastPacket = await sink.getPacket(Infinity); + const lastPacket = await reader.getAt(Infinity); assert(lastPacket); expect(lastPacket.timestamp).toBeCloseTo(47.26133333333333); @@ -384,8 +396,7 @@ test.concurrent('Out-of-band audio track via ADTS', { timeout: 15_000 }, async ( expect(await videoTrack.hasOnlyKeyPackets()).toBe(false); let lastTimestamp = -Infinity; - const sink = new EncodedPacketSink(audioTrack); - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(audioTrack)) { expect(packet.timestamp).toBeGreaterThan(lastTimestamp); lastTimestamp = packet.timestamp; } @@ -540,13 +551,13 @@ test.concurrent('Single-value PDT', { timeout: 15_000 }, async () => { const endDateTime = new Date(endTimestamp * 1000).toISOString(); expect(endDateTime).toBe('2013-05-08T17:41:40.000Z'); - const sink = new EncodedPacketSink(track); - const firstPacket = await sink.getFirstPacket(); + const reader = new PacketReader(track); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBe(firstTimestamp); expect(firstPacket.sequenceNumber).toBe(752); // Make sure it's not huge - const lastPacket = await sink.getPacket(Infinity); + const lastPacket = await reader.getAt(Infinity); assert(lastPacket); expect(lastPacket.timestamp).toBeCloseTo(firstTimestamp + 50, 1); expect(lastPacket.sequenceNumber).toBeLessThan(1e8 * 60); @@ -564,9 +575,8 @@ test.concurrent('Duplicate PDT', { timeout: 30_000 }, async () => { expect(await audioTrack.getFirstTimestamp()).toBe(Date.parse('2013-06-17T18:32:00Z') / 1000); // Ensure the timestamps are monotonically increasing - const sink = new EncodedPacketSink(audioTrack); let lastTimestamp = -Infinity; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(audioTrack)) { expect(packet.timestamp).toBeGreaterThan(lastTimestamp); lastTimestamp = packet.timestamp; } @@ -581,11 +591,11 @@ test.concurrent('PDT with large gaps', { timeout: 15_000 }, async () => { const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); - const sink = new EncodedPacketSink(audioTrack); - const packet = await sink.getPacket(Date.parse('2012-12-06T19:10:03+00:00') / 1000 + 6); + const reader = new PacketReader(audioTrack); + const packet = await reader.getAt(Date.parse('2012-12-06T19:10:03+00:00') / 1000 + 6); assert(packet); - const nextPacket = await sink.getNextPacket(packet); + const nextPacket = await reader.getNext(packet); assert(nextPacket); expect(nextPacket.timestamp - packet.timestamp).toBeGreaterThan(59); @@ -625,7 +635,7 @@ test.concurrent('Single-value PDT with unix offsets disabled', { timeout: 15_000 const endTimestamp = await track.computeDuration(); expect(endTimestamp).toBe(firstTimestamp + 50); - const firstPacket = await new EncodedPacketSink(track).getFirstPacket(); + const firstPacket = await new PacketReader(track).getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBe(firstTimestamp); // Kinda obvious check tbh @@ -798,14 +808,14 @@ test.concurrent('Live HLS', { timeout: 30_000 }, async () => { ]); expect(duration).toBeGreaterThan((Date.now() / 1000) - 3600); - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); - let currentLastPacket = await sink.getPacket(Infinity, { skipLiveWait: true }); + let currentLastPacket = await reader.getAt(Infinity, { skipLiveWait: true }); assert(currentLastPacket); // Actually find the last packet in decode order while (true) { - const nextKnownPacket = await sink.getNextPacket(currentLastPacket, { skipLiveWait: true }); + const nextKnownPacket = await reader.getNext(currentLastPacket, { skipLiveWait: true }); if (!nextKnownPacket) { break; } @@ -814,7 +824,7 @@ test.concurrent('Live HLS', { timeout: 30_000 }, async () => { } // This tests waiting for the live stream to advance - const nextPacket = await sink.getNextPacket(currentLastPacket); + const nextPacket = await reader.getNext(currentLastPacket); assert(nextPacket); expect(nextPacket.sequenceNumber).toBeGreaterThan(currentLastPacket.sequenceNumber); @@ -966,8 +976,8 @@ test.concurrent('Widevine encryption (SAMPLE-AES-CTR) fails without keys', async const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new EncodedPacketSink(videoTrack); - await expect(sink.getPacket(Infinity)).rejects.toThrow(); + const reader = new PacketReader(videoTrack); + await expect(reader.getAt(Infinity)).rejects.toThrow(); }); test.concurrent('Widevine encryption (SAMPLE-AES-CTR) succeeds with string keys', async () => { @@ -1007,8 +1017,8 @@ test.concurrent('Widevine encryption (SAMPLE-AES-CTR) succeeds with string keys' const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new EncodedPacketSink(videoTrack); - const lastPacket = await sink.getPacket(Infinity); + const reader = new PacketReader(videoTrack); + const lastPacket = await reader.getAt(Infinity); assert(lastPacket); expect(lastPacket.timestamp + lastPacket.duration).toBe(60); }); @@ -1040,8 +1050,8 @@ test.concurrent('Widevine encryption (SAMPLE-AES-CTR) succeeds with buffer keys' const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new EncodedPacketSink(videoTrack); - const lastPacket = await sink.getPacket(Infinity); + const reader = new PacketReader(videoTrack); + const lastPacket = await reader.getAt(Infinity); assert(lastPacket); expect(lastPacket.timestamp + lastPacket.duration).toBe(60); }); @@ -1084,8 +1094,8 @@ test.concurrent('Widevine HLS passes #EXT-X-KEY PSSH boxes to key resolver', asy const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new EncodedPacketSink(videoTrack); - const lastPacket = await sink.getPacket(Infinity); + const reader = new PacketReader(videoTrack); + const lastPacket = await reader.getAt(Infinity); assert(lastPacket); expect(lastPacket.timestamp + lastPacket.duration).toBe(60); }); diff --git a/test/node/hls-output.test.ts b/test/node/hls-output.test.ts index 9bc30979c..0a161b171 100644 --- a/test/node/hls-output.test.ts +++ b/test/node/hls-output.test.ts @@ -25,7 +25,8 @@ import { Input } from '../../src/input.js'; import { BufferSource, CustomPathedSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { InputAudioTrack, InputVideoTrack } from '../../src/input-track.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; + const videoSource = (codec: VideoCodec = 'avc') => new EncodedVideoPacketSource(codec); const audioSource = (codec: AudioCodec = 'aac') => new EncodedAudioPacketSource(codec); @@ -858,9 +859,9 @@ const setUpSegmentationEnvironment = async (options: { const videoTrack = await input.getPrimaryVideoTrack() as InputVideoTrack; if (videoTrack) { - const sink = new EncodedPacketSink(videoTrack); + const cursor = new PacketCursor(videoTrack); const timestamps: number[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); } videoBundle.resolve(timestamps); @@ -870,9 +871,9 @@ const setUpSegmentationEnvironment = async (options: { const audioTrack = await input.getPrimaryAudioTrack() as InputAudioTrack; if (audioTrack) { - const sink = new EncodedPacketSink(audioTrack); + const cursor = new PacketCursor(audioTrack); const timestamps: number[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); } audioBundle.resolve(timestamps); @@ -2152,9 +2153,9 @@ test('Single-file mode with fragmented MP4 produces proper standalone segment fi }); const track = (await input.getPrimaryVideoTrack())!; const timestamps: number[] = []; - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); } @@ -2346,10 +2347,10 @@ segment-1-2.m4s const videoTrack = await segmentInput.getPrimaryVideoTrack() as InputVideoTrack; expect(videoTrack).toBeTruthy(); - const sink = new EncodedPacketSink(videoTrack); + const cursor = new PacketCursor(videoTrack); let packetCount = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const packet of sink.packets()) { + for await (const packet of cursor) { packetCount++; } expect(packetCount).toBe(4); @@ -2597,16 +2598,16 @@ const runSparseTracksInSegments = async (hlsOptions: HlsOutputFormatOptions) => const audioTrack = await input.getPrimaryAudioTrack() as InputAudioTrack; expect(audioTrack).toBeTruthy(); - const videoSink = new EncodedPacketSink(videoTrack); + const videoCursor = new PacketCursor(videoTrack); const videoTimestamps: number[] = []; - for await (const packet of videoSink.packets()) { + for await (const packet of videoCursor) { videoTimestamps.push(packet.timestamp); } expect(videoTimestamps).toEqual([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5]); - const audioSink = new EncodedPacketSink(audioTrack); + const audioCursor = new PacketCursor(audioTrack); const audioTimestamps: number[] = []; - for await (const packet of audioSink.packets()) { + for await (const packet of audioCursor) { audioTimestamps.push(packet.timestamp); } expect(audioTimestamps).toEqual([2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5]); diff --git a/test/node/isobmff-muxer.test.ts b/test/node/isobmff-muxer.test.ts index 91a28b164..13e4fcd09 100644 --- a/test/node/isobmff-muxer.test.ts +++ b/test/node/isobmff-muxer.test.ts @@ -4,14 +4,14 @@ import { fileURLToPath } from 'node:url'; import { Input } from '../../src/input.js'; import { BufferSource, FilePathSource } from '../../src/source.js'; import { ADTS, ALL_FORMATS } from '../../src/input-format.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; import { Output } from '../../src/output.js'; import { BufferTarget } from '../../src/target.js'; import { Mp4OutputFormat } from '../../src/output-format.js'; import { Conversion } from '../../src/conversion.js'; import { assert, toDataView } from '../../src/misc.js'; import { EncodedAudioPacketSource, EncodedVideoPacketSource } from '../../src/media-source.js'; -import { EncodedPacket } from '../../src/packet.js'; +import { EncodedPacket, PacketReader } from '../../src/packet.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -52,10 +52,10 @@ test('ISOBMFF muxer internally converts ADTS to AAC', async () => { const outputDecoderConfig = await outputTrack.getDecoderConfig(); expect(outputDecoderConfig!.description).toBeDefined(); - const outputSink = new EncodedPacketSink(outputTrack); + const outputCursor = new PacketCursor(outputTrack); let count = 0; - for await (const packet of outputSink.packets()) { + for await (const packet of outputCursor) { // Packets should NOT be ADTS frames (should not start with 0xFFF sync word) const isAdts = packet.data[0] === 0xff && (packet.data[1]! & 0xf0) === 0xf0; expect(isAdts).toBe(false); @@ -76,9 +76,9 @@ test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => { assert(videoTrack); assert(audioTrack); - const originalVideoSink = new EncodedPacketSink(videoTrack); + const originalVideoCursor = new PacketCursor(videoTrack); const originalTimestamps: number[] = []; - for await (const packet of originalVideoSink.packets()) { + for await (const packet of originalVideoCursor) { originalTimestamps.push(packet.timestamp); } @@ -98,10 +98,10 @@ test('Fragmented fMP4 with video+audio preserves B-frame CTS', async () => { const outputVideoTrack = await outputAsInput.getPrimaryVideoTrack(); assert(outputVideoTrack); - const videoSink = new EncodedPacketSink(outputVideoTrack); + const videoCursor = new PacketCursor(outputVideoTrack); const timestamps: number[] = []; - for await (const packet of videoSink.packets()) { + for await (const packet of videoCursor) { timestamps.push(packet.timestamp); } @@ -164,11 +164,11 @@ test('Non-zero start timestamp, regular MP4', async () => { const track = await input.getPrimaryVideoTrack(); assert(track); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); const timestamps: number[] = []; const durations: number[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); durations.push(packet.duration); } @@ -208,11 +208,11 @@ test('Non-zero start timestamp, fragmented MP4', async () => { const track = await input.getPrimaryVideoTrack(); assert(track); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); const timestamps: number[] = []; const durations: number[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); durations.push(packet.duration); } @@ -273,10 +273,10 @@ const testNegativeTimestampRoundTrip = async ( const track = await input.getPrimaryVideoTrack(); assert(track); - const sink = new EncodedPacketSink(track); + const packetReader = new PacketReader(track); const outputPackets: EncodedPacket[] = []; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(track)) { outputPackets.push(packet); } @@ -289,7 +289,7 @@ const testNegativeTimestampRoundTrip = async ( }))); for (const inputPacket of inputPackets) { - const outputPacket = await sink.getPacket(inputPacket.timestamp); + const outputPacket = await packetReader.getAt(inputPacket.timestamp); assert(outputPacket); expect({ @@ -385,10 +385,10 @@ test('PCM audio, silence padding', async () => { const numChannels = await audioTrack.getNumberOfChannels(); const expectedFrameCount = 48000 + 256; - const sink = new EncodedPacketSink(audioTrack); + const cursor = new PacketCursor(audioTrack); let frameCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { frameCount += packet.byteLength / 2 / numChannels; } @@ -425,10 +425,10 @@ test('PCM audio, no silence padding with approximate timestamps', async () => { const numChannels = await audioTrack.getNumberOfChannels(); const expectedFrameCount = 256 + 256; - const sink = new EncodedPacketSink(audioTrack); + const cursor = new PacketCursor(audioTrack); let frameCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { frameCount += packet.byteLength / 2 / numChannels; } @@ -545,12 +545,12 @@ test('No btrt boxes, fragmented MP4 without bitrate metadata', async () => { await output.start(); const videoMeta = { decoderConfig: (await videoTrack.getDecoderConfig())! }; - for await (const packet of new EncodedPacketSink(videoTrack).packets()) { + for await (const packet of new PacketCursor(videoTrack)) { await videoSource.add(packet, videoMeta); } const audioMeta = { decoderConfig: (await audioTrack.getDecoderConfig())! }; - for await (const packet of new EncodedPacketSink(audioTrack).packets()) { + for await (const packet of new PacketCursor(audioTrack)) { await audioSource.add(packet, audioMeta); } diff --git a/test/node/matroska-muxer.test.ts b/test/node/matroska-muxer.test.ts index 0087a911d..cdf7f72af 100644 --- a/test/node/matroska-muxer.test.ts +++ b/test/node/matroska-muxer.test.ts @@ -4,14 +4,14 @@ import { fileURLToPath } from 'node:url'; import { Input } from '../../src/input.js'; import { BufferSource, FilePathSource } from '../../src/source.js'; import { ADTS, ALL_FORMATS } from '../../src/input-format.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; import { Output } from '../../src/output.js'; import { BufferTarget } from '../../src/target.js'; import { MkvOutputFormat } from '../../src/output-format.js'; import { Conversion } from '../../src/conversion.js'; import { assert } from '../../src/misc.js'; import { EncodedVideoPacketSource } from '../../src/media-source.js'; -import { EncodedPacket } from '../../src/packet.js'; +import { EncodedPacket, PacketReader } from '../../src/packet.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -52,10 +52,10 @@ test('Matroska muxer internally converts ADTS to AAC', async () => { const outputDecoderConfig = await outputTrack.getDecoderConfig(); expect(outputDecoderConfig!.description).toBeDefined(); - const outputSink = new EncodedPacketSink(outputTrack); + const outputCursor = new PacketCursor(outputTrack); let count = 0; - for await (const packet of outputSink.packets()) { + for await (const packet of outputCursor) { // Packets should NOT be ADTS frames (should not start with 0xFFF sync word) const isAdts = packet.data[0] === 0xff && (packet.data[1]! & 0xf0) === 0xf0; expect(isAdts).toBe(false); @@ -109,10 +109,10 @@ const testNegativeTimestampRoundTrip = async (timestamps: number[], duration: nu const track = await input.getPrimaryVideoTrack(); assert(track); - const sink = new EncodedPacketSink(track); + const packetReader = new PacketReader(track); const outputPackets: EncodedPacket[] = []; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(track)) { outputPackets.push(packet); } @@ -125,7 +125,7 @@ const testNegativeTimestampRoundTrip = async (timestamps: number[], duration: nu }))); for (const inputPacket of inputPackets) { - const outputPacket = await sink.getPacket(inputPacket.timestamp); + const outputPacket = await packetReader.getAt(inputPacket.timestamp); assert(outputPacket); expect({ diff --git a/test/node/mp3-encoder-extension.test.ts b/test/node/mp3-encoder-extension.test.ts index 9d2c0f83e..7e55a2fc3 100644 --- a/test/node/mp3-encoder-extension.test.ts +++ b/test/node/mp3-encoder-extension.test.ts @@ -6,7 +6,8 @@ import { Output } from '../../src/output.js'; import { BufferTarget } from '../../src/target.js'; import { canEncode, Quality } from '../../src/encode.js'; import { AudioSampleSource } from '../../src/media-source.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { Mp3OutputFormat, Mp4OutputFormat } from '../../src/output-format.js'; import { AudioSample } from '../../src/sample.js'; import { registerMp3Encoder } from '@mediabunny/mp3-encoder'; @@ -73,9 +74,9 @@ test('MP3 encoding', async () => { expect(await track.getSampleRate()).toBe(sampleRate); expect(await track.getNumberOfChannels()).toBe(channels); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { expect(packet.type).toBe('key'); packetCount++; } @@ -121,8 +122,8 @@ test('MP3 with huge timestamps', async () => { }); const track = (await input.getPrimaryAudioTrack())!; - const sink = new EncodedPacketSink(track); - const firstPacket = await sink.getFirstPacket(); + const reader = new PacketReader(track); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.timestamp).toBe(timestamp); diff --git a/test/node/mp3-muxer.test.ts b/test/node/mp3-muxer.test.ts index e4c33d157..f40e234b3 100644 --- a/test/node/mp3-muxer.test.ts +++ b/test/node/mp3-muxer.test.ts @@ -8,7 +8,7 @@ import { Output } from '../../src/output.js'; import { Mp3OutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { EncodedAudioPacketSource } from '../../src/media-source.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; import { assert } from '../../src/misc.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -32,7 +32,7 @@ test('Muxing without Xing header', async () => { await output.start(); let inputPacketCount = 0; - for await (const packet of new EncodedPacketSink(inputTrack).packets()) { + for await (const packet of new PacketCursor(inputTrack)) { await source.add(packet); inputPacketCount++; } @@ -48,8 +48,7 @@ test('Muxing without Xing header', async () => { assert(outputTrack); let outputPacketCount = 0; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const packet of new EncodedPacketSink(outputTrack).packets()) { + for await (const _ of new PacketCursor(outputTrack)) { outputPacketCount++; } diff --git a/test/node/mpeg-ts-demuxing.test.ts b/test/node/mpeg-ts-demuxing.test.ts index 325b59111..44dfb11b7 100644 --- a/test/node/mpeg-ts-demuxing.test.ts +++ b/test/node/mpeg-ts-demuxing.test.ts @@ -7,7 +7,8 @@ import fs from 'node:fs'; import { Readable } from 'node:stream'; import { ALL_FORMATS, MPEG_TS } from '../../src/input-format.js'; import { assert } from '../../src/misc.js'; -import { EncodedPacketSink } from '../../src/media-sink.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { EncodedPacket } from '../../src/packet.js'; import { MpegTsDemuxer } from '../../src/mpeg-ts/mpeg-ts-demuxer.js'; import { MpegTsStreamType } from '../../src/mpeg-ts/mpeg-ts-misc.js'; @@ -117,9 +118,9 @@ test('MPEG-TS AVC video packets', async () => { const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect([...firstPacket.data.slice(0, 4)]).toEqual([0, 0, 0, 1]); @@ -129,12 +130,12 @@ test('MPEG-TS AVC video packets', async () => { expect(firstPacket.duration).toBe(0.016666666666666666); expect(firstPacket.sequenceNumber).not.toBe(-1); - const firstPacketMetadataOnly = await sink.getFirstPacket({ metadataOnly: true }); + const firstPacketMetadataOnly = await reader.getFirst({ metadataOnly: true }); assert(firstPacketMetadataOnly); expect(firstPacketMetadataOnly.data).toHaveLength(0); expect(firstPacketMetadataOnly.byteLength).toBe(23813); - const secondPacket = await sink.getNextPacket(firstPacket); + const secondPacket = await reader.getNext(firstPacket); assert(secondPacket); expect([...secondPacket.data.slice(0, 4)]).toEqual([0, 0, 0, 1]); @@ -150,7 +151,7 @@ test('MPEG-TS AVC video packets', async () => { while (currentPacket) { expect(currentPacket.duration).toBe(0.016666666666666666); - currentPacket = await sink.getNextPacket(currentPacket); + currentPacket = await reader.getNext(currentPacket); count++; } @@ -166,9 +167,9 @@ test('MPEG-TS AAC audio packets', async () => { const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); - const sink = new EncodedPacketSink(audioTrack); + const reader = new PacketReader(audioTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.data[0]).toEqual(0xff); @@ -178,7 +179,7 @@ test('MPEG-TS AAC audio packets', async () => { expect(firstPacket.duration).toBe(0.021333333333333333); expect(firstPacket.sequenceNumber).not.toBe(-1); - const secondPacket = await sink.getNextPacket(firstPacket); + const secondPacket = await reader.getNext(firstPacket); assert(secondPacket); expect(secondPacket.data[0]).toEqual(0xff); @@ -194,7 +195,7 @@ test('MPEG-TS AAC audio packets', async () => { while (currentPacket) { expect(currentPacket.duration).toBe(0.021333333333333333); - currentPacket = await sink.getNextPacket(currentPacket); + currentPacket = await reader.getNext(currentPacket); count++; } @@ -216,24 +217,24 @@ test('MPEG-TS video seeking', async () => { demuxer.seekChunkSize = 250_000; // Try it again with a smaller chunk size } - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); const firstTimestamp = await videoTrack.getFirstTimestamp(); - const firstPacket = await sink.getPacket(firstTimestamp); + const firstPacket = await reader.getAt(firstTimestamp); assert(firstPacket); expect(firstPacket.timestamp).toBe(firstTimestamp); expect(firstPacket.duration).toBe(0.016666666666666666); - expect(firstPacket.sequenceNumber).toBe((await sink.getFirstPacket())?.sequenceNumber); + expect(firstPacket.sequenceNumber).toBe((await reader.getFirst())?.sequenceNumber); - const lastPacket = await sink.getPacket(Infinity); + const lastPacket = await reader.getAt(Infinity); assert(lastPacket); expect(lastPacket.timestamp).toBeCloseTo(14.983333333333333); expect(lastPacket.duration).toBe(0.016666666666666666); - const beforeFirst = await sink.getPacket(-10); + const beforeFirst = await reader.getAt(-10); expect(beforeFirst).toBeNull(); - const middlePacket = await sink.getPacket(12.5); + const middlePacket = await reader.getAt(12.5); assert(middlePacket); expect(middlePacket.timestamp).toBeCloseTo(12.5); expect(middlePacket.duration).toBe(0.016666666666666666); @@ -243,13 +244,13 @@ test('MPEG-TS video seeking', async () => { while (currentPacket) { allPackets.push(currentPacket); - currentPacket = await sink.getNextPacket(currentPacket); + currentPacket = await reader.getNext(currentPacket); } expect(allPackets).toHaveLength(298); for (const packet of allPackets) { - const seekedPacket = await sink.getPacket(packet.timestamp); + const seekedPacket = await reader.getAt(packet.timestamp); assert(seekedPacket); expect(seekedPacket.timestamp).toBe(packet.timestamp); expect(seekedPacket.duration).toBe(packet.duration); @@ -273,24 +274,24 @@ test('MPEG-TS audio seeking', async () => { demuxer.seekChunkSize = 250_000; // Try it again with a smaller chunk size } - const sink = new EncodedPacketSink(audioTrack); + const reader = new PacketReader(audioTrack); const firstTimestamp = await audioTrack.getFirstTimestamp(); - const firstPacket = await sink.getPacket(firstTimestamp); + const firstPacket = await reader.getAt(firstTimestamp); assert(firstPacket); expect(firstPacket.timestamp).toBe(firstTimestamp); expect(firstPacket.duration).toBe(0.021333333333333333); - expect(firstPacket.sequenceNumber).toBe((await sink.getFirstPacket())?.sequenceNumber); + expect(firstPacket.sequenceNumber).toBe((await reader.getFirst())?.sequenceNumber); - const lastPacket = await sink.getPacket(Infinity); + const lastPacket = await reader.getAt(Infinity); assert(lastPacket); expect(lastPacket.timestamp).toBeCloseTo(14.982666666666667); expect(lastPacket.duration).toBe(0.021333333333333333); - const beforeFirst = await sink.getPacket(-10); + const beforeFirst = await reader.getAt(-10); expect(beforeFirst).toBeNull(); - const middlePacket = await sink.getPacket(12.5); + const middlePacket = await reader.getAt(12.5); assert(middlePacket); expect(middlePacket.timestamp).toBeCloseTo(12.486666666666666); expect(middlePacket.duration).toBe(0.021333333333333333); @@ -300,13 +301,13 @@ test('MPEG-TS audio seeking', async () => { while (currentPacket) { allPackets.push(currentPacket); - currentPacket = await sink.getNextPacket(currentPacket); + currentPacket = await reader.getNext(currentPacket); } expect(allPackets).toHaveLength(234); for (const packet of allPackets) { - const seekedPacket = await sink.getPacket(packet.timestamp); + const seekedPacket = await reader.getAt(packet.timestamp); assert(seekedPacket); expect(seekedPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved expect(seekedPacket.duration).toBe(packet.duration); // The correct duration was retrieved @@ -330,18 +331,19 @@ test('MPEG-TS seeking race condition test', async () => { demuxer.seekChunkSize = 250_000; // Try it again with a smaller chunk size } - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); const allPackets: EncodedPacket[] = []; - let currentPacket: EncodedPacket | null = await sink.getFirstPacket(); + let currentPacket: EncodedPacket | null = await reader.getFirst(); while (currentPacket) { allPackets.push(currentPacket); - currentPacket = await sink.getNextPacket(currentPacket); + currentPacket = await reader.getNext(currentPacket); } // Perform all seeks concurrently - const seekPromises = allPackets.map(packet => sink.getPacket(packet.timestamp)); + const seekPromises = allPackets.map(packet => reader.getAt(packet.timestamp)); + // eslint-disable-next-line @typescript-eslint/await-thenable const seekedPackets = await Promise.all(seekPromises); for (let j = 0; j < allPackets.length; j++) { @@ -370,37 +372,37 @@ test('MPEG-TS video key packets', { timeout: 30_000 }, async () => { demuxer.seekChunkSize = 250_000; // Try it again with a smaller chunk size } - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.type).toBe('key'); - const secondPacket = await sink.getNextPacket(firstPacket); + const secondPacket = await reader.getNext(firstPacket); assert(secondPacket); expect(secondPacket.type).toBe('delta'); - const nextKeyPacket = await sink.getNextKeyPacket(firstPacket); + const nextKeyPacket = await reader.getNextKey(firstPacket); assert(nextKeyPacket); expect(nextKeyPacket.type).toBe('key'); expect(nextKeyPacket.sequenceNumber).toBeGreaterThan(secondPacket.sequenceNumber); - const firstKeyPacket = await sink.getKeyPacket(firstPacket.timestamp + 0.5); + const firstKeyPacket = await reader.getKeyAt(firstPacket.timestamp + 0.5); assert(firstKeyPacket); expect(firstKeyPacket.type).toBe('key'); expect(firstKeyPacket.sequenceNumber).toBe(firstPacket.sequenceNumber); - const afterKeyPacket = await sink.getNextPacket(firstKeyPacket); + const afterKeyPacket = await reader.getNext(firstKeyPacket); expect(afterKeyPacket).not.toBe(null); expect(afterKeyPacket!.type).toBe('delta'); expect(afterKeyPacket!.sequenceNumber).toBe(secondPacket.sequenceNumber); - const secondKeyPacket = await sink.getKeyPacket(2.5); + const secondKeyPacket = await reader.getKeyAt(2.5); assert(secondKeyPacket); expect(secondKeyPacket.type).toBe('key'); expect(secondKeyPacket.sequenceNumber).toBe(nextKeyPacket.sequenceNumber); - const lastKeyPacket = await sink.getKeyPacket(Infinity); + const lastKeyPacket = await reader.getKeyAt(Infinity); assert(lastKeyPacket); expect(lastKeyPacket.type).toBe('key'); expect(lastKeyPacket.sequenceNumber).toBeGreaterThan(secondKeyPacket.sequenceNumber); @@ -410,11 +412,11 @@ test('MPEG-TS video key packets', { timeout: 30_000 }, async () => { while (currentKeyPacket) { allKeyPackets.push(currentKeyPacket); - currentKeyPacket = await sink.getNextKeyPacket(currentKeyPacket); + currentKeyPacket = await reader.getNextKey(currentKeyPacket); } for (const packet of allKeyPackets) { - const keyPacket = await sink.getKeyPacket(packet.timestamp); + const keyPacket = await reader.getKeyAt(packet.timestamp); assert(keyPacket); expect(keyPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet expect(keyPacket.duration).toBe(packet.duration); // The correct duration was retrieved for this packet @@ -438,33 +440,33 @@ test('MPEG-TS audio key packets', async () => { demuxer.seekChunkSize = 250_000; // Try it again with a smaller chunk size } - const sink = new EncodedPacketSink(audioTrack); + const reader = new PacketReader(audioTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.type).toBe('key'); - const secondPacket = await sink.getNextPacket(firstPacket); + const secondPacket = await reader.getNext(firstPacket); assert(secondPacket); expect(secondPacket.type).toBe('key'); - const nextKeyPacket = await sink.getNextKeyPacket(firstPacket); + const nextKeyPacket = await reader.getNextKey(firstPacket); assert(nextKeyPacket); expect(nextKeyPacket.type).toBe('key'); expect(nextKeyPacket.sequenceNumber).toBe(secondPacket.sequenceNumber); // All audio packets are key packets - const middleKeyPacket = await sink.getKeyPacket(12.5); + const middleKeyPacket = await reader.getKeyAt(12.5); assert(middleKeyPacket); expect(middleKeyPacket.type).toBe('key'); - const afterKeyPacket = await sink.getNextPacket(middleKeyPacket); + const afterKeyPacket = await reader.getNext(middleKeyPacket); expect(afterKeyPacket).not.toBe(null); expect(afterKeyPacket!.type).toBe('key'); - const lastPacket = await sink.getPacket(Infinity); + const lastPacket = await reader.getAt(Infinity); assert(lastPacket); - const lastKeyPacket = await sink.getKeyPacket(Infinity); + const lastKeyPacket = await reader.getKeyAt(Infinity); assert(lastKeyPacket); expect(lastKeyPacket.type).toBe('key'); expect(lastKeyPacket.sequenceNumber).toBe(lastPacket.sequenceNumber); // It's actually the last packet @@ -474,11 +476,11 @@ test('MPEG-TS audio key packets', async () => { while (currentKeyPacket) { allKeyPackets.push(currentKeyPacket); - currentKeyPacket = await sink.getNextKeyPacket(currentKeyPacket); + currentKeyPacket = await reader.getNextKey(currentKeyPacket); } for (const packet of allKeyPackets) { - const keyPacket = await sink.getKeyPacket(packet.timestamp); + const keyPacket = await reader.getKeyAt(packet.timestamp); assert(keyPacket); expect(keyPacket.timestamp).toBe(packet.timestamp); // The correct timestamp was retrieved for this packet expect(keyPacket.duration).toBe(packet.duration); // The correct duration was retrieved for this packet @@ -506,14 +508,14 @@ test('MPEG-TS with unknown file size (ReadableStreamSource)', async () => { demuxer.seekChunkSize = 250_000; // Try it again with a smaller chunk size } - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.type).toBe('key'); expect(firstPacket.timestamp).toBe(10.033333333333333); - const middlePacket = await sink.getPacket(12.5); + const middlePacket = await reader.getAt(12.5); assert(middlePacket); expect(middlePacket.timestamp).toBeCloseTo(12.5); @@ -569,10 +571,10 @@ test('MPEG-TS with HEVC video', async () => { // No description, it's Annex B }); - const sink = new EncodedPacketSink(videoTrack); + const cursor = new PacketCursor(videoTrack); let i = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { expect(packet.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); // Annex B expect(packet.duration).toBeCloseTo(0.04166666666); expect(packet.type).toBe(i > 0 ? 'delta' : 'key'); @@ -599,13 +601,13 @@ test('MPEG-TS with MP3 audio', async () => { sampleRate: 48000, }); - const sink = new EncodedPacketSink(audioTrack); + const reader = new PacketReader(audioTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); let count = 0; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(audioTrack)) { expect(packet.data[0]).toBe(0xff); expect(packet.type).toBe('key'); count++; @@ -633,13 +635,13 @@ test('MPEG-TS with AC-3 audio (System A)', async () => { sampleRate: 48000, }); - const sink = new EncodedPacketSink(audioTrack); + const reader = new PacketReader(audioTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); let count = 0; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(audioTrack)) { expect(packet.data[0]).toBe(0x0b); expect(packet.data[1]).toBe(0x77); expect(packet.type).toBe('key'); @@ -668,13 +670,13 @@ test('MPEG-TS with AC-3 audio (System B)', async () => { sampleRate: 48000, }); - const sink = new EncodedPacketSink(audioTrack); + const reader = new PacketReader(audioTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); let count = 0; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(audioTrack)) { expect(packet.data[0]).toBe(0x0b); expect(packet.data[1]).toBe(0x77); expect(packet.type).toBe('key'); @@ -703,13 +705,13 @@ test('MPEG-TS with E-AC-3 audio (System A)', async () => { sampleRate: 48000, }); - const sink = new EncodedPacketSink(audioTrack); + const reader = new PacketReader(audioTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); let count = 0; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(audioTrack)) { expect(packet.data[0]).toBe(0x0b); expect(packet.data[1]).toBe(0x77); expect(packet.type).toBe('key'); @@ -738,13 +740,13 @@ test('MPEG-TS with E-AC-3 audio (System B)', async () => { sampleRate: 48000, }); - const sink = new EncodedPacketSink(audioTrack); + const reader = new PacketReader(audioTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); let count = 0; - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(audioTrack)) { expect(packet.data[0]).toBe(0x0b); expect(packet.data[1]).toBe(0x77); expect(packet.type).toBe('key'); @@ -793,19 +795,19 @@ test('MPEG-TS first packet key packet forcing', async () => { demuxer.seekChunkSize = 250_000; // Try it again with a smaller chunk size } - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); // Test that the first packet is indeed a key packet even if the file doesn't label it that way (L muxer) - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.type).toBe('key'); - const firstPacketSeeked = await sink.getPacket(firstPacket.timestamp); + const firstPacketSeeked = await reader.getAt(firstPacket.timestamp); assert(firstPacketSeeked); expect(firstPacketSeeked.type).toBe('key'); expect(firstPacketSeeked.sequenceNumber).toBe(firstPacket.sequenceNumber); - const firstKeyPacketSeeked = await sink.getKeyPacket(firstPacket.timestamp); + const firstKeyPacketSeeked = await reader.getKeyAt(firstPacket.timestamp); assert(firstKeyPacketSeeked); expect(firstKeyPacketSeeked.type).toBe('key'); expect(firstKeyPacketSeeked.sequenceNumber).toBe(firstPacket.sequenceNumber); @@ -827,16 +829,16 @@ test('MPEG-TS without initial key packet', async () => { demuxer.seekChunkSize = 250_000; // Try it again with a smaller chunk size } - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); expect(firstPacket.type).toBe('delta'); // First packet is delta - const noPacket = await sink.getKeyPacket(firstPacket.timestamp); + const noPacket = await reader.getKeyAt(firstPacket.timestamp); expect(noPacket).toBeNull(); - const aKeyPacket = await sink.getKeyPacket(Infinity); + const aKeyPacket = await reader.getKeyAt(Infinity); assert(aKeyPacket); expect(aKeyPacket.type).toBe('key'); } @@ -852,12 +854,12 @@ test('MPEG-TS with "extension" PES packets without PTS', async () => { assert(videoTrack); const packetSizes: number[] = []; - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); - for await (const packet of sink.packets()) { + for await (const packet of new PacketCursor(videoTrack)) { packetSizes.push(packet.data.byteLength); - const seeked = await sink.getPacket(packet.timestamp); + const seeked = await reader.getAt(packet.timestamp); expect(seeked!.timestamp).toBe(packet.timestamp); expect(seeked!.sequenceNumber).toBe(packet.sequenceNumber); } @@ -886,13 +888,13 @@ test('MPEG-TS with AUD-less video packets', async () => { const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new EncodedPacketSink(videoTrack); + const reader = new PacketReader(videoTrack); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await reader.getFirst(); assert(firstPacket); - const secondPacket = await sink.getNextPacket(firstPacket); + const secondPacket = await reader.getNext(firstPacket); assert(secondPacket); - const thirdPacket = await sink.getNextPacket(secondPacket); + const thirdPacket = await reader.getNext(secondPacket); assert(thirdPacket); expect(firstPacket.data.byteLength).toBe(331774); diff --git a/test/node/packet-cache.test.ts b/test/node/packet-cache.test.ts new file mode 100644 index 000000000..e01588d58 --- /dev/null +++ b/test/node/packet-cache.test.ts @@ -0,0 +1,1647 @@ +import { beforeAll, expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { InputVideoTrack } from '../../src/input-track.js'; +import { EncodedVideoPacketSource } from '../../src/media-source.js'; +import { isThenable, MaybePromise, MaybeRelevantPromise, ResultValue } from '../../src/misc.js'; +import { Output } from '../../src/output.js'; +import { Mp4OutputFormat } from '../../src/output-format.js'; +import { + EncodedPacket, + PACKET_SIZE_OVERHEAD, + PacketCache, + PacketReader, + PacketRetrievalOptions, + PacketRetrievalResult, + PacketType, +} from '../../src/packet.js'; +import { BufferSource } from '../../src/source.js'; +import { BufferTarget } from '../../src/target.js'; + +type PacketSpec = { + type: PacketType; + timestamp: number; + actualType?: PacketType; +}; + +// Decode order. Features an open GOP (key packet at 6 with leading packets at 4 and 5), a fake key packet at 8 whose +// bitstream says otherwise, duplicate timestamps (8 and 9), and a key packet sharing its timestamp with the delta +// packet right before it. All of it adheres to the GOP rule (no packets after a key frame have a timestamp less than +// any packet before the key frame). +const MAIN_PACKETS: PacketSpec[] = [ + { type: 'key', timestamp: 2 }, // 0 + { type: 'delta', timestamp: 0 }, // 1 + { type: 'delta', timestamp: 1 }, // 2 + { type: 'delta', timestamp: 3 }, // 3 + { type: 'key', timestamp: 6 }, // 4 + { type: 'delta', timestamp: 4 }, // 5 + { type: 'delta', timestamp: 5 }, // 6 + { type: 'delta', timestamp: 7 }, // 7 + { type: 'key', timestamp: 8, actualType: 'delta' }, // 8 + { type: 'delta', timestamp: 8 }, // 9 + { type: 'delta', timestamp: 9 }, // 10 + { type: 'key', timestamp: 9 }, // 11 + { type: 'delta', timestamp: 10 }, // 12 +]; + +// The very first key packet is fake here +const FAKE_FIRST_KEY_PACKETS: PacketSpec[] = [ + { type: 'key', timestamp: 0, actualType: 'delta' }, // 0 + { type: 'delta', timestamp: 1 }, // 1 + { type: 'key', timestamp: 2 }, // 2 + { type: 'delta', timestamp: 3 }, // 3 +]; + +const QUERY_TIMESTAMPS = [ + -Infinity, -1, 0, 0.5, 1, 2, 2.5, 3, 3.5, 4, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5, 9, 9.5, 10, 100, Infinity, +]; + +const PACKET_DATA_SIZE = 8; +const FULL_PACKET_SIZE = PACKET_SIZE_OVERHEAD + PACKET_DATA_SIZE; + +const BACKING_METHODS = ['getFirstPacket', 'getPacket', 'getKeyPacket', 'getNextPacket', 'getNextKeyPacket'] as const; +type BackingMethod = typeof BACKING_METHODS[number]; + +type PacketCacheTrackInfo = ReturnType; +type CacheEntry = NonNullable; + +let mainFile: ArrayBuffer; +let fakeFirstKeyFile: ArrayBuffer; +let emptyFile: ArrayBuffer; + +beforeAll(async () => { + mainFile = await createFile(MAIN_PACKETS); + fakeFirstKeyFile = await createFile(FAKE_FIRST_KEY_PACKETS); + emptyFile = await createFile([]); +}); + +test('Synthetic file matches the reference model', async () => { + using ctx = await setup(mainFile); + const reader = new PacketReader(ctx.track); + + for (const timestamp of QUERY_TIMESTAMPS) { + expectPacket(await reader.getAt(timestamp), MAIN_PACKETS, modelAt(MAIN_PACKETS, timestamp)); + expectPacket(await reader.getKeyAt(timestamp), MAIN_PACKETS, modelKeyAt(MAIN_PACKETS, timestamp)); + expectPacket( + await reader.getKeyAt(timestamp, { verifyKeyPackets: true }), + MAIN_PACKETS, + modelKeyAt(MAIN_PACKETS, timestamp, true), + ); + } + + for (let i = 0; i < ctx.packets.length; i++) { + expectPacket(ctx.packets[i], MAIN_PACKETS, i); + expectPacket(await reader.getNext(ctx.packets[i]!), MAIN_PACKETS, modelNext(MAIN_PACKETS, i)); + expectPacket(await reader.getNextKey(ctx.packets[i]!), MAIN_PACKETS, modelNextKey(MAIN_PACKETS, i)); + } +}); + +test('Sequential reading', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + + expect(cached.first()).toBe(undefined); + + let packet = await reader.getFirst(); + expectPacket(packet, MAIN_PACKETS, 0); + expectPacket(cached.first(), MAIN_PACKETS, 0); + + for (let i = 0; i < ctx.packets.length; i++) { + expect(cached.next(packet!)).toBe(undefined); + packet = await reader.getNext(packet!); + expectPacket(packet, MAIN_PACKETS, modelNext(MAIN_PACKETS, i)); + expectPacket(cached.next(ctx.packets[i]!), MAIN_PACKETS, modelNext(MAIN_PACKETS, i)); + } + + expectConsistentCache(ctx.cache); + + // Having read everything, every GOP is complete, so everything can be answered by the cache now + for (const timestamp of QUERY_TIMESTAMPS) { + expectPacket(cached.at(timestamp), MAIN_PACKETS, modelAt(MAIN_PACKETS, timestamp)); + expectPacket(cached.keyAt(timestamp), MAIN_PACKETS, modelKeyAt(MAIN_PACKETS, timestamp)); + } + + for (let i = 0; i < ctx.packets.length; i++) { + expectPacket(cached.nextKey(ctx.packets[i]!), MAIN_PACKETS, modelNextKey(MAIN_PACKETS, i)); + } + + // And the reader no longer needs the backing at all + const callCount = backing.totalCalls(); + for (const timestamp of QUERY_TIMESTAMPS) { + expectPacket(await reader.getAt(timestamp), MAIN_PACKETS, modelAt(MAIN_PACKETS, timestamp)); + expectPacket(await reader.getKeyAt(timestamp), MAIN_PACKETS, modelKeyAt(MAIN_PACKETS, timestamp)); + } + for (let i = 0; i < ctx.packets.length; i++) { + expectPacket(await reader.getNext(ctx.packets[i]!), MAIN_PACKETS, modelNext(MAIN_PACKETS, i)); + expectPacket(await reader.getNextKey(ctx.packets[i]!), MAIN_PACKETS, modelNextKey(MAIN_PACKETS, i)); + } + expectPacket(await reader.getFirst(), MAIN_PACKETS, 0); + expectPacket(await reader.getFirstKey(), MAIN_PACKETS, 0); + expect(backing.totalCalls()).toBe(callCount); +}); + +test('Seeking', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + + expect(cached.at(5.5)).toBe(undefined); + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expectPacket(cached.at(5.5), MAIN_PACKETS, 6); + expectPacket(cached.at(5), MAIN_PACKETS, 6); + expect(cached.at(5.6)).toBe(undefined); // Something could lie in between + expect(cached.at(4.5)).toBe(undefined); // No packet known here + expect(cached.keyAt(5.5)).toBe(undefined); // It's a delta packet + + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expect(backing.calls.getPacket).toBe(1); + + // The same packet, learned via a larger timestamp + expectPacket(await reader.getAt(5.8), MAIN_PACKETS, 6); + expectPacket(cached.at(5.7), MAIN_PACKETS, 6); + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expect(backing.calls.getPacket).toBe(2); + + // Retrieving a key packet also makes it available to key packet seeking + expectPacket(await reader.getAt(6.5), MAIN_PACKETS, 4); + expectPacket(cached.at(6.5), MAIN_PACKETS, 4); + expectPacket(cached.keyAt(6.5), MAIN_PACKETS, 4); + expect(cached.keyAt(6.6)).toBe(undefined); + + expect(await reader.getAt(-1)).toBe(null); + expect(cached.at(-1)).toBe(null); + expect(cached.at(-5)).toBe(null); + expect(cached.keyAt(-5)).toBe(null); + expect(cached.at(-0.5)).toBe(undefined); + + expectConsistentCache(ctx.cache); +}); + +test('Key packet seeking', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + + expect(cached.keyAt(5.5)).toBe(undefined); + expectPacket(await reader.getKeyAt(5.5), MAIN_PACKETS, 0); + expectPacket(cached.keyAt(5.5), MAIN_PACKETS, 0); + expectPacket(cached.keyAt(2), MAIN_PACKETS, 0); + expect(cached.keyAt(5.6)).toBe(undefined); + expect(cached.at(5.5)).toBe(undefined); // Doesn't tell us anything about non-key packets + + expectPacket(await reader.getKeyAt(5.5), MAIN_PACKETS, 0); + expect(backing.calls.getKeyPacket).toBe(1); + + // The cache walks back over delta packets to find the relevant key packet + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expectPacket(cached.keyAt(5.5), MAIN_PACKETS, 0); + + expect(await reader.getKeyAt(1)).toBe(null); + expect(cached.keyAt(1)).toBe(null); + expect(cached.keyAt(0.5)).toBe(null); + expect(cached.at(0.5)).toBe(undefined); // There are delta packets before the first key packet + + expectPacket(await reader.getKeyAt(100), MAIN_PACKETS, 11); + expectPacket(cached.keyAt(100), MAIN_PACKETS, 11); + expect(cached.keyAt(101)).toBe(undefined); +}); + +test('Infinite timestamps', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + + expect(cached.at(-Infinity)).toBe(null); + expect(cached.keyAt(-Infinity)).toBe(null); + expect(await reader.getAt(-Infinity)).toBe(null); + expect(await reader.getKeyAt(-Infinity)).toBe(null); + expect(backing.totalCalls()).toBe(0); + + // The last packet in presentation order + expect(cached.at(Infinity)).toBe(undefined); + expectPacket(await reader.getAt(Infinity), MAIN_PACKETS, 12); + expectPacket(cached.at(Infinity), MAIN_PACKETS, 12); + expectPacket(cached.at(1e300), MAIN_PACKETS, 12); + expectPacket(cached.at(10), MAIN_PACKETS, 12); + expect(cached.at(9.99)).toBe(undefined); + expectPacket(await reader.getAt(Infinity), MAIN_PACKETS, 12); + expect(backing.calls.getPacket).toBe(1); + + expect(cached.keyAt(Infinity)).toBe(undefined); + expectPacket(await reader.getKeyAt(Infinity), MAIN_PACKETS, 11); + expectPacket(cached.keyAt(Infinity), MAIN_PACKETS, 11); + expectPacket(cached.keyAt(9.5), MAIN_PACKETS, 11); + expectPacket(await reader.getKeyAt(Infinity), MAIN_PACKETS, 11); + expect(backing.calls.getKeyPacket).toBe(1); + + expectPacket( + await reader.getKeyAt(Infinity, { verifyKeyPackets: true }), + MAIN_PACKETS, + modelKeyAt(MAIN_PACKETS, Infinity, true), + ); + + // Concurrent requests for infinity are deduplicated too + using ctx2 = await setup(mainFile); + ctx2.backing.forceAsync = true; + const results = await Promise.all([ctx2.reader.getAt(Infinity), ctx2.reader.getAt(Infinity)]); + expectPacket(results[0], MAIN_PACKETS, 12); + expectPacket(results[1], MAIN_PACKETS, 12); + expect(ctx2.backing.calls.getPacket).toBe(1); + + // Reading up to the end also makes the cache aware of what's last + using ctx3 = await setup(mainFile); + expectPacket(await ctx3.reader.getKeyAt(9), MAIN_PACKETS, 11); + expect(await readSequentially(ctx3.reader, ctx3.packets[11]!, 2)).toBe(null); + expectPacket(ctx3.cached.at(Infinity), MAIN_PACKETS, 12); + expectPacket(ctx3.cached.keyAt(Infinity), MAIN_PACKETS, 11); +}); + +test('Open GOP', async () => { + using ctx = await setup(mainFile); + const { reader, cached } = ctx; + + // Read the first GOP, including the key packet that starts the next one + await readSequentially(reader, (await reader.getFirst())!, 4); + + // The next GOP has leading packets at 4 and 5, below its key packet at 6, so the first GOP's packets must not be + // considered valid beyond the first GOP's maximum timestamp + expectPacket(cached.at(3), MAIN_PACKETS, 3); + expect(cached.at(3.5)).toBe(undefined); + expect(cached.at(4.5)).toBe(undefined); + expectPacket(await reader.getAt(4.5), MAIN_PACKETS, 5); + + // Key packets on the other hand are monotonic, so key packet seeking may extend up to the next key packet + expectPacket(cached.keyAt(5.9), MAIN_PACKETS, 0); + expect(cached.keyAt(6)).toBe(undefined); // Another key packet might share its timestamp +}); + +test('Reverse GOP finalization', async () => { + using ctx = await setup(mainFile); + const { reader, cached } = ctx; + + // Read the second GOP first + expectPacket(await reader.getKeyAt(6), MAIN_PACKETS, 4); + await readSequentially(reader, ctx.packets[4]!, 4); + + expectPacket(cached.at(4.5), MAIN_PACKETS, 5); + expectPacket(cached.at(7), MAIN_PACKETS, 7); + expect(cached.at(7.5)).toBe(undefined); // The GOP's minimum timestamp after it is not yet known + expect(cached.at(3.5)).toBe(undefined); + + // Now the first one. Since the second GOP's minimum timestamp is known, the first GOP can extend up to it + await readSequentially(reader, (await reader.getFirst())!, 4); + expectPacket(cached.at(3.5), MAIN_PACKETS, 3); + expectPacket(cached.at(3.99), MAIN_PACKETS, 3); + + // Reading the third GOP extends the second one up to the third's minimum timestamp + await readSequentially(reader, ctx.packets[8]!, 3); + expectPacket(cached.at(7.5), MAIN_PACKETS, 7); + expectPacket(cached.at(8), MAIN_PACKETS, 9); + expectPacket(cached.at(8.5), MAIN_PACKETS, 9); + expect(cached.at(9)).toBe(undefined); // The fourth GOP is only known by its key packet + + expectConsistentCache(ctx.cache); +}); + +test('Mid-GOP seek joined later', async () => { + using ctx = await setup(mainFile); + const { reader, cached } = ctx; + + expectPacket(await reader.getAt(4.5), MAIN_PACKETS, 5); + await readSequentially(reader, ctx.packets[5]!, 1); + + // The chain starting at the GOP's key packet joins the existing one, but it doesn't reach the GOP's end yet + expectPacket(await reader.getKeyAt(6), MAIN_PACKETS, 4); + expectPacket(await reader.getNext(ctx.packets[4]!), MAIN_PACKETS, 5); + expect(cached.at(5.5)).toBe(undefined); + expectPacket(cached.keyAt(6), MAIN_PACKETS, 4); + expect(cached.keyAt(6.5)).toBe(undefined); + + // Now it does + await readSequentially(reader, ctx.packets[6]!, 2); + expectPacket(cached.at(5.5), MAIN_PACKETS, 6); + expectPacket(cached.at(7), MAIN_PACKETS, 7); + expectPacket(cached.keyAt(7.9), MAIN_PACKETS, 4); + expectPacket(cached.nextKey(ctx.packets[5]!), MAIN_PACKETS, 8); + + // The same, but this time the join completes the GOP right away + using ctx2 = await setup(mainFile); + expectPacket(await ctx2.reader.getAt(4.5), MAIN_PACKETS, 5); + await readSequentially(ctx2.reader, ctx2.packets[5]!, 3); + expect(ctx2.cached.at(5.5)).toBe(undefined); + + expectPacket(await ctx2.reader.getKeyAt(6), MAIN_PACKETS, 4); + expectPacket(await ctx2.reader.getNext(ctx2.packets[4]!), MAIN_PACKETS, 5); + expectPacket(ctx2.cached.at(5.5), MAIN_PACKETS, 6); + expectPacket(ctx2.cached.keyAt(7.9), MAIN_PACKETS, 4); +}); + +test('Duplicate timestamps', async () => { + using ctx = await setup(mainFile); + const { reader, cached, info } = ctx; + + // Of the two packets at 8, getAt returns the later one + expectPacket(await reader.getAt(8), MAIN_PACKETS, 9); + expectPacket(cached.at(8), MAIN_PACKETS, 9); + + // The earlier one gets inserted before it + expectPacket(await reader.getKeyAt(8.5), MAIN_PACKETS, 8); + expect(getCachedSequenceNumbers(info)).toEqual([8, 9]); + expectPacket(cached.at(8), MAIN_PACKETS, 9); + + // Retrieving the later one again doesn't duplicate it + expectPacket(await reader.getNext(ctx.packets[8]!), MAIN_PACKETS, 9); + expect(getCachedSequenceNumbers(info)).toEqual([8, 9]); + + expectPacket(await reader.getAt(9), MAIN_PACKETS, 11); + expectPacket(await reader.getNext(ctx.packets[9]!), MAIN_PACKETS, 10); + expectPacket(await reader.getNext(ctx.packets[10]!), MAIN_PACKETS, 11); + expect(getCachedSequenceNumbers(info)).toEqual([8, 9, 10, 11]); + expectPacket(cached.at(9), MAIN_PACKETS, 11); + + expectConsistentCache(ctx.cache); +}); + +test('Key packet sharing its timestamp with the preceding delta packet', async () => { + using ctx = await setup(mainFile); + const { reader, cached } = ctx; + + expectPacket(await reader.getKeyAt(8), MAIN_PACKETS, 8); + await readSequentially(reader, ctx.packets[8]!, 2); + + // The next key packet may still lie at 9, so key packet seeking can only go right up to it + expectPacket(cached.keyAt(8.99), MAIN_PACKETS, 8); + expect(cached.keyAt(9)).toBe(undefined); + expectPacket(await reader.getKeyAt(9), MAIN_PACKETS, 11); + + using ctx2 = await setup(mainFile); + expectPacket(await ctx2.reader.getKeyAt(8), MAIN_PACKETS, 8); + await readSequentially(ctx2.reader, ctx2.packets[8]!, 3); + expectPacket(ctx2.cached.keyAt(8.99), MAIN_PACKETS, 8); + expect(ctx2.cached.keyAt(9)).toBe(undefined); // The key packet at 9 has no validity of its own yet + + await readSequentially(ctx2.reader, ctx2.packets[11]!, 1); + expectPacket(ctx2.cached.keyAt(9), MAIN_PACKETS, 11); + expectPacket(ctx2.cached.keyAt(9.99), MAIN_PACKETS, 11); +}); + +test('Key packet seeking from regular seeking', async () => { + using ctx = await setup(mainFile); + const { reader, cached } = ctx; + + expectPacket(await reader.getKeyAt(6), MAIN_PACKETS, 4); + await readSequentially(reader, ctx.packets[4]!, 3); + expect(cached.keyAt(7.5)).toBe(undefined); // A key packet could still share packet 7's timestamp + + // Seeking to a packet of a known GOP rules out any key packet between the two + expectPacket(await reader.getAt(7.5), MAIN_PACKETS, 7); + expectPacket(cached.keyAt(7.5), MAIN_PACKETS, 4); + expect(cached.keyAt(7.6)).toBe(undefined); +}); + +test('Next key packets', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + + expectPacket(await reader.getFirst(), MAIN_PACKETS, 0); + expect(cached.nextKey(ctx.packets[0]!)).toBe(undefined); + expectPacket(await reader.getNextKey(ctx.packets[0]!), MAIN_PACKETS, 4); + expectPacket(cached.nextKey(ctx.packets[0]!), MAIN_PACKETS, 4); + expectPacket(await reader.getNextKey(ctx.packets[0]!), MAIN_PACKETS, 4); + expect(backing.calls.getNextKeyPacket).toBe(1); + + // Knowing the next key packet after a key packet bounds key packet seeking + expectPacket(cached.keyAt(5.9), MAIN_PACKETS, 0); + expect(cached.keyAt(6)).toBe(undefined); // The next key packet has no validity of its own yet + + // Not so for packets whose GOP is unknown + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expectPacket(await reader.getNextKey(ctx.packets[6]!), MAIN_PACKETS, 8); + expectPacket(cached.nextKey(ctx.packets[6]!), MAIN_PACKETS, 8); + expect(cached.keyAt(7.9)).toBe(undefined); + + // But for packets whose GOP is known + expectPacket(await reader.getKeyAt(8), MAIN_PACKETS, 8); + await readSequentially(reader, ctx.packets[8]!, 1); + expect(cached.keyAt(8.99)).toBe(undefined); + expectPacket(await reader.getNextKey(ctx.packets[9]!), MAIN_PACKETS, 11); + expectPacket(cached.keyAt(8.99), MAIN_PACKETS, 8); + + // There's no key packet after the last one, so it's valid indefinitely + expectPacket(await reader.getKeyAt(9.5), MAIN_PACKETS, 11); + expect(await reader.getNextKey(ctx.packets[11]!)).toBe(null); + expect(cached.nextKey(ctx.packets[11]!)).toBe(null); + expectPacket(cached.keyAt(1000), MAIN_PACKETS, 11); + expectPacket(await reader.getAt(10), MAIN_PACKETS, 12); + expect(await reader.getNextKey(ctx.packets[12]!)).toBe(null); + expect(cached.nextKey(ctx.packets[12]!)).toBe(null); +}); + +test('Uncached anchor packets', async () => { + using ctx = await setup(mainFile); + const { reader, cached, info } = ctx; + + // The packets handed to next() and nextKey() only serve as anchors. If they aren't cached, no link is made, but the + // returned packets are still remembered. + expectPacket(await reader.getNext(ctx.packets[5]!), MAIN_PACKETS, 6); + expect(cached.next(ctx.packets[5]!)).toBe(undefined); + expectPacket(await reader.getNextKey(ctx.packets[5]!), MAIN_PACKETS, 8); + expect(cached.nextKey(ctx.packets[5]!)).toBe(undefined); + expect(getCachedSequenceNumbers(info)).toEqual([6, 8]); + for (const entry of info.sortedEntries) { + expect(entry.next).toBe(undefined); + expect(entry.prev).toBe(undefined); + expect(entry.nextKey).toBe(undefined); + } + + // Once the anchor is cached, links are made, pointing to the cache's own instances + expectPacket(await reader.getAt(4.5), MAIN_PACKETS, 5); + expectPacket(await reader.getNext(ctx.packets[5]!), MAIN_PACKETS, 6); + expectPacket(cached.next(ctx.packets[5]!), MAIN_PACKETS, 6); + expect(info.entries.get(6)!.prev).toBe(info.entries.get(5)); + expect(info.entries.get(5)!.packet).not.toBe(ctx.packets[5]); + + expectConsistentCache(ctx.cache); +}); + +test('Next key packets from the next() chain', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + + expectPacket(await reader.getAt(4.5), MAIN_PACKETS, 5); + await readSequentially(reader, ctx.packets[5]!, 2); + expect(cached.nextKey(ctx.packets[5]!)).toBe(undefined); // Chain incomplete + + await readSequentially(reader, ctx.packets[7]!, 1); + expectPacket(cached.nextKey(ctx.packets[5]!), MAIN_PACKETS, 8); + expectPacket(await reader.getNextKey(ctx.packets[5]!), MAIN_PACKETS, 8); + expect(backing.calls.getNextKeyPacket).toBe(0); + + expectPacket(await reader.getAt(10), MAIN_PACKETS, 12); + await readSequentially(reader, ctx.packets[12]!, 1); + expect(cached.nextKey(ctx.packets[12]!)).toBe(null); +}); + +test('Next key packets from earlier packets in the GOP', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing, info } = ctx; + + // Delta packets following a packet share its next key packet + expectPacket(await reader.getAt(4.5), MAIN_PACKETS, 5); + expectPacket(await reader.getNextKey(ctx.packets[5]!), MAIN_PACKETS, 8); + await readSequentially(reader, ctx.packets[5]!, 2); + expectPacket(cached.nextKey(ctx.packets[7]!), MAIN_PACKETS, 8); + expectPacket(await reader.getNextKey(ctx.packets[6]!), MAIN_PACKETS, 8); + expect(backing.calls.getNextKeyPacket).toBe(1); + expect(info.entries.get(7)!.nextKey).toBe(info.entries.get(8)); // Remembered + + // Also when it's learned for the GOP's key packet + expectPacket(await reader.getFirst(), MAIN_PACKETS, 0); + expectPacket(await reader.getNextKey(ctx.packets[0]!), MAIN_PACKETS, 4); + await readSequentially(reader, ctx.packets[0]!, 2); + expectPacket(cached.nextKey(ctx.packets[2]!), MAIN_PACKETS, 4); + + // And when there is none + expectPacket(await reader.getKeyAt(9.5), MAIN_PACKETS, 11); + expect(await reader.getNextKey(ctx.packets[11]!)).toBe(null); + await readSequentially(reader, ctx.packets[11]!, 1); + expect(cached.nextKey(ctx.packets[12]!)).toBe(null); + + // A key packet in between starts a new GOP, so what came before it doesn't apply + await readSequentially(reader, ctx.packets[7]!, 2); + expect(cached.nextKey(ctx.packets[9]!)).toBe(undefined); + + // An evicted next key packet teaches us nothing + using ctx2 = await setup(mainFile, new PacketCache({ maxCacheSize: 2.5 * FULL_PACKET_SIZE, autoEvict: false })); + expectPacket(await ctx2.reader.getAt(4.5), MAIN_PACKETS, 5); + expectPacket(await ctx2.reader.getNextKey(ctx2.packets[5]!), MAIN_PACKETS, 8); + expectPacket(await ctx2.reader.getNext(ctx2.packets[5]!), MAIN_PACKETS, 6); + expectPacket(ctx2.cached.at(4.5), MAIN_PACKETS, 5); + + ctx2.cache.evict(); + expect(getLruOrder(ctx2.cache)).toEqual([5, 6]); + expect(ctx2.cached.nextKey(ctx2.packets[6]!)).toBe(undefined); + expectConsistentCache(ctx2.cache); + + expectConsistentCache(ctx.cache); +}); + +test('Next key packets from key packet seeking', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + + // The key packet that's valid indefinitely is the last one + expectPacket(await reader.getKeyAt(Infinity), MAIN_PACKETS, 11); + expect(cached.nextKey(ctx.packets[11]!)).toBe(null); + + // And so its whole GOP has no next key packet + await readSequentially(reader, ctx.packets[11]!, 1); + expect(await reader.getNextKey(ctx.packets[12]!)).toBe(null); + expect(backing.calls.getNextKeyPacket).toBe(0); +}); + +test('Next key packets learned before joining the GOP', async () => { + using ctx = await setup(mainFile); + const { reader, cached } = ctx; + + expectPacket(await reader.getAt(4.5), MAIN_PACKETS, 5); + expectPacket(await reader.getNextKey(ctx.packets[5]!), MAIN_PACKETS, 8); + expectPacket(await reader.getKeyAt(6), MAIN_PACKETS, 4); + expect(cached.keyAt(7.5)).toBe(undefined); + + // Joining packet 5 onto its GOP's key packet bounds key packet seeking + expectPacket(await reader.getNext(ctx.packets[4]!), MAIN_PACKETS, 5); + expectPacket(cached.keyAt(7.99), MAIN_PACKETS, 4); + expect(cached.keyAt(8)).toBe(undefined); + + // Same when there's no next key packet + expectPacket(await reader.getAt(10), MAIN_PACKETS, 12); + expect(await reader.getNextKey(ctx.packets[12]!)).toBe(null); + expectPacket(await reader.getKeyAt(9), MAIN_PACKETS, 11); + await readSequentially(reader, ctx.packets[11]!, 1); + expectPacket(cached.keyAt(1000), MAIN_PACKETS, 11); +}); + +test('Metadata-only packets', async () => { + using ctx = await setup(mainFile); + const { reader, cached, info } = ctx; + const metadataOnly = { metadataOnly: true }; + + const metadataPacket = await reader.getAt(5.5, metadataOnly); + expectPacket(metadataPacket, MAIN_PACKETS, 6); + expect(metadataPacket!.isMetadataOnly).toBe(true); + expect((cached.at(5.5, metadataOnly) as EncodedPacket).isMetadataOnly).toBe(true); + expect(cached.at(5.5)).toBe(undefined); // The data isn't known + + // The cache stores its own instance, which it upgrades in place once the data becomes known + const storedPacket = info.sortedEntries[0]!.packet!; + expect(storedPacket).not.toBe(metadataPacket); + + const fullPacket = await reader.getAt(5.5); + expectPacket(fullPacket, MAIN_PACKETS, 6); + expect(fullPacket!.isMetadataOnly).toBe(false); + expect(info.sortedEntries[0]!.packet).toBe(storedPacket); + expect(storedPacket.isMetadataOnly).toBe(false); + expect(metadataPacket!.isMetadataOnly).toBe(true); + expectPacket(cached.at(5.5), MAIN_PACKETS, 6); + expect((cached.at(5.5, metadataOnly) as EncodedPacket).isMetadataOnly).toBe(true); + + // A full packet is never downgraded + expectPacket(await reader.getAt(5.7, metadataOnly), MAIN_PACKETS, 6); + expect(info.sortedEntries[0]!.packet!.isMetadataOnly).toBe(false); + + // The upgrade is seen by everything referencing the packet + expectPacket(await reader.getFirst(metadataOnly), MAIN_PACKETS, 0); + expectPacket(await reader.getNext(ctx.packets[0]!, metadataOnly), MAIN_PACKETS, 1); + expectPacket(await reader.getAt(3, metadataOnly), MAIN_PACKETS, 3); + expectPacket(await reader.getNextKey(ctx.packets[3]!, metadataOnly), MAIN_PACKETS, 4); + expect(cached.first()).toBe(undefined); + expect(cached.next(ctx.packets[0]!)).toBe(undefined); + expect(cached.nextKey(ctx.packets[3]!)).toBe(undefined); + + expectPacket(await reader.getAt(2), MAIN_PACKETS, 0); + expectPacket(await reader.getAt(0), MAIN_PACKETS, 1); + expectPacket(await reader.getAt(6), MAIN_PACKETS, 4); + expectPacket(cached.first(), MAIN_PACKETS, 0); + expectPacket(cached.next(ctx.packets[0]!), MAIN_PACKETS, 1); + expectPacket(cached.nextKey(ctx.packets[3]!), MAIN_PACKETS, 4); + expect((cached.first() as EncodedPacket).isMetadataOnly).toBe(false); + + // A chain learned without data, then traversed with data + expectPacket(await reader.getNext(ctx.packets[4]!, metadataOnly), MAIN_PACKETS, 5); + expectPacket(await reader.getNext(ctx.packets[4]!), MAIN_PACKETS, 5); + + expectConsistentCache(ctx.cache); +}); + +test('Fresh packet instances', async () => { + using ctx = await setup(mainFile); + const { reader, info } = ctx; + + const a = (await reader.getAt(5.5))!; + const b = (await reader.getAt(5.5))!; + const c = (await reader.getAt(5.5))!; + expect(a).not.toBe(b); + expect(b).not.toBe(c); + expect(b).not.toBe(info.sortedEntries[0]!.packet); + expect(c).not.toBe(info.sortedEntries[0]!.packet); + expect(b.data).toBe(c.data); +}); + +test('Key packet verification', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + const verify = { verifyKeyPackets: true }; + + expectPacket(await reader.getFirst(verify), MAIN_PACKETS, 0); + expectPacket(cached.first(verify), MAIN_PACKETS, 0); + + expectPacket(await reader.getAt(3), MAIN_PACKETS, 3); + expectPacket(await reader.getNext(ctx.packets[3]!, verify), MAIN_PACKETS, 4); + expectPacket(cached.next(ctx.packets[3]!, verify), MAIN_PACKETS, 4); + + // The fake key packet comes back as what it actually is when verifying, and as what the container says otherwise + expectPacket(await reader.getAt(7), MAIN_PACKETS, 7); + expectPacket(await reader.getNext(ctx.packets[7]!, verify), MAIN_PACKETS, 8, 'delta'); + expectPacket(cached.next(ctx.packets[7]!, verify), MAIN_PACKETS, 8, 'delta'); + expectPacket(cached.next(ctx.packets[7]!), MAIN_PACKETS, 8, 'key'); + expectPacket(await reader.getNext(ctx.packets[7]!), MAIN_PACKETS, 8, 'key'); + + expectPacket(await reader.getAt(6.5, verify), MAIN_PACKETS, 4); + expectPacket(cached.at(6.5, verify), MAIN_PACKETS, 4); + + expectPacket(await reader.getAt(8.5, verify), MAIN_PACKETS, 9); + + // Verified key packet seeking skips over the fake key packet, and remembers everything it found along the way + const keyPacketCalls = backing.calls.getKeyPacket; + expectPacket(await reader.getKeyAt(8.5, verify), MAIN_PACKETS, 4); + expect(backing.calls.getKeyPacket).toBeGreaterThan(keyPacketCalls); + expectPacket(cached.keyAt(8.5, verify), MAIN_PACKETS, 8, 'delta'); + expectPacket(cached.keyAt(8.5), MAIN_PACKETS, 8); + + const callCount = backing.totalCalls(); + expectPacket(await reader.getKeyAt(8.5, verify), MAIN_PACKETS, 4); + expectPacket(reader.getKeyAt(8.5, verify) as EncodedPacket, MAIN_PACKETS, 4); // Fully synchronous + expectPacket(await reader.getKeyAt(8.5), MAIN_PACKETS, 8); + expect(backing.totalCalls()).toBe(callCount); + + // Same for next key packets + expectPacket(await reader.getAt(4.5), MAIN_PACKETS, 5); + expectPacket(await reader.getNextKey(ctx.packets[5]!, verify), MAIN_PACKETS, 11); + expectPacket(cached.nextKey(ctx.packets[5]!, verify), MAIN_PACKETS, 8, 'delta'); + const nextKeyCallCount = backing.totalCalls(); + expectPacket(await reader.getNextKey(ctx.packets[5]!, verify), MAIN_PACKETS, 11); + expectPacket(await reader.getNextKey(ctx.packets[5]!), MAIN_PACKETS, 8); + expect(backing.totalCalls()).toBe(nextKeyCallCount); + expectPacket(await reader.getNextKey(ctx.packets[11]!, verify), MAIN_PACKETS, null); +}); + +test('Key packet verification of cached packets', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + const verify = { verifyKeyPackets: true }; + + // Packets cached without verification get verified on demand + expectPacket(await reader.getKeyAt(6.5), MAIN_PACKETS, 4); + const verified = cached.keyAt(6.5, verify); + expect(verified).toBeInstanceOf(Promise); + expectPacket(await verified, MAIN_PACKETS, 4); + expectPacket(cached.keyAt(6.5, verify), MAIN_PACKETS, 4); + + expectPacket(await reader.getKeyAt(8.5), MAIN_PACKETS, 8); + const verifiedFake = cached.keyAt(8.5, verify); + expect(verifiedFake).toBeInstanceOf(Promise); + expectPacket(await verifiedFake, MAIN_PACKETS, 8, 'delta'); + + // The reader steps back from the fake key packet, partly using the cache + const callCount = backing.calls.getKeyPacket; + expectPacket(await reader.getKeyAt(8.5, verify), MAIN_PACKETS, 4); + expect(backing.calls.getKeyPacket).toBe(callCount + 1); + + // Same for next key packets + expectPacket(await reader.getAt(4.5), MAIN_PACKETS, 5); + expectPacket(await reader.getNextKey(ctx.packets[5]!), MAIN_PACKETS, 8); + expectPacket(await reader.getNextKey(ctx.packets[5]!, verify), MAIN_PACKETS, 11); + expectPacket(cached.nextKey(ctx.packets[8]!, verify), MAIN_PACKETS, 11); + + // The reader itself also verifies cached packets on demand + expectPacket(await reader.getKeyAt(2.5), MAIN_PACKETS, 0); + expectPacket(await reader.getKeyAt(2.5, verify), MAIN_PACKETS, 0); + + using ctx2 = await setup(mainFile); + expectPacket(await ctx2.reader.getFirst(), MAIN_PACKETS, 0); + expectPacket(await ctx2.reader.getNextKey(ctx2.packets[0]!), MAIN_PACKETS, 4); + expectPacket(await ctx2.reader.getNextKey(ctx2.packets[0]!, verify), MAIN_PACKETS, 4); + expect(ctx2.backing.calls.getNextKeyPacket).toBe(1); +}); + +test('Fake first key packet', async () => { + using ctx = await setup(fakeFirstKeyFile); + const { reader, cached } = ctx; + const verify = { verifyKeyPackets: true }; + + const first = (await reader.getFirst(verify))!; + expectPacket(first, FAKE_FIRST_KEY_PACKETS, 0, 'delta'); + expectPacket(cached.first(verify), FAKE_FIRST_KEY_PACKETS, 0, 'delta'); + expectPacket(cached.first(), FAKE_FIRST_KEY_PACKETS, 0, 'key'); + + // Continuing from the packet that claims to be a delta packet, the first packet still starts the first GOP + const second = (await reader.getNext(first))!; + expectPacket(second, FAKE_FIRST_KEY_PACKETS, 1); + expectPacket(await reader.getNext(second), FAKE_FIRST_KEY_PACKETS, 2); + expectPacket(cached.at(0.5), FAKE_FIRST_KEY_PACKETS, 0); + expectPacket(cached.at(1), FAKE_FIRST_KEY_PACKETS, 1); + expect(cached.at(1.5)).toBe(undefined); + expectPacket(await reader.getAt(0.5, verify), FAKE_FIRST_KEY_PACKETS, 0, 'delta'); + + using ctx2 = await setup(fakeFirstKeyFile); + expectPacket(await ctx2.reader.getAt(0.5, verify), FAKE_FIRST_KEY_PACKETS, 0, 'delta'); + expectPacket(ctx2.cached.at(0.5, verify), FAKE_FIRST_KEY_PACKETS, 0, 'delta'); + + expectPacket(await reader.getKeyAt(1, verify), FAKE_FIRST_KEY_PACKETS, null); + expectPacket(await reader.getKeyAt(3, verify), FAKE_FIRST_KEY_PACKETS, 2); + expectPacket(await reader.getFirstKey(verify), FAKE_FIRST_KEY_PACKETS, 2); + expect(ctx.packets).toHaveLength(FAKE_FIRST_KEY_PACKETS.length); +}); + +test('First GOP completed by learning the first packet', async () => { + using ctx = await setup(fakeFirstKeyFile); + const { reader, cached } = ctx; + const verify = { verifyKeyPackets: true }; + const metadataOnly = { metadataOnly: true }; + + // Continue from the first packet, which claims to be a delta packet, without knowing that it's the first packet + const first = (await reader.getAt(0.5, verify))!; + expectPacket(first, FAKE_FIRST_KEY_PACKETS, 0, 'delta'); + const second = (await reader.getNext(first))!; + expectPacket(await reader.getNext(second), FAKE_FIRST_KEY_PACKETS, 2); + expect(cached.at(1.5)).toBe(undefined); + + // Read the second GOP, but without data + expectPacket(await reader.getNext(ctx.packets[2]!, metadataOnly), FAKE_FIRST_KEY_PACKETS, 3); + expect(await reader.getNext(ctx.packets[3]!, metadataOnly)).toBe(null); + + // This completes the first GOP, which then extends right up to the second one + expectPacket(await reader.getFirst(), FAKE_FIRST_KEY_PACKETS, 0); + expectPacket(cached.at(1.5), FAKE_FIRST_KEY_PACKETS, 1); + expectPacket(cached.at(1.99), FAKE_FIRST_KEY_PACKETS, 1); +}); + +test('Start of the track', async () => { + using ctx = await setup(mainFile); + const { reader, cached } = ctx; + + // No key packet comes before the first one + expectPacket(await reader.getFirst(), MAIN_PACKETS, 0); + expect(cached.keyAt(1.99)).toBe(null); + expect(cached.at(-1)).toBe(undefined); + + // Once the first GOP is complete, no packet comes before its smallest timestamp + await readSequentially(reader, ctx.packets[0]!, 4); + expect(cached.at(-0.01)).toBe(null); + expectPacket(cached.at(0), MAIN_PACKETS, 1); + + // Muxers won't write tracks starting with a delta packet, so these get inserted directly. There, the key packet + // ending the first GOP is the first one. + const cache = new PacketCache(); + const info = cache._getTrackInfo(ctx.track); + insertChain(cache, info, [ + { type: 'delta', timestamp: 1 }, + { type: 'delta', timestamp: 0 }, + { type: 'key', timestamp: 3 }, + ]); + expect(cache._getKeyAt(info, 2.99, {})).toBe(null); + expect(cache._getAt(info, -0.01, {})).toBe(null); + + // And without any key packet, there's nothing to find at all + const keylessCache = new PacketCache(); + const keylessInfo = keylessCache._getTrackInfo(ctx.track); + insertChain(keylessCache, keylessInfo, [ + { type: 'delta', timestamp: 0 }, + { type: 'delta', timestamp: 1 }, + ]); + expect(keylessCache._getKeyAt(keylessInfo, Infinity, {})).toBe(null); +}); + +test('Empty track', async () => { + using ctx = await setup(emptyFile); + const { reader, cached, backing } = ctx; + + expect(await reader.getFirst()).toBe(null); + expect(cached.first()).toBe(null); + expect(await reader.getFirst()).toBe(null); + expect(await reader.getFirstKey()).toBe(null); + expect(backing.calls.getFirstPacket).toBe(1); + + expect(await reader.getAt(5)).toBe(null); + expect(cached.at(5)).toBe(null); + expect(cached.at(6)).toBe(undefined); + + expect(await reader.getKeyAt(6)).toBe(null); + expect(cached.keyAt(6)).toBe(null); + expect(await reader.getKeyAt(6)).toBe(null); + expect(backing.calls.getKeyPacket).toBe(1); +}); + +test('Provisional results', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing, info } = ctx; + backing.markProvisional = true; + + expectPacket(await reader.getFirst(), MAIN_PACKETS, 0); + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expectPacket(await reader.getKeyAt(5.5), MAIN_PACKETS, 0); + expectPacket(await reader.getNext(ctx.packets[0]!), MAIN_PACKETS, 1); + expectPacket(await reader.getNextKey(ctx.packets[0]!), MAIN_PACKETS, 4); + expectPacket(await reader.getFirst({ verifyKeyPackets: true }), MAIN_PACKETS, 0); + expectPacket(await reader.getKeyAt(8.5, { verifyKeyPackets: true }), MAIN_PACKETS, 4); + expectPacket(await reader.getNextKey(ctx.packets[5]!, { verifyKeyPackets: true }), MAIN_PACKETS, 11); + expect(await reader.getAt(-1)).toBe(null); + + expect(cached.first()).toBe(undefined); + expect(cached.at(5.5)).toBe(undefined); + expect(cached.keyAt(5.5)).toBe(undefined); + expect(cached.next(ctx.packets[0]!)).toBe(undefined); + expect(cached.nextKey(ctx.packets[0]!)).toBe(undefined); + expect(cached.at(-1)).toBe(undefined); + expect(info.sortedEntries).toHaveLength(0); + + // Asking again goes to the backing again + const callCount = backing.totalCalls(); + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expect(backing.totalCalls()).toBe(callCount + 1); + + backing.markProvisional = false; + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expectPacket(cached.at(5.5), MAIN_PACKETS, 6); +}); + +test('Concurrent identical requests', async () => { + using ctx = await setup(mainFile); + const { reader, backing, info } = ctx; + backing.forceAsync = true; + + const firsts = await Promise.all([reader.getFirst(), reader.getFirst()]); + expectPacket(firsts[0], MAIN_PACKETS, 0); + expectPacket(firsts[1], MAIN_PACKETS, 0); + expect(firsts[0]).not.toBe(firsts[1]); + expect(backing.calls.getFirstPacket).toBe(1); + + const ats = await Promise.all([reader.getAt(5.5), reader.getAt(5.5), reader.getAt(5.6)]); + expectPacket(ats[0], MAIN_PACKETS, 6); + expectPacket(ats[1], MAIN_PACKETS, 6); + expectPacket(ats[2], MAIN_PACKETS, 6); + expect(backing.calls.getPacket).toBe(2); // Different timestamps aren't deduplicated + + const keyAts = await Promise.all([reader.getKeyAt(6.5), reader.getKeyAt(6.5)]); + expectPacket(keyAts[0], MAIN_PACKETS, 4); + expectPacket(keyAts[1], MAIN_PACKETS, 4); + expect(backing.calls.getKeyPacket).toBe(1); + + // Make the anchors known first + expectPacket(await reader.getAt(7), MAIN_PACKETS, 7); + expectPacket(await reader.getAt(8), MAIN_PACKETS, 9); + + const nexts = await Promise.all([reader.getNext(ctx.packets[7]!), reader.getNext(ctx.packets[7]!)]); + expectPacket(nexts[0], MAIN_PACKETS, 8); + expectPacket(nexts[1], MAIN_PACKETS, 8); + expect(backing.calls.getNextPacket).toBe(1); + + const nextKeys = await Promise.all([reader.getNextKey(ctx.packets[9]!), reader.getNextKey(ctx.packets[9]!)]); + expectPacket(nextKeys[0], MAIN_PACKETS, 11); + expectPacket(nextKeys[1], MAIN_PACKETS, 11); + expect(backing.calls.getNextKeyPacket).toBe(1); + + // Different options aren't deduplicated + const mixed = await Promise.all([ + reader.getAt(9.5), + reader.getAt(9.5, { metadataOnly: true }), + reader.getAt(9.5), + ]); + expectPacket(mixed[0], MAIN_PACKETS, 11); + expectPacket(mixed[1], MAIN_PACKETS, 11); + expectPacket(mixed[2], MAIN_PACKETS, 11); + expect(mixed[1]!.isMetadataOnly).toBe(true); + expect(backing.calls.getPacket).toBe(6); + + const mixedFirsts = await Promise.all([ + reader.getFirst({ verifyKeyPackets: true }), + reader.getFirst({ metadataOnly: true }), + reader.getFirst({ verifyKeyPackets: true }), + ]); + expectPacket(mixedFirsts[0], MAIN_PACKETS, 0); + expectPacket(mixedFirsts[1], MAIN_PACKETS, 0); + expectPacket(mixedFirsts[2], MAIN_PACKETS, 0); + + expect(info.pendingFirstCalls).toHaveLength(0); + expect(info.pendingAtCalls.size).toBe(0); + expect(info.pendingKeyAtCalls.size).toBe(0); + expect(info.pendingNextCalls.size).toBe(0); + expect(info.pendingNextKeyCalls.size).toBe(0); +}); + +test('Concurrent identical requests after a failure', async () => { + using ctx = await setup(mainFile); + const { reader, backing, info } = ctx; + backing.forceAsync = true; + + backing.failNextCall = true; + const results = await Promise.allSettled([reader.getAt(5.5), reader.getAt(5.5)]); + expect(results[0].status).toBe('rejected'); + expect(results[1].status).toBe('fulfilled'); + expectPacket((results[1] as PromiseFulfilledResult).value, MAIN_PACKETS, 6); + expect(backing.calls.getPacket).toBe(2); + + backing.failNextCall = true; + const firsts = await Promise.allSettled([reader.getFirst(), reader.getFirst()]); + expect(firsts[0].status).toBe('rejected'); + expect(firsts[1].status).toBe('fulfilled'); + + backing.failNextCall = true; + const keyAts = await Promise.allSettled([reader.getKeyAt(6.5), reader.getKeyAt(6.5)]); + expect(keyAts[0].status).toBe('rejected'); + expect(keyAts[1].status).toBe('fulfilled'); + + backing.failNextCall = true; + const nexts = await Promise.allSettled([reader.getNext(ctx.packets[1]!), reader.getNext(ctx.packets[1]!)]); + expect(nexts[0].status).toBe('rejected'); + expect(nexts[1].status).toBe('fulfilled'); + + backing.failNextCall = true; + const nextKeys = await Promise.allSettled([ + reader.getNextKey(ctx.packets[1]!), + reader.getNextKey(ctx.packets[1]!), + ]); + expect(nextKeys[0].status).toBe('rejected'); + expect(nextKeys[1].status).toBe('fulfilled'); + + expect(info.pendingFirstCalls).toHaveLength(0); + expect(info.pendingAtCalls.size).toBe(0); + expect(info.pendingKeyAtCalls.size).toBe(0); + expect(info.pendingNextCalls.size).toBe(0); + expect(info.pendingNextKeyCalls.size).toBe(0); +}); + +test('Asynchronous backing', async () => { + using ctx = await setup(mainFile); + const { reader, cached, backing } = ctx; + backing.forceAsync = true; + const verify = { verifyKeyPackets: true }; + + const firstKey = reader.getFirstKey(); + expect(firstKey).toBeInstanceOf(Promise); + expectPacket(await firstKey, MAIN_PACKETS, 0); + expectPacket(cached.first(), MAIN_PACKETS, 0); + + // Stepping back from the fake key packet requires the time resolution, which is also asynchronous now + expectPacket(await reader.getKeyAt(8.5, verify), MAIN_PACKETS, 4); + + // A new reader, this time stepping back based on what's in the cache + const reader2 = new PacketReader(ctx.track, { cache: ctx.cache }); + const callCount = backing.totalCalls(); + expectPacket(await reader2.getKeyAt(8.5, verify), MAIN_PACKETS, 4); + expect(backing.totalCalls()).toBe(callCount); +}); + +test('Shared cache', async () => { + const cache = new PacketCache(); + using ctx1 = await setup(mainFile, cache); + using ctx2 = await setup(mainFile, cache); + + // Different tracks don't share their cache entries + expectPacket(await ctx1.reader.getAt(5.5), MAIN_PACKETS, 6); + expect(ctx2.cached.at(5.5)).toBe(undefined); + + // Different readers of the same track do + const reader = new PacketReader(ctx1.track, { cache }); + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expect(ctx1.backing.calls.getPacket).toBe(1); +}); + +test('Cache size', async () => { + using ctx = await setup(mainFile); + const { reader, cache } = ctx; + + expect(cache._cacheSize).toBe(0); + + // A packet takes up a fixed overhead plus its data + expectPacket(await reader.getAt(5.5, { metadataOnly: true }), MAIN_PACKETS, 6); + expect(cache._cacheSize).toBe(PACKET_SIZE_OVERHEAD); + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expect(cache._cacheSize).toBe(FULL_PACKET_SIZE); + + await readSequentially(reader, (await reader.getFirst())!, MAIN_PACKETS.length); + expect(cache._cacheSize).toBe(MAIN_PACKETS.length * FULL_PACKET_SIZE); + + // Alpha data counts too. The synthetic files have none, so let's hand it to the cache directly. + using ctx2 = await setup(mainFile); + const alphaPacket = ctx.packets[6]!.clone({ sideData: { alpha: new Uint8Array(100) } }); + ctx2.cache._insertAt(ctx2.info, 5.5, alphaPacket, undefined); + expect(ctx2.cache._cacheSize).toBe(FULL_PACKET_SIZE + 100); + + expectConsistentCache(cache); + expectConsistentCache(ctx2.cache); +}); + +test('Least recently used packets get evicted', async () => { + using ctx = await setup(mainFile, new PacketCache({ maxCacheSize: 3 * FULL_PACKET_SIZE })); + const { reader, cache, cached, backing } = ctx; + + expectPacket(await reader.getAt(2.5), MAIN_PACKETS, 0); + expectPacket(await reader.getAt(0.5), MAIN_PACKETS, 1); + expectPacket(await reader.getAt(1.5), MAIN_PACKETS, 2); + expect(getLruOrder(cache)).toEqual([2, 1, 0]); + + // Returning a packet makes it the most recently used one, no matter if it comes from the cache or the backing + expectPacket(await reader.getAt(2.5), MAIN_PACKETS, 0); + expect(backing.calls.getPacket).toBe(3); + expect(getLruOrder(cache)).toEqual([0, 2, 1]); + + // Exceeding the maximum size by more than 10% evicts packets until the cache is down to 90% of it + expectPacket(await reader.getAt(3.5), MAIN_PACKETS, 3); + expect(getLruOrder(cache)).toEqual([3, 0]); + expect(cached.at(0.5)).toBe(undefined); + expect(cached.at(1.5)).toBe(undefined); + expectPacket(cached.at(2.5), MAIN_PACKETS, 0); + expect(getLruOrder(cache)).toEqual([0, 3]); + + // Metadata-only retrievals count too + expectPacket(cached.at(3.5, { metadataOnly: true }), MAIN_PACKETS, 3); + expect(getLruOrder(cache)).toEqual([3, 0]); + + // Requests the cache can't satisfy don't + expectPacket(await reader.getAt(5.5, { metadataOnly: true }), MAIN_PACKETS, 6); + expect(getLruOrder(cache)).toEqual([6, 3, 0]); + expect(cached.at(5.5)).toBe(undefined); + expect(getLruOrder(cache)).toEqual([6, 3, 0]); + + expectConsistentCache(cache); + expect(cache._cacheSize).toBeLessThanOrEqual(3 * FULL_PACKET_SIZE); +}); + +test('Batched eviction', async () => { + using ctx = await setup(mainFile, new PacketCache({ maxCacheSize: 10 * FULL_PACKET_SIZE })); + const { reader, cache } = ctx; + + // Up to 10% above the maximum size is tolerated + await readSequentially(reader, (await reader.getFirst())!, 10); + expect(cache._cacheSize).toBe(11 * FULL_PACKET_SIZE); + + // Beyond that, the cache gets brought down to 90% of it in one go + await readSequentially(reader, ctx.packets[10]!, 1); + expect(getLruOrder(cache)).toEqual([11, 10, 9, 8, 7, 6, 5, 4, 3]); + expect(cache._cacheSize).toBe(9 * FULL_PACKET_SIZE); + expectConsistentCache(cache); + + // Explicit eviction behaves the same + using ctx2 = await setup(mainFile, new PacketCache({ maxCacheSize: 10 * FULL_PACKET_SIZE, autoEvict: false })); + await readSequentially(ctx2.reader, (await ctx2.reader.getFirst())!, 10); + ctx2.cache.evict(); + expect(ctx2.cache._cacheSize).toBe(11 * FULL_PACKET_SIZE); + + await readSequentially(ctx2.reader, ctx2.packets[10]!, 1); + ctx2.cache.evict(); + expect(getLruOrder(ctx2.cache)).toEqual([11, 10, 9, 8, 7, 6, 5, 4, 3]); + expectConsistentCache(ctx2.cache); +}); + +test('Eviction within a complete GOP', async () => { + // 90% of the maximum size leaves room for exactly four packets + using ctx = await setup(mainFile, new PacketCache({ maxCacheSize: 4.5 * FULL_PACKET_SIZE, autoEvict: false })); + const { reader, cache, cached } = ctx; + + expectPacket(await reader.getKeyAt(6), MAIN_PACKETS, 4); + await readSequentially(reader, ctx.packets[4]!, 4); + expect(cache._cacheSize).toBe(5 * FULL_PACKET_SIZE); // No automatic eviction + + // Use everything but the packet at 5 + expectPacket(cached.at(4.5), MAIN_PACKETS, 5); + expectPacket(cached.at(6.5), MAIN_PACKETS, 4); + expectPacket(cached.at(7), MAIN_PACKETS, 7); + expectPacket(cached.next(ctx.packets[7]!), MAIN_PACKETS, 8); + expect(getLruOrder(cache).at(-1)).toBe(6); + + cache.evict(); + expect(getLruOrder(cache)).toEqual([8, 7, 4, 5]); + expectConsistentCache(cache); + + // Lookups that would've landed on the evicted packet now land on the one before it, which must not answer them + expectPacket(cached.at(4.99), MAIN_PACKETS, 5); + expect(cached.at(5)).toBe(undefined); + expect(cached.at(5.5)).toBe(undefined); + expectPacket(cached.at(6), MAIN_PACKETS, 4); + expect(cached.next(ctx.packets[5]!)).toBe(undefined); + expect(cached.next(ctx.packets[6]!)).toBe(undefined); + + // Which is, of course, still answered correctly + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + expectPacket(cached.at(5.5), MAIN_PACKETS, 6); + + // Key packet seeking isn't affected by evicting delta packets + expectPacket(cached.keyAt(7.99), MAIN_PACKETS, 4); +}); + +test('Eviction of key packets', async () => { + // 90% of the maximum size leaves room for exactly three packets + using ctx = await setup(mainFile, new PacketCache({ maxCacheSize: 3.5 * FULL_PACKET_SIZE, autoEvict: false })); + const { reader, cache, cached } = ctx; + + expectPacket(await reader.getKeyAt(6), MAIN_PACKETS, 4); + await readSequentially(reader, ctx.packets[4]!, 4); + expectPacket(cached.keyAt(7.99), MAIN_PACKETS, 4); + + // Evict the key packet, leaving the packets of its GOP behind + expectPacket(cached.at(4.5), MAIN_PACKETS, 5); + expectPacket(cached.at(5.5), MAIN_PACKETS, 6); + expectPacket(cached.at(7), MAIN_PACKETS, 7); + cache.evict(); + expect(getLruOrder(cache)).toEqual([7, 6, 5]); + expectConsistentCache(cache); + + expect(cached.keyAt(7.99)).toBe(undefined); + expectPacket(cached.at(5.5), MAIN_PACKETS, 6); + + // The GOP's key packet isn't known anymore, so continuing the chain teaches us nothing about it + expectPacket(await reader.getNext(ctx.packets[7]!), MAIN_PACKETS, 8); + expectPacket(await reader.getNextKey(ctx.packets[6]!), MAIN_PACKETS, 8); + expect(cached.keyAt(7.99)).toBe(undefined); + expectConsistentCache(cache); + + cache.evict(); + expect(getLruOrder(cache)).toEqual([8, 6, 7]); + expectConsistentCache(cache); + + // A next key packet that's linked to directly can be evicted too + using ctx2 = await setup(mainFile, new PacketCache({ maxCacheSize: 1.5 * FULL_PACKET_SIZE, autoEvict: false })); + expectPacket(await ctx2.reader.getAt(4.5), MAIN_PACKETS, 5); + expectPacket(await ctx2.reader.getNextKey(ctx2.packets[5]!), MAIN_PACKETS, 8); + expectPacket(ctx2.cached.nextKey(ctx2.packets[5]!), MAIN_PACKETS, 8); + expectPacket(ctx2.cached.at(4.5), MAIN_PACKETS, 5); + + ctx2.cache.evict(); + expect(getLruOrder(ctx2.cache)).toEqual([5]); + expect(ctx2.cached.nextKey(ctx2.packets[5]!)).toBe(undefined); + expectPacket(await ctx2.reader.getNextKey(ctx2.packets[5]!), MAIN_PACKETS, 8); + expectConsistentCache(ctx2.cache); +}); + +test('Evicted anchor packets', async () => { + using ctx = await setup(mainFile, new PacketCache({ maxCacheSize: 0, autoEvict: false })); + const { reader, cache, info } = ctx; + + const first = (await reader.getFirst())!; + cache.evict(); + expect(info.sortedEntries).toHaveLength(0); + expect(info.first).toBe(undefined); + + expectPacket(await reader.getNext(first), MAIN_PACKETS, 1); + expect(getCachedSequenceNumbers(info)).toEqual([1]); + expect(info.entries.get(1)!.next).toBe(undefined); + expect(info.entries.get(1)!.prev).toBe(undefined); + expectConsistentCache(cache); +}); + +test('Eviction while verifying a key packet', async () => { + using ctx = await setup(mainFile, new PacketCache({ maxCacheSize: 0, autoEvict: false })); + const { reader, cache, cached, info } = ctx; + + expectPacket(await reader.getKeyAt(6.5), MAIN_PACKETS, 4); + const entry = info.entries.get(4)!; + const verified = cached.keyAt(6.5, { verifyKeyPackets: true }); + expect(verified).toBeInstanceOf(Promise); + cache.evict(); + + expectPacket(await verified, MAIN_PACKETS, 4); + expect(entry.packet).toBe(null); + expect(entry.determinedType).toBe(undefined); + expectConsistentCache(cache); +}); + +test('Results stay correct under eviction', async () => { + for (const maxCacheSize of [0, FULL_PACKET_SIZE, 3 * FULL_PACKET_SIZE, 6 * FULL_PACKET_SIZE, Infinity]) { + using ctx = await setup(mainFile, new PacketCache({ maxCacheSize })); + const { reader, cache } = ctx; + const random = createRandom(1234); + const pick = (items: T[]) => items[Math.floor(random() * items.length)]!; + + const anchors: EncodedPacket[] = []; + + for (let i = 0; i < 500; i++) { + const options = pick([{}, { metadataOnly: true }, { verifyKeyPackets: true }]); + const verified = !!options.verifyKeyPackets; + const operation = anchors.length === 0 ? 0 : Math.floor(random() * 5); + + let packet: EncodedPacket | null; + let expectedIndex: number | null; + let returnsKeyPackets = false; + + if (operation === 0) { + packet = await reader.getFirst(options); + expectedIndex = 0; + } else if (operation === 1) { + const timestamp = pick(QUERY_TIMESTAMPS); + packet = await reader.getAt(timestamp, options); + expectedIndex = modelAt(MAIN_PACKETS, timestamp); + } else if (operation === 2) { + const timestamp = pick(QUERY_TIMESTAMPS); + packet = await reader.getKeyAt(timestamp, options); + expectedIndex = modelKeyAt(MAIN_PACKETS, timestamp, verified); + returnsKeyPackets = true; + } else if (operation === 3) { + const anchor = pick(anchors); + packet = await reader.getNext(anchor, options); + expectedIndex = modelNext(MAIN_PACKETS, anchor.sequenceNumber); + } else { + const anchor = pick(anchors); + packet = await reader.getNextKey(anchor, options); + expectedIndex = modelNextKey(MAIN_PACKETS, anchor.sequenceNumber, verified); + returnsKeyPackets = true; + } + + // Verification may turn key packets into delta packets + const spec = expectedIndex !== null ? MAIN_PACKETS[expectedIndex]! : null; + const expectedType = spec && verified && !returnsKeyPackets ? spec.actualType ?? spec.type : undefined; + + expectPacket(packet, MAIN_PACKETS, expectedIndex, expectedType); + if (packet) { + expect(packet.isMetadataOnly).toBe(!!options.metadataOnly); + anchors.push(packet); + } + + expectConsistentCache(cache); + expect(cache._cacheSize).toBeLessThanOrEqual(1.1 * maxCacheSize); + } + } +}); + +test('Cache size across tracks', async () => { + // 90% of the maximum size leaves room for exactly three packets + const cache = new PacketCache({ maxCacheSize: 3.5 * FULL_PACKET_SIZE }); + using ctx1 = await setup(mainFile, cache); + using ctx2 = await setup(mainFile, cache); + + expectPacket(await ctx1.reader.getAt(2.5), MAIN_PACKETS, 0); + expectPacket(await ctx1.reader.getAt(0.5), MAIN_PACKETS, 1); + expectPacket(await ctx2.reader.getAt(2.5), MAIN_PACKETS, 0); + expectPacket(await ctx2.reader.getAt(0.5), MAIN_PACKETS, 1); + + expect(getCachedSequenceNumbers(ctx1.info)).toEqual([1]); + expect(getCachedSequenceNumbers(ctx2.info)).toEqual([1, 0]); + expectConsistentCache(cache); +}); + +test('Disabled eviction', async () => { + using ctx = await setup(mainFile, new PacketCache({ maxCacheSize: Infinity })); + const { reader, cache } = ctx; + + await readSequentially(reader, (await reader.getFirst())!, MAIN_PACKETS.length); + expectPacket(await reader.getAt(5.5), MAIN_PACKETS, 6); + cache.evict(); + expect(ctx.info.sortedEntries).toHaveLength(MAIN_PACKETS.length); + + // No LRU bookkeeping happens at all + expect(cache._lruHead).toBe(null); + expect(cache._cacheSize).toBe(0); + expectConsistentCache(cache); +}); + +test('Clearing', async () => { + using ctx = await setup(mainFile); + const { reader, cache, cached, info, backing } = ctx; + + await readSequentially(reader, (await reader.getFirst())!, MAIN_PACKETS.length); + expect(await reader.getAt(-1)).toBe(null); + expectPacket(await reader.getKeyAt(Infinity), MAIN_PACKETS, 11); + + cache.clear(); + expect(info.sortedEntries).toHaveLength(0); + expect(cache._cacheSize).toBe(0); + expect(cached.first()).toBe(undefined); + expect(cached.at(-1)).toBe(undefined); + expect(cached.keyAt(Infinity)).toBe(undefined); + expect(cached.next(ctx.packets[0]!)).toBe(undefined); + expectConsistentCache(cache); + + // Requests that are underway while clearing still end up in the cache + backing.forceAsync = true; + const pending = reader.getAt(5.5); + cache.clear(); + expectPacket(await pending, MAIN_PACKETS, 6); + expectPacket(cached.at(5.5), MAIN_PACKETS, 6); + expect(info.pendingAtCalls.size).toBe(0); + expectConsistentCache(cache); +}); + +const createFile = async (packets: PacketSpec[]) => { + const output = new Output({ + format: new Mp4OutputFormat(), + target: new BufferTarget(), + }); + const source = new EncodedVideoPacketSource('vp8'); + output.addVideoTrack(source, { + decoderConfig: { + codec: 'vp8', + codedWidth: 64, + codedHeight: 64, + }, + }); + + await output.start(); + + for (let i = 0; i < packets.length; i++) { + const spec = packets[i]!; + + const data = new Uint8Array(PACKET_DATA_SIZE); + data[0] = (spec.actualType ?? spec.type) === 'key' ? 0 : 1; // VP8 frame tag + data[1] = i; + + await source.add(new EncodedPacket(data, spec.type, spec.timestamp, 1)); + } + + await output.finalize(); + return output.target.buffer!; +}; + +const setup = async (file: ArrayBuffer, cache = new PacketCache()) => { + const { input, track } = await openFile(file); + + // All of the track's packets, read without any cache involvement, so that tests have packets to hand to getNext + // & co. This happens before instrumenting, so it doesn't count towards the backing calls. + const packets: EncodedPacket[] = []; + const plainReader = new PacketReader(track); + let packet = await plainReader.getFirst(); + while (packet) { + packets.push(packet); + packet = await plainReader.getNext(packet); + } + + const reader = new PacketReader(track, { cache }); + const info = cache._getTrackInfo(track); + const backing = instrumentBacking(track); + + return { + track, + packets, + cache, + reader, + info, + backing, + // Direct access to the cache, so we know whether something was answered by it or not + cached: { + first: (options: PacketRetrievalOptions = {}) => cache._getFirst(info, options), + at: (timestamp: number, options: PacketRetrievalOptions = {}) => cache._getAt(info, timestamp, options), + keyAt: (timestamp: number, options: PacketRetrievalOptions = {}) => { + return cache._getKeyAt(info, timestamp, options); + }, + next: (packet: EncodedPacket, options: PacketRetrievalOptions = {}) => { + return cache._getNext(info, packet, options); + }, + nextKey: (packet: EncodedPacket, options: PacketRetrievalOptions = {}) => { + return cache._getNextKey(info, packet, options); + }, + }, + [Symbol.dispose]: () => input.dispose(), + }; +}; + +const openFile = async (file: ArrayBuffer) => { + const input = new Input({ + source: new BufferSource(file), + formats: ALL_FORMATS, + }); + const track = (await input.getPrimaryVideoTrack())!; + + return { + input, + track, + }; +}; + +const instrumentBacking = (track: InputVideoTrack) => { + const instrumentation = { + calls: { + getFirstPacket: 0, + getPacket: 0, + getKeyPacket: 0, + getNextPacket: 0, + getNextKeyPacket: 0, + }, + markProvisional: false, + failNextCall: false, + // The MP4 backing answers synchronously for in-memory files, which would never let calls overlap + forceAsync: false, + totalCalls: () => Object.values(instrumentation.calls).reduce((a, b) => a + b, 0), + }; + + const getTimeResolution = track._backing.getTimeResolution.bind(track._backing); + track._backing.getTimeResolution = () => { + return instrumentation.forceAsync ? Promise.resolve(getTimeResolution()) : getTimeResolution(); + }; + + type Method = (res: ResultValue, ...args: unknown[]) => MaybeRelevantPromise; + const backing = track._backing as unknown as Record; + + for (const name of BACKING_METHODS) { + const original = backing[name].bind(backing); + + backing[name] = (res, ...args) => { + instrumentation.calls[name]++; + + if (instrumentation.failNextCall) { + instrumentation.failNextCall = false; + return Promise.reject(new Error('Backing failure')); + } + + const markProvisional = () => { + if (instrumentation.markProvisional) { + res.value.provisional = true; + } + }; + + if (instrumentation.forceAsync) { + return (async (): MaybeRelevantPromise => { + await Promise.resolve(); + + const promise = original(res, ...args); + await promise; + markProvisional(); + + return promise; + })(); + } + + const promise = original(res, ...args); + if (res.pending) { + return promise.then((value) => { + markProvisional(); + return value; + }); + } + + markProvisional(); + return promise; + }; + } + + return instrumentation; +}; + +const readSequentially = async (reader: PacketReader, start: EncodedPacket, count: number) => { + let packet: EncodedPacket | null = start; + for (let i = 0; i < count; i++) { + packet = await reader.getNext(packet!); + } + + return packet; +}; + +// Teaches the cache a whole track, from its first packet to its end +const insertChain = (cache: PacketCache, info: PacketCacheTrackInfo, specs: PacketSpec[]) => { + const packets = specs.map((spec, i) => { + return new EncodedPacket(new Uint8Array(PACKET_DATA_SIZE), spec.type, spec.timestamp, 1, i); + }); + + cache._insertFirst(info, packets[0]!, undefined); + for (let i = 0; i < packets.length; i++) { + cache._insertNext(info, packets[i]!, packets[i + 1] ?? null, undefined); + } +}; + +const expectPacket = ( + maybePacket: MaybePromise, + packets: PacketSpec[], + index: number | null, + type?: PacketType, +) => { + expect(isThenable(maybePacket)).toBe(false); + const packet = maybePacket as EncodedPacket | null | undefined; + + if (index === null) { + expect(packet).toBe(null); + return; + } + + expect(packet).toBeInstanceOf(EncodedPacket); + expect(packet!.sequenceNumber).toBe(index); + expect(packet!.timestamp).toBe(packets[index]!.timestamp); + expect(packet!.type).toBe(type ?? packets[index]!.type); + if (!packet!.isMetadataOnly) { + expect(packet!.data[1]).toBe(index); + } +}; + +// Checks the cache's internal invariants +const expectConsistentCache = (cache: PacketCache) => { + let entryCount = 0; + + for (const info of cache._trackInfos.values()) { + for (let i = 1; i < info.sortedEntries.length; i++) { + const a = info.sortedEntries[i - 1]!.packet!; + const b = info.sortedEntries[i]!.packet!; + + expect( + a.timestamp < b.timestamp + || (a.timestamp === b.timestamp && a.sequenceNumber < b.sequenceNumber), + ).toBe(true); + } + + const isCached = (entry: CacheEntry) => { + return entry.packet !== null && info.entries.get(entry.packet.sequenceNumber) === entry; + }; + + // Exactly the entries in the list are cached + expect(info.entries.size).toBe(info.sortedEntries.length); + for (const entry of info.sortedEntries) { + const packet = entry.packet!; + expect(packet).toBeInstanceOf(EncodedPacket); + expect(isCached(entry)).toBe(true); + expect(entry.trackInfo).toBe(info); + expect(entry.timestamp).toBe(packet.timestamp); + expect(entry.size).toBe(cache._evictionEnabled + ? PACKET_SIZE_OVERHEAD + packet.data.byteLength + (packet.sideData.alpha?.byteLength ?? 0) + : 0, + ); + + // Links only ever connect cached entries, and always go both ways + if (entry.next) { + expect(isCached(entry.next)).toBe(true); + expect(entry.next.prev).toBe(entry); + } + if (entry.prev) { + expect(isCached(entry.prev)).toBe(true); + expect(entry.prev.next).toBe(entry); + } + + // These may point to evicted entries, which then must be marked as such + for (const other of [entry.nextKey, entry.gopKey]) { + if (other) { + expect(other.packet === null || isCached(other)).toBe(true); + } + } + } + entryCount += info.entries.size; + + if (info.first) { + expect(isCached(info.first)).toBe(true); + expect(info.first.prev).toBe(null); + } + } + + // The LRU list contains every cached entry exactly once, and the sizes add up + let size = 0; + let count = 0; + let previous: CacheEntry | null = null; + for (let entry = cache._lruHead; entry; entry = entry.lruNext) { + expect(entry.lruPrev).toBe(previous); + expect(entry.trackInfo.entries.get(entry.packet!.sequenceNumber)).toBe(entry); + size += entry.size; + count++; + previous = entry; + } + expect(cache._lruTail).toBe(previous); + // Without eviction, there's no LRU bookkeeping at all + expect(count).toBe(cache._evictionEnabled ? entryCount : 0); + expect(cache._cacheSize).toBe(size); +}; + +// Sequence numbers of the cached packets, from most to least recently used +const getLruOrder = (cache: PacketCache) => { + const result: number[] = []; + for (let entry = cache._lruHead; entry; entry = entry.lruNext) { + result.push(entry.packet!.sequenceNumber); + } + + return result; +}; + +// Sequence numbers of the cached packets, in timestamp order +const getCachedSequenceNumbers = (info: PacketCacheTrackInfo) => { + return info.sortedEntries.map(x => x.packet!.sequenceNumber); +}; + +const modelKeyAt = (packets: PacketSpec[], timestamp: number, verified = false) => { + let result = modelAt(packets, timestamp, i => packets[i]!.type === 'key'); + while (verified && result !== null && packets[result]!.actualType === 'delta') { + result = modelAt(packets, packets[result]!.timestamp - 1e-6, i => packets[i]!.type === 'key'); + } + + return result; +}; + +const modelAt = (packets: PacketSpec[], timestamp: number, predicate = (_index: number) => true) => { + let result: number | null = null; + for (let i = 0; i < packets.length; i++) { + if ( + packets[i]!.timestamp <= timestamp + && predicate(i) + && (result === null || packets[i]!.timestamp >= packets[result]!.timestamp) + ) { + result = i; + } + } + + return result; +}; + +const modelNext = (packets: PacketSpec[], index: number) => { + return index + 1 < packets.length ? index + 1 : null; +}; + +const modelNextKey = (packets: PacketSpec[], index: number, verified = false) => { + for (let i = index + 1; i < packets.length; i++) { + if (packets[i]!.type === 'key' && (!verified || packets[i]!.actualType !== 'delta')) { + return i; + } + } + + return null; +}; + +// Deterministic pseudo-random numbers in [0, 1) (mulberry32), so that failures are reproducible +const createRandom = (seed: number) => { + let state = seed; + + return () => { + state = (state + 0x6d2b79f5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +}; diff --git a/test/node/packet-reading.test.ts b/test/node/packet-reading.test.ts new file mode 100644 index 000000000..b9c498f4d --- /dev/null +++ b/test/node/packet-reading.test.ts @@ -0,0 +1,382 @@ +import { expect, test } from 'vitest'; +import { Input, InputDisposedError } from '../../src/input.js'; +import { BufferSource, FilePathSource } from '../../src/source.js'; +import path from 'node:path'; +import fs from 'node:fs'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { promiseAllEnsureOrder } from '../../src/misc.js'; +import { EncodedPacket, PacketReader } from '../../src/packet.js'; + +const __dirname = new URL('.', import.meta.url).pathname; + +test('Packet reader', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/trim-buck-bunny.mov')), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const reader = new PacketReader(videoTrack); + + const packet1 = (await reader.getFirst())!; + expect(packet1.timestamp).toBe(0); + + const packet3 = (await reader.getNext(packet1))!; + expect(packet3.sequenceNumber).toBeGreaterThan(packet1.sequenceNumber); + + const packet4 = (await reader.getNextKey(packet1))!; + expect(packet4.sequenceNumber).toBeGreaterThan(packet3.sequenceNumber); + expect(packet4.type).toBe('key'); + + const packet5 = (await reader.getNext(packet3))!; + expect(packet5.sequenceNumber).toBeGreaterThan(packet3.sequenceNumber); + expect(packet5.sequenceNumber).toBeLessThan(packet4.sequenceNumber); + + const packet6 = (await reader.getAt(2.4))!; + expect(packet6.timestamp).toBeGreaterThan(2); + expect(packet6.timestamp).toBeLessThanOrEqual(2.4); +}); + +test('Packet reader rejecting packet inputs it didn\'t create', async () => { + const filePath = path.join(__dirname, '../public/trim-buck-bunny.mov'); + using input = new Input({ + source: new FilePathSource(filePath), + formats: ALL_FORMATS, + }); + using otherInput = new Input({ + source: new FilePathSource(filePath), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const reader = new PacketReader(videoTrack); + const first = (await reader.getFirst())!; + expect(await reader.getNext(first)).not.toBe(null); + + // Clones, synthetic packets and packets of other tracks could carry arbitrary data, so they're rejected + expect(() => reader.getNext(first.clone())).toThrow('Packet was not created from this track.'); + expect(() => reader.getNextKey(first.clone({ sequenceNumber: 5 }))).toThrow( + 'Packet was not created from this track.', + ); + + const synthetic = new EncodedPacket(first.data, 'key', 0, 0.04, first.sequenceNumber); + expect(() => reader.getNext(synthetic)).toThrow('Packet was not created from this track.'); + + const otherReader = new PacketReader((await otherInput.getPrimaryVideoTrack())!); + const otherFirst = (await otherReader.getFirst())!; + expect(() => reader.getNext(otherFirst)).toThrow('Packet was not created from this track.'); + expect(() => otherReader.getNext(first)).toThrow('Packet was not created from this track.'); +}); + +test('Packet reading throwing after Input disposal', async () => { + const input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/trim-buck-bunny.mov'))), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const reader = new PacketReader(videoTrack); + + const first = await reader.getFirst(); + + input.dispose(); + + expect(() => reader.getFirst()).toThrow(InputDisposedError); + expect(() => reader.getAt(0)).toThrow(InputDisposedError); + expect(() => reader.getKeyAt(0)).toThrow(InputDisposedError); + expect(() => reader.getNext(first!)).toThrow(InputDisposedError); + expect(() => reader.getNextKey(first!)).toThrow(InputDisposedError); +}); + +test('Packet cursor seeking', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/trim-buck-bunny.mov')), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const cursor = new PacketCursor(videoTrack); + + expect(cursor.current).toBe(null); + + const packet1 = (await cursor.seekToFirst())!; + expect(packet1).not.toBe(null); + expect(packet1).toBe(cursor.current); + expect(packet1.timestamp).toBe(0); + + const packet2 = (await cursor.seekTo(0.01))!; + expect(packet1.sequenceNumber).toBe(packet2.sequenceNumber); // Same packet + + const packet3 = (await cursor.seekTo(0.1))!; + expect(packet3).toBe(cursor.current); + expect(packet3.timestamp).toBeGreaterThan(0); + expect(packet3.sequenceNumber).toBeGreaterThan(packet1.sequenceNumber); + + const packet4 = (await cursor.seekToKey(0.1))!; + expect(packet4).toBe(cursor.current); + expect(packet4.sequenceNumber).toBe(packet1.sequenceNumber); + + const packet5 = (await cursor.seekTo(Infinity))!; + expect(packet5).toBe(cursor.current); + expect(packet5.timestamp).toBe(5); + + const packet6 = (await cursor.seekTo(-Infinity))!; + expect(packet6).toBe(cursor.current); + expect(packet6).toBe(null); +}); + +test('Packet cursor iteration', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/trim-buck-bunny.mov')), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const cursor = new PacketCursor(videoTrack); + + const packet0 = (await cursor.seekToFirst())!; + expect(cursor.current!.timestamp).toBe(0); + + const packet1 = (await cursor.next())!; + expect(packet1.sequenceNumber).toBeGreaterThan(packet0.sequenceNumber); + expect(packet1).toBe(cursor.current); + + const packet2 = (await cursor.next())!; + expect(packet2.sequenceNumber).toBeGreaterThan(packet1.sequenceNumber); + expect(packet2).toBe(cursor.current); + + const packet3 = (await cursor.nextKey())!; + expect(packet3.sequenceNumber).toBeGreaterThan(packet2.sequenceNumber); + expect(packet3.type).toBe('key'); + expect(packet3).toBe(cursor.current); + + await cursor.seekTo(Infinity); + expect(cursor.current).not.toBe(null); + + const packet4 = await cursor.next(); + expect(packet4).toBe(null); + expect(packet4).toBe(cursor.current); + + const packet5 = await cursor.next(); + expect(packet5).toBe(null); + + await cursor.seekTo(-Infinity); + expect(cursor.current).toBe(null); + + const packet6 = (await cursor.next())!; + expect(packet6.sequenceNumber).toBe(packet0.sequenceNumber); + expect(packet6).toBe(cursor.current); + + await cursor.seekTo(-Infinity); + expect(cursor.current).toBe(null); + + const packet7 = (await cursor.next())!; + expect(packet7.sequenceNumber).toBe(packet0.sequenceNumber); + expect(packet7).toBe(cursor.current); + + const packet8 = (await cursor.next())!; + expect(packet8.sequenceNumber).toBeGreaterThan(packet7.sequenceNumber); + expect(packet8).toBe(cursor.current); + + await cursor.seekToFirst(); + + let total = 0; + let lastSeqNum = -Infinity; + for await (const packet of cursor) { + if (total === 0) { + expect(packet.sequenceNumber).toBe(packet0.sequenceNumber); + } + + expect(packet.sequenceNumber).toBeGreaterThan(lastSeqNum); + + lastSeqNum = packet.sequenceNumber; + total++; + } + + expect(total).toBe(121); + + for await (const _ of cursor) { + throw new Error('Unreachable'); + } + + total = 0; + await cursor.seekTo(1); + for await (const _ of cursor) { + total++; + } + + expect(total).toBe(97); + + total = 0; + await cursor.seekToFirst(); + await cursor.iterate(() => void total++); + + expect(total).toBe(121); + expect(cursor.current).toBe(null); + + total = 0; + await cursor.seekToFirst(); + await cursor.iterate((packet) => { + if (packet.timestamp === 1) { + return false; + } + + total++; + }); + + expect(total).toBe(24); + expect(cursor.current!.timestamp).toBe(1); + + await cursor.seekTo(-Infinity); + total = 0; + for await (const _ of cursor) total++; + + expect(total).toBe(121); + + await cursor.seekTo(-Infinity); + total = 0; + await cursor.iterate(() => void total++); + + expect(total).toBe(121); + + const cursor2 = new PacketCursor(videoTrack); + const packet9 = (await cursor2.next())!; // Without any prior seeks + expect(packet9.sequenceNumber).toBe(packet0.sequenceNumber); +}); + +test('Synchronous packet reading', async () => { + using input = new Input({ + source: new BufferSource(fs.readFileSync(path.join(__dirname, '../public/trim-buck-bunny.mov'))), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const reader = new PacketReader(videoTrack); + const cursor = new PacketCursor(videoTrack); + + expect(reader.getFirst()).not.toBeInstanceOf(Promise); + + expect(cursor.seekToFirst()).not.toBeInstanceOf(Promise); + expect(cursor.seekTo(0.1)).not.toBeInstanceOf(Promise); + expect(cursor.seekToKey(0.1)).not.toBeInstanceOf(Promise); + expect(cursor.seekTo(2)).not.toBeInstanceOf(Promise); + expect(cursor.seekTo(Infinity)).not.toBeInstanceOf(Promise); + expect(cursor.seekTo(-Infinity)).not.toBeInstanceOf(Promise); + + void cursor.seekToFirst(); + + expect(cursor.next()).not.toBeInstanceOf(Promise); +}); + +test('Command queuing', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/trim-buck-bunny.mov'), { + maxCacheSize: 0, // So all commands return promises + }), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const cursor = new PacketCursor(videoTrack); + + expect(cursor.waitUntilIdle()).toBe(null); + + const commands = [ + cursor.seekToFirst(), + cursor.next(), + cursor.next(), + cursor.seekTo(2.4), + cursor.next(), + cursor.nextKey(), + cursor.waitUntilIdle()!.then(() => cursor.current), + cursor.nextKey(), + cursor.nextKey(), + cursor.next(), + cursor.seekTo(Infinity), + cursor.seekTo(-Infinity), + cursor.seekToKey(2.4), + ]; + + expect(commands.every(x => x instanceof Promise)).toBe(true); + + const resolved = await promiseAllEnsureOrder(commands); + + expect(resolved[0]!.timestamp).toBe(0); + + expect(resolved[1]!.sequenceNumber).toBeGreaterThan(resolved[0]!.sequenceNumber); + + expect(resolved[2]!.sequenceNumber).toBeGreaterThan(resolved[1]!.sequenceNumber); + + expect(resolved[3]!.timestamp).toBeGreaterThan(2); + expect(resolved[3]!.timestamp).toBeLessThanOrEqual(2.4); + + expect(resolved[4]!.sequenceNumber).toBeGreaterThan(resolved[3]!.sequenceNumber); + + expect(resolved[5]!.timestamp).toBe(3); + + expect(resolved[6]!.sequenceNumber).toBe(resolved[5]!.sequenceNumber); + + expect(resolved[7]!.timestamp).toBe(4); + + expect(resolved[8]!.timestamp).toBe(5); + + expect(resolved[9]).toBe(null); + + expect(resolved[10]!.sequenceNumber).toBe(resolved[8]!.sequenceNumber); + + expect(resolved[11]).toBe(null); + + expect(resolved[12]!.timestamp).toBe(2); + expect(resolved[12]!.type).toBe('key'); + + void cursor.seekTo(1); + await cursor.iterate((packet) => { + expect(packet.timestamp).toBe(1); + return false; + }); + + void cursor.seekTo(3); + for await (const packet of cursor) { + expect(packet.timestamp).toBe(3); + break; + } +}); + +test('verifyKeyPackets with faultily-labeled key frames', async () => { + using input = new Input({ + source: new FilePathSource(path.join(__dirname, '../public/fucked-keyframes.mp4')), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const reader = new PacketReader(videoTrack); + + const firstPacket = (await reader.getFirst())!; + expect(firstPacket.type).toBe('key'); + + const fakeKeyPacket = (await reader.getNextKey(firstPacket))!; + expect(fakeKeyPacket).not.toBe(null); + expect(fakeKeyPacket.type).toBe('key'); // Metadata says it's a key frame + expect(fakeKeyPacket.sequenceNumber).toBeGreaterThan(firstPacket.sequenceNumber); + + const verifiedPacket = (await reader.getAt(fakeKeyPacket.timestamp, { verifyKeyPackets: true }))!; + expect(verifiedPacket.sequenceNumber).toBe(fakeKeyPacket.sequenceNumber); + expect(verifiedPacket.type).toBe('delta'); // After verification, it's actually a delta frame + + const unverifiedKeyAt = (await reader.getKeyAt(fakeKeyPacket.timestamp))!; + expect(unverifiedKeyAt.sequenceNumber).toBe(fakeKeyPacket.sequenceNumber); + expect(unverifiedKeyAt.type).toBe('key'); + + const verifiedKeyAt = (await reader.getKeyAt(fakeKeyPacket.timestamp, { verifyKeyPackets: true }))!; + expect(verifiedKeyAt.sequenceNumber).toBe(firstPacket.sequenceNumber); + expect(verifiedKeyAt.type).toBe('key'); + + const unverifiedNextKey = (await reader.getNextKey(firstPacket))!; + expect(unverifiedNextKey).not.toBe(null); + expect(unverifiedNextKey.type).toBe('key'); + expect(unverifiedNextKey.sequenceNumber).toBe(fakeKeyPacket.sequenceNumber); + + const verifiedNextKey = await reader.getNextKey(firstPacket, { verifyKeyPackets: true }); + expect(verifiedNextKey).toBe(null); +}); diff --git a/test/node/pcm.test.ts b/test/node/pcm.test.ts index 5286bc13a..7816ae529 100644 --- a/test/node/pcm.test.ts +++ b/test/node/pcm.test.ts @@ -6,7 +6,7 @@ import { Output } from '../../src/output.js'; import { BufferTarget } from '../../src/target.js'; import { WavOutputFormat } from '../../src/output-format.js'; import { AudioSampleSource } from '../../src/media-source.js'; -import { AudioSampleSink } from '../../src/media-sink.js'; +import { AudioSampleCursor } from '../../src/cursors.js'; import { AudioSample } from '../../src/sample.js'; import { PcmAudioCodec } from '../../src/codec.js'; import { toUint8Array, uint8ArraysAreEqual } from '../../src/misc.js'; @@ -106,11 +106,11 @@ const testEncodeDecodeRoundtrip = async (codec: PcmAudioCodec, format: SampleFor const track = (await input.getPrimaryAudioTrack())!; expect(await track.getCodec()).toBe(codec); - const sink = new AudioSampleSink(track); + await using cursor = new AudioSampleCursor(track); const decoded = data.slice().fill(0); let frameOffset = 0; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { sample.copyTo(decoded.subarray(frameOffset), { format, planeIndex: 0, @@ -219,11 +219,11 @@ const expectG711FileRoundtrip = async (codec: 'ulaw' | 'alaw', decodeCodeword: ( const track = (await input.getPrimaryAudioTrack())!; expect(await track.getCodec()).toBe(codec); - const sink = new AudioSampleSink(track); + await using cursor = new AudioSampleCursor(track); const decoded = new Int16Array(data.length); let frameOffset = 0; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { sample.copyTo(decoded.subarray(frameOffset), { format: 's16', planeIndex: 0, diff --git a/test/node/prores.test.ts b/test/node/prores.test.ts index 84ce42061..09f8d9b00 100644 --- a/test/node/prores.test.ts +++ b/test/node/prores.test.ts @@ -7,12 +7,13 @@ import { Output } from '../../src/output.js'; import { MkvOutputFormat, MovOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { Conversion } from '../../src/conversion.js'; -import { EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; +import { VideoSampleCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; import { assert } from '../../src/misc.js'; const SAMPLE_URL = 'https://pub-1ee78aacb848486482b20a72b55b3121.r2.dev/turbores-sample.mov'; -test.concurrent('ProRes MOV file reading', { timeout: 20_000 }, async () => { +test('ProRes MOV file reading', { timeout: 20_000 }, async () => { using input = new Input({ source: new UrlSource(SAMPLE_URL), formats: ALL_FORMATS, @@ -28,7 +29,7 @@ test.concurrent('ProRes MOV file reading', { timeout: 20_000 }, async () => { expect(decoderConfig.description).toBeUndefined(); }); -test.concurrent('ProRes transmuxing into MOV', { timeout: 20_000 }, async () => { +test('ProRes transmuxing into MOV', { timeout: 20_000 }, async () => { using input = new Input({ source: new UrlSource(SAMPLE_URL), formats: ALL_FORMATS, @@ -65,7 +66,7 @@ test.concurrent('ProRes transmuxing into MOV', { timeout: 20_000 }, async () => expect(decoderConfig.description).toBeUndefined(); }); -test.concurrent('ProRes transmuxing into MKV', { timeout: 20_000 }, async () => { +test('ProRes transmuxing into MKV', { timeout: 20_000 }, async () => { using input = new Input({ source: new UrlSource(SAMPLE_URL), formats: ALL_FORMATS, @@ -105,8 +106,8 @@ test.concurrent('ProRes transmuxing into MKV', { timeout: 20_000 }, async () => expect(decoderConfig.codec).toBe('apch'); expect(decoderConfig.description).toBeUndefined(); - const sink = new EncodedPacketSink(videoTrack); - const firstPacket = await sink.getFirstPacket(); + const reader = new PacketReader(videoTrack); + const firstPacket = await reader.getFirst(); assert(firstPacket); // The frame container atom headers are added back when reading out the packets @@ -142,8 +143,8 @@ test('ProRes decoding', { timeout: 20_000 }, async () => { const videoTrack = (await input.getPrimaryVideoTrack())!; const firstTimestamp = await videoTrack.getFirstTimestamp(); - const sink = new VideoSampleSink(videoTrack); - using sample = await sink.getSample(firstTimestamp); + await using cursor = new VideoSampleCursor(videoTrack); + const sample = await cursor.seekToFirst(); assert(sample); expect(sample.timestamp).toBe(firstTimestamp); diff --git a/test/node/read-mp4.test.ts b/test/node/read-mp4.test.ts index 4f153838c..298ea863b 100644 --- a/test/node/read-mp4.test.ts +++ b/test/node/read-mp4.test.ts @@ -6,7 +6,6 @@ import { BufferSource, BufferTarget, EncodedPacket, - EncodedPacketSink, EncodedVideoPacketSource, FilePathSource, Input, @@ -14,6 +13,8 @@ import { MovOutputFormat, Mp4OutputFormat, Output, + PacketCursor, + PacketReader, } from '../../src/index.js'; import { assert, toUint8Array } from '../../src/misc.js'; @@ -33,12 +34,12 @@ test('Should be able to get packets from a .MP4 file', async () => { const track = await input.getPrimaryVideoTrack(); if (!track) throw new Error('No video track found'); - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); let samples = 0; const timestamps: number[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); samples++; } @@ -158,8 +159,8 @@ test('Annex B', async () => { expect(decoderConfig.codec).toBe('avc1.424028'); expect(decoderConfig.description).toBeUndefined(); - const sink = new EncodedPacketSink(track); - const firstPacket = (await sink.getFirstPacket())!; + const packetReader = new PacketReader(track); + const firstPacket = (await packetReader.getFirst())!; expect([...firstPacket.data.slice(0, 4)]).toEqual([0, 0, 0, 1]); }); diff --git a/test/node/read-wav.test.ts b/test/node/read-wav.test.ts index fb524ce83..06a6d3cc7 100644 --- a/test/node/read-wav.test.ts +++ b/test/node/read-wav.test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { ALL_FORMATS, EncodedPacketSink, Input, FilePathSource } from '../../src/index.js'; +import { ALL_FORMATS, Input, FilePathSource, PacketCursor } from '../../src/index.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -17,9 +17,9 @@ test('Should handle WAV file with oversized ID3 chunk', async () => { throw new Error('No audio track found'); } - const sink = new EncodedPacketSink(track); + const cursor = new PacketCursor(track); - for await (const packet of sink.packets()) { + for await (const packet of cursor) { void packet; } }); diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index 5422a91ef..b3d8ebb75 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -4,7 +4,7 @@ import { Input } from '../../src/input.js'; import { BufferSource, FilePathSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { assert, last, toUint8Array } from '../../src/misc.js'; -import { AudioSampleSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; +import { AudioSampleCursor, PacketCursor, VideoSampleCursor } from '../../src/cursors.js'; import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js'; import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js'; import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js'; @@ -72,9 +72,9 @@ describe('Video', async () => { await decoder.init(); - const sink = new EncodedPacketSink(videoTrack); + const cursor = new PacketCursor(videoTrack); let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { packetTimestamps.push(packet.timestamp); packetTimestamps.sort((a, b) => a - b); // Because of B-frames await decoder.decode(packet); @@ -93,7 +93,8 @@ describe('Video', async () => { packetTimestamps.length = 0; packetCount = 0; - for await (const packet of sink.packets((await sink.getKeyPacket(2))!)) { + await cursor.seekToKey(2); + for await (const packet of cursor) { packetTimestamps.push(packet.timestamp); packetTimestamps.sort((a, b) => a - b); // Because of B-frames await decoder.decode(packet); @@ -659,8 +660,8 @@ describe('Video', async () => { const videoTrack = (await input.getPrimaryVideoTrack())!; expect(await videoTrack.canBeTransparent()).toBe(true); - const sink = new VideoSampleSink(videoTrack); - using sample = (await sink.getSample(0))!; + await using cursor = new VideoSampleCursor(videoTrack); + const sample = (await cursor.seekTo(0))!; expect(sample.format).toBe('I444AP12'); @@ -685,7 +686,7 @@ describe('Video', async () => { }); test('Non-square pixels encode and decode #1', async () => { - await encodeDecodeTest('avc', { displayWidth: 2 * 1280, displayHeight: 720 }, async (packet, meta, i) => { + await encodeDecodeTest('avc', { displayWidth: 2 * 1280, displayHeight: 720 }, async (_, meta, i) => { if (i === 0) { expect(meta.decoderConfig).toBeDefined(); expect(meta.decoderConfig!.codedWidth).toBe(1280); @@ -702,7 +703,7 @@ describe('Video', async () => { }); test('Non-square pixels encode and decode #2', async () => { - await encodeDecodeTest('avc', { displayWidth: 1280, displayHeight: 2 * 720 }, async (packet, meta, i) => { + await encodeDecodeTest('avc', { displayWidth: 1280, displayHeight: 2 * 720 }, async (_, meta, i) => { if (i === 0) { expect(meta.decoderConfig).toBeDefined(); expect(meta.decoderConfig!.codedWidth).toBe(1280); @@ -901,9 +902,9 @@ describe('Video', async () => { assert(inputTrack); const packetTimestamps: number[] = []; - const packetSink = new EncodedPacketSink(inputTrack); + const packetCursor = new PacketCursor(inputTrack); - for await (const packet of packetSink.packets()) { + for await (const packet of packetCursor) { if (duration !== undefined && packet.timestamp >= duration) { break; } @@ -938,10 +939,10 @@ describe('Video', async () => { expect(await newInputTrack.getCodec()).toBe(codec); - const sink = new VideoSampleSink(newInputTrack); + await using cursor = new VideoSampleCursor(newInputTrack); let sampleCount = 0; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { expect(sample.codedWidth).toBe(await inputTrack.getCodedWidth()); expect(sample.codedHeight).toBe(await inputTrack.getCodedHeight()); expect(sample.timestamp).toBe(packetTimestamps[sampleCount]); @@ -961,8 +962,8 @@ describe('Video', async () => { const videoTrack = await input.getPrimaryVideoTrack(); assert(videoTrack); - const sink = new VideoSampleSink(videoTrack); - using sample = await sink.getSample(0); + await using cursor = new VideoSampleCursor(videoTrack); + const sample = await cursor.seekTo(0); assert(sample); expect(sample._data).toBeInstanceOf(AvFrameVideoSampleResource); @@ -1232,13 +1233,13 @@ describe('Video', async () => { const track = await input.getPrimaryVideoTrack(); assert(track); - const sink = new VideoSampleSink(track, { + await using cursor = new VideoSampleCursor(track, { hardwareAcceleration: 'prefer-software', }); let count = 0; // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (using sample of sink.samples()) { + for await (const sample of cursor) { count++; } @@ -1396,8 +1397,8 @@ describe('Video', async () => { }); test('Sample flip metadata composed with additional rotation', async () => { - using sample = makeSample(); - sample.setFlip(true); + using unflippedSample = makeSample(); + using sample = unflippedSample.clone({ flip: true }); // The sample's own flip comes first, then the additional rotation: blue goes bottom-left -> bottom-right // -> bottom-left @@ -1519,9 +1520,9 @@ describe('Audio', async () => { await decoder.init(); - const sink = new EncodedPacketSink(audioTrack); + const cursor = new PacketCursor(audioTrack); let packetCount = 0; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { packetTimestamps.push(packet.timestamp); await decoder.decode(packet); @@ -1539,7 +1540,8 @@ describe('Audio', async () => { packetTimestamps.length = 0; packetCount = 0; - for await (const packet of sink.packets((await sink.getKeyPacket(5))!)) { + await cursor.seekToKey(5); + for await (const packet of cursor) { packetTimestamps.push(packet.timestamp); await decoder.decode(packet); @@ -2083,10 +2085,10 @@ describe('Audio', async () => { expect(await newInputTrack.getCodec()).toBe(codec); expect(await newInputTrack.computeDuration()).toBeCloseTo(await inputTrack.computeDuration(), 0); - const sink = new AudioSampleSink(newInputTrack); + await using cursor = new AudioSampleCursor(newInputTrack); let sampleCount = 0; - for await (using sample of sink.samples()) { + for await (const sample of cursor) { expect([await inputTrack.getNumberOfChannels(), 2].includes(sample.numberOfChannels)).toBe(true); expect(sample.sampleRate).toBe(await inputTrack.getSampleRate()); @@ -2105,8 +2107,8 @@ describe('Audio', async () => { const audioTrack = await input.getPrimaryAudioTrack(); assert(audioTrack); - const sink = new AudioSampleSink(audioTrack); - using sample = await sink.getSample(await audioTrack.getFirstTimestamp()); + await using cursor = new AudioSampleCursor(audioTrack); + const sample = await cursor.seekToFirst(); assert(sample); expect(sample._data).toBeInstanceOf(AvFrameAudioSampleResource); diff --git a/test/node/url-source.test.ts b/test/node/url-source.test.ts index 8b80a044c..b2749efa1 100644 --- a/test/node/url-source.test.ts +++ b/test/node/url-source.test.ts @@ -9,13 +9,14 @@ import { Reader, readBytes } from '../../src/reader.js'; import { ALL_FORMATS, EncodedPacket, - EncodedPacketSink, FilePathSource, Input, Logging, LogLevel, UrlSource, } from '../../src/index.js'; +import { PacketCursor } from '../../src/cursors.js'; +import { PacketReader } from '../../src/packet.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); const videoFilePath = path.join(__dirname, '..', 'public/video.mp4'); @@ -33,17 +34,18 @@ test('UrlSource works against a server without range request support', async () const track = await input.getPrimaryVideoTrack(); if (!track) throw new Error('No video track found'); - const sink = new EncodedPacketSink(track); + const packetReader = new PacketReader(track); + const cursor = new PacketCursor(track); const timestamps: number[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); } expect(timestamps).toHaveLength(125); // The default cache size exceeds the file size, so random access back to the start of the file still works - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await packetReader.getFirst(); if (!firstPacket) throw new Error('No first packet found'); expect(firstPacket.timestamp).toBe(0); @@ -72,16 +74,17 @@ test('UrlSource throws when reading from an evicted region in sequential mode', const track = await input.getPrimaryVideoTrack(); if (!track) throw new Error('No video track found'); - const sink = new EncodedPacketSink(track); + const packetReader = new PacketReader(track); + const cursor = new PacketCursor(track); const timestamps: number[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); } expect(timestamps).toHaveLength(125); - await expect(sink.getFirstPacket()).rejects.toThrow(/already-evicted part of the cache/); + await expect(async () => packetReader.getFirst()).rejects.toThrow(/already-evicted part of the cache/); expect(logs.warnings.filter( x => x.includes('did not respond to a range request with 206 Partial Content'), @@ -109,7 +112,7 @@ test('UrlSource resumes with correct data when the connection dies in sequential if (!referenceTrack) throw new Error('No video track found'); const referencePackets: EncodedPacket[] = []; - for await (const packet of new EncodedPacketSink(referenceTrack).packets()) { + for await (const packet of new PacketCursor(referenceTrack)) { referencePackets.push(packet); } @@ -122,7 +125,7 @@ test('UrlSource resumes with correct data when the connection dies in sequential if (!track) throw new Error('No video track found'); let packetIndex = 0; - for await (const packet of new EncodedPacketSink(track).packets()) { + for await (const packet of new PacketCursor(track)) { const referencePacket = referencePackets[packetIndex]!; expect(packet.timestamp).toBe(referencePacket.timestamp); @@ -157,16 +160,17 @@ test('UrlSource with maxCacheSize: Infinity allows random access against a range const track = await input.getPrimaryVideoTrack(); if (!track) throw new Error('No video track found'); - const sink = new EncodedPacketSink(track); + const packetReader = new PacketReader(track); + const cursor = new PacketCursor(track); const timestamps: number[] = []; - for await (const packet of sink.packets()) { + for await (const packet of cursor) { timestamps.push(packet.timestamp); } expect(timestamps).toHaveLength(125); - const firstPacket = await sink.getFirstPacket(); + const firstPacket = await packetReader.getFirst(); if (!firstPacket) throw new Error('No first packet found'); expect(firstPacket.timestamp).toBe(0); @@ -207,8 +211,8 @@ test('UrlSource in sequential mode downloads lazily and aborts the response on d expect(server.bytesSent()).toBeLessThan(totalSize / 2); // The response is merely suspended, not dead: reading still works - const sink = new EncodedPacketSink(track); - const firstPacket = await sink.getFirstPacket(); + const packetReader = new PacketReader(track); + const firstPacket = await packetReader.getFirst(); if (!firstPacket) throw new Error('No first packet found'); expect(firstPacket.timestamp).toBe(0); diff --git a/test/public/AudacityTest1.mp3 b/test/public/AudacityTest1.mp3 new file mode 100644 index 000000000..8b3115f49 Binary files /dev/null and b/test/public/AudacityTest1.mp3 differ diff --git a/test/public/beach-party.ogg b/test/public/beach-party.ogg new file mode 100644 index 000000000..0939a0c19 Binary files /dev/null and b/test/public/beach-party.ogg differ diff --git a/test/public/frag-buck-bunny.mp4 b/test/public/frag-buck-bunny.mp4 new file mode 100644 index 000000000..5caaecd1f Binary files /dev/null and b/test/public/frag-buck-bunny.mp4 differ diff --git a/test/public/fucked-keyframes.mp4 b/test/public/fucked-keyframes.mp4 new file mode 100644 index 000000000..5005caeee Binary files /dev/null and b/test/public/fucked-keyframes.mp4 differ diff --git a/test/public/sylvie-trimmed.mp4 b/test/public/sylvie-trimmed.mp4 new file mode 100644 index 000000000..18e59ba9e Binary files /dev/null and b/test/public/sylvie-trimmed.mp4 differ diff --git a/test/public/trim-buck-bunny.aac b/test/public/trim-buck-bunny.aac new file mode 100644 index 000000000..8d8416336 Binary files /dev/null and b/test/public/trim-buck-bunny.aac differ diff --git a/test/public/trim-buck-bunny.mov b/test/public/trim-buck-bunny.mov new file mode 100644 index 000000000..4780a5e0a Binary files /dev/null and b/test/public/trim-buck-bunny.mov differ diff --git a/test/public/trim-buck-bunny.webm b/test/public/trim-buck-bunny.webm new file mode 100644 index 000000000..f0ee7dc1d Binary files /dev/null and b/test/public/trim-buck-bunny.webm differ