Repository navigation
Expand file tree
/
Copy pathindex.html
More file actions
488 lines (463 loc) · 26.6 KB
/
Copy pathindex.html
File metadata and controls
488 lines (463 loc) · 26.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>The Missing Temporal Link — Temporal Context Routing</title>
<meta name="description" content="Temporal Context Routing for script-driven audio-video generation.">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Caveat:wght@500;600;700&family=Newsreader:ital,opsz,wght@0,6..72,400;0,6..72,500;0,6..72,600;1,6..72,400;1,6..72,500&family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
<link rel="stylesheet" href="style.css?v=title1">
</head>
<body>
<header class="hero">
<div class="hero-task" aria-hidden="true">
<div class="hero-clock">
<svg viewBox="0 -22 1200 194" fill="none" xmlns="http://www.w3.org/2000/svg">
<defs>
<!-- Cool family is what: shots take steel, speech takes sage, matching
the paper's blue and green towers. Gold is when, and only when. -->
<linearGradient id="gShotA" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#d8e6f2"/>
<stop offset=".16" stop-color="#7fa8c8"/>
<stop offset="1" stop-color="#3d5770"/>
</linearGradient>
<linearGradient id="gShotB" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#c5d5e4"/>
<stop offset=".16" stop-color="#6a8eaa"/>
<stop offset="1" stop-color="#33485c"/>
</linearGradient>
<linearGradient id="gDia" x1="0" y1="0" x2="0" y2="1">
<stop offset="0" stop-color="#d4e2dc"/>
<stop offset=".18" stop-color="#7a9a8c"/>
<stop offset="1" stop-color="#3d524c"/>
</linearGradient>
<!-- When. The bump is at its floor on its own endpoints, so the fill is given
no height to occupy there and dies out on its own; brightest at the crest,
which is where the timing term peaks. -->
<linearGradient id="gBumpUp" gradientUnits="userSpaceOnUse" x1="0" y1="-10" x2="0" y2="22">
<stop offset="0" stop-color="#ffd79a" stop-opacity=".9"/>
<stop offset="1" stop-color="#e0a13a" stop-opacity="0"/>
</linearGradient>
<linearGradient id="gBumpDn" gradientUnits="userSpaceOnUse" x1="0" y1="113" x2="0" y2="96">
<stop offset="0" stop-color="#ffd79a" stop-opacity=".9"/>
<stop offset="1" stop-color="#e0a13a" stop-opacity="0"/>
</linearGradient>
<!-- The clock both streams read, brass warming towards the middle. -->
<linearGradient id="gClock" x1="0" y1="0" x2="1" y2="0">
<stop offset="0" stop-color="#8a6a2e" stop-opacity=".25"/>
<stop offset=".2" stop-color="#dcb264" stop-opacity=".9"/>
<stop offset=".8" stop-color="#dcb264" stop-opacity=".9"/>
<stop offset="1" stop-color="#8a6a2e" stop-opacity=".25"/>
</linearGradient>
<!-- The playhead is a beam struck on the clock, not a bar laid across it. -->
<linearGradient id="gHead" gradientUnits="userSpaceOnUse" x1="0" y1="-18" x2="0" y2="130">
<stop offset="0" stop-color="#ffffff" stop-opacity=".12"/>
<stop offset=".6" stop-color="#ffffff" stop-opacity=".75"/>
<stop offset="1" stop-color="#ffffff"/>
</linearGradient>
<filter id="glow" x="-60%" y="-60%" width="220%" height="220%">
<feGaussianBlur stdDeviation="5" result="b"/>
<feMerge><feMergeNode in="b"/><feMergeNode in="SourceGraphic"/></feMerge>
</filter>
</defs>
<text x="6" y="40" fill="#9a93ad" font-size="12.5" letter-spacing="1.4" font-family="Inter, sans-serif">SHOTS</text>
<text x="6" y="92" fill="#9a93ad" font-size="12.5" letter-spacing="1.4" font-family="Inter, sans-serif">DIALOGUE</text>
<!-- Names the gold layer, so the two colours cannot be read as two streams. -->
<text x="6" y="134" fill="#c9a24f" font-size="12.5" letter-spacing="1.4" font-family="Inter, sans-serif">CLOCK</text>
<!-- When, for the shots. Each interval also arrives as a timing term on a gold
side channel. Leaving the path open lets one element carry both the crest
line and the mass under it: fill closes the curve, stroke does not. -->
<g fill="url(#gBumpUp)" stroke="#ffd79a" stroke-width="1.5" stroke-linecap="round">
<path d="M80 22 C 144 -20, 244 -20, 308 22" fill-opacity=".13" stroke-opacity=".26">
<animate attributeName="fill-opacity" values=".13;.55;.55;.13;.13" keyTimes="0;0;0.217;0.235;1" dur="10s" repeatCount="indefinite"/>
<animate attributeName="stroke-opacity" values=".26;1;1;.26;.26" keyTimes="0;0;0.217;0.235;1" dur="10s" repeatCount="indefinite"/>
</path>
<path d="M312 22 C 359 -20, 432 -20, 478 22" fill-opacity=".13" stroke-opacity=".26">
<animate attributeName="fill-opacity" values=".13;.13;.55;.55;.13;.13" keyTimes="0;0.205;0.221;0.378;0.396;1" dur="10s" repeatCount="indefinite"/>
<animate attributeName="stroke-opacity" values=".26;.26;1;1;.26;.26" keyTimes="0;0.205;0.221;0.378;0.396;1" dur="10s" repeatCount="indefinite"/>
</path>
<path d="M482 22 C 546 -20, 646 -20, 710 22" fill-opacity=".13" stroke-opacity=".26">
<animate attributeName="fill-opacity" values=".13;.13;.55;.55;.13;.13" keyTimes="0;0.366;0.382;0.599;0.617;1" dur="10s" repeatCount="indefinite"/>
<animate attributeName="stroke-opacity" values=".26;.26;1;1;.26;.26" keyTimes="0;0.366;0.382;0.599;0.617;1" dur="10s" repeatCount="indefinite"/>
</path>
<path d="M714 22 C 778 -20, 878 -20, 942 22" fill-opacity=".13" stroke-opacity=".26">
<animate attributeName="fill-opacity" values=".13;.13;.55;.55;.13;.13" keyTimes="0;0.587;0.603;0.819;0.837;1" dur="10s" repeatCount="indefinite"/>
<animate attributeName="stroke-opacity" values=".26;.26;1;1;.26;.26" keyTimes="0;0.587;0.603;0.819;0.837;1" dur="10s" repeatCount="indefinite"/>
</path>
<path d="M946 22 C 998 -20, 1080 -20, 1132 22" fill-opacity=".13" stroke-opacity=".26">
<animate attributeName="fill-opacity" values=".13;.13;.55;.55" keyTimes="0;0.807;0.823;1" dur="10s" repeatCount="indefinite"/>
<animate attributeName="stroke-opacity" values=".26;.26;1;1" keyTimes="0;0.807;0.823;1" dur="10s" repeatCount="indefinite"/>
</path>
</g>
<!-- What, for the shots. The outline is always there because the script declared
the interval; the fill arrives only when the clock reaches it. Idle sits
high enough to read the lip, short of the highlighter pop at 1. -->
<g stroke="#c5d4e4" stroke-opacity=".45" stroke-width=".9">
<rect x="80" y="22" width="228" height="26" rx="6" fill="url(#gShotA)" fill-opacity=".42">
<animate attributeName="fill-opacity" values=".42;.92;.92;.42;.42" keyTimes="0;0;0.217;0.235;1" dur="10s" repeatCount="indefinite"/>
</rect>
<rect x="312" y="22" width="166" height="26" rx="6" fill="url(#gShotB)" fill-opacity=".42">
<animate attributeName="fill-opacity" values=".42;.42;.92;.92;.42;.42" keyTimes="0;0.205;0.221;0.378;0.396;1" dur="10s" repeatCount="indefinite"/>
</rect>
<rect x="482" y="22" width="228" height="26" rx="6" fill="url(#gShotA)" fill-opacity=".42">
<animate attributeName="fill-opacity" values=".42;.42;.92;.92;.42;.42" keyTimes="0;0.366;0.382;0.599;0.617;1" dur="10s" repeatCount="indefinite"/>
</rect>
<rect x="714" y="22" width="228" height="26" rx="6" fill="url(#gShotB)" fill-opacity=".42">
<animate attributeName="fill-opacity" values=".42;.42;.92;.92;.42;.42" keyTimes="0;0.587;0.603;0.819;0.837;1" dur="10s" repeatCount="indefinite"/>
</rect>
<rect x="946" y="22" width="186" height="26" rx="6" fill="url(#gShotA)" fill-opacity=".42">
<animate attributeName="fill-opacity" values=".42;.42;.92;.92" keyTimes="0;0.807;0.823;1" dur="10s" repeatCount="indefinite"/>
</rect>
</g>
<!-- What, for the speech. Sage sits next to the steel of the shots the way
the paper's audio tower sits next to its video tower. -->
<g stroke="#c5d4cc" stroke-opacity=".40" stroke-width=".9">
<rect x="122" y="74" width="250" height="22" rx="6" fill="url(#gDia)" fill-opacity=".4">
<animate attributeName="fill-opacity" values=".4;.4;.88;.88;.4;.4" keyTimes="0;0.024;0.04;0.277;0.295;1" dur="10s" repeatCount="indefinite"/>
</rect>
<rect x="392" y="74" width="270" height="22" rx="6" fill="url(#gDia)" fill-opacity=".4">
<animate attributeName="fill-opacity" values=".4;.4;.88;.88;.4;.4" keyTimes="0;0.281;0.297;0.553;0.571;1" dur="10s" repeatCount="indefinite"/>
</rect>
<rect x="724" y="74" width="332" height="22" rx="6" fill="url(#gDia)" fill-opacity=".4">
<animate attributeName="fill-opacity" values=".4;.4;.88;.88;.4;.4" keyTimes="0;0.596;0.612;0.928;0.946;1" dur="10s" repeatCount="indefinite"/>
</rect>
</g>
<!-- When, for the speech: the same gold bump, mirrored under its own lane. -->
<g fill="url(#gBumpDn)" stroke="#ffd79a" stroke-width="1.5" stroke-linecap="round">
<path d="M122 96 C 192 118, 302 118, 372 96" fill-opacity=".13" stroke-opacity=".26">
<animate attributeName="fill-opacity" values=".13;.13;.55;.55;.13;.13" keyTimes="0;0.024;0.04;0.277;0.295;1" dur="10s" repeatCount="indefinite"/>
<animate attributeName="stroke-opacity" values=".26;.26;1;1;.26;.26" keyTimes="0;0.024;0.04;0.277;0.295;1" dur="10s" repeatCount="indefinite"/>
</path>
<path d="M392 96 C 468 118, 586 118, 662 96" fill-opacity=".13" stroke-opacity=".26">
<animate attributeName="fill-opacity" values=".13;.13;.55;.55;.13;.13" keyTimes="0;0.281;0.297;0.553;0.571;1" dur="10s" repeatCount="indefinite"/>
<animate attributeName="stroke-opacity" values=".26;.26;1;1;.26;.26" keyTimes="0;0.281;0.297;0.553;0.571;1" dur="10s" repeatCount="indefinite"/>
</path>
<path d="M724 96 C 817 118, 963 118, 1056 96" fill-opacity=".13" stroke-opacity=".26">
<animate attributeName="fill-opacity" values=".13;.13;.55;.55;.13;.13" keyTimes="0;0.596;0.612;0.928;0.946;1" dur="10s" repeatCount="indefinite"/>
<animate attributeName="stroke-opacity" values=".26;.26;1;1;.26;.26" keyTimes="0;0.596;0.612;0.928;0.946;1" dur="10s" repeatCount="indefinite"/>
</path>
</g>
<!-- the one clock both lanes are written against, ruled every second -->
<line x1="80" y1="130" x2="1132" y2="130" stroke="url(#gClock)" stroke-width="1.5"/>
<g stroke="#a8842f" stroke-width="1" stroke-opacity=".45">
<line x1="185" y1="130" x2="185" y2="134"/>
<line x1="290" y1="130" x2="290" y2="134"/>
<line x1="396" y1="130" x2="396" y2="134"/>
<line x1="501" y1="130" x2="501" y2="134"/>
<line x1="711" y1="130" x2="711" y2="134"/>
<line x1="816" y1="130" x2="816" y2="134"/>
<line x1="922" y1="130" x2="922" y2="134"/>
<line x1="1027" y1="130" x2="1027" y2="134"/>
</g>
<g stroke="#dcb264" stroke-width="1.5" stroke-opacity=".85">
<line x1="80" y1="130" x2="80" y2="139"/>
<line x1="343" y1="130" x2="343" y2="136"/>
<line x1="606" y1="130" x2="606" y2="139"/>
<line x1="869" y1="130" x2="869" y2="136"/>
<line x1="1132" y1="130" x2="1132" y2="139"/>
</g>
<text x="80" y="156" fill="#c2a271" font-size="11.5" font-family="JetBrains Mono, monospace">0s</text>
<text x="588" y="156" fill="#c2a271" font-size="11.5" font-family="JetBrains Mono, monospace">5s</text>
<text x="1104" y="156" fill="#c2a271" font-size="11.5" font-family="JetBrains Mono, monospace">10s</text>
<g filter="url(#glow)">
<line x1="80" y1="-18" x2="80" y2="130" stroke="url(#gHead)" stroke-width="2">
<animate attributeName="x1" values="80;1132" dur="10s" repeatCount="indefinite"/>
<animate attributeName="x2" values="80;1132" dur="10s" repeatCount="indefinite"/>
</line>
<circle cx="80" cy="130" r="7.5" fill="none" stroke="#ffd79a" stroke-opacity=".55" stroke-width="1.5">
<animate attributeName="cx" values="80;1132" dur="10s" repeatCount="indefinite"/>
</circle>
<circle cx="80" cy="130" r="4" fill="#ffffff">
<animate attributeName="cx" values="80;1132" dur="10s" repeatCount="indefinite"/>
</circle>
</g>
</svg>
</div>
</div>
<div class="wrap">
<p class="venue">Under review</p>
<h1>
<span class="title-kicker">The Missing Temporal Link:</span>
<span class="title-method">Temporal Context Routing</span>
<span class="title-tail">for Script-Driven Audio-Video Generation</span>
</h1>
<p class="authors">
Yichen Liu<sup>1</sup>,
Quanwei Zhang<sup>2</sup>,
Haozhe Wang<sup>3</sup>,
Donghao Zhou<sup>4</sup>,
Jiankun Zhang<sup>5</sup>,
Xiaojie Li,
Yang Shi<sup>2</sup>,
Jiaming Liu<sup>2</sup>,
Ruihua Huang<sup>2</sup>,
Yingtian Zou<sup>6</sup>,
Daquan Zhou<sup>1</sup>
</p>
<ul class="affils">
<li><sup>1</sup>Peking University</li>
<li><sup>2</sup>Qwen Applications</li>
<li><sup>3</sup>The Hong Kong University of Science and Technology</li>
<li><sup>4</sup>The Chinese University of Hong Kong</li>
<li><sup>5</sup>University of Chicago</li>
<li><sup>6</sup>Shanghai Jiao Tong University</li>
</ul>
<nav class="links">
<a class="btn" href="https://huggingface.co/starry0929/Temporal-Context-Routing">Hugging Face</a>
<a class="btn" href="https://github.com/DAGroup-PKU/Temporal-Context-Routing">GitHub</a>
<a class="btn" href="https://arxiv.org/abs/2609.02367">arXiv</a>
<a class="btn" href="https://huggingface.co/papers/2609.02367">HF Paper</a>
<a class="btn" href="https://huggingface.co/spaces/hugging-apps/temporal-context-routing">HF Space</a>
<a class="btn btn-primary" href="./" aria-current="page">Project Page</a>
</nav>
<div class="stat">
<div class="stat-num">0.042<span>s</span></div>
<p class="stat-cap"><b>Mean shot boundary error</b>
about one frame at 24 fps, down from 1.11 s on the same base model</p>
</div>
</div>
</header>
<nav class="tocbar">
<div class="wrap toc">
<a href="#film">Video</a>
<a href="#demos">Examples</a>
<a href="#abstract">Abstract</a>
<a href="#method">Method</a>
<a href="#data">Data</a>
<a href="#results">Comparison</a>
<a href="#acknowledgements">Acknowledgements</a>
<a href="#cite">Citation</a>
</div>
</nav>
<main>
<section class="wrap section promo-section" id="film">
<div class="h2row"><span class="h2num">01</span><h2>Video overview</h2></div>
<figure class="promo-frame">
<video controls playsinline preload="metadata"
poster="assets/posters/tcr_promo_film_poster.jpg?v=32"
aria-label="Video overview of Temporal Context Routing">
<source src="assets/videos/tcr_promo_film.mp4?v=31" type="video/mp4">
Your browser does not support embedded MP4 video.
</video>
</figure>
</section>
<section class="wrap section" id="demos">
<div class="h2row"><span class="h2num">02</span><h2>Generated examples</h2></div>
<div class="legend">
<div class="legend-item"><span class="swatch swatch-req"></span>
<div><b>Requested</b><br>the shot or line written in the script</div></div>
<div class="legend-item"><span class="swatch swatch-now"></span>
<div><b>Now</b><br>the interval under the playhead</div></div>
<div class="legend-item"><span class="swatch swatch-head"></span>
<div><b>Playhead</b><br>current position, synchronised with the video</div></div>
</div>
<div class="grid" id="grid"></div>
</section>
<section class="wrap section" id="abstract">
<div class="h2row"><span class="h2num">03</span><h2>Abstract</h2></div>
<p class="abstract">
Audio-video generation is shifting from prompt following to script execution, where success
depends on placing the requested content at the requested time. A director-style script
schedules shots and spoken lines on one clock, so a plausible clip can still fail by cutting
late or letting dialogue cross the wrong boundary. An interval defines the target, but the
generator still needs a way to vary each script element's influence over video and audio
queries. We formulate this missing operation as <strong>temporal routing</strong>.
<strong>Temporal Context Routing (TCR)</strong> aligns script-specified shot and dialogue timing
with the shared video-audio timeline. Timing bypasses the text encoder and enters
cross-attention logits as a routing score, separating <em>what</em> to generate from
<em>when</em> it should appear. Built on LTX-2.3 22B, TCR reduces shot-boundary error from
<strong>1.11 s to 0.042 s</strong>—about one frame at 24 fps—and raises dialogue
timing accuracy from <strong>28.3% to 84.1%</strong>.
</p>
</section>
<section class="wrap section" id="method">
<div class="h2row"><span class="h2num">04</span><h2>Method</h2></div>
<figure class="figure">
<img src="assets/figures/fig_method_v10.png"
alt="TCR applied to the video and audio towers of a joint backbone">
<figcaption>
<b>Figure 1.</b> Blue and green are the video and audio towers in <b>(b)</b>, but a shot and a
spoken line in <b>(a)</b> and <b>(c)</b>; gold always marks time. <b>(a)</b> The script, each
interval a gold bar. <b>(b)</b> The intervals bypass the text encoder and arrive as a routing
term on a gold side channel into text cross-attention. <b>(c)</b> Each interval becomes a
duration-normalized arc reaching its floor at its own endpoints.
</figcaption>
</figure>
<div class="cols">
<div class="col">
<h3>Time never enters the text encoder</h3>
<p>
Every <code>time_range</code> field is stripped from the script before tokenization and kept
in an index-aligned timing map. Shot, Event, Reference and Global records are matched back to
tokenizer positions, so each content token inherits the interval of the record it belongs to.
Tokens the script never placed — padding, special tokens, register slots — take a
sentinel interval and receive no routing term at all.
</p>
</div>
<div class="col">
<h3>One term added to the attention logits</h3>
<p>
For a token with interval centre <i>c</i> and radius <i>r</i>, and a latent query at wall-clock
time <i>t</i>, we add
<span class="math">B = −β(t−c)<sup>2</sup> / 2r<sup>2</sup></span>
to the pre-softmax text cross-attention logit. Because the terms combine additively, the
unnormalized attention weight factorizes: semantic evidence selects what fits, the routing
term decides when it matters. TCR has no learned parameters, and β is fixed at 5
everywhere.
</p>
</div>
<div class="col">
<h3>Normalized by duration, not by a fixed width</h3>
<p>
Dividing by each record's own radius makes the profile relative to its duration, so a
0.8-second reaction and a 4-second establishing shot are treated alike rather than gated
uniformly. Adjacent intervals meet their shared boundary at the same value, making the
handoff between them duration-independent.
</p>
</div>
<div class="col">
<h3>Both towers read the same clock</h3>
<p>
The routing term reads only wall-clock seconds, so a single function compiled once from the
script is evaluated at the query times of each branch. The two routing matrices differ in
shape because the latent lattices differ; what is shared is the function and its clock. A
dialogue token and the shot it crosses are therefore routed against the same timeline.
</p>
</div>
</div>
</section>
<section class="wrap section" id="data">
<div class="h2row"><span class="h2num">05</span><h2>Construction of data</h2></div>
<figure class="figure">
<img src="assets/figures/fig_data_pipeline_v3.png"
alt="Three-stage pipeline for building multi-shot audio-visual clips with corrected timings">
<figcaption>
<b>Figure 2.</b> The pipeline traced on real data; each panel header names the tool.
<b>(a)</b> A clip is kept only when both boundaries fall in speech-free regions; interior cuts
are retained, making the unit multi-shot. <b>(b)</b> Gemini proposes a structured record with
provisional intervals. <b>(c)</b> Shot detection relocates those cuts and word alignment
re-times each line from audio.
</figcaption>
</figure>
<ol class="stages">
<li>
<h3>Cut on sound, not on picture</h3>
<p>
Source episodes are split where the speech band goes silent, under an asymmetric rule: a clip
must begin and end inside a speech-free region, while visual cuts inside it are deliberately
kept. The resulting units are multi-shot clips in which dialogue routinely continues across a
shot change and a line is often spoken by someone momentarily off screen — exactly the
structure that fixed-length windows or per-shot clipping would destroy.
</p>
</li>
<li>
<h3>Propose structure with a captioner</h3>
<p>
Gemini proposes grounded Reference, Shot, Event and Global records, using stable identifiers
instead of repeated appearance descriptions and keeping dialogue in its original spoken
language. Fields that must not reach the model, most importantly transcribed burned-in
subtitles, are removed at this stage.
</p>
</li>
<li>
<h3>Replace guessed times with measured ones</h3>
<p>
A record is retained only when its shot count agrees with a separately run content-based
detector; a disagreement discards the clip rather than forcing an alignment. On agreement the
detector's cut times replace the provisional boundaries, and WhisperX supplies word-level
timestamps from the audio alone to re-time every line. The captioner supplies semantic
structure; the detectors supply the clock.
</p>
</li>
</ol>
</section>
<section class="wrap section" id="results">
<div class="h2row"><span class="h2num">06</span><h2>Comparison with open-source generators</h2></div>
<div class="tablewrap">
<table class="results">
<thead>
<tr>
<th rowspan="2" class="lead">System</th>
<th colspan="2">Video quality</th>
<th colspan="4">Temporal accuracy</th>
<th colspan="3">Speech & A/V sync</th>
</tr>
<tr>
<th>IQ ↑</th><th>AES ↑</th>
<th>Shot B-MAE (s) ↓</th><th>Shot IoU ↑</th>
<th>Count Acc. (%) ↑</th><th>Acc@0.5s (%) ↑</th>
<th>WER (%) ↓</th><th>Sync-C ↑</th><th>Off. Acc (%) ↑</th>
</tr>
</thead>
<tbody>
<tr>
<td class="lead">Wan2.2 <span class="tag">video only</span></td>
<td class="second">0.6820</td><td>0.5150</td>
<td>2.31</td><td>0.422</td><td>9.0</td><td>0.0</td>
<td class="na">n/a</td><td class="na">n/a</td><td class="na">n/a</td>
</tr>
<tr>
<td class="lead">OVI</td>
<td>0.6640</td><td class="best">0.5602</td>
<td>1.84</td><td>0.457</td><td>17.0</td><td>8.6</td>
<td>56.9</td><td>2.37</td><td>17.2</td>
</tr>
<tr>
<td class="lead">JoyAI-Echo</td>
<td>0.6680</td><td>0.5167</td>
<td>1.81</td><td>0.464</td><td>15.5</td><td class="second">23.2</td>
<td class="second">11.9</td><td class="second">2.71</td><td>7.3</td>
</tr>
<tr>
<td class="lead">LTX-2.3 <span class="tag">our base</span></td>
<td>0.6819</td><td>0.5354</td>
<td class="second">1.11</td><td class="second">0.532</td>
<td class="second">36.0</td><td>0.0</td>
<td class="na">n/a</td><td>2.55</td><td class="best">31.2</td>
</tr>
<tr class="ours">
<td class="lead">TCR <span class="tag tag-ours">ours</span></td>
<td class="best">0.7032</td><td class="second">0.5477</td>
<td class="best">0.042</td><td class="best">0.957</td>
<td class="best">93.0</td><td class="best">84.1</td>
<td class="best">8.48</td><td class="best">2.78</td><td class="second">30.5</td>
</tr>
</tbody>
</table>
</div>
<p class="tnote">
<b>Bold</b> is best, <u>underline</u> second best. Wan2.2 emits no audio, and LTX-2.3 produces no
intelligible speech, so their speech columns are not applicable. Adding TCR to the same base
lowers shot Boundary MAE from 1.11 s to 0.042 s, about one frame at 24 fps.
</p>
</section>
<section class="wrap section" id="acknowledgements">
<div class="h2row"><span class="h2num">07</span><h2>Acknowledgements</h2></div>
<p>Our work builds on <a href="https://github.com/Lightricks/LTX-2">LTX-2</a> and
<a href="https://huggingface.co/google/gemma-3-12b-it">Gemma</a>. We sincerely thank the
Hugging Face team for creating the official
<a href="https://huggingface.co/spaces/hugging-apps/temporal-context-routing">interactive demo Space</a>.</p>
</section>
<section class="wrap section" id="cite">
<div class="h2row"><span class="h2num">08</span><h2>Citation</h2></div>
<pre class="cite">@misc{liu2026missingtemporallinktemporal,
title = {The Missing Temporal Link: Temporal Context Routing for Script-Driven Audio-Video Generation},
author = {Yichen Liu and Quanwei Zhang and Haozhe Wang and Donghao Zhou and Jiankun Zhang and Xiaojie Li and Yang Shi and Jiaming Liu and Ruihua Huang and Yingtian Zou and Daquan Zhou},
year = {2026},
eprint = {2609.02367},
archivePrefix = {arXiv},
primaryClass = {cs.MM},
url = {https://arxiv.org/abs/2609.02367}
}</pre>
</section>
</main>
<footer class="wrap">
<p>Videos on this page are generated by our model from held-out scripts. No cherry-picking across
checkpoints: every clip comes from the same model at the same training step used in the paper.</p>
</footer>
<script src="assets/data.js?v=cards1"></script>
<script src="app.js?v=cards1"></script>
</body>
</html>