@@ -998,21 +998,23 @@ fn sanitize_video_json(value: &mut serde_json::Value) {
998998 serde_json:: json!( { "version" : "" , "repository" : "" } ) ,
999999 ) ;
10001000 }
1001- // Unread arrays/objects: one sparse entry must not fail the video.
1001+ // Unread arrays/objects: one sparse entry must not fail the video,
1002+ // so they are always reset — not just when absent. Some extractors
1003+ // (TikTok) also emit explicit nulls, which serde defaults don't
1004+ // cover (those only fill in missing keys). Formats are read by the
1005+ // pipeline, so they are repaired entry by entry below instead.
10021006 for key in [ "thumbnails" , "chapters" , "tags" , "categories" ] {
1003- if obj. contains_key ( key) {
1004- obj. insert ( key. to_string ( ) , serde_json:: json!( [ ] ) ) ;
1005- }
1007+ obj. insert ( key. to_string ( ) , serde_json:: json!( [ ] ) ) ;
10061008 }
10071009 for key in [ "subtitles" , "automatic_captions" ] {
1008- if obj. contains_key ( key) {
1009- obj. insert ( key. to_string ( ) , serde_json:: json!( { } ) ) ;
1010- }
1010+ obj. insert ( key. to_string ( ) , serde_json:: json!( { } ) ) ;
10111011 }
1012- if obj. contains_key ( "heatmap" ) {
1013- obj. insert ( "heatmap" . to_string ( ) , serde_json:: Value :: Null ) ;
1012+ obj. insert ( "heatmap" . to_string ( ) , serde_json:: Value :: Null ) ;
1013+ if !obj. get ( "formats" ) . is_some_and ( |v| v. is_array ( ) ) {
1014+ obj. insert ( "formats" . to_string ( ) , serde_json:: json!( [ ] ) ) ;
10141015 }
10151016 if let Some ( formats) = obj. get_mut ( "formats" ) . and_then ( |f| f. as_array_mut ( ) ) {
1017+ formats. retain ( |f| f. is_object ( ) ) ;
10161018 for format in formats. iter_mut ( ) {
10171019 if let Some ( entry) = format. as_object_mut ( ) {
10181020 coerce_int_fields ( entry) ;
@@ -1323,6 +1325,235 @@ fn parse_progress_size(line: &str) -> Option<u64> {
13231325 raw. trim ( ) . parse ( ) . ok ( )
13241326}
13251327
1328+ /// One variant (`EXT-X-STREAM-INF`) of an HLS master playlist.
1329+ #[ derive( Debug , Clone , PartialEq , Eq ) ]
1330+ struct HlsVariant {
1331+ uri : String ,
1332+ height : Option < u32 > ,
1333+ audio_group : Option < String > ,
1334+ }
1335+
1336+ /// One audio rendition (`EXT-X-MEDIA TYPE=AUDIO`) of a master playlist.
1337+ #[ derive( Debug , Clone , PartialEq , Eq ) ]
1338+ struct HlsAudio {
1339+ group : String ,
1340+ uri : Option < String > ,
1341+ }
1342+
1343+ /// Split an HLS tag attribute list on commas outside quotes
1344+ /// (`BANDWIDTH=123,RESOLUTION=640x360,AUDIO="a"`).
1345+ fn split_hls_attrs ( line : & str ) -> Vec < & str > {
1346+ let mut parts = Vec :: new ( ) ;
1347+ let mut start = 0 ;
1348+ let mut quoted = false ;
1349+ for ( i, c) in line. char_indices ( ) {
1350+ match c {
1351+ '"' => quoted = !quoted,
1352+ ',' if !quoted => {
1353+ parts. push ( line[ start..i] . trim ( ) ) ;
1354+ start = i + 1 ;
1355+ }
1356+ _ => { }
1357+ }
1358+ }
1359+ parts. push ( line[ start..] . trim ( ) ) ;
1360+ parts
1361+ }
1362+
1363+ /// Parse one `KEY=VALUE` attribute pair, unquoting the value.
1364+ fn hls_attr ( part : & str ) -> Option < ( & str , & str ) > {
1365+ let ( key, value) = part. split_once ( '=' ) ?;
1366+ Some ( ( key. trim ( ) , value. trim ( ) . trim_matches ( '"' ) ) )
1367+ }
1368+
1369+ /// Height from a `RESOLUTION=WxH` value.
1370+ fn hls_resolution_height ( value : & str ) -> Option < u32 > {
1371+ value. split_once ( 'x' ) ?. 1 . parse ( ) . ok ( )
1372+ }
1373+
1374+ /// Parse an HLS master playlist into variants + audio renditions,
1375+ /// resolving relative URIs against the playlist URL. Returns `None`
1376+ /// when the text is a media playlist (segments, no variants), which
1377+ /// the caller downloads directly.
1378+ fn parse_hls_master ( text : & str , base : & url:: Url ) -> Option < ( Vec < HlsVariant > , Vec < HlsAudio > ) > {
1379+ let mut variants = Vec :: new ( ) ;
1380+ let mut audios = Vec :: new ( ) ;
1381+ let mut pending: Option < HlsVariant > = None ;
1382+ let mut is_master = false ;
1383+ for line in text. lines ( ) . map ( str:: trim) . filter ( |l| !l. is_empty ( ) ) {
1384+ if let Some ( rest) = line. strip_prefix ( "#EXT-X-STREAM-INF:" ) {
1385+ is_master = true ;
1386+ let mut variant = HlsVariant {
1387+ uri : String :: new ( ) ,
1388+ height : None ,
1389+ audio_group : None ,
1390+ } ;
1391+ for part in split_hls_attrs ( rest) {
1392+ match hls_attr ( part) {
1393+ Some ( ( "RESOLUTION" , v) ) => variant. height = hls_resolution_height ( v) ,
1394+ Some ( ( "AUDIO" , v) ) => variant. audio_group = Some ( v. to_string ( ) ) ,
1395+ _ => { }
1396+ }
1397+ }
1398+ pending = Some ( variant) ;
1399+ } else if let Some ( rest) = line. strip_prefix ( "#EXT-X-MEDIA:" ) {
1400+ let ( mut kind, mut group, mut uri) = ( "" , "" , None ) ;
1401+ for part in split_hls_attrs ( rest) {
1402+ match hls_attr ( part) {
1403+ Some ( ( "TYPE" , v) ) => kind = v,
1404+ Some ( ( "GROUP-ID" , v) ) => group = v,
1405+ Some ( ( "URI" , v) ) => uri = Some ( v. to_string ( ) ) ,
1406+ _ => { }
1407+ }
1408+ }
1409+ if kind == "AUDIO" && !group. is_empty ( ) {
1410+ let uri = uri. and_then ( |u| base. join ( & u) . ok ( ) ) . map ( |u| u. to_string ( ) ) ;
1411+ audios. push ( HlsAudio {
1412+ group : group. to_string ( ) ,
1413+ uri,
1414+ } ) ;
1415+ }
1416+ } else if !line. starts_with ( '#' )
1417+ && let Some ( mut variant) = pending. take ( )
1418+ && let Ok ( uri) = base. join ( line)
1419+ {
1420+ variant. uri = uri. to_string ( ) ;
1421+ variants. push ( variant) ;
1422+ }
1423+ }
1424+ if !is_master {
1425+ return None ;
1426+ }
1427+ Some ( ( variants, audios) )
1428+ }
1429+
1430+ /// Concrete ffmpeg inputs for one HLS capture: the video playlist
1431+ /// plus, when the variant carries separate audio, its rendition URL.
1432+ #[ derive( Debug , Clone , PartialEq , Eq ) ]
1433+ struct HlsInput {
1434+ video : String ,
1435+ audio : Option < String > ,
1436+ }
1437+
1438+ /// Pick the variant for a height cap (smallest at or above, else
1439+ /// tallest — the same rule as [`select_hls_format`]), preferring
1440+ /// variants that actually carry audio, and attach their rendition.
1441+ fn pick_hls_variant (
1442+ variants : & [ HlsVariant ] ,
1443+ audios : & [ HlsAudio ] ,
1444+ want : Option < u32 > ,
1445+ ) -> Option < HlsInput > {
1446+ let mut cands: Vec < & HlsVariant > = variants. iter ( ) . filter ( |v| !v. uri . is_empty ( ) ) . collect ( ) ;
1447+ if cands. is_empty ( ) {
1448+ return None ;
1449+ }
1450+ // Audio-bearing first (muxed variants have no AUDIO tag, which
1451+ // also counts), then closest height at or above the cap.
1452+ cands. sort_by_key ( |v| {
1453+ let has_audio = v. audio_group . is_none ( )
1454+ || audios
1455+ . iter ( )
1456+ . any ( |a| Some ( a. group . as_str ( ) ) == v. audio_group . as_deref ( ) && a. uri . is_some ( ) ) ;
1457+ ( !has_audio, v. height . unwrap_or ( 0 ) )
1458+ } ) ;
1459+ let pick = match want {
1460+ Some ( h) => cands
1461+ . iter ( )
1462+ . find ( |v| v. height . is_some_and ( |x| x >= h) )
1463+ . or_else ( || cands. iter ( ) . max_by_key ( |v| v. height . unwrap_or ( 0 ) ) )
1464+ . cloned ( ) ,
1465+ None => cands. into_iter ( ) . max_by_key ( |v| v. height . unwrap_or ( 0 ) ) ,
1466+ } ?;
1467+ let audio = pick. audio_group . as_deref ( ) . and_then ( |group| {
1468+ audios
1469+ . iter ( )
1470+ . find ( |a| a. group == group)
1471+ . and_then ( |a| a. uri . clone ( ) )
1472+ } ) ;
1473+ Some ( HlsInput {
1474+ video : pick. uri . clone ( ) ,
1475+ audio,
1476+ } )
1477+ }
1478+
1479+ /// Request headers for playlist fetches: extractor-resolved headers
1480+ /// with the configured user agent winning. Empty values are skipped.
1481+ fn hls_request_headers ( headers : & HttpHeaders , user_agent : & str ) -> reqwest:: header:: HeaderMap {
1482+ use reqwest:: header:: { HeaderMap , HeaderName , HeaderValue } ;
1483+ let ua = if user_agent. trim ( ) . is_empty ( ) {
1484+ headers. user_agent . as_str ( )
1485+ } else {
1486+ user_agent. trim ( )
1487+ } ;
1488+ let mut map = HeaderMap :: new ( ) ;
1489+ for ( name, value) in [
1490+ ( "user-agent" , ua) ,
1491+ ( "accept" , headers. accept . as_str ( ) ) ,
1492+ ( "accept-language" , headers. accept_language . as_str ( ) ) ,
1493+ ( "sec-fetch-mode" , headers. sec_fetch_mode . as_str ( ) ) ,
1494+ ] {
1495+ let value = value. trim ( ) ;
1496+ if value. is_empty ( ) {
1497+ continue ;
1498+ }
1499+ if let ( Ok ( name) , Ok ( value) ) = (
1500+ HeaderName :: from_bytes ( name. as_bytes ( ) ) ,
1501+ HeaderValue :: from_str ( value) ,
1502+ ) {
1503+ map. insert ( name, value) ;
1504+ }
1505+ }
1506+ map
1507+ }
1508+
1509+ /// Resolve what ffmpeg should open for one HLS format URL: when the URL
1510+ /// is a master playlist, pick the height-appropriate variant (plus its
1511+ /// audio rendition, if separate); otherwise — media playlist, fetch
1512+ /// failure, anything unexpected — hand ffmpeg the URL untouched, which
1513+ /// is exactly today's behavior.
1514+ async fn resolve_hls_input (
1515+ url : & str ,
1516+ headers : & HttpHeaders ,
1517+ user_agent : & str ,
1518+ want : Option < u32 > ,
1519+ ) -> HlsInput {
1520+ let fallback = || HlsInput {
1521+ video : url. to_string ( ) ,
1522+ audio : None ,
1523+ } ;
1524+ let Ok ( base) = url:: Url :: parse ( url) else {
1525+ return fallback ( ) ;
1526+ } ;
1527+ let client = reqwest:: Client :: builder ( )
1528+ . timeout ( std:: time:: Duration :: from_secs ( 30 ) )
1529+ . build ( )
1530+ . ok ( ) ;
1531+ let Some ( client) = client else {
1532+ return fallback ( ) ;
1533+ } ;
1534+ let text = match client
1535+ . get ( url)
1536+ . headers ( hls_request_headers ( headers, user_agent) )
1537+ . send ( )
1538+ . await
1539+ {
1540+ Ok ( resp) if resp. status ( ) . is_success ( ) => match resp. text ( ) . await {
1541+ Ok ( text) => text,
1542+ Err ( _) => return fallback ( ) ,
1543+ } ,
1544+ _ => return fallback ( ) ,
1545+ } ;
1546+ if !text. contains ( "#EXT-X-STREAM-INF" ) {
1547+ return fallback ( ) ;
1548+ }
1549+ match parse_hls_master ( & text, & base) {
1550+ Some ( ( variants, audios) ) => {
1551+ pick_hls_variant ( & variants, & audios, want) . unwrap_or_else ( fallback)
1552+ }
1553+ None => fallback ( ) ,
1554+ }
1555+ }
1556+
13261557/// Find one format by id, accepting only what the pipeline can fetch.
13271558/// `None` covers unknown ids and HLS/DRM/missing-URL formats alike: the
13281559/// caller falls back to the quality preset.
@@ -2132,21 +2363,36 @@ async fn run_hls_download(
21322363 . await
21332364 . map_err ( VideoError :: staging) ?;
21342365 let out_path = staging. join ( if job. audio_only { "hls.m4a" } else { "hls.mp4" } ) ;
2366+ // Master playlists resolve to a height-appropriate variant plus its
2367+ // audio rendition when separate; anything else passes through as today's
2368+ // single input.
2369+ let input = resolve_hls_input ( & hls. url , & hls. headers , & job. user_agent , hls. height ) . await ;
21352370 let mut cmd = tokio:: process:: Command :: new ( ffmpeg) ;
21362371 cmd. arg ( "-hide_banner" )
21372372 . arg ( "-loglevel" )
21382373 . arg ( "warning" )
21392374 . arg ( "-nostats" )
21402375 . arg ( "-progress" )
2141- . arg ( "pipe:1" )
2142- . arg ( "-i" )
2143- . arg ( & hls. url ) ;
2376+ . arg ( "pipe:1" ) ;
21442377 let headers = ffmpeg_headers ( & hls. headers , & job. user_agent ) ;
2145- if !headers. is_empty ( ) {
2146- cmd. arg ( "-headers" ) . arg ( headers) ;
2147- }
2378+ // `-headers` is per-input: it must precede each `-i` it covers.
2379+ let input_arg = |cmd : & mut tokio:: process:: Command , url : & str | {
2380+ if !headers. is_empty ( ) {
2381+ cmd. arg ( "-headers" ) . arg ( & headers) ;
2382+ }
2383+ cmd. arg ( "-i" ) . arg ( url) ;
2384+ } ;
21482385 if job. audio_only {
2386+ // Audio rendition alone when split (skips the video segments),
2387+ // else the variant with video dropped.
2388+ input_arg ( & mut cmd, input. audio . as_deref ( ) . unwrap_or ( & input. video ) ) ;
21492389 cmd. arg ( "-vn" ) ;
2390+ } else if let Some ( audio) = & input. audio {
2391+ input_arg ( & mut cmd, & input. video ) ;
2392+ input_arg ( & mut cmd, audio) ;
2393+ cmd. arg ( "-map" ) . arg ( "0:v:0" ) . arg ( "-map" ) . arg ( "1:a:0" ) ;
2394+ } else {
2395+ input_arg ( & mut cmd, & input. video ) ;
21502396 }
21512397 cmd. arg ( "-c" )
21522398 . arg ( "copy" )
0 commit comments