Skip to content

Commit 27cf347

Browse files
authored
Merge pull request #2 from houssemko/site-topup
Broaden video-site support + pristine CI builds
2 parents 5b1413d + 4ec580b commit 27cf347

5 files changed

Lines changed: 339 additions & 46 deletions

File tree

‎.github/workflows/flatpak-build.yml‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
name: Flatpak build
22

3-
# Shared build chain for release bundles and PR test bundles. Release mode
4-
# (--force-clean + attach to the release) vs test mode (SDK cache + upload
5-
# artifact) is picked by the inputs below.
3+
# Shared build chain for release bundles and PR test bundles. Every build
4+
# is pristine (--force-clean, no SDK cache): validation must never depend
5+
# on what a previous run left behind. Release mode attaches the bundle to
6+
# the release; test mode uploads it as a workflow artifact instead.
67
on:
78
workflow_call:
89
inputs:
@@ -11,17 +12,13 @@ on:
1112
required: true
1213
type: string
1314
force-clean:
14-
description: "Pristine release build"
15-
default: false
15+
description: "Pristine build directory"
16+
default: true
1617
type: boolean
1718
release-tag:
1819
description: "Attach the bundle here; empty uploads an artifact instead"
1920
default: ""
2021
type: string
21-
cache-sdk:
22-
description: "Cache the SDK between runs"
23-
default: false
24-
type: boolean
2522

2623
jobs:
2724
bundle:
@@ -37,15 +34,6 @@ jobs:
3734
sudo apt-get update -qq
3835
sudo apt-get install -y -qq flatpak flatpak-builder python3
3936
40-
- name: Restore SDK cache
41-
if: inputs.cache-sdk
42-
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
43-
with:
44-
path: ~/.local/share/flatpak
45-
key: flatpak-sdk-${{ hashFiles('build-aux/io.github.houssemko.Grab.json') }}
46-
restore-keys: |
47-
flatpak-sdk-
48-
4937
- name: Install GNOME SDK and Platform
5038
run: |
5139
RUNTIME=$(python3 -c "import json; print(json.load(open('build-aux/io.github.houssemko.Grab.json'))['runtime-version'])")

‎.github/workflows/flatpak-test.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ name: Flatpak test bundle
33
# On-demand test builds for local testing: builds Grab.flatpak from any
44
# branch, tag, or SHA in CI and uploads it as a workflow artifact to
55
# download. Nothing touches releases. Also runs automatically on pull
6-
# requests touching code, data, or packaging. No --force-clean: that flag
7-
# is for release hygiene (and runners are ephemeral anyway, so the speed
8-
# lever is the SDK cache, not incrementality).
6+
# requests touching code, data, translations, or packaging. Always a full
7+
# pristine build (--force-clean, no SDK cache): validation must never
8+
# depend on what a previous run left behind.
99
on:
1010
workflow_dispatch:
1111
inputs:
@@ -17,6 +17,7 @@ on:
1717
paths:
1818
- "src/**"
1919
- "data/**"
20+
- "po/**"
2021
- "build-aux/**"
2122
- "Cargo.toml"
2223
- "Cargo.lock"
@@ -30,4 +31,4 @@ jobs:
3031
with:
3132
# Dispatch input when manually run, PR merge commit otherwise.
3233
ref: ${{ inputs.ref || github.sha }}
33-
cache-sdk: true
34+
force-clean: true

‎src/video.rs‎

Lines changed: 105 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,11 @@ use thiserror::Error;
3030
use tokio::sync::oneshot;
3131
use yt_dlp::Downloader;
3232
use yt_dlp::client::deps::{Libraries, LibraryInstaller};
33-
use yt_dlp::model::format::{Format, HttpHeaders};
33+
use yt_dlp::model::format::{Format, HttpHeaders, Protocol};
3434
use yt_dlp::model::selector::{
3535
AudioCodecPreference, AudioQuality, VideoCodecPreference, VideoQuality,
3636
};
37-
use yt_dlp::model::{FORMAT_URL_LIFETIME, Video};
37+
use yt_dlp::model::{DrmStatus, FORMAT_URL_LIFETIME, Video};
3838
use yt_dlp::{DownloadPriority, DownloadStatus as YtDownloadStatus};
3939

4040
/// Values for the `video-quality` GSettings key and the per-item quality
@@ -77,9 +77,11 @@ pub fn quality_labels() -> Vec<String> {
7777
]
7878
}
7979
/// Hosts routed through the video extractor instead of the plain HTTP
80-
/// engine. Suffix-matched (`music.youtube.com` counts), lowercase. Curated
81-
/// for v1; extend here as supported sites grow — this is the only place
82-
/// that decides what a "video page" is.
80+
/// engine. Suffix-matched (`music.youtube.com` counts), lowercase.
81+
/// DRM-free sites only: DRM-walled services (Netflix and kin) fail cleanly
82+
/// at selection time, so listing them here would only promise what the
83+
/// pipeline refuses to fetch. Extend here as verified sites grow — this is
84+
/// the only place that decides what a "video page" is.
8385
const VIDEO_DOMAINS: &[&str] = &[
8486
"youtube.com",
8587
"youtu.be",
@@ -94,8 +96,38 @@ const VIDEO_DOMAINS: &[&str] = &[
9496
"twitter.com",
9597
"x.com",
9698
"bilibili.com",
99+
"instagram.com",
100+
"facebook.com",
101+
"fb.watch",
102+
"threads.com",
103+
"bsky.app",
104+
"pinterest.com",
105+
"pin.it",
106+
"tumblr.com",
107+
"vk.com",
108+
"ok.ru",
109+
"coub.com",
110+
"bitchute.com",
111+
"odysee.com",
112+
"rutube.ru",
113+
"nicovideo.jp",
114+
"ted.com",
115+
"archive.org",
116+
"drive.google.com",
117+
"dropbox.com",
118+
"mediafire.com",
119+
"loom.com",
120+
"wistia.com",
121+
"wistia.net",
122+
"soundcloud.com",
123+
"bandcamp.com",
97124
];
98125

126+
/// Domains whose pages carry no video tracks (audio-first services). The
127+
/// New Download dialog presets Audio-only for these; the worker also falls
128+
/// back to audio-only on its own when no usable video format selects.
129+
const AUDIO_FIRST_DOMAINS: &[&str] = &["soundcloud.com", "bandcamp.com"];
130+
99131
/// Where a to-be-downloaded resource comes from. Serialized into the queue
100132
/// file, so it stays stable across releases.
101133
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
@@ -155,6 +187,20 @@ fn video_domain(host: &str) -> bool {
155187
.any(|d| host == *d || host.ends_with(&format!(".{d}")))
156188
}
157189

190+
/// Whether a video-page URL belongs to an audio-first service (no video
191+
/// tracks expected). Used to preset the dialog; the worker re-derives the
192+
/// effective mode from the selected formats anyway.
193+
pub fn is_audio_first(url: &str) -> bool {
194+
url::Url::parse(url)
195+
.ok()
196+
.and_then(|u| u.host_str().map(|h| h.to_ascii_lowercase()))
197+
.is_some_and(|host| {
198+
AUDIO_FIRST_DOMAINS
199+
.iter()
200+
.any(|d| host == *d || host.ends_with(&format!(".{d}")))
201+
})
202+
}
203+
158204
/// Unix timestamp now, seconds.
159205
pub fn now_unix() -> i64 {
160206
SystemTime::now()
@@ -412,10 +458,23 @@ struct StreamSel {
412458
url: String,
413459
headers: HttpHeaders,
414460
size: Option<u64>,
461+
/// Whether the stream carries an audio track (muxed files do).
462+
has_audio: bool,
415463
}
416464

417465
impl StreamSel {
466+
/// Build from an extractor format, accepting only what the pipeline
467+
/// can actually fetch: plain-HTTPS, DRM-free streams with a URL.
468+
/// Anything else (HLS manifests, encrypted formats) is a clean
469+
/// unavailable error — never playlist bytes merged as media, never
470+
/// encrypted garbage saved as a finished file.
418471
fn from_format(f: &Format) -> Result<Self, VideoError> {
472+
if f.protocol != Protocol::Https {
473+
return Err(VideoError::unavailable());
474+
}
475+
if matches!(f.has_drm, Some(DrmStatus::Yes)) {
476+
return Err(VideoError::unavailable());
477+
}
419478
Ok(Self {
420479
format_id: f.format_id.clone(),
421480
ext: f.download_info.ext.as_str().to_string(),
@@ -426,6 +485,11 @@ impl StreamSel {
426485
.ok_or_else(VideoError::unavailable)?,
427486
headers: f.download_info.http_headers.clone(),
428487
size: filesize_of(f),
488+
has_audio: f
489+
.codec_info
490+
.audio_codec
491+
.as_deref()
492+
.is_some_and(|c| c != "none"),
429493
})
430494
}
431495
}
@@ -637,24 +701,41 @@ pub async fn run_video_download(
637701
};
638702

639703
// Select streams: AVC1 default for compat (VP9/AV1 need no merge but
640-
// play back on fewer targets), best audio.
641-
let video_sel: Option<StreamSel> = if job.audio_only {
642-
None
643-
} else {
644-
Some(StreamSel::from_format(
704+
// play back on fewer targets), best audio. Rejections (HLS/DRM/missing
705+
// URL) degrade candidates to absent here; the plan below decides
706+
// between split, single-file and audio-only from what's fetchable.
707+
let mut video_sel: Option<StreamSel> = (!job.audio_only)
708+
.then(|| {
709+
video.select_video_format(
710+
selector_for_quality(&job.quality),
711+
VideoCodecPreference::AVC1,
712+
)
713+
})
714+
.flatten()
715+
.and_then(|f| StreamSel::from_format(f).ok());
716+
let mut audio_sel: Option<StreamSel> = video
717+
.select_audio_format(AudioQuality::Best, AudioCodecPreference::Any)
718+
.and_then(|f| StreamSel::from_format(f).ok());
719+
// Muxed-only sources (one file, both tracks — archive.org, file
720+
// lockers): adopt the file directly instead of failing on the missing
721+
// split counterpart. A downloaded track beats a failed row; the
722+
// manifest records the effective single-part mode so retries agree.
723+
let mut audio_only = job.audio_only;
724+
if audio_sel.is_none() {
725+
let muxed = video_sel.take_if(|v| v.has_audio).or_else(|| {
645726
video
646-
.select_video_format(
647-
selector_for_quality(&job.quality),
648-
VideoCodecPreference::AVC1,
649-
)
650-
.ok_or_else(VideoError::unavailable)?,
651-
)?)
727+
.best_audio_video_format()
728+
.ok()
729+
.and_then(|m| StreamSel::from_format(m).ok())
730+
});
731+
if let Some(m) = muxed {
732+
audio_sel = Some(m);
733+
audio_only = true;
734+
}
735+
}
736+
let Some(audio_sel) = audio_sel else {
737+
return Err(VideoError::unavailable());
652738
};
653-
let audio_sel = StreamSel::from_format(
654-
video
655-
.select_audio_format(AudioQuality::Best, AudioCodecPreference::Any)
656-
.ok_or_else(VideoError::unavailable)?,
657-
)?;
658739

659740
// Retry discipline from the sidecar.
660741
let manifest = read_manifest(&staging);
@@ -664,7 +745,7 @@ pub async fn run_video_download(
664745
dest: &job.dest,
665746
page_url: &job.page_url,
666747
quality: &job.quality,
667-
audio_only: job.audio_only,
748+
audio_only,
668749
video: video_sel
669750
.as_ref()
670751
.map(|s| (s.format_id.as_str(), s.ext.as_str())),
@@ -692,7 +773,7 @@ pub async fn run_video_download(
692773
let combined_total: Option<u64> =
693774
match (video_sel.as_ref().and_then(|s| s.size), audio_sel.size) {
694775
(Some(v), Some(a)) => Some(v + a),
695-
(None, Some(a)) if job.audio_only => Some(a),
776+
(None, Some(a)) if audio_only => Some(a),
696777
_ => None,
697778
};
698779
let v_done = Arc::new(AtomicU64::new(0));
@@ -779,7 +860,7 @@ pub async fn run_video_download(
779860
&VideoManifest {
780861
page_url: job.page_url.clone(),
781862
quality: job.quality.clone(),
782-
audio_only: job.audio_only,
863+
audio_only,
783864
video_format_id: video_sel.as_ref().map(|s| s.format_id.clone()),
784865
video_ext: video_sel
785866
.as_ref()

0 commit comments

Comments
 (0)