Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,12 @@ jobs:
- name: Run Clippy
run: cargo clippy --all-targets --all-features -- -D warnings

- name: Check minimal search feature dependency boundary
working-directory: .
run: |
cargo test --manifest-path rust/Cargo.toml --no-default-features --features search --test search_feature
node scripts/rust-check-search-feature.mjs

# Test matrix: Rust on multiple OS - only runs when Rust code changes
test:
name: Rust - Test (${{ matrix.os }})
Expand Down
19 changes: 19 additions & 0 deletions js/tests/unit/search.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,25 @@ describe('search module (#130)', () => {
});
});

it('keeps URL construction and response parsing transport-independent', () => {
const sourceUrl = buildSearchUrl('wikipedia', 'formal ai', 1);
const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
limit: 1,
});

expect(sourceUrl).toBe(
'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal%20ai&limit=1'
);
expect(results).toEqual([
{
rank: 1,
title: 'Formal methods',
url: 'https://en.wikipedia.org/wiki/Formal_methods',
snippet: 'the study of formal',
},
]);
});

describe('parseSearchResults', () => {
it('normalizes Wikipedia REST JSON and strips markup', () => {
const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
Expand Down
2 changes: 1 addition & 1 deletion rust/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

91 changes: 66 additions & 25 deletions rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,80 +11,121 @@ keywords = ["web", "capture", "screenshot", "markdown", "html"]
categories = ["command-line-utilities", "web-programming"]
rust-version = "1.96"

[features]
default = ["runtime"]
search = [
"dep:html-escape",
"dep:scraper",
"dep:serde",
"dep:serde_json",
"dep:thiserror",
"dep:url",
]
runtime = [
"search",
"dep:anyhow",
"dep:async-tungstenite",
"dep:axum",
"dep:base64",
"dep:browser-commander",
"dep:clap",
"dep:encoding_rs",
"dep:futures",
"dep:html-to-markdown-rs",
"dep:html2md",
"dep:lino-arguments",
"dep:regex",
"dep:reqwest",
"dep:tokio",
"dep:tower",
"dep:tower-http",
"dep:tracing",
"dep:tracing-subscriber",
"dep:zip",
]

[lib]
path = "src/lib.rs"

[[bin]]
name = "web-capture"
path = "src/main.rs"
required-features = ["runtime"]

[[test]]
name = "unit"
path = "tests/unit/mod.rs"
required-features = ["runtime"]

[[test]]
name = "integration"
path = "tests/integration/mod.rs"
required-features = ["runtime"]

[[test]]
name = "search_feature"
path = "tests/search_feature.rs"
required-features = ["search"]

[dependencies]
# Browser automation using browser-commander from link-foundation
browser-commander = "0.9"
async-tungstenite = { version = "0.27", features = ["tokio-runtime"] }
futures = "0.3"
browser-commander = { version = "0.9", optional = true }
async-tungstenite = { version = "0.27", features = ["tokio-runtime"], optional = true }
futures = { version = "0.3", optional = true }

# Unified configuration from CLI args, env vars, and .lenv files
lino-arguments = "0.3"
lino-arguments = { version = "0.3", optional = true }

# Async runtime
tokio = { version = "1.0", features = ["full"] }
tokio = { version = "1.0", features = ["full"], optional = true }

# Web framework
axum = "0.8"
tower = "0.5"
tower-http = { version = "0.6", features = ["cors", "trace"] }
axum = { version = "0.8", optional = true }
tower = { version = "0.5", optional = true }
tower-http = { version = "0.6", features = ["cors", "trace"], optional = true }

# HTTP client
reqwest = { version = "0.12", features = ["cookies", "gzip"] }
reqwest = { version = "0.12", features = ["cookies", "gzip"], optional = true }

# HTML parsing and manipulation
scraper = "0.21"
scraper = { version = "0.21", optional = true }

# HTML to Markdown conversion
html2md = "0.2"
html-to-markdown-rs = { version = "3.6", features = ["inline-images", "metadata", "serde"] }
html2md = { version = "0.2", optional = true }
html-to-markdown-rs = { version = "3.6", features = ["inline-images", "metadata", "serde"], optional = true }

# Command line argument parsing
clap = { version = "4.5", features = ["derive", "env"] }
clap = { version = "4.5", features = ["derive", "env"], optional = true }

# Serialization
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
serde = { version = "1.0", features = ["derive"], optional = true }
serde_json = { version = "1.0", optional = true }

# Error handling
thiserror = "2.0"
anyhow = "1.0"
thiserror = { version = "2.0", optional = true }
anyhow = { version = "1.0", optional = true }

# Logging
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tracing = { version = "0.1", optional = true }
tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true }

# URL handling
url = "2.5"
url = { version = "2.5", optional = true }

# Encoding
encoding_rs = "0.8"
encoding_rs = { version = "0.8", optional = true }

# Base64 for image data
base64 = "0.22"
base64 = { version = "0.22", optional = true }

# Regex for URL conversion
regex = "1.11"
regex = { version = "1.11", optional = true }

# HTML entity decoding
html-escape = "0.2"
html-escape = { version = "0.2", optional = true }

# ZIP archive creation
zip = { version = "4.0", default-features = false, features = ["deflate"] }
zip = { version = "4.0", default-features = false, features = ["deflate"], optional = true }

[dev-dependencies]
tokio-test = "0.4"
Expand Down
18 changes: 18 additions & 0 deletions rust/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,24 @@ cd rust
cargo build --release
```

## Cargo Features

The default `runtime` feature preserves the full CLI, HTTP server, HTTP client,
and browser capture behavior. Applications that only build provider URLs,
parse provider responses, or supply their own transport can disable default
features and select `search`:

```toml
[dependencies]
web-capture = { version = "0.3", default-features = false, features = ["search"] }
```

The `search` feature exposes the pure URL/parser API and the caller-owned
transport contract, including `build_search_url`, `parse_search_results`, and
`search_with_transport`. It does not select `browser-commander`, `reqwest`,
Tokio, Axum, or OpenSSL. The convenience `search` function, which performs an
HTTP request with reqwest, remains part of `runtime`.

## Quick Start

### CLI Usage
Expand Down
Loading
Loading