(null);
@@ -77,6 +83,25 @@ const PageDomainBlocklist = () => {
e.target.value = '';
};
+ const handleSync = async (e: React.FormEvent) => {
+ e.preventDefault();
+ setSyncing(true);
+ setSyncResult(null);
+ try {
+ const { total, submitted } = await adminApi.syncDomainBlocklist(syncUrl);
+ setSyncResult(
+ `Read ${total} entries and sent ${submitted} unique domains to the blocklist. Entries already on the list were ignored.`
+ );
+ await loadEntries();
+ } catch (e) {
+ setSyncResult(
+ `Error: ${e instanceof Error ? e.message : 'Unknown error'}`
+ );
+ } finally {
+ setSyncing(false);
+ }
+ };
+
const handleDelete = async (domain: string) => {
try {
await adminApi.removeDomainBlocklistEntry(domain);
@@ -169,6 +194,40 @@ const PageDomainBlocklist = () => {
+ Sync from a list URL
+
+ Fetches a newline-delimited domain list and imports it in batches.
+ Comments (#) and invalid entries are skipped. A large list
+ can take a minute.
+
+
+ {syncResult && (
+
+ {syncResult}
+
+ )}
+
+
+
Current Blocklist
{entries.length > 0 && (
diff --git a/packages/fxa-admin-panel/src/lib/api.ts b/packages/fxa-admin-panel/src/lib/api.ts
index e4932e9eccb..bf78e326e61 100644
--- a/packages/fxa-admin-panel/src/lib/api.ts
+++ b/packages/fxa-admin-panel/src/lib/api.ts
@@ -18,6 +18,7 @@ import type {
WafBypassTokenDto,
WafBypassTokenCreateDto,
DomainBlocklistEntry,
+ DomainBlocklistSyncResult,
OAuthScopeDto,
OAuthScopeCreateDto,
} from 'fxa-admin-server/src/types';
@@ -343,6 +344,13 @@ export const adminApi = {
return apiFetch('/api/domain-blocklist/all', { method: 'DELETE' });
},
+ syncDomainBlocklist(url: string): Promise {
+ return apiFetch('/api/domain-blocklist/sync', {
+ method: 'POST',
+ body: JSON.stringify({ url }),
+ });
+ },
+
// ---- OAuth scopes ----
getOAuthScopes(): Promise {
diff --git a/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.spec.ts b/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.spec.ts
index eecc821de5e..5a34ed75b2c 100644
--- a/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.spec.ts
+++ b/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.spec.ts
@@ -20,9 +20,40 @@ jest.mock('fxa-shared/db/models/auth', () => ({
},
}));
+const SYNC_URL = 'https://lists.example.com/disposable.conf';
+
+/** Minimal stand-in for the parts of `Response` the controller reads. */
+function mockResponse(
+ chunks: string[] | string,
+ overrides: {
+ ok?: boolean;
+ status?: number;
+ statusText?: string;
+ url?: string;
+ } = {}
+): Response {
+ const parts = Array.isArray(chunks) ? chunks : [chunks];
+ return {
+ ok: overrides.ok ?? true,
+ status: overrides.status ?? 200,
+ statusText: overrides.statusText ?? 'OK',
+ url: overrides.url ?? SYNC_URL,
+ body: new ReadableStream({
+ start(controller) {
+ for (const part of parts) {
+ controller.enqueue(new TextEncoder().encode(part));
+ }
+ controller.close();
+ },
+ }),
+ } as unknown as Response;
+}
+
describe('DomainBlocklistController', () => {
let controller: DomainBlocklistController;
let logger: { debug: jest.Mock; error: jest.Mock; info: jest.Mock };
+ let fetchMock: jest.Mock;
+ const realFetch = global.fetch;
beforeEach(async () => {
logger = { debug: jest.fn(), error: jest.fn(), info: jest.fn() };
@@ -58,9 +89,13 @@ describe('DomainBlocklistController', () => {
controller = module.get(
DomainBlocklistController
);
+
+ fetchMock = jest.fn();
+ global.fetch = fetchMock as unknown as typeof fetch;
});
afterEach(() => {
+ global.fetch = realFetch;
jest.clearAllMocks();
});
@@ -157,6 +192,154 @@ describe('DomainBlocklistController', () => {
});
});
+ describe('sync', () => {
+ beforeEach(() => {
+ (DomainBlocklist.addMany as jest.Mock).mockResolvedValue(undefined);
+ });
+
+ it('imports the fetched list in batches of 500', async () => {
+ const list = Array.from({ length: 1200 }, (_, i) => `spam${i}.example`);
+ fetchMock.mockResolvedValue(mockResponse(list.join('\n')));
+
+ const result = await controller.sync(SYNC_URL, 'admin@example.com');
+
+ expect(result).toEqual({ ok: true, total: 1200, submitted: 1200 });
+ expect(DomainBlocklist.addMany).toHaveBeenCalledTimes(3);
+ const batches = (DomainBlocklist.addMany as jest.Mock).mock.calls.map(
+ ([domains]) => domains.length
+ );
+ expect(batches).toEqual([500, 500, 200]);
+ expect(fetchMock).toHaveBeenCalledWith(SYNC_URL, {
+ signal: expect.any(AbortSignal),
+ });
+ });
+
+ it('skips blank lines, comments and invalid entries', async () => {
+ fetchMock.mockResolvedValue(
+ mockResponse(
+ [
+ '# disposable domains',
+ '',
+ ' evil.com ',
+ 'spam.net # inline comment',
+ 'not a domain',
+ 'nodot',
+ `${'a'.repeat(64)}.com`,
+ ].join('\n')
+ )
+ );
+
+ const result = await controller.sync(SYNC_URL, 'admin@example.com');
+
+ expect(DomainBlocklist.addMany).toHaveBeenCalledWith([
+ 'evil.com',
+ 'spam.net',
+ ]);
+ expect(result).toEqual({ ok: true, total: 5, submitted: 2 });
+ });
+
+ it('imports repeated entries once', async () => {
+ fetchMock.mockResolvedValue(
+ mockResponse(['evil.com', '@Evil.com', 'evil.com'].join('\n'))
+ );
+
+ const result = await controller.sync(SYNC_URL, 'admin@example.com');
+
+ expect(DomainBlocklist.addMany).toHaveBeenCalledWith(['evil.com']);
+ expect(result).toEqual({ ok: true, total: 3, submitted: 1 });
+ });
+
+ it('logs the summary', async () => {
+ fetchMock.mockResolvedValue(mockResponse('evil.com'));
+
+ await controller.sync(SYNC_URL, 'admin@example.com');
+
+ expect(logger.info).toHaveBeenCalledWith('domainBlocklist.sync', {
+ user: 'admin@example.com',
+ url: SYNC_URL,
+ total: 1,
+ submitted: 1,
+ });
+ });
+
+ it('imports nothing when the list has no valid entries', async () => {
+ fetchMock.mockResolvedValue(mockResponse('# nothing here\n\n'));
+
+ const result = await controller.sync(SYNC_URL, 'admin@example.com');
+
+ expect(result).toEqual({ ok: true, total: 0, submitted: 0 });
+ expect(DomainBlocklist.addMany).not.toHaveBeenCalled();
+ });
+
+ it('throws for a url that is not https', async () => {
+ await expect(
+ controller.sync(
+ 'http://lists.example.com/list.conf',
+ 'admin@example.com'
+ )
+ ).rejects.toThrow(BadRequestException);
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it('throws for a malformed url', async () => {
+ await expect(
+ controller.sync('not-a-url', 'admin@example.com')
+ ).rejects.toThrow(BadRequestException);
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it('throws when url is not a string', async () => {
+ await expect(
+ controller.sync([SYNC_URL] as any, 'admin@example.com')
+ ).rejects.toThrow('url must be a string');
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it('throws when the list redirects to a plaintext url', async () => {
+ fetchMock.mockResolvedValue(
+ mockResponse('evil.com', { url: 'http://lists.example.com/list.conf' })
+ );
+
+ await expect(
+ controller.sync(SYNC_URL, 'admin@example.com')
+ ).rejects.toThrow('url redirected away from https');
+ expect(DomainBlocklist.addMany).not.toHaveBeenCalled();
+ });
+
+ it('throws when the body exceeds the size cap', async () => {
+ const oneMb = 'a'.repeat(1024 * 1024);
+ fetchMock.mockResolvedValue(mockResponse(Array(11).fill(oneMb)));
+
+ await expect(
+ controller.sync(SYNC_URL, 'admin@example.com')
+ ).rejects.toThrow('List is larger than the 10485760 byte limit');
+ expect(DomainBlocklist.addMany).not.toHaveBeenCalled();
+ });
+
+ it('throws when the list responds with an error status', async () => {
+ fetchMock.mockResolvedValue(
+ mockResponse('', { ok: false, status: 404, statusText: 'Not Found' })
+ );
+
+ await expect(
+ controller.sync(SYNC_URL, 'admin@example.com')
+ ).rejects.toThrow('Could not fetch the list: 404 Not Found');
+ expect(DomainBlocklist.addMany).not.toHaveBeenCalled();
+ });
+
+ it('throws and logs when the fetch fails', async () => {
+ fetchMock.mockRejectedValue(new Error('ECONNREFUSED'));
+
+ await expect(
+ controller.sync(SYNC_URL, 'admin@example.com')
+ ).rejects.toThrow('Could not fetch the list: ECONNREFUSED');
+ expect(logger.error).toHaveBeenCalledWith(
+ 'domainBlocklist.sync.fetchFailed',
+ expect.objectContaining({ url: SYNC_URL })
+ );
+ });
+ });
+
describe('remove', () => {
it('removes an existing domain', async () => {
(DomainBlocklist.removeByDomain as jest.Mock).mockResolvedValue(true);
diff --git a/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.ts b/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.ts
index 58d48086215..1afa6bd3af6 100644
--- a/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.ts
+++ b/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.ts
@@ -18,12 +18,38 @@ import { AuthHeaderGuard } from '../../auth/auth-header.guard';
import { AuditLog } from '../../auth/audit-log.decorator';
import { Features } from '../../auth/user-group-header.decorator';
import { CurrentUser } from '../../auth/auth-header.decorator';
-import type { DomainBlocklistEntry } from '../../types';
+import type {
+ DomainBlocklistEntry,
+ DomainBlocklistSyncResult,
+} from '../../types';
// RFC 1035 hostname label pattern — used to validate each domain entry
const DOMAIN_RE =
/^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/i;
+const SYNC_BATCH_SIZE = 500;
+/** Pause between batches so a large import does not saturate the database. */
+const SYNC_BATCH_DELAY_MS = 50;
+const SYNC_FETCH_TIMEOUT_MS = 60_000;
+/** The ~40k entry reference list is under 1MB. 10MB leaves room for a bigger list. */
+const SYNC_MAX_BYTES = 10 * 1024 * 1024;
+
+function normalizeDomain(domain: string): string {
+ return domain.trim().toLowerCase().replace(/^@/, '');
+}
+
+function isValidDomain(domain: string): boolean {
+ return (
+ domain.length <= 253 &&
+ domain.split('.').every((label) => label.length <= 63) &&
+ DOMAIN_RE.test(domain)
+ );
+}
+
+function delay(ms: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
@UseGuards(AuthHeaderGuard)
@Controller('/api/domain-blocklist')
export class DomainBlocklistController {
@@ -57,9 +83,7 @@ export class DomainBlocklistController {
);
}
- const trimmed = domains
- .map((d) => d.trim().toLowerCase().replace(/^@/, ''))
- .filter((d) => d.length > 0);
+ const trimmed = domains.map(normalizeDomain).filter((d) => d.length > 0);
if (trimmed.length === 0) {
throw new BadRequestException(
@@ -93,6 +117,139 @@ export class DomainBlocklistController {
return { ok: true };
}
+ /**
+ * Imports a newline-delimited domain list from a public URL. The server
+ * fetches the list so the panel does not need a cross-origin request.
+ */
+ @Post('sync')
+ @Features(AdminPanelFeature.DomainBlocklist)
+ @AuditLog()
+ public async sync(
+ @Body('url') url: string,
+ @CurrentUser() user: string
+ ): Promise {
+ const source = this.parseSyncUrl(url);
+ const body = await this.fetchList(source);
+ const { total, domains } = this.parseList(body);
+
+ for (let i = 0; i < domains.length; i += SYNC_BATCH_SIZE) {
+ await DomainBlocklist.addMany(domains.slice(i, i + SYNC_BATCH_SIZE));
+ if (i + SYNC_BATCH_SIZE < domains.length) {
+ await delay(SYNC_BATCH_DELAY_MS);
+ }
+ }
+
+ this.log.info('domainBlocklist.sync', {
+ user,
+ url: `${source.origin}${source.pathname}`,
+ total,
+ submitted: domains.length,
+ });
+ return { ok: true, total, submitted: domains.length };
+ }
+
+ /** Accepts https URLs only, so the fetch cannot reach a plaintext target. */
+ private parseSyncUrl(url: string): URL {
+ if (typeof url !== 'string') {
+ throw new BadRequestException('url must be a string');
+ }
+ let parsed: URL;
+ try {
+ parsed = new URL(url);
+ } catch {
+ throw new BadRequestException('url must be a valid URL');
+ }
+ if (parsed.protocol !== 'https:') {
+ throw new BadRequestException('url must use https');
+ }
+ return parsed;
+ }
+
+ private async fetchList(source: URL): Promise {
+ const abort = new AbortController();
+ const timer = setTimeout(() => abort.abort(), SYNC_FETCH_TIMEOUT_MS);
+ try {
+ const res = await fetch(source.toString(), { signal: abort.signal });
+ if (!res.ok) {
+ throw new BadRequestException(
+ `Could not fetch the list: ${res.status} ${res.statusText}`
+ );
+ }
+ // fetch follows redirects, so a https URL can still land on a plaintext one.
+ // This checks the final hop only; fetch has already made every request.
+ if (res.url && new URL(res.url).protocol !== 'https:') {
+ throw new BadRequestException('url redirected away from https');
+ }
+ return await this.readCappedBody(res);
+ } catch (err) {
+ if (err instanceof BadRequestException) {
+ throw err;
+ }
+ this.log.error('domainBlocklist.sync.fetchFailed', {
+ url: `${source.origin}${source.pathname}`,
+ err,
+ });
+ throw new BadRequestException(
+ `Could not fetch the list: ${err instanceof Error ? err.message : 'unknown error'}`
+ );
+ } finally {
+ clearTimeout(timer);
+ }
+ }
+
+ /**
+ * Reads the body a chunk at a time and stops at SYNC_MAX_BYTES. `content-length`
+ * is not enough on its own: a chunked response does not send one.
+ */
+ private async readCappedBody(res: Response): Promise {
+ const reader = res.body?.getReader();
+ if (!reader) {
+ return '';
+ }
+
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) {
+ break;
+ }
+ size += value.length;
+ if (size > SYNC_MAX_BYTES) {
+ await reader.cancel();
+ throw new BadRequestException(
+ `List is larger than the ${SYNC_MAX_BYTES} byte limit`
+ );
+ }
+ chunks.push(value);
+ }
+
+ return Buffer.concat(chunks).toString('utf8');
+ }
+
+ /**
+ * Splits the list into unique valid domains. Invalid entries are dropped
+ * rather than rejected, because a public list of ~40k entries usually has
+ * some junk in it and one bad line must not fail the whole import.
+ */
+ private parseList(body: string): { total: number; domains: string[] } {
+ const domains = new Set();
+ let total = 0;
+
+ for (const line of body.split(/\r?\n/)) {
+ const entry = normalizeDomain(line.split('#')[0]);
+ if (entry.length === 0) {
+ continue;
+ }
+ total++;
+ if (isValidDomain(entry)) {
+ domains.add(entry);
+ }
+ }
+
+ return { total, domains: [...domains] };
+ }
+
@Delete()
@Features(AdminPanelFeature.DomainBlocklist)
@AuditLog()
@@ -103,7 +260,7 @@ export class DomainBlocklistController {
if (typeof domain !== 'string' || domain.trim().length === 0) {
throw new BadRequestException('domain must be a non-empty string');
}
- const trimmedDomain = domain.trim().toLowerCase().replace(/^@/, '');
+ const trimmedDomain = normalizeDomain(domain);
const removed = await DomainBlocklist.removeByDomain(trimmedDomain);
this.log.info('domainBlocklist.remove', {
user,
diff --git a/packages/fxa-admin-server/src/types.ts b/packages/fxa-admin-server/src/types.ts
index 82da58ef395..0b4463262a3 100644
--- a/packages/fxa-admin-server/src/types.ts
+++ b/packages/fxa-admin-server/src/types.ts
@@ -320,6 +320,14 @@ export interface DomainBlocklistEntry {
createdAt: number;
}
+export interface DomainBlocklistSyncResult {
+ ok: boolean;
+ /** non-empty, non-comment lines read from the list */
+ total: number;
+ /** unique valid domains sent to the database; already-blocked ones are ignored there */
+ submitted: number;
+}
+
export interface OAuthScopeDto {
id: number;
scope: string;