From 737be045a67c07e42068fb558386c397f6ad0798 Mon Sep 17 00:00:00 2001 From: Vijay Budhram Date: Mon, 24 Aug 2026 17:22:40 -0400 Subject: [PATCH] feat(admin): add domain blocklist sync from a list URL ## Because - FXA-13596 shipped the domain blocklist panel and the auth-server enforcement, but not a seed path. Today an admin can only paste or upload domains by hand. - The public disposable-domain list holds about 40k entries. Nobody is going to paste that. ## This pull request - Adds `POST /api/domain-blocklist/sync`. It fetches a list URL over https, parses it, and imports it through the existing `DomainBlocklist.addMany()`. - Parses the list line by line. Blank lines and `#` comments are skipped, entries are normalized, invalid domains and repeats are dropped. - Imports in batches of 500 with a 50 ms pause between them. `addMany` ignores conflicts, so a domain that is already blocked does not error. - Adds an editable URL field to the panel, pre-filled with the disposable-email-domains list, plus a sync button and a result line. - Adds `syncDomainBlocklist()` to the panel API client and `DomainBlocklistSyncResult` to the admin-server types. ## Issue that this pull request solves Closes: https://mozilla-hub.atlassian.net/browse/FXA-13674 --- .../PageDomainBlocklist/index.test.tsx | 72 +++++++ .../components/PageDomainBlocklist/index.tsx | 59 ++++++ packages/fxa-admin-panel/src/lib/api.ts | 8 + .../domain-blocklist.controller.spec.ts | 183 ++++++++++++++++++ .../domain-blocklist.controller.ts | 167 +++++++++++++++- packages/fxa-admin-server/src/types.ts | 8 + 6 files changed, 492 insertions(+), 5 deletions(-) diff --git a/packages/fxa-admin-panel/src/components/PageDomainBlocklist/index.test.tsx b/packages/fxa-admin-panel/src/components/PageDomainBlocklist/index.test.tsx index eb80a68a549..69230e78f3f 100644 --- a/packages/fxa-admin-panel/src/components/PageDomainBlocklist/index.test.tsx +++ b/packages/fxa-admin-panel/src/components/PageDomainBlocklist/index.test.tsx @@ -14,6 +14,7 @@ jest.mock('../../lib/api', () => ({ addDomainBlocklistEntries: jest.fn(), removeDomainBlocklistEntry: jest.fn(), deleteAllDomainBlocklistEntries: jest.fn(), + syncDomainBlocklist: jest.fn(), }, })); @@ -22,6 +23,9 @@ const mockEntries = [ { domain: 'spam.net', createdAt: 1000 }, ]; +const DEFAULT_SYNC_URL = + 'https://raw.githubusercontent.com/disposable/disposable-email-domains/master/disposable_email_blocklist.conf'; + describe('PageDomainBlocklist', () => { let user: UserEvent; let confirmSpy: jest.SpyInstance; @@ -121,6 +125,74 @@ describe('PageDomainBlocklist', () => { }); }); + it('pre-fills the sync URL with the disposable-email-domains list', async () => { + render(); + await screen.findByText('No entries yet.'); + + expect(screen.getByTestId('domain-blocklist-sync-url')).toHaveValue( + DEFAULT_SYNC_URL + ); + }); + + it('syncs from the default URL and reports the submitted count', async () => { + (adminApi.getDomainBlocklist as jest.Mock) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce(mockEntries); + (adminApi.syncDomainBlocklist as jest.Mock).mockResolvedValue({ + ok: true, + total: 40000, + submitted: 39990, + }); + + render(); + await screen.findByText('No entries yet.'); + + await user.click(screen.getByTestId('domain-blocklist-sync-btn')); + + expect(adminApi.syncDomainBlocklist).toHaveBeenCalledWith(DEFAULT_SYNC_URL); + expect( + await screen.findByTestId('domain-blocklist-sync-result') + ).toHaveTextContent( + 'Read 40000 entries and sent 39990 unique domains to the blocklist. Entries already on the list were ignored.' + ); + expect(screen.getByText('evil.com')).toBeInTheDocument(); + }); + + it('syncs from an edited URL', async () => { + (adminApi.syncDomainBlocklist as jest.Mock).mockResolvedValue({ + ok: true, + total: 2, + submitted: 2, + }); + + render(); + await screen.findByText('No entries yet.'); + + const input = screen.getByTestId('domain-blocklist-sync-url'); + await user.clear(input); + await user.type(input, 'https://lists.example.com/other.conf'); + await user.click(screen.getByTestId('domain-blocklist-sync-btn')); + + expect(adminApi.syncDomainBlocklist).toHaveBeenCalledWith( + 'https://lists.example.com/other.conf' + ); + }); + + it('shows the error message when the sync fails', async () => { + (adminApi.syncDomainBlocklist as jest.Mock).mockRejectedValue( + new Error('API error 400: url must use https') + ); + + render(); + await screen.findByText('No entries yet.'); + + await user.click(screen.getByTestId('domain-blocklist-sync-btn')); + + expect( + await screen.findByTestId('domain-blocklist-sync-result') + ).toHaveTextContent('Error: API error 400: url must use https'); + }); + it('deletes a single entry after successful API call', async () => { (adminApi.getDomainBlocklist as jest.Mock).mockResolvedValue(mockEntries); (adminApi.removeDomainBlocklistEntry as jest.Mock).mockResolvedValue({ diff --git a/packages/fxa-admin-panel/src/components/PageDomainBlocklist/index.tsx b/packages/fxa-admin-panel/src/components/PageDomainBlocklist/index.tsx index 0b7817683de..dd1025c5286 100644 --- a/packages/fxa-admin-panel/src/components/PageDomainBlocklist/index.tsx +++ b/packages/fxa-admin-panel/src/components/PageDomainBlocklist/index.tsx @@ -9,12 +9,18 @@ import type { DomainBlocklistEntry } from 'fxa-admin-server/src/types'; const btnClass = 'bg-grey-10 border-2 p-1 border-grey-100 font-small leading-6 rounded'; +const DEFAULT_SYNC_URL = + 'https://raw.githubusercontent.com/disposable/disposable-email-domains/master/disposable_email_blocklist.conf'; + const PageDomainBlocklist = () => { const [entries, setEntries] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const [submitting, setSubmitting] = useState(false); const [hasInput, setHasInput] = useState(false); + const [syncUrl, setSyncUrl] = useState(DEFAULT_SYNC_URL); + const [syncing, setSyncing] = useState(false); + const [syncResult, setSyncResult] = useState(null); const textareaRef = useRef(null); const fileInputRef = useRef(null); @@ -77,6 +83,25 @@ const PageDomainBlocklist = () => { e.target.value = ''; }; + const handleSync = async (e: React.FormEvent) => { + e.preventDefault(); + setSyncing(true); + setSyncResult(null); + try { + const { total, submitted } = await adminApi.syncDomainBlocklist(syncUrl); + setSyncResult( + `Read ${total} entries and sent ${submitted} unique domains to the blocklist. Entries already on the list were ignored.` + ); + await loadEntries(); + } catch (e) { + setSyncResult( + `Error: ${e instanceof Error ? e.message : 'Unknown error'}` + ); + } finally { + setSyncing(false); + } + }; + const handleDelete = async (domain: string) => { try { await adminApi.removeDomainBlocklistEntry(domain); @@ -169,6 +194,40 @@ const PageDomainBlocklist = () => {
+

Sync from a list URL

+

+ Fetches a newline-delimited domain list and imports it in batches. + Comments (#) and invalid entries are skipped. A large list + can take a minute. +

+
+ setSyncUrl(e.target.value)} + /> +
+ +
+ {syncResult && ( +

+ {syncResult} +

+ )} + +
+

Current Blocklist

{entries.length > 0 && ( diff --git a/packages/fxa-admin-panel/src/lib/api.ts b/packages/fxa-admin-panel/src/lib/api.ts index e4932e9eccb..bf78e326e61 100644 --- a/packages/fxa-admin-panel/src/lib/api.ts +++ b/packages/fxa-admin-panel/src/lib/api.ts @@ -18,6 +18,7 @@ import type { WafBypassTokenDto, WafBypassTokenCreateDto, DomainBlocklistEntry, + DomainBlocklistSyncResult, OAuthScopeDto, OAuthScopeCreateDto, } from 'fxa-admin-server/src/types'; @@ -343,6 +344,13 @@ export const adminApi = { return apiFetch('/api/domain-blocklist/all', { method: 'DELETE' }); }, + syncDomainBlocklist(url: string): Promise { + return apiFetch('/api/domain-blocklist/sync', { + method: 'POST', + body: JSON.stringify({ url }), + }); + }, + // ---- OAuth scopes ---- getOAuthScopes(): Promise { diff --git a/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.spec.ts b/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.spec.ts index eecc821de5e..5a34ed75b2c 100644 --- a/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.spec.ts +++ b/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.spec.ts @@ -20,9 +20,40 @@ jest.mock('fxa-shared/db/models/auth', () => ({ }, })); +const SYNC_URL = 'https://lists.example.com/disposable.conf'; + +/** Minimal stand-in for the parts of `Response` the controller reads. */ +function mockResponse( + chunks: string[] | string, + overrides: { + ok?: boolean; + status?: number; + statusText?: string; + url?: string; + } = {} +): Response { + const parts = Array.isArray(chunks) ? chunks : [chunks]; + return { + ok: overrides.ok ?? true, + status: overrides.status ?? 200, + statusText: overrides.statusText ?? 'OK', + url: overrides.url ?? SYNC_URL, + body: new ReadableStream({ + start(controller) { + for (const part of parts) { + controller.enqueue(new TextEncoder().encode(part)); + } + controller.close(); + }, + }), + } as unknown as Response; +} + describe('DomainBlocklistController', () => { let controller: DomainBlocklistController; let logger: { debug: jest.Mock; error: jest.Mock; info: jest.Mock }; + let fetchMock: jest.Mock; + const realFetch = global.fetch; beforeEach(async () => { logger = { debug: jest.fn(), error: jest.fn(), info: jest.fn() }; @@ -58,9 +89,13 @@ describe('DomainBlocklistController', () => { controller = module.get( DomainBlocklistController ); + + fetchMock = jest.fn(); + global.fetch = fetchMock as unknown as typeof fetch; }); afterEach(() => { + global.fetch = realFetch; jest.clearAllMocks(); }); @@ -157,6 +192,154 @@ describe('DomainBlocklistController', () => { }); }); + describe('sync', () => { + beforeEach(() => { + (DomainBlocklist.addMany as jest.Mock).mockResolvedValue(undefined); + }); + + it('imports the fetched list in batches of 500', async () => { + const list = Array.from({ length: 1200 }, (_, i) => `spam${i}.example`); + fetchMock.mockResolvedValue(mockResponse(list.join('\n'))); + + const result = await controller.sync(SYNC_URL, 'admin@example.com'); + + expect(result).toEqual({ ok: true, total: 1200, submitted: 1200 }); + expect(DomainBlocklist.addMany).toHaveBeenCalledTimes(3); + const batches = (DomainBlocklist.addMany as jest.Mock).mock.calls.map( + ([domains]) => domains.length + ); + expect(batches).toEqual([500, 500, 200]); + expect(fetchMock).toHaveBeenCalledWith(SYNC_URL, { + signal: expect.any(AbortSignal), + }); + }); + + it('skips blank lines, comments and invalid entries', async () => { + fetchMock.mockResolvedValue( + mockResponse( + [ + '# disposable domains', + '', + ' evil.com ', + 'spam.net # inline comment', + 'not a domain', + 'nodot', + `${'a'.repeat(64)}.com`, + ].join('\n') + ) + ); + + const result = await controller.sync(SYNC_URL, 'admin@example.com'); + + expect(DomainBlocklist.addMany).toHaveBeenCalledWith([ + 'evil.com', + 'spam.net', + ]); + expect(result).toEqual({ ok: true, total: 5, submitted: 2 }); + }); + + it('imports repeated entries once', async () => { + fetchMock.mockResolvedValue( + mockResponse(['evil.com', '@Evil.com', 'evil.com'].join('\n')) + ); + + const result = await controller.sync(SYNC_URL, 'admin@example.com'); + + expect(DomainBlocklist.addMany).toHaveBeenCalledWith(['evil.com']); + expect(result).toEqual({ ok: true, total: 3, submitted: 1 }); + }); + + it('logs the summary', async () => { + fetchMock.mockResolvedValue(mockResponse('evil.com')); + + await controller.sync(SYNC_URL, 'admin@example.com'); + + expect(logger.info).toHaveBeenCalledWith('domainBlocklist.sync', { + user: 'admin@example.com', + url: SYNC_URL, + total: 1, + submitted: 1, + }); + }); + + it('imports nothing when the list has no valid entries', async () => { + fetchMock.mockResolvedValue(mockResponse('# nothing here\n\n')); + + const result = await controller.sync(SYNC_URL, 'admin@example.com'); + + expect(result).toEqual({ ok: true, total: 0, submitted: 0 }); + expect(DomainBlocklist.addMany).not.toHaveBeenCalled(); + }); + + it('throws for a url that is not https', async () => { + await expect( + controller.sync( + 'http://lists.example.com/list.conf', + 'admin@example.com' + ) + ).rejects.toThrow(BadRequestException); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('throws for a malformed url', async () => { + await expect( + controller.sync('not-a-url', 'admin@example.com') + ).rejects.toThrow(BadRequestException); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('throws when url is not a string', async () => { + await expect( + controller.sync([SYNC_URL] as any, 'admin@example.com') + ).rejects.toThrow('url must be a string'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('throws when the list redirects to a plaintext url', async () => { + fetchMock.mockResolvedValue( + mockResponse('evil.com', { url: 'http://lists.example.com/list.conf' }) + ); + + await expect( + controller.sync(SYNC_URL, 'admin@example.com') + ).rejects.toThrow('url redirected away from https'); + expect(DomainBlocklist.addMany).not.toHaveBeenCalled(); + }); + + it('throws when the body exceeds the size cap', async () => { + const oneMb = 'a'.repeat(1024 * 1024); + fetchMock.mockResolvedValue(mockResponse(Array(11).fill(oneMb))); + + await expect( + controller.sync(SYNC_URL, 'admin@example.com') + ).rejects.toThrow('List is larger than the 10485760 byte limit'); + expect(DomainBlocklist.addMany).not.toHaveBeenCalled(); + }); + + it('throws when the list responds with an error status', async () => { + fetchMock.mockResolvedValue( + mockResponse('', { ok: false, status: 404, statusText: 'Not Found' }) + ); + + await expect( + controller.sync(SYNC_URL, 'admin@example.com') + ).rejects.toThrow('Could not fetch the list: 404 Not Found'); + expect(DomainBlocklist.addMany).not.toHaveBeenCalled(); + }); + + it('throws and logs when the fetch fails', async () => { + fetchMock.mockRejectedValue(new Error('ECONNREFUSED')); + + await expect( + controller.sync(SYNC_URL, 'admin@example.com') + ).rejects.toThrow('Could not fetch the list: ECONNREFUSED'); + expect(logger.error).toHaveBeenCalledWith( + 'domainBlocklist.sync.fetchFailed', + expect.objectContaining({ url: SYNC_URL }) + ); + }); + }); + describe('remove', () => { it('removes an existing domain', async () => { (DomainBlocklist.removeByDomain as jest.Mock).mockResolvedValue(true); diff --git a/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.ts b/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.ts index 58d48086215..1afa6bd3af6 100644 --- a/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.ts +++ b/packages/fxa-admin-server/src/rest/domain-blocklist/domain-blocklist.controller.ts @@ -18,12 +18,38 @@ import { AuthHeaderGuard } from '../../auth/auth-header.guard'; import { AuditLog } from '../../auth/audit-log.decorator'; import { Features } from '../../auth/user-group-header.decorator'; import { CurrentUser } from '../../auth/auth-header.decorator'; -import type { DomainBlocklistEntry } from '../../types'; +import type { + DomainBlocklistEntry, + DomainBlocklistSyncResult, +} from '../../types'; // RFC 1035 hostname label pattern — used to validate each domain entry const DOMAIN_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/i; +const SYNC_BATCH_SIZE = 500; +/** Pause between batches so a large import does not saturate the database. */ +const SYNC_BATCH_DELAY_MS = 50; +const SYNC_FETCH_TIMEOUT_MS = 60_000; +/** The ~40k entry reference list is under 1MB. 10MB leaves room for a bigger list. */ +const SYNC_MAX_BYTES = 10 * 1024 * 1024; + +function normalizeDomain(domain: string): string { + return domain.trim().toLowerCase().replace(/^@/, ''); +} + +function isValidDomain(domain: string): boolean { + return ( + domain.length <= 253 && + domain.split('.').every((label) => label.length <= 63) && + DOMAIN_RE.test(domain) + ); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + @UseGuards(AuthHeaderGuard) @Controller('/api/domain-blocklist') export class DomainBlocklistController { @@ -57,9 +83,7 @@ export class DomainBlocklistController { ); } - const trimmed = domains - .map((d) => d.trim().toLowerCase().replace(/^@/, '')) - .filter((d) => d.length > 0); + const trimmed = domains.map(normalizeDomain).filter((d) => d.length > 0); if (trimmed.length === 0) { throw new BadRequestException( @@ -93,6 +117,139 @@ export class DomainBlocklistController { return { ok: true }; } + /** + * Imports a newline-delimited domain list from a public URL. The server + * fetches the list so the panel does not need a cross-origin request. + */ + @Post('sync') + @Features(AdminPanelFeature.DomainBlocklist) + @AuditLog() + public async sync( + @Body('url') url: string, + @CurrentUser() user: string + ): Promise { + const source = this.parseSyncUrl(url); + const body = await this.fetchList(source); + const { total, domains } = this.parseList(body); + + for (let i = 0; i < domains.length; i += SYNC_BATCH_SIZE) { + await DomainBlocklist.addMany(domains.slice(i, i + SYNC_BATCH_SIZE)); + if (i + SYNC_BATCH_SIZE < domains.length) { + await delay(SYNC_BATCH_DELAY_MS); + } + } + + this.log.info('domainBlocklist.sync', { + user, + url: `${source.origin}${source.pathname}`, + total, + submitted: domains.length, + }); + return { ok: true, total, submitted: domains.length }; + } + + /** Accepts https URLs only, so the fetch cannot reach a plaintext target. */ + private parseSyncUrl(url: string): URL { + if (typeof url !== 'string') { + throw new BadRequestException('url must be a string'); + } + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new BadRequestException('url must be a valid URL'); + } + if (parsed.protocol !== 'https:') { + throw new BadRequestException('url must use https'); + } + return parsed; + } + + private async fetchList(source: URL): Promise { + const abort = new AbortController(); + const timer = setTimeout(() => abort.abort(), SYNC_FETCH_TIMEOUT_MS); + try { + const res = await fetch(source.toString(), { signal: abort.signal }); + if (!res.ok) { + throw new BadRequestException( + `Could not fetch the list: ${res.status} ${res.statusText}` + ); + } + // fetch follows redirects, so a https URL can still land on a plaintext one. + // This checks the final hop only; fetch has already made every request. + if (res.url && new URL(res.url).protocol !== 'https:') { + throw new BadRequestException('url redirected away from https'); + } + return await this.readCappedBody(res); + } catch (err) { + if (err instanceof BadRequestException) { + throw err; + } + this.log.error('domainBlocklist.sync.fetchFailed', { + url: `${source.origin}${source.pathname}`, + err, + }); + throw new BadRequestException( + `Could not fetch the list: ${err instanceof Error ? err.message : 'unknown error'}` + ); + } finally { + clearTimeout(timer); + } + } + + /** + * Reads the body a chunk at a time and stops at SYNC_MAX_BYTES. `content-length` + * is not enough on its own: a chunked response does not send one. + */ + private async readCappedBody(res: Response): Promise { + const reader = res.body?.getReader(); + if (!reader) { + return ''; + } + + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + size += value.length; + if (size > SYNC_MAX_BYTES) { + await reader.cancel(); + throw new BadRequestException( + `List is larger than the ${SYNC_MAX_BYTES} byte limit` + ); + } + chunks.push(value); + } + + return Buffer.concat(chunks).toString('utf8'); + } + + /** + * Splits the list into unique valid domains. Invalid entries are dropped + * rather than rejected, because a public list of ~40k entries usually has + * some junk in it and one bad line must not fail the whole import. + */ + private parseList(body: string): { total: number; domains: string[] } { + const domains = new Set(); + let total = 0; + + for (const line of body.split(/\r?\n/)) { + const entry = normalizeDomain(line.split('#')[0]); + if (entry.length === 0) { + continue; + } + total++; + if (isValidDomain(entry)) { + domains.add(entry); + } + } + + return { total, domains: [...domains] }; + } + @Delete() @Features(AdminPanelFeature.DomainBlocklist) @AuditLog() @@ -103,7 +260,7 @@ export class DomainBlocklistController { if (typeof domain !== 'string' || domain.trim().length === 0) { throw new BadRequestException('domain must be a non-empty string'); } - const trimmedDomain = domain.trim().toLowerCase().replace(/^@/, ''); + const trimmedDomain = normalizeDomain(domain); const removed = await DomainBlocklist.removeByDomain(trimmedDomain); this.log.info('domainBlocklist.remove', { user, diff --git a/packages/fxa-admin-server/src/types.ts b/packages/fxa-admin-server/src/types.ts index 82da58ef395..0b4463262a3 100644 --- a/packages/fxa-admin-server/src/types.ts +++ b/packages/fxa-admin-server/src/types.ts @@ -320,6 +320,14 @@ export interface DomainBlocklistEntry { createdAt: number; } +export interface DomainBlocklistSyncResult { + ok: boolean; + /** non-empty, non-comment lines read from the list */ + total: number; + /** unique valid domains sent to the database; already-blocked ones are ignored there */ + submitted: number; +} + export interface OAuthScopeDto { id: number; scope: string;