From 8964243585bc17f98bea39035d8bc4c88948c504 Mon Sep 17 00:00:00 2001 From: Faiz Khairi Date: Wed, 25 Mar 2026 01:11:41 +0800 Subject: [PATCH 1/2] feat(cpex-scraper): add file logger for better debugging and tracing Add a lightweight file logger that writes timestamped lines to both stdout and a log file in the logs/ directory. The log directory is created automatically at runtime (already gitignored by root .gitignore). This matches the logging pattern used by the nus-v2 scraper, making it easier to debug and trace CPEx scraper runs. - Add src/logger.ts with createFileLogger() utility - Update src/index.ts to use the file logger instead of console --- scrapers/cpex-scraper/src/index.ts | 10 ++++- scrapers/cpex-scraper/src/logger.ts | 59 +++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 scrapers/cpex-scraper/src/logger.ts diff --git a/scrapers/cpex-scraper/src/index.ts b/scrapers/cpex-scraper/src/index.ts index 71e8f5a50a..1f91169ac1 100644 --- a/scrapers/cpex-scraper/src/index.ts +++ b/scrapers/cpex-scraper/src/index.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; +import { createFileLogger } from './logger'; import { scrapeCPEx, type ScraperEnv } from './scraper'; const ACADEMIC_YEAR = '2026/27'; @@ -12,10 +13,17 @@ const threshold = 1500; const envPath = path.join(__dirname, '../../env.json'); const env = JSON.parse(fs.readFileSync(envPath, 'utf8')) as ScraperEnv; +const logger = createFileLogger(); + scrapeCPEx({ academicYear: ACADEMIC_YEAR, env, + logger, threshold, +}).then(() => { + logger.close(); }).catch((error) => { - console.error(`Failed to scrape: ${error}`); + logger.log(`Failed to scrape: ${error}`); + logger.close(); + process.exitCode = 1; }); diff --git a/scrapers/cpex-scraper/src/logger.ts b/scrapers/cpex-scraper/src/logger.ts new file mode 100644 index 0000000000..91fad3e714 --- /dev/null +++ b/scrapers/cpex-scraper/src/logger.ts @@ -0,0 +1,59 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const LOG_DIR = path.join(__dirname, '../logs'); + +function pad2(n: number): string { + return n < 10 ? `0${n}` : String(n); +} + +function formatTimestamp(date: Date): string { + return ( + date.getFullYear().toString() + + '-' + + pad2(date.getMonth() + 1) + + '-' + + pad2(date.getDate()) + + '.' + + pad2(date.getHours()) + + '-' + + pad2(date.getMinutes()) + + '-' + + pad2(date.getSeconds()) + ); +} + +export type FileLogger = Pick & { + close: () => void; +}; + +/** + * Creates a logger that writes timestamped lines to both stdout and a log file. + * The log file is written to `scrapers/cpex-scraper/logs/cpex-YYYY-MM-DD.HH-mm-ss.log`. + * + * Satisfies the `Pick` interface used by the CPEx scraper. + */ +export function createFileLogger(now: Date = new Date()): FileLogger { + if (!fs.existsSync(LOG_DIR)) { + fs.mkdirSync(LOG_DIR, { recursive: true }); + } + + const filename = `cpex-${formatTimestamp(now)}.log`; + const logPath = path.join(LOG_DIR, filename); + const stream = fs.createWriteStream(logPath, { flags: 'a' }); + + function log(...args: Array): void { + const timestamp = new Date().toISOString(); + const message = args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '); + const line = `[${timestamp}] ${message}`; + + console.log(line); + stream.write(`${line}\n`); + } + + function close(): void { + stream.end(); + } + + return { close, log }; +} From b1610aa4366542081e5d0267d700200541dc44e7 Mon Sep 17 00:00:00 2001 From: Faiz Khairi Date: Wed, 25 Mar 2026 01:24:25 +0800 Subject: [PATCH 2/2] fix: correct log path and await stream flush - Fix log directory path: '../logs' resolved to build/logs/ at runtime (inside gitignored build dir). Changed to '../../logs' to resolve to scrapers/cpex-scraper/logs/ (the scraper root) - Make close() return Promise to ensure all buffered data is flushed to disk before process exits - Await close() in both success and error paths in index.ts - Add error handler on write stream to prevent unhandled crash --- scrapers/cpex-scraper/src/index.ts | 8 ++++---- scrapers/cpex-scraper/src/logger.ts | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scrapers/cpex-scraper/src/index.ts b/scrapers/cpex-scraper/src/index.ts index 1f91169ac1..760b2e9d07 100644 --- a/scrapers/cpex-scraper/src/index.ts +++ b/scrapers/cpex-scraper/src/index.ts @@ -20,10 +20,10 @@ scrapeCPEx({ env, logger, threshold, -}).then(() => { - logger.close(); -}).catch((error) => { +}).then(async () => { + await logger.close(); +}).catch(async (error) => { logger.log(`Failed to scrape: ${error}`); - logger.close(); + await logger.close(); process.exitCode = 1; }); diff --git a/scrapers/cpex-scraper/src/logger.ts b/scrapers/cpex-scraper/src/logger.ts index 91fad3e714..fe340ae242 100644 --- a/scrapers/cpex-scraper/src/logger.ts +++ b/scrapers/cpex-scraper/src/logger.ts @@ -1,7 +1,9 @@ import fs from 'node:fs'; import path from 'node:path'; -const LOG_DIR = path.join(__dirname, '../logs'); +// At runtime, __dirname is build/src/ (compiled from src/). +// Two levels up reaches the scraper root: build/src/ -> build/ -> scrapers/cpex-scraper/ +const LOG_DIR = path.join(__dirname, '../../logs'); function pad2(n: number): string { return n < 10 ? `0${n}` : String(n); @@ -24,7 +26,7 @@ function formatTimestamp(date: Date): string { } export type FileLogger = Pick & { - close: () => void; + close: () => Promise; }; /** @@ -42,6 +44,10 @@ export function createFileLogger(now: Date = new Date()): FileLogger { const logPath = path.join(LOG_DIR, filename); const stream = fs.createWriteStream(logPath, { flags: 'a' }); + stream.on('error', (err) => { + console.error(`[FileLogger] write stream error: ${err.message}`); + }); + function log(...args: Array): void { const timestamp = new Date().toISOString(); const message = args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '); @@ -51,8 +57,10 @@ export function createFileLogger(now: Date = new Date()): FileLogger { stream.write(`${line}\n`); } - function close(): void { - stream.end(); + function close(): Promise { + return new Promise((resolve, reject) => { + stream.end((err?: Error | null) => (err ? reject(err) : resolve())); + }); } return { close, log };