From 99e4927640a758d38f8b8ba2d1c921cefd8c2724 Mon Sep 17 00:00:00 2001 From: null Date: Mon, 6 Jul 2026 11:20:29 -0500 Subject: [PATCH] refactor(server): migrate csvTransactionImportService to TypeScript (.cts) The CSV parser + import primitives (also shared by the OFX importer). Behavior-preserving. Verified: typecheck:server 0, check:server 0, suite 226/226. Co-Authored-By: Claude Opus 4.8 --- routes/import.js | 2 +- ...ice.js => csvTransactionImportService.cts} | 113 +++++++++--------- services/ofxImportService.cts | 2 +- tests/csvTransactionImportService.test.js | 2 +- 4 files changed, 60 insertions(+), 59 deletions(-) rename services/{csvTransactionImportService.js => csvTransactionImportService.cts} (83%) diff --git a/routes/import.js b/routes/import.js index 17404f4..420c8d7 100644 --- a/routes/import.js +++ b/routes/import.js @@ -15,7 +15,7 @@ const { const { previewCsvTransactions, commitCsvTransactions, -} = require('../services/csvTransactionImportService'); +} = require('../services/csvTransactionImportService.cts'); const { previewOfxTransactions, commitOfxTransactions, diff --git a/services/csvTransactionImportService.js b/services/csvTransactionImportService.cts similarity index 83% rename from services/csvTransactionImportService.js rename to services/csvTransactionImportService.cts index 4eb1fd6..b0e55e0 100644 --- a/services/csvTransactionImportService.js +++ b/services/csvTransactionImportService.cts @@ -1,14 +1,18 @@ 'use strict'; +import type { Db } from '../types/db'; + const crypto = require('crypto'); const { getDb } = require('../db/database'); const { decorateTransaction, ensureManualDataSource } = require('./transactionService.cts'); +type Row = Record; + const SESSION_TTL_MS = 24 * 60 * 60 * 1000; const MAX_ROWS = 25000; const SAMPLE_SIZE = 10; -const FIELD_LABELS = { +const FIELD_LABELS: Record = { posted_date: 'Posted date', transacted_at: 'Transaction date/time', amount: 'Amount', @@ -24,32 +28,32 @@ const FIELD_LABELS = { currency: 'Currency', }; -function importError(status, message, code, details = []) { - const err = new Error(message); +function importError(status: number, message: string, code: string, details: any[] = []): Error { + const err = new Error(message) as any; err.status = status; err.code = code; err.details = details; return err; } -function makeSessionId() { +function makeSessionId(): string { return crypto.randomBytes(16).toString('hex'); } -function cleanFilename(value) { +function cleanFilename(value: any): string | null { return value ? String(value).replace(/[^a-zA-Z0-9._\-\s]/g, '').trim().slice(0, 255) : null; } -function normalizeHeader(value, index) { +function normalizeHeader(value: any, index: number): string { const text = String(value ?? '').trim(); return text || `Column ${index + 1}`; } -function parseCsv(text) { - const rows = []; - let row = []; +function parseCsv(text: string): any[] { + const rows: any[] = []; + let row: any[] = []; let cell = ''; let inQuotes = false; @@ -98,38 +102,38 @@ function parseCsv(text) { rows.push(row); } - return rows.filter(r => r.some(c => String(c ?? '').trim() !== '')); + return rows.filter((r: any[]) => r.some((c: any) => String(c ?? '').trim() !== '')); } -function csvBufferToText(buffer) { +function csvBufferToText(buffer: any): string { if (!Buffer.isBuffer(buffer) || buffer.length === 0) { throw importError(400, 'CSV file is required.', 'CSV_REQUIRED'); } - return buffer.toString('utf8').replace(/^\uFEFF/, ''); + return buffer.toString('utf8').replace(/^/, ''); } -function rowToObject(headers, row) { - const out = {}; - headers.forEach((header, index) => { +function rowToObject(headers: string[], row: any[]): Row { + const out: Row = {}; + headers.forEach((header: string, index: number) => { out[header] = String(row[index] ?? '').trim(); }); return out; } -function normalizeHeaderToken(value) { +function normalizeHeaderToken(value: any): string { return String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim(); } -function headerMatches(header, patterns) { +function headerMatches(header: any, patterns: any[]): boolean { const token = normalizeHeaderToken(header); - return patterns.some(pattern => ( + return patterns.some((pattern: any) => ( pattern instanceof RegExp ? pattern.test(token) : token === pattern )); } -function suggestMapping(headers) { - const mapping = {}; - const candidates = [ +function suggestMapping(headers: string[]): Row { + const mapping: Row = {}; + const candidates: [string, any[]][] = [ ['posted_date', [/^date$/, 'posted date', 'post date', 'posting date', 'transaction date', 'trans date']], ['transacted_at', ['authorized date', 'authorization date', 'datetime', 'date time', 'timestamp']], ['transaction_id', ['transaction id', 'transaction number', 'trans id', 'id', 'fitid', 'reference', 'reference number']], @@ -146,19 +150,19 @@ function suggestMapping(headers) { ]; for (const [field, patterns] of candidates) { - const found = headers.find(header => !Object.values(mapping).includes(header) && headerMatches(header, patterns)); + const found = headers.find((header: string) => !Object.values(mapping).includes(header) && headerMatches(header, patterns)); if (found) mapping[field] = found; } if (!mapping.payee && mapping.description) { - const payee = headers.find(header => header !== mapping.description && headerMatches(header, ['name', 'merchant name'])); + const payee = headers.find((header: string) => header !== mapping.description && headerMatches(header, ['name', 'merchant name'])); if (payee) mapping.payee = payee; } return mapping; } -function parseCsvPreview(buffer, options = {}) { +function parseCsvPreview(buffer: any, options: any = {}): any { const text = csvBufferToText(buffer); const parsed = parseCsv(text); if (parsed.length < 2) { @@ -166,8 +170,8 @@ function parseCsvPreview(buffer, options = {}) { } const headers = parsed[0].map(normalizeHeader); - const seenHeaders = new Set(); - const duplicateHeaders = []; + const seenHeaders = new Set(); + const duplicateHeaders: any[] = []; for (const header of headers) { const key = header.toLowerCase(); if (seenHeaders.has(key)) duplicateHeaders.push(header); @@ -177,10 +181,10 @@ function parseCsvPreview(buffer, options = {}) { throw importError(400, `CSV contains duplicate headers: ${duplicateHeaders.join(', ')}`, 'CSV_DUPLICATE_HEADERS'); } - const dataRows = parsed.slice(1).slice(0, MAX_ROWS).map(row => rowToObject(headers, row)); + const dataRows = parsed.slice(1).slice(0, MAX_ROWS).map((row: any[]) => rowToObject(headers, row)); const truncated = parsed.length - 1 > MAX_ROWS; const suggestedMapping = suggestMapping(headers); - const errors = []; + const errors: any[] = []; if (!suggestedMapping.posted_date) { errors.push({ field: 'posted_date', message: 'Could not detect a posted date column.' }); @@ -206,11 +210,11 @@ function parseCsvPreview(buffer, options = {}) { }; } -function pruneExpiredSessions(db) { +function pruneExpiredSessions(db: Db): void { db.prepare('DELETE FROM import_sessions WHERE expires_at <= ?').run(new Date().toISOString()); } -function saveImportSession(db, userId, sessionData) { +function saveImportSession(db: Db, userId: number, sessionData: any): string { const id = makeSessionId(); const now = new Date().toISOString(); const expiresAt = new Date(Date.now() + SESSION_TTL_MS).toISOString(); @@ -221,7 +225,7 @@ function saveImportSession(db, userId, sessionData) { return id; } -function loadImportSession(db, userId, sessionId) { +function loadImportSession(db: Db, userId: number, sessionId: any): any { const row = db.prepare(` SELECT preview_json FROM import_sessions @@ -234,12 +238,12 @@ function loadImportSession(db, userId, sessionId) { return JSON.parse(row.preview_json); } -function deleteImportSession(db, sessionId) { +function deleteImportSession(db: Db, sessionId: any): void { db.prepare('DELETE FROM import_sessions WHERE id = ?').run(sessionId); } -function previewCsvTransactions(userId, buffer, options = {}) { - const db = getDb(); +function previewCsvTransactions(userId: number, buffer: any, options: any = {}): any { + const db: Db = getDb(); pruneExpiredSessions(db); const preview = parseCsvPreview(buffer, options); const sessionId = saveImportSession(db, userId, { @@ -258,13 +262,13 @@ function previewCsvTransactions(userId, buffer, options = {}) { }; } -function headerValue(row, mapping, field) { +function headerValue(row: Row, mapping: any, field: string): string { const header = mapping?.[field]; if (!header) return ''; return String(row[header] ?? '').trim(); } -function parseDateValue(value, field, rowNumber) { +function parseDateValue(value: any, field: string, rowNumber: number): string { const text = String(value || '').trim(); if (!text) { throw importError(400, `${FIELD_LABELS[field] || field} is required`, 'CSV_ROW_VALIDATION', [ @@ -290,7 +294,7 @@ function parseDateValue(value, field, rowNumber) { ]); } -function isRealDate(value) { +function isRealDate(value: any): boolean { const [year, month, day] = String(value).split('-').map(Number); const date = new Date(Date.UTC(year, month - 1, day)); return date.getUTCFullYear() === year @@ -298,7 +302,7 @@ function isRealDate(value) { && date.getUTCDate() === day; } -function parseCents(value, { negative = false } = {}) { +function parseCents(value: any, { negative = false }: { negative?: boolean } = {}): number | null { const text = String(value ?? '').trim(); if (!text) return null; const parenNegative = /^\(.*\)$/.test(text); @@ -311,7 +315,7 @@ function parseCents(value, { negative = false } = {}) { return Math.round(Math.abs(number) * 100) * sign; } -function parseMappedAmount(row, mapping) { +function parseMappedAmount(row: Row, mapping: any): number | null { const amount = parseCents(headerValue(row, mapping, 'amount')); if (amount !== null) return amount; @@ -324,15 +328,15 @@ function parseMappedAmount(row, mapping) { return null; } -function stableHash(parts) { +function stableHash(parts: any[]): string { return crypto .createHash('sha256') - .update(parts.map(part => String(part || '').trim().toLowerCase()).join('\u001f')) + .update(parts.map((part: any) => String(part || '').trim().toLowerCase()).join('')) .digest('hex') .slice(0, 48); } -function getOrCreateCsvDataSource(db, userId) { +function getOrCreateCsvDataSource(db: Db, userId: number): Row { ensureManualDataSource(db, userId); const existing = db.prepare(` SELECT * @@ -350,7 +354,7 @@ function getOrCreateCsvDataSource(db, userId) { return db.prepare('SELECT * FROM data_sources WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId); } -function getOrCreateAccount(db, userId, dataSourceId, name) { +function getOrCreateAccount(db: Db, userId: number, dataSourceId: any, name: any): Row | null { const accountName = String(name || '').trim(); if (!accountName) return null; const providerAccountId = stableHash([accountName]).slice(0, 32); @@ -369,13 +373,13 @@ function getOrCreateAccount(db, userId, dataSourceId, name) { return db.prepare('SELECT * FROM financial_accounts WHERE id = ? AND user_id = ?').get(result.lastInsertRowid, userId); } -function validateMapping(headers, mapping = {}) { +function validateMapping(headers: any[], mapping: any = {}): void { const headerSet = new Set(headers); - const required = []; + const required: any[] = []; if (!mapping.posted_date) required.push('posted_date'); if (!mapping.amount && !(mapping.debit_amount || mapping.credit_amount)) required.push('amount'); if (required.length) { - throw importError(400, `Missing required mapping: ${required.map(f => FIELD_LABELS[f] || f).join(', ')}`, 'CSV_MAPPING_REQUIRED'); + throw importError(400, `Missing required mapping: ${required.map((f: string) => FIELD_LABELS[f] || f).join(', ')}`, 'CSV_MAPPING_REQUIRED'); } for (const [field, header] of Object.entries(mapping)) { @@ -388,7 +392,7 @@ function validateMapping(headers, mapping = {}) { } } -function parseOptionalDateTimeValue(value, field, rowNumber) { +function parseOptionalDateTimeValue(value: any, field: string, rowNumber: number): string | null { const text = String(value || '').trim(); if (!text) return null; @@ -401,7 +405,7 @@ function parseOptionalDateTimeValue(value, field, rowNumber) { return text; } -function normalizeCsvTransaction(row, mapping, rowNumber) { +function normalizeCsvTransaction(row: Row, mapping: any, rowNumber: number): Row { const postedDate = parseDateValue(headerValue(row, mapping, 'posted_date'), 'posted_date', rowNumber); const transactedAt = parseOptionalDateTimeValue(headerValue(row, mapping, 'transacted_at'), 'transacted_at', rowNumber); const amount = parseMappedAmount(row, mapping); @@ -436,8 +440,8 @@ function normalizeCsvTransaction(row, mapping, rowNumber) { }; } -function commitCsvTransactions(userId, importSessionId, mapping, options = {}) { - const db = getDb(); +function commitCsvTransactions(userId: number, importSessionId: any, mapping: any, options: any = {}): any { + const db: Db = getDb(); const session = loadImportSession(db, userId, importSessionId); if (session.kind !== 'csv_transactions') { throw importError(400, 'Import session is not a CSV transaction preview.', 'CSV_SESSION_INVALID'); @@ -445,7 +449,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) { validateMapping(session.headers, mapping); const dataSource = getOrCreateCsvDataSource(db, userId); - const details = []; + const details: any[] = []; const counts = { imported: 0, skipped: 0, failed: 0 }; const insert = db.prepare(` INSERT INTO transactions @@ -461,7 +465,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) { `); const run = db.transaction(() => { - session.rows.forEach((row, index) => { + session.rows.forEach((row: Row, index: number) => { const rowNumber = index + 2; try { const tx = normalizeCsvTransaction(row, mapping, rowNumber); @@ -508,7 +512,7 @@ function commitCsvTransactions(userId, importSessionId, mapping, options = {}) { ignored: 0, }), }); - } catch (err) { + } catch (err: any) { counts.failed++; details.push({ row: rowNumber, result: 'failed', message: err.message, details: err.details || [] }); } @@ -553,9 +557,6 @@ module.exports = { FIELD_LABELS, commitCsvTransactions, previewCsvTransactions, - // Reusable transaction-import primitives (shared by the OFX/QFX importer so it - // dedupes and sessions identically — same import_sessions table, same - // (user_id, data_source_id, provider_transaction_id) dedupe scope). saveImportSession, loadImportSession, deleteImportSession, diff --git a/services/ofxImportService.cts b/services/ofxImportService.cts index a8943cf..39e0992 100644 --- a/services/ofxImportService.cts +++ b/services/ofxImportService.cts @@ -21,7 +21,7 @@ const { getOrCreateAccount, stableHash, parseCents, -} = require('./csvTransactionImportService'); +} = require('./csvTransactionImportService.cts'); type Row = Record; diff --git a/tests/csvTransactionImportService.test.js b/tests/csvTransactionImportService.test.js index f315759..a547de9 100644 --- a/tests/csvTransactionImportService.test.js +++ b/tests/csvTransactionImportService.test.js @@ -11,7 +11,7 @@ const { getDb, closeDb } = require('../db/database'); const { commitCsvTransactions, previewCsvTransactions, -} = require('../services/csvTransactionImportService'); +} = require('../services/csvTransactionImportService.cts'); function createUser(db) { return db.prepare(`